more work on msd & shift to overlay key from overlay_services but wit…

…h backward compat.

plugins/action/common/nac_dc_validate.py

@@ -29,6 +29,7 @@
 from ansible.errors import AnsibleError
 
 try:
+    from iac_validate.yaml import load_yaml_files
     import iac_validate.validator
     from iac_validate.cli.options import DEFAULT_SCHEMA
 except ImportError as imp_exc:
@@ -37,6 +38,7 @@
     IAC_VALIDATE_IMPORT_ERROR = None
 
 import os
+from ...plugin_utils.helper_functions import data_model_key_check
 
 display = Display()
 
@@ -77,18 +79,64 @@ def run(self, tmp=None, task_vars=None):
         if schema == '':
             schema = DEFAULT_SCHEMA
 
-        validator = iac_validate.validator.Validator(schema, rules)
-        if schema:
-            validator.validate_syntax([mdata])
-        if rules:
-            validator.validate_semantics([mdata])
-
-        msg = ""
-        for error in validator.errors:
-            msg += error + "\n"
-
-        if msg:
-            results['failed'] = True
-            results['msg'] = msg
+        rules_list = []
+        if rules and task_vars['role_path'] in rules:
+            # Load in-memory data model using iac-validate
+            # Perform the load in this if block to avoid loading the data model multiple times when custom enhanced rules are provided
+            results['data'] = load_yaml_files([mdata])
+
+            # Introduce common directory to the rules list by default once vrf and network rules are updated
+            # rules_to_run.append(f'{rules}common')
+            parent_keys = ['vxlan', 'fabric']
+            check = data_model_key_check(results['data'], parent_keys)
+            if 'fabric' in check['keys_found'] and 'fabric' in check['keys_data']:
+                if 'type' in results['data']['vxlan']['fabric']:
+                    if results['data']['vxlan']['fabric']['type'] in ('VXLAN_EVPN'):
+                        rules_list.append(f'{rules}vxlan/')
+                    elif results['data']['vxlan']['fabric']['type'] in ('MSD', 'MCF'):
+                        rules_list.append(f'{rules}multisite/')
+                    else:
+                        results['failed'] = True
+                        results['msg'] = f"vxlan.fabric.type {results['data']['vxlan']['fabric']['type']} is not a supported fabric type."
+                else:
+                    results['failed'] = True
+                    results['msg'] = "vxlan.fabric.type is not defined in the data model."
+            else:
+                # This else block is to be removed after the deprecation of vxlan.global.fabric_type
+                parent_keys = ['vxlan', 'global']
+                check = data_model_key_check(results['data'], parent_keys)
+                if 'global' in check['keys_found'] and 'global' in check['keys_data']:
+                    if 'fabric_type' in results['data']['vxlan']['global']:
+                        display.deprecated("Attempting to use vxlan.global.fabric_type due to vxlan.fabric.type not being found. vxlan.global.fabric_type is being deprecated. Please use vxlan.fabric.type.")
+
+                        if results['data']['vxlan']['global']['fabric_type'] in ('VXLAN_EVPN'):
+                            rules_list.append(f'{rules}vxlan/')
+                        elif results['data']['vxlan']['global']['fabric_type'] in ('MSD', 'MCF'):
+                            rules_list.append(f'{rules}multisite/')
+                        else:
+                            results['failed'] = True
+                            results['msg'] = f"vxlan.fabric.type {results['data']['vxlan']['global']['fabric_type']} is not a supported fabric type."
+                    else:
+                        results['failed'] = True
+                        results['msg'] = "vxlan.fabric.type is not defined in the data model."
+        else:
+            # Else block to pickup custom enhanced rules provided by the user
+            rules_list.append(f'{rules}')
+
+        for rules_item in rules_list:
+            validator = iac_validate.validator.Validator(schema, rules_item)
+            if schema:
+                validator.validate_syntax([mdata])
+            if rules_item:
+                validator.validate_semantics([mdata])
+
+            msg = ""
+            for error in validator.errors:
+                msg += error + "\n"
+
+            if msg:
+                results['failed'] = True
+                results['msg'] = msg
+                break
 
         return results

plugins/action/common/prepare_plugins/prep_001_list_defaults.py

@@ -127,6 +127,55 @@ def prepare(self):
         # Fabric Overlay List Defaults
         # --------------------------------------------------------------------
 
+        # Check vxlan.overlay list elements
+        parent_keys = ['vxlan', 'overlay']
+        dm_check = data_model_key_check(self.model_data, parent_keys)
+        if 'overlay' in dm_check['keys_not_found'] or 'overlay_services' in dm_check['keys_no_data']:
+            self.model_data['vxlan']['overlay'] = {'vrfs': []}
+            self.model_data['vxlan']['overlay'] = {'vrf_attach_groups': []}
+            self.model_data['vxlan']['overlay'] = {'networks': []}
+            self.model_data['vxlan']['overlay'] = {'network_attach_groups': []}
+
+        # Check vxlan.overlay_services.vrfs list element
+        target_key = 'vrfs'
+        self.set_list_default(parent_keys, target_key)
+
+        # Check vxlan.overlay.vrf_attach_groups list element
+        target_key = 'vrf_attach_groups'
+        self.set_list_default(parent_keys, target_key)
+
+        # Check vxlan.overlay.vrf_attach_groups[index].switches list elements
+        list_index = 0
+        for group in self.model_data['vxlan']['overlay']['vrf_attach_groups']:
+            dm_check = data_model_key_check(group, ['switches'])
+            if 'switches' in dm_check['keys_not_found'] or \
+               'switches' in dm_check['keys_no_data']:
+                self.model_data['vxlan']['overlay']['vrf_attach_groups'][list_index]['switches'] = []
+
+            list_index += 1
+
+        # Check vxlan.overlay.networks list element
+        target_key = 'networks'
+        self.set_list_default(parent_keys, target_key)
+
+        # Check vxlan.overlay.network_attach_groups list element
+        target_key = 'network_attach_groups'
+        self.set_list_default(parent_keys, target_key)
+
+        # Check vxlan.overlay.network_attach_groups[index].switches list elements
+        list_index = 0
+        for group in self.model_data['vxlan']['overlay']['network_attach_groups']:
+            dm_check = data_model_key_check(group, ['switches'])
+            if 'switches' in dm_check['keys_not_found'] or \
+               'switches' in dm_check['keys_no_data']:
+                self.model_data['vxlan']['overlay']['network_attach_groups'][list_index]['switches'] = []
+
+            list_index += 1
+
+        # --------------------------------------------------------------------
+        # Fabric Overlay List Defaults - Backwards Compatibility
+        # --------------------------------------------------------------------
+
         # Check vxlan.overlay_services list elements
         parent_keys = ['vxlan', 'overlay_services']
         dm_check = data_model_key_check(self.model_data, parent_keys)

plugins/action/common/prepare_plugins/prep_101_fabric.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2024 Cisco Systems, Inc. and its affiliates
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# SPDX-License-Identifier: MIT
+
+from ansible.utils.display import Display
+from ....plugin_utils.helper_functions import data_model_key_check
+
+display = Display()
+
+
+class PreparePlugin:
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        self.keys = []
+
+    def prepare(self):
+        model_data = self.kwargs['results']['model_extended']
+
+        parent_keys = ['vxlan', 'fabric']
+        dm_check = data_model_key_check(model_data, parent_keys)
+        if 'fabric' in dm_check['keys_not_found'] or 'fabric' in dm_check['keys_no_data']:
+            display.deprecated(
+                "Attempting to use vxlan.global.name and vxlan.global.fabric_type due to vxlan.fabric.name and vxlan.fabric.type not being found. "
+                "vxlan.global.name and vxlan.global.fabric_type is being deprecated. Please use vxlan.fabric."
+            )
+
+            # Prepare the data model to ensure vxlan.fabric.name is set
+        #     global_data_check = False
+        #     parent_keys = ['vxlan', 'global']
+        #     dm_check = data_model_key_check(model_data, parent_keys)
+        #     if 'global' in dm_check['keys_found'] and 'global' in dm_check['keys_data']:
+        #         parent_keys = ['vxlan', 'global', 'name']
+        #         dm_check = data_model_key_check(model_data, parent_keys)
+        #         if 'name' in dm_check['keys_found'] and 'name' in dm_check['keys_data']:
+        #             model_data['vxlan']['fabric']['name'] = model_data['vxlan']['global']['name']
+        #         else:
+        #             global_data_check = True
+
+        #         parent_keys = ['vxlan', 'global', 'fabric_type']
+        #         dm_check = data_model_key_check(model_data, parent_keys)
+        #         if 'fabric_type' in dm_check['keys_found'] and 'fabric_type' in dm_check['keys_data']:
+        #             model_data['vxlan']['fabric']['type'] = model_data['vxlan']['global']['fabric_type']
+        #         else:
+        #             self.kwargs['results']['failed'] = True
+        #             self.kwargs['results']['msg'] = "vxlan.fabric is not set in the model data."
+        #             global_data_check = True
+        #     else:
+        #         self.kwargs['results']['failed'] = True
+        #         self.kwargs['results']['msg'] = "vxlan.fabric is not set in the model data."
+
+        #     if global_data_check:
+        #         self.kwargs['results']['failed'] = True
+        #         self.kwargs['results']['msg'] = "vxlan.fabric is not set in the model data."
+        # else:
+        #     parent_keys = ['vxlan', 'fabric', 'name']
+        #     dm_check = data_model_key_check(model_data, parent_keys)
+        #     if 'name' in dm_check['keys_no_data'] or 'name' in dm_check['keys_not_found']:
+        #         parent_keys = ['vxlan', 'global', 'name']
+        #         dm_check = data_model_key_check(model_data, parent_keys)
+        #         if 'name' in dm_check['keys_data']:
+        #             # Insert warning about deprecation and where found
+        #             model_data['vxlan']['fabric']['name'] = model_data['vxlan']['global']['name']
+        #         else:
+        #             self.kwargs['results']['failed'] = True
+        #             self.kwargs['results']['msg'] = "vxlan.fabric.name is not set in the model data."
+
+        #     # Prepare the data model to ensure vxlan.fabric.type is set
+        #     parent_keys = ['vxlan', 'fabric', 'type']
+        #     dm_check = data_model_key_check(model_data, parent_keys)
+        #     if 'type' in dm_check['keys_no_data'] or 'name' in dm_check['keys_not_found']:
+        #         parent_keys = ['vxlan', 'global', 'fabric_type']
+        #         dm_check = data_model_key_check(model_data, parent_keys)
+        #         if 'fabric_type' in dm_check['keys_data']:
+        #             # Insert warning about deprecation and where found
+        #             model_data['vxlan']['fabric']['type'] = model_data['vxlan']['global']['fabric_type']
+        #         else:
+        #             self.kwargs['results']['failed'] = True
+        #             self.kwargs['results']['msg'] = "vxlan.fabric.type is not set in the model data."
+
+        # 1 - no fabric key data model
+        # 2a - fabric key data model with no data (i.e. no name or type) > semantic valdiation failure with schema
+        # 2b - fabric key data model with no data (i.e. name or type) > check if we have global name and global fabric_type 
+
+
+        # insert comment to indicate this is a oneoff check for fabric key as this should really be done in a rule, 
+        # but fabric name and key are foundational for the collection so we need to ensure it is set.
+        # this prepare plugin also helps retain backwards compatibility with the global fabric name and fabric_type keys previously used.
+
+        # Prepare the data model to ensure vxlan.fabric.name is set
+
+        self.kwargs['results']['model_extended'] = model_data
+        return self.kwargs['results']

plugins/action/common/prepare_plugins/prep_104_fabric_overlay_services.py

@@ -29,6 +29,56 @@ def prepare(self):
         model_data = self.kwargs['results']['model_extended']
         switches = model_data['vxlan']['topology']['switches']
 
+        # Rebuild sm_data['vxlan']['overlay']['vrf_attach_groups'] into
+        # a structure that is easier to use.
+        vrf_grp_name_list = []
+        model_data['vxlan']['overlay']['vrf_attach_groups_dict'] = {}
+        for grp in model_data['vxlan']['overlay']['vrf_attach_groups']:
+            model_data['vxlan']['overlay']['vrf_attach_groups_dict'][grp['name']] = []
+            vrf_grp_name_list.append(grp['name'])
+            for switch in grp['switches']:
+                model_data['vxlan']['overlay']['vrf_attach_groups_dict'][grp['name']].append(switch)
+            # If the switch is in the switch list and a hostname is used, replace the hostname with the management IP
+            for switch in model_data['vxlan']['overlay']['vrf_attach_groups_dict'][grp['name']]:
+                if any(sw['name'] == switch['hostname'] for sw in switches):
+                    found_switch = next((item for item in switches if item["name"] == switch['hostname']))
+                    if found_switch.get('management').get('management_ipv4_address'):
+                        switch['hostname'] = found_switch['management']['management_ipv4_address']
+                    elif found_switch.get('management').get('management_ipv6_address'):
+                        switch['hostname'] = found_switch['management']['management_ipv6_address']
+
+        # Remove vrf_attach_group from vrf if the group_name is not defined
+        for vrf in model_data['vxlan']['overlay']['vrfs']:
+            if 'vrf_attach_group' in vrf:
+                if vrf.get('vrf_attach_group') not in vrf_grp_name_list:
+                    del vrf['vrf_attach_group']
+
+        # Rebuild sm_data['vxlan']['overlay']['network_attach_groups'] into
+        # a structure that is easier to use.
+        net_grp_name_list = []
+        model_data['vxlan']['overlay']['network_attach_groups_dict'] = {}
+        for grp in model_data['vxlan']['overlay']['network_attach_groups']:
+            model_data['vxlan']['overlay']['network_attach_groups_dict'][grp['name']] = []
+            net_grp_name_list.append(grp['name'])
+            for switch in grp['switches']:
+                model_data['vxlan']['overlay']['network_attach_groups_dict'][grp['name']].append(switch)
+            # If the switch is in the switch list and a hostname is used, replace the hostname with the management IP
+            for switch in model_data['vxlan']['overlay']['network_attach_groups_dict'][grp['name']]:
+                if any(sw['name'] == switch['hostname'] for sw in switches):
+                    found_switch = next((item for item in switches if item["name"] == switch['hostname']))
+                    if found_switch.get('management').get('management_ipv4_address'):
+                        switch['hostname'] = found_switch['management']['management_ipv4_address']
+                    elif found_switch.get('management').get('management_ipv6_address'):
+                        switch['hostname'] = found_switch['management']['management_ipv6_address']
+
+        # Remove network_attach_group from net if the group_name is not defined
+        for net in model_data['vxlan']['overlay']['networks']:
+            if 'network_attach_group' in net:
+                if net.get('network_attach_group') not in net_grp_name_list:
+                    del net['network_attach_group']
+
+        # Backwards compatibility for vxlan.overlay_services
+
         # Rebuild sm_data['vxlan']['overlay_services']['vrf_attach_groups'] into
         # a structure that is easier to use.
         vrf_grp_name_list = []

roles/dtc/common/tasks/main.yml

@@ -21,12 +21,12 @@
 
 ---
 
-- name: Import Role Tasks for MSD Fabric
-  ansible.builtin.import_tasks: sub_main_msd.yml
-  tags: "{{ nac_tags.common_role }}" # Tags defined in roles/common_global/vars/main.yml
-  when: MD_Extended.vxlan.fabric_type == 'MSD'
-
 - name: Import Role Tasks for VXLAN Fabric
   ansible.builtin.import_tasks: sub_main_vxlan.yml
   tags: "{{ nac_tags.common_role }}" # Tags defined in roles/common_global/vars/main.yml
   when: MD_Extended.vxlan.fabric_type == 'VXLAN_EVPN'
+
+- name: Import Role Tasks for MSD Fabric
+  ansible.builtin.import_tasks: sub_main_msd.yml
+  tags: "{{ nac_tags.common_role }}" # Tags defined in roles/common_global/vars/main.yml
+  when: MD_Extended.vxlan.fabric_type == 'MSD'

roles/dtc/common/tasks/ndfc_fabric.yml

@@ -33,6 +33,7 @@
         {{ MD.vxlan.name }}_ndfc_fabric.yml
       {% elif MD.vxlan.global.name is defined and MD.vxlan.global.name %}
         {{ MD.vxlan.global.name }}_ndfc_fabric.yml
+      {% endif %}
   delegate_to: localhost
 
 - name: Stat Previous File If It Exists

roles/dtc/common/tasks/sub_main_vxlan.yml

@@ -38,7 +38,7 @@
 # --------------------------------------------------------------------
 
 - name: Build Fabric Create Parameters
-  ansible.builtin.include_tasks: vxlan/ndfc_fabric.yml
+  ansible.builtin.include_tasks: ndfc_fabric.yml
 
 # --------------------------------------------------------------------
 # Build NDFC Fabric Switch Inventory List From Template