Updating index and metadata of packs with changed tags (demisto#11927)

Co-authored-by: hod-alpert <haplert@paloaltonetworks.com>

Tests/scripts/collect_tests_and_content_packs.py

@@ -14,6 +14,7 @@
 from typing import Dict, Tuple, Union, Optional
 
 import demisto_sdk.commands.common.tools as tools
+from Tests.scripts.utils.collect_helpers import LANDING_PAGE_SECTIONS_JSON_PATH
 from demisto_sdk.commands.common.constants import *  # noqa: E402
 
 from Tests.scripts.utils import collect_helpers
@@ -793,6 +794,27 @@ def extract_matching_object_from_id_set(obj_id, obj_set, server_version='0'):
     return None
 
 
+def get_packs_from_landing_page(branch_name: str) -> set:
+    """
+    Parses the changes made by the current branch to the landing page sections file into modified packs.
+    These modified packs will have the metadata files changed.
+    Args:
+        branch_name: The name of the branch to compare changes from origin/master
+
+    Returns:
+        A set of packs that needs to be updated.
+    """
+    with open(LANDING_PAGE_SECTIONS_JSON_PATH, 'r') as file:
+        landing_page_sections_content = json.load(file)
+    landing_page_sections_keys = landing_page_sections_content.keys()
+    change_string = tools.run_command(f'git diff origin/master...{branch_name} {LANDING_PAGE_SECTIONS_JSON_PATH}')
+    changed_packs = set()
+    for changed_group in re.findall(r'[-+][ ]+"(.*)"', change_string):
+        if changed_group not in landing_page_sections_keys:
+            changed_packs.add(changed_group)
+    return changed_packs
+
+
 def get_test_from_conf(branch_name, conf=deepcopy(CONF)):
     tests = set([])
     changed = set([])
@@ -1081,8 +1103,17 @@ def get_test_list_and_content_packs_to_install(files_string,
                                                conf=deepcopy(CONF),
                                                id_set=deepcopy(ID_SET)):
     """Create a test list that should run"""
-    (modified_files_with_relevant_tests, modified_tests_list, changed_common, is_conf_json, sample_tests,
-     modified_packs, is_reputations_json, is_indicator_json) = get_modified_files_for_testing(files_string)
+    modified_files_instance = get_modified_files_for_testing(files_string)
+
+    modified_files_with_relevant_tests = modified_files_instance.modified_files
+    modified_tests_list = modified_files_instance.modified_tests
+    changed_common = modified_files_instance.changed_common_files
+    is_conf_json = modified_files_instance.is_conf_json
+    sample_tests = modified_files_instance.sample_tests
+    modified_packs = modified_files_instance.modified_metadata
+    is_reputations_json = modified_files_instance.is_reputations_json
+    is_indicator_json = modified_files_instance.is_indicator_json
+    is_landing_page_sections_json = modified_files_instance.is_landing_page_sections_json
 
     all_modified_files_paths = set(
         modified_files_with_relevant_tests + modified_tests_list + changed_common + sample_tests
@@ -1122,6 +1153,9 @@ def get_test_list_and_content_packs_to_install(files_string,
     if is_conf_json:
         tests = tests.union(get_test_from_conf(branch_name, conf))
 
+    if is_landing_page_sections_json:
+        packs_to_install |= get_packs_from_landing_page(branch_name)
+
     if changed_common:
         tests.add('TestCommonPython')
 

Tests/scripts/infrastructure_tests/test_collect_tests_and_content_packs.py

@@ -17,7 +17,7 @@
     create_filter_envs_file, get_from_version_and_to_version_bounderies,
     get_test_list_and_content_packs_to_install, is_documentation_changes_only,
     remove_ignored_tests, remove_tests_for_non_supported_packs)
-from Tests.scripts.utils.get_modified_files_for_testing import get_modified_files_for_testing
+from Tests.scripts.utils.get_modified_files_for_testing import get_modified_files_for_testing, ModifiedFiles
 from Tests.scripts.utils import content_packs_util
 
 from TestSuite import repo, test_tools
@@ -290,9 +290,12 @@ def test_get_from_version_and_to_version_from_modified_files(self):
 
         """
         test_path = 'Tests/scripts/infrastructure_tests/tests_data/mock_test_playbooks/fake_test_playbook.yml'
-        modified_files_list, modified_tests_list, changed_common, _, sample_tests, modified_metadata_list, _, _ = \
-            create_get_modified_files_ret(modified_files_list=[test_path], modified_tests_list=[test_path])
-        all_modified_files_paths = set(modified_files_list + modified_tests_list + changed_common + sample_tests)
+        modified_files_instance = create_get_modified_files_ret(modified_files_list=[test_path],
+                                                                modified_tests_list=[test_path])
+        all_modified_files_paths = set(modified_files_instance.modified_files
+                                       + modified_files_instance.modified_tests
+                                       + modified_files_instance.changed_common_files
+                                       + modified_files_instance.sample_tests)
         from_version, to_version = get_from_version_and_to_version_bounderies(all_modified_files_paths,
                                                                               MOCK_ID_SET)
 
@@ -567,17 +570,10 @@ class TestPackageFilesModified:
 """
 
     def test_changed_runnable_test_non_mocked_get_modified_files(self):
-        (files_list,
-         tests_list,
-         all_tests,
-         is_conf_json,
-         sample_tests,
-         modified_metadata_list,
-         is_reputations_json,
-         is_indicator_json) = get_modified_files_for_testing(self.GIT_DIFF_RET)
-        assert len(sample_tests) == 0
+        modified_files_instance = get_modified_files_for_testing(self.GIT_DIFF_RET)
+        assert len(modified_files_instance.sample_tests) == 0
         assert 'Packs/Active_Directory_Query/Integrations/' \
-               'Active_Directory_Query/Active_Directory_Query.yml' in files_list
+               'Active_Directory_Query/Active_Directory_Query.yml' in modified_files_instance.modified_files
 
 
 class TestNoChange:
@@ -591,19 +587,20 @@ def test_no_change(self, mocker):
 
 def create_get_modified_files_ret(modified_files_list=None, modified_tests_list=None, changed_common=None,
                                   is_conf_json=False, sample_tests=None, modified_metadata_list=None,
-                                  is_reputations_json=None, is_indicator_json=None):
+                                  is_reputations_json=None, is_indicator_json=None, is_landing_page_sections_json=None):
     """
     Returns return value for get_modified_files() to be used with a mocker patch
     """
-    return (
+    return ModifiedFiles(
         modified_files_list if modified_files_list is not None else [],
         modified_tests_list if modified_tests_list is not None else [],
         changed_common if changed_common is not None else [],
         is_conf_json,
         sample_tests if sample_tests is not None else [],
         modified_metadata_list if modified_metadata_list is not None else [],
         is_reputations_json if is_reputations_json is not None else [],
-        is_indicator_json if is_indicator_json is not None else []
+        is_indicator_json if is_indicator_json is not None else [],
+        is_landing_page_sections_json if is_landing_page_sections_json is not None else False
     )
 
 

Tests/scripts/utils/collect_helpers.py

@@ -61,6 +61,8 @@
 # secrets white list file to be ignored in tests to prevent full tests running each time it is updated
 SECRETS_WHITE_LIST = 'secrets_white_list.json'
 
+LANDING_PAGE_SECTIONS_JSON_PATH = 'Tests/Marketplace/landingPage_sections.json'
+
 
 def checked_type(file_path: str, regex_list: Iterable[str]) -> bool:
     """

Tests/scripts/utils/get_modified_files_for_testing.py

@@ -4,21 +4,56 @@
 import glob
 import os
 from enum import Enum
-from typing import Dict, Set, Optional, Tuple, List
+from typing import Dict, Set, Optional
 
 import demisto_sdk.commands.common.constants as constants
-from demisto_sdk.commands.common import tools
-
 from Tests.scripts.utils.collect_helpers import (
     COMMON_YML_LIST,
-    is_pytest_file, checked_type, SECRETS_WHITE_LIST,
+    is_pytest_file, checked_type, SECRETS_WHITE_LIST, LANDING_PAGE_SECTIONS_JSON_PATH,
 )
+from demisto_sdk.commands.common import tools
 
 
 class FileType(constants.FileType, Enum):
     CONF_JSON = "confjson"
     METADATA = "metadata"
     WHITE_LIST = 'whitelist'
+    LANDING_PAGE_SECTIONS_JSON = 'landingPage_sections.json'
+
+
+class ModifiedFiles:
+    def __init__(self,
+                 modified_files: list,
+                 modified_tests: list,
+                 changed_common_files: list,
+                 is_conf_json: bool,
+                 sample_tests: list,
+                 modified_metadata: set,
+                 is_reputations_json: bool,
+                 is_indicator_json: bool,
+                 is_landing_page_sections_json: bool):
+        """
+        A holder for the 'get_modified_files_for_testing' method's response
+        Args:
+            modified_files: Modified YMLs for testing (Integrations, Scripts, Playbooks).
+            modified_tests: Test playbooks.
+            changed_common_files: Globally used YMLs (Like CommonServerPython).
+            is_conf_json: If Tests/Conf.json has been changed.
+            sample_tests: Files to test, Like the infrastructures files.
+            modified_metadata: Pack names of changed metadata files.
+            is_reputations_json: If any reputation file changed.
+            is_indicator_json: If any indicator file changed.
+            is_landing_page_sections_json: If Tests/Marketplace/landingPage_sections.json has been changed
+        """
+        self.modified_files = modified_files
+        self.modified_tests = modified_tests
+        self.changed_common_files = changed_common_files
+        self.is_conf_json = is_conf_json
+        self.sample_tests = sample_tests
+        self.modified_metadata = modified_metadata
+        self.is_reputations_json = is_reputations_json
+        self.is_indicator_json = is_indicator_json
+        self.is_landing_page_sections_json = is_landing_page_sections_json
 
 
 def resolve_type(file_path: str) -> Optional[FileType]:
@@ -33,6 +68,9 @@ def resolve_type(file_path: str) -> Optional[FileType]:
     # if conf.json file
     if checked_type(file_path, [constants.CONF_PATH]):
         return FileType.CONF_JSON
+    # landingPage_sections.json file
+    if checked_type(file_path, [LANDING_PAGE_SECTIONS_JSON_PATH]):
+        return FileType.LANDING_PAGE_SECTIONS_JSON
     # MetaData files
     elif any(
         file in file_path
@@ -120,23 +158,14 @@ def remove_common_files(
     return types_to_files
 
 
-def get_modified_files_for_testing(
-    git_diff: str,
-) -> Tuple[List[str], List[str], List[str], bool, List[str], set, bool, bool]:
+def get_modified_files_for_testing(git_diff: str) -> ModifiedFiles:
     """
     Gets git diff string and filters those files into tests:
 
     Args:
         git_diff: a git diff output (with --name-only flag)
     Returns:
-        modified_files: Modified YMLs for testing (Integrations, Scripts, Playbooks).
-        modified_tests: Test playbooks.
-        changed_common_files: Globally used YMLs (Like CommonServerPython).
-        is_conf_json: If Tests/Conf.json has been changed.
-        sample_tests: Files to test, Like the infrastructures files.
-        modified_metadata: Pack names of changed metadata files.
-        is_reputations_json: If any reputation file changed.
-        is_indicator_json: If any indicator file changed.
+        ModifiedFiles instance
     """
     types_to_files: Dict[FileType, Set[str]] = create_type_to_file(git_diff)  # Mapping of the files FileType: file path
 
@@ -162,11 +191,13 @@ def get_modified_files_for_testing(
     # Booleans. If this kind of file is inside, its exists
     is_conf_json = FileType.CONF_JSON in types_to_files
 
+    is_landing_page_sections_json = FileType.LANDING_PAGE_SECTIONS_JSON in types_to_files
+
     is_reputations_json = FileType.REPUTATION in types_to_files
 
     is_indicator_json = FileType.INDICATOR_FIELD in types_to_files
 
-    return (
+    modified_files_instance = ModifiedFiles(
         list(modified_files),
         list(modified_tests),
         list(changed_common_files),
@@ -175,7 +206,9 @@ def get_modified_files_for_testing(
         modified_metadata,
         is_reputations_json,
         is_indicator_json,
-    )
+        is_landing_page_sections_json)
+
+    return modified_files_instance
 
 
 def get_corresponding_yml_file(file_path: str) -> Optional[str]:

Tests/scripts/utils/tests/content_packs_util_test.py

@@ -1,14 +1,15 @@
 import json
 import os
+
 import pytest
-from demisto_sdk.commands.common.constants import (PACK_METADATA_SUPPORT,
-                                                   PACKS_DIR,
-                                                   PACKS_PACK_META_FILE_NAME)
 
+from Tests.scripts.collect_tests_and_content_packs import get_packs_from_landing_page
 from Tests.scripts.utils.content_packs_util import (is_pack_xsoar_supported,
                                                     is_pack_deprecated,
                                                     should_test_content_pack, should_install_content_pack)
-
+from demisto_sdk.commands.common.constants import (PACK_METADATA_SUPPORT,
+                                                   PACKS_DIR,
+                                                   PACKS_PACK_META_FILE_NAME)
 
 with open('Tests/scripts/infrastructure_tests/tests_data/mock_id_set.json', 'r') as mock_id_set_f:
     MOCK_ID_SET = json.load(mock_id_set_f)
@@ -146,3 +147,44 @@ def test_should_install_content_pack(mocker, tmp_path, pack_metadata_content, pa
     # Mocking os.path.join to return the temp path created instead the path in content
     mocker.patch.object(os.path, 'join', return_value=str(pack_metadata_file))
     assert should_install_content_pack(pack_name) == expected
+
+
+def test_get_packs_from_landing_page(mocker, tmp_path):
+    """
+    Given:
+        The diff output for the landing page sections file
+
+    When:
+        - A pack is modified (added or removed)
+
+    Then:
+        - Ensure that pack is returned by the 'get_packs_from_landing_page' method.
+    """
+    git_diff = '''
+    diff --git a/Tests/Marketplace/landingPage_sections.json b/Tests/Marketplace/landingPage_sections.json
+    index 953a91ed0f..eb96c36117 100644
+    --- a/Tests/Marketplace/landingPage_sections.json
+    +++ b/Tests/Marketplace/landingPage_sections.json
+    @@ -17,7 +17,8 @@
+         "NIST",
+         "GenericWebhook",
+         "GraphQL",
+    -    "GenericSQL"
+    +    "GenericSQL",
+    +    "my test pack"
+       ]'''
+    landing_page_sections_mock = {"description": "description",
+                                  "sections": ["Featured"],
+                                  "Featured": [
+                                      "NIST",
+                                      "GenericWebhook",
+                                      "GraphQL",
+                                      "GenericSQL",
+                                      "my test pack"
+                                  ]}
+    landing_page_sections_file = tmp_path / 'landing_page.json'
+    landing_page_sections_file.write_text(json.dumps(landing_page_sections_mock))
+    mocker.patch('Tests.scripts.collect_tests_and_content_packs.LANDING_PAGE_SECTIONS_JSON_PATH',
+                 landing_page_sections_file)
+    mocker.patch('Tests.scripts.collect_tests_and_content_packs.tools.run_command', return_value=git_diff)
+    assert 'my test pack' in get_packs_from_landing_page('branch_name')