feat: added support for YAML file (splunk#536)

* feat(code): ADDON-56381 Added support for YAML file

feat(code): ADDON-56381 Updated global config validator file

feat(code): ADDON-56381 Doc changes

chore(deps): ADDON-56381 adding pyYaml to poetry dependency

ci(pre-commit): add pyYAML as additional deps for mypy

* docs(code): ADDON-56381 Doc changes

.pre-commit-config.yaml

@@ -39,6 +39,7 @@ repos:
     hooks:
     -   id: mypy
         exclude: ^docs/
+        additional_dependencies: ['types-PyYAML']
 -   repo: https://github.com/PyCQA/flake8
     rev: 4.0.1
     hooks:

docs/example.md

@@ -5,7 +5,7 @@ Splunk index that is going to be used in the add-on. Let's also assume
 that you do not want to show indexes that are for internal use only
 (like, `_internal`).
 
-For this you can create a `globalConfig.json` file and specify that you
+For this you can create a `globalConfig.json` or `globalConfig.yaml` it can be of json or yaml type and specify that you
 want one configuration tab called "Global Settings", one UI component on
 that tab that will handle index management and store selected index in
 specific add-on configuration file.
@@ -24,7 +24,7 @@ installed, please refer to installation section.
 The structure of the add-on before running `ucc-gen` command should be
 like this:
 
-    ├── globalConfig.json
+    ├── globalConfig.json or globalConfig.yaml
     └── package
         ├── LICENSE.txt
         ├── README.txt
@@ -64,7 +64,7 @@ After that, `output` folder should be created. It should contain
     │   │       |   ├── 4.licenses.txt
     │   │       |   ├── entry_page.js
     │   │       |   ├── entry_page.licenses.txt
-    │   │       |   └── globalConfig.json
+    │   │       |   └── globalConfig.json or globalConfig.yaml
     |   |       └── dependencies.txt
     │   └── templates
     │       └── base.html

docs/how_to_use.md

@@ -2,16 +2,20 @@
 
 ## Prerequisites
 
--   `globalConfig.json`
+-   `globalConfig.json` or `globalConfig.yaml`
 -   `package` folder
 -   `git` for automatic versioning (when no `ta-version` argument is
     specified)
 
 Example of `globalConfig.json` and `package` folder can be found [here](https://github.com/splunk/splunk-add-on-for-ucc-example).
 
-The JSON schema for the `globalConfig.json` can be found 
+The JSON schema for the globalConfig file can be found
 [here](https://github.com/splunk/addonfactory-ucc-base-ui/blob/main/src/main/webapp/schema/schema.json).
 
+```
+If both globalConfig.json and globalConfig.yaml files are present, then the globalConfig.json file will take precedence.
+```
+
 ## Steps
 
 * Use Python virtual environment:
@@ -26,9 +30,9 @@ The JSON schema for the `globalConfig.json` can be found
 * `source` - [optional] folder containing the `app.manifest` and app 
     source.
 * `config` - [optional] path to the configuration file, defaults to
-    `globalConfig.json` in the parent directory of source provided.
+    globalConfig file in the parent directory of source provided.
 * `ta-version` - [optional] override current version of TA, default
-    version is version specified in `globalConfig.json`. Splunkbase
+    version is version specified in `globalConfig.json` or `globalConfig.yaml`. Splunkbase
     compatible version of SEMVER will be used by default.
 * `python-binary-name` - [optional] Python binary name to use when
     installing Python libraries.
@@ -46,7 +50,7 @@ package for distribution.
 * Cleans the output folder.
 * Retrieve the package ID of addon.
 * Copy UCC template directory under `output/<package_ID>` directory.
-* Copy globalConfig.json file to
+* Copy globalConfig.json or globalConfig.yaml file to
     `output/<package_ID>/appserver/static/js/build` directory.
 * Collect and install Addon's requirements into
     `output/<package_ID>/lib` directory of addon's package.
@@ -64,7 +68,7 @@ package for distribution.
 
 ## `additional_packaging.py` file
 
-To extend the build process, you can create `additional_packaging.py` file in the same file level where you have your `globalConfig.json`.
+To extend the build process, you can create `additional_packaging.py` file in the same file level where you have your globalConfig file.
 
 This file should have `additional_packaging` function which accepts 1 argument: add-on name.
 

pyproject.toml

@@ -40,6 +40,7 @@ jinja2 = ">=2,<4"
 addonfactory-splunk-conf-parser-lib = "^0.3.3"
 dunamai = "^1.9.0"
 jsonschema = "^4.4.0"
+PyYAML = "^6.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.1"

splunk_add_on_ucc_framework/commands/build.py

@@ -14,12 +14,14 @@
 # limitations under the License.
 #
 import configparser
+import functools
 import json
 import logging
 import os
 import shutil
 import sys
 
+import yaml
 from jinja2 import Environment, FileSystemLoader
 
 from splunk_add_on_ucc_framework import (
@@ -46,20 +48,29 @@
     loader=FileSystemLoader(os.path.join(internal_root_dir, "templates"))
 )
 
+Loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
+yaml_load = functools.partial(yaml.load, Loader=Loader)
 
-def _update_ta_version(config, ta_version):
+
+def _update_ta_version(config, ta_version, is_global_config_yaml):
     """
-    Update version of TA in globalConfig.json.
+    Update version of TA in globalConfig file.
 
     Args:
         args (argparse.Namespace): Object with command-line arguments.
     """
 
     with open(config) as config_file:
-        schema_content = json.load(config_file)
+        if is_global_config_yaml:
+            schema_content = yaml_load(config_file)
+        else:
+            schema_content = json.load(config_file)
     schema_content.setdefault("meta", {})["version"] = ta_version
     with open(config, "w") as config_file:
-        json.dump(schema_content, config_file, indent=4)
+        if is_global_config_yaml:
+            yaml.dump(schema_content, config_file)
+        else:
+            json.dump(schema_content, config_file, ensure_ascii=False, indent=4)
 
 
 def _recursive_overwrite(src, dest, ignore_list=None):
@@ -140,10 +151,10 @@ def _generate_rest(ta_name, scheme, import_declare_name, outputdir):
 
 def _is_oauth_configured(ta_tabs):
     """
-    Check if oauth is configured in globalConfig.json.
+    Check if oauth is configured in globalConfig file.
 
     Args:
-        ta_tabs (list): List of tabs mentioned in globalConfig.json.
+        ta_tabs (list): List of tabs mentioned in globalConfig file.
 
     Returns:
         bool: True if oauth is configured, False otherwise.
@@ -190,7 +201,7 @@ def _modify_and_replace_token_for_oauth_templates(
     Args:
         ta_name (str): Name of TA.
         ta_version (str): Version of TA.
-        ta_tabs (list): List of tabs mentioned in globalConfig.json.
+        ta_tabs (list): List of tabs mentioned in globalConfig file.
         outputdir (str): output directory.
     """
     redirect_xml_src = os.path.join(
@@ -246,7 +257,7 @@ def _add_modular_input(ta_name, schema_content, import_declare_name, outputdir):
 
     Args:
         ta_name (str): Name of TA.
-        schema_content (dict): JSON schema of globalConfig.json.
+        schema_content (dict): schema of globalConfig file.
         outputdir (str): output directory.
     """
 
@@ -298,7 +309,7 @@ def _make_modular_alerts(ta_name, ta_namespace, schema_content, outputdir):
     Args:
         ta_name (str): Name of TA.
         ta_namespace (str): restRoot of TA.
-        schema_content (dict): JSON schema of globalConfig.json.
+        schema_content (dict): schema of globalConfig file.
         outputdir (str): output directory.
     """
 
@@ -434,7 +445,13 @@ def generate(source, config, ta_version, outputdir=None, python_binary_name="pyt
 
     # Setting default value to Config argument
     if not config:
+        is_global_config_yaml = False
         config = os.path.abspath(os.path.join(source, PARENT_DIR, "globalConfig.json"))
+        if not os.path.isfile(config):
+            config = os.path.abspath(
+                os.path.join(source, PARENT_DIR, "globalConfig.yaml")
+            )
+            is_global_config_yaml = True
 
     logger.info(f"Cleaning out directory {outputdir}")
     shutil.rmtree(os.path.join(outputdir), ignore_errors=True)
@@ -458,22 +475,31 @@ def generate(source, config, ta_version, outputdir=None, python_binary_name="pyt
         sys.exit(1)
     ta_name = manifest.get_addon_name()
 
-    if os.path.exists(config):
+    if os.path.isfile(config):
         try:
             with open(config) as f_config:
                 config_raw = f_config.read()
-            validator = global_config_validator.GlobalConfigValidator(
-                internal_root_dir, json.loads(config_raw)
-            )
+
+            if is_global_config_yaml:
+                validator = global_config_validator.GlobalConfigValidator(
+                    internal_root_dir, yaml_load(config_raw)
+                )
+            else:
+                validator = global_config_validator.GlobalConfigValidator(
+                    internal_root_dir, json.loads(config_raw)
+                )
+
             validator.validate()
             logger.info("Config is valid")
         except global_config_validator.GlobalConfigValidatorException as e:
             logger.error(f"Config is not valid. Error: {e}")
             sys.exit(1)
 
-        _update_ta_version(config, ta_version)
+        _update_ta_version(config, ta_version, is_global_config_yaml)
 
-        schema_content = global_config_update.handle_global_config_update(config)
+        schema_content = global_config_update.handle_global_config_update(
+            config, is_global_config_yaml
+        )
 
         scheme = global_config.GlobalConfigBuilderSchema(schema_content, j2_env)
 
@@ -492,6 +518,9 @@ def generate(source, config, ta_version, outputdir=None, python_binary_name="pyt
         )
 
         logger.info("Copy globalConfig to output")
+        global_config_file = (
+            "globalConfig.yaml" if is_global_config_yaml else "globalConfig.json"
+        )
         shutil.copyfile(
             config,
             os.path.join(
@@ -501,7 +530,7 @@ def generate(source, config, ta_version, outputdir=None, python_binary_name="pyt
                 "static",
                 "js",
                 "build",
-                "globalConfig.json",
+                global_config_file,
             ),
         )
         ucc_lib_target = os.path.join(outputdir, ta_name, "lib")
@@ -535,7 +564,7 @@ def generate(source, config, ta_version, outputdir=None, python_binary_name="pyt
     else:
         logger.info("Addon Version : " + ta_version)
         logger.warning(
-            "Skipped generating UI components as globalConfig.json does not exist."
+            "Skipped generating UI components as globalConfig file does not exist."
         )
         ucc_lib_target = os.path.join(outputdir, ta_name, "lib")
 

splunk_add_on_ucc_framework/global_config_update.py

@@ -13,11 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import functools
 import json
 import logging
 
+import yaml
+
 logger = logging.getLogger("ucc_gen")
 
+Loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
+yaml_load = functools.partial(yaml.load, Loader=Loader)
+
 
 def _version_tuple(version_str):
     """
@@ -71,25 +77,32 @@ def _handle_dropping_api_version_update(schema_content: dict) -> dict:
     return schema_content
 
 
-def handle_global_config_update(config_path: str) -> dict:
-    """Handle changes in globalConfig.json.
+def handle_global_config_update(config_path: str, is_global_config_yaml: bool) -> dict:
+    """Handle changes in globalConfig file.
 
     Args:
         logger: Logger instance
-        config_path: Path to globalConfig.json file
+        config_path: Path to globalConfig file
+        is_global_config_yaml: True if globalconfig file is of type yaml
 
     Returns:
-        Content of the updated globalConfig.json file in a dictionary format.
+        Content of the updated globalConfig file in a dictionary format.
     """
     with open(config_path) as config_file:
-        schema_content = json.load(config_file)
+        if is_global_config_yaml:
+            schema_content = yaml_load(config_file)
+        else:
+            schema_content = json.load(config_file)
 
     version = schema_content.get("meta").get("schemaVersion", "0.0.0")
 
     if _version_tuple(version) < _version_tuple("0.0.1"):
         schema_content = _handle_biased_terms_update(schema_content)
         with open(config_path, "w") as config_file:
-            json.dump(schema_content, config_file, ensure_ascii=False, indent=4)
+            if is_global_config_yaml:
+                yaml.dump(schema_content, config_file, indent=4)
+            else:
+                json.dump(schema_content, config_file, ensure_ascii=False, indent=4)
 
     if _version_tuple(version) < _version_tuple("0.0.2"):
         ta_tabs = schema_content.get("pages").get("configuration", {}).get("tabs", {})
@@ -104,7 +117,7 @@ def handle_global_config_update(config_path: str) -> dict:
                             "oauth_state_enabled field is no longer a separate "
                             "entity since UCC version 5.0.0. It is now an "
                             "option in the oauth field. Please update the "
-                            "globalconfig.json file accordingly."
+                            "globalconfig file accordingly."
                         )
                         oauth_state_enabled_entity = entity
 
@@ -154,11 +167,17 @@ def handle_global_config_update(config_path: str) -> dict:
 
         schema_content["meta"]["schemaVersion"] = "0.0.2"
         with open(config_path, "w") as config_file:
-            json.dump(schema_content, config_file, ensure_ascii=False, indent=4)
+            if is_global_config_yaml:
+                yaml.dump(schema_content, config_file, indent=4)
+            else:
+                json.dump(schema_content, config_file, ensure_ascii=False, indent=4)
 
     if _version_tuple(version) < _version_tuple("0.0.3"):
         schema_content = _handle_dropping_api_version_update(schema_content)
         with open(config_path, "w") as config_file:
-            json.dump(schema_content, config_file, ensure_ascii=False, indent=4)
+            if is_global_config_yaml:
+                yaml.dump(schema_content, config_file, indent=4)
+            else:
+                json.dump(schema_content, config_file, ensure_ascii=False, indent=4)
 
     return schema_content

splunk_add_on_ucc_framework/global_config_validator.py

@@ -26,7 +26,7 @@ class GlobalConfigValidatorException(Exception):
 
 class GlobalConfigValidator:
     """
-    GlobalConfigValidator implements different validation for globalConfig.json.
+    GlobalConfigValidator implements different validation for globalConfig file.
     Simple validation should go to JSON schema in
     https://github.com/splunk/addonfactory-ucc-base-ui repository.
     Custom validation should be implemented here.

tests/unit/helpers.py

@@ -13,7 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import functools
+import json
 import os
+from typing import Dict
+
+import yaml
+
+Loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
+yaml_load = functools.partial(yaml.load, Loader=Loader)
 
 
 def assert_identical_files(expected_file_path: str, file_path: str) -> bool:
@@ -34,3 +42,11 @@ def get_testdata_file(file_name: str) -> str:
     file_path = get_testdata_file_path(file_name)
     with open(file_path) as fp:
         return fp.read()
+
+
+def get_testdata(file_name: str) -> Dict:
+    config = get_testdata_file(file_name)
+    if file_name[-4] == "json":
+        return json.loads(config)
+    else:
+        return yaml_load(config)