Validated SPEC fields uppon PR and commit

.github/workflows/validate-spec-fields.yml

@@ -0,0 +1,39 @@
+name: Validate SPEC fields
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  pytest:
+    name: Validate SPEC fields
+    runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.10'
+
+    - uses: docker-practice/actions-setup-docker@master
+
+    - name: Install Poetry
+      run: |
+        curl -sSL https://install.python-poetry.org | python3 -
+
+    - name: Install Dev requirements
+      run: |
+        pip install -U setuptools wheel
+        pip install -r dev_requirements.txt
+
+    - name: Install Dependencies
+      run: |
+        poetry install
+
+    - name: Validate SPEC fields
+      run: |
+        poetry run redis-benchmarks-spec-cli --tool stats --fail-on-required-diff

redis_benchmarks_specification/__cli__/args.py

@@ -40,6 +40,12 @@ def spec_cli_args(parser):
         action="store_true",
         help="Override test specs.",
     )
+    parser.add_argument(
+        "--fail-on-required-diff",
+        default=False,
+        action="store_true",
+        help="Fail tool when there is difference between required parameters.",
+    )
     parser.add_argument(
         "--push-stats-redis",
         default=False,

redis_benchmarks_specification/__cli__/stats.py

@@ -44,11 +44,15 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
         )
     )
     tracked_groups = []
+    override_enabled = args.override_tests
+    fail_on_required_diff = args.fail_on_required_diff
+    overall_result = True
     for test_file in testsuite_spec_files:
         benchmark_config = {}
         requires_override = False
-        override_enabled = args.override_tests
+        test_result = True
         with open(test_file, "r") as stream:
+
             try:
                 benchmark_config = yaml.safe_load(stream)
                 test_name = benchmark_config["name"]
@@ -126,8 +130,13 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                         test_file, e.__str__()
                     )
                 )
+                test_result = False
                 pass
 
+        if requires_override:
+            test_result = False
+        overall_result &= test_result
+
         if requires_override and override_enabled:
             logging.info(
                 "Saving a new version of the file {} with the overrided data".format(
@@ -146,6 +155,12 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
     total_tracked_groups = len(tracked_groups)
     logging.info("Total tracked groups: {}".format(total_tracked_groups))
 
+    if overall_result is False and fail_on_required_diff:
+        logging.error(
+            "Failing given there were changes required to be made and --fail-on-required-diff was enabled"
+        )
+        exit(1)
+
     if args.commandstats_csv != "":
         logging.info(
             "Reading commandstats csv {} to determine commands/test coverage".format(