Annotate dashboard and recommended monitors validation

.github/workflows/validations.yaml

@@ -35,6 +35,10 @@ jobs:
       run: |
         ddev validate config
 
+    - name: Run dashboard validation
+      run: |
+        ddev validate dashboards
+
     - name: Run HTTP wrapper validation
       run: |
         ddev validate http
@@ -43,6 +47,10 @@ jobs:
       run: |
         ddev validate legacy-signature
 
+    - name: Run recommended monitors validation
+      run: |
+        ddev validate recommended-monitors
+
     - name: Run service checks validation
       run: |
         ddev validate service-checks

datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/dashboards.py

@@ -2,12 +2,14 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import json
+import os
 
 import click
 
 from ....utils import read_file
+from ...annotations import annotate_display_queue, annotate_error
 from ...testing import process_checks_option
-from ...utils import complete_valid_checks, get_assets_from_manifest
+from ...utils import complete_valid_checks, get_assets_from_manifest, get_manifest_file
 from ..console import CONTEXT_SETTINGS, abort, echo_failure, echo_info, echo_success
 
 REQUIRED_ATTRIBUTES = {"description", "template_variables", "widgets"}
@@ -47,42 +49,49 @@ def dashboards(check):
         file_failed = False
 
         dashboard_relative_locations, invalid_files = get_assets_from_manifest(check_name, 'dashboards')
+        manifest_file = get_manifest_file(check_name)
         for invalid in invalid_files:
+            message = f'{invalid} does not exist'
             echo_info(f'{check_name}... ', nl=False)
             echo_info(' FAILED')
-            echo_failure(f'  {invalid} does not exist')
+            echo_failure('  ' + message)
             failed_checks += 1
+            annotate_error(manifest_file, message)
 
         for dashboard_file in dashboard_relative_locations:
+            dashboard_filename = os.path.basename(dashboard_file)
             try:
                 decoded = json.loads(read_file(dashboard_file).strip())
             except json.JSONDecodeError as e:
                 failed_checks += 1
+                message = f'invalid json: {e}'
                 echo_info(f'{check_name}... ', nl=False)
                 echo_failure(' FAILED')
-                echo_failure(f'  invalid json: {e}')
+                echo_failure('  ' + message)
+                annotate_error(dashboard_file, message)
                 continue
 
             all_keys = set(decoded.keys())
             if not REQUIRED_ATTRIBUTES.issubset(all_keys):
                 missing_fields = REQUIRED_ATTRIBUTES.difference(all_keys)
                 file_failed = True
                 display_queue.append(
-                    (echo_failure, f"    {dashboard_file} does not contain the required fields: {missing_fields}"),
+                    (echo_failure, f"    {dashboard_filename} does not contain the required fields: {missing_fields}"),
                 )
 
             # Confirm the dashboard payload comes from the old API for now
             if not _is_dashboard_format(decoded):
                 file_failed = True
                 display_queue.append(
-                    (echo_failure, f'    {dashboard_file} is not using the new /dashboard payload format.'),
+                    (echo_failure, f'    {dashboard_filename} is not using the new /dashboard payload format.'),
                 )
 
             if file_failed:
                 failed_checks += 1
                 # Display detailed info if file is invalid
                 echo_info(f'{check_name}... ', nl=False)
                 echo_failure(' FAILED')
+                annotate_display_queue(dashboard_file, display_queue)
                 for display_func, message in display_queue:
                     display_func(message)
                 display_queue = []

datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/recommended_monitors.py

@@ -8,8 +8,9 @@
 import click
 
 from ....utils import read_file
+from ...annotations import annotate_display_queue, annotate_error
 from ...testing import process_checks_option
-from ...utils import complete_valid_checks, get_assets_from_manifest, load_manifest
+from ...utils import complete_valid_checks, get_assets_from_manifest, get_manifest_file, load_manifest
 from ..console import CONTEXT_SETTINGS, abort, echo_failure, echo_info, echo_success
 
 REQUIRED_ATTRIBUTES = {'name', 'type', 'query', 'message', 'tags', 'options', 'recommended_monitor_metadata'}
@@ -41,11 +42,14 @@ def recommended_monitors(check):
         file_failed = False
         manifest = load_manifest(check_name)
         monitors_relative_locations, invalid_files = get_assets_from_manifest(check_name, 'monitors')
+        manifest_file = get_manifest_file(check_name)
         for file in invalid_files:
             echo_info(f'{check_name}... ', nl=False)
             echo_info(' FAILED')
-            echo_failure(f'  {file} does not exist')
+            message = f'{file} does not exist'
+            echo_failure('  ' + message)
             failed_checks += 1
+            annotate_error(manifest_file, message)
 
         for monitor_file in monitors_relative_locations:
             monitor_filename = os.path.basename(monitor_file)
@@ -55,7 +59,9 @@ def recommended_monitors(check):
                 failed_checks += 1
                 echo_info(f'{check_name}... ', nl=False)
                 echo_failure(' FAILED')
-                echo_failure(f'  invalid json: {e}')
+                message = f'invalid json: {e}'
+                echo_failure('  ' + message)
+                annotate_error(monitor_file, message)
                 continue
 
             all_keys = set(decoded.keys())
@@ -75,8 +81,7 @@ def recommended_monitors(check):
                     ),
                 )
             else:
-                # If all required keys exist, validate values
-
+                # If all required keys exist, validate value
                 monitor_type = decoded.get('type')
                 if monitor_type not in ALLOWED_MONITOR_TYPES:
                     file_failed = True
@@ -112,16 +117,17 @@ def recommended_monitors(check):
                         (echo_failure, f"    {monitor_filename} name must contain the integration name"),
                     )
 
-        if file_failed:
-            failed_checks += 1
-            # Display detailed info if file is invalid
-            echo_info(f'{check_name}... ', nl=False)
-            echo_failure(' FAILED')
-            for display_func, message in display_queue:
-                display_func(message)
-            display_queue = []
-        else:
-            ok_checks += 1
+            if file_failed:
+                failed_checks += 1
+                # Display detailed info if file is invalid
+                echo_info(f'{check_name}... ', nl=False)
+                echo_failure(' FAILED')
+                annotate_display_queue(monitor_file, display_queue)
+                for display_func, message in display_queue:
+                    display_func(message)
+                display_queue = []
+            else:
+                ok_checks += 1
 
     if ok_checks:
         echo_success(f"{ok_checks} valid files")