markers / validation (RasaHQ#10012)

* validation; name vs tag; specify expected number of sub-markers;... * Delete old Marker test fixtures and associated files. * Fix tests after rebasing main * First pass on domain validation * Add some tests around domain validation Need to work out why set union over a generator expression wasn't working * Remove unused markers test data * Config validation tests * Fix lint * add tests for from_path; simplify other tests * make domain path optional for _run_markers * Codeclimate fixes * Codeclimate fixes * Swap over warnings for errors Co-authored-by: aeshky <aciel.eshky@gmail.com> Co-authored-by: Matthew Summers <m.summers@rasa.com>
kode-git · Nov 2, 2021 · 9142bd4 · 9142bd4
1 parent 6df732c
commit 9142bd4
Show file tree

Hide file tree

Showing 14 changed files with 676 additions and 642 deletions.
diff --git a/data/test_markers/config_dir/config_1.yml b/data/test_markers/config_dir/config_1.yml
diff --git a/data/test_markers/config_dir/config_2.yml b/data/test_markers/config_dir/config_2.yml
diff --git a/data/test_markers/config_dir/config_3.yml b/data/test_markers/config_dir/config_3.yml
diff --git a/data/test_markers/config_dir/not_a_config.txt b/data/test_markers/config_dir/not_a_config.txt
diff --git a/data/test_markers/config_invalid.yml b/data/test_markers/config_invalid.yml
diff --git a/data/test_markers/config_operators.yml b/data/test_markers/config_operators.yml
diff --git a/data/test_markers/config_simple.yml b/data/test_markers/config_simple.yml
diff --git a/data/test_markers/extracted_markers.json b/data/test_markers/extracted_markers.json
diff --git a/rasa/cli/evaluate.py b/rasa/cli/evaluate.py
@@ -1,4 +1,5 @@
 import argparse
+from rasa.shared.core.domain import Domain
 from typing import List, Text, Optional
 
 from rasa.core.utils import AvailableEndpoints
@@ -87,6 +88,7 @@ def _run_markers_cli(args: argparse.Namespace) -> None:
         seed,
         count,
         args.endpoints,
+        args.domain,
         args.strategy,
         args.config,
         args.output_filename,
@@ -98,6 +100,7 @@ def _run_markers(
     seed: Optional[int],
     count: Optional[int],
     endpoint_config: Text,
+    domain_path: Optional[Text],
     strategy: Text,
     config: Text,
     output_filename: Text,
@@ -112,6 +115,8 @@ def _run_markers(
                except 'all').
         endpoint_config: Path to the endpoint configuration defining the tracker
                          store to use.
+        domain_path: Path to the domain specification to use when validating the
+                     marker definitions.
         strategy: Strategy to use when selecting trackers to extract from.
         config: Path to the markers definition file to use.
         output_filename: Path to write out the extracted markers.
@@ -128,8 +133,16 @@ def _run_markers(
             "A file with the stats filename already exists"
         )
 
-    tracker_loader = _create_tracker_loader(endpoint_config, strategy, count, seed)
+    domain = Domain.load(domain_path) if domain_path else None
     markers = Marker.from_path(config)
+
+    if domain and not markers.validate_against_domain(domain):
+        rasa.shared.utils.cli.print_error_and_exit(
+            "Validation errors were found in the markers definition. "
+            "Please see errors listed above and fix before running again."
+        )
+
+    tracker_loader = _create_tracker_loader(endpoint_config, strategy, count, seed)
     markers.export_markers(tracker_loader.load(), output_filename, stats_file)