Specified events.json

docs/docs/telemetry/events.json

@@ -3,6 +3,7 @@
     "Model Training",
     "Model Testing",
     "Model Serving",
+    "Markers Evaluation",
     "Data Handling",
     "Miscellaneous"
   ],
@@ -422,6 +423,63 @@
         "num_synonyms",
         "num_regexes"
       ]
+    },
+    "Markers Evaluation Initiated": {
+      "description": "Triggered when markers evaluation has been initiated.",
+      "type": "object",
+      "section": "Markers Evaluation",
+      "properties": {
+        "strategy": {
+          "type": "string",
+          "description": "Strategy to use when selecting trackers to extract from."
+        },
+        "only_extract": {
+          "type": "boolean",
+          "description": "Indicates if path to write out statistics hasn't been specified."
+        },
+        "seed": {
+          "type": ["string", "null"],
+          "description": "The seed to initialise the random number generator for use with the 'sample' strategy."
+        },
+        "count": {
+          "type": ["string", "null"],
+          "description": "Number of trackers to extract from (for any strategy except 'all')."
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "count"
+      ]
+    },
+    "Markers Extracted": {
+      "description": "Triggered when markers has been extracted.",
+      "type": "object",
+      "section": "Markers Evaluation",
+      "properties": {
+        "trackers_count": {
+          "type": "integer",
+          "description": "Number of processed trackers."
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "trackers_count"
+      ]
+    },
+    "Markers Stats Computed": {
+      "description": "Triggered when marker stats has been computed.",
+      "type": "object",
+      "section": "Markers Evaluation",
+      "properties": {
+        "trackers_count": {
+          "type": "integer",
+          "description": "Number of processed trackers."
+        }
+      },
+      "additionalProperties": false,
+      "required": [
+        "trackers_count"
+      ]
     }
   }
 }

rasa/cli/evaluate.py

@@ -132,11 +132,8 @@ def _run_markers(
             computed per session will be stored in
             '<path-to-stats-folder>/statistics-per-session.csv'.
     """
-    telemetry.track_markers_evaluation_triggered(
-        strategy=strategy,
-        only_extract=stats_file is None,
-        seed=seed,
-        count=count,
+    telemetry.track_markers_evaluation_initiated(
+        strategy=strategy, only_extract=stats_file is not None, seed=seed, count=count,
     )
 
 	domain = Domain.load(domain_path) if domain_path else None

rasa/telemetry.py

@@ -94,7 +94,7 @@
 TELEMETRY_VISUALIZATION_STARTED_EVENT = "Story Visualization Started"
 TELEMETRY_TEST_CORE_EVENT = "Model Core Tested"
 TELEMETRY_TEST_NLU_EVENT = "Model NLU Tested"
-TELEMETRY_MARKERS_EVALUATION_TRIGGERED_EVENT = "Markers Evaluation Triggered"
+TELEMETRY_MARKERS_EVALUATION_INITIATED_EVENT = "Markers Evaluation Initiated"
 TELEMETRY_MARKERS_EXTRACTED_EVENT = "Markers Extracted"
 TELEMETRY_MARKERS_STATS_COMPUTED_EVENT = "Markers Stats Computed"
 
@@ -1011,14 +1011,11 @@ def track_nlu_model_test(test_data: "TrainingData") -> None:
 
 
 @ensure_telemetry_enabled
-def track_markers_evaluation_triggered(
-        strategy: Text,
-        only_extract: bool,
-        seed: Optional[int],
-        count: Optional[int],
+def track_markers_evaluation_initiated(
+    strategy: Text, only_extract: bool, seed: Optional[int], count: Optional[int],
 ) -> None:
     _track(
-        TELEMETRY_MARKERS_EVALUATION_TRIGGERED_EVENT,
+        TELEMETRY_MARKERS_EVALUATION_INITIATED_EVENT,
         {
             "strategy": strategy,
             "only_extract": only_extract,
@@ -1029,20 +1026,14 @@ def track_markers_evaluation_triggered(
 
 
 @ensure_telemetry_enabled
-def track_markers_extracted(count: int) -> None:
+def track_markers_extracted(trackers_count: int) -> None:
     _track(
-        TELEMETRY_MARKERS_EXTRACTED_EVENT,
-        {
-            "count": count
-        },
+        TELEMETRY_MARKERS_EXTRACTED_EVENT, {"trackers_count": trackers_count},
     )
 
 
 @ensure_telemetry_enabled
-def track_markers_stats_computed(count: int) -> None:
+def track_markers_stats_computed(trackers_count: int) -> None:
     _track(
-        TELEMETRY_MARKERS_STATS_COMPUTED_EVENT,
-        {
-            "count": count
-        },
-    )
+        TELEMETRY_MARKERS_STATS_COMPUTED_EVENT, {"trackers_count": trackers_count},
+    )