Start changing over args to new format

rasa/__main__.py

@@ -23,6 +23,7 @@
     train,
     visualize,
     x,
+    evaluate,
 )
 from rasa.cli.arguments.default_arguments import add_logging_options
 from rasa.cli.utils import parse_last_positional_argument_as_model_path
@@ -69,6 +70,7 @@ def create_argument_parser() -> argparse.ArgumentParser:
     data.add_subparser(subparsers, parents=parent_parsers)
     export.add_subparser(subparsers, parents=parent_parsers)
     x.add_subparser(subparsers, parents=parent_parsers)
+    evaluate.add_subparser(subparsers, parents=parent_parsers)
 
     return parser
 

rasa/cli/arguments/evaluate.py

@@ -0,0 +1,77 @@
+import argparse
+from rasa.cli.arguments.default_arguments import add_endpoint_param
+
+
+def set_markers_arguments(parser: argparse.ArgumentParser):
+    """Specifies arguments for `rasa markers extract`"""
+    parser.add_argument(
+        "--by",
+        choices=["first-n", "sample", "all"],
+        required=True,
+        dest="strategy",
+        help="The strategy used to select trackers for marker extraction",
+    )
+
+    parser.add_subparsers()
+
+    parser.add_argument(
+        "output_filename",
+        type=argparse.FileType("w"),
+        help="The filename to write the extracted markers to",
+    )
+
+    stats = parser.add_mutually_exclusive_group()
+
+    stats.add_argument(
+        "--no-stats",
+        default=True,
+        action="store_false",
+        dest="stats",
+        help="Do not compute summary statistics",
+    )
+
+    stats.add_argument(
+        "--stats-file",
+        type=argparse.FileType("w"),
+        help="Che filename to write out computed summary statistics",
+    )
+
+    first_n_arguments = parser.add_argument_group("Arguments for strategy 'by_first_n'")
+    set_markers_first_n_arguments(first_n_arguments)
+
+    sample_arguments = parser.add_argument_group("Arguments for strategy 'by_sample'")
+    set_markers_sample_arguments(sample_arguments)
+
+    add_endpoint_param(parser)
+
+
+def set_markers_first_n_arguments(parser: argparse.ArgumentParser):
+    """Specifies arguments for `rasa markers by_first_n`"""
+    parser.add_argument(
+        "--num-trackers",
+        type=int,
+        dest="count",
+        help="The number of trackers to extract markers from",
+    )
+
+
+def set_markers_sample_arguments(parser: argparse.ArgumentParser):
+    """Specifies arguments for `rasa markers by_sample"""
+    parser.add_argument(
+        "--seed", type=int, help="Seed to use if selecting trackers by 'sample'"
+    )
+    parser.add_argument(
+        "--num-trackers",
+        type=int,
+        dest="count",
+        help="The number of trackers to extract markers from",
+    )
+
+
+def set_markers_all_arguments(parser: argparse.ArgumentParser):
+    pass
+
+
+def set_evaluate_arguments(parser: argparse.ArgumentParser):
+    # Two subparsers
+    pass

rasa/cli/markers.py → rasa/cli/evaluate.py

@@ -18,7 +18,17 @@ def add_subparser(
         subparsers: subparser we are going to attach to
         parents: Parent parsers, needed to ensure tree structure in argparse
     """
-    marker_parser = subparsers.add_parser(
+    evaluate_parser = subparsers.add_parser(
+        "evaluate",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Tools for evaluating models.",
+    )
+
+    evaluate_subparsers = evaluate_parser.add_subparsers()
+
+    marker_parser = evaluate_subparsers.add_parser(
         "marker",
         parents=parents,
         conflict_handler="resolve",
@@ -29,26 +39,35 @@ def add_subparser(
     markers_arguments.set_markers_arguments(marker_parser)
 
     markers_subparser = marker_parser.add_subparsers()
-    marker_extract_parser = markers_subparser.add_parser(
-        "extract",
+
+    markers_first_n_subparser = markers_subparser.add_parser(
+        "by_first_n",
         parents=parents,
         conflict_handler="resolve",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        help="Extract markers from trackers in a tracker store.",
+        help="Tests Rasa models using your test NLU data and stories.",
     )
-    markers_arguments.set_markers_extract_arguments(marker_extract_parser)
+    markers_arguments.set_markers_first_n_arguments(markers_first_n_subparser)
 
-    marker_extract_parser.set_defaults(func=_run_extract)
+    markers_sample_subparser = markers_subparser.add_parser(
+        "by_sample",
+        parents=parents,
+        conflict_handler="resolve",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        help="Tests Rasa models using your test NLU data and stories.",
+    )
+    markers_arguments.set_markers_sample_arguments(markers_sample_subparser)
 
-    marker_stats_parser = markers_subparser.add_parser(
-        "stats",
+    markers_all_subparser = markers_subparser.add_parser(
+        "by_all",
         parents=parents,
+        conflict_handler="resolve",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-        help="Compute stats from previously extracted markers.",
+        help="Tests Rasa models using your test NLU data and stories.",
     )
-    markers_arguments.set_markers_stats_arguments(marker_stats_parser)
+    markers_arguments.set_markers_all_arguments(markers_all_subparser)
 
-    marker_stats_parser.set_defaults(func=_run_stats)
+    marker_parser.set_defaults(func=_run_extract)
 
 
 def _run_extract(args: argparse.Namespace):
@@ -65,12 +84,6 @@ def _run_extract(args: argparse.Namespace):
     pass
 
 
-def _run_stats(args: argparse.Namespace):
-    # markers = ConversationMarkers.from_file(args.input_filename)
-    # _stats(markers, args.output_filename)
-    pass
-
-
 def _stats(markers, out_file: TextIO):
     # TODO: Figure out how this is done
     pass