Allow to specify a test to run on cli runner; Included pipeline 10 redis-benchmark tests

pyproject.toml

@@ -16,8 +16,8 @@ argparse = "^1.4.0"
 Flask-HTTPAuth = "^4.4.0"
 PyYAML = "^5.4.1"
 docker = "^5.0.0"
-redisbench-admin = "^0.5.27"
-# redisbench-admin = {path = "../redisbench-admin", develop = true}
+redisbench-admin = "^0.6.12"
+#redisbench-admin = {path = "../redisbench-admin", develop = true}
 psutil = "^5.8.0"
 tox-docker = "^3.1.0"
 PyGithub = "^1.55"

redis_benchmarks_specification/__runner__/args.py

@@ -36,8 +36,16 @@ def create_client_runner_args(project_name):
         default=SPECS_PATH_TEST_SUITES,
         help="Test suites folder, containing the different test variations",
     )
+    parser.add_argument(
+        "--test",
+        type=str,
+        default="",
+        help="specify a test to run. By default will run all the tests"
+        + " present in the folder specified in --test-suites-folder.",
+    )
     parser.add_argument("--db_server_host", type=str, default="localhost")
     parser.add_argument("--db_server_port", type=int, default=6379)
+    parser.add_argument("--cpuset_start_pos", type=int, default=0)
     parser.add_argument(
         "--datasink_redistimeseries_host", type=str, default=DATASINK_RTS_HOST
     )

redis_benchmarks_specification/__runner__/runner.py

@@ -72,7 +72,7 @@ def main():
     logging.info(get_version_string(project_name, project_version))
     testsuites_folder = os.path.abspath(args.test_suites_folder)
     logging.info("Using test-suites folder dir {}".format(testsuites_folder))
-    testsuite_spec_files = get_benchmark_specs(testsuites_folder)
+    testsuite_spec_files = get_benchmark_specs(testsuites_folder, args.test)
     logging.info(
         "There are a total of {} test-suites in folder {}".format(
             len(testsuite_spec_files), testsuites_folder
@@ -175,7 +175,7 @@ def process_self_contained_coordinator_stream(
             for topology_spec_name in benchmark_config["redis-topologies"]:
                 test_result = False
                 try:
-                    current_cpu_pos = 0
+                    current_cpu_pos = args.cpuset_start_pos
                     temporary_dir_client = tempfile.mkdtemp(dir=home)
 
                     tf_github_org = args.github_org
@@ -318,7 +318,7 @@ def process_self_contained_coordinator_stream(
                         full_result_path = "{}/{}".format(
                             temporary_dir_client, local_benchmark_output_filename
                         )
-                    logging.critical(
+                    logging.info(
                         "Reading results json from {}".format(full_result_path)
                     )
 
@@ -330,7 +330,7 @@ def process_self_contained_coordinator_stream(
                         logging.info("Final JSON result {}".format(results_dict))
                     dataset_load_duration_seconds = 0
 
-                    logging.error(
+                    logging.info(
                         "Using datapoint_time_ms: {}".format(datapoint_time_ms)
                     )
 
@@ -464,12 +464,19 @@ def data_prepopulation_step(
         )
 
 
-def get_benchmark_specs(testsuites_folder):
-    files = pathlib.Path(testsuites_folder).glob("*.yml")
-    files = [str(x) for x in files]
-    logging.info(
-        "Running all specified benchmarks: {}".format(" ".join([str(x) for x in files]))
-    )
+def get_benchmark_specs(testsuites_folder, test):
+    if test == "":
+        files = pathlib.Path(testsuites_folder).glob("*.yml")
+        files = [str(x) for x in files]
+        logging.info(
+            "Running all specified benchmarks: {}".format(
+                " ".join([str(x) for x in files])
+            )
+        )
+    else:
+        files = test.split(",")
+        files = ["{}/{}".format(testsuites_folder, x) for x in files]
+        logging.info("Running specific benchmark in file: {}".format(files))
     return files
 
 

redis_benchmarks_specification/test-suites/redis-benchmark-full-suite-1Mkeys-100B-pipeline-10.yml

@@ -0,0 +1,58 @@
+version: 0.4
+name: "redis-benchmark-full-suite-1Mkeys-100B-pipeline-10"
+description: "Runs the default redis-benchmark test suite, for a keyspace length of 1M keys
+              with a data size of 100 Bytes for each key. On total 50 concurrent connections 
+              will be used, sending 1M requests."
+dbconfig:
+  - configuration-parameters:
+    - save: '""'
+tested-commands:
+  - PING
+  - SET
+  - GET
+  - INCR
+  - LPUSH
+  - RPUSH
+  - LPOP
+  - RPOP
+  - SADD
+  - SPOP
+  - ZADD
+  - ZPOPMIN
+  - LRANGE
+  - MSET
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redis:6.2.4
+  tool: redis-benchmark
+  min-tool-version: "6.2.0"
+  parameters:
+    - clients: 50
+    - requests: 1000000
+    - threads: 3
+    - pipeline: 1
+    - keyspacelen: 1000000
+    - size: 100
+  resources:
+    requests:
+      cpus: "3"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: "$.StartTime"
+    metrics:
+      - "$.Tests.*.rps"
+      - "$.Tests.*.avg_latency_ms"
+      - "$.Tests.*.p50_latency_ms"
+      - "$.Tests.*.p95_latency_ms"
+      - "$.Tests.*.p99_latency_ms"
+      - "$.Tests.*.max_latency_ms"
+      - "$.Tests.*.min_latency_ms"

redis_benchmarks_specification/test-suites/redis-benchmark-full-suite-1Mkeys-1KiB-pipeline-10.yml

@@ -0,0 +1,58 @@
+version: 0.4
+name: "redis-benchmark-full-suite-1Mkeys-1KiB-pipeline-10"
+description: "Runs the default redis-benchmark test suite, for a keyspace length of 1M keys
+              with a data size of 1000 Bytes for each key. On total 50 concurrent connections
+              will be used, sending 1M requests."
+dbconfig:
+  - configuration-parameters:
+    - save: '""'
+tested-commands:
+  - PING
+  - SET
+  - GET
+  - INCR
+  - LPUSH
+  - RPUSH
+  - LPOP
+  - RPOP
+  - SADD
+  - SPOP
+  - ZADD
+  - ZPOPMIN
+  - LRANGE
+  - MSET
+redis-topologies:
+  - oss-standalone
+
+build-variants:
+  - gcc:8.5.0-amd64-debian-buster-default
+
+clientconfig:
+  run_image: redis:6.2.4
+  tool: redis-benchmark
+  min-tool-version: "6.2.0"
+  parameters:
+    - clients: 50
+    - requests: 1000000
+    - threads: 3
+    - pipeline: 10
+    - keyspacelen: 1000000
+    - size: 1000
+  resources:
+    requests:
+      cpus: "3"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: "$.StartTime"
+    metrics:
+      - "$.Tests.*.rps"
+      - "$.Tests.*.avg_latency_ms"
+      - "$.Tests.*.p50_latency_ms"
+      - "$.Tests.*.p95_latency_ms"
+      - "$.Tests.*.p99_latency_ms"
+      - "$.Tests.*.max_latency_ms"
+      - "$.Tests.*.min_latency_ms"