Extended runner functional coverage

redis_benchmarks_specification/__runner__/runner.py

@@ -65,6 +65,10 @@ def main():
     )
     args = parser.parse_args()
 
+    run_client_runner_logic(args, project_name, project_name_suffix, project_version)
+
+
+def run_client_runner_logic(args, project_name, project_name_suffix, project_version):
     if args.logname is not None:
         print("Writting log to {}".format(args.logname))
         logging.basicConfig(
@@ -83,7 +87,6 @@ def main():
         )
     logging.info(get_version_string(project_name, project_version))
     testsuite_spec_files = extract_testsuites(args)
-
     datasink_conn = None
     if args.datasink_push_results_redistimeseries:
         logging.info(
@@ -113,7 +116,6 @@ def main():
             )
             logging.error("Error message {}".format(e.__str__()))
             exit(1)
-
     running_platform = args.platform_name
     tls_enabled = args.tls
     tls_skip_verify = args.tls_skip_verify
@@ -124,7 +126,6 @@ def main():
     preserve_temporary_client_dirs = args.preserve_temporary_client_dirs
     docker_client = docker.from_env()
     home = str(Path.home())
-
     profilers_list = []
     profilers_enabled = args.enable_profilers
     if profilers_enabled:
@@ -137,9 +138,7 @@ def main():
                 )
             )
             exit(1)
-
     logging.info("Running the benchmark specs.")
-
     process_self_contained_coordinator_stream(
         args,
         args.datasink_push_results_redistimeseries,

tox.ini

@@ -20,10 +20,17 @@ commands =
 
 docker =
     datasink
+    db_server
 
 [docker:datasink]
 image = redis/redis-stack-server:7.0.2-RC4
 ports =
     16379:6379/tcp
 volumes =
     bind:rw:{toxinidir}/utils/tests/test_data/:/data
+
+
+[docker:db_server]
+image = redis/redis-stack-server:7.0.2-RC4
+ports =
+    6380:6379/tcp

utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml

@@ -0,0 +1,38 @@
+version: 0.4
+name: "memtier_benchmark-10keys-100B-expire-use-case"
+description: "Runs memtier_benchmark, for a keyspace length of 1M keys
+              with a data size of 100 Bytes for each key, which 50% of the keys have expiration set during the benchmark."
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--command" "SETEX __key__ 360 __value__" "--command-key-pattern" "P" "-c" "5" "-t" "2" "--hide-histogram" "--key-minimum" 1 "--key-maximum" "10"'
+tested-commands:
+  - SET
+  - SETX
+  - GET
+redis-topologies:
+  - oss-standalone
+build-variants:
+  - abc
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: '"--key-minimum" 1 "--key-maximum" "10" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" -c 5 -t 1 --hide-histogram --test-time 10'
+  resources:
+    requests:
+      cpus: "1"
+      memory: "2g"
+exporter:
+  redistimeseries:
+    break_by:
+      - version
+      - commit
+    timemetric: '$."ALL STATS".Runtime."Start time"'
+    metrics:
+      - '$."ALL STATS".Totals."Ops/sec"'
+      - '$."ALL STATS".Totals."Latency"'
+      - '$."ALL STATS".Totals."Misses/sec"'
+      - '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

utils/tests/test_runner.py

@@ -1,8 +1,14 @@
+import argparse
+
+import redis
 import yaml
 
+from redis_benchmarks_specification.__common__.package import get_version_string
 from redis_benchmarks_specification.__common__.spec import extract_client_tool
+from redis_benchmarks_specification.__runner__.args import create_client_runner_args
 from redis_benchmarks_specification.__runner__.runner import (
     prepare_memtier_benchmark_parameters,
+    run_client_runner_logic,
 )
 
 
@@ -140,3 +146,65 @@ def test_prepare_memtier_benchmark_parameters():
             benchmark_command_str
             == 'memtier_benchmark --port 12000 --server localhost --json-out-file 1.json --tls --cert cert.file --key key.file --cacert cacert.file "--data-size" "100" --command "SETEX __key__ 10 __value__" --command-key-pattern="R" --command "SET __key__ __value__" --command-key-pattern="R" --command "GET __key__" --command-key-pattern="R" --command "DEL __key__" --command-key-pattern="R"  -c 50 -t 2 --hide-histogram --test-time 300'
         )
+
+
+def test_run_client_runner_logic():
+    project_name = "tool"
+    project_version = "v0"
+    parser = argparse.ArgumentParser(
+        description="test",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser = create_client_runner_args(
+        get_version_string(project_name, project_version)
+    )
+    db_host = "localhost"
+    db_port = "6380"
+    datasink_port = "16379"
+    db_port_int = int(db_port)
+    datasink_port_int = int(db_port)
+    args = parser.parse_args(
+        args=[
+            "--test",
+            "../../utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml",
+            "--db_server_host",
+            "{}".format(db_host),
+            "--db_server_port",
+            "{}".format(db_port),
+            "--flushall_on_every_test_start",
+        ]
+    )
+    try:
+        run_client_runner_logic(args, "tool", "", "v0")
+    except SystemExit as e:
+        assert e.code == 0
+
+    r = redis.Redis(host=db_host, port=db_port_int)
+    total_keys = r.info("keyspace")["db0"]["keys"]
+    assert total_keys == 10
+
+    # run while pushing to redistimeseries
+    args = parser.parse_args(
+        args=[
+            "--test",
+            "../../utils/tests/test_data/test-suites/memtier_benchmark-10keys-100B-expire-use-case-with-variant.yml",
+            "--datasink_push_results_redistimeseries",
+            "--datasink_redistimeseries_host",
+            "{}".format(db_host),
+            "--datasink_redistimeseries_port",
+            "{}".format(datasink_port),
+            "--db_server_host",
+            "{}".format(db_host),
+            "--db_server_port",
+            "{}".format(db_port),
+            "--flushall_on_every_test_start",
+        ]
+    )
+    try:
+        run_client_runner_logic(args, "tool", "", "v0")
+    except SystemExit as e:
+        assert e.code == 0
+
+    r = redis.Redis(host=db_host, port=db_port_int)
+    total_keys = r.info("keyspace")["db0"]["keys"]
+    assert total_keys == 10