Skip to content

Commit c5e80aa

Browse files
filipecosta90filipecosta90
authored
Added --client_aggregated_results_folder and --preserve_temporary_client_dirs to redis-benchmarks-spec-client-runner (#95)
1 parent 863a647 commit c5e80aa

File tree

55 files changed

+392
-106
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+392
-106
lines changed

‎pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redis-benchmarks-specification"
3-
version = "0.1.27"
3+
version = "0.1.29"
44
description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
55
authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
66
readme = "Readme.md"

‎redis_benchmarks_specification/__runner__/args.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,18 @@ def create_client_runner_args(project_name):
7676
action="store_true",
7777
help="At the end of every test send a FLUSHALL",
7878
)
79+
parser.add_argument(
80+
"--preserve_temporary_client_dirs",
81+
default=False,
82+
action="store_true",
83+
help="Preserve the temporary client dirs",
84+
)
85+
parser.add_argument(
86+
"--client_aggregated_results_folder",
87+
type=str,
88+
default="",
89+
help="Client tool aggregated results folder ( contains all results from all runs ). If specified then all results will be copied there at the end of each run.",
90+
)
7991
parser.add_argument(
8092
"--tls",
8193
default=False,

‎redis_benchmarks_specification/__runner__/runner.py

Lines changed: 137 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,16 @@
1212
import docker
1313
import redis
1414
from docker.models.containers import Container
15+
from pytablewriter import MarkdownTableWriter
16+
from pytablewriter import CsvTableWriter
17+
1518
from redisbench_admin.run.common import (
1619
get_start_time_vars,
1720
prepare_benchmark_parameters,
1821
execute_init_commands,
22+
merge_default_and_config_metrics,
1923
)
24+
from redisbench_admin.run.metrics import extract_results_table
2025
from redisbench_admin.run.redistimeseries import timeseries_test_sucess_flow
2126
from redisbench_admin.run.run import calculate_client_tool_duration_and_check
2227
from redisbench_admin.utils.benchmark_config import (
@@ -115,6 +120,8 @@ def main():
115120
tls_cert = args.cert
116121
tls_key = args.key
117122
tls_cacert = args.cacert
123+
client_aggregated_results_folder = args.client_aggregated_results_folder
124+
preserve_temporary_client_dirs = args.preserve_temporary_client_dirs
118125
docker_client = docker.from_env()
119126
home = str(Path.home())
120127
logging.info("Running the benchmark specs.")
@@ -134,6 +141,8 @@ def main():
134141
tls_cert,
135142
tls_key,
136143
tls_cacert,
144+
client_aggregated_results_folder,
145+
preserve_temporary_client_dirs,
137146
)
138147

139148

@@ -194,8 +203,11 @@ def process_self_contained_coordinator_stream(
194203
tls_cert=None,
195204
tls_key=None,
196205
tls_cacert=None,
206+
client_aggregated_results_folder="",
207+
preserve_temporary_client_dirs=False,
197208
):
198209
overall_result = True
210+
results_matrix = []
199211
total_test_suite_runs = 0
200212
for test_file in testsuite_spec_files:
201213
client_containers = []
@@ -204,6 +216,7 @@ def process_self_contained_coordinator_stream(
204216
_, benchmark_config, test_name = get_final_benchmark_config(
205217
None, stream, ""
206218
)
219+
default_metrics = []
207220

208221
if tls_enabled:
209222
test_name = test_name + "-tls"
@@ -389,7 +402,12 @@ def process_self_contained_coordinator_stream(
389402
benchmark_end_time, benchmark_start_time
390403
)
391404
)
392-
logging.info("output {}".format(client_container_stdout))
405+
logging.info(
406+
"Printing client tool stdout output".format(
407+
client_container_stdout
408+
)
409+
)
410+
print()
393411
if args.flushall_on_every_test_end:
394412
logging.info("Sending FLUSHALL to the DB")
395413
r.flushall()
@@ -417,7 +435,22 @@ def process_self_contained_coordinator_stream(
417435
"r",
418436
) as json_file:
419437
results_dict = json.load(json_file)
420-
logging.info("Final JSON result {}".format(results_dict))
438+
print_results_table_stdout(
439+
benchmark_config,
440+
default_metrics,
441+
results_dict,
442+
setup_type,
443+
test_name,
444+
None,
445+
)
446+
prepare_overall_total_test_results(
447+
benchmark_config,
448+
default_metrics,
449+
results_dict,
450+
test_name,
451+
results_matrix,
452+
)
453+
421454
dataset_load_duration_seconds = 0
422455

423456
logging.info(
@@ -472,9 +505,64 @@ def process_self_contained_coordinator_stream(
472505
)
473506
)
474507
pass
475-
shutil.rmtree(temporary_dir_client, ignore_errors=True)
508+
509+
if preserve_temporary_client_dirs is True:
510+
logging.info(
511+
"Preserving temporary client dir {}".format(
512+
temporary_dir_client
513+
)
514+
)
515+
else:
516+
logging.info(
517+
"Removing temporary client dir {}".format(temporary_dir_client)
518+
)
519+
shutil.rmtree(temporary_dir_client, ignore_errors=True)
520+
if client_aggregated_results_folder != "":
521+
os.makedirs(client_aggregated_results_folder, exist_ok=True)
522+
dest_fpath = "{}/{}".format(
523+
client_aggregated_results_folder,
524+
local_benchmark_output_filename,
525+
)
526+
logging.info(
527+
"Preserving local results file {} into {}".format(
528+
full_result_path, dest_fpath
529+
)
530+
)
531+
shutil.copy(full_result_path, dest_fpath)
476532
overall_result &= test_result
477533

534+
table_name = "Results for entire test-suite".format(test_name)
535+
results_matrix_headers = [
536+
"Test Name",
537+
"Metric JSON Path",
538+
"Metric Value",
539+
]
540+
writer = MarkdownTableWriter(
541+
table_name=table_name,
542+
headers=results_matrix_headers,
543+
value_matrix=results_matrix,
544+
)
545+
writer.write_table()
546+
547+
if client_aggregated_results_folder != "":
548+
os.makedirs(client_aggregated_results_folder, exist_ok=True)
549+
dest_fpath = "{}/{}".format(
550+
client_aggregated_results_folder,
551+
"aggregate-results.csv",
552+
)
553+
logging.info(
554+
"Storing an aggregated results CSV into {}".format(
555+
full_result_path, dest_fpath
556+
)
557+
)
558+
559+
csv_writer = CsvTableWriter(
560+
table_name=table_name,
561+
headers=results_matrix_headers,
562+
value_matrix=results_matrix,
563+
)
564+
csv_writer.dump(dest_fpath)
565+
478566

479567
def cp_to_workdir(benchmark_tool_workdir, srcfile):
480568
head, filename = os.path.split(srcfile)
@@ -488,6 +576,52 @@ def cp_to_workdir(benchmark_tool_workdir, srcfile):
488576
return dstfile, filename
489577

490578

579+
def print_results_table_stdout(
580+
benchmark_config,
581+
default_metrics,
582+
results_dict,
583+
setup_name,
584+
test_name,
585+
cpu_usage=None,
586+
):
587+
# check which metrics to extract
588+
(_, metrics,) = merge_default_and_config_metrics(
589+
benchmark_config,
590+
default_metrics,
591+
None,
592+
)
593+
table_name = "Results for {} test-case on {} topology".format(test_name, setup_name)
594+
results_matrix_headers = [
595+
"Metric JSON Path",
596+
"Metric Value",
597+
]
598+
results_matrix = extract_results_table(metrics, results_dict)
599+
600+
results_matrix = [[x[0], "{:.3f}".format(x[3])] for x in results_matrix]
601+
writer = MarkdownTableWriter(
602+
table_name=table_name,
603+
headers=results_matrix_headers,
604+
value_matrix=results_matrix,
605+
)
606+
writer.write_table()
607+
608+
609+
def prepare_overall_total_test_results(
610+
benchmark_config, default_metrics, results_dict, test_name, overall_results_matrix
611+
):
612+
# check which metrics to extract
613+
(_, metrics,) = merge_default_and_config_metrics(
614+
benchmark_config,
615+
default_metrics,
616+
None,
617+
)
618+
current_test_results_matrix = extract_results_table(metrics, results_dict)
619+
current_test_results_matrix = [
620+
[test_name, x[0], "{:.3f}".format(x[3])] for x in current_test_results_matrix
621+
]
622+
overall_results_matrix.extend(current_test_results_matrix)
623+
624+
491625
def data_prepopulation_step(
492626
benchmark_config,
493627
benchmark_tool_workdir,

‎redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values-pipeline-10.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,7 @@ exporter:
2929
- commit
3030
timemetric: '$."ALL STATS".Runtime."Start time"'
3131
metrics:
32-
- '$."ALL STATS".*."Ops/sec"'
33-
- '$."ALL STATS".*."Latency"'
32+
- '$."ALL STATS".Totals."Ops/sec"'
33+
- '$."ALL STATS".Totals."Latency"'
34+
- '$."ALL STATS".Totals."Misses/sec"'
35+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-100B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,7 @@ exporter:
2929
- commit
3030
timemetric: '$."ALL STATS".Runtime."Start time"'
3131
metrics:
32-
- '$."ALL STATS".*."Ops/sec"'
33-
- '$."ALL STATS".*."Latency"'
32+
- '$."ALL STATS".Totals."Ops/sec"'
33+
- '$."ALL STATS".Totals."Latency"'
34+
- '$."ALL STATS".Totals."Misses/sec"'
35+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values-pipeline-10.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,7 @@ exporter:
2929
- commit
3030
timemetric: '$."ALL STATS".Runtime."Start time"'
3131
metrics:
32-
- '$."ALL STATS".*."Ops/sec"'
33-
- '$."ALL STATS".*."Latency"'
32+
- '$."ALL STATS".Totals."Ops/sec"'
33+
- '$."ALL STATS".Totals."Latency"'
34+
- '$."ALL STATS".Totals."Misses/sec"'
35+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-10Mkeys-load-hash-5-fields-with-10B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,7 @@ exporter:
2929
- commit
3030
timemetric: '$."ALL STATS".Runtime."Start time"'
3131
metrics:
32-
- '$."ALL STATS".*."Ops/sec"'
33-
- '$."ALL STATS".*."Latency"'
32+
- '$."ALL STATS".Totals."Ops/sec"'
33+
- '$."ALL STATS".Totals."Latency"'
34+
- '$."ALL STATS".Totals."Misses/sec"'
35+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,7 @@ exporter:
3535
- commit
3636
timemetric: '$."ALL STATS".Runtime."Start time"'
3737
metrics:
38-
- '$."ALL STATS".*."Ops/sec"'
39-
- '$."ALL STATS".*."Latency"'
38+
- '$."ALL STATS".Totals."Ops/sec"'
39+
- '$."ALL STATS".Totals."Latency"'
40+
- '$."ALL STATS".Totals."Misses/sec"'
41+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-10B-expire-use-case.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,7 @@ exporter:
3535
- commit
3636
timemetric: '$."ALL STATS".Runtime."Start time"'
3737
metrics:
38-
- '$."ALL STATS".*."Ops/sec"'
39-
- '$."ALL STATS".*."Latency"'
38+
- '$."ALL STATS".Totals."Ops/sec"'
39+
- '$."ALL STATS".Totals."Latency"'
40+
- '$."ALL STATS".Totals."Misses/sec"'
41+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-1KiB-expire-use-case.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,7 @@ exporter:
3535
- commit
3636
timemetric: '$."ALL STATS".Runtime."Start time"'
3737
metrics:
38-
- '$."ALL STATS".*."Ops/sec"'
39-
- '$."ALL STATS".*."Latency"'
38+
- '$."ALL STATS".Totals."Ops/sec"'
39+
- '$."ALL STATS".Totals."Latency"'
40+
- '$."ALL STATS".Totals."Misses/sec"'
41+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-4KiB-expire-use-case.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,7 @@ exporter:
3535
- commit
3636
timemetric: '$."ALL STATS".Runtime."Start time"'
3737
metrics:
38-
- '$."ALL STATS".*."Ops/sec"'
39-
- '$."ALL STATS".*."Latency"'
38+
- '$."ALL STATS".Totals."Ops/sec"'
39+
- '$."ALL STATS".Totals."Latency"'
40+
- '$."ALL STATS".Totals."Misses/sec"'
41+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-hash-hget-hgetall-hkeys-hvals-with-100B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,7 @@ exporter:
3737
- commit
3838
timemetric: '$."ALL STATS".Runtime."Start time"'
3939
metrics:
40-
- '$."ALL STATS".*."Ops/sec"'
41-
- '$."ALL STATS".*."Latency"'
40+
- '$."ALL STATS".Totals."Ops/sec"'
41+
- '$."ALL STATS".Totals."Latency"'
42+
- '$."ALL STATS".Totals."Misses/sec"'
43+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-list-lpop-rpop-with-100B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,7 @@ exporter:
3535
- commit
3636
timemetric: '$."ALL STATS".Runtime."Start time"'
3737
metrics:
38-
- '$."ALL STATS".*."Ops/sec"'
39-
- '$."ALL STATS".*."Latency"'
38+
- '$."ALL STATS".Totals."Ops/sec"'
39+
- '$."ALL STATS".Totals."Latency"'
40+
- '$."ALL STATS".Totals."Misses/sec"'
41+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-list-lpop-rpop-with-10B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,7 @@ exporter:
3535
- commit
3636
timemetric: '$."ALL STATS".Runtime."Start time"'
3737
metrics:
38-
- '$."ALL STATS".*."Ops/sec"'
39-
- '$."ALL STATS".*."Latency"'
38+
- '$."ALL STATS".Totals."Ops/sec"'
39+
- '$."ALL STATS".Totals."Latency"'
40+
- '$."ALL STATS".Totals."Misses/sec"'
41+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-list-lpop-rpop-with-1KiB-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,7 @@ exporter:
3535
- commit
3636
timemetric: '$."ALL STATS".Runtime."Start time"'
3737
metrics:
38-
- '$."ALL STATS".*."Ops/sec"'
39-
- '$."ALL STATS".*."Latency"'
38+
- '$."ALL STATS".Totals."Ops/sec"'
39+
- '$."ALL STATS".Totals."Latency"'
40+
- '$."ALL STATS".Totals."Misses/sec"'
41+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values-pipeline-10.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,7 @@ exporter:
2929
- commit
3030
timemetric: '$."ALL STATS".Runtime."Start time"'
3131
metrics:
32-
- '$."ALL STATS".*."Ops/sec"'
33-
- '$."ALL STATS".*."Latency"'
32+
- '$."ALL STATS".Totals."Ops/sec"'
33+
- '$."ALL STATS".Totals."Latency"'
34+
- '$."ALL STATS".Totals."Misses/sec"'
35+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-hash-5-fields-with-1000B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,7 @@ exporter:
2929
- commit
3030
timemetric: '$."ALL STATS".Runtime."Start time"'
3131
metrics:
32-
- '$."ALL STATS".*."Ops/sec"'
33-
- '$."ALL STATS".*."Latency"'
32+
- '$."ALL STATS".Totals."Ops/sec"'
33+
- '$."ALL STATS".Totals."Latency"'
34+
- '$."ALL STATS".Totals."Misses/sec"'
35+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-with-100B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,7 @@ exporter:
2828
- commit
2929
timemetric: '$."ALL STATS".Runtime."Start time"'
3030
metrics:
31-
- '$."ALL STATS".*."Ops/sec"'
32-
- '$."ALL STATS".*."Latency"'
31+
- '$."ALL STATS".Totals."Ops/sec"'
32+
- '$."ALL STATS".Totals."Latency"'
33+
- '$."ALL STATS".Totals."Misses/sec"'
34+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

‎redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-load-list-with-10B-values.yml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,7 @@ exporter:
2828
- commit
2929
timemetric: '$."ALL STATS".Runtime."Start time"'
3030
metrics:
31-
- '$."ALL STATS".*."Ops/sec"'
32-
- '$."ALL STATS".*."Latency"'
31+
- '$."ALL STATS".Totals."Ops/sec"'
32+
- '$."ALL STATS".Totals."Latency"'
33+
- '$."ALL STATS".Totals."Misses/sec"'
34+
- '$."ALL STATS".Totals."Percentile Latencies"."p50.00"'

0 commit comments

Comments
 (0)