Skip to content
This repository was archived by the owner on Mar 23, 2023. It is now read-only.

Commit 5e42372

Browse files
KFilipekKFilipek
committed
[utils] add benchmark for YCSB
This tools allows to put multiple suites and run them one-by-one and parse the output to easy to use form as CSV files.
1 parent df8221c commit 5e42372

File tree

3 files changed

+317
-0
lines changed

3 files changed

+317
-0
lines changed

‎utils/parser.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import os
2+
from os.path import join, getsize
3+
4+
for root, dirs, filenames in os.walk('results'):
5+
if len(dirs) == 0:
6+
parsed_results = []
7+
for filename in filenames:
8+
if filename.split('_')[0] == 'run':
9+
with open(root + '/' + filename) as file_object:
10+
file_object.readline()
11+
trimmed_lines = []
12+
for line in file_object.readlines():
13+
record = tuple(line.replace(',','').split(' '))
14+
if record[0] != '[CLEANUP]' or record[0] != '[READ-FAILED]':
15+
if record[0] == '[READ]' or record[0] == '[INSERT]' or record[0] == '[UPDATE]' or record[0] == '[OVERALL]': #in case of READ
16+
try:
17+
int(record[1])
18+
except ValueError: #if cannot cast it's fine
19+
trimmed_lines.append(record)
20+
parsed_results.append([int(filename.split('_')[1].split('.')[0]), trimmed_lines])
21+
22+
parsed_results = sorted(parsed_results, key=lambda x: x[0], reverse=False)
23+
csv = []
24+
print root
25+
threads = 'Threads;#;'
26+
if len(parsed_results) <= 0:
27+
continue
28+
print '------CSV------'
29+
for i in range(0, len(parsed_results[0][1])):
30+
csv.append(parsed_results[0][1][i][0] + ';' + parsed_results[0][1][i][1] + ';')
31+
for test_result in parsed_results:
32+
threads += str(test_result[0]) + ';'
33+
for i, line in enumerate(test_result[1]):
34+
csv[i] += line[2].replace('\n','').replace('.',',') + ';'
35+
csv.insert(0, threads)
36+
with open(root + '/results.csv','w') as csv_file:
37+
for x in csv:
38+
csv_file.write(x + '\n')
39+
print x
40+
csv_file.close()
41+

‎utils/run_suite.py‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
#!/usr/bin/python2
2+
import json
3+
import os
4+
import subprocess
5+
6+
#comment
7+
# SUITE write_workload
8+
# THREADS 1 2 4 8 16 32 48 64 96
9+
# JOURNALING enabled/disabled
10+
# RECORDS 1000
11+
# OPERATIONS 100
12+
# READ_PROPORTION 0.0
13+
# UPDATE_PROPORTION 0.0
14+
# INSERT_PROPORTION 1.0
15+
# YCSB_NUMA 1
16+
# DROP_BEFORE
17+
# ENDSUITE
18+
19+
#GET PATHS FROM CONFIG FILE
20+
PATH_TO_YCSB = ''
21+
22+
path_configuration = open("path_configuration.txt", "r")
23+
for line in path_configuration:
24+
if line.startswith('YCSB_PATH='):
25+
arg = line.split("=")
26+
if len(arg) > 1:
27+
PATH_TO_YCSB = arg[1].replace('\n','')
28+
else:
29+
raise NameError('No path in YCSB_PATH!')
30+
31+
if not os.path.isdir(PATH_TO_YCSB):
32+
raise NameError('Wrong path to YCSB!')
33+
34+
class Test:
35+
def __init__(self):
36+
self.pmemkv_engine = "cmap"
37+
self.pmemkv_dbsize = 0
38+
self.pmemkv_dbpath = "/dev/shm/"
39+
self.workload_type = "workloada"
40+
self.testName = ""
41+
self.threads = []
42+
# self.journaling = ""
43+
self.records = 0
44+
self.operations = 0
45+
self.read_proportion = -1.0
46+
self.update_proportion = -1.0
47+
self.insert_proportion = -1.0
48+
self.ycsb_numa = -1
49+
# Actually we don't need creation
50+
# self.drop_before = -1
51+
# self.create_after_drop = -1
52+
self.is_load = -1
53+
def toJSON(self):
54+
return json.dumps(self, default=lambda o: o.__dict__,
55+
sort_keys=True, indent=4)
56+
57+
def getArgs(str):
58+
arguments = []
59+
for i in range(1, len(str)):
60+
arguments.append(str[i])
61+
return arguments
62+
63+
KEYWORDS = set(["THREADS", "JOURNALING", "RECORDS", "OPERATIONS",
64+
"READ_PROPORTION", "LOAD", "UPDATE_PROPORTION",
65+
"INSERT_PROPORTION", "YCSB_NUMA", "SUITE", "ENDSUITE",
66+
"DROP_BEFORE", "CREATE_AFTER_DROP", "PMEMKV_ENGINE",
67+
"PMEMKV_DBSIZE", "PMEMKV_DBPATH", "WORKLOAD_TYPE"]) #Add keyword if you need to extend implementation
68+
69+
# open meta file
70+
with open("test_suite.txt", "r") as configfile:
71+
configurations = []
72+
for line in configfile:
73+
splittedLine = line.split()
74+
if line == '\n' or line.startswith('#'):
75+
continue
76+
if len(set.intersection(KEYWORDS, splittedLine)) != 1:
77+
print(splittedLine)
78+
raise NameError('Too many keywords in single line!')
79+
80+
#get args if exists
81+
args = getArgs(splittedLine)
82+
83+
#if line starts from keyword we must read arguments
84+
if splittedLine[0] == "SUITE":
85+
configurations.append(Test())
86+
configurations[len(configurations)-1].testName = args[0]
87+
elif splittedLine[0] == "THREADS":
88+
configurations[len(configurations)-1].threads = args
89+
elif splittedLine[0] == "LOAD":
90+
configurations[len(configurations)-1].is_load = 1
91+
elif splittedLine[0] == "RECORDS":
92+
configurations[len(configurations)-1].records = args[0]
93+
elif splittedLine[0] == "OPERATIONS":
94+
configurations[len(configurations)-1].operations = args[0]
95+
elif splittedLine[0] == "READ_PROPORTION":
96+
configurations[len(configurations)-1].read_proportion = args[0]
97+
elif splittedLine[0] == "UPDATE_PROPORTION":
98+
configurations[len(configurations)-1].update_proportion = args[0]
99+
elif splittedLine[0] == "INSERT_PROPORTION":
100+
configurations[len(configurations)-1].insert_proportion = args[0]
101+
elif splittedLine[0] == "YCSB_NUMA":
102+
configurations[len(configurations)-1].ycsb_numa = args[0]
103+
elif splittedLine[0] == "PMEMKV_ENGINE":
104+
configurations[len(configurations)-1].pmemkv_engine = args[0]
105+
elif splittedLine[0] == "PMEMKV_DBSIZE":
106+
configurations[len(configurations)-1].pmemkv_dbsize = args[0]
107+
elif splittedLine[0] == "PMEMKV_DBPATH":
108+
configurations[len(configurations)-1].pmemkv_dbpath = args[0]
109+
elif splittedLine[0] == "WORKLOAD_TYPE":
110+
configurations[len(configurations)-1].workload_type = args[0]
111+
elif splittedLine[0] == "ENDSUITE":
112+
continue
113+
else:
114+
raise NameError('Unrecognized keyword')
115+
configfile.close()
116+
117+
print('Script read those tests:')
118+
i = 1
119+
for conf in configurations:
120+
print('{:>20} {:<12}'.format('Test#: ', str(i)))
121+
print('{:>20} {:<12}'.format("Name: ", conf.testName))
122+
print('{:>20} {:<12}'.format("Threads: " ,str(conf.threads)))
123+
print('{:>20} {:<12}'.format("Records: ", conf.records))
124+
print('{:>20} {:<12}'.format("Operation: ", conf.operations))
125+
print('{:>20} {:<12}'.format("Read proportion: ", str(conf.read_proportion)))
126+
print('{:>20} {:<12}'.format("Update proportion: ", str(conf.update_proportion)))
127+
print('{:>20} {:<12}'.format("Insert proportion: ", str(conf.insert_proportion)))
128+
print('{:>20} {:<12}'.format("Is load: ", str(conf.is_load)))
129+
print('{:>20} {:<12}'.format("NUMA for YCSB: ", conf.ycsb_numa))
130+
print('{:>20} {:<12}'.format("Workload type: ", conf.workload_type))
131+
print('{:>20} {:<12}'.format("Pmemkv engine: ", conf.pmemkv_engine))
132+
print('{:>20} {:<12}'.format("Pmemkv size: ", conf.pmemkv_dbsize))
133+
print('{:>20} {:<12}'.format("Pmemkv path: ", conf.pmemkv_dbpath))
134+
print("")
135+
i = i + 1
136+
137+
# PUT CONFIGURATION TO FILE IN PROPER PATH
138+
results_directory = "results/"
139+
if not os.path.exists(results_directory):
140+
os.makedirs(results_directory)
141+
i = 1
142+
with open(results_directory + '/configurations.json', 'w') as jsonconfig:
143+
for conf in configurations:
144+
jsonconfig.write(conf.toJSON() + '\n')
145+
if not os.path.exists(results_directory + conf.testName + '/'):
146+
os.makedirs(results_directory + conf.testName + '/')
147+
with open(results_directory + conf.testName + '/test_description.txt', 'a') as test_description:
148+
test_description.write('{:>20} {:<12}'.format('Test#: ', str(i)) + '\n') # 'Test #' + str(i)
149+
test_description.write('{:>20} {:<12}'.format("Name: ", conf.testName) + '\n')
150+
test_description.write('{:>20} {:<12}'.format("Threads: " ,str(conf.threads)) + '\n')
151+
test_description.write('{:>20} {:<12}'.format("Records: ", conf.records) + '\n')
152+
test_description.write('{:>20} {:<12}'.format("Operation: ", conf.operations) + '\n')
153+
test_description.write('{:>20} {:<12}'.format("Read proportion: ", str(conf.read_proportion)) + '\n')
154+
test_description.write('{:>20} {:<12}'.format("Update proportion: ", str(conf.update_proportion)) + '\n')
155+
test_description.write('{:>20} {:<12}'.format("Insert proportion: ", str(conf.insert_proportion)) + '\n')
156+
test_description.write('{:>20} {:<12}'.format("NUMA for YCSB: ", conf.ycsb_numa) + '\n')
157+
test_description.write('{:>20} {:<12}'.format("Workload type: ", conf.workload_type) + '\n')
158+
test_description.write('{:>20} {:<12}'.format("Pmemkv engine: ", conf.pmemkv_engine) + '\n')
159+
test_description.write('{:>20} {:<12}'.format("Pmemkv size: ", conf.pmemkv_dbsize) + '\n')
160+
test_description.write('{:>20} {:<12}'.format("Pmemkv path: ", conf.pmemkv_dbpath) + '\n')
161+
test_description.write('\n')
162+
i = i + 1
163+
164+
# run specified configurations
165+
generated_commands = []
166+
for test in configurations:
167+
command_prefix = ''
168+
command_suffix = ''
169+
170+
command_prefix = './run_workload.sh ' + test.testName
171+
172+
if not test.is_load == 1:
173+
command_prefix += ' run '
174+
else:
175+
command_prefix += ' load '
176+
177+
178+
# Put path to YCSB main directory
179+
command_suffix += PATH_TO_YCSB + ' '
180+
# Put operation numbers
181+
command_suffix += test.records + ' ' + test.operations + ' '
182+
# Put workload ratios
183+
command_suffix += test.read_proportion + ' ' + test.update_proportion + ' ' + test.insert_proportion + ' '
184+
# Put NUMA node
185+
if test.ycsb_numa == -1:
186+
print('NUMA node is not set for test: ' + test.testName + '.')
187+
command_suffix += test.ycsb_numa + ' '
188+
# Put workload type
189+
command_suffix += test.workload_type + ' '
190+
# Put engine specific fields
191+
command_suffix += test.pmemkv_engine + ' ' + test.pmemkv_dbsize + ' ' + test.pmemkv_dbpath + ' '
192+
193+
for thread_no in test.threads:
194+
# DROP&CREATE BEFORE NEXT INSERTS
195+
generated_commands.append(command_prefix + thread_no + ' ' + command_suffix)
196+
197+
# Generate script
198+
with open('testplan.sh','w') as testplan:
199+
testplan.write('#!/bin/bash\n')
200+
for x in generated_commands:
201+
testplan.write(x + '\n')
202+
print(generated_commands)

‎utils/run_workload.sh‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/bin/bash
2+
# Run workload from command line
3+
#
4+
# e.g. ./run_workload.sh run_cmap run 12 PATH_TO_YCSB 1000000 1000000
5+
# {0} {1} {2} {3} {4} {5} {6}
6+
# -1.0 -1.0 -1.0 1 workloadb csmap 80000000 DBPATH
7+
# {7} {8} {9} {10} {11} {12} {13} {14}
8+
# 1 - suite name
9+
# 2 - ycsb phase: load/run
10+
# 3 - thread count
11+
# 4 - path to YCSB
12+
# 5 - record count
13+
# 6 - operation count
14+
# 7 - read proportion
15+
# 8 - insert proportion
16+
# 9 - update proportion
17+
# 10 - NUMA node for YCSB
18+
# 11 - workload scenario (workload[a-f])
19+
####### Engine related args
20+
# 12 - pmemkv: engine name
21+
# 13 - pmemkv: pool size
22+
# 14 - pmemkv: path to pool
23+
24+
YCSB_PATH=/home/kfilipek/Development/work/YCSB/ # TODO(kfilipek): remove hardcoding
25+
echo $YCSB_PATH
26+
OLD_PATH=$(pwd)
27+
28+
echo $@
29+
echo "Passed $# argumets to script"
30+
31+
if [ "$#" -ne "14" ];
32+
then
33+
echo "Illegal number of parameters, should be 11. Check script documentation."
34+
exit 0
35+
fi
36+
37+
mkdir -p "results/$1/" # Create results directory: results/{test_suite_name}/
38+
# Prepare future arguments for YCSB
39+
NUMA_ARG=""
40+
READ_RATIO=""
41+
INSERT_RATIO=""
42+
UPDATE_RATIO=""
43+
if [ "$7" != "-1.0" ];
44+
then
45+
READ_RATIO=" -p readproportion=$7 "
46+
fi
47+
if [ "$8" != "-1.0" ];
48+
then
49+
INSERT_RATIO=" -p insertproportion=$8 "
50+
fi
51+
if [ "$9" != "-1.0" ];
52+
then
53+
UPDATE_RATIO=" -p updateproportion=$9 "
54+
fi
55+
if [ "${10}" != "-1" ];
56+
then
57+
NUMA_ARG=" numactl -N ${10} "
58+
fi
59+
# echo "READ_RATIO param: $READ_RATIO"
60+
# echo "INSERT_RATIO param: $INSERT_RATIO"
61+
# echo "UPDATE_RATIO param: $UPDATE_RATIO"
62+
# echo "NUMA NODE param: $NUMA_ARG"
63+
#exit
64+
65+
# TODOD(kfilipek): Implement splitting threads into processes
66+
cd $YCSB_PATH
67+
if [ "${2}" == "load" ];
68+
then
69+
# Remove old DB before new load phase
70+
rm -rf ${14}
71+
fi
72+
echo "PMEM_IS_PMEM_FORCE=1 $NUMA_ARG bin/ycsb.sh $2 pmemkv -P workloads/${11} -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p pmemkv.engine=${12} -p pmemkv.dbsize=${13} -p pmemkv.dbpath=${14} > $OLD_PATH/results/$1/${2}_${3}.log" >> $OLD_PATH/results/$1/cmds_executed.log
73+
PMEM_IS_PMEM_FORCE=1 $NUMA_ARG bin/ycsb.sh $2 pmemkv -P workloads/${11} -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p pmemkv.engine=${12} -p pmemkv.dbsize=${13} -p pmemkv.dbpath=${14} > $OLD_PATH/results/$1/${2}_${3}.log
74+
cd $OLD_PATH

0 commit comments

Comments
 (0)