Skip to content
This repository was archived by the owner on Mar 23, 2023. It is now read-only.

Commit a390706

Browse files
KFilipekKFilipek
committed
[utils] add benchmark for YCSB
This tools allows to put multiple suites and run them one-by-one and parse the output to easy to use form as CSV files.
1 parent acd56d4 commit a390706

File tree

3 files changed

+341
-0
lines changed

3 files changed

+341
-0
lines changed

‎utils/parser.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import os
2+
from os.path import join, getsize
3+
4+
for root, dirs, filenames in os.walk('results'):
5+
if len(dirs) == 0:
6+
parsed_results = []
7+
for filename in filenames:
8+
if filename.split('_')[0] == 'run':
9+
with open(root + '/' + filename) as file_object:
10+
file_object.readline()
11+
trimmed_lines = []
12+
for line in file_object.readlines():
13+
record = tuple(line.replace(',','').split(' '))
14+
if record[0] != '[CLEANUP]' or record[0] != '[READ-FAILED]':
15+
if record[0] == '[READ]' or record[0] == '[INSERT]' or record[0] == '[UPDATE]' or record[0] == '[OVERALL]': #in case of READ
16+
try:
17+
int(record[1])
18+
except ValueError: #if cannot cast it's fine
19+
trimmed_lines.append(record)
20+
parsed_results.append([int(filename.split('_')[1].split('.')[0]), trimmed_lines])
21+
22+
parsed_results = sorted(parsed_results, key=lambda x: x[0], reverse=False)
23+
csv = []
24+
threads = 'Threads;#;'
25+
if len(parsed_results) <= 0:
26+
continue
27+
print '------CSV------'
28+
for i in range(0, len(parsed_results[0][1])):
29+
csv.append(parsed_results[0][1][i][0] + ';' + parsed_results[0][1][i][1] + ';')
30+
for test_result in parsed_results:
31+
threads += str(test_result[0]) + ';'
32+
for i, line in enumerate(test_result[1]):
33+
csv[i] += line[2].replace('\n','').replace('.',',') + ';'
34+
csv.insert(0, threads)
35+
with open(root + '/results.csv','w') as csv_file:
36+
for x in csv:
37+
csv_file.write(x + '\n')
38+
print x
39+
csv_file.close()

‎utils/run_suite.py‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
#!/usr/bin/python2
2+
import json
3+
import os
4+
import subprocess
5+
6+
#comment
7+
# SUITE write_workload
8+
# THREADS 1 2 4 8 16 32 48 64 96
9+
# JOURNALING enabled/disabled
10+
# RECORDS 1000
11+
# OPERATIONS 100
12+
# READ_PROPORTION 0.0
13+
# UPDATE_PROPORTION 0.0
14+
# INSERT_PROPORTION 1.0
15+
# YCSB_NUMA 1
16+
# DROP_BEFORE
17+
# ENDSUITE
18+
19+
#GET PATHS FROM CONFIG FILE
20+
PATH_TO_YCSB = ''
21+
22+
path_configuration = open("path_configuration.txt", "r")
23+
for line in path_configuration:
24+
if line.startswith('YCSB_PATH='):
25+
arg = line.split("=")
26+
if len(arg) > 1:
27+
PATH_TO_YCSB = arg[1].replace('\n','')
28+
else:
29+
raise NameError('No path in YCSB_PATH!')
30+
31+
if not os.path.isdir(PATH_TO_YCSB):
32+
raise NameError('Wrong path to YCSB!')
33+
34+
class Test:
35+
def __init__(self):
36+
self.pmemkv_engine = "cmap"
37+
self.pmemkv_dbsize = 0
38+
self.pmemkv_dbpath = "/dev/shm/"
39+
self.workload_type = "workloada"
40+
self.testName = ""
41+
self.threads = []
42+
# self.journaling = ""
43+
self.records = 0
44+
self.operations = 0
45+
self.read_proportion = -1.0
46+
self.update_proportion = -1.0
47+
self.insert_proportion = -1.0
48+
self.ycsb_numa = -1
49+
# Actually we don't need creation
50+
# self.drop_before = -1
51+
# self.create_after_drop = -1
52+
self.is_load = -1
53+
def toJSON(self):
54+
return json.dumps(self, default=lambda o: o.__dict__,
55+
sort_keys=True, indent=4)
56+
57+
def getArgs(str):
58+
arguments = []
59+
for i in range(1, len(str)):
60+
arguments.append(str[i])
61+
return arguments
62+
63+
KEYWORDS = set(["THREADS", "JOURNALING", "RECORDS", "OPERATIONS",
64+
"READ_PROPORTION", "LOAD", "UPDATE_PROPORTION",
65+
"INSERT_PROPORTION", "YCSB_NUMA", "SUITE", "ENDSUITE",
66+
"DROP_BEFORE", "CREATE_AFTER_DROP", "PMEMKV_ENGINE",
67+
"PMEMKV_DBSIZE", "PMEMKV_DBPATH", "WORKLOAD_TYPE"]) #Add keyword if you need to extend implementation
68+
69+
# open meta file
70+
with open("test_suite.txt", "r") as configfile:
71+
configurations = []
72+
for line in configfile:
73+
splittedLine = line.split()
74+
if line == '\n' or line.startswith('#'):
75+
continue
76+
if len(set.intersection(KEYWORDS, splittedLine)) != 1:
77+
print(splittedLine)
78+
raise NameError('Too many keywords in single line!')
79+
80+
#get args if exists
81+
args = getArgs(splittedLine)
82+
83+
#if line starts from keyword we must read arguments
84+
if splittedLine[0] == "SUITE":
85+
configurations.append(Test())
86+
configurations[len(configurations)-1].testName = args[0]
87+
elif splittedLine[0] == "THREADS":
88+
configurations[len(configurations)-1].threads = args
89+
elif splittedLine[0] == "LOAD":
90+
configurations[len(configurations)-1].is_load = 1
91+
elif splittedLine[0] == "RECORDS":
92+
configurations[len(configurations)-1].records = args[0]
93+
elif splittedLine[0] == "OPERATIONS":
94+
configurations[len(configurations)-1].operations = args[0]
95+
elif splittedLine[0] == "READ_PROPORTION":
96+
configurations[len(configurations)-1].read_proportion = args[0]
97+
elif splittedLine[0] == "UPDATE_PROPORTION":
98+
configurations[len(configurations)-1].update_proportion = args[0]
99+
elif splittedLine[0] == "INSERT_PROPORTION":
100+
configurations[len(configurations)-1].insert_proportion = args[0]
101+
elif splittedLine[0] == "YCSB_NUMA":
102+
configurations[len(configurations)-1].ycsb_numa = args[0]
103+
elif splittedLine[0] == "PMEMKV_ENGINE":
104+
configurations[len(configurations)-1].pmemkv_engine = args[0]
105+
elif splittedLine[0] == "PMEMKV_DBSIZE":
106+
configurations[len(configurations)-1].pmemkv_dbsize = args[0]
107+
elif splittedLine[0] == "PMEMKV_DBPATH":
108+
configurations[len(configurations)-1].pmemkv_dbpath = args[0]
109+
elif splittedLine[0] == "WORKLOAD_TYPE":
110+
configurations[len(configurations)-1].workload_type = args[0]
111+
elif splittedLine[0] == "ENDSUITE":
112+
continue
113+
else:
114+
raise NameError('Unrecognized keyword')
115+
configfile.close()
116+
117+
print('Script read those tests:')
118+
i = 1
119+
for conf in configurations:
120+
print('{:>20} {:<12}'.format('Test#: ', str(i)))
121+
print('{:>20} {:<12}'.format("Name: ", conf.testName))
122+
print('{:>20} {:<12}'.format("Threads: " ,str(conf.threads)))
123+
print('{:>20} {:<12}'.format("Records: ", conf.records))
124+
print('{:>20} {:<12}'.format("Operation: ", conf.operations))
125+
print('{:>20} {:<12}'.format("Read proportion: ", str(conf.read_proportion)))
126+
print('{:>20} {:<12}'.format("Update proportion: ", str(conf.update_proportion)))
127+
print('{:>20} {:<12}'.format("Insert proportion: ", str(conf.insert_proportion)))
128+
print('{:>20} {:<12}'.format("Is load: ", str(conf.is_load)))
129+
print('{:>20} {:<12}'.format("NUMA for YCSB: ", conf.ycsb_numa))
130+
print('{:>20} {:<12}'.format("Workload type: ", conf.workload_type))
131+
print('{:>20} {:<12}'.format("Pmemkv engine: ", conf.pmemkv_engine))
132+
print('{:>20} {:<12}'.format("Pmemkv size: ", conf.pmemkv_dbsize))
133+
print('{:>20} {:<12}'.format("Pmemkv path: ", conf.pmemkv_dbpath))
134+
print("")
135+
i = i + 1
136+
137+
# PUT CONFIGURATION TO FILE IN PROPER PATH
138+
results_directory = "results/"
139+
if not os.path.exists(results_directory):
140+
os.makedirs(results_directory)
141+
i = 1
142+
with open(results_directory + '/configurations.json', 'w') as jsonconfig:
143+
for conf in configurations:
144+
jsonconfig.write(conf.toJSON() + '\n')
145+
if not os.path.exists(results_directory + conf.testName + '/'):
146+
os.makedirs(results_directory + conf.testName + '/')
147+
with open(results_directory + conf.testName + '/test_description.txt', 'a') as test_description:
148+
test_description.write('{:>20} {:<12}'.format('Test#: ', str(i)) + '\n') # 'Test #' + str(i)
149+
test_description.write('{:>20} {:<12}'.format("Name: ", conf.testName) + '\n')
150+
test_description.write('{:>20} {:<12}'.format("Threads: " ,str(conf.threads)) + '\n')
151+
test_description.write('{:>20} {:<12}'.format("Records: ", conf.records) + '\n')
152+
test_description.write('{:>20} {:<12}'.format("Operation: ", conf.operations) + '\n')
153+
test_description.write('{:>20} {:<12}'.format("Read proportion: ", str(conf.read_proportion)) + '\n')
154+
test_description.write('{:>20} {:<12}'.format("Update proportion: ", str(conf.update_proportion)) + '\n')
155+
test_description.write('{:>20} {:<12}'.format("Insert proportion: ", str(conf.insert_proportion)) + '\n')
156+
test_description.write('{:>20} {:<12}'.format("NUMA for YCSB: ", conf.ycsb_numa) + '\n')
157+
test_description.write('{:>20} {:<12}'.format("Workload type: ", conf.workload_type) + '\n')
158+
test_description.write('{:>20} {:<12}'.format("Pmemkv engine: ", conf.pmemkv_engine) + '\n')
159+
test_description.write('{:>20} {:<12}'.format("Pmemkv size: ", conf.pmemkv_dbsize) + '\n')
160+
test_description.write('{:>20} {:<12}'.format("Pmemkv path: ", conf.pmemkv_dbpath) + '\n')
161+
test_description.write('\n')
162+
i = i + 1
163+
164+
# run specified configurations
165+
generated_commands = []
166+
for test in configurations:
167+
command_prefix = ''
168+
command_suffix = ''
169+
170+
command_prefix = './run_workload.sh ' + test.testName
171+
172+
if not test.is_load == 1:
173+
command_prefix += ' run '
174+
else:
175+
command_prefix += ' load '
176+
177+
178+
# Put path to YCSB main directory
179+
command_suffix += PATH_TO_YCSB + ' '
180+
# Put operation numbers
181+
command_suffix += test.records + ' ' + test.operations + ' '
182+
# Put workload ratios
183+
command_suffix += test.read_proportion + ' ' + test.update_proportion + ' ' + test.insert_proportion + ' '
184+
# Put NUMA node
185+
if test.ycsb_numa == -1:
186+
print('NUMA node is not set for test: ' + test.testName + '.')
187+
command_suffix += test.ycsb_numa + ' '
188+
# Put workload type
189+
command_suffix += test.workload_type + ' '
190+
# Put engine specific fields
191+
command_suffix += test.pmemkv_engine + ' ' + test.pmemkv_dbsize + ' ' + test.pmemkv_dbpath + ' '
192+
193+
for thread_no in test.threads:
194+
# DROP&CREATE BEFORE NEXT INSERTS
195+
generated_commands.append(command_prefix + thread_no + ' ' + command_suffix)
196+
197+
# Generate script
198+
with open('testplan.sh','w') as testplan:
199+
testplan.write('#!/bin/bash\n')
200+
for x in generated_commands:
201+
testplan.write(x + '\n')
202+
print(generated_commands)

‎utils/run_workload.sh‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#!/bin/bash
2+
# Run workload from command line
3+
# ./run_workload.sh suite_name(string) workload_type(string) no_threads(uint)
4+
# journal(bool) record_count(uint) operation_count(uint)
5+
# readproportion(uint) updateproportion(uint) insertproportion(uint)
6+
# numa_node(uint) ycsb_path
7+
#
8+
# workload_type can be: run or load according to YCSB documentation
9+
# {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10} {11} {12} {13} {14}
10+
# e.g. ./run_workload.sh run_cmap run 12 PATH_TO_YCSB 1000000 1000000 -1.0 -1.0 -1.0 1 workloadb csmap 80000000 DBPATH
11+
#
12+
# 1 - suite name
13+
# 2 - ycsb phase: load/run
14+
# 3 - thread count
15+
# 4 - path to YCSB
16+
# 5 - record count
17+
# 6 - operation count
18+
# 7 - read proportion
19+
# 8 - insert proportion
20+
# 9 - update proportion
21+
# 10 - NUMA node for YCSB
22+
# 11 - workload scenario (workload[a-f])
23+
####### Engine related args
24+
# 12 - pmemkv: engine name
25+
# 13 - pmemkv: pool size
26+
# 14 - pmemkv: path to pool
27+
28+
YCSB_PATH=/home/kfilipek/Development/work/YCSB/ # TODO(kfilipek): remove hardcoding
29+
echo $YCSB_PATH
30+
OLD_PATH=$(pwd)
31+
32+
echo $@
33+
echo "Passed $# argumets to script"
34+
35+
if [ "$#" -ne "14" ];
36+
then
37+
echo "Illegal number of parameters, should be 11. Check script documentation."
38+
exit 0
39+
fi
40+
41+
mkdir -p "results/$1/" # Create results directory
42+
# Prepare future arguments for YCSB
43+
NUMA_ARG=""
44+
READ_RATIO=""
45+
INSERT_RATIO=""
46+
UPDATE_RATIO=""
47+
if [ "$7" != "-1.0" ];
48+
then
49+
echo $7
50+
READ_RATIO=" -p readproportion=$7 "
51+
fi
52+
if [ "$8" != "-1.0" ];
53+
then
54+
echo $8
55+
INSERT_RATIO=" -p insertproportion=$8 "
56+
fi
57+
if [ "$9" != "-1.0" ];
58+
then
59+
echo $9
60+
UPDATE_RATIO=" -p updateproportion=$9 "
61+
fi
62+
if [ "${10}" != "-1" ];
63+
then
64+
echo ${10}
65+
NUMA_ARG=" -N ${10} "
66+
fi
67+
echo "READ_RATIO param: $READ_RATIO"
68+
echo "INSERT_RATIO param: $INSERT_RATIO"
69+
echo "UPDATE_RATIO param: $UPDATE_RATIO"
70+
echo "NUMA NODE param: $NUMA_ARG"
71+
exit
72+
73+
if [ $2 = "load" ];
74+
then
75+
# LOAD PHASE
76+
echo "load chosen"
77+
if [ ${10} -lt 0 ];
78+
then
79+
cd $YCSB_PATH
80+
./bin/ycsb load mongodb -s -threads $3 -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p readproportion=$7 -p updateproportion=$8 -p insertproportion=$9 -P ./workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb -p mongodb.writeConcern=$JOURNALING > $OLD_PATH/results/$1/load_$3.log
81+
cd $OLD_PATH
82+
else
83+
cd $YCSB_PATH
84+
numactl -N ${10} ./bin/ycsb load mongodb -s -threads $3 -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p readproportion=$7 -p updateproportion=$8 -p insertproportion=$9 -P ./workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb -p mongodb.writeConcern=$JOURNALING > $OLD_PATH/results/$1/load_$3.log
85+
cd $OLD_PATH
86+
fi
87+
else
88+
# RUN PHASE
89+
echo "run chosen"
90+
if [ ${10} -lt 0 ];
91+
then
92+
cd $YCSB_PATH
93+
./bin/ycsb run mongodb -s -threads $3 -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p readproportion=$7 -p updateproportion=$8 -p insertproportion=$9 -P ./workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb -p mongodb.writeConcern=$JOURNALING > $OLD_PATH/results/$1/run_$3.log
94+
cd $OLD_PATH
95+
else
96+
cd $YCSB_PATH
97+
numactl -N ${10} ./bin/ycsb run mongodb -s -threads $3 -p hdrhistogram.percentiles=95,99,99.9,99.99 -p recordcount=$5 -p operationcount=$6 -p readproportion=$7 -p updateproportion=$8 -p insertproportion=$9 -P ./workloads/workloada -p mongodb.url=mongodb://localhost:27017/ycsb -p mongodb.writeConcern=$JOURNALING > $OLD_PATH/results/$1/run_$3.log
98+
cd $OLD_PATH
99+
fi
100+
fi

0 commit comments

Comments
 (0)