Skip to content

Commit 8dffd61

Browse files
vladl2802vladl2802
authored
Merge 44e0a8e into d425c83 
2 parents d425c83 + 44e0a8e commit 8dffd61

File tree

6 files changed

+183
-11
lines changed

6 files changed

+183
-11
lines changed

‎ydb/library/benchmarks/runner/run_tests/run_tests.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
import argparse
2+
import subprocess
3+
import pathlib
4+
import os
5+
from sys import stderr
6+
7+
def parse_args(passed=None):
8+
YDB_ROOT = "../../../"
9+
10+
def variant(string):
11+
if string not in ["h", "ds"]:
12+
raise ValueError("variant must be h or ds")
13+
return string
14+
15+
parser = argparse.ArgumentParser()
16+
17+
parser.add_argument('--datasize', type=int, default=1)
18+
parser.add_argument('--variant', type=variant, default='h')
19+
parser.add_argument('--tasks', type=int, default=1)
20+
parser.add_argument('--ydb-root', type=lambda path: pathlib.Path(path).resolve(), default=YDB_ROOT)
21+
parser.add_argument('-o', '--output', default="./results")
22+
parser.add_argument('--clean-old', action="store_true", default=False)
23+
parser.add_argument('--query-filter', action="append", default=[])
24+
25+
return parser.parse_args(passed)
26+
27+
class Runner:
28+
def prepare_queries_dir(self, custom_pragmas):
29+
print("Preparing queries...")
30+
self.queries_dir.mkdir(parents=True, exist_ok=True)
31+
cmd = [self.args.gen_queries]
32+
cmd += ["--output", f"{self.queries_dir}"]
33+
cmd += ["--variant", f"{self.args.variant}"]
34+
cmd += ["--syntax", "yql"]
35+
cmd += ["--dataset-size", f"{self.args.datasize}"]
36+
for it in custom_pragmas:
37+
cmd += ["--pragma", it]
38+
subprocess.run(cmd)
39+
40+
def prepare_tpc_dir(self):
41+
print("Preparing tpc...")
42+
cmd = [f"{self.args.downloaders_dir}/download_files_{self.args.variant}_{self.args.datasize}.sh"]
43+
subprocess.run(cmd)
44+
os.symlink(f"{self.args.downloaders_dir}/tpc", f"{pathlib.Path("./tpc")}", target_is_directory=True)
45+
46+
def __init__(self, args, enable_spilling):
47+
self.args = args
48+
self.enable_spilling = enable_spilling
49+
50+
self.queries_dir = pathlib.Path(f"queries{"+" if self.enable_spilling else "-"}spilling-{args.datasize}-{args.tasks}")
51+
if self.args.clean_old or not self.queries_dir.exists():
52+
self.prepare_queries_dir([
53+
f"dq.MaxTasksPerStage={self.args.tasks}",
54+
"dq.OptLLVM=ON"
55+
] + [
56+
"dq.UseFinalizeByKey=true",
57+
"dq.EnableSpillingNodes=All",
58+
] if self.enable_spilling else [])
59+
60+
self.tpc_dir = pathlib.Path(f"{self.args.downloaders_dir}/tpc/{self.args.variant}/{self.args.datasize}").resolve()
61+
if self.args.clean_old or not self.tpc_dir.exists():
62+
self.prepare_tpc_dir()
63+
64+
self.result_dir = pathlib.Path(f"{self.args.output}/{"with" if self.enable_spilling else "no"}-spilling/{args.variant}-{args.datasize}-{args.tasks}").resolve()
65+
self.result_dir.mkdir(parents=True, exist_ok=True)
66+
67+
def run(self):
68+
cmd = ["/usr/bin/time", f"{self.args.runner_path}"]
69+
cmd += ["--perf"]
70+
for it in self.args.query_filter:
71+
cmd += ["--query-filter", it]
72+
cmd += ["--query-dir", f"{self.queries_dir}/{self.args.variant}"]
73+
cmd += ["--bindings", f"{self.queries_dir}/{self.args.variant}/bindings.json"]
74+
cmd += ["--result-dir", f"{self.result_dir}"]
75+
cmd += [f"{self.args.dqrun}", "-s"]
76+
cmd += ["--enable-spilling"] if self.enable_spilling else []
77+
cmd += ["--udfs-dir", f"{self.args.udfs_dir}"]
78+
cmd += ["--fs-cfg", f"{self.args.fs_cfg}"]
79+
cmd += ["--gateways-cfg", f"{self.args.gateways_cfg}"]
80+
subprocess.run(cmd)
81+
82+
return self.result_dir
83+
84+
def result_compare(args, to_compare):
85+
cmd = [f"{args.result_compare}"]
86+
cmd += ["-v"]
87+
cmd += to_compare
88+
print(cmd, file=stderr)
89+
with open(f"{args.output}/result-{args.variant}-{args.datasize}-{args.tasks}.htm", "w") as result_table:
90+
subprocess.run(cmd, stdout=result_table)
91+
92+
def main(passed=None):
93+
args, _ = parse_args(passed)
94+
args.dqrun = args.ydb_root / "library" / "yql" / "tools" / "dqrun" / "dqrun"
95+
args.gen_queries = args.ydb_root / "library" / "benchmarks" / "gen_queries" / "gen_queries"
96+
args.downloaders_dir = args.ydb_root / "library" / "benchmarks" / "runner"
97+
args.udfs_dir = args.ydb_root / "library" / "yql" / "udfs" / "common"
98+
args.fs_cfg = args.ydb_root / "library" / "yql" / "tools" / "dqrun" / "examples" / "fs.conf"
99+
args.result_compare = args.ydb_root / "library" / "benchmarks" / "runner" / "result_compare" / "result_compare"
100+
args.gateways_cfg = args.ydb_root / "library" / "benchmarks" / "runner" / "runner" / "test-gateways.conf"
101+
args.runner_path = args.ydb_root / "library" / "benchmarks" / "runner" / "runner" / "runner"
102+
103+
print(args.query_filter)
104+
105+
results = []
106+
print("With spilling...", file=stderr)
107+
results.append(Runner(args, True).run())
108+
print("No spilling...", file=stderr)
109+
results.append(Runner(args, False).run())
110+
111+
print(results, file=stderr)
112+
113+
result_compare(args, results)
114+
115+
if __name__ == "__main__":
116+
main()

‎ydb/library/benchmarks/runner/run_tests/ya.make‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
PY3_PROGRAM()
2+
3+
PY_SRCS(
4+
MAIN run_tests.py
5+
)
6+
7+
PEERDIR(
8+
)
9+
10+
END()

‎ydb/library/benchmarks/runner/runner/runner.py‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -65,25 +65,25 @@ def run(argv, out, err, timeout=30*60, hard_timeout=5):
6565

6666

6767
def main():
68-
6968
parser = argparse.ArgumentParser()
7069
parser.add_argument('--query-dir', type=str, default='q/scalar')
7170
parser.add_argument('--bindings', type=str, default='bindings.json')
72-
parser.add_argument('--result-dir', type=str, default="result-{:%Y%m%dT%H%M%S}".format(datetime.datetime.now()))
71+
parser.add_argument('--result-dir', type=Path, default="result-{:%Y%m%dT%H%M%S}".format(datetime.datetime.now()))
7372
parser.add_argument('--timeout', type=int, default=30*60)
7473
parser.add_argument('--perf', action='store_true')
7574
parser.add_argument('--arc-path', type=str, default='{}/arcadia'.format(os.environ['HOME']))
7675
parser.add_argument('--include-q', default=[], action='append')
7776
parser.add_argument('--exclude-q', default=[], action='append')
77+
parser.add_argument('--query-filter', action="append", default=[])
78+
7879
args, argv = parser.parse_known_intermixed_args()
7980
qdir = args.query_dir
8081
bindings = args.bindings
8182
outdir = args.result_dir
8283
assert len(argv)
8384
querydir = Path(qdir)
84-
os.makedirs(outdir + '/' + qdir, exist_ok=True)
85-
with open(outdir + '/' + qdir + "/summary.tsv", "w") as outf, \
86-
open(outdir + '/' + qdir + "/summary.json", "w") as outj:
85+
with open(outdir / "summary.tsv", "w") as outf, \
86+
open(outdir / "summary.json", "w") as outj:
8787
print(' '.join(argv + ['-p', qdir, '--bindings-file', bindings]), file=outf)
8888
print(json.dumps({
8989
'cmdline': argv,
@@ -92,8 +92,13 @@ def main():
9292
'version': 100
9393
}), file=outj)
9494
for query in sorted(querydir.glob('**/*.sql'), key=lambda x: tuple(map(lambda y: int(y) if re.match(RE_DIGITS, y) else y, re.split(RE_DIGITS, str(x))))):
95-
q = str(query)
96-
name = outdir + '/' + q
95+
q = str(query.stem)
96+
# q<num>.sql
97+
num = q[1:-4]
98+
if args.query_filter != [] and num not in args.query_filter:
99+
continue
100+
print(f"{q}", end="", flush=True)
101+
name = str(outdir / q)
97102
if len(args.include_q):
98103
include = False
99104
for r in args.include_q:
@@ -112,6 +117,7 @@ def main():
112117
continue
113118
print(q, end='\t', file=outf)
114119
outname = name + '-result.yson'
120+
print(".", end="", flush=True)
115121
exitcode, rusage, elapsed, iostat = run(
116122
argv + [
117123
'--result-file', outname,
@@ -120,7 +126,7 @@ def main():
120126
'--err-file', name + '-err.txt',
121127
'--expr-file', name + '-expr.txt',
122128
'--stat', name + '-stat.yson',
123-
'-p', q
129+
'-p', str(query)
124130
],
125131
name + '-stdout.txt',
126132
name + '-stderr.txt',
@@ -164,25 +170,27 @@ def main():
164170
}
165171
}), file=outj)
166172
outj.flush()
173+
print(".", end="", flush=True)
167174
if args.perf:
168175
exitcode, rusage, elapsed, iostat = run(
169-
['{}/ya'.format(args.arc_path), 'tool', 'perf', 'record', '-F250', '-g', '--call-graph', 'dwarf', '-o', '{}/perf.data'.format(outdir), '--'] +
176+
['/usr/bin/perf', 'record', '-F250', '-g', '--call-graph', 'dwarf', '-o', '{}/perf.data'.format(outdir), '--'] +
170177
argv + [
171178
'--result-file', '/dev/null',
172179
'--bindings-file', bindings,
173180
'--plan-file', '/dev/null',
174181
'--err-file', '/dev/null',
175182
'--expr-file', '/dev/null',
176-
'-p', q
183+
'-p', str(query)
177184
],
178185
name + '-stdout-perf.txt',
179186
name + '-stderr-perf.txt',
180187
timeout=args.timeout)
181188
os.system('''
182-
{0}/ya tool perf script -i {2}/perf.data --header |
189+
/usr/bin/perf script -i {2}/perf.data --header |
183190
{0}/contrib/tools/flame-graph/stackcollapse-perf.pl |
184191
{0}/contrib/tools/flame-graph/flamegraph.pl > {1}.svg
185192
'''.format(args.arc_path, name, outdir))
193+
print(".", flush=True)
186194

187195

188196
if __name__ == "__main__":

‎ydb/library/benchmarks/runner/runner/ya.make‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ PY_SRCS(
55
)
66

77
PEERDIR(
8+
ydb/library/yql/tools/dqrun
9+
ydb/library/benchmarks/gen_queries
10+
11+
ydb/library/yql/udfs/common/set
12+
ydb/library/yql/udfs/common/url_base
13+
ydb/library/yql/udfs/common/datetime2
14+
ydb/library/yql/udfs/common/re2
815
)
916

1017
END()

‎ydb/library/benchmarks/runner/tpc_tests.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import run_tests.run_tests as run_tests
2+
3+
4+
def wrapped_run(variant, datasize, tasks):
5+
cmd = []
6+
cmd += ["--variant", f"{variant}"]
7+
cmd += ["--datasize", f"{datasize}"]
8+
cmd += ["--tasks", f"{tasks}"]
9+
cmd += ["--query-filter", "1"]
10+
cmd += ["--ydb-root", "/home/vladluk/ydbwork/ydb/ydb"]
11+
cmd += ["-o", "/home/vladluk/ydbwork/ydb/ydb/library/benchmarks/runner/results"]
12+
run_tests.main(cmd)
13+
14+
15+
def test_tpc_h_1_1():
16+
wrapped_run("h", 1, 1)

‎ydb/library/benchmarks/runner/ya.make‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
1+
PY3TEST()
2+
3+
SIZE(MEDIUM)
4+
5+
PY_SRCS(
6+
run_tests/run_tests.py
7+
)
8+
9+
TEST_SRCS(
10+
tpc_tests.py
11+
)
12+
13+
END()
14+
115
RECURSE(
16+
run_tests
217
runner
318
result_convert
419
result_compare

0 commit comments

Comments
 (0)