Skip to content

Commit 55707b9

Browse files
maximyurchukmaximyurchuk
authored
Merge 644866f into 4e131ff 
2 parents 4e131ff + 644866f commit 55707b9

File tree

2 files changed

+308
-0
lines changed

2 files changed

+308
-0
lines changed

‎ydb/tests/functional/compatibility/test_stress.py‎

Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
# -*- coding: utf-8 -*-
2+
import os
3+
4+
import pytest
5+
6+
import yatest
7+
8+
from ydb.tests.library.harness.kikimr_runner import KiKiMR
9+
from ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
10+
from ydb.tests.library.harness.param_constants import kikimr_driver_path
11+
from ydb.tests.library.common.types import Erasure
12+
from ydb.tests.stress.simple_queue.workload import Workload
13+
14+
last_stable_binary_path = yatest.common.binary_path("ydb/tests/library/compatibility/ydbd-last-stable")
15+
current_binary_path = kikimr_driver_path()
16+
17+
all_binary_combinations = [
18+
[last_stable_binary_path],
19+
[current_binary_path],
20+
[last_stable_binary_path, current_binary_path],
21+
]
22+
all_binary_combinations_ids = ["last_stable", "current", "mixed"]
23+
24+
25+
class TestStress(object):
26+
@pytest.fixture(autouse=True, params=all_binary_combinations, ids=all_binary_combinations_ids)
27+
def setup(self, request):
28+
binary_paths = request.param
29+
self.config = KikimrConfigGenerator(
30+
erasure=Erasure.MIRROR_3_DC,
31+
binary_paths=binary_paths,
32+
# uncomment for 64 datetime in tpc-h/tpc-ds
33+
# extra_feature_flags={"enable_table_datetime64": True},
34+
)
35+
36+
self.cluster = KiKiMR(self.config)
37+
self.cluster.start()
38+
self.endpoint = "%s:%s" % (self.cluster.nodes[1].host, self.cluster.nodes[1].port)
39+
output_path = yatest.common.test_output_path()
40+
self.output_f = open(os.path.join(output_path, "out.log"), "w")
41+
yield
42+
self.cluster.stop()
43+
44+
def get_command_prefix_log(self, subcmds: list[str], path: str) -> list[str]:
45+
return (
46+
[
47+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
48+
"--verbose",
49+
"--endpoint",
50+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
51+
"--database=/Root",
52+
"workload",
53+
"log",
54+
]
55+
+ subcmds
56+
+ ["--path", path]
57+
)
58+
59+
@pytest.mark.parametrize("store_type", ["row"])
60+
def test_log(self, store_type):
61+
timeout_scale = 60
62+
63+
upload_commands = [
64+
# bulk upsert workload
65+
self.get_command_prefix_log(subcmds=["run", "bulk_upsert"], path=store_type)
66+
+ ["--seconds", str(timeout_scale), "--threads", "10", "--rows", "2000"],
67+
# upsert workload
68+
self.get_command_prefix_log(subcmds=["run", "upsert"], path=store_type)
69+
+ ["--seconds", str(timeout_scale), "--threads", "10"],
70+
# insert workload
71+
self.get_command_prefix_log(subcmds=["run", "insert"], path=store_type)
72+
+ ["--seconds", str(timeout_scale), "--threads", "10"],
73+
]
74+
75+
# init
76+
yatest.common.execute(
77+
self.get_command_prefix_log(subcmds=["init"], path=store_type)
78+
+ [
79+
"--store",
80+
store_type,
81+
"--min-partitions",
82+
"100",
83+
"--partition-size",
84+
"10",
85+
"--auto-partition",
86+
"0",
87+
"--ttl",
88+
"10",
89+
],
90+
stdout=self.output_f,
91+
stderr=self.output_f,
92+
)
93+
94+
yatest.common.execute(
95+
self.get_command_prefix_log(subcmds=["import", "--bulk-size", "1000", "-t", "1", "generator"], path=store_type),
96+
stdout=self.output_f,
97+
stderr=self.output_f,
98+
wait=True,
99+
)
100+
101+
select = yatest.common.execute(
102+
self.get_command_prefix_log(subcmds=["run", "select"], path=store_type)
103+
+ [
104+
"--client-timeout",
105+
"10000",
106+
"--threads",
107+
"10",
108+
"--seconds",
109+
str(timeout_scale * len(upload_commands)),
110+
],
111+
wait=False,
112+
stdout=self.output_f,
113+
stderr=self.output_f,
114+
)
115+
116+
for i, command in enumerate(upload_commands):
117+
yatest.common.execute(
118+
command,
119+
wait=True,
120+
stdout=self.output_f,
121+
stderr=self.output_f,
122+
)
123+
124+
select.wait()
125+
126+
@pytest.mark.parametrize("mode", ["row"])
127+
def test_simple_queue(self, mode: str):
128+
with Workload(f"grpc://localhost:{self.cluster.nodes[1].grpc_port}", "/Root", 180, mode) as workload:
129+
for handle in workload.loop():
130+
handle()
131+
132+
@pytest.mark.parametrize("store_type", ["row"])
133+
def test_kv(self, store_type):
134+
init_command_prefix = [
135+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
136+
"--verbose",
137+
"--endpoint",
138+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
139+
"--database=/Root",
140+
"workload",
141+
"kv",
142+
"init",
143+
"--min-partitions",
144+
"10",
145+
"--partition-size",
146+
"10",
147+
"--auto-partition",
148+
"0",
149+
"--init-upserts",
150+
"0",
151+
"--cols",
152+
"5",
153+
"--int-cols",
154+
"2",
155+
"--key-cols",
156+
"3",
157+
]
158+
159+
run_command_prefix = [
160+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
161+
"--verbose",
162+
"--endpoint",
163+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
164+
"--database=/Root",
165+
"workload",
166+
"kv",
167+
"run",
168+
"mixed",
169+
"--seconds",
170+
"180",
171+
"--threads",
172+
"10",
173+
"--cols",
174+
"5",
175+
"--len",
176+
"200",
177+
"--int-cols",
178+
"2",
179+
"--key-cols",
180+
"3",
181+
]
182+
183+
init_command = init_command_prefix
184+
init_command.extend(
185+
[
186+
"--path",
187+
store_type,
188+
"--store",
189+
store_type,
190+
]
191+
)
192+
run_command = run_command_prefix
193+
run_command.extend(
194+
[
195+
"--path",
196+
store_type,
197+
]
198+
)
199+
yatest.common.execute(init_command, wait=True, stdout=self.output_f, stderr=self.output_f)
200+
yatest.common.execute(run_command, wait=True, stdout=self.output_f, stderr=self.output_f)
201+
202+
@pytest.mark.parametrize("store_type", ["row"])
203+
def test_tpch1(self, store_type):
204+
init_command = [
205+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
206+
"--verbose",
207+
"--endpoint",
208+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
209+
"--database=/Root",
210+
"workload",
211+
"tpch",
212+
"-p",
213+
"tpch/s1",
214+
"init",
215+
"--store={}".format(store_type),
216+
"--datetime", # use 32 bit dates instead of 64 (not supported in 24-4)
217+
]
218+
import_command = [
219+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
220+
"--verbose",
221+
"--endpoint",
222+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
223+
"--database=/Root",
224+
"workload",
225+
"tpch",
226+
"-p",
227+
"tpch/s1",
228+
"import",
229+
"generator",
230+
"--scale=1",
231+
]
232+
run_command = [
233+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
234+
"--verbose",
235+
"--endpoint",
236+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
237+
"--database=/Root",
238+
"workload",
239+
"tpch",
240+
"-p",
241+
"tpch/s1",
242+
"run",
243+
"--scale=1",
244+
"--exclude",
245+
# not working for row tables
246+
"17",
247+
"--check-canonical",
248+
]
249+
250+
yatest.common.execute(init_command, wait=True, stdout=self.output_f, stderr=self.output_f)
251+
yatest.common.execute(import_command, wait=True, stdout=self.output_f, stderr=self.output_f)
252+
yatest.common.execute(run_command, wait=True, stdout=self.output_f, stderr=self.output_f)
253+
254+
@pytest.mark.parametrize("store_type", ["row"])
255+
def test_tpcds1(self, store_type):
256+
init_command = [
257+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
258+
"--verbose",
259+
"--endpoint",
260+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
261+
"--database=/Root",
262+
"workload",
263+
"tpcds",
264+
"-p",
265+
"tpcds/s1",
266+
"init",
267+
"--store={}".format(store_type),
268+
"--datetime", # use 32 bit dates instead of 64 (not supported in 24-4)
269+
]
270+
import_command = [
271+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
272+
"--verbose",
273+
"--endpoint",
274+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
275+
"--database=/Root",
276+
"workload",
277+
"tpcds",
278+
"-p",
279+
"olap_yatests/tpcds/s1",
280+
"import",
281+
"generator",
282+
"--scale=1",
283+
]
284+
run_command = [
285+
yatest.common.binary_path(os.getenv("YDB_CLI_BINARY")),
286+
"--verbose",
287+
"--endpoint",
288+
"grpc://localhost:%d" % self.cluster.nodes[1].grpc_port,
289+
"--database=/Root",
290+
"workload",
291+
"tpcds",
292+
"-p",
293+
"tpcds/s1",
294+
"run",
295+
"--scale=1",
296+
"--check-canonical",
297+
"--exclude",
298+
# not working for row tables
299+
"5,7,14,18,22,23,24,26,27,31,33,39,46,51,54,56,58,60,61,64,66,67,68,72,75,77,78,79,80,93",
300+
]
301+
302+
yatest.common.execute(init_command, wait=True, stdout=self.output_f, stderr=self.output_f)
303+
yatest.common.execute(import_command, wait=True, stdout=self.output_f, stderr=self.output_f)
304+
yatest.common.execute(run_command, wait=True, stdout=self.output_f, stderr=self.output_f)

‎ydb/tests/functional/compatibility/ya.make‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,25 @@
11
PY3TEST()
22
ENV(YDB_DRIVER_BINARY="ydb/apps/ydbd/ydbd")
3+
ENV(YDB_CLI_BINARY="ydb/apps/ydb/ydb")
34

45
TEST_SRCS(
56
test_followers.py
67
test_compatibility.py
8+
test_stress.py
79
)
810

911
SIZE(LARGE)
1012
INCLUDE(${ARCADIA_ROOT}/ydb/tests/large.inc)
1113

1214
DEPENDS(
15+
ydb/apps/ydb
1316
ydb/apps/ydbd
1417
ydb/tests/library/compatibility
1518
)
1619

1720
PEERDIR(
1821
ydb/tests/library
22+
ydb/tests/stress/simple_queue/workload
1923
)
2024

2125
END()

0 commit comments

Comments
 (0)