Skip to content

Commit 25a7a96

Browse files
ryv-odooryv-odoo
committed
[IMP] perf unity script
1 parent 431032d commit 25a7a96

File tree

2 files changed

+357
-0
lines changed

2 files changed

+357
-0
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
!not_in_selection/*.py
2828
!not_in_selection/*.log
2929
!new_read_group/*.py
30+
!perf_unity/*.py
3031
!new_read_group/*.txt
3132
!expression_tests.sql
3233
!aggregate_perf.py

‎perf_unity/read_group_unity_perf.py‎

Lines changed: 356 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,356 @@
1+
from time import time_ns
2+
import grequests
3+
import requests
4+
import itertools
5+
from functools import partial
6+
from statistics import fmean, stdev
7+
8+
# Branch for test:
9+
10+
READ_SPEC_CRM = {
11+
12+
}
13+
14+
READ_SPEC_TASK = {
15+
16+
}
17+
18+
READ_SPEC_PROJECT = {
19+
20+
}
21+
22+
BASE_URL = "https://127.0.0.1/"
23+
COOKIES = {
24+
"session_id": "",
25+
}
26+
27+
SCENARIO = [
28+
# ------- crm.lead
29+
# Kanban: Open CRM - No filter - all open lead are loaded
30+
{
31+
32+
},
33+
# Kanban: Open CRM - FZilter: Creation Date = 2025 - Groupby month
34+
{
35+
36+
},
37+
38+
39+
# ------- project.task
40+
# Kanban: Open Framework Python project - No filters
41+
{
42+
43+
},
44+
# Kanban: Open Help project - No filter
45+
{
46+
47+
},
48+
# Kanban:
49+
50+
51+
]
52+
53+
54+
def get_url(model, method):
55+
return f"{BASE_URL}/web/dataset/call_kw/RYV/{model}/{method}"
56+
57+
def get_default_json_data(model, method, domain):
58+
return {
59+
"id": 13, # balec
60+
"jsonrpc": "2.0",
61+
"method": "call",
62+
"params": {
63+
"model": model,
64+
"method": method,
65+
"args": [],
66+
"kwargs": {
67+
"context": {
68+
"allowed_company_ids": [1],
69+
"lang": "en_US",
70+
"tz": "Europe/Brussels",
71+
},
72+
"domain": domain,
73+
},
74+
},
75+
}
76+
77+
78+
def old_way(scenario, with_issue=True):
79+
(
80+
model,
81+
domain,
82+
groupby,
83+
aggregates,
84+
read_specification,
85+
search_limit,
86+
search_order,
87+
) = scenario
88+
json_data = get_default_json_data(model, "web_read_group", domain)
89+
json_data["params"]["kwargs"]["fields"] = aggregates
90+
json_data["params"]["kwargs"]["groupby"] = groupby
91+
92+
url = get_url(model, "web_read_group")
93+
94+
start = time_ns()
95+
res = requests.post(url, json=json_data, cookies=COOKIES)
96+
delay = time_ns() - start
97+
groups = res.json()["result"]["groups"]
98+
99+
def web_search_read_data(group):
100+
data = get_default_json_data(model, "web_search_read", [])
101+
data["params"]["kwargs"]["domain"] = group["__domain"]
102+
data["params"]["kwargs"]["specification"] = read_specification
103+
data["params"]["kwargs"]["count_limit"] = 10001 if with_issue else search_limit
104+
data["params"]["kwargs"]["limit"] = search_limit
105+
data["params"]["kwargs"]["order"] = search_order
106+
return data
107+
108+
def is_unfold(group):
109+
return (not group.get("__fold", False)) and (group[f"{groupby[0]}_count"] != 0)
110+
111+
unfold_groups = list(itertools.islice(filter(is_unfold, groups), 10))
112+
113+
parallel_web_search_read = [
114+
grequests.post(
115+
get_url(model, "web_search_read"),
116+
json=web_search_read_data(g),
117+
cookies=cookies,
118+
)
119+
for g in unfold_groups
120+
]
121+
122+
# TODO: should be 5 because of the read_group_progress_bar ?
123+
start = time_ns()
124+
results = grequests.map(parallel_web_search_read, size=6)
125+
delay += time_ns() - start
126+
127+
for group, res in zip(unfold_groups, results):
128+
group["__records"] = res.json()["result"]["records"]
129+
130+
return groups, delay / 1_000_000
131+
132+
133+
def new_way(method, scenario):
134+
(
135+
model,
136+
domain,
137+
groupby,
138+
aggregates,
139+
read_specification,
140+
search_limit,
141+
search_order,
142+
) = scenario
143+
json_data = get_default_json_data(model, method, domain)
144+
json_data["params"]["kwargs"]["aggregates"] = aggregates
145+
json_data["params"]["kwargs"]["groupby"] = groupby
146+
json_data["params"]["kwargs"]["read_specification"] = read_specification
147+
json_data["params"]["kwargs"]["search_limit"] = search_limit
148+
json_data["params"]["kwargs"]["search_order"] = search_order
149+
start = time_ns()
150+
res = requests.post(get_url(model, method), json=json_data, cookies=cookies)
151+
delay = time_ns() - start
152+
return res.json()["result"]["groups"], (delay / 1_000_000)
153+
154+
155+
NB_TEST = 10
156+
157+
158+
def time_test(method):
159+
res = []
160+
for _ in range(NB_TEST):
161+
_, delay = method()
162+
res.append(delay)
163+
164+
res.sort()
165+
return res[2:-2] # Remove outlier
166+
167+
168+
if __name__ == "__main__":
169+
# model, domain, groupby, aggregates, read_specification, limit_search, order_search
170+
scenarios = [
171+
( # When you open Help project (fixed aggregates)
172+
"project.task",
173+
[
174+
"&",
175+
"&",
176+
["project_id", "=", 49],
177+
["display_in_project", "=", True],
178+
[
179+
"state",
180+
"in",
181+
[
182+
"01_in_progress",
183+
"02_changes_requested",
184+
"03_approved",
185+
"04_waiting_normal",
186+
],
187+
],
188+
],
189+
["stage_id"],
190+
["__count"],
191+
project_spec,
192+
20,
193+
None,
194+
),
195+
( # When you open CRM -> search 'test' (fixed aggregates)
196+
"crm.lead",
197+
[
198+
"&",
199+
["type", "=", "opportunity"],
200+
"|",
201+
"|",
202+
"|",
203+
"|",
204+
["partner_id", "ilike", "test"],
205+
["partner_name", "ilike", "test"],
206+
["email_from", "ilike", "test"],
207+
["name", "ilike", "test"],
208+
["contact_name", "ilike", "test"],
209+
],
210+
["stage_id"],
211+
["__count", "expected_revenue:sum", "recurring_revenue_monthly:sum"],
212+
spec_lead,
213+
40,
214+
None,
215+
),
216+
( # When you open CRM -> search 'infinity' included archived
217+
"crm.lead",
218+
[
219+
"&",
220+
["type", "=", "opportunity"],
221+
"&",
222+
"|",
223+
"|",
224+
"|",
225+
"|",
226+
["partner_id", "ilike", "infinity i"],
227+
["partner_name", "ilike", "infinity i"],
228+
["email_from", "ilike", "infinity i"],
229+
["name", "ilike", "infinity i"],
230+
["contact_name", "ilike", "infinity i"],
231+
["active", "in", [True, False]],
232+
],
233+
["stage_id"],
234+
["__count", "expected_revenue:sum", "recurring_revenue_monthly:sum"],
235+
spec_lead,
236+
40,
237+
None,
238+
),
239+
( # When you open CRM -> search 'Won t find anything' (fixed aggregates)
240+
"crm.lead",
241+
[
242+
"&",
243+
["type", "=", "opportunity"],
244+
"|",
245+
"|",
246+
"|",
247+
"|",
248+
["partner_id", "ilike", "Won t find anything"],
249+
["partner_name", "ilike", "Won t find anything"],
250+
["email_from", "ilike", "Won t find anything"],
251+
["name", "ilike", "Won t find anything"],
252+
["contact_name", "ilike", "Won t find anything"],
253+
],
254+
["stage_id"],
255+
["__count", "expected_revenue:sum", "recurring_revenue_monthly:sum"],
256+
spec_lead,
257+
40,
258+
None,
259+
),
260+
( # When you open CRM -> groupby 'Sales Team' (fixed aggregates)
261+
"crm.lead",
262+
[["type", "=", "opportunity"]],
263+
["team_id"],
264+
["__count", "expected_revenue:sum", "recurring_revenue_monthly:sum"],
265+
spec_lead,
266+
40,
267+
None,
268+
),
269+
( # When you open CRM -> custom groupby language (fixed aggregates)
270+
"crm.lead",
271+
[["type", "=", "opportunity"]],
272+
["lang_id"],
273+
["__count", "expected_revenue:sum", "recurring_revenue_monthly:sum"],
274+
spec_lead,
275+
40,
276+
None,
277+
),
278+
( # Open Project -> Groupby Status (fixed aggregates)
279+
"project.project",
280+
[["is_internal_project", "=", False]],
281+
["last_update_status"],
282+
["__count"],
283+
project_project_spec,
284+
80,
285+
"is_favorite DESC, sequence ASC, name ASC, id ASC",
286+
),
287+
# TODO: with date ??? But _records_by_group_union_all_cte doesn't work
288+
]
289+
290+
scenario_names = [
291+
# "Open CRM",
292+
"Open CRM",
293+
"Open ORM project",
294+
"Open Help project",
295+
"Open CRM -> search 'test'",
296+
"Open CRM -> search 'infinity' included archived",
297+
"Open CRM -> search 'Won t find anything'",
298+
"Open CRM -> groupby 'Sales Team'",
299+
"Open CRM -> custom groupby language",
300+
"Open Project -> Groupby Status",
301+
]
302+
303+
# Warmup
304+
NB_WARMUP = 1
305+
for i in range(NB_WARMUP):
306+
print(f"Warmup worker {i}/{NB_WARMUP} :")
307+
for scenario, name_s in zip(scenarios, scenario_names):
308+
old_groups, _ = old_way(scenario, with_issue=True)
309+
old_groups, _ = old_way(scenario, with_issue=False)
310+
311+
new_groups, _ = new_way("web_read_group_unity_naive_search", scenario)
312+
assert (
313+
old_groups == new_groups
314+
), f"web_read_group_unity_naive_search fail assert: {name_s}\n{old_groups}\nVS\n{new_groups}"
315+
316+
# new_groups, _ = new_way("web_read_group_unity_union_all", scenario)
317+
# assert (
318+
# old_groups == new_groups
319+
# ), f"web_read_group_unity_union_all fail assert: {name_s}\n{old_groups}\nVS\n{new_groups}"
320+
321+
new_groups, _ = new_way("web_read_group_unity_union_all_cte", scenario)
322+
assert (
323+
old_groups == new_groups
324+
), f"web_read_group_unity_union_all_cte fail assert: {name_s}\n{old_groups}\nVS\n{new_groups}"
325+
if i == 0:
326+
print(
327+
f"\t- {name_s} : {sum(1 for group in new_groups if '__records' in group)} groups open"
328+
)
329+
330+
print("Launching test")
331+
for scenario, name_s in zip(scenarios, scenario_names):
332+
to_test = [
333+
(
334+
"union_all_cte",
335+
partial(new_way, "web_read_group_unity_union_all_cte", scenario),
336+
),
337+
(
338+
"naive_search",
339+
partial(new_way, "web_read_group_unity_naive_search", scenario),
340+
),
341+
("Old way (Fixed)", partial(old_way, scenario, with_issue=False)),
342+
("Old way", partial(old_way, scenario, with_issue=True)),
343+
# ('union_all', partial(new_way, "web_read_group_unity_union_all", scenario)),
344+
]
345+
print(f"For {name_s!r}:")
346+
for name, method in to_test:
347+
res = time_test(method)
348+
349+
avg_res = fmean(res)
350+
stdev_res = stdev(res)
351+
max_res = max(res)
352+
min_res = min(res)
353+
354+
print(
355+
f"\t- {name}: {avg_res:.3f} +- {stdev_res:.3f} ms / min={min_res:.3f}, max={max_res:.3f} ms"
356+
)

0 commit comments

Comments
 (0)