Begin performance module

ryv-odoo · ryv-odoo · commit 3fe683846707 · 2023-04-18T17:01:31.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -29,5 +29,6 @@
 !new_read_group/*.txt
 !expression_tests.sql
 !aggregate_perf.py
+!expression_performance/**
 
 !read_group_replacement.py
diff --git a/expression_performance/__init__.py b/expression_performance/__init__.py
@@ -0,0 +1 @@
+from . import models
diff --git a/expression_performance/__manifest__.py b/expression_performance/__manifest__.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Part of Odoo. See LICENSE file for full copyright and licensing details.
+
+{
+    'name': 'Performance SQL',
+    'version': '1.0',
+    'category': 'Hidden',
+    'description': """
+Module to test performance of query generated by expression.py
+""",
+    'data': [
+
+    ],
+    'demo': [
+
+    ],
+    'installable': True,
+    'auto_install': False,
+    'license': 'LGPL-3',
+}
diff --git a/expression_performance/models/__init__.py b/expression_performance/models/__init__.py
@@ -0,0 +1 @@
+from . import perf_models
diff --git a/expression_performance/models/perf_models.py b/expression_performance/models/perf_models.py
@@ -0,0 +1,3 @@
+from odoo import api, fields, models, tools, SUPERUSER_ID, _
+
+
diff --git a/psql_exists_in_join/exists_in_join_compare.py b/psql_exists_in_join/exists_in_join_compare.py
@@ -1,4 +1,5 @@
 from collections import defaultdict
+from itertools import islice
 import random
 
 import psycopg2
@@ -17,6 +18,8 @@
     'sale_order',
 ]
 
+rng = random.Random(1)
+
 def drop_tables(cr):
     for t in tables:
         cr.execute(f"DROP TABLE IF EXISTS {t}")
@@ -30,17 +33,17 @@ def create_tables(cr):
     cr.execute("""
         CREATE TABLE sale_order (
             id SERIAL PRIMARY KEY,
-            state VARCHAR NOT NULL,  -- Can be 'draft', 'to confirm', 'confirmed', 'cancel' or 'done'
             partner VARCHAR NOT NULL
+            uniform_100 INT,
+            float_uniform_100 FLOAT,
         )
     """)
     cr.execute("""
         CREATE TABLE sale_order_line (
             id SERIAL PRIMARY KEY,
             sale_id INT NOT NULL,
-            delivery_date DATE,
-            qty FLOAT,
-            amount FLOAT,
+            uniform_100 INT,
+            float_uniform_100 FLOAT,
             FOREIGN KEY (sale_id) REFERENCES sale_order (id)
         )
     """)
@@ -63,19 +66,11 @@ def create_tables(cr):
 
 data_cases = {
     'small': {
-        'nb_sale_order': 50000,
-        'sale_order_state_proportion': {
-            'draft': 0.2,
-            'to confirm': 0.2,
-            'confirmed': 0.2,
-            'cancel': 0.2,
-            'done': 0.2,
-        },
-
-        'nb_sale_order_line': 500000,
-        'qty_sale_order_line': [0, 100000],
+        'nb_sale_order': 50_000,
+        'nb_sale_order_line': 500_000,
 
         'sale_order_tags': ['urgent', 'safe', 'boring client', 'water', 'other'],
+        'sale_order_tags_weight': [0.1, 0.1, 0.1, 0.1, 0.1],
         'nb_sale_order_tag_line_rel': 10000,
     },
 }
@@ -85,33 +80,53 @@ def create_indexes(cr):
     
     """)
 
-def fill_database(data_case):
+def get_values_generator_so(**kwargs):
+    states = list(kwargs['sale_order_state_proportion'].keys())
+    states_w = list(kwargs['sale_order_state_proportion'].values())
+    for i in range(kwargs['nb_sale_order']):
+        yield (
+            rng.choices(states, states_w, k=1)[0],
+            f'partern_{i}',
+        )
 
-    with psycopg2.connect(CONNECTION_PARAMS) as conn, conn.cursor() as cr:
-        states = []
-        for state, proportion in data_case['sale_order_state_proportion'].items():
-            states += [state] * int(proportion * data_case['nb_sale_order'])
-        values_sale_order = tuple(
-            (state, f'partern_{i}')
-            for i, state in zip(range(data_case['nb_sale_order']), random.sample(states, k=len(states)))
+def get_values_generator_sol(**kwargs):
+    for i in range(kwargs['nb_sale_order_line']):
+        yield (
+            rng.randint(1, kwargs['nb_sale_order']),
+            rng.randint(0, 100),
+            rng.random() * 100,
         )
 
-        cr.execute(f"""
-            INSERT INTO sale_order (state, partner) VALUES {','.join(['%s'] * len(values_sale_order))}
-        """, values_sale_order)
+def split_every(n, iterator, piece_maker=tuple):
+    """Splits an iterable into length-n pieces. The last piece will be shorter
+       if ``n`` does not evenly divide the iterable length.
+
+       :param int n: maximum size of each generated chunk
+       :param Iterable iterable: iterable to chunk into pieces
+       :param piece_maker: callable taking an iterable and collecting each
+                           chunk from its slice, *must consume the entire slice*.
+    """
+    piece = piece_maker(islice(iterator, n))
+    while piece:
+        yield piece
+        piece = piece_maker(islice(iterator, n))
+
+def fill_database(data_case):
+    BATCH_SIZE_CREATION = 100_000
 
     with psycopg2.connect(CONNECTION_PARAMS) as conn, conn.cursor() as cr:
-        values_sale_order_line = tuple(
-            (
-                random.randint(1, data_case['nb_sale_order']),
-                random.randint(data_case['qty_sale_order_line'][0], data_case['qty_sale_order_line'][1]),
-            )
-            for i in range(data_case['nb_sale_order_line'])
-        )
+        for i, pieces in enumerate(split_every(BATCH_SIZE_CREATION, get_values_generator_so(**data_case))):
+            print(f"Insert {i * BATCH_SIZE_CREATION}/{data_case['nb_sale_order']} SO")
+            cr.execute(f"""
+                INSERT INTO sale_order (state, partner, uniform_100, float_uniform_100) VALUES {','.join(['%s'] * len(pieces))}
+            """, pieces)
 
-        cr.execute(f"""
-            INSERT INTO sale_order_line (sale_id, qty) VALUES {','.join(['%s'] * len(values_sale_order_line))}
-        """, values_sale_order_line)
+    with psycopg2.connect(CONNECTION_PARAMS) as conn, conn.cursor() as cr:
+        for pieces in split_every(BATCH_SIZE_CREATION, get_values_generator_sol(**data_case)):
+            print(f"Insert {i * BATCH_SIZE_CREATION}/{data_case['nb_sale_order_line']} SOL ")
+            cr.execute(f"""
+                INSERT INTO sale_order_line (sale_id, qty, price) VALUES {','.join(['%s'] * len(pieces))}
+            """, pieces)
 
 
 SQL_alternative = {
@@ -139,7 +154,7 @@ def fill_database(data_case):
             ORDER BY "sale_order_line"."id"
         """,
 
-        'join':
+        'left join':
         """
             SELECT "sale_order_line"."id"
             FROM "sale_order_line"
@@ -197,12 +212,12 @@ def fill_database(data_case):
             ORDER BY "sale_order_line"."id"
         """,
 
-        'join':
+        'left join':
         """
             SELECT "sale_order_line"."id"
             FROM "sale_order_line"
                 LEFT JOIN "sale_order" ON "sale_order"."id" = "sale_order_line"."sale_id"
-            WHERE "sale_order"."state" = 'draft' OR "sale_order_line"."qty" > 50000
+            WHERE ("sale_order"."state" = 'draft') OR "sale_order_line"."qty" > 50000
             ORDER BY "sale_order_line"."id"
         """,
 
@@ -216,18 +231,6 @@ def fill_database(data_case):
             WHERE "sale_order_line"."sale_id" IN (SELECT "id" FROM "subquery") OR "sale_order_line"."qty" > 50000
             ORDER BY "sale_order_line"."id"
         """,
-
-        # 'CTE, join':
-        # """
-        #     WITH subquery AS (
-        #         SELECT * FROM "sale_order" WHERE "sale_order"."state" = 'draft'
-        #     )
-        #     SELECT "sale_order_line"."id"
-        #     FROM "sale_order_line"
-        #         LEFT JOIN "subquery" ON "subquery"."id" = "sale_order_line"."sale_id"
-        #     WHERE "subquery"."id" IS NOT NULL OR "sale_order_line"."qty" > 50000
-        #     ORDER BY "sale_order_line"."id"
-        # """,
     },
     "sale.order.line - search - [order_id.state = 'draft' AND qty > 50000]": {
         'in': """
@@ -255,7 +258,7 @@ def fill_database(data_case):
             ORDER BY "sale_order_line"."id"
         """,
 
-        'join':
+        'left join':
         """
             SELECT "sale_order_line"."id"
             FROM "sale_order_line"
@@ -274,23 +277,10 @@ def fill_database(data_case):
             WHERE "sale_order_line"."sale_id" IN (SELECT "id" FROM "subquery") AND "sale_order_line"."qty" > 50000
             ORDER BY "sale_order_line"."id"
         """,
-
-        # 'CTE, join':
-        # """
-        #     WITH subquery AS (
-        #         SELECT * FROM "sale_order" WHERE "sale_order"."state" = 'draft'
-        #     )
-        #     SELECT "sale_order_line"."id"
-        #     FROM "sale_order_line"
-        #         LEFT JOIN "subquery" ON "subquery"."id" = "sale_order_line"."sale_id"
-        #     WHERE "subquery"."id" IS NOT NULL AND "sale_order_line"."qty" > 50000
-        #     ORDER BY "sale_order_line"."id"
-        # """,
     },
 }
 
 def one_data_case_test():
-
     result_explain = defaultdict(dict)
     for alternative, sql_requests in SQL_alternative.items():
         with psycopg2.connect(CONNECTION_PARAMS) as conn, conn.cursor() as cr:
@@ -300,18 +290,18 @@ def one_data_case_test():
                 current_res = tuple(cr.fetchall())
                 if res and current_res != res:
                     print(f"{res} != {current_res}")
-                    raise ValueError(f"{case_str} doens't have the same result than the previous one for {alternative}")
+                    raise ValueError(f"ERROR: {case_str} doens't have the same result than the previous one for {alternative}")
                 res = current_res
 
-        for case_str, sql_request in sql_requests.items():
-            with psycopg2.connect(CONNECTION_PARAMS) as conn, conn.cursor() as cr:
+        with psycopg2.connect(CONNECTION_PARAMS) as conn, conn.cursor() as cr:
+            for case_str, sql_request in sql_requests.items():
                 plan = psql_explain(cr, sql_request)
                 result_explain[alternative][case_str] = plan
 
     for alternative, case_plans in result_explain.items():
         list_plan = list(case_plans.values())
         if len(set(list_plan)) != 1:
-            print(f"\n-> Some queries has a different plan for {alternative} ({len(set(list_plan))}):")
+            print(f"\n=> Some queries has a different plan for {alternative} ({len(set(list_plan))}):")
             case_by_plan = defaultdict(list)
             for case, plan in case_plans.items():
                 case_by_plan[plan].append(case)
@@ -320,7 +310,7 @@ def one_data_case_test():
                 print(f"These cases {cases} has this plan:\n{plan}")
                 print("  ------------------------")
         else:
-            print(f"\n-> {alternative} all plan is equals")
+            print(f"\n=> {alternative} all plan is equals")
 
 
 def main():
@@ -352,8 +342,7 @@ def main():
 # https://github.com/postgres/postgres/commit/41efb8340877e8ffd0023bb6b2ef22ffd1ca014d
 
 
-
-
+# TODO Convert this stuff in a Odoo module to test it with expression.py 
 
 if __name__ == '__main__':
     main()

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+from odoo import api, fields, models, tools, SUPERUSER_ID, _`
	`2`	`+`
	`3`	`+`