Add test module perf

Author: ryv-odoo

expression_performance/models/perf_models.py

@@ -66,9 +66,13 @@ class PerfLine(models.Model):
     def _custom_populate_factories(self, rng: Random):
         ids_container = self.env['perf.container'].search([])._ids
         generator_ab = super()._custom_populate_factories(rng)
-        for __ in itertools.count():
+        for i in itertools.count():
+            parent_id = False
+            if i > 1 and rng.random() > 0.01:  # 25 % of set
+                parent_id = rng.randint(1, i-1)
             yield next(generator_ab) | {
                 'uniform_container_id': rng.choice(ids_container),
+                'parent_id': parent_id,
             }
 
 

expression_performance/tests/test_sql.py

@@ -16,6 +16,15 @@ def get_db_name():
     return db
 
 
+ratios_possible = [
+    500, # 50 %
+    50,  # 5 %
+    5,   # 0.5 %
+    0.5,  # 0.05 %
+    0.01,  # 0.001 %
+]
+
+
 class TestPerformanceSQL(BaseCase):
 
     @classmethod
@@ -31,7 +40,7 @@ def setUpClass(cls):
 
         cls.env = api.Environment(cls.cr, odoo.SUPERUSER_ID, {})
 
-    def populate(self, nb_tag=0, nb_container=100_000, nb_line=500_000):
+    def populate(self, nb_tag=0, nb_container=10_000, nb_line=50_000):
         models = {
             'perf.tag': nb_tag,
             'perf.container': nb_container,
@@ -41,23 +50,31 @@ def populate(self, nb_tag=0, nb_container=100_000, nb_line=500_000):
             self.env[model]._custom_populate(nb)
 
         self.env.cr.execute('ANALYSE;')
+        self.env.cr.commit()
 
     def create_indexes_many2one(self):
         self.env.cr.execute("CREATE INDEX IF NOT EXISTS perf_line_uniform_container_id ON perf_line(uniform_container_id)")
         self.env.cr.execute("CREATE INDEX IF NOT EXISTS perf_line_parent_id ON perf_line(parent_id)")
         self.env.cr.commit()
 
-    def delete_all_indexes(self):
+    def create_indexes_uniform(self):
+        for table in ['perf_line', 'perf_container', 'perf_tag']:
+            self.env.cr.execute(f"CREATE INDEX IF NOT EXISTS {table}_float_uniform_1000 ON {table}(float_uniform_1000)")
+        self.env.cr.commit()
+
+    def delete_indexes_uniform(self):
         for table in ['perf_line', 'perf_container', 'perf_tag']:
             self.env.cr.execute(f"DROP INDEX IF EXISTS {table}_float_uniform_1000")
+        self.env.cr.commit()
 
+    def delete_indexes_many2one(self):
         self.env.cr.execute("DROP INDEX IF EXISTS perf_line_uniform_container_id")
+        self.env.cr.execute("DROP INDEX IF EXISTS perf_line_parent_id")
         self.env.cr.commit()
 
-    def create_indexes_uniform(self):
-        for table in ['perf_line', 'perf_container', 'perf_tag']:
-            self.env.cr.execute(f"CREATE INDEX IF NOT EXISTS {table}_float_uniform_1000 ON {table}(float_uniform_1000)")
-        self.env.cr.commit()
+    def delete_all_indexes(self):
+        self.delete_indexes_uniform()
+        self.delete_indexes_many2one()
 
     def test_create_indexes_many2one(self):
         self.create_indexes_many2one()
@@ -68,19 +85,23 @@ def test_create_indexes_uniform(self):
     def test_delete_all_indexes(self):
         self.delete_all_indexes()
 
-    def test_one_level_condition(self):
-        self.populate()
+    def test_truncate_everything(self):
+        print('Truncate everything')
+        for table in ['perf_line', 'perf_container', 'perf_tag']:
+            self.env.cr.execute(f'TRUNCATE "{table}" RESTART IDENTITY CASCADE')
         self.env.cr.commit()
 
-        ratios_possible = [
-            500, # 50 %
-            50,  # 5 %
-            5,   # 0.5 %
-            0.5,  # 0.05 %
-            0.01,  # 0.001 %
-        ]
+    def test_one_sub_level_1_and(self):
+        self.test_truncate_everything()
+        self.populate()
+
+        def domain_like(args):
+            return [
+                '&',
+                ('float_uniform_1000', '<', args[0]),
+                ('uniform_container_id.float_uniform_1000', '<', args[1]),
+            ]
 
-        # For [<condition_1> line AND <condition_2> line.parent_id] on line
         queries = {
             'join': """
 SELECT "perf_line"."id"
@@ -91,6 +112,7 @@ def test_one_level_condition(self):
     "perf_container"."float_uniform_1000" < %s
 ORDER BY "perf_line"."id"
             """,
+
             'subselect': """
 SELECT "perf_line"."id"
 FROM "perf_line"
@@ -99,6 +121,7 @@ def test_one_level_condition(self):
     "perf_line"."uniform_container_id" IN (SELECT "perf_container"."id" FROM "perf_container" WHERE "perf_container"."float_uniform_1000" < %s)
 ORDER BY "perf_line"."id"
             """,
+
             'exists': """
 SELECT "perf_line"."id"
 FROM "perf_line"
@@ -109,31 +132,24 @@ def test_one_level_condition(self):
             """,
         }
 
-        for line_ratio, container_ratio in combinations_with_replacement(ratios_possible, 2):
-            arguments = [line_ratio, container_ratio]
-            res = None
-            for name, query in queries.items():
-                self.env.cr.execute(query, arguments)
-                res_new = [id_ for id_, in self.env.cr.fetchall()]
-                if res and res != res_new:
-                    raise Exception(f'{name} not the same result than the previous one')
+        self.launch_queries(
+            'Level 1 - AND',
+            queries,
+            combinations_with_replacement(ratios_possible, 2),
+            domain_like,
+        )
 
-            explain_dict = {}
-            for name, query in queries.items():
-                self.env.cr.execute("EXPLAIN " + query, arguments)
-                explain_dict[name] = "\n".join(s for s, in self.env.cr.fetchall())
+    def test_one_sub_level_1_or(self):
+        self.test_truncate_everything()
+        self.populate()
 
-            if len(set(explain_dict.values())) == 1:
-                print("AND - For arguments - All the same", arguments)
-                continue
-            
-            print()
-            print('AND - For arguments', arguments)
-            for name, explain in explain_dict.items():
-                print(f'=> {name} : ')
-                print(explain)
+        def domain_like(args):
+            return [
+                '|',
+                ('float_uniform_1000', '<', args[0]),
+                ('uniform_container_id.float_uniform_1000', '<', args[1]),
+            ]
 
-        # For [<condition_1> line OR <condition_2> line.parent_id] on line
         queries = {
             'join': """
 SELECT "perf_line"."id"
@@ -144,6 +160,7 @@ def test_one_level_condition(self):
     "perf_container"."float_uniform_1000" < %s
 ORDER BY "perf_line"."id"
             """,
+
             'subselect': """
 SELECT "perf_line"."id"
 FROM "perf_line"
@@ -152,6 +169,7 @@ def test_one_level_condition(self):
     "perf_line"."uniform_container_id" IN (SELECT "perf_container"."id" FROM "perf_container" WHERE "perf_container"."float_uniform_1000" < %s)
 ORDER BY "perf_line"."id"
             """,
+
             'exists': """
 SELECT "perf_line"."id"
 FROM "perf_line"
@@ -162,25 +180,90 @@ def test_one_level_condition(self):
             """,
         }
 
-        for line_ratio, container_ratio in combinations_with_replacement(ratios_possible, 2):
-            arguments = [line_ratio, container_ratio]
+        self.launch_queries(
+            'Level 1 - OR',
+            queries,
+            combinations_with_replacement(ratios_possible, 2),
+            domain_like,
+        )
+
+    def test_two_level_condition(self):
+        self.test_truncate_everything()
+        self.populate()
+
+        def domain_like(args):
+            return [
+                ('parent_id.uniform_container_id.float_uniform_1000', '<', args[0])
+            ]
+
+        queries = {
+            'join': """
+SELECT "perf_line"."id"
+FROM "perf_line"
+    LEFT JOIN "perf_line" AS "perf_line__parent_id" ON "perf_line__parent_id"."id" = "perf_line"."parent_id"
+    LEFT JOIN "perf_container" ON "perf_line__parent_id"."uniform_container_id" = "perf_container"."id"
+WHERE
+    "perf_container"."float_uniform_1000" < %s
+ORDER BY "perf_line"."id"
+            """,
+
+            'subselect': """
+SELECT "perf_line"."id"
+FROM "perf_line"
+WHERE
+    "perf_line"."parent_id" IN (
+        SELECT "perf_line"."id" FROM "perf_line"
+        WHERE "perf_line"."uniform_container_id" IN (
+            SELECT "perf_container"."id" FROM "perf_container"
+            WHERE "perf_container"."float_uniform_1000" < %s
+        )
+    )
+ORDER BY "perf_line"."id"
+            """,
+
+            'exists': """
+SELECT "perf_line"."id"
+FROM "perf_line"
+WHERE
+    EXISTS (
+        SELECT FROM "perf_line" AS "perf_line__parent_id"
+        WHERE EXISTS (
+            SELECT FROM "perf_container"
+            WHERE "perf_container"."float_uniform_1000" < %s AND "perf_line__parent_id"."uniform_container_id" = "perf_container"."id"
+        ) AND "perf_line"."parent_id" = "perf_line__parent_id"."id"
+    )
+ORDER BY "perf_line"."id"
+            """,
+        }
+
+        self.launch_queries(
+            'Level 2 - For arguments',
+            queries,
+            combinations_with_replacement(ratios_possible, 1),
+            domain_like,
+        )
+
+    def launch_queries(self, name_test, queries, args_combinaison, domain_like: callable):
+        for arguments in args_combinaison:
             res = None
             for name, query in queries.items():
                 self.env.cr.execute(query, arguments)
                 res_new = [id_ for id_, in self.env.cr.fetchall()]
                 if res and res != res_new:
-                    raise Exception(f'{name} not the same result than the previous one')
+                    raise Exception(f'{name} not the same result than the previous one: {name_test}')
+                res = res_new
 
             explain_dict = {}
             for name, query in queries.items():
                 self.env.cr.execute("EXPLAIN " + query, arguments)
                 explain_dict[name] = "\n".join(s for s, in self.env.cr.fetchall())
 
-            if set(explain_dict.values()) == 1:
+            if len(set(explain_dict.values())) == 1:
+                print(f"{name_test}: {domain_like(arguments)} : ALL the same")
                 continue
 
             print()
-            print('OR - For arguments', arguments)
+            print(f"{name_test}: {domain_like(arguments)}")
             for name, explain in explain_dict.items():
                 print(f'=> {name} : ')
                 print(explain)