fix(db-index): Add missing index from FRI prover jobs (#334)

# What ❔ Add missing index from FRI prover job ## Why ❔ Required to run query efficiently ## Checklist - [ *] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ *] Tests for the changes have been added / updated. - [ *] Documentation comments have been added / updated. - [ *] Code has been formatted via `zk fmt` and `zk lint`.
matter-labs · Oct 27, 2023 · 730447f · 730447f
1 parent eefaad0
commit 730447f
Show file tree

Hide file tree

Showing 4 changed files with 88 additions and 67 deletions.
diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql
@@ -0,0 +1,9 @@
+CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts
+    ON prover_jobs_fri (processing_started_at, attempts)
+    WHERE status IN ('in_progress', 'failed');
+
+DROP INDEX IF EXISTS prover_jobs_fri_status_processing_started_at_idx_2;
+
+DROP INDEX IF EXISTS idx_prover_jobs_fri_status;
+
+DROP INDEX IF EXISTS idx_prover_jobs_fri_circuit_id_agg_batch_num;
diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql
@@ -0,0 +1,11 @@
+CREATE INDEX IF NOT EXISTS prover_jobs_fri_status_processing_started_at_idx_2 ON prover_jobs_fri (status, processing_started_at)
+    WHERE (attempts < 20);
+
+DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts;
+
+CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status ON prover_jobs_fri (circuit_id, aggregation_round)
+    WHERE (status != 'successful' and status != 'skipped');
+
+CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_circuit_id_agg_batch_num
+    ON prover_jobs_fri (circuit_id, aggregation_round, l1_batch_number)
+    WHERE status IN ('queued', 'in_progress', 'in_gpu_proof', 'failed');
diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json
@@ -285,36 +285,6 @@
     },
     "query": "\n                    INSERT INTO node_aggregation_witness_jobs\n                        (l1_batch_number, protocol_version, status, created_at, updated_at)\n                    VALUES ($1, $2, 'waiting_for_artifacts', now(), now())\n                    "
   },
-  "0c729d441aceba247e36c08a89661c35b476d4d7c73882147699009affe78472": {
-    "describe": {
-      "columns": [
-        {
-          "name": "l1_batch_number!",
-          "ordinal": 0,
-          "type_info": "Int8"
-        },
-        {
-          "name": "circuit_id",
-          "ordinal": 1,
-          "type_info": "Int2"
-        },
-        {
-          "name": "aggregation_round",
-          "ordinal": 2,
-          "type_info": "Int2"
-        }
-      ],
-      "nullable": [
-        null,
-        false,
-        false
-      ],
-      "parameters": {
-        "Left": []
-      }
-    },
-    "query": "\n                    SELECT MIN(l1_batch_number) as \"l1_batch_number!\", circuit_id, aggregation_round\n                    FROM prover_jobs_fri\n                    WHERE status IN('queued', 'in_progress', 'failed')\n                    GROUP BY circuit_id, aggregation_round\n                "
-  },
   "0d1bed183c38304ff1a6c8c78dca03964e2e188a6d01f98eaf0c6b24f19b8b6f": {
     "describe": {
       "columns": [],
@@ -1374,42 +1344,6 @@
     },
     "query": "\n                UPDATE node_aggregation_witness_jobs_fri\n                SET status='queued'\n                WHERE (l1_batch_number, circuit_id, depth) IN\n                      (SELECT prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth\n                       FROM prover_jobs_fri\n                                JOIN node_aggregation_witness_jobs_fri nawj ON\n                                prover_jobs_fri.l1_batch_number = nawj.l1_batch_number\n                                AND prover_jobs_fri.circuit_id = nawj.circuit_id\n                                AND prover_jobs_fri.depth = nawj.depth\n                       WHERE nawj.status = 'waiting_for_proofs'\n                         AND prover_jobs_fri.status = 'successful'\n                         AND prover_jobs_fri.aggregation_round = 1\n                         AND prover_jobs_fri.depth = 0\n                       GROUP BY prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth, nawj.number_of_dependent_jobs\n                       HAVING COUNT(*) = nawj.number_of_dependent_jobs)\n                RETURNING l1_batch_number, circuit_id, depth;\n            "
   },
-  "1e68306cbd83eb6b5de59fb8f638c8f3252732b9074e2455f7b5aedf6fdc886f": {
-    "describe": {
-      "columns": [
-        {
-          "name": "count!",
-          "ordinal": 0,
-          "type_info": "Int8"
-        },
-        {
-          "name": "circuit_id!",
-          "ordinal": 1,
-          "type_info": "Int2"
-        },
-        {
-          "name": "aggregation_round!",
-          "ordinal": 2,
-          "type_info": "Int2"
-        },
-        {
-          "name": "status!",
-          "ordinal": 3,
-          "type_info": "Text"
-        }
-      ],
-      "nullable": [
-        null,
-        false,
-        false,
-        false
-      ],
-      "parameters": {
-        "Left": []
-      }
-    },
-    "query": "\n                SELECT COUNT(*) as \"count!\", circuit_id as \"circuit_id!\", aggregation_round as \"aggregation_round!\", status as \"status!\"\n                FROM prover_jobs_fri\n                GROUP BY circuit_id, aggregation_round, status\n                "
-  },
   "1ed353a16e8d0abaf426e5c235b20a79c727c08bc23fb1708a833a6930131691": {
     "describe": {
       "columns": [],
@@ -4038,6 +3972,42 @@
     },
     "query": "SELECT bootloader_code_hash, default_account_code_hash FROM protocol_versions\n                WHERE id = $1\n            "
   },
+  "51d02f2e314ebf78c27949cc10997bd2171755400cc3a13c63994c85e15cb3df": {
+    "describe": {
+      "columns": [
+        {
+          "name": "count!",
+          "ordinal": 0,
+          "type_info": "Int8"
+        },
+        {
+          "name": "circuit_id!",
+          "ordinal": 1,
+          "type_info": "Int2"
+        },
+        {
+          "name": "aggregation_round!",
+          "ordinal": 2,
+          "type_info": "Int2"
+        },
+        {
+          "name": "status!",
+          "ordinal": 3,
+          "type_info": "Text"
+        }
+      ],
+      "nullable": [
+        null,
+        false,
+        false,
+        false
+      ],
+      "parameters": {
+        "Left": []
+      }
+    },
+    "query": "\n                SELECT COUNT(*) as \"count!\", circuit_id as \"circuit_id!\", aggregation_round as \"aggregation_round!\", status as \"status!\"\n                FROM prover_jobs_fri\n                WHERE status <> 'skipped' and status <> 'successful'\n                GROUP BY circuit_id, aggregation_round, status\n                "
+  },
   "52eeb8c529efb796fdefb30a381fcf6c931512f30e55e24c155f6c649e662909": {
     "describe": {
       "columns": [
@@ -7762,6 +7732,36 @@
     },
     "query": "UPDATE prover_jobs_fri SET status = $1, updated_at = now() WHERE id = $2"
   },
+  "98c81ee6f73859c6cd6ba54ab438c900dda646b70a700f936e5218d9ba3bd0ec": {
+    "describe": {
+      "columns": [
+        {
+          "name": "l1_batch_number!",
+          "ordinal": 0,
+          "type_info": "Int8"
+        },
+        {
+          "name": "circuit_id",
+          "ordinal": 1,
+          "type_info": "Int2"
+        },
+        {
+          "name": "aggregation_round",
+          "ordinal": 2,
+          "type_info": "Int2"
+        }
+      ],
+      "nullable": [
+        null,
+        false,
+        false
+      ],
+      "parameters": {
+        "Left": []
+      }
+    },
+    "query": "\n                    SELECT MIN(l1_batch_number) as \"l1_batch_number!\", circuit_id, aggregation_round\n                    FROM prover_jobs_fri\n                    WHERE status IN('queued', 'in_gpu_proof', 'in_progress', 'failed')\n                    GROUP BY circuit_id, aggregation_round\n                "
+  },
   "9970bb69f5ca9ab9f103e1547eb40c1d4f5dd3a540ff6f1b9724821350c9501a": {
     "describe": {
       "columns": [

diff --git a/core/lib/dal/src/fri_prover_dal.rs b/core/lib/dal/src/fri_prover_dal.rs
@@ -268,6 +268,7 @@ impl FriProverDal<'_, '_> {
                 r#"
                 SELECT COUNT(*) as "count!", circuit_id as "circuit_id!", aggregation_round as "aggregation_round!", status as "status!"
                 FROM prover_jobs_fri
+                WHERE status <> 'skipped' and status <> 'successful'
                 GROUP BY circuit_id, aggregation_round, status
                 "#
             )
@@ -301,7 +302,7 @@ impl FriProverDal<'_, '_> {
                 r#"
                     SELECT MIN(l1_batch_number) as "l1_batch_number!", circuit_id, aggregation_round
                     FROM prover_jobs_fri
-                    WHERE status IN('queued', 'in_progress', 'failed')
+                    WHERE status IN('queued', 'in_gpu_proof', 'in_progress', 'failed')
                     GROUP BY circuit_id, aggregation_round
                 "#
             )