From 730447f90efb2478097f06c2ed5d965ac65b7874 Mon Sep 17 00:00:00 2001 From: Akash <112477155+akash-chandrakar@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:15:15 +0400 Subject: [PATCH] fix(db-index): Add missing index from FRI prover jobs (#334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What ❔ Add missing index from FRI prover job ## Why ❔ Required to run query efficiently ## Checklist - [ *] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ *] Tests for the changes have been added / updated. - [ *] Documentation comments have been added / updated. - [ *] Code has been formatted via `zk fmt` and `zk lint`. --- ...ssing_index_for_fri_prover_tables.down.sql | 9 ++ ...missing_index_for_fri_prover_tables.up.sql | 11 ++ core/lib/dal/sqlx-data.json | 132 +++++++++--------- core/lib/dal/src/fri_prover_dal.rs | 3 +- 4 files changed, 88 insertions(+), 67 deletions(-) create mode 100644 core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql create mode 100644 core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql new file mode 100644 index 00000000000..6c80db7472a --- /dev/null +++ b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.down.sql @@ -0,0 +1,9 @@ +CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status_processing_attempts + ON prover_jobs_fri (processing_started_at, attempts) + WHERE status IN ('in_progress', 'failed'); + +DROP INDEX IF EXISTS prover_jobs_fri_status_processing_started_at_idx_2; + +DROP INDEX IF EXISTS idx_prover_jobs_fri_status; + +DROP INDEX IF EXISTS idx_prover_jobs_fri_circuit_id_agg_batch_num; diff --git a/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql new file mode 100644 index 00000000000..1a9eaece6a3 --- /dev/null +++ b/core/lib/dal/migrations/20231027074517_add_missing_index_for_fri_prover_tables.up.sql @@ -0,0 +1,11 @@ +CREATE INDEX IF NOT EXISTS prover_jobs_fri_status_processing_started_at_idx_2 ON prover_jobs_fri (status, processing_started_at) + WHERE (attempts < 20); + +DROP INDEX IF EXISTS idx_prover_jobs_fri_status_processing_attempts; + +CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_status ON prover_jobs_fri (circuit_id, aggregation_round) + WHERE (status != 'successful' and status != 'skipped'); + +CREATE INDEX IF NOT EXISTS idx_prover_jobs_fri_circuit_id_agg_batch_num + ON prover_jobs_fri (circuit_id, aggregation_round, l1_batch_number) + WHERE status IN ('queued', 'in_progress', 'in_gpu_proof', 'failed'); diff --git a/core/lib/dal/sqlx-data.json b/core/lib/dal/sqlx-data.json index 29aea31db1f..7fc1026a3d5 100644 --- a/core/lib/dal/sqlx-data.json +++ b/core/lib/dal/sqlx-data.json @@ -285,36 +285,6 @@ }, "query": "\n INSERT INTO node_aggregation_witness_jobs\n (l1_batch_number, protocol_version, status, created_at, updated_at)\n VALUES ($1, $2, 'waiting_for_artifacts', now(), now())\n " }, - "0c729d441aceba247e36c08a89661c35b476d4d7c73882147699009affe78472": { - "describe": { - "columns": [ - { - "name": "l1_batch_number!", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "circuit_id", - "ordinal": 1, - "type_info": "Int2" - }, - { - "name": "aggregation_round", - "ordinal": 2, - "type_info": "Int2" - } - ], - "nullable": [ - null, - false, - false - ], - "parameters": { - "Left": [] - } - }, - "query": "\n SELECT MIN(l1_batch_number) as \"l1_batch_number!\", circuit_id, aggregation_round\n FROM prover_jobs_fri\n WHERE status IN('queued', 'in_progress', 'failed')\n GROUP BY circuit_id, aggregation_round\n " - }, "0d1bed183c38304ff1a6c8c78dca03964e2e188a6d01f98eaf0c6b24f19b8b6f": { "describe": { "columns": [], @@ -1374,42 +1344,6 @@ }, "query": "\n UPDATE node_aggregation_witness_jobs_fri\n SET status='queued'\n WHERE (l1_batch_number, circuit_id, depth) IN\n (SELECT prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth\n FROM prover_jobs_fri\n JOIN node_aggregation_witness_jobs_fri nawj ON\n prover_jobs_fri.l1_batch_number = nawj.l1_batch_number\n AND prover_jobs_fri.circuit_id = nawj.circuit_id\n AND prover_jobs_fri.depth = nawj.depth\n WHERE nawj.status = 'waiting_for_proofs'\n AND prover_jobs_fri.status = 'successful'\n AND prover_jobs_fri.aggregation_round = 1\n AND prover_jobs_fri.depth = 0\n GROUP BY prover_jobs_fri.l1_batch_number, prover_jobs_fri.circuit_id, prover_jobs_fri.depth, nawj.number_of_dependent_jobs\n HAVING COUNT(*) = nawj.number_of_dependent_jobs)\n RETURNING l1_batch_number, circuit_id, depth;\n " }, - "1e68306cbd83eb6b5de59fb8f638c8f3252732b9074e2455f7b5aedf6fdc886f": { - "describe": { - "columns": [ - { - "name": "count!", - "ordinal": 0, - "type_info": "Int8" - }, - { - "name": "circuit_id!", - "ordinal": 1, - "type_info": "Int2" - }, - { - "name": "aggregation_round!", - "ordinal": 2, - "type_info": "Int2" - }, - { - "name": "status!", - "ordinal": 3, - "type_info": "Text" - } - ], - "nullable": [ - null, - false, - false, - false - ], - "parameters": { - "Left": [] - } - }, - "query": "\n SELECT COUNT(*) as \"count!\", circuit_id as \"circuit_id!\", aggregation_round as \"aggregation_round!\", status as \"status!\"\n FROM prover_jobs_fri\n GROUP BY circuit_id, aggregation_round, status\n " - }, "1ed353a16e8d0abaf426e5c235b20a79c727c08bc23fb1708a833a6930131691": { "describe": { "columns": [], @@ -4038,6 +3972,42 @@ }, "query": "SELECT bootloader_code_hash, default_account_code_hash FROM protocol_versions\n WHERE id = $1\n " }, + "51d02f2e314ebf78c27949cc10997bd2171755400cc3a13c63994c85e15cb3df": { + "describe": { + "columns": [ + { + "name": "count!", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "circuit_id!", + "ordinal": 1, + "type_info": "Int2" + }, + { + "name": "aggregation_round!", + "ordinal": 2, + "type_info": "Int2" + }, + { + "name": "status!", + "ordinal": 3, + "type_info": "Text" + } + ], + "nullable": [ + null, + false, + false, + false + ], + "parameters": { + "Left": [] + } + }, + "query": "\n SELECT COUNT(*) as \"count!\", circuit_id as \"circuit_id!\", aggregation_round as \"aggregation_round!\", status as \"status!\"\n FROM prover_jobs_fri\n WHERE status <> 'skipped' and status <> 'successful'\n GROUP BY circuit_id, aggregation_round, status\n " + }, "52eeb8c529efb796fdefb30a381fcf6c931512f30e55e24c155f6c649e662909": { "describe": { "columns": [ @@ -7762,6 +7732,36 @@ }, "query": "UPDATE prover_jobs_fri SET status = $1, updated_at = now() WHERE id = $2" }, + "98c81ee6f73859c6cd6ba54ab438c900dda646b70a700f936e5218d9ba3bd0ec": { + "describe": { + "columns": [ + { + "name": "l1_batch_number!", + "ordinal": 0, + "type_info": "Int8" + }, + { + "name": "circuit_id", + "ordinal": 1, + "type_info": "Int2" + }, + { + "name": "aggregation_round", + "ordinal": 2, + "type_info": "Int2" + } + ], + "nullable": [ + null, + false, + false + ], + "parameters": { + "Left": [] + } + }, + "query": "\n SELECT MIN(l1_batch_number) as \"l1_batch_number!\", circuit_id, aggregation_round\n FROM prover_jobs_fri\n WHERE status IN('queued', 'in_gpu_proof', 'in_progress', 'failed')\n GROUP BY circuit_id, aggregation_round\n " + }, "9970bb69f5ca9ab9f103e1547eb40c1d4f5dd3a540ff6f1b9724821350c9501a": { "describe": { "columns": [ diff --git a/core/lib/dal/src/fri_prover_dal.rs b/core/lib/dal/src/fri_prover_dal.rs index af1c218195c..845e793948d 100644 --- a/core/lib/dal/src/fri_prover_dal.rs +++ b/core/lib/dal/src/fri_prover_dal.rs @@ -268,6 +268,7 @@ impl FriProverDal<'_, '_> { r#" SELECT COUNT(*) as "count!", circuit_id as "circuit_id!", aggregation_round as "aggregation_round!", status as "status!" FROM prover_jobs_fri + WHERE status <> 'skipped' and status <> 'successful' GROUP BY circuit_id, aggregation_round, status "# ) @@ -301,7 +302,7 @@ impl FriProverDal<'_, '_> { r#" SELECT MIN(l1_batch_number) as "l1_batch_number!", circuit_id, aggregation_round FROM prover_jobs_fri - WHERE status IN('queued', 'in_progress', 'failed') + WHERE status IN('queued', 'in_gpu_proof', 'in_progress', 'failed') GROUP BY circuit_id, aggregation_round "# )