diff --git a/core/lib/dal/.sqlx/query-362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395.json b/core/lib/dal/.sqlx/query-362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395.json deleted file mode 100644 index ef84a26a6e8..00000000000 --- a/core/lib/dal/.sqlx/query-362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM storage_logs\n WHERE\n storage_logs.miniblock_number < $1\n AND hashed_key IN (\n SELECT\n hashed_key\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395" -} diff --git a/core/lib/dal/.sqlx/query-6ebb549e274b7e684cde480c78e1cf4274a870c0ff7801e61807ff78cfe398f8.json b/core/lib/dal/.sqlx/query-6ebb549e274b7e684cde480c78e1cf4274a870c0ff7801e61807ff78cfe398f8.json new file mode 100644 index 00000000000..fc65c45e323 --- /dev/null +++ b/core/lib/dal/.sqlx/query-6ebb549e274b7e684cde480c78e1cf4274a870c0ff7801e61807ff78cfe398f8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM storage_logs\n WHERE\n ctid IN (\n SELECT\n prev_logs.ctid\n FROM\n storage_logs AS prev_logs\n INNER JOIN LATERAL (\n SELECT\n 1\n FROM\n storage_logs AS current_logs\n WHERE\n current_logs.miniblock_number BETWEEN $1 AND $2\n AND current_logs.hashed_key = prev_logs.hashed_key\n ) AS current_logs ON TRUE\n WHERE\n prev_logs.miniblock_number < $1\n LIMIT\n $3\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6ebb549e274b7e684cde480c78e1cf4274a870c0ff7801e61807ff78cfe398f8" +} diff --git a/core/lib/dal/src/pruning_dal/mod.rs b/core/lib/dal/src/pruning_dal/mod.rs index 9a5356202ae..702a301e743 100644 --- a/core/lib/dal/src/pruning_dal/mod.rs +++ b/core/lib/dal/src/pruning_dal/mod.rs @@ -318,29 +318,51 @@ impl PruningDal<'_, '_> { &mut self, l2_blocks_to_prune: ops::RangeInclusive, ) -> DalResult { - let execution_result = sqlx::query!( - r#" - DELETE FROM storage_logs - WHERE - storage_logs.miniblock_number < $1 - AND hashed_key IN ( - SELECT - hashed_key - FROM - storage_logs - WHERE - miniblock_number BETWEEN $1 AND $2 - ) - "#, - i64::from(l2_blocks_to_prune.start().0), - i64::from(l2_blocks_to_prune.end().0) - ) - .instrument("hard_prune_batches_range#prune_storage_logs_from_past_l2_blocks") - .with_arg("l2_blocks_to_prune", &l2_blocks_to_prune) - .report_latency() - .execute(self.storage) - .await?; - Ok(execution_result.rows_affected()) + /// Number of past logs to delete in a single query run. + const BATCHING_LIMIT: i64 = 10_000; + + let mut total_rows_affected = 0; + loop { + let execution_result = sqlx::query!( + r#" + DELETE FROM storage_logs + WHERE + ctid IN ( + SELECT + prev_logs.ctid + FROM + storage_logs AS prev_logs + INNER JOIN LATERAL ( + SELECT + 1 + FROM + storage_logs AS current_logs + WHERE + current_logs.miniblock_number BETWEEN $1 AND $2 + AND current_logs.hashed_key = prev_logs.hashed_key + ) AS current_logs ON TRUE + WHERE + prev_logs.miniblock_number < $1 + LIMIT + $3 + ) + "#, + i64::from(l2_blocks_to_prune.start().0), + i64::from(l2_blocks_to_prune.end().0), + BATCHING_LIMIT + ) + .instrument("hard_prune_batches_range#prune_storage_logs_from_past_l2_blocks") + .with_arg("l2_blocks_to_prune", &l2_blocks_to_prune) + .report_latency() + .execute(self.storage) + .await?; + + if execution_result.rows_affected() > 0 { + total_rows_affected += execution_result.rows_affected(); + } else { + return Ok(total_rows_affected); + } + } } async fn prune_storage_logs_in_range( diff --git a/core/lib/dal/src/pruning_dal/tests.rs b/core/lib/dal/src/pruning_dal/tests.rs index 7583065a8ec..4b2c6befcfa 100644 --- a/core/lib/dal/src/pruning_dal/tests.rs +++ b/core/lib/dal/src/pruning_dal/tests.rs @@ -361,7 +361,7 @@ async fn storage_logs_pruning_works_correctly() { ) .await; - transaction + let stats = transaction .pruning_dal() .hard_prune_batches_range(L1BatchNumber(4), L2BlockNumber(9)) .await @@ -377,8 +377,10 @@ async fn storage_logs_pruning_works_correctly() { &[random_storage_log(2, 3), random_storage_log(3, 4)], ); assert_l2_block_storage_logs_equal(L2BlockNumber(1), &actual_logs, &[random_storage_log(1, 1)]); + assert_eq!(stats.deleted_storage_logs_from_past_batches, 0); + assert_eq!(stats.deleted_storage_logs_from_pruned_batches, 1); - transaction + let stats = transaction .pruning_dal() .hard_prune_batches_range(L1BatchNumber(10), L2BlockNumber(21)) .await @@ -400,6 +402,8 @@ async fn storage_logs_pruning_works_correctly() { &actual_logs, &[random_storage_log(5, 7)], ); + assert_eq!(stats.deleted_storage_logs_from_past_batches, 1); + assert_eq!(stats.deleted_storage_logs_from_pruned_batches, 1); } #[tokio::test]