experiment: improve migration performance

Author: gfyrag

internal/storage/bucket/migrations/17-moves-fill-transaction-id/up.sql

@@ -5,40 +5,38 @@ do $$
 	begin
 		set search_path = '{{.Schema}}';
 
-		select count(seq)
+		-- todo: debug
+		--create temporary table transactions_ids as
+		create table transactions_ids as
+		select row_number() over (order by transactions.seq) as row_number,
+		       moves.seq as moves_seq, transactions.id, transactions.seq as transactions_seq
 		from moves
-		where transactions_id is null
+		join transactions on transactions.seq = moves.transactions_seq
+		where transactions_id is null;
+
+		create index transactions_ids_rows on transactions_ids(row_number) include (moves_seq, transactions_seq, id);
+
+		select count(*)
+		from transactions_ids
 		into _max;
 
 		perform pg_notify('migrations-{{ .Schema }}', 'init: ' || _max);
-		loop
 
-			with _outdated_moves as (
+		for i in 0.._max-1 by _batch_size loop
+			with _rows as (
 				select *
-				from moves
-				where transactions_id is null
-				limit _batch_size
+				from transactions_ids
+				where row_number >= i and row_number < i + _batch_size
 			)
 			update moves
-			set transactions_id = (
-				select id
-				from transactions
-				where seq = moves.transactions_seq
-			)
-			from _outdated_moves
-			where moves.seq in (_outdated_moves.seq);
+			set transactions_id = _rows.id
+			from _rows
+			where seq = _rows.moves_seq;
 
-			exit when not found;
+			commit;
 
 			perform pg_notify('migrations-{{ .Schema }}', 'continue: ' || _batch_size);
-
-			commit ;
 		end loop;
-
-		alter table moves
-		add constraint transactions_id_not_null
-		check (transactions_id is not null)
-		not valid;
 	end
 $$
 language plpgsql;

internal/storage/bucket/migrations/18-transactions-fill-inserted-at/up.sql

@@ -3,7 +3,6 @@ do $$
 		_ledger record;
 		_vsql text;
 		_batch_size integer := 1000;
-		_date timestamp without time zone;
 		_count integer := 0;
 	begin
 		set search_path = '{{.Schema}}';
@@ -19,17 +18,12 @@ do $$
 			execute _vsql;
 		end loop;
 
-		-- select the date where the "11-make-stateless" migration has been applied
-		select tstamp into _date
-		from goose_db_version
-		where version_id = 12;
-
 		create temporary table logs_transactions as
-		select id, ledger, date, (data->'transaction'->>'id')::bigint as transaction_id
+		select row_number() over (order by ledger, id) as row_number, ledger, date, (data->'transaction'->>'id')::bigint as transaction_id
 		from logs
-		where date <= _date;
+		where type = 'NEW_TRANSACTION' or type = 'REVERTED_TRANSACTION';
 
-		create index on logs_transactions (ledger, transaction_id) include (id, date);
+		create index on logs_transactions (row_number) include (ledger, date, transaction_id);
 
 		select count(*) into _count
 		from logs_transactions;
@@ -40,14 +34,12 @@ do $$
 			with _rows as (
 				select *
 				from logs_transactions
-				order by ledger, transaction_id
-				offset i
-				limit _batch_size
+				where row_number >= i and row_number < i + _batch_size
 			)
 			update transactions
 			set inserted_at = _rows.date
 			from _rows
-			where transactions.ledger = _rows.ledger and transactions.id = _rows.transaction_id;
+			where transactions.ledger = _rows.ledger and transactions.id = _rows.transaction_id and inserted_at is null;
 
 			commit;