Skip to content

Commit

Permalink
PS-8836 fix: MyRocks serialization violation? Read after write is fai…
Browse files Browse the repository at this point in the history 
…ling (#5120)

https://jira.percona.com/browse/PS-8836

This is a cherry-pick of the
"Fix for snapshot conflict on SK due to concurrent update"
(commit facebook/mysql-5.6@98da68c)

Summary: The issue that was being hit has two different connections to the MySQL server trying to update two different rows via a secondary index scan with RC isolation level set. During the SK index scan, either one of the iterators may end up returning the value updated by the other connection. This leads to the iterator range condition failing, and the iterator returns an early EOF back, and the update fails to apply on the remaining rows that were actually valid candidates for update based on the WHERE conditions.

Differential Revision: D48740767

fbshipit-source-id: f9b58edc249b6deace936dbafc70f79456eb49ca

Co-authored-by: Saumitr Pathak <saumitr@meta.com>
  • Loading branch information
@percona-ysorokin @saumitrp
percona-ysorokin and saumitrp authored Sep 11, 2023
1 parent 3abdf9b commit ce5983c
Show file tree
Hide file tree
Showing 10 changed files with 610 additions and 16 deletions.
6 changes: 3 additions & 3 deletions mysql-test/suite/rocksdb/include/rocksdb_concurrent_delete_sk.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ INSERT INTO t1 VALUES(1,1), (2,2), (3,3), (4,4), (5,5);
--echo --SK first row delete
connection con;
eval SET SESSION TRANSACTION ISOLATION LEVEL $isolation_level;
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
send_eval SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;

# While that connection is waiting, delete the first row (the one con
Expand All @@ -34,7 +34,7 @@ reap;

# Deleting a middle row
--echo --SK middle row delete
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
send_eval SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;

connection default;
Expand All @@ -55,7 +55,7 @@ if ($isolation_level == "READ COMMITTED")

# Deleting the end row
--echo --SK end row delete
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
send_eval SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;

connection default;
Expand Down
12 changes: 6 additions & 6 deletions mysql-test/suite/rocksdb/r/rocksdb_concurrent_delete.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, a INT, index a(a)) ENGINE=RocksDB;
INSERT INTO t1 VALUES(1,1), (2,2), (3,3), (4,4), (5,5);
--SK first row delete
SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;
SET debug_sync='now WAIT_FOR parked';
DELETE FROM t1 WHERE pk = 1;
Expand All @@ -47,14 +47,14 @@ a
4
5
--SK middle row delete
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;
SET debug_sync='now WAIT_FOR parked';
DELETE FROM t1 WHERE pk = 3;
SET debug_sync='now SIGNAL go';
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
--SK end row delete
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;
SET debug_sync='now WAIT_FOR parked';
DELETE FROM t1 WHERE pk = 5;
Expand Down Expand Up @@ -289,7 +289,7 @@ CREATE TABLE t1 (pk INT PRIMARY KEY, a INT, index a(a)) ENGINE=RocksDB;
INSERT INTO t1 VALUES(1,1), (2,2), (3,3), (4,4), (5,5);
--SK first row delete
SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;
SET debug_sync='now WAIT_FOR parked';
DELETE FROM t1 WHERE pk = 1;
Expand All @@ -300,7 +300,7 @@ a
4
5
--SK middle row delete
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;
SET debug_sync='now WAIT_FOR parked';
DELETE FROM t1 WHERE pk = 3;
Expand All @@ -310,7 +310,7 @@ a
4
5
--SK end row delete
SET debug_sync='rocksdb_concurrent_delete_sk SIGNAL parked WAIT_FOR go';
SET debug_sync='rocksdb_concurrent_upd_or_delete_sk SIGNAL parked WAIT_FOR go';
SELECT a FROM t1 FORCE INDEX(a) FOR UPDATE;
SET debug_sync='now WAIT_FOR parked';
DELETE FROM t1 WHERE pk = 5;
Expand Down
106 changes: 106 additions & 0 deletions mysql-test/suite/rocksdb/r/sk_rc_concurrent_point_update.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
Conn A creating table
CREATE TABLE table1 (
row_key BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
val1 TINYINT NOT NULL,
val2 VARCHAR(128) NOT NULL,
PRIMARY KEY (row_key),
KEY idx_val1 (val1)
) ENGINE=RocksDB;
INSERT INTO table1 (val1, val2) VALUES (14, 'Alfa'), (14, 'Bravo'), (14, 'Charlie'), (14, 'Delta');
Conn A: `table1` created with 4 rows
Conn A: Table before
SELECT * FROM table1;
row_key val1 val2
1 14 Alfa
2 14 Bravo
3 14 Charlie
4 14 Delta
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn A: Started TRANSACTION A (SELECT .. FOR UPDATE )
set DEBUG_SYNC = "rocksdb_concurrent_upd_or_delete_sk SIGNAL waiting_for_update WAIT_FOR update_done";
Conn A: activate DEBUG_SYNC point rocksdb_concurrent_upd_or_delete_sk
SELECT * from table1 FORCE INDEX(idx_val1) WHERE val1 = 14 AND val2 = 'Alfa' FOR UPDATE;
Conn A: Sent SELECT
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn B: Started TRANSACTION B (Concurrent update)
Conn B: Waiting for Conn A to hit `waiting_for_update`
set DEBUG_SYNC = "now WAIT_FOR waiting_for_update";
Conn B: Conn A triggered `waiting_for_update`
UPDATE table1 SET val1 = 15 WHERE val1 = 14 AND val2 = 'Alfa';
SELECT * FROM table1;
row_key val1 val2
1 15 Alfa
2 14 Bravo
3 14 Charlie
4 14 Delta
COMMIT;
Conn B: COMMIT for update done
set DEBUG_SYNC = "now SIGNAL update_done";
Conn B: signalled Conn A with event `update_done`
Conn A: reaping SELECT * from table1 FORCE INDEX(idx_val1) WHERE val1 = 14 AND val2 = 'Alfa';
The SELECT output should be empty
row_key val1 val2
ROLLBACK;
Conn A: Table after
SELECT * FROM table1;
row_key val1 val2
1 15 Alfa
2 14 Bravo
3 14 Charlie
4 14 Delta
DROP TABLE table1;
Conn A creating table
CREATE TABLE table1 (

val1 TINYINT NOT NULL,
val2 VARCHAR(128) NOT NULL,

KEY idx_val1 (val1)
) ENGINE=RocksDB;
INSERT INTO table1 (val1, val2) VALUES (14, 'Alfa'), (14, 'Bravo'), (14, 'Charlie'), (14, 'Delta');
Conn A: `table1` created with 4 rows
Conn A: Table before
SELECT * FROM table1;
val1 val2
14 Alfa
14 Bravo
14 Charlie
14 Delta
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn A: Started TRANSACTION A (SELECT .. FOR UPDATE )
set DEBUG_SYNC = "rocksdb_concurrent_upd_or_delete_sk SIGNAL waiting_for_update WAIT_FOR update_done";
Conn A: activate DEBUG_SYNC point rocksdb_concurrent_upd_or_delete_sk
SELECT * from table1 FORCE INDEX(idx_val1) WHERE val1 = 14 AND val2 = 'Alfa' FOR UPDATE;
Conn A: Sent SELECT
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn B: Started TRANSACTION B (Concurrent update)
Conn B: Waiting for Conn A to hit `waiting_for_update`
set DEBUG_SYNC = "now WAIT_FOR waiting_for_update";
Conn B: Conn A triggered `waiting_for_update`
UPDATE table1 SET val1 = 15 WHERE val1 = 14 AND val2 = 'Alfa';
SELECT * FROM table1;
val1 val2
15 Alfa
14 Bravo
14 Charlie
14 Delta
COMMIT;
Conn B: COMMIT for update done
set DEBUG_SYNC = "now SIGNAL update_done";
Conn B: signalled Conn A with event `update_done`
Conn A: reaping SELECT * from table1 FORCE INDEX(idx_val1) WHERE val1 = 14 AND val2 = 'Alfa';
The SELECT output should be empty
val1 val2
ROLLBACK;
Conn A: Table after
SELECT * FROM table1;
val1 val2
15 Alfa
14 Bravo
14 Charlie
14 Delta
DROP TABLE table1;
88 changes: 88 additions & 0 deletions mysql-test/suite/rocksdb/r/sk_rc_concurrent_update.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
Conn A creating table
CREATE TABLE table1 (
row_key BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
val1 TINYINT NOT NULL,
val2 VARCHAR(128) NOT NULL,
PRIMARY KEY (row_key),
KEY idx_val1 (val1)
) ENGINE=RocksDB;
INSERT INTO table1 (val1, val2) VALUES (14, 'Alfa'), (14, 'Bravo'), (14, 'Charlie'), (14, 'Delta');
Conn A: `table1` created with 4 rows
Conn A: Table before
SELECT * FROM table1;
row_key val1 val2
1 14 Alfa
2 14 Bravo
3 14 Charlie
4 14 Delta
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn A: Started TRANSACTION A (Update )
set DEBUG_SYNC = "rocksdb_concurrent_upd_or_delete_sk SIGNAL waiting_for_update WAIT_FOR update_done";
Conn A: activate DEBUG_SYNC point rocksdb_concurrent_upd_or_delete_sk
UPDATE table1 SET val1 = 15 WHERE val1 = 14 AND val2 = 'Bravo';
Conn A: Sent UPDATE
Conn B: Start transaction B
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn B: Started TRANSACTION B (Update)
Conn B: Waiting for Conn A to hit `waiting_for_update`
set DEBUG_SYNC = "now WAIT_FOR waiting_for_update";
Conn B: Conn A triggered `waiting_for_update`
UPDATE table1 SET val1 = 15 WHERE val1 = 14 AND val2 = 'Alfa';
COMMIT;
Conn B: COMMIT for update done
set DEBUG_SYNC = "now SIGNAL update_done";
Conn B: signalled Conn A with event `update_done`
Conn A: Table after
SELECT * FROM table1;
row_key val1 val2
1 15 Alfa
2 15 Bravo
3 14 Charlie
4 14 Delta
DROP TABLE table1;
Conn A creating table
CREATE TABLE table1 (

val1 TINYINT NOT NULL,
val2 VARCHAR(128) NOT NULL,

KEY idx_val1 (val1)
) ENGINE=RocksDB;
INSERT INTO table1 (val1, val2) VALUES (14, 'Alfa'), (14, 'Bravo'), (14, 'Charlie'), (14, 'Delta');
Conn A: `table1` created with 4 rows
Conn A: Table before
SELECT * FROM table1;
val1 val2
14 Alfa
14 Bravo
14 Charlie
14 Delta
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn A: Started TRANSACTION A (Update )
set DEBUG_SYNC = "rocksdb_concurrent_upd_or_delete_sk SIGNAL waiting_for_update WAIT_FOR update_done";
Conn A: activate DEBUG_SYNC point rocksdb_concurrent_upd_or_delete_sk
UPDATE table1 SET val1 = 15 WHERE val1 = 14 AND val2 = 'Bravo';
Conn A: Sent UPDATE
Conn B: Start transaction B
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn B: Started TRANSACTION B (Update)
Conn B: Waiting for Conn A to hit `waiting_for_update`
set DEBUG_SYNC = "now WAIT_FOR waiting_for_update";
Conn B: Conn A triggered `waiting_for_update`
UPDATE table1 SET val1 = 15 WHERE val1 = 14 AND val2 = 'Alfa';
COMMIT;
Conn B: COMMIT for update done
set DEBUG_SYNC = "now SIGNAL update_done";
Conn B: signalled Conn A with event `update_done`
Conn A: Table after
SELECT * FROM table1;
val1 val2
15 Alfa
15 Bravo
14 Charlie
14 Delta
DROP TABLE table1;
86 changes: 86 additions & 0 deletions mysql-test/suite/rocksdb/r/sk_rc_read_set.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
Creating TABLE `table1`
CREATE TABLE table1 (
row_key BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
val1 TINYINT NOT NULL,
val2 VARCHAR(128) NOT NULL,
PRIMARY KEY (row_key),
KEY idx_val1 (val1, val2(1))
) ENGINE=RocksDB;
INSERT INTO table1 (val1, val2) VALUES (14, 'Alfa'), (14, 'Bravo'), (14, 'Charlie'), (14, 'Delta');
`table1` created with 4 rows
Table before
SELECT * FROM table1;
row_key val1 val2
1 14 Alfa
2 14 Bravo
3 14 Charlie
4 14 Delta
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn A: Started RC TRANSACTION
Conn A: SELECT with val1 referenced - NO LOCKS
SELECT val1 from table1 WHERE val1 = 14;
val1
14
14
14
14
Conn A: SELECT .. FOR SHARE with val1 referenced - READ LOCK
SELECT val1 from table1 WHERE val1 = 14 FOR SHARE;
val1
14
14
14
14
Conn A: SELECT .. FOR UPDATE with val1 referenced - WRITE LOCK
SELECT val1 from table1 WHERE val1 = 14 FOR UPDATE;
val1
14
14
14
14
ROLLBACK;
DROP TABLE table1;
Creating TABLE `table1`
CREATE TABLE table1 (

val1 TINYINT NOT NULL,
val2 VARCHAR(128) NOT NULL,

KEY idx_val1 (val1, val2(1))
) ENGINE=RocksDB;
INSERT INTO table1 (val1, val2) VALUES (14, 'Alfa'), (14, 'Bravo'), (14, 'Charlie'), (14, 'Delta');
`table1` created with 4 rows
Table before
SELECT * FROM table1;
val1 val2
14 Alfa
14 Bravo
14 Charlie
14 Delta
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;
START TRANSACTION;
Conn A: Started RC TRANSACTION
Conn A: SELECT with val1 referenced - NO LOCKS
SELECT val1 from table1 WHERE val1 = 14;
val1
14
14
14
14
Conn A: SELECT .. FOR SHARE with val1 referenced - READ LOCK
SELECT val1 from table1 WHERE val1 = 14 FOR SHARE;
val1
14
14
14
14
Conn A: SELECT .. FOR UPDATE with val1 referenced - WRITE LOCK
SELECT val1 from table1 WHERE val1 = 14 FOR UPDATE;
val1
14
14
14
14
ROLLBACK;
DROP TABLE table1;
Loading

0 comments on commit ce5983c

Please sign in to comment.