Skip to content

PS-10067 [8.0]: Consume per-field metadata with Rdb_convert_to_record_key_decoder::skip #5707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 25, 2025
Merged

PS-10067 [8.0]: Consume per-field metadata with Rdb_convert_to_record_key_decoder::skip #5707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4f84234
PS-10075 [8.0]: RocksDB: Checksum mismatch in key of key-value pair f…
inikep Sep 18, 2025
89d7b97
PS-10067 [8.0]: Consume per-field metadata with `Rdb_convert_to_recor…
inikep Sep 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions mysql-test/suite/rocksdb/r/percona_bugs.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,54 @@ PRIMARY KEY (`id`)
SET unique_checks=OFF;
INSERT INTO t1 (`id`, `ttl_ts`) VALUES ('helloworld', 1738737225);
DROP TABLE t1;
#
# PS-10075: RocksDB: Checksum mismatch in key of key-value pair for index
# https://perconadev.atlassian.net/browse/PS-10075
#
CREATE TABLE t1 (lt LONGTEXT, INDEX idx(lt(8)) ) ENGINE=RocksDB;
INSERT INTO t1 (lt) VALUES (
'START-1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890..-END'
);
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT lt, LENGTH(lt) FROM t1;
lt LENGTH(lt)
START-1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890..-END 257
DROP TABLE t1;
#
# PS-10067: Consume per-field metadata with `Rdb_convert_to_record_key_decoder::skip`
# https://perconadev.atlassian.net/browse/PS-10067
#
# Test Rdb_key_def::make_unpack_unknown_varlength for MYSQL_TYPE_VARCHAR and MYSQL_TYPE_VARCHAR
# (uses Rdb_key_def::skip_variable_length_encoding)
CREATE TABLE t1 (v7 VARCHAR(7), v32 VARCHAR(32), INDEX i(v32(5), v7) ) ENGINE=RocksDB;
INSERT INTO t1 (v7, v32) VALUES ('percona', '1234567890.1234567890.1234567890');
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
SELECT * FROM t1 WHERE v32 LIKE '1234567890%';
v7 v32
percona 1234567890.1234567890.1234567890
DROP TABLE t1;
# Test Rdb_key_def::make_unpack_unknown_varlength for MYSQL_TYPE_VARCHAR and MYSQL_TYPE_BLOB
# (uses Rdb_key_def::skip_variable_length_encoding)
CREATE TABLE t2 (v7 VARCHAR(7), v32 TEXT, INDEX i(v32(11), v7) ) COLLATE=utf8mb4_0900_ai_ci ENGINE=RocksDB;
INSERT INTO t2 (v7, v32) VALUES ('percona', '1234567890.1234567890.1234567890');
CHECK TABLE t2;
Table Op Msg_type Msg_text
test.t2 check status OK
SELECT * FROM t2 WHERE v32 LIKE '1234567890%';
v7 v32
percona 1234567890.1234567890.1234567890
DROP TABLE t2;
# Test Rdb_key_def::skip_variable_space_pad
CREATE TABLE t3 (v7 VARCHAR(7), v32 TEXT, INDEX i(v32(11), v7) ) COLLATE=utf8mb4_bin ENGINE=RocksDB;
INSERT INTO t3 (v7, v32) VALUES ('percona', '1234567890.1234567890.1234567890');
CHECK TABLE t3;
Table Op Msg_type Msg_text
test.t3 check status OK
SELECT * FROM t3 WHERE v32 LIKE '1234567890%';
v7 v32
percona 1234567890.1234567890.1234567890
DROP TABLE t3;
50 changes: 49 additions & 1 deletion mysql-test/suite/rocksdb/t/percona_bugs.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
--source include/have_debug.inc
--source include/have_rocksdb.inc

--echo #
Expand Down Expand Up @@ -71,3 +70,52 @@ SET unique_checks=OFF;
INSERT INTO t1 (`id`, `ttl_ts`) VALUES ('helloworld', 1738737225);

DROP TABLE t1;



--echo #
--echo # PS-10075: RocksDB: Checksum mismatch in key of key-value pair for index
--echo # https://perconadev.atlassian.net/browse/PS-10075
--echo #

CREATE TABLE t1 (lt LONGTEXT, INDEX idx(lt(8)) ) ENGINE=RocksDB;

# string has 257 bytes (little endian 4-byte representation of 257 is 01 01 00 00)
# what causes conflict with RDB_CHECKSUM_DATA_TAG = 0x01
INSERT INTO t1 (lt) VALUES (
'START-1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890.1234567890123456789012345678901234567890..-END'
);

CHECK TABLE t1;
SELECT lt, LENGTH(lt) FROM t1;
DROP TABLE t1;



--echo #
--echo # PS-10067: Consume per-field metadata with `Rdb_convert_to_record_key_decoder::skip`
--echo # https://perconadev.atlassian.net/browse/PS-10067
--echo #

--echo # Test Rdb_key_def::make_unpack_unknown_varlength for MYSQL_TYPE_VARCHAR and MYSQL_TYPE_VARCHAR
--echo # (uses Rdb_key_def::skip_variable_length_encoding)
CREATE TABLE t1 (v7 VARCHAR(7), v32 VARCHAR(32), INDEX i(v32(5), v7) ) ENGINE=RocksDB;
INSERT INTO t1 (v7, v32) VALUES ('percona', '1234567890.1234567890.1234567890');
CHECK TABLE t1;
SELECT * FROM t1 WHERE v32 LIKE '1234567890%';
DROP TABLE t1;

--echo # Test Rdb_key_def::make_unpack_unknown_varlength for MYSQL_TYPE_VARCHAR and MYSQL_TYPE_BLOB
--echo # (uses Rdb_key_def::skip_variable_length_encoding)
CREATE TABLE t2 (v7 VARCHAR(7), v32 TEXT, INDEX i(v32(11), v7) ) COLLATE=utf8mb4_0900_ai_ci ENGINE=RocksDB;
INSERT INTO t2 (v7, v32) VALUES ('percona', '1234567890.1234567890.1234567890');
CHECK TABLE t2;
SELECT * FROM t2 WHERE v32 LIKE '1234567890%';
DROP TABLE t2;

--echo # Test Rdb_key_def::skip_variable_space_pad
CREATE TABLE t3 (v7 VARCHAR(7), v32 TEXT, INDEX i(v32(11), v7) ) COLLATE=utf8mb4_bin ENGINE=RocksDB;
INSERT INTO t3 (v7, v32) VALUES ('percona', '1234567890.1234567890.1234567890');
CHECK TABLE t3;
SELECT * FROM t3 WHERE v32 LIKE '1234567890%';
DROP TABLE t3;
26 changes: 22 additions & 4 deletions storage/rocksdb/rdb_datadic.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,8 @@ int Rdb_convert_to_record_key_decoder::decode(
*/
int Rdb_convert_to_record_key_decoder::skip(
const Rdb_field_packing *fpi, const Field *field MY_ATTRIBUTE((__unused__)),
Rdb_string_reader *reader, Rdb_string_reader *unp_reader,
bool covered_bitmap_format_enabled) {
bool has_unpack_info, Rdb_string_reader *reader,
Rdb_string_reader *unp_reader, bool covered_bitmap_format_enabled) {
/* It is impossible to unpack the column. Skip it. */
if (fpi->m_field_is_nullable) {
const char *nullp;
Expand All @@ -180,6 +180,23 @@ int Rdb_convert_to_record_key_decoder::skip(
return HA_ERR_ROCKSDB_CORRUPT_DATA;
}

const uint len_bytes = fpi->m_varlength_bytes;
if (fpi->m_unpack_func == &Rdb_key_def::unpack_unknown_varlength &&
unp_reader && has_unpack_info && len_bytes > 0) {
const uchar *ptr = (uchar *)unp_reader->read(len_bytes);
if (!ptr) return HA_ERR_ROCKSDB_CORRUPT_DATA;

uint len = 0;
if (fpi->m_field_real_type == MYSQL_TYPE_VARCHAR) {
len = len_bytes == 1 ? (uint)*ptr : uint2korr(ptr);
} else {
len = Field_blob::get_length(ptr, fpi->m_varlength_bytes);
}

ptr = (const uchar *)unp_reader->read(len);
if (!ptr) return HA_ERR_ROCKSDB_CORRUPT_DATA;
}

// If this is a space padded varchar, we need to skip the indicator
// bytes for trailing bytes. They're useless since we can't restore the
// field anyway.
Expand Down Expand Up @@ -257,7 +274,7 @@ int Rdb_key_field_iterator::next() {
} else {
auto field = fpi->get_field_in_table(m_table);
status = Rdb_convert_to_record_key_decoder::skip(
fpi, field, m_reader, m_unp_reader,
fpi, field, m_has_unpack_info, m_reader, m_unp_reader,
this->m_key_def->use_covered_bitmap_format());
if (status) {
return status;
Expand Down Expand Up @@ -2216,7 +2233,8 @@ int Rdb_key_def::unpack_record(TABLE *const table, uchar *const buf,
Check checksum values if present
*/
const char *ptr;
if (unlikely((ptr = unp_reader.read(1)) && *ptr == RDB_CHECKSUM_DATA_TAG)) {
if (unlikely(unp_reader.remaining_bytes() == RDB_CHECKSUM_CHUNK_SIZE &&
(ptr = unp_reader.read(1)) && *ptr == RDB_CHECKSUM_DATA_TAG)) {
if (verify_row_debug_checksums) {
uint32_t stored_key_chksum = rdb_netbuf_to_uint32(
(const uchar *)unp_reader.read(RDB_CHECKSUM_SIZE));
Expand Down
3 changes: 2 additions & 1 deletion storage/rocksdb/rdb_datadic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ class Rdb_convert_to_record_key_decoder {
bool has_unpack_info, Rdb_string_reader *reader,
Rdb_string_reader *unpack_reader);
static int skip(const Rdb_field_packing *fpi, const Field *field,
Rdb_string_reader *reader, Rdb_string_reader *unpack_reader,
bool has_unpack_info, Rdb_string_reader *reader,
Rdb_string_reader *unpack_reader,
bool covered_bitmap_format_enabled);

private:
Expand Down
Loading