Skip to content

Commit

Permalink
fix: disable layered memtable in overwrite mode (#1533)
Browse files Browse the repository at this point in the history
## Rationale
Layered memtable is only designed for append mode table now, and it
shouldn't be used in overwrite mode table.

## Detailed Changes
- Make default values in config used.
- Add `enable` field to control layered memtable's on/off.
- Add check to prevent invalid options during table create/alter.
- Add related it cases.

## Test Plan
Test manually.

Following cases are considered:

Check and intercept the invalid table options during table create/alter
- enable layered memtable but mutable switch threshold is 0 
- enable layered memtable for overwrite mode table

Table options new field `layered_enable`'s default value when it is not
found in pb
- false, when whole `layered_memtable_options` not exist
- false, when `layered_memtable_options` exist, and
`mutable_segment_switch_threshold` == 0
- true, when `layered_memtable_options` exist, and
`mutable_segment_switch_threshold` > 0
  • Loading branch information
Rachelint authored Sep 5, 2024
1 parent 039af9f commit 2c9cd1a
Show file tree
Hide file tree
Showing 22 changed files with 294 additions and 100 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ thiserror = "1"
bytes_ext = { path = "src/components/bytes_ext" }
catalog = { path = "src/catalog" }
catalog_impls = { path = "src/catalog_impls" }
horaedbproto = { git = "https://github.com/apache/incubator-horaedb-proto.git", rev = "19ece8f771fc0b3e8e734072cc3d8040de6c74cb" }
horaedbproto = { git = "https://github.com/apache/incubator-horaedb-proto.git", rev = "a5874d9fedee32ab1292252c4eb6defc4f6e245a" }
codec = { path = "src/components/codec" }
chrono = "0.4"
clap = { version = "4.5.1", features = ["derive"] }
Expand Down
4 changes: 2 additions & 2 deletions integration_tests/cases/common/dml/case_sensitive.result
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ Failed to execute query, err: Server(ServerError { code: 500, msg: "Failed to cr
SHOW CREATE TABLE case_SENSITIVE_table1;

Table,Create Table,
String("case_SENSITIVE_table1"),String("CREATE TABLE `case_SENSITIVE_table1` (`tsid` uint64 NOT NULL, `ts` timestamp NOT NULL, `VALUE1` double, PRIMARY KEY(tsid,ts), TIMESTAMP KEY(ts)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='false', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),
String("case_SENSITIVE_table1"),String("CREATE TABLE `case_SENSITIVE_table1` (`tsid` uint64 NOT NULL, `ts` timestamp NOT NULL, `VALUE1` double, PRIMARY KEY(tsid,ts), TIMESTAMP KEY(ts)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='false', layered_enable='false', layered_mutable_switch_threshold='3145728', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),


SHOW CREATE TABLE CASE_SENSITIVE_TABLE1;
Expand All @@ -84,7 +84,7 @@ Failed to execute query, err: Server(ServerError { code: 500, msg: "Failed to cr
SHOW CREATE TABLE `case_SENSITIVE_table1`;

Table,Create Table,
String("case_SENSITIVE_table1"),String("CREATE TABLE `case_SENSITIVE_table1` (`tsid` uint64 NOT NULL, `ts` timestamp NOT NULL, `VALUE1` double, PRIMARY KEY(tsid,ts), TIMESTAMP KEY(ts)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='false', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),
String("case_SENSITIVE_table1"),String("CREATE TABLE `case_SENSITIVE_table1` (`tsid` uint64 NOT NULL, `ts` timestamp NOT NULL, `VALUE1` double, PRIMARY KEY(tsid,ts), TIMESTAMP KEY(ts)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='false', layered_enable='false', layered_mutable_switch_threshold='3145728', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),


SHOW CREATE TABLE `CASE_SENSITIVE_TABLE1`;
Expand Down
6 changes: 3 additions & 3 deletions integration_tests/cases/common/show/show_create_table.result
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ affected_rows: 0
SHOW CREATE TABLE `06_show_a`;

Table,Create Table,
String("06_show_a"),String("CREATE TABLE `06_show_a` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` bigint, `b` int DEFAULT 3, `c` string DEFAULT 'x', `d` smallint, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),
String("06_show_a"),String("CREATE TABLE `06_show_a` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` bigint, `b` int DEFAULT 3, `c` string DEFAULT 'x', `d` smallint, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', layered_enable='false', layered_mutable_switch_threshold='3145728', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),


CREATE TABLE `06_show_b` (a bigint, b int null default null, c string, d smallint null, t timestamp NOT NULL, TIMESTAMP KEY(t)) ENGINE = Analytic;
Expand All @@ -45,7 +45,7 @@ affected_rows: 0
SHOW CREATE TABLE `06_show_b`;

Table,Create Table,
String("06_show_b"),String("CREATE TABLE `06_show_b` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` bigint, `b` int DEFAULT NULL, `c` string, `d` smallint, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),
String("06_show_b"),String("CREATE TABLE `06_show_b` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` bigint, `b` int DEFAULT NULL, `c` string, `d` smallint, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', layered_enable='false', layered_mutable_switch_threshold='3145728', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),


CREATE TABLE `06_show_c` (a int, t timestamp NOT NULL, TIMESTAMP KEY(t)) ENGINE = Analytic;
Expand All @@ -55,7 +55,7 @@ affected_rows: 0
SHOW CREATE TABLE `06_show_c`;

Table,Create Table,
String("06_show_c"),String("CREATE TABLE `06_show_c` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` int, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),
String("06_show_c"),String("CREATE TABLE `06_show_c` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` int, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', layered_enable='false', layered_mutable_switch_threshold='3145728', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='33554432')"),


DROP TABLE `06_show_a`;
Expand Down
4 changes: 2 additions & 2 deletions integration_tests/cases/env/cluster/ddl/alter_table.result
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ affected_rows: 0
show create table 05_alter_table_t1;

Table,Create Table,
String("05_alter_table_t1"),String("CREATE TABLE `05_alter_table_t1` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` int, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='314572800')"),
String("05_alter_table_t1"),String("CREATE TABLE `05_alter_table_t1` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `a` int, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', layered_enable='false', layered_mutable_switch_threshold='3145728', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='7d', update_mode='OVERWRITE', write_buffer_size='314572800')"),


drop table 05_alter_table_t1;
Expand All @@ -120,7 +120,7 @@ affected_rows: 0
show create table 05_alter_table_t1;

Table,Create Table,
String("05_alter_table_t1"),String("CREATE TABLE `05_alter_table_t1` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `sid` uint64 NOT NULL, `a` int, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='10d', update_mode='OVERWRITE', write_buffer_size='314572800')"),
String("05_alter_table_t1"),String("CREATE TABLE `05_alter_table_t1` (`tsid` uint64 NOT NULL, `t` timestamp NOT NULL, `sid` uint64 NOT NULL, `a` int, PRIMARY KEY(tsid,t), TIMESTAMP KEY(t)) ENGINE=Analytic WITH(arena_block_size='2097152', compaction_strategy='default', compression='ZSTD', enable_ttl='true', layered_enable='false', layered_mutable_switch_threshold='3145728', memtable_type='skiplist', num_rows_per_row_group='8192', segment_duration='', storage_format='AUTO', ttl='10d', update_mode='OVERWRITE', write_buffer_size='314572800')"),


drop table 05_alter_table_t1;
Expand Down
Loading

0 comments on commit 2c9cd1a

Please sign in to comment.