From 7fff925a31df93ab334b582dc70ac79aae1a4c35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=B5=B5=E7=A9=BA=E4=BA=8B=E3=82=B9=E3=83=94=E3=83=AA?=
 =?UTF-8?q?=E3=83=83=E3=83=88?= <wanglichen@starrocks.com>
Date: Wed, 15 Jan 2025 09:55:17 +0800
Subject: [PATCH 01/71] [Doc] Files() with Regex (#55078)

---
 .../sql-functions/table-functions/files.md           | 12 +++++++++++-
 .../sql-functions/table-functions/files.md           | 12 +++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/sql-functions/table-functions/files.md b/docs/en/sql-reference/sql-functions/table-functions/files.md
index dfc70a519602ee..f6926266d2f65c 100644
--- a/docs/en/sql-reference/sql-functions/table-functions/files.md
+++ b/docs/en/sql-reference/sql-functions/table-functions/files.md
@@ -43,7 +43,17 @@ All parameters are in the `"key" = "value"` pairs.
 
 #### data_location
 
-The URI used to access the files. You can specify a path or a file.
+The URI used to access the files.
+
+You can specify a path or a file. For example, you can specify this parameter as `"hdfs://<hdfs_host>:<hdfs_port>/user/data/tablename/20210411"` to load a data file named `20210411` from the path `/user/data/tablename` on the HDFS server.
+
+You can also specify this parameter as the save path of multiple data files by using wildcards `?`, `*`, `[]`, `{}`, or `^`. For example, you can specify this parameter as `"hdfs://<hdfs_host>:<hdfs_port>/user/data/tablename/*/*"` or `"hdfs://<hdfs_host>:<hdfs_port>/user/data/tablename/dt=202104*/*"` to load the data files from all partitions or only `202104` partitions in the path `/user/data/tablename` on the HDFS server.
+
+:::note
+
+Wildcards can also be used to specify intermediate paths.
+
+:::
 
 - To access HDFS, you need to specify this parameter as:
 
diff --git a/docs/zh/sql-reference/sql-functions/table-functions/files.md b/docs/zh/sql-reference/sql-functions/table-functions/files.md
index e0f032d8f32a65..e8e0ebec4601ba 100644
--- a/docs/zh/sql-reference/sql-functions/table-functions/files.md
+++ b/docs/zh/sql-reference/sql-functions/table-functions/files.md
@@ -42,7 +42,17 @@ FILES( data_location , [data_format] [, schema_detect ] [, StorageCredentialPara
 
 #### data_location
 
-用于访问文件的 URI。可以指定路径或文件名。
+用于访问文件的 URI。
+
+可以指定路径或文件名。例如，通过指定 `"hdfs://<hdfs_host>:<hdfs_port>/user/data/tablename/20210411"` 可以匹配 HDFS 服务器上 `/user/data/tablename` 目录下名为 `20210411` 的数据文件。
+
+您也可以用通配符指定导入某个路径下所有的数据文件。FILES 支持如下通配符：`?`、`*`、`[]`、`{}` 和 `^`。例如， 通过指定 `"hdfs://<hdfs_host>:<hdfs_port>/user/data/tablename/*/*"` 路径可以匹配 HDFS 服务器上 `/user/data/tablename` 目录下所有分区内的数据文件，通过 `"hdfs://<hdfs_host>:<hdfs_port>/user/data/tablename/dt=202104*/*"` 路径可以匹配 HDFS 服务器上 `/user/data/tablename` 目录下所有 `202104` 分区内的数据文件。
+
+:::note
+
+中间的目录也可以使用通配符匹配。
+
+:::
 
 - 要访问 HDFS，您需要将此参数指定为：
 

From 79a03f9067cd9e886168c9533e0ed5ff2b502a78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=B5=B5=E7=A9=BA=E4=BA=8B=E3=82=B9=E3=83=94=E3=83=AA?=
 =?UTF-8?q?=E3=83=83=E3=83=88?= <wanglichen@starrocks.com>
Date: Wed, 15 Jan 2025 10:33:28 +0800
Subject: [PATCH 02/71] [Doc] Remove incorrect description (#55062)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 絵空事スピリット <wanglichen@starrocks.com>
---
 docs/en/sql-reference/sql-functions/table-functions/files.md | 2 +-
 docs/zh/sql-reference/sql-functions/table-functions/files.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/sql-functions/table-functions/files.md b/docs/en/sql-reference/sql-functions/table-functions/files.md
index f6926266d2f65c..0bbda3326a7b38 100644
--- a/docs/en/sql-reference/sql-functions/table-functions/files.md
+++ b/docs/en/sql-reference/sql-functions/table-functions/files.md
@@ -220,7 +220,7 @@ The system unionizes the schema of Parquet and ORC files based on the column nam
 
 ##### Infer STRUCT type from Parquet
 
-From v3.4.0 onwards, FILES() supports inferring the STRUCT type data from Parquet files. Although Parquet file itself does not support the STRUCT type, the system can infer STRUCT and nested STRUCT values from the STRING type column of the file. 
+From v3.4.0 onwards, FILES() supports inferring the STRUCT type data from Parquet files.
 
 #### StorageCredentialParams
 
diff --git a/docs/zh/sql-reference/sql-functions/table-functions/files.md b/docs/zh/sql-reference/sql-functions/table-functions/files.md
index e8e0ebec4601ba..64c4711cc78997 100644
--- a/docs/zh/sql-reference/sql-functions/table-functions/files.md
+++ b/docs/zh/sql-reference/sql-functions/table-functions/files.md
@@ -219,7 +219,7 @@ FILES() 的 Schema 检测并不是完全严格的。例如，在读取 CSV 文
 
 ##### 推断 Parquet 文件中的 STRUCT 类型
 
-从 v3.4.0 版本开始，FILES() 支持从 Parquet 文件中推断 STRUCT 类型的数据。尽管 Parquet 文件本身不支持 STRUCT 类型，但系统可以从文件中的 STRING 类型列推断 STRUCT 及嵌套 STRUCT 值。
+从 v3.4.0 版本开始，FILES() 支持从 Parquet 文件中推断 STRUCT 类型的数据。
 
 #### StorageCredentialParams
 

From 6e3d945bb492c47e82d00fa8d701cd6e4dd2f510 Mon Sep 17 00:00:00 2001
From: Dan Roscigno <dan@roscigno.com>
Date: Tue, 14 Jan 2025 21:54:19 -0500
Subject: [PATCH 03/71] [Doc] use package.json from main (#55083)

Signed-off-by: DanRoscigno <dan@roscigno.com>
---
 .github/workflows/ci-doc-checker.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/ci-doc-checker.yml b/.github/workflows/ci-doc-checker.yml
index 1adaad950a1ee0..e03210602e8490 100644
--- a/.github/workflows/ci-doc-checker.yml
+++ b/.github/workflows/ci-doc-checker.yml
@@ -130,6 +130,11 @@ jobs:
           rm -rf ./docs/release_notes ./docs/ecosystem_release
           mv ../zh ./i18n/zh/docusaurus-plugin-content-docs/current
           rm -rf ./i18n/zh/docusaurus-plugin-content-docs/current/release_notes ./i18n/zh/docusaurus-plugin-content-docs/current/ecosystem_release
+          # Using package.json and yarn.lock from a PR is not safe, so copy from main branch.
+          rm package.json
+          rm yarn.lock
+          curl -O https://raw.githubusercontent.com/StarRocks/starrocks/refs/heads/main/docs/docusaurus/package.json
+          curl -O https://raw.githubusercontent.com/StarRocks/starrocks/refs/heads/main/docs/docusaurus/yarn.lock
           yarn install --frozen-lockfile
           yarn clear
           yarn build

From 2533f88f812a4384f0c56b291db2ec4898b79adf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=B5=B5=E7=A9=BA=E4=BA=8B=E3=82=B9=E3=83=94=E3=83=AA?=
 =?UTF-8?q?=E3=83=83=E3=83=88?= <wanglichen@starrocks.com>
Date: Wed, 15 Jan 2025 11:00:22 +0800
Subject: [PATCH 04/71] [Doc] Add links to release notes (#55050)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 絵空事スピリット <wanglichen@starrocks.com>
---
 docs/en/release_notes/release-3.4.md | 28 ++++++++++++++--------------
 docs/zh/release_notes/release-3.4.md | 28 ++++++++++++++--------------
 2 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/docs/en/release_notes/release-3.4.md b/docs/en/release_notes/release-3.4.md
index 4e1f6770ac4942..e69beb57883830 100644
--- a/docs/en/release_notes/release-3.4.md
+++ b/docs/en/release_notes/release-3.4.md
@@ -11,12 +11,12 @@ Release date: January 13, 2025
 ### Data Lake Analytics
 
 - Optimized Iceberg V2 query performance and lowered memory usage by reducing repeated reads of delete-files.
-- Supports column mapping for Delta Lake tables, allowing queries against data after Delta Schema Evolution.
+- Supports column mapping for Delta Lake tables, allowing queries against data after Delta Schema Evolution. For more information, see [Delta Lake catalog - Feature support](https://docs.starrocks.io/docs/data_source/catalog/deltalake_catalog/#feature-support).
 - Data Cache related improvements:
-  - Introduces a Segmented LRU (SLRU) Cache eviction strategy, which significantly defends against cache pollution from occasional large queries, improves cache hit rate, and reduces fluctuations in query performance. In simulated test cases with large queries, SLRU-based query performance can be improved by 70% or even higher. 
-  - Unified the Data Cache instance used in both shared-data architecture and data lake query scenarios to simplify the configuration and improve resource utilization.
+  - Introduces a Segmented LRU (SLRU) Cache eviction strategy, which significantly defends against cache pollution from occasional large queries, improves cache hit rate, and reduces fluctuations in query performance. In simulated test cases with large queries, SLRU-based query performance can be improved by 70% or even higher. For more information, see [Data Cache - Cache replacement policies](https://docs.starrocks.io/docs/data_source/data_cache/#cache-replacement-policies).
+  - Unified the Data Cache instance used in both shared-data architecture and data lake query scenarios to simplify the configuration and improve resource utilization. For more information, see [Data Cache](https://docs.starrocks.io/docs/using_starrocks/caching/block_cache/).
   - Provides an adaptive I/O strategy optimization for Data Cache, which flexibly routes some query requests to remote storage based on the cache disk's load and performance, thereby enhancing overall access throughput.
-- Supports automatic collection of external table statistics through automatic ANALYZE tasks triggered by queries. It can provide more accurate NDV information compared to metadata files, thereby optimizing the query plan and improving query performance.
+- Supports automatic collection of external table statistics through automatic ANALYZE tasks triggered by queries. It can provide more accurate NDV information compared to metadata files, thereby optimizing the query plan and improving query performance. For more information, see [Query-triggered collection](https://docs.starrocks.io/docs/using_starrocks/Cost_based_optimizer/#query-triggered-collection).
 
 <!--
 - Provides Time Travel query capability for Iceberg, allowing data to be read from a specified BRANCH or TAG by specifying TIMESTAMP or VERSION.
@@ -25,8 +25,8 @@ Release date: January 13, 2025
 
 ### Performance Improvement and Query Optimization
 
-- [Experimental] Offers a preliminary Query Feedback feature for automatic optimization of slow queries. The system will collect the execution details of slow queries, automatically analyze its query plan for potential opportunities for optimization, and generate a tailored optimization guide for the query. If CBO generates the same bad plan for subsequent identical queries, the system will locally optimize this query plan based on the guide.
-- [Experimental] Supports Python UDFs, offering more convenient function customization compared to Java UDFs.
+- [Experimental] Offers a preliminary Query Feedback feature for automatic optimization of slow queries. The system will collect the execution details of slow queries, automatically analyze its query plan for potential opportunities for optimization, and generate a tailored optimization guide for the query. If CBO generates the same bad plan for subsequent identical queries, the system will locally optimize this query plan based on the guide. For more information, see [Query Feedback](https://docs.starrocks.io/docs/using_starrocks/query_feedback/).
+- [Experimental] Supports Python UDFs, offering more convenient function customization compared to Java UDFs. For more information, see [Python UDF](https://docs.starrocks.io/docs/sql-reference/sql-functions/Python_UDF/).
 
 <!--
 - [Experimental] Supports Arrow Flight interface for more efficient reading of large data volumes in query results. It also allows BE, instead of FE, to process the returned results, greatly reducing the pressure on FE. It is especially suitable for business scenarios involving big data analysis and processing, and machine learning.
@@ -41,7 +41,7 @@ Release date: January 13, 2025
 
 ### Storage Engine
 
-- Unified all partitioning methods into the expression partitioning and supported multi-level partitioning, where each level can be any expression.
+- Unified all partitioning methods into the expression partitioning and supported multi-level partitioning, where each level can be any expression. For more information, see [Expression Partitioning](https://docs.starrocks.io/docs/table_design/data_distribution/expression_partitioning/).
 
 <!--
 - [Preview] Supports all native aggregate functions in Aggregate tables. By introducing a generic aggregate function state storage framework, all native aggregate functions supported by StarRocks can be used to define an Aggregate table.
@@ -50,14 +50,14 @@ Release date: January 13, 2025
 
 ### Loading
 
-- INSERT OVERWRITE now supports a new semantic - Dynamic Overwrite. When this semantic is enabled, the ingested data will either create new partitions or overwrite existing partitions that correspond to the new data records. Partitions not involved will not be truncated or deleted. This semantic is especially useful when users want to recover data in specific partitions without specifying the partition names.
+- INSERT OVERWRITE now supports a new semantic - Dynamic Overwrite. When this semantic is enabled, the ingested data will either create new partitions or overwrite existing partitions that correspond to the new data records. Partitions not involved will not be truncated or deleted. This semantic is especially useful when users want to recover data in specific partitions without specifying the partition names. For more information, see [Dynamic Overwrite](https://docs.starrocks.io/docs/loading/InsertInto/#dynamic-overwrite).
 - Optimized the data ingestion with INSERT from FILES to replace Broker Load as the preferred loading method:
-  - FILES now supports listing files in remote storage, and providing basic statistics of the files.
-  - INSERT now supports matching columns by name, which is especially useful when users load data from numerous columns with identical names. (The default behavior matches columns by their position.)
-  - INSERT supports specifying PROPERTIES, aligning with other loading methods. Users can specify `strict_mode`, `max_filter_ratio`, and `timeout` for INSERT operations to control and behavior and quality of the data ingestion.
-  - INSERT from FILES supports pushing down the target table schema check to the Scan stage of FILES to infer a more accurate source data schema.
-  - FILES supports unionizing files with different schema. The schema of Parquet and ORC files are unionized based on the column names, and that of CSV files are unionized based on the position (order) of the columns. When there are mismatched columns, users can choose to fill the columns with NULL or return an error by specifying the property `fill_mismatch_column_with`.
-  - FILES supports inferring the STRUCT type data from Parquet files. (In earlier versions, STRUCT data is inferred as STRING type.)
+  - FILES now supports listing files in remote storage, and providing basic statistics of the files. For more information, see [FILES - list_files_only](https://docs.starrocks.io/docs/sql-reference/sql-functions/table-functions/files/#list_files_only).
+  - INSERT now supports matching columns by name, which is especially useful when users load data from numerous columns with identical names. (The default behavior matches columns by their position.) For more information, see [Match column by name](https://docs.starrocks.io/docs/loading/InsertInto/#match-column-by-name).
+  - INSERT supports specifying PROPERTIES, aligning with other loading methods. Users can specify `strict_mode`, `max_filter_ratio`, and `timeout` for INSERT operations to control and behavior and quality of the data ingestion. For more information, see [INSERT - PROPERTIES](https://docs.starrocks.io/docs/sql-reference/sql-statements/loading_unloading/INSERT/#properties).
+  - INSERT from FILES supports pushing down the target table schema check to the Scan stage of FILES to infer a more accurate source data schema. For more information, see see [Push down target table schema check](https://docs.starrocks.io/docs/sql-reference/sql-functions/table-functions/files/#push-down-target-table-schema-check).
+  - FILES supports unionizing files with different schema. The schema of Parquet and ORC files are unionized based on the column names, and that of CSV files are unionized based on the position (order) of the columns. When there are mismatched columns, users can choose to fill the columns with NULL or return an error by specifying the property `fill_mismatch_column_with`. For more information, see [Union files with different schema](https://docs.starrocks.io/docs/sql-reference/sql-functions/table-functions/files/#union-files-with-different-schema).
+  - FILES supports inferring the STRUCT type data from Parquet files. (In earlier versions, STRUCT data is inferred as STRING type.) For more information, see [Infer STRUCT type from Parquet](https://docs.starrocks.io/docs/sql-reference/sql-functions/table-functions/files/#infer-struct-type-from-parquet).
 
 <!--
 - Supports merging multiple concurrent Stream Load requests into a single transaction and committing data in a batch, thus improving the throughput of real-time data ingestion. It is designed for high-concurrency, small-batch (from KB to tens of MB) real-time loading scenarios. It can reduce the excessive data versions caused by frequent loading operations, resource consumption during Compaction, and IOPS and I/O latency brought by excessive small files.
diff --git a/docs/zh/release_notes/release-3.4.md b/docs/zh/release_notes/release-3.4.md
index 2edb4e732a1354..df4f74f6d7e906 100644
--- a/docs/zh/release_notes/release-3.4.md
+++ b/docs/zh/release_notes/release-3.4.md
@@ -11,12 +11,12 @@ displayed_sidebar: docs
 ### 数据湖分析
 
 - 优化了针对 Iceberg V2 的查询性能：通过减少对 Delete-file 的重复读取，提升了查询性能并降低了内存使用量。
-- 支持 Delta Lake 表的列映射功能，允许查询经过 Delta Schema Evolution 的 Delta Lake 数据。
+- 支持 Delta Lake 表的列映射功能，允许查询经过 Delta Schema Evolution 的 Delta Lake 数据。更多内容，参考 [Delta Lake Catalog - 功能支持](https://docs.starrocks.io/zh/docs/data_source/catalog/deltalake_catalog/#%E5%8A%9F%E8%83%BD%E6%94%AF%E6%8C%81)。
 - Data Cache 相关优化： 
-  - 引入了分段 LRU (SLRU) 缓存淘汰策略，有效防止偶发大查询导致的缓存污染，提高缓存命中率，减少查询性能波动。在有大查询污染的模拟测试中，基于 SLRU 的查询性能能提升 70% 至数倍。 
+  - 引入了分段 LRU (SLRU) 缓存淘汰策略，有效防止偶发大查询导致的缓存污染，提高缓存命中率，减少查询性能波动。在有大查询污染的模拟测试中，基于 SLRU 的查询性能能提升 70% 至数倍。 更多内容，参考 [Data Cache - 缓存淘汰机制](https://docs.starrocks.io/zh/docs/data_source/data_cache/#%E7%BC%93%E5%AD%98%E6%B7%98%E6%B1%B0%E6%9C%BA%E5%88%B6)。
   - 优化了 Data Cache 的自适应 I/O 策略。系统会根据缓存磁盘的负载和性能，自适应地将部分查询请求路由到远端存储，从而提升整体访问吞吐能力。
-  - 统一了存算分离架构和数据湖查询场景中使用的 Data Cache 实例，以及相关的参数和指标，简化配置，并提升资源使用率。
-- 支持通过查询自动触发 ANALYZE 任务自动收集外部表统计信息，相较于元数据文件，可提供更准确的 NDV 信息，从而优化查询计划并提升查询性能。
+  - 统一了存算分离架构和数据湖查询场景中使用的 Data Cache 实例，以及相关的参数和指标，简化配置，并提升资源使用率。更多内容，参考 [Data Cache](https://docs.starrocks.io/zh/docs/using_starrocks/caching/block_cache/)。
+- 支持通过查询自动触发 ANALYZE 任务自动收集外部表统计信息，相较于元数据文件，可提供更准确的 NDV 信息，从而优化查询计划并提升查询性能。更多内容，参考 [查询触发采集](https://docs.starrocks.io/zh/docs/using_starrocks/Cost_based_optimizer/#%E6%9F%A5%E8%AF%A2%E8%A7%A6%E5%8F%91%E9%87%87%E9%9B%86)。
 
 <!--
 - 提供针对 Iceberg 的 Time Travel 查询功能，可通过指定 TIMESTAMP 或 VERSION，从指定的 BRANCH 或 TAG 读取数据。
@@ -25,8 +25,8 @@ displayed_sidebar: docs
 
 ### 性能提升与查询优化
 
-- [Experimental] 初步支持 Query Feedback 功能，用于慢查询的自动优化。系统将收集慢查询的执行详情，自动分析查询计划中是否存在需要调优的地方，并生成专属的 Tuning Guide。当后续相同查询生成相同的 Bad Plan 时，系统会基于先前生成的 Tuning Guide 局部调优该 Query Plan。
-- [Experimental] 支持 Python UDF，相较于 Java UDF 提供了更便捷的函数自定义能力。
+- [Experimental] 初步支持 Query Feedback 功能，用于慢查询的自动优化。系统将收集慢查询的执行详情，自动分析查询计划中是否存在需要调优的地方，并生成专属的 Tuning Guide。当后续相同查询生成相同的 Bad Plan 时，系统会基于先前生成的 Tuning Guide 局部调优该 Query Plan。更多内容，参考 [Query Feedback](https://docs.starrocks.io/zh/docs/using_starrocks/query_feedback/)。
+- [Experimental] 支持 Python UDF，相较于 Java UDF 提供了更便捷的函数自定义能力。更多内容，参考 [Python UDF](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/Python_UDF/)。
 
 <!--
 - [Experimental] 支持 Arrow Flight 接口，可更高效读取大数据量的查询结果，并使 BE 替代 FE 直接处理返回结果，显著降低 FE 压力，特别适用于大数据分析、处理和机器学习等场景。
@@ -41,7 +41,7 @@ displayed_sidebar: docs
 
 ### 存储引擎
 
-- 统一多种分区方式为表达式分区，支持多级分区，每级分区均可为任意表达式。
+- 统一多种分区方式为表达式分区，支持多级分区，每级分区均可为任意表达式。更多内容，参考 [表达式分区](https://docs.starrocks.io/zh/docs/table_design/data_distribution/expression_partitioning/)。
 
 <!--
 - [Preview] 通过引入通用聚合函数状态存储框架，聚合表可以支持所有 StarRocks 原生聚合函数。
@@ -50,14 +50,14 @@ displayed_sidebar: docs
 
 ### 数据导入
 
-- INSERT OVERWRITE 新增 Dynamic Overwrite 语义，启用后，系统将根据导入的数据自动创建分区或覆盖对应的现有分区，导入不涉及的分区不会被清空或删除，适用于恢复特定分区数据的场景。
+- INSERT OVERWRITE 新增 Dynamic Overwrite 语义，启用后，系统将根据导入的数据自动创建分区或覆盖对应的现有分区，导入不涉及的分区不会被清空或删除，适用于恢复特定分区数据的场景。更多内容，参考 []()。
 - 优化了 INSERT from FILES 导入，使其可以基本取代 Broker Load 成为首选导入方式： 
-  - FILES 支持 LIST 远程存储中的文件，并提供文件的基本统计信息。
-  - INSERT 支持按名称匹配列，特别适用于导入列很多且列名相同的数据。（默认按位置匹配列。）
-  - INSERT 支持指定 PROPERTIES，与其他导入方式保持一致。用户可通过指定 `strict_mode`、`max_filter_ratio` 和 `timeout` 来控制数据导入的行为和质量。
-  - INSERT from FILES 支持将目标表的 Schema 检查下推到 FILES 的扫描阶段，从而更准确地推断源数据 Schema。
-  - FILES 支持合并不同 Schema 的文件。Parquet 和 ORC 文件基于列名合并，CSV 文件基于列位置（顺序）合并。对于不匹配的列，用户可通过指定 `fill_mismatch_column_with` 属性选择填充 NULL 值或报错。
-  - FILES 支持从 Parquet 文件推断 STRUCT 类型数据。（在早期版本中，STRUCT 数据被推断为 STRING 类型。）
+  - FILES 支持 LIST 远程存储中的文件，并提供文件的基本统计信息。更多内容，参考 [FILES - list_files_only](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#list_files_only)。
+  - INSERT 支持按名称匹配列，特别适用于导入列很多且列名相同的数据。（默认按位置匹配列。）更多内容，参考 [INSERT 按名称匹配列](https://docs.starrocks.io/zh/docs/loading/InsertInto/#insert-%E6%8C%89%E5%90%8D%E7%A7%B0%E5%8C%B9%E9%85%8D%E5%88%97)。
+  - INSERT 支持指定 PROPERTIES，与其他导入方式保持一致。用户可通过指定 `strict_mode`、`max_filter_ratio` 和 `timeout` 来控制数据导入的行为和质量。更多内容，参考 [](https://docs.starrocks.io/zh/docs/sql-reference/sql-statements/loading_unloading/INSERT/#properties)。
+  - INSERT from FILES 支持将目标表的 Schema 检查下推到 FILES 的扫描阶段，从而更准确地推断源数据 Schema。更多内容，参考 [Target Table Schema 检查下推](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#target-table-schema-%E6%A3%80%E6%9F%A5%E4%B8%8B%E6%8E%A8)。
+  - FILES 支持合并不同 Schema 的文件。Parquet 和 ORC 文件基于列名合并，CSV 文件基于列位置（顺序）合并。对于不匹配的列，用户可通过指定 `fill_mismatch_column_with` 属性选择填充 NULL 值或报错。更多内容，参考 [合并具有不同 Schema 的文件](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#%E5%90%88%E5%B9%B6%E5%85%B7%E6%9C%89%E4%B8%8D%E5%90%8C-schema-%E7%9A%84%E6%96%87%E4%BB%B6)。
+  - FILES 支持从 Parquet 文件推断 STRUCT 类型数据。（在早期版本中，STRUCT 数据被推断为 STRING 类型。）更多内容，参考 [推断 Parquet 文件中的 STRUCT 类型](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#%E6%8E%A8%E6%96%AD-parquet-%E6%96%87%E4%BB%B6%E4%B8%AD%E7%9A%84-struct-%E7%B1%BB%E5%9E%8B)。
 
 <!--
 - 支持将多个并发的 Stream Load 请求合并为一个事务批量提交，从而提高实时数据导入的吞吐能力，对于高并发、小批量（KB到几十MB）实时导入场景特别有用，可以减少频繁导入操作导致的数据版本过多、避免 Compaction 过程中的大量资源消耗，并且降低过多小文件带来的 IOPS 和 I/O 延迟。

From 70e5cbb80b6e33036056099a20d8fb48aaacac6d Mon Sep 17 00:00:00 2001
From: Youngwb <yangwenbo_mailbox@163.com>
Date: Wed, 15 Jan 2025 14:17:45 +0800
Subject: [PATCH 05/71] [Enhancement] deletion bitmap fill skip rows filter in
 batches (#55022)

Signed-off-by: Youngwb <yangwenbo_mailbox@163.com>
---
 .../deletion_vector/deletion_bitmap.cpp        | 10 +++++++---
 .../deletion_vector/deletion_bitmap.h          |  2 ++
 be/src/exec/hdfs_scanner.cpp                   | 18 +++++++++++++-----
 be/src/exec/hdfs_scanner.h                     |  3 ++-
 be/src/exec/hdfs_scanner_parquet.cpp           |  5 +++--
 5 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/be/src/connector/deletion_vector/deletion_bitmap.cpp b/be/src/connector/deletion_vector/deletion_bitmap.cpp
index 341011da50c591..ef5100db3309f4 100644
--- a/be/src/connector/deletion_vector/deletion_bitmap.cpp
+++ b/be/src/connector/deletion_vector/deletion_bitmap.cpp
@@ -23,6 +23,7 @@ StatusOr<bool> DeletionBitmap::fill_filter(uint64_t start, uint64_t end, Filter&
     if (start >= end) {
         return false;
     }
+
     roaring64_iterator_t* it = roaring64_iterator_create(_bitmap);
     DeferOp defer([&it] { roaring64_iterator_free(it); });
     if (!roaring64_iterator_move_equalorlarger(it, start)) {
@@ -30,11 +31,14 @@ StatusOr<bool> DeletionBitmap::fill_filter(uint64_t start, uint64_t end, Filter&
     }
 
     bool has_filter = false;
+    std::vector<uint64_t> buf(kBatchSize, 0);
     while (roaring64_iterator_has_value(it) && roaring64_iterator_value(it) < end) {
-        uint64_t value = roaring64_iterator_value(it);
-        filter[value - start] = 0;
+        uint64_t count = roaring64_iterator_read(it, buf.data(), kBatchSize);
+        for (uint64_t i = 0; i < count && buf[i] < end; ++i) {
+            filter[buf[i] - start] = 0;
+        }
+
         has_filter = true;
-        roaring64_iterator_advance(it);
     }
 
     return has_filter;
diff --git a/be/src/connector/deletion_vector/deletion_bitmap.h b/be/src/connector/deletion_vector/deletion_bitmap.h
index da3ba31aab1f3c..7f6a6fef202cb7 100644
--- a/be/src/connector/deletion_vector/deletion_bitmap.h
+++ b/be/src/connector/deletion_vector/deletion_bitmap.h
@@ -40,6 +40,8 @@ class DeletionBitmap {
     void to_array(std::vector<uint64_t>& array) const;
 
 private:
+    static const uint64_t kBatchSize = 256;
+
     roaring64_bitmap_t* _bitmap = nullptr;
 };
 
diff --git a/be/src/exec/hdfs_scanner.cpp b/be/src/exec/hdfs_scanner.cpp
index 23140df880c910..00e22b72321060 100644
--- a/be/src/exec/hdfs_scanner.cpp
+++ b/be/src/exec/hdfs_scanner.cpp
@@ -335,8 +335,8 @@ void HdfsScanner::do_update_iceberg_v2_counter(RuntimeProfile* parent_profile, c
     COUNTER_UPDATE(delete_file_per_scan_counter, _app_stats.iceberg_delete_files_per_scan);
 }
 
-void HdfsScanner::do_update_deletion_vector_counter(RuntimeProfile* parent_profile) {
-    if (_scanner_ctx.enable_split_tasks && !has_split_tasks()) {
+void HdfsScanner::do_update_deletion_vector_build_counter(RuntimeProfile* parent_profile) {
+    if (_app_stats.deletion_vector_build_count == 0) {
         return;
     }
     const std::string DV_TIMER = DeletionVector::DELETION_VECTOR;
@@ -344,16 +344,24 @@ void HdfsScanner::do_update_deletion_vector_counter(RuntimeProfile* parent_profi
 
     RuntimeProfile::Counter* delete_build_timer =
             ADD_CHILD_COUNTER(parent_profile, "DeletionVectorBuildTime", TUnit::TIME_NS, DV_TIMER);
-    RuntimeProfile::Counter* delete_file_build_filter_timer =
-            ADD_CHILD_COUNTER(parent_profile, "DeletionVectorBuildRowIdFilterTime", TUnit::TIME_NS, DV_TIMER);
+
     RuntimeProfile::Counter* delete_file_per_scan_counter =
             ADD_CHILD_COUNTER(parent_profile, "DeletionVectorBuildCount", TUnit::UNIT, DV_TIMER);
 
     COUNTER_UPDATE(delete_build_timer, _app_stats.deletion_vector_build_ns);
-    COUNTER_UPDATE(delete_file_build_filter_timer, _app_stats.build_rowid_filter_ns);
+
     COUNTER_UPDATE(delete_file_per_scan_counter, _app_stats.deletion_vector_build_count);
 }
 
+void HdfsScanner::do_update_deletion_vector_filter_counter(RuntimeProfile* parent_profile) {
+    const std::string DV_TIMER = DeletionVector::DELETION_VECTOR;
+    ADD_COUNTER(parent_profile, DV_TIMER, TUnit::NONE);
+
+    RuntimeProfile::Counter* delete_file_build_filter_timer =
+            ADD_CHILD_COUNTER(parent_profile, "DeletionVectorBuildRowIdFilterTime", TUnit::TIME_NS, DV_TIMER);
+    COUNTER_UPDATE(delete_file_build_filter_timer, _app_stats.build_rowid_filter_ns);
+}
+
 int64_t HdfsScanner::estimated_mem_usage() const {
     if (_scanner_ctx.estimated_mem_usage_per_split_task != 0) {
         return _scanner_ctx.estimated_mem_usage_per_split_task;
diff --git a/be/src/exec/hdfs_scanner.h b/be/src/exec/hdfs_scanner.h
index 59859f4559bb33..7f1e786111c504 100644
--- a/be/src/exec/hdfs_scanner.h
+++ b/be/src/exec/hdfs_scanner.h
@@ -419,7 +419,8 @@ class HdfsScanner {
     static CompressionTypePB get_compression_type_from_path(const std::string& filename);
 
     void do_update_iceberg_v2_counter(RuntimeProfile* parquet_profile, const std::string& parent_name);
-    void do_update_deletion_vector_counter(RuntimeProfile* parent_profile);
+    void do_update_deletion_vector_build_counter(RuntimeProfile* parent_profile);
+    void do_update_deletion_vector_filter_counter(RuntimeProfile* parent_profile);
 
 private:
     bool _opened = false;
diff --git a/be/src/exec/hdfs_scanner_parquet.cpp b/be/src/exec/hdfs_scanner_parquet.cpp
index 2301277f2a8713..37682f9623a613 100644
--- a/be/src/exec/hdfs_scanner_parquet.cpp
+++ b/be/src/exec/hdfs_scanner_parquet.cpp
@@ -62,9 +62,9 @@ Status HdfsParquetScanner::do_init(RuntimeState* runtime_state, const HdfsScanne
 
 void HdfsParquetScanner::do_update_counter(HdfsScanProfile* profile) {
     RuntimeProfile* root = profile->runtime_profile;
-    // deletion vector build only in the first task which used for splite sub-tasks,
+    // deletion vector build only in the first task which used for split sub-tasks,
     // and do not need to re-build in sub io tasks.
-    do_update_deletion_vector_counter(root);
+    do_update_deletion_vector_build_counter(root);
     // if we have split tasks, we don't need to update counter
     // and we will update those counters in sub io tasks.
     if (has_split_tasks()) {
@@ -171,6 +171,7 @@ void HdfsParquetScanner::do_update_counter(HdfsScanProfile* profile) {
     COUNTER_UPDATE(page_skip, _app_stats.page_skip);
     group_min_round_cost->set(_app_stats.group_min_round_cost);
     do_update_iceberg_v2_counter(root, kParquetProfileSectionPrefix);
+    do_update_deletion_vector_filter_counter(root);
     COUNTER_UPDATE(rows_before_page_index, _app_stats.rows_before_page_index);
     COUNTER_UPDATE(page_index_timer, _app_stats.page_index_ns);
     COUNTER_UPDATE(total_row_groups, _app_stats.parquet_total_row_groups);

From 1abe63d19d006fe341ee55eb4080ab5bf5761f16 Mon Sep 17 00:00:00 2001
From: satanson <ranpanf@gmail.com>
Date: Wed, 15 Jan 2025 15:12:23 +0800
Subject: [PATCH 06/71] [BugFix] Fix MemTracker::release_without_root (#55095)

Signed-off-by: satanson <ranpanf@gmail.com>
---
 be/src/runtime/mem_tracker.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h
index d22296ee22b680..bd96729a6fba6e 100644
--- a/be/src/runtime/mem_tracker.h
+++ b/be/src/runtime/mem_tracker.h
@@ -267,7 +267,7 @@ class MemTracker {
     }
 
     void release_without_root(int64_t bytes) {
-        if (bytes == 0) {
+        if (bytes == 0 || _all_trackers.empty()) {
             return;
         }
 

From 3015cd78bc9b02394e3f6ab9b02ea197025d27f8 Mon Sep 17 00:00:00 2001
From: Binglin Chang <decstery@gmail.com>
Date: Wed, 15 Jan 2025 17:17:01 +0800
Subject: [PATCH 07/71] [BugFix] Fix spill merge options in various key
 type/order by/merge condition configs (#54862)

---
 be/src/storage/lake/delta_writer.cpp         |  4 +++-
 be/src/storage/lake/spill_mem_table_sink.cpp |  6 +++++-
 be/src/storage/primary_index.cpp             | 20 --------------------
 be/src/storage/tablet_schema.cpp             | 10 ++++++++++
 be/src/storage/tablet_schema.h               |  2 ++
 be/test/storage/tablet_schema_test.cpp       |  2 ++
 6 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/be/src/storage/lake/delta_writer.cpp b/be/src/storage/lake/delta_writer.cpp
index 0b2c359efb4c75..4efbc32094dfe5 100644
--- a/be/src/storage/lake/delta_writer.cpp
+++ b/be/src/storage/lake/delta_writer.cpp
@@ -272,7 +272,9 @@ Status DeltaWriterImpl::build_schema_and_writer() {
                                                                              _txn_id, false);
         }
         RETURN_IF_ERROR(_tablet_writer->open());
-        if (config::enable_load_spill) {
+        if (config::enable_load_spill &&
+            !(_tablet_schema->keys_type() == KeysType::PRIMARY_KEYS &&
+              (!_merge_condition.empty() || is_partial_update() || _tablet_schema->has_separate_sort_key()))) {
             if (_load_spill_block_mgr == nullptr) {
                 _load_spill_block_mgr =
                         std::make_unique<LoadSpillBlockManager>(UniqueId(_load_id).to_thrift(), _tablet_id, _txn_id,
diff --git a/be/src/storage/lake/spill_mem_table_sink.cpp b/be/src/storage/lake/spill_mem_table_sink.cpp
index 57d547819b391a..69a5a4b8d447af 100644
--- a/be/src/storage/lake/spill_mem_table_sink.cpp
+++ b/be/src/storage/lake/spill_mem_table_sink.cpp
@@ -216,7 +216,11 @@ Status SpillMemTableSink::merge_blocks_to_segments() {
     size_t current_input_bytes = 0;
     auto merge_func = [&] {
         total_merges++;
-        auto merge_itr = new_heap_merge_iterator(merge_inputs);
+        // PK shouldn't do agg because pk support order key different from primary key,
+        // in that case, data is sorted by order key and cannot be aggregated by primary key
+        bool do_agg = _schema->keys_type() == TKeysType::AGG_KEYS || _schema->keys_type() == TKeysType::UNIQUE_KEYS;
+        auto tmp_itr = new_heap_merge_iterator(merge_inputs);
+        auto merge_itr = do_agg ? new_aggregate_iterator(tmp_itr) : tmp_itr;
         RETURN_IF_ERROR(merge_itr->init_encoded_schema(EMPTY_GLOBAL_DICTMAPS));
         auto chunk_shared_ptr = ChunkHelper::new_chunk(*_schema, config::vector_chunk_size);
         auto chunk = chunk_shared_ptr.get();
diff --git a/be/src/storage/primary_index.cpp b/be/src/storage/primary_index.cpp
index 198379635b432e..1bc133fdc4da6b 100644
--- a/be/src/storage/primary_index.cpp
+++ b/be/src/storage/primary_index.cpp
@@ -173,10 +173,6 @@ class HashIndexImpl : public HashIndex {
             auto p = _map.insert({keys[i], v});
             if (!p.second) {
                 uint64_t old = p.first->second.value;
-                if ((old >> 32) == rssid) {
-                    LOG(ERROR) << "found duplicate in upsert data rssid:" << rssid << " key=" << keys[i] << " idx=" << i
-                               << " rowid=" << rowid_start + i;
-                }
                 (*deletes)[(uint32_t)(old >> 32)].push_back((uint32_t)(old & ROWID_MASK));
                 p.first->second = v;
             }
@@ -395,10 +391,6 @@ class FixSliceHashIndex : public HashIndex {
                 auto p = _map.emplace_with_hash(prefetch_hashes[pslot], prefetch_keys[pslot], v);
                 if (!p.second) {
                     uint64_t old = p.first->second.value;
-                    if ((old >> 32) == rssid) {
-                        LOG(ERROR) << "found duplicate in upsert data rssid:" << rssid << " key=" << keys[i].to_string()
-                                   << " [" << hexdump(keys[i].data, keys[i].size) << "]";
-                    }
                     (*deletes)[(uint32_t)(old >> 32)].push_back((uint32_t)(old & ROWID_MASK));
                     p.first->second = v;
                 }
@@ -415,10 +407,6 @@ class FixSliceHashIndex : public HashIndex {
                 auto p = _map.emplace(FixSlice<S>(keys[i]), v);
                 if (!p.second) {
                     uint64_t old = p.first->second.value;
-                    if ((old >> 32) == rssid) {
-                        LOG(ERROR) << "found duplicate in upsert data rssid:" << rssid << " key=" << keys[i].to_string()
-                                   << " [" << hexdump(keys[i].data, keys[i].size) << "]";
-                    }
                     (*deletes)[(uint32_t)(old >> 32)].push_back((uint32_t)(old & ROWID_MASK));
                     p.first->second = v;
                 }
@@ -667,10 +655,6 @@ class SliceHashIndex : public HashIndex {
             auto p = _map.insert({keys[i].to_string(), v});
             if (!p.second) {
                 uint64_t old = p.first->second;
-                if ((old >> 32) == rssid) {
-                    LOG(ERROR) << "found duplicate in upsert data rssid:" << rssid << " key=" << keys[i].to_string()
-                               << " [" << hexdump(keys[i].data, keys[i].size) << "]";
-                }
                 (*deletes)[(uint32_t)(old >> 32)].push_back((uint32_t)(old & ROWID_MASK));
                 p.first->second = v;
             } else {
@@ -1353,10 +1337,6 @@ Status PrimaryIndex::_upsert_into_persistent_index(uint32_t rssid, uint32_t rowi
     RETURN_IF_ERROR(_persistent_index->upsert(n, vkeys, reinterpret_cast<IndexValue*>(values.data()),
                                               reinterpret_cast<IndexValue*>(old_values.data()), stat));
     for (unsigned long old : old_values) {
-        if ((old != NullIndexValue) && (old >> 32) == rssid) {
-            LOG(ERROR) << "found duplicate in upsert data rssid:" << rssid;
-            st = Status::InternalError("found duplicate in upsert data");
-        }
         if (old != NullIndexValue) {
             (*deletes)[(uint32_t)(old >> 32)].push_back((uint32_t)(old & ROWID_MASK));
         }
diff --git a/be/src/storage/tablet_schema.cpp b/be/src/storage/tablet_schema.cpp
index 410f22e069fae3..61c398c777d276 100644
--- a/be/src/storage/tablet_schema.cpp
+++ b/be/src/storage/tablet_schema.cpp
@@ -781,6 +781,16 @@ bool operator!=(const TabletSchema& a, const TabletSchema& b) {
     return !(a == b);
 }
 
+bool TabletSchema::has_separate_sort_key() const {
+    RETURN_IF(_sort_key_idxes.size() != _num_key_columns, true);
+    for (size_t i = 0; i < _sort_key_idxes.size(); ++i) {
+        if (_sort_key_idxes[i] != i) {
+            return true;
+        }
+    }
+    return false;
+}
+
 std::string TabletSchema::debug_string() const {
     std::stringstream ss;
     ss << "column=[";
diff --git a/be/src/storage/tablet_schema.h b/be/src/storage/tablet_schema.h
index 26f4a5174341dd..837ae139c6e835 100644
--- a/be/src/storage/tablet_schema.h
+++ b/be/src/storage/tablet_schema.h
@@ -331,6 +331,8 @@ class TabletSchema {
     }
     void set_num_short_key_columns(uint16_t num_short_key_columns) { _num_short_key_columns = num_short_key_columns; }
 
+    bool has_separate_sort_key() const;
+
     std::string debug_string() const;
 
     int64_t mem_usage() const;
diff --git a/be/test/storage/tablet_schema_test.cpp b/be/test/storage/tablet_schema_test.cpp
index 5eb8a2875f6542..f4931492b7a013 100644
--- a/be/test/storage/tablet_schema_test.cpp
+++ b/be/test/storage/tablet_schema_test.cpp
@@ -128,6 +128,8 @@ TEST(TabletSchemaTest, test_schema_with_index) {
     std::unordered_map<IndexType, TabletIndex> res;
     ASSERT_TRUE(tablet_schema.get_indexes_for_column(1, &res).ok());
     ASSERT_FALSE(res.empty());
+
+    ASSERT_FALSE(tablet_schema.has_separate_sort_key());
 }
 
 TEST(TabletSchemaTest, test_is_support_checksum) {

From d650aa8b3af3b636e72890be757f2443635c999e Mon Sep 17 00:00:00 2001
From: PengFei Li <lpengfei2016@gmail.com>
Date: Wed, 15 Jan 2025 19:02:01 +0800
Subject: [PATCH 08/71] [Enhancement] Introduce TxnStateDispatcher for merge
 commit sync mode #55001 (#55071)

This is the second PR of merge commit sync mode optimization https://github.com/StarRocks/starrocks/issues/54995. Introduce TxnStateDispatcher on FE side. You can see https://github.com/StarRocks/starrocks/issues/54995 for details

Signed-off-by: PengFei Li <lpengfei2016@gmail.com>
---
 .../java/com/starrocks/common/Config.java     |  26 +-
 .../load/batchwrite/BatchWriteMgr.java        |  11 +-
 .../CoordinatorBackendAssignerImpl.java       |   4 +-
 .../load/batchwrite/IsomorphicBatchWrite.java |  29 +-
 .../load/batchwrite/LoadExecuteCallback.java  |   4 +-
 .../load/batchwrite/LoadExecutor.java         |  29 +-
 .../load/batchwrite/TxnStateDispatcher.java   | 301 ++++++++++++++
 .../com/starrocks/rpc/PBackendService.java    |   4 +
 .../transaction/TransactionStatus.java        |  26 ++
 .../CoordinatorBackendAssignerTest.java       |   6 +-
 .../batchwrite/IsomorphicBatchWriteTest.java  |  16 +-
 .../load/batchwrite/LoadExecutorTest.java     |   4 +-
 .../batchwrite/TxnStateDispatcherTest.java    | 381 ++++++++++++++++++
 .../pseudocluster/PseudoBackend.java          |   7 +
 .../com/starrocks/utframe/MockedBackend.java  |   7 +
 gensrc/proto/internal_service.proto           |  13 +-
 gensrc/proto/types.proto                      |  15 +
 17 files changed, 846 insertions(+), 37 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/load/batchwrite/TxnStateDispatcher.java
 create mode 100644 fe/fe-core/src/test/java/com/starrocks/load/batchwrite/TxnStateDispatcherTest.java

diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
index 2198f1e793eb87..ea78c0cf00f81a 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/Config.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
@@ -3366,12 +3366,26 @@ public class Config extends ConfigBase {
     public static int lake_remove_table_thread_num = 4;
 
     @ConfField(mutable = true)
-    public static int batch_write_gc_check_interval_ms = 60000;
+    public static int merge_commit_gc_check_interval_ms = 60000;
 
     @ConfField(mutable = true)
-    public static int batch_write_idle_ms = 3600000;
+    public static int merge_commit_idle_ms = 3600000;
 
-    public static int batch_write_executor_threads_num = 4096;
+    @ConfField(mutable = false)
+    public static int merge_commit_executor_threads_num = 4096;
+
+    @ConfField(mutable = true)
+    public static int merge_commit_txn_state_dispatch_retry_times = 3;
+
+    @ConfField(mutable = true)
+    public static int merge_commit_txn_state_dispatch_retry_interval_ms = 200;
+
+    @ConfField(mutable = true)
+    public static int merge_commit_be_assigner_schedule_interval_ms = 5000;
+
+    @ConfField(mutable = true, comment = "Defines the maximum balance factor allowed " +
+            "between any two nodes before triggering a balance")
+    public static double merge_commit_be_assigner_balance_factor_threshold = 0.1;
 
     /**
      * Enable Arrow Flight SQL server only when the port is set to positive value.
@@ -3385,12 +3399,6 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true)
     public static int arrow_token_cache_expire = 3600;
 
-    public static int batch_write_be_assigner_schedule_interval_ms = 5000;
-
-    @ConfField(mutable = true, comment = "Defines the maximum balance factor allowed " +
-            "between any two nodes before triggering a balance")
-    public static double batch_write_be_assigner_balance_factor_threshold = 0.1;
-
     @ConfField(mutable = false)
     public static int query_deploy_threadpool_size = max(50, getRuntime().availableProcessors() * 10);
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/BatchWriteMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/BatchWriteMgr.java
index dc3e5d6afb763e..82ab0214735ba0 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/BatchWriteMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/BatchWriteMgr.java
@@ -60,14 +60,17 @@ public class BatchWriteMgr extends FrontendDaemon {
     // A thread pool executor for executing batch write tasks.
     private final ThreadPoolExecutor threadPoolExecutor;
 
+    private final TxnStateDispatcher txnStateDispatcher;
+
     public BatchWriteMgr() {
-        super("group-commit-mgr", Config.batch_write_gc_check_interval_ms);
+        super("merge-commit-mgr", Config.merge_commit_gc_check_interval_ms);
         this.idGenerator = new AtomicLong(0L);
         this.isomorphicBatchWriteMap = new ConcurrentHashMap<>();
         this.lock = new ReentrantReadWriteLock();
         this.coordinatorBackendAssigner = new CoordinatorBackendAssignerImpl();
         this.threadPoolExecutor = ThreadPoolManager.newDaemonCacheThreadPool(
-                        Config.batch_write_executor_threads_num, "batch-write-load", true);
+                        Config.merge_commit_executor_threads_num, "batch-write-load", true);
+        this.txnStateDispatcher = new TxnStateDispatcher(threadPoolExecutor);
     }
 
     @Override
@@ -79,7 +82,7 @@ public synchronized void start() {
 
     @Override
     protected void runAfterCatalogReady() {
-        setInterval(Config.batch_write_gc_check_interval_ms);
+        setInterval(Config.merge_commit_gc_check_interval_ms);
         cleanupInactiveBatchWrite();
     }
 
@@ -194,7 +197,7 @@ private Pair<TStatus, IsomorphicBatchWrite> getOrCreateTableBatchWrite(TableId t
                 long id = idGenerator.getAndIncrement();
                 IsomorphicBatchWrite newLoad = new IsomorphicBatchWrite(
                         id, tableId, warehouseName, streamLoadInfo, batchWriteIntervalMs, batchWriteParallel,
-                        params, coordinatorBackendAssigner, threadPoolExecutor);
+                        params, coordinatorBackendAssigner, threadPoolExecutor, txnStateDispatcher);
                 coordinatorBackendAssigner.registerBatchWrite(id, newLoad.getWarehouseId(), tableId,
                         newLoad.getBatchWriteParallel());
                 return newLoad;
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerImpl.java b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerImpl.java
index 87562d3ab72356..e9fd7fa8f4de18 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerImpl.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerImpl.java
@@ -122,7 +122,7 @@ private void runSchedule() {
                     checkIntervalMs = Integer.MAX_VALUE;
                     LOG.info("Disable periodical schedule because there is no load");
                 } else {
-                    checkIntervalMs = Math.max(MIN_CHECK_INTERVAL_MS, Config.batch_write_be_assigner_schedule_interval_ms);
+                    checkIntervalMs = Math.max(MIN_CHECK_INTERVAL_MS, Config.merge_commit_be_assigner_schedule_interval_ms);
                     LOG.debug("Set schedule interval to {} ms", checkIntervalMs);
                 }
                 task = taskPriorityQueue.poll(checkIntervalMs, TimeUnit.MILLISECONDS);
@@ -379,7 +379,7 @@ void runPeriodicalCheck() {
                     LOG.info("Remove empty warehouse {}", warehouseMeta.warehouseId);
                 } else {
                     checkNodeStatusAndReassignment(warehouseMeta);
-                    doBalanceIfNeeded(warehouseMeta, Config.batch_write_be_assigner_balance_factor_threshold);
+                    doBalanceIfNeeded(warehouseMeta, Config.merge_commit_be_assigner_balance_factor_threshold);
                     if (LOG.isDebugEnabled()) {
                         logStatistics(warehouseMeta);
                     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/IsomorphicBatchWrite.java b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/IsomorphicBatchWrite.java
index e6be1543bf2763..812d8c43ea25b4 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/IsomorphicBatchWrite.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/IsomorphicBatchWrite.java
@@ -49,7 +49,7 @@ public class IsomorphicBatchWrite implements LoadExecuteCallback {
 
     private static final Logger LOG = LoggerFactory.getLogger(IsomorphicBatchWrite.class);
 
-    private static final String LABEL_PREFIX = "batch_write_";
+    private static final String LABEL_PREFIX = "merge_commit_";
 
     private final long id;
     private final TableId tableId;
@@ -57,6 +57,7 @@ public class IsomorphicBatchWrite implements LoadExecuteCallback {
     private final StreamLoadInfo streamLoadInfo;
     private final int batchWriteIntervalMs;
     private final int batchWriteParallel;
+    private final boolean asyncMode;
     private final StreamLoadKvParams loadParameters;
 
     /**
@@ -69,6 +70,9 @@ public class IsomorphicBatchWrite implements LoadExecuteCallback {
      */
     private final Executor executor;
 
+    /** Update the transaction state of the backend if this is a sync mode. */
+    private final TxnStateDispatcher txnUpdateDispatch;
+
     /**
      * The factory to create query coordinators.
      */
@@ -95,16 +99,19 @@ public IsomorphicBatchWrite(
             int batchWriteParallel,
             StreamLoadKvParams loadParameters,
             CoordinatorBackendAssigner coordinatorBackendAssigner,
-            Executor executor) {
+            Executor executor,
+            TxnStateDispatcher txnUpdateDispatch) {
         this.id = id;
         this.tableId = tableId;
         this.warehouseName = warehouseName;
         this.streamLoadInfo = streamLoadInfo;
         this.batchWriteIntervalMs = batchWriteIntervalMs;
         this.batchWriteParallel = batchWriteParallel;
+        this.asyncMode = loadParameters.getBatchWriteAsync().orElse(false);
         this.loadParameters = loadParameters;
         this.coordinatorBackendAssigner = coordinatorBackendAssigner;
         this.executor = executor;
+        this.txnUpdateDispatch = txnUpdateDispatch;
         this.queryCoordinatorFactory = new DefaultCoordinator.Factory();
         this.loadExecutorMap = new ConcurrentHashMap<>();
         this.lock = new ReentrantReadWriteLock();
@@ -244,17 +251,29 @@ public RequestLoadResult requestLoad(long backendId, String backendHost) {
      */
     public boolean isActive() {
         long idleTime = System.currentTimeMillis() - lastLoadCreateTimeMs.get();
-        return !loadExecutorMap.isEmpty() || idleTime < Config.batch_write_idle_ms;
+        return !loadExecutorMap.isEmpty() || idleTime < Config.merge_commit_idle_ms;
     }
 
     @Override
-    public void finishLoad(String label) {
+    public void finishLoad(LoadExecutor executor) {
         lock.writeLock().lock();
         try {
-            loadExecutorMap.remove(label);
+            loadExecutorMap.remove(executor.getLabel());
         } finally {
             lock.writeLock().unlock();
         }
+
+        long txnId = executor.getTxnId();
+        if (!asyncMode && txnId > 0) {
+            for (long backendId : executor.getBackendIds()) {
+                try {
+                    txnUpdateDispatch.submitTask(tableId.getDbName(), txnId, backendId);
+                } catch (Exception e) {
+                    LOG.error("Fail to submit transaction state update task, db: {}, txn_id: {}, backend id: {}",
+                            tableId.getDbName(), txnId, backendId, e);
+                }
+            }
+        }
     }
 
     @VisibleForTesting
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecuteCallback.java b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecuteCallback.java
index fdad8c476cd9a9..77618e7a80390d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecuteCallback.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecuteCallback.java
@@ -22,7 +22,7 @@ public interface LoadExecuteCallback {
     /**
      * Called when the load operation is finished.
      *
-     * @param label The label associated with the load operation.
+     * @param loadExecutor The executor associated with the load operation.
      */
-    void finishLoad(String label);
+    void finishLoad(LoadExecutor loadExecutor);
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecutor.java b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecutor.java
index 6bdca621b0d7fa..3c97291b9da95e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecutor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/LoadExecutor.java
@@ -45,6 +45,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
@@ -114,7 +115,7 @@ public void run() {
             LOG.error("Failed to execute load, label: {}, load id: {}, txn id: {}",
                     label, DebugUtil.printId(loadId), txnId, e);
         } finally {
-            loadExecuteCallback.finishLoad(label);
+            loadExecuteCallback.finishLoad(this);
             timeTrace.finishTimeMs = System.currentTimeMillis();
             LOG.debug("Finish load, label: {}, load id: {}, txn_id: {}, {}",
                     label, DebugUtil.printId(loadId), txnId, timeTrace.summary());
@@ -125,6 +126,14 @@ public String getLabel() {
         return label;
     }
 
+    public long getTxnId() {
+        return txnId;
+    }
+
+    public Set<Long> getBackendIds() {
+        return Collections.unmodifiableSet(coordinatorBackendIds);
+    }
+
     /**
      * Checks if the given backend id is contained in the coordinator backend IDs.
      */
@@ -159,11 +168,11 @@ private void beginTxn() throws Exception {
 
     private void commitAndPublishTxn() throws Exception {
         timeTrace.commitTxnTimeMs = System.currentTimeMillis();
-        Pair<Database, OlapTable> pair = getDbAndTable();
+        Database database = getDb();
         long publishTimeoutMs =
                 streamLoadInfo.getTimeout() * 1000L - (timeTrace.commitTxnTimeMs - timeTrace.beginTxnTimeMs);
         boolean publishSuccess = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr().commitAndPublishTransaction(
-                pair.first, txnId, tabletCommitInfo, tabletFailInfo, publishTimeoutMs, null);
+                database, txnId, tabletCommitInfo, tabletFailInfo, publishTimeoutMs, null);
         if (!publishSuccess) {
             LOG.warn("Publish timeout, txn_id: {}, label: {}, total timeout: {} ms, publish timeout: {} ms",
                         txnId, label, streamLoadInfo.getTimeout() * 1000, publishTimeoutMs);
@@ -175,9 +184,9 @@ private void abortTxn(Throwable reason) {
             return;
         }
         try {
-            Pair<Database, OlapTable> pair = getDbAndTable();
+            Database database = getDb();
             GlobalStateMgr.getCurrentState().getGlobalTransactionMgr().abortTransaction(
-                    pair.first.getId(), txnId, reason == null ? "" : reason.getMessage());
+                    database.getId(), txnId, reason == null ? "" : reason.getMessage());
         } catch (Exception e) {
             LOG.error("Failed to abort transaction {}", txnId, e);
         }
@@ -253,6 +262,16 @@ private void executeLoad() throws Exception {
         }
     }
 
+    private Database getDb() throws Exception {
+        GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState();
+        Database db = globalStateMgr.getLocalMetastore().getDb(tableId.getDbName());
+        if (db == null) {
+            throw new LoadException(String.format("Database %s does not exist", tableId.getDbName()));
+        }
+
+        return db;
+    }
+
     private Pair<Database, OlapTable> getDbAndTable() throws Exception {
         GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState();
         Database db = globalStateMgr.getLocalMetastore().getDb(tableId.getDbName());
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/TxnStateDispatcher.java b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/TxnStateDispatcher.java
new file mode 100644
index 00000000000000..98ba42b970333e
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/load/batchwrite/TxnStateDispatcher.java
@@ -0,0 +1,301 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.load.batchwrite;
+
+import com.google.common.base.Preconditions;
+import com.starrocks.catalog.Database;
+import com.starrocks.common.Config;
+import com.starrocks.proto.PUpdateTransactionStateRequest;
+import com.starrocks.proto.PUpdateTransactionStateResponse;
+import com.starrocks.proto.TransactionStatePB;
+import com.starrocks.rpc.BrpcProxy;
+import com.starrocks.rpc.PBackendService;
+import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.system.ComputeNode;
+import com.starrocks.thrift.TNetworkAddress;
+import com.starrocks.transaction.TransactionStateSnapshot;
+import com.starrocks.transaction.TransactionStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.annotation.Nullable;
+
+/**
+ * Dispatch transaction state to backends after the load finished.
+ * TODO send txn states on the same backend in batch
+ */
+public class TxnStateDispatcher {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TxnStateDispatcher.class);
+
+    private final Executor rpcExecutor;
+    private final AtomicLong numSubmittedTasks;
+    private final ConcurrentMap<TaskId, Task> pendingTasks;
+
+    // For testing
+    private TaskExecuteListener taskExecuteListener;
+
+    public TxnStateDispatcher(Executor rpcExecutor) {
+        this.rpcExecutor = rpcExecutor;
+        this.numSubmittedTasks = new AtomicLong(0);
+        this.pendingTasks = new ConcurrentHashMap<>();
+    }
+
+    public void submitTask(String dbName, long txnId, long backendId) throws Exception {
+        TaskId taskId = new TaskId(txnId, backendId);
+        Task task = new Task(dbName, taskId);
+        Task oldTask = pendingTasks.putIfAbsent(taskId, task);
+        if (oldTask != null) {
+            return;
+        }
+        int numRetry = 0;
+        boolean success = false;
+        Throwable lastException = null;
+        while (true) {
+            try {
+                rpcExecutor.execute(() -> runTask(task));
+                success = true;
+                break;
+            } catch (Throwable e) {
+                LOG.warn("Failed to submit txn state update task, db: {}, txn_id: {}, backend_id: {}, num retry: {}",
+                        dbName, txnId, backendId, numRetry, e);
+                lastException = e;
+            }
+            if (numRetry >= 3) {
+                break;
+            }
+            numRetry += 1;
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException e) {
+                break;
+            }
+        }
+        if (success) {
+            numSubmittedTasks.incrementAndGet();
+            LOG.debug("Success to submit txn state update task, db: {}, txn_id: {}, backend_id: {}, num retry: {}",
+                    dbName, txnId, backendId, numRetry);
+        } else {
+            pendingTasks.remove(taskId);
+            throw new Exception(String.format("Failed to submit txn state update task, db: %s, txn_id: %s, " +
+                            "backend_id: %s, num retry: %s", dbName, txnId, backendId, numRetry), lastException);
+        }
+    }
+
+    private void runTask(Task task) {
+        int maxRetries = Config.merge_commit_txn_state_dispatch_retry_times;
+        int numRetry = 0;
+        DispatchResult result;
+        while (true) {
+            result = dispatchTxnState(task.dbName, task.taskId.txnId, task.taskId.backendId);
+            if (result.getStatus() != DispatchStatus.RETRYABLE) {
+                break;
+            }
+            LOG.warn("Dispatch txn state failed, db: {}, txn_id: {}, backend_id: {}, num retry: {}, fail reason: {}," +
+                            " txn state: {}", task.dbName, task.taskId.txnId, task.taskId.backendId, numRetry,
+                                    result.getFailReason(), result.getState());
+            if (numRetry >= maxRetries) {
+                break;
+            }
+            numRetry += 1;
+            try {
+                Thread.sleep(Config.merge_commit_txn_state_dispatch_retry_interval_ms);
+            } catch (InterruptedException e) {
+                break;
+            }
+        }
+        if (result.getStatus() == DispatchStatus.SUCCESS) {
+            LOG.debug("Success to dispatch txn state, db: {}, txn_id: {}, backend_id: {}, num retry: {}, txn state: {}",
+                    task.dbName, task.taskId.txnId, task.taskId.backendId, numRetry, result.getState());
+        } else {
+            LOG.warn("Failed to dispatch txn state, db: {}, txn_id: {}, backend_id: {}, fail reason: {}, txn state: {}",
+                    task.dbName, task.taskId.txnId, task.taskId.backendId, result.getFailReason(), result.getState());
+        }
+        pendingTasks.remove(task.taskId);
+        if (taskExecuteListener != null) {
+            taskExecuteListener.onFinish(task, result, numRetry);
+        }
+    }
+
+    private DispatchResult dispatchTxnState(String dbName, long txnId, long backendId) {
+        GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState();
+        ComputeNode computeNode = globalStateMgr.getNodeMgr().getClusterInfo().getBackendOrComputeNode(backendId);
+        if (computeNode == null) {
+            return DispatchResult.fail(DispatchStatus.ABORT, "can't find backend");
+        }
+        if (!computeNode.isAlive()) {
+            return DispatchResult.fail(DispatchStatus.RETRYABLE,
+                    String.format("backend [%s] does not alive", computeNode.getHost()));
+        }
+        TNetworkAddress address = new TNetworkAddress(computeNode.getHost(), computeNode.getBrpcPort());
+        Database db = globalStateMgr.getLocalMetastore().getDb(dbName);
+        TransactionStateSnapshot state;
+        if (db == null) {
+            state = new TransactionStateSnapshot(TransactionStatus.UNKNOWN, "can't find database " + dbName);
+        } else {
+            try {
+                state = globalStateMgr.getGlobalTransactionMgr().getTxnState(db, txnId);
+            } catch (Throwable e) {
+                state = new TransactionStateSnapshot(TransactionStatus.UNKNOWN,
+                        "can't get txn state, exception: " + e.getMessage());
+            }
+        }
+        try {
+            TransactionStatePB statePB = new TransactionStatePB();
+            statePB.setTxnId(txnId);
+            statePB.setStatus(state.getStatus().toProto());
+            statePB.setReason(state.getReason());
+            PUpdateTransactionStateRequest request = new PUpdateTransactionStateRequest();
+            request.setStates(Collections.singletonList(statePB));
+            PBackendService service = BrpcProxy.getBackendService(address);
+            Future<PUpdateTransactionStateResponse> future = service.updateTransactionState(request);
+            future.get();
+            return DispatchResult.success(state);
+        } catch (Throwable e) {
+            return DispatchResult.fail(DispatchStatus.RETRYABLE, state,
+                    "failed to update txn state, exception: " + e.getMessage());
+        }
+    }
+
+    public long getNumSubmittedTasks() {
+        return numSubmittedTasks.get();
+    }
+
+    Task getPendingTask(long txnId, long backendId) {
+        return pendingTasks.get(new TaskId(txnId, backendId));
+    }
+
+    void setTaskExecuteListener(TaskExecuteListener taskExecuteListener) {
+        this.taskExecuteListener = taskExecuteListener;
+    }
+
+    static class TaskId {
+        private final long txnId;
+        private final long backendId;
+
+        public TaskId(long txnId, long backendId) {
+            this.txnId = txnId;
+            this.backendId = backendId;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            TaskId taskId = (TaskId) o;
+            return txnId == taskId.txnId && backendId == taskId.backendId;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(txnId, backendId);
+        }
+    }
+
+    static class Task {
+        final String dbName;
+        final TaskId taskId;
+
+        public Task(String dbName, TaskId taskId) {
+            this.dbName = dbName;
+            this.taskId = taskId;
+        }
+
+        public String getDbName() {
+            return dbName;
+        }
+
+        public long getTxnId() {
+            return taskId.txnId;
+        }
+
+        public long getBackendId() {
+            return taskId.backendId;
+        }
+    }
+
+    enum DispatchStatus {
+        // Dispatch transaction state to backend successfully
+        SUCCESS,
+        // Dispatch transaction state to backend failed, and can't retry
+        ABORT,
+        // Dispatch transaction state to backend failed, but can retry
+        RETRYABLE
+    }
+
+    static class DispatchResult {
+        private DispatchStatus status;
+        // Maybe null if status is not SUCCESS
+        private @Nullable TransactionStateSnapshot state;
+        // Maybe null if status is SUCCESS
+        private @Nullable String failReason;
+
+        public DispatchStatus getStatus() {
+            return status;
+        }
+
+        public TransactionStateSnapshot getState() {
+            return state;
+        }
+
+        public String getFailReason() {
+            return failReason;
+        }
+
+        public static DispatchResult success(TransactionStateSnapshot stateSnapshot) {
+            Preconditions.checkNotNull(stateSnapshot);
+            DispatchResult result = new DispatchResult();
+            result.status = DispatchStatus.SUCCESS;
+            result.state = stateSnapshot;
+            return result;
+        }
+
+        public static DispatchResult fail(DispatchStatus type, String reason) {
+            Preconditions.checkNotNull(type);
+            Preconditions.checkNotNull(reason);
+            DispatchResult result = new DispatchResult();
+            result.status = type;
+            result.failReason = reason;
+            return result;
+        }
+
+        public static DispatchResult fail(DispatchStatus type, TransactionStateSnapshot stateSnapshot, String reason) {
+            Preconditions.checkNotNull(type);
+            Preconditions.checkNotNull(stateSnapshot);
+            Preconditions.checkNotNull(reason);
+            DispatchResult result = new DispatchResult();
+            result.status = type;
+            result.state = stateSnapshot;
+            result.failReason = reason;
+            return result;
+        }
+    }
+
+    // For testing
+    interface TaskExecuteListener {
+        void onFinish(Task task, DispatchResult result, int numRetry);
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/rpc/PBackendService.java b/fe/fe-core/src/main/java/com/starrocks/rpc/PBackendService.java
index 3ff596a0dce161..78464189abbec8 100644
--- a/fe/fe-core/src/main/java/com/starrocks/rpc/PBackendService.java
+++ b/fe/fe-core/src/main/java/com/starrocks/rpc/PBackendService.java
@@ -38,6 +38,8 @@
 import com.starrocks.proto.PTriggerProfileReportResult;
 import com.starrocks.proto.PUpdateFailPointStatusRequest;
 import com.starrocks.proto.PUpdateFailPointStatusResponse;
+import com.starrocks.proto.PUpdateTransactionStateRequest;
+import com.starrocks.proto.PUpdateTransactionStateResponse;
 
 import java.util.concurrent.Future;
 
@@ -101,5 +103,7 @@ public interface PBackendService {
     @ProtobufRPC(serviceName = "PInternalService", methodName = "fetch_arrow_schema", onceTalkTimeout = 600000)
     Future<PFetchArrowSchemaResult> fetchArrowSchema(PFetchArrowSchemaRequest request);
 
+    @ProtobufRPC(serviceName = "PInternalService", methodName = "update_transaction_state", onceTalkTimeout = 600000)
+    Future<PUpdateTransactionStateResponse> updateTransactionState(PUpdateTransactionStateRequest request);
 }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStatus.java b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStatus.java
index 899bccfac0588c..8b1a4e8a6e8a40 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStatus.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStatus.java
@@ -17,6 +17,7 @@
 
 package com.starrocks.transaction;
 
+import com.starrocks.proto.TransactionStatusPB;
 import com.starrocks.thrift.TTransactionStatus;
 
 import java.util.Arrays;
@@ -67,6 +68,31 @@ public TTransactionStatus toThrift() {
         }
     }
 
+    public TransactionStatusPB toProto() {
+        switch (this.getFlag()) {
+            // UNKNOWN
+            case 0:
+                return TransactionStatusPB.TRANS_UNKNOWN;
+            // PREPARE
+            case 1:
+                return TransactionStatusPB.TRANS_PREPARE;
+            // COMMITTED
+            case 2:
+                return TransactionStatusPB.TRANS_COMMITTED;
+            // VISIBLE
+            case 3:
+                return TransactionStatusPB.TRANS_VISIBLE;
+            // ABORTED
+            case 4:
+                return TransactionStatusPB.TRANS_ABORTED;
+            // PREPARED
+            case 5:
+                return TransactionStatusPB.TRANS_PREPARED;
+            default:
+                return TransactionStatusPB.TRANS_UNKNOWN;
+        }
+    }
+
     public boolean isFailed() {
         return this == UNKNOWN || this == ABORTED;
     }
diff --git a/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerTest.java
index 83f3e9b1459732..3e881b871edbe9 100644
--- a/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/CoordinatorBackendAssignerTest.java
@@ -173,7 +173,7 @@ public void testPeriodicalCheck() throws Exception {
             nodes.get().forEach(node -> backendIds.add(node.getId()));
         }
         assertEquals(5, backendIds.size());
-        assertTrue(assigner.currentLoadDiffRatio(1) < Config.batch_write_be_assigner_balance_factor_threshold);
+        assertTrue(assigner.currentLoadDiffRatio(1) < Config.merge_commit_be_assigner_balance_factor_threshold);
 
 
         // create empty warehouse meta
@@ -195,7 +195,7 @@ public void testPeriodicalCheck() throws Exception {
             backendIds.clear();
             assigner.runPeriodicalCheck();
             assertNull(assigner.getWarehouseMeta(2));
-            assertTrue(assigner.currentLoadDiffRatio(1) < Config.batch_write_be_assigner_balance_factor_threshold);
+            assertTrue(assigner.currentLoadDiffRatio(1) < Config.merge_commit_be_assigner_balance_factor_threshold);
             for (int i = 1; i <= 100; i++) {
                 Optional<List<ComputeNode>> nodes = assigner.getBackends(i);
                 assertTrue(nodes.isPresent());
@@ -203,7 +203,7 @@ public void testPeriodicalCheck() throws Exception {
                 nodes.get().forEach(node -> backendIds.add(node.getId()));
             }
             assertEquals(10, backendIds.size());
-            assertTrue(assigner.currentLoadDiffRatio(1) < Config.batch_write_be_assigner_balance_factor_threshold);
+            assertTrue(assigner.currentLoadDiffRatio(1) < Config.merge_commit_be_assigner_balance_factor_threshold);
         } finally {
             for (int i = 10006; i <= 10010; i++) {
                 UtFrameUtils.dropMockBackend(i);
diff --git a/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/IsomorphicBatchWriteTest.java b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/IsomorphicBatchWriteTest.java
index 8dc3e97af80c92..c4afa010cb082c 100644
--- a/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/IsomorphicBatchWriteTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/IsomorphicBatchWriteTest.java
@@ -60,6 +60,7 @@ public class IsomorphicBatchWriteTest extends BatchWriteTestBase {
     @Mocked
     private CoordinatorBackendAssigner assigner;
     private TestThreadPoolExecutor executor;
+    private TxnStateDispatcher txnStateDispatcher;
     private int parallel;
 
     private IsomorphicBatchWrite load;
@@ -67,12 +68,13 @@ public class IsomorphicBatchWriteTest extends BatchWriteTestBase {
     @Before
     public void setup() throws Exception {
         executor = new TestThreadPoolExecutor();
+        txnStateDispatcher = new TxnStateDispatcher(executor);
         parallel = 4;
         assertTrue("Number nodes " + allNodes.size(), parallel < allNodes.size());
         Map<String, String> map = new HashMap<>();
         map.put(StreamLoadHttpHeader.HTTP_FORMAT, "json");
         map.put(StreamLoadHttpHeader.HTTP_ENABLE_BATCH_WRITE, "true");
-        map.put(StreamLoadHttpHeader.HTTP_BATCH_WRITE_ASYNC, "true");
+        map.put(StreamLoadHttpHeader.HTTP_BATCH_WRITE_ASYNC, "false");
         StreamLoadKvParams params = new StreamLoadKvParams(map);
         StreamLoadInfo streamLoadInfo =
                 StreamLoadInfo.fromHttpStreamLoadRequest(null, -1, Optional.empty(), params);
@@ -85,7 +87,8 @@ public void setup() throws Exception {
                 parallel,
                 params,
                 assigner,
-                executor);
+                executor,
+                txnStateDispatcher);
     }
 
     @Test
@@ -156,6 +159,7 @@ public void testRequestLoadFromCoordinatorBackend() throws Exception {
         assertEquals(TransactionStatus.VISIBLE, getTxnStatus(label));
         assertNull(load.getLoadExecutor(label));
         assertEquals(0, load.numRunningLoads());
+        assertEquals(loadExecutor.getBackendIds().size(), txnStateDispatcher.getNumSubmittedTasks());
     }
 
     @Test
@@ -193,10 +197,14 @@ public void testRequestLoadFromNoneCoordinatorBackend() throws Exception {
         assertEquals(expectNodeIds, loadExecutor2.getCoordinatorBackendIds());
 
         executor.manualRun(loadExecutor1);
-        executor.manualRun(loadExecutor2);
-
         assertEquals(TransactionStatus.VISIBLE, getTxnStatus(label1));
+        assertEquals(loadExecutor1.getCoordinatorBackendIds().size(), txnStateDispatcher.getNumSubmittedTasks());
+
+        executor.manualRun(loadExecutor2);
         assertEquals(TransactionStatus.VISIBLE, getTxnStatus(label2));
+        assertEquals(loadExecutor1.getBackendIds().size() + loadExecutor2.getBackendIds().size(),
+                txnStateDispatcher.getNumSubmittedTasks());
+
         assertEquals(0, load.numRunningLoads());
     }
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/LoadExecutorTest.java b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/LoadExecutorTest.java
index 93919cfb21e348..76894280c211ef 100644
--- a/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/LoadExecutorTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/LoadExecutorTest.java
@@ -321,8 +321,8 @@ public List<String> getFinishedLoads() {
         }
 
         @Override
-        public void finishLoad(String label) {
-            finishedLoads.add(label);
+        public void finishLoad(LoadExecutor loadExecutor) {
+            finishedLoads.add(loadExecutor.getLabel());
         }
     }
     
diff --git a/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/TxnStateDispatcherTest.java b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/TxnStateDispatcherTest.java
new file mode 100644
index 00000000000000..b74052a65afb62
--- /dev/null
+++ b/fe/fe-core/src/test/java/com/starrocks/load/batchwrite/TxnStateDispatcherTest.java
@@ -0,0 +1,381 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.load.batchwrite;
+
+import com.starrocks.catalog.Database;
+import com.starrocks.common.Config;
+import com.starrocks.proto.PUpdateTransactionStateRequest;
+import com.starrocks.proto.PUpdateTransactionStateResponse;
+import com.starrocks.proto.StatusPB;
+import com.starrocks.rpc.BrpcProxy;
+import com.starrocks.rpc.PBackendService;
+import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.system.Backend;
+import com.starrocks.system.SystemInfoService;
+import com.starrocks.thrift.TNetworkAddress;
+import com.starrocks.transaction.TransactionStateSnapshot;
+import com.starrocks.transaction.TransactionStatus;
+import com.starrocks.utframe.MockedBackend;
+import mockit.Expectations;
+import mockit.Mock;
+import mockit.MockUp;
+import mockit.Mocked;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TxnStateDispatcherTest {
+
+    @Mocked
+    private GlobalStateMgr globalStateMgr;
+    private MockedPBackendService backendService;
+    private SystemInfoService systemInfoService;
+
+    @Before
+    public void setUp() throws Exception {
+        backendService = new MockedPBackendService();
+        new MockUp<BrpcProxy>() {
+            @Mock
+            private synchronized PBackendService getBackendService(TNetworkAddress address) {
+                return backendService;
+            }
+        };
+
+        systemInfoService = new SystemInfoService();
+        Backend node1 = new Backend(1, "10.0.0.1", 9050);
+        node1.setAlive(true);
+        node1.setBrpcPort(8060);
+        systemInfoService.addBackend(node1);
+        Backend node2 = new Backend(2, "10.0.0.2", 9050);
+        node2.setAlive(false);
+        node2.setBrpcPort(8060);
+        systemInfoService.addBackend(node2);
+    }
+
+    @Test
+    public void testSubmitTaskSuccess() throws Exception {
+        MockExecutor executor = new MockExecutor();
+        TxnStateDispatcher dispatcher = new TxnStateDispatcher(executor);
+
+        // success without retry
+        dispatcher.submitTask("db1", 1, 1);
+        assertEquals(1, executor.getNumExecuteCalled());
+        assertEquals(1, executor.numPendingTasks());
+        TxnStateDispatcher.Task task1 = dispatcher.getPendingTask(1, 1);
+        assertNotNull(task1);
+        assertEquals("db1", task1.getDbName());
+        assertEquals(1, task1.getTxnId());
+        assertEquals(1, task1.getBackendId());
+        assertEquals(1, dispatcher.getNumSubmittedTasks());
+
+        // success with retry
+        executor.setArtificialRejectNum(3);
+        dispatcher.submitTask("db2", 2, 2);
+        assertEquals(5, executor.getNumExecuteCalled());
+        assertEquals(2, executor.numPendingTasks());
+        assertEquals(0, executor.getArtificialRejectNum());
+        TxnStateDispatcher.Task task2 = dispatcher.getPendingTask(2, 2);
+        assertNotNull(task2);
+        assertEquals("db2", task2.getDbName());
+        assertEquals(2, task2.getTxnId());
+        assertEquals(2, task2.getBackendId());
+        assertEquals(2, dispatcher.getNumSubmittedTasks());
+    }
+
+    @Test
+    public void testSubmitTaskFail() throws Exception {
+        MockExecutor executor = new MockExecutor();
+        TxnStateDispatcher dispatcher = new TxnStateDispatcher(executor);
+
+        executor.setArtificialRejectNum(4);
+        try {
+            dispatcher.submitTask("db1", 1, 2);
+            fail();
+        } catch (Exception e) {
+            assertTrue(e.getCause() instanceof RejectedExecutionException);
+        }
+        assertEquals(4, executor.getNumExecuteCalled());
+        assertEquals(0, executor.getArtificialRejectNum());
+        assertEquals(0, executor.numPendingTasks());
+        assertNull(dispatcher.getPendingTask(1, 2));
+        assertEquals(0, dispatcher.getNumSubmittedTasks());
+    }
+
+    @Test
+    public void testDispatchSuccess() throws Exception {
+        Database db1 = new Database(1, "db1");
+        TransactionStateSnapshot txnState1 = new TransactionStateSnapshot(TransactionStatus.VISIBLE, "");
+        new Expectations() {
+            {
+                globalStateMgr.getNodeMgr().getClusterInfo();
+                result = systemInfoService;
+                globalStateMgr.getLocalMetastore().getDb("db1");
+                result = db1;
+                globalStateMgr.getGlobalTransactionMgr().getTxnState(db1, 1L);
+                result = txnState1;
+            }
+        };
+        PUpdateTransactionStateResponse response = new PUpdateTransactionStateResponse();
+        StatusPB statusPB = new StatusPB();
+        statusPB.setStatusCode(0);
+        response.setResults(Collections.singletonList(statusPB));
+        CompletableFuture<PUpdateTransactionStateResponse> future = new CompletableFuture<>();
+        future.complete(response);
+        backendService.addResponseFuture(future);
+        TxnStateDispatcher.DispatchResult expected = TxnStateDispatcher.DispatchResult.success(txnState1);
+        testDispatchBase("db1", 1, 1, expected, 0);
+    }
+
+    @Test
+    public void testDispatchRetrySuccess() throws Exception {
+        Database db1 = new Database(1, "db1");
+        TransactionStateSnapshot txnState1 = new TransactionStateSnapshot(TransactionStatus.VISIBLE, "");
+        new Expectations() {
+            {
+                globalStateMgr.getNodeMgr().getClusterInfo();
+                result = systemInfoService;
+                globalStateMgr.getLocalMetastore().getDb("db1");
+                result = db1;
+                globalStateMgr.getGlobalTransactionMgr().getTxnState(db1, 1L);
+                result = txnState1;
+            }
+        };
+        CompletableFuture<PUpdateTransactionStateResponse> failureFuture = new CompletableFuture<>();
+        failureFuture.completeExceptionally(new Exception("artificial failure"));
+        for (int i = 0; i < Config.merge_commit_txn_state_dispatch_retry_times; i++) {
+            backendService.addResponseFuture(failureFuture);
+        }
+        PUpdateTransactionStateResponse response = new PUpdateTransactionStateResponse();
+        StatusPB statusPB = new StatusPB();
+        statusPB.setStatusCode(0);
+        response.setResults(Collections.singletonList(statusPB));
+        CompletableFuture<PUpdateTransactionStateResponse> successFuture = new CompletableFuture<>();
+        successFuture.complete(response);
+        backendService.addResponseFuture(successFuture);
+        TxnStateDispatcher.DispatchResult expected = TxnStateDispatcher.DispatchResult.success(txnState1);
+        testDispatchBase("db1", 1, 1, expected,
+                Config.merge_commit_txn_state_dispatch_retry_times);
+    }
+
+    @Test
+    public void testBackendNotExist() throws Exception {
+        new Expectations() {
+            {
+                globalStateMgr.getNodeMgr().getClusterInfo();
+                result = systemInfoService;
+            }
+        };
+        TxnStateDispatcher.DispatchResult expected = TxnStateDispatcher.DispatchResult.fail(
+                TxnStateDispatcher.DispatchStatus.ABORT, "can't find backend");
+        testDispatchBase("db1", 1, 3, expected, 0);
+    }
+
+    @Test
+    public void testDbNotExist() throws Exception {
+        new Expectations() {
+            {
+                globalStateMgr.getNodeMgr().getClusterInfo();
+                result = systemInfoService;
+                globalStateMgr.getLocalMetastore().getDb("db1");
+                result = null;
+            }
+        };
+        TransactionStateSnapshot txnState = new TransactionStateSnapshot(TransactionStatus.UNKNOWN, "can't find database db1");
+        TxnStateDispatcher.DispatchResult expected = TxnStateDispatcher.DispatchResult.success(txnState);
+        PUpdateTransactionStateResponse response = new PUpdateTransactionStateResponse();
+        StatusPB statusPB = new StatusPB();
+        statusPB.setStatusCode(0);
+        response.setResults(Collections.singletonList(statusPB));
+        CompletableFuture<PUpdateTransactionStateResponse> future = new CompletableFuture<>();
+        future.complete(response);
+        backendService.addResponseFuture(future);
+        testDispatchBase("db1", 1, 1, expected, 0);
+    }
+
+    @Test
+    public void testGetTxnStateFail() throws Exception {
+        Database db1 = new Database(1, "db1");
+        new Expectations() {
+            {
+                globalStateMgr.getNodeMgr().getClusterInfo();
+                result = systemInfoService;
+                globalStateMgr.getLocalMetastore().getDb("db1");
+                result = db1;
+                globalStateMgr.getGlobalTransactionMgr().getTxnState(db1, 1L);
+                result = new Exception("artificial failure");
+            }
+        };
+        TransactionStateSnapshot txnState = new TransactionStateSnapshot(TransactionStatus.UNKNOWN,
+                "can't get txn state, exception: artificial failure");
+        TxnStateDispatcher.DispatchResult expected = TxnStateDispatcher.DispatchResult.success(txnState);
+        PUpdateTransactionStateResponse response = new PUpdateTransactionStateResponse();
+        StatusPB statusPB = new StatusPB();
+        statusPB.setStatusCode(0);
+        response.setResults(Collections.singletonList(statusPB));
+        CompletableFuture<PUpdateTransactionStateResponse> future = new CompletableFuture<>();
+        future.complete(response);
+        backendService.addResponseFuture(future);
+        testDispatchBase("db1", 1, 1, expected, 0);
+    }
+
+    @Test
+    public void testReachMaxRetry() throws Exception {
+        new Expectations() {
+            {
+                globalStateMgr.getNodeMgr().getClusterInfo();
+                result = systemInfoService;
+            }
+        };
+        TxnStateDispatcher.DispatchResult expected = TxnStateDispatcher.DispatchResult.fail(
+                TxnStateDispatcher.DispatchStatus.RETRYABLE, "backend [10.0.0.2] does not alive");
+        testDispatchBase("db1", 1, 2, expected,
+                Config.merge_commit_txn_state_dispatch_retry_times);
+    }
+
+    private void testDispatchBase(String dbName, long txnId, long backendId,
+                                  TxnStateDispatcher.DispatchResult expected, int numRetry) throws Exception {
+        MockExecutor executor = new MockExecutor();
+        TxnStateDispatcher dispatcher = new TxnStateDispatcher(executor);
+        TaskExecuteListener taskExecuteListener = new TaskExecuteListener();
+        dispatcher.setTaskExecuteListener(taskExecuteListener);
+        dispatcher.submitTask(dbName, txnId, backendId);
+        executor.runOneTask();
+        assertNull(dispatcher.getPendingTask(txnId, backendId));
+        TaskExecuteResult result = taskExecuteListener.getResult(txnId, backendId);
+        assertNotNull(result);
+        assertTask(dbName, txnId, backendId, result.task);
+        assertEquals(expected.getStatus(), result.dispatchResult.getStatus());
+        assertTxnState(expected.getState(), result.dispatchResult.getState());
+        assertEquals(expected.getFailReason(), result.dispatchResult.getFailReason());
+        assertEquals(numRetry, result.numRetry);
+    }
+
+    private static void assertTask(String dbName, long txnId, long backendId, TxnStateDispatcher.Task actual) {
+        assertEquals(dbName, actual.getDbName());
+        assertEquals(txnId, actual.getTxnId());
+        assertEquals(backendId, actual.getBackendId());
+    }
+
+    private static void assertTxnState(TransactionStateSnapshot expected, TransactionStateSnapshot actual) {
+        if (expected == null) {
+            assertNull(actual);
+            return;
+        }
+        assertNotNull(actual);
+        assertEquals(expected.getStatus(), actual.getStatus());
+        assertEquals(expected.getReason(), actual.getReason());
+    }
+
+    private static class MockExecutor implements Executor {
+
+        private int numExecuteCalled = 0;
+        private int artificialRejectNum = 0;
+        private final LinkedList<Runnable> pendingTasks = new LinkedList<>();
+
+        @Override
+        public void execute(Runnable command) {
+            numExecuteCalled += 1;
+            if (artificialRejectNum > 0) {
+                artificialRejectNum -= 1;
+                throw new RejectedExecutionException("reject submit");
+            }
+            pendingTasks.add(command);
+        }
+
+        public void runOneTask() {
+            if (pendingTasks.isEmpty()) {
+                return;
+            }
+            Runnable task = pendingTasks.pollFirst();
+            task.run();
+        }
+
+        public void setArtificialRejectNum(int artificialRejectNum) {
+            this.artificialRejectNum = artificialRejectNum;
+        }
+
+        public int getArtificialRejectNum() {
+            return artificialRejectNum;
+        }
+
+        public int getNumExecuteCalled() {
+            return numExecuteCalled;
+        }
+
+        public int numPendingTasks() {
+            return pendingTasks.size();
+        }
+    }
+
+    private static class MockedPBackendService extends MockedBackend.MockPBackendService {
+
+        private LinkedList<Future<PUpdateTransactionStateResponse>> responseList = new LinkedList<>();
+
+        @Override
+        public Future<PUpdateTransactionStateResponse> updateTransactionState(PUpdateTransactionStateRequest request) {
+            if (responseList.isEmpty()) {
+                CompletableFuture<PUpdateTransactionStateResponse> future = new CompletableFuture<>();
+                future.completeExceptionally(new Exception("reponse not available"));
+                return future;
+            }
+            return responseList.pollFirst();
+        }
+
+        public void addResponseFuture(Future<PUpdateTransactionStateResponse> responseFuture) {
+            this.responseList.add(responseFuture);
+        }
+    }
+
+    private static class TaskExecuteResult {
+        TxnStateDispatcher.Task task;
+        TxnStateDispatcher.DispatchResult dispatchResult;
+        int numRetry;
+
+        public TaskExecuteResult(
+                TxnStateDispatcher.Task task, TxnStateDispatcher.DispatchResult dispatchResult, int numRetry) {
+            this.task = task;
+            this.dispatchResult = dispatchResult;
+            this.numRetry = numRetry;
+        }
+    }
+
+    private static class TaskExecuteListener implements TxnStateDispatcher.TaskExecuteListener {
+
+        private final Map<TxnStateDispatcher.TaskId, TaskExecuteResult> results = new HashMap<>();
+
+        @Override
+        public void onFinish(TxnStateDispatcher.Task task, TxnStateDispatcher.DispatchResult result, int numRetry) {
+            results.put(task.taskId, new TaskExecuteResult(task, result, numRetry));
+        }
+
+        public TaskExecuteResult getResult(long txnId, long backendId) {
+            return results.get(new TxnStateDispatcher.TaskId(txnId, backendId));
+        }
+    }
+}
diff --git a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoBackend.java b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoBackend.java
index af8fbe73efa05d..436ad43389df33 100644
--- a/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoBackend.java
+++ b/fe/fe-core/src/test/java/com/starrocks/pseudocluster/PseudoBackend.java
@@ -74,6 +74,8 @@
 import com.starrocks.proto.PUniqueId;
 import com.starrocks.proto.PUpdateFailPointStatusRequest;
 import com.starrocks.proto.PUpdateFailPointStatusResponse;
+import com.starrocks.proto.PUpdateTransactionStateRequest;
+import com.starrocks.proto.PUpdateTransactionStateResponse;
 import com.starrocks.proto.PublishLogVersionBatchRequest;
 import com.starrocks.proto.PublishLogVersionRequest;
 import com.starrocks.proto.PublishLogVersionResponse;
@@ -1096,6 +1098,11 @@ public Future<PListFailPointResponse> listFailPointAsync(PListFailPointRequest r
         public Future<PExecShortCircuitResult> execShortCircuit(PExecShortCircuitRequest request) {
             return null;
         }
+
+        @Override
+        public Future<PUpdateTransactionStateResponse> updateTransactionState(PUpdateTransactionStateRequest request) {
+            throw new org.apache.commons.lang.NotImplementedException("TODO");
+        }
     }
 
     public static class PseudoLakeService implements LakeService {
diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/MockedBackend.java b/fe/fe-core/src/test/java/com/starrocks/utframe/MockedBackend.java
index 9d8098682fa61b..e4f032625073f4 100644
--- a/fe/fe-core/src/test/java/com/starrocks/utframe/MockedBackend.java
+++ b/fe/fe-core/src/test/java/com/starrocks/utframe/MockedBackend.java
@@ -60,6 +60,8 @@
 import com.starrocks.proto.PTriggerProfileReportResult;
 import com.starrocks.proto.PUpdateFailPointStatusRequest;
 import com.starrocks.proto.PUpdateFailPointStatusResponse;
+import com.starrocks.proto.PUpdateTransactionStateRequest;
+import com.starrocks.proto.PUpdateTransactionStateResponse;
 import com.starrocks.proto.PublishLogVersionBatchRequest;
 import com.starrocks.proto.PublishLogVersionRequest;
 import com.starrocks.proto.PublishLogVersionResponse;
@@ -535,6 +537,11 @@ public Future<PFetchArrowSchemaResult> fetchArrowSchema(PFetchArrowSchemaRequest
         public Future<PProcessDictionaryCacheResult> processDictionaryCache(PProcessDictionaryCacheRequest request) {
             return null;
         }
+
+        @Override
+        public Future<PUpdateTransactionStateResponse> updateTransactionState(PUpdateTransactionStateRequest request) {
+            throw new NotImplementedException("TODO");
+        }
     }
 
     private static class MockLakeService implements LakeService {
diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto
index 7d3f3b030508f5..209e5d47db5af1 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -661,6 +661,15 @@ message PStreamLoadResponse {
     optional string json_result = 1;
 }
 
+message PUpdateTransactionStateRequest {
+    repeated TransactionStatePB states = 1;
+}
+
+// The update result for each transaction state
+message PUpdateTransactionStateResponse {
+    repeated StatusPB results = 1;
+}
+
 service PInternalService {
     rpc transmit_data(PTransmitDataParams) returns (PTransmitDataResult);
     rpc exec_plan_fragment(PExecPlanFragmentRequest) returns (PExecPlanFragmentResult);
@@ -704,6 +713,8 @@ service PInternalService {
     rpc process_dictionary_cache(starrocks.PProcessDictionaryCacheRequest) returns (starrocks.PProcessDictionaryCacheResult);
 
     rpc fetch_arrow_schema(PFetchArrowSchemaRequest) returns (PFetchArrowSchemaResult);
-    
+
     rpc stream_load(PStreamLoadRequest) returns (PStreamLoadResponse);
+
+    rpc update_transaction_state(PUpdateTransactionStateRequest) returns (PUpdateTransactionStateResponse);
 };
diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto
index b203dcc4910426..0d3305ec993374 100644
--- a/gensrc/proto/types.proto
+++ b/gensrc/proto/types.proto
@@ -205,3 +205,18 @@ message PersistentIndexSstableMetaPB {
     // sstables are ordered with the smaller version on the left.
     repeated PersistentIndexSstablePB sstables = 1;
 }
+
+enum TransactionStatusPB {
+    TRANS_UNKNOWN = 0;
+    TRANS_PREPARE = 1;
+    TRANS_COMMITTED = 2;
+    TRANS_VISIBLE = 3;
+    TRANS_ABORTED = 4;
+    TRANS_PREPARED = 5;
+}
+
+message TransactionStatePB {
+    optional int64 txn_id = 1;
+    optional TransactionStatusPB status = 2;
+    optional string reason = 3;
+}

From c653f83d17a931d636d6a1c33b38c447275dfb86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=B5=B5=E7=A9=BA=E4=BA=8B=E3=82=B9=E3=83=94=E3=83=AA?=
 =?UTF-8?q?=E3=83=83=E3=83=88?= <wanglichen@starrocks.com>
Date: Thu, 16 Jan 2025 09:56:37 +0800
Subject: [PATCH 09/71] [Doc] Add Trino Translate doc (#55107)

---
 .../sql-statements/TRANSLATE_TRINO.md         | 35 +++++++++++++++++++
 .../sql-statements/TRANSLATE_TRINO.md         | 35 +++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 docs/en/sql-reference/sql-statements/TRANSLATE_TRINO.md
 create mode 100644 docs/zh/sql-reference/sql-statements/TRANSLATE_TRINO.md

diff --git a/docs/en/sql-reference/sql-statements/TRANSLATE_TRINO.md b/docs/en/sql-reference/sql-statements/TRANSLATE_TRINO.md
new file mode 100644
index 00000000000000..22441c81e35821
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/TRANSLATE_TRINO.md
@@ -0,0 +1,35 @@
+---
+displayed_sidebar: docs
+---
+
+# Translate Trino SQL
+
+From v3.3.9, StarRocks supports translating Trino SQL statements into StarRocks SQL statements.
+
+## Syntax
+
+```SQL
+TRANSLATE TRINO <SELECT_statement>
+```
+
+## Parameters
+
+`SELECT_statement`: The Trino SQL statement you want to translate.
+
+## Return
+
+Returns the StarRocks SQL statement.
+
+## Examples
+
+```Plain
+mysql> TRANSLATE TRINO SELECT id, name, category FROM products WHERE name = 'Dell XPS 13'  AND category = "Electronics"  AND `price` > 500;
++---------------------------------------------------------------------------------------------------------------------------------------+
+| Translated SQL                                                                                                                        |
++---------------------------------------------------------------------------------------------------------------------------------------+
+| SELECT `id`, `name`, `category`
+FROM `products`
+WHERE ((`name` = 'Dell XPS 13') AND (`category` = 'Electronics')) AND (`price` > 500) |
++---------------------------------------------------------------------------------------------------------------------------------------+
+1 row in set (0.30 sec)
+```
diff --git a/docs/zh/sql-reference/sql-statements/TRANSLATE_TRINO.md b/docs/zh/sql-reference/sql-statements/TRANSLATE_TRINO.md
new file mode 100644
index 00000000000000..c6e6b7ded39089
--- /dev/null
+++ b/docs/zh/sql-reference/sql-statements/TRANSLATE_TRINO.md
@@ -0,0 +1,35 @@
+---
+displayed_sidebar: docs
+---
+
+# 翻译 Trino SQL
+
+自 v3.3.9 起，StarRocks 支持将 Trino SQL 语句翻译为 StarRocks SQL 语句。
+
+## 语法
+
+```SQL
+TRANSLATE TRINO <SELECT_statement>
+```
+
+## 参数说明
+
+`SELECT_statement`：需要翻译的 Trino SQL 语句。
+
+## 返回
+
+返回 StarRocks SQL 语句。
+
+## 示例
+
+```Plain
+mysql> TRANSLATE TRINO SELECT id, name, category FROM products WHERE name = 'Dell XPS 13'  AND category = "Electronics"  AND `price` > 500;
++---------------------------------------------------------------------------------------------------------------------------------------+
+| Translated SQL                                                                                                                        |
++---------------------------------------------------------------------------------------------------------------------------------------+
+| SELECT `id`, `name`, `category`
+FROM `products`
+WHERE ((`name` = 'Dell XPS 13') AND (`category` = 'Electronics')) AND (`price` > 500) |
++---------------------------------------------------------------------------------------------------------------------------------------+
+1 row in set (0.30 sec)
+```

From 1fa3cd01cf04f405d70e4b938c029ba80bcd4bf7 Mon Sep 17 00:00:00 2001
From: Seaven <seaven_7@qq.com>
Date: Thu, 16 Jan 2025 10:17:04 +0800
Subject: [PATCH 10/71] [Refactor] refactor pattern logical (#55048)

Signed-off-by: Seaven <seaven_7@qq.com>
---
 .../sql/optimizer/MaterializationContext.java |   4 +-
 .../starrocks/sql/optimizer/Optimizer.java    |  10 +-
 .../operator/pattern/AnyPattern.java          |  33 +++++
 .../operator/pattern/MultiJoinPattern.java    |  33 +++++
 .../operator/pattern/MultiLeafPattern.java    |  33 +++++
 .../operator/pattern/MultiOpPattern.java      |  62 +++++++++
 .../optimizer/operator/pattern/OpPattern.java |  46 +++++++
 .../optimizer/operator/pattern/Pattern.java   | 119 +++++-------------
 .../starrocks/sql/optimizer/rule/Binder.java  |  25 ++--
 .../starrocks/sql/optimizer/rule/RuleSet.java |  94 ++------------
 .../rule/transformation/CombinationRule.java  |  15 +--
 .../ExternalScanPartitionPruneRule.java       |  41 +++---
 .../transformation/MergeLimitDirectRule.java  |  62 ++++-----
 .../transformation/PruneEmptyScanRule.java    |  24 ++--
 .../PruneHDFSScanColumnRule.java              |  37 +++---
 .../transformation/PruneScanColumnRule.java   |  22 ++--
 .../PushDownLimitDirectRule.java              |  16 +--
 .../PushDownPredicateScanRule.java            |  63 ++++------
 ...hDownPredicateToExternalTableScanRule.java |  15 +--
 .../RewriteSimpleAggToHDFSScanRule.java       |  39 +++---
 .../transformation/SkewJoinOptimizeRule.java  |   7 ++
 .../sql/optimizer/task/RewriteTreeTask.java   |   4 +-
 .../sql/optimizer/task/TaskScheduler.java     |   3 +-
 .../connector/paimon/PaimonMetadataTest.java  |   2 +-
 .../PruneHDFSScanColumnRuleTest.java          |   6 +-
 .../transformation/PushDownScanRuleTest.java  |   2 +-
 .../com/starrocks/sql/plan/SkewJoinTest.java  |   4 +-
 27 files changed, 431 insertions(+), 390 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java

diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java
index 6c22ad99947f5b..7eabf32ca15e30 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializationContext.java
@@ -31,7 +31,7 @@
 import com.starrocks.sql.optimizer.operator.logical.LogicalAggregationOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
-import com.starrocks.sql.optimizer.operator.pattern.Pattern;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
 import com.starrocks.sql.optimizer.rule.transformation.materialization.MaterializedViewRewriter;
@@ -323,7 +323,7 @@ private static int getOperatorOrdering(OperatorType op) {
                 return 1;
             } else if (op == OperatorType.LOGICAL_JOIN) {
                 return 2;
-            } else if (Pattern.isScanOperator(op)) {
+            } else if (MultiOpPattern.ALL_SCAN_TYPES.contains(op)) {
                 return 3;
             } else {
                 return 4;
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
index 60a5634ed611ed..eaf325e43c288a 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
@@ -695,9 +695,7 @@ private OptExpression logicalRuleRewrite(
         ruleBasedMaterializedViewRewrite(tree, rootTaskContext, requiredColumns);
 
         // this rewrite rule should be after mv.
-        scheduler.rewriteIterative(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.HIVE_SCAN_NO_PROJECT);
-        scheduler.rewriteIterative(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.ICEBERG_SCAN_NO_PROJECT);
-        scheduler.rewriteIterative(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.FILE_SCAN_NO_PROJECT);
+        scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT);
 
         // NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle
         // select count(distinct c) from t group by a, b
@@ -842,16 +840,12 @@ private OptExpression pushDownAggregation(OptExpression tree, TaskContext rootTa
     }
 
     private void skewJoinOptimize(OptExpression tree, TaskContext rootTaskContext) {
-        SkewJoinOptimizeRule rule = new SkewJoinOptimizeRule();
         if (context.getSessionVariable().isEnableStatsToOptimizeSkewJoin()) {
             // merge projects before calculate statistics
             scheduler.rewriteOnce(tree, rootTaskContext, new MergeTwoProjectRule());
             Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext());
         }
-        if (scheduler.rewriteOnce(tree, rootTaskContext, rule)) {
-            // skew join generate new join and on predicate, need to push down join on expression to child project again
-            scheduler.rewriteOnce(tree, rootTaskContext, new PushDownJoinOnExpressionToChildProject());
-        }
+        scheduler.rewriteOnce(tree, rootTaskContext, new SkewJoinOptimizeRule());
     }
 
     private OptExpression pruneSubfield(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java
new file mode 100644
index 00000000000000..a3139cee6902a1
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/AnyPattern.java
@@ -0,0 +1,33 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer.operator.pattern;
+
+import com.starrocks.sql.optimizer.operator.OperatorType;
+
+public class AnyPattern extends OpPattern {
+    protected AnyPattern() {
+        super(OperatorType.PATTERN_LEAF);
+    }
+
+    @Override
+    public boolean isFixedPattern() {
+        return false;
+    }
+
+    @Override
+    protected boolean matchWithoutChild(OperatorType op) {
+        return true;
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java
new file mode 100644
index 00000000000000..119d7ff8f3d722
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiJoinPattern.java
@@ -0,0 +1,33 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer.operator.pattern;
+
+import com.starrocks.sql.optimizer.operator.OperatorType;
+
+public class MultiJoinPattern extends OpPattern {
+    protected MultiJoinPattern() {
+        super(OperatorType.PATTERN_MULTIJOIN);
+    }
+
+    @Override
+    public boolean isFixedPattern() {
+        return false;
+    }
+
+    @Override
+    protected boolean matchWithoutChild(OperatorType op) {
+        return op.equals(OperatorType.LOGICAL_JOIN) || MultiOpPattern.ALL_SCAN_TYPES.contains(op);
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java
new file mode 100644
index 00000000000000..59e58e9aec9f8d
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiLeafPattern.java
@@ -0,0 +1,33 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer.operator.pattern;
+
+import com.starrocks.sql.optimizer.operator.OperatorType;
+
+public class MultiLeafPattern extends OpPattern {
+    protected MultiLeafPattern() {
+        super(OperatorType.PATTERN_MULTI_LEAF);
+    }
+
+    @Override
+    public boolean isFixedPattern() {
+        return false;
+    }
+
+    @Override
+    protected boolean matchWithoutChild(OperatorType op) {
+        return true;
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java
new file mode 100644
index 00000000000000..19017b75456080
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/MultiOpPattern.java
@@ -0,0 +1,62 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer.operator.pattern;
+
+import com.google.common.collect.ImmutableSet;
+import com.starrocks.sql.optimizer.operator.OperatorType;
+
+import java.util.Set;
+
+public class MultiOpPattern extends Pattern {
+    public static final ImmutableSet<OperatorType> ALL_SCAN_TYPES = ImmutableSet.<OperatorType>builder()
+            .add(OperatorType.LOGICAL_OLAP_SCAN)
+            .add(OperatorType.LOGICAL_HIVE_SCAN)
+            .add(OperatorType.LOGICAL_ICEBERG_SCAN)
+            .add(OperatorType.LOGICAL_HUDI_SCAN)
+            .add(OperatorType.LOGICAL_FILE_SCAN)
+            .add(OperatorType.LOGICAL_SCHEMA_SCAN)
+            .add(OperatorType.LOGICAL_MYSQL_SCAN)
+            .add(OperatorType.LOGICAL_ES_SCAN)
+            .add(OperatorType.LOGICAL_META_SCAN)
+            .add(OperatorType.LOGICAL_JDBC_SCAN)
+            .add(OperatorType.LOGICAL_BINLOG_SCAN)
+            .add(OperatorType.LOGICAL_VIEW_SCAN)
+            .add(OperatorType.LOGICAL_PAIMON_SCAN)
+            .add(OperatorType.PATTERN_SCAN)
+            .build();
+
+    private final Set<OperatorType> ops;
+    protected MultiOpPattern(Set<OperatorType> ops) {
+        super();
+        this.ops = ops;
+    }
+
+    @Override
+    protected boolean matchWithoutChild(OperatorType op) {
+        return ops.contains(op);
+    }
+
+    public static Pattern ofAllScan() {
+        return of(ALL_SCAN_TYPES);
+    }
+
+    public static Pattern of(OperatorType... types) {
+        return new MultiOpPattern(Set.of(types));
+    }
+
+    public static Pattern of(Set<OperatorType> types) {
+        return new MultiOpPattern(types);
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java
new file mode 100644
index 00000000000000..626e5ba599fd3f
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/OpPattern.java
@@ -0,0 +1,46 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer.operator.pattern;
+
+import com.starrocks.sql.optimizer.operator.OperatorType;
+
+public class OpPattern extends Pattern {
+
+    protected final OperatorType opType;
+
+    protected OpPattern(OperatorType opType) {
+        super();
+        this.opType = opType;
+    }
+
+    public OperatorType getOpType() {
+        return opType;
+    }
+
+    @Override
+    public boolean is(OperatorType opType) {
+        return this.opType.equals(opType);
+    }
+
+    @Override
+    public boolean isFixedPattern() {
+        return true;
+    }
+
+    @Override
+    protected boolean matchWithoutChild(OperatorType op) {
+        return opType.equals(op);
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java
index 35921e0a046832..bc39cdf2b9350b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/pattern/Pattern.java
@@ -15,7 +15,6 @@
 package com.starrocks.sql.optimizer.operator.pattern;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.starrocks.sql.optimizer.GroupExpression;
 import com.starrocks.sql.optimizer.OptExpression;
@@ -23,47 +22,26 @@
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
 /**
  * Pattern is used in rules as a placeholder for group
  */
-public class Pattern {
-    public static final ImmutableSet<OperatorType> ALL_SCAN_TYPES = ImmutableSet.<OperatorType>builder()
-            .add(OperatorType.LOGICAL_OLAP_SCAN)
-            .add(OperatorType.LOGICAL_HIVE_SCAN)
-            .add(OperatorType.LOGICAL_ICEBERG_SCAN)
-            .add(OperatorType.LOGICAL_HUDI_SCAN)
-            .add(OperatorType.LOGICAL_FILE_SCAN)
-            .add(OperatorType.LOGICAL_SCHEMA_SCAN)
-            .add(OperatorType.LOGICAL_MYSQL_SCAN)
-            .add(OperatorType.LOGICAL_ES_SCAN)
-            .add(OperatorType.LOGICAL_META_SCAN)
-            .add(OperatorType.LOGICAL_JDBC_SCAN)
-            .add(OperatorType.LOGICAL_BINLOG_SCAN)
-            .add(OperatorType.LOGICAL_VIEW_SCAN)
-            .add(OperatorType.LOGICAL_PAIMON_SCAN)
-            .build();
+public abstract class Pattern {
+    private static final Map<OperatorType, Function<Void, Pattern>> PATTERN_MAP = Map.of(
+            OperatorType.PATTERN_LEAF, p -> new AnyPattern(),
+            OperatorType.PATTERN_MULTI_LEAF, p -> new MultiLeafPattern(),
+            OperatorType.PATTERN_SCAN, p -> MultiOpPattern.ofAllScan(),
+            OperatorType.PATTERN_MULTIJOIN, p -> new MultiJoinPattern()
+    );
 
-    private final OperatorType opType;
     private final List<Pattern> children;
 
-    protected Pattern(OperatorType opType) {
-        this.opType = opType;
+    protected Pattern() {
         this.children = Lists.newArrayList();
     }
 
-    public OperatorType getOpType() {
-        return opType;
-    }
-
-    public static Pattern create(OperatorType type, OperatorType... children) {
-        Pattern p = new Pattern(type);
-        for (OperatorType child : children) {
-            p.addChildren(new Pattern(child));
-        }
-        return p;
-    }
-
     public List<Pattern> children() {
         return children;
     }
@@ -73,89 +51,50 @@ public Pattern childAt(int i) {
     }
 
     public Pattern addChildren(Pattern... children) {
-        Preconditions.checkArgument(opType != OperatorType.PATTERN_MULTIJOIN,
-                "MULTI_JOIN cannot has children");
         this.children.addAll(Arrays.asList(children));
         return this;
     }
 
-    public boolean isPatternLeaf() {
-        return OperatorType.PATTERN_LEAF.equals(opType);
+    public boolean is(OperatorType opType) {
+        return false;
     }
 
-    public boolean isPatternMultiLeaf() {
-        return OperatorType.PATTERN_MULTI_LEAF.equals(opType);
+    public boolean isFixedPattern() {
+        return false;
     }
 
-    public boolean isPatternScan() {
-        return OperatorType.PATTERN_SCAN.equals(opType);
-    }
-
-    public boolean isPatternMultiJoin() {
-        return OperatorType.PATTERN_MULTIJOIN.equals(opType);
-    }
-
-    public static boolean isScanOperator(OperatorType operatorType) {
-        return ALL_SCAN_TYPES.contains(operatorType);
-    }
+    protected abstract boolean matchWithoutChild(OperatorType op);
 
     public boolean matchWithoutChild(GroupExpression expression) {
         if (expression == null) {
             return false;
         }
-
         if (expression.getInputs().size() < children.size()
-                && children.stream().noneMatch(p -> OperatorType.PATTERN_MULTI_LEAF.equals(p.getOpType()))) {
+                && children.stream().noneMatch(p -> p.is(OperatorType.PATTERN_MULTI_LEAF))) {
             return false;
         }
-
-        if (OperatorType.PATTERN_LEAF.equals(getOpType()) || OperatorType.PATTERN_MULTI_LEAF.equals(getOpType())) {
-            return true;
-        }
-
-        if (isPatternScan() && ALL_SCAN_TYPES.contains(expression.getOp().getOpType())) {
-            return true;
-        }
-
-        if (isPatternMultiJoin() && isMultiJoin(expression.getOp().getOpType())) {
-            return true;
-        }
-
-        return getOpType().equals(expression.getOp().getOpType());
+        return matchWithoutChild(expression.getOp().getOpType());
     }
 
     public boolean matchWithoutChild(OptExpression expression) {
-        if (expression == null) {
-            return false;
-        }
-
+        Preconditions.checkNotNull(expression);
         if (expression.getInputs().size() < this.children().size()
-                && children.stream().noneMatch(p -> OperatorType.PATTERN_MULTI_LEAF.equals(p.getOpType()))) {
+                && children.stream().noneMatch(p -> p.is(OperatorType.PATTERN_MULTI_LEAF))) {
             return false;
         }
-
-        if (OperatorType.PATTERN_LEAF.equals(getOpType()) || OperatorType.PATTERN_MULTI_LEAF.equals(getOpType())) {
-            return true;
-        }
-
-        if (isPatternScan() && ALL_SCAN_TYPES.contains(expression.getOp().getOpType())) {
-            return true;
-        }
-
-        if (isPatternMultiJoin() && isMultiJoin(expression.getOp().getOpType())) {
-            return true;
-        }
-
-        return getOpType().equals(expression.getOp().getOpType());
+        return matchWithoutChild(expression.getOp().getOpType());
     }
 
-    private boolean isMultiJoin(OperatorType operatorType) {
-        if (ALL_SCAN_TYPES.contains(operatorType)) {
-            return true;
-        } else if (operatorType.equals(OperatorType.LOGICAL_JOIN)) {
-            return true;
+    public static Pattern create(OperatorType type, OperatorType... children) {
+        Pattern p;
+        if (PATTERN_MAP.containsKey(type)) {
+            p = PATTERN_MAP.get(type).apply(null);
         } else {
-            return false;
+            p = new OpPattern(type);
+        }
+        for (OperatorType child : children) {
+            p.addChildren(create(child));
         }
+        return p;
     }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java
index 179eaba2ab819e..eaf34a5ceba8cf 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/Binder.java
@@ -23,6 +23,7 @@
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
 import com.starrocks.sql.optimizer.operator.OperatorType;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 
 import java.util.List;
@@ -30,7 +31,6 @@
 
 // Used to extract matched expression from GroupExpression
 public class Binder {
-    private final OptimizerContext optimizerContext;
     private final Pattern pattern;
     private final GroupExpression groupExpression;
     // binder status
@@ -57,7 +57,6 @@ public class Binder {
      */
     public Binder(OptimizerContext optimizerContext, Pattern pattern,
                   GroupExpression groupExpression, Stopwatch stopwatch) {
-        this.optimizerContext = optimizerContext;
         this.pattern = pattern;
         this.groupExpression = groupExpression;
         this.groupExpressionIndex = Lists.newArrayList(0);
@@ -65,9 +64,9 @@ public Binder(OptimizerContext optimizerContext, Pattern pattern,
         this.multiJoinBinder = new MultiJoinBinder(optimizerContext, stopwatch);
         // MULTI_JOIN is a special pattern which can contain children groups if the input group expression
         // is not a scan node.
-        this.isPatternWithoutChildren = pattern.isPatternMultiJoin()
-                ? Pattern.ALL_SCAN_TYPES.contains(groupExpression.getOp().getOpType())
-                : pattern.children().size() == 0;
+        this.isPatternWithoutChildren = pattern.is(OperatorType.PATTERN_MULTIJOIN)
+                ? MultiOpPattern.ALL_SCAN_TYPES.contains(groupExpression.getOp().getOpType())
+                : pattern.children().isEmpty();
     }
 
     /*
@@ -108,12 +107,11 @@ public OptExpression next() {
         return expression;
     }
 
-
     /**
      * Pattern tree match groupExpression tree
      */
     private OptExpression match(Pattern pattern, GroupExpression groupExpression) {
-        if (pattern.isPatternMultiJoin()) {
+        if (pattern.is(OperatorType.PATTERN_MULTIJOIN)) {
             return multiJoinBinder.match(groupExpression);
         }
 
@@ -142,7 +140,7 @@ private OptExpression match(Pattern pattern, GroupExpression groupExpression) {
                 resultInputs.add(opt);
             }
 
-            if (!(childPattern.isPatternMultiLeaf() &&
+            if (!(childPattern.is(OperatorType.PATTERN_MULTI_LEAF) &&
                     geSize - groupExpressionIndex > patternSize - patternIndex)) {
                 patternIndex++;
             }
@@ -165,7 +163,7 @@ private void trace() {
      */
     private GroupExpression extractGroupExpression(Pattern pattern, Group group) {
         final int valueIndex = groupExpressionIndex.get(groupTraceKey);
-        if (pattern.isPatternLeaf() || pattern.isPatternMultiLeaf()) {
+        if (pattern.is(OperatorType.PATTERN_LEAF) || pattern.is(OperatorType.PATTERN_MULTI_LEAF)) {
             if (valueIndex > 0) {
                 groupExpressionIndex.remove(groupTraceKey);
                 return null;
@@ -190,7 +188,6 @@ private GroupExpression extractGroupExpression(Pattern pattern, Group group) {
      * binding state and check the expression at the same time. But MULTI_JOIN could enumerate the GE without any check
      */
     private class MultiJoinBinder {
-        private final SessionVariable sessionVariable;
         // Stopwatch to void infinite loop
         private final Stopwatch watch;
         // Time limit for the entire optimization
@@ -200,7 +197,7 @@ private class MultiJoinBinder {
         private long loopCount = 0;
 
         public MultiJoinBinder(OptimizerContext optimizerContext, Stopwatch stopwatch) {
-            this.sessionVariable = optimizerContext.getSessionVariable();
+            SessionVariable sessionVariable = optimizerContext.getSessionVariable();
             this.watch = stopwatch;
             this.timeLimit = Math.min(sessionVariable.getOptimizerMaterializedViewTimeLimitMillis(),
                     sessionVariable.getOptimizerExecuteTimeout());
@@ -276,7 +273,7 @@ private GroupExpression extractGroupExpression(Group group) {
             }
 
             // shortcut for no child group expression
-            if (valueIndex > 0 && Pattern.ALL_SCAN_TYPES.contains(next.getOp().getOpType())) {
+            if (valueIndex > 0 && MultiOpPattern.ALL_SCAN_TYPES.contains(next.getOp().getOpType())) {
                 groupExpressionIndex.remove(groupTraceKey);
                 return null;
             }
@@ -307,7 +304,8 @@ private GroupExpression extractGroupExpression(Group group) {
 
         private boolean isMultiJoinOp(GroupExpression ge) {
             OperatorType operatorType = ge.getOp().getOpType();
-            return operatorType.equals(OperatorType.LOGICAL_JOIN) || Pattern.ALL_SCAN_TYPES.contains(operatorType);
+            return operatorType.equals(OperatorType.LOGICAL_JOIN) ||
+                    MultiOpPattern.ALL_SCAN_TYPES.contains(operatorType);
         }
 
         private boolean isMultiJoin(GroupExpression ge) {
@@ -342,6 +340,7 @@ private boolean isMultiJoinRecursive(GroupExpression ge) {
 
         /**
          * Check Group's logical expressions except the first has already been rewritten by mv rules.
+         *
          * @param g : Group to check whether it has been rewritten by mv rules.
          * @return : true if the Group has GroupExpression which is rewritten by mv rules.
          */
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java
index 99eaf59e7b63fc..b5887591b6afe5 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/RuleSet.java
@@ -222,47 +222,14 @@ public class RuleSet {
             new PushDownLimitCTEAnchor(),
             new PushDownLimitUnionRule(),
             new MergeLimitWithLimitRule(),
-            PushDownLimitDirectRule.PROJECT,
-            PushDownLimitDirectRule.ASSERT_ONE_ROW,
-            PushDownLimitDirectRule.CTE_CONSUME,
-            MergeLimitDirectRule.AGGREGATE,
-            MergeLimitDirectRule.OLAP_SCAN,
-            MergeLimitDirectRule.VIEW_SCAN,
-            MergeLimitDirectRule.HIVE_SCAN,
-            MergeLimitDirectRule.ICEBERG_SCAN,
-            MergeLimitDirectRule.HUDI_SCAN,
-            MergeLimitDirectRule.DELTALAKE_SCAN,
-            MergeLimitDirectRule.FILE_SCAN,
-            MergeLimitDirectRule.PAIMON_SCAN,
-            MergeLimitDirectRule.ODPS_SCAN,
-            MergeLimitDirectRule.KUDU_SCAN,
-            MergeLimitDirectRule.SCHEMA_SCAN,
-            MergeLimitDirectRule.MYSQL_SCAN,
-            MergeLimitDirectRule.ES_SCAN,
-            MergeLimitDirectRule.JDBC_SCAN,
-            MergeLimitDirectRule.ICEBERG_METADATA_SCAN,
-            MergeLimitDirectRule.WINDOW,
-            MergeLimitDirectRule.INTERSECT,
-            MergeLimitDirectRule.EXCEPT,
-            MergeLimitDirectRule.VALUES,
-            MergeLimitDirectRule.FILTER,
-            MergeLimitDirectRule.CTE_CONSUMER,
-            MergeLimitDirectRule.TABLE_FUNCTION,
-            MergeLimitDirectRule.TABLE_FUNCTION_TABLE_SCAN
+            new PushDownLimitDirectRule(),
+            new MergeLimitDirectRule()
     ));
 
     public static final Rule PARTITION_PRUNE_RULES = new CombinationRule(RuleType.GP_PARTITION_PRUNE, ImmutableList.of(
             new PartitionPruneRule(),
             new DistributionPruneRule(),
-            ExternalScanPartitionPruneRule.HIVE_SCAN,
-            ExternalScanPartitionPruneRule.HUDI_SCAN,
-            ExternalScanPartitionPruneRule.ICEBERG_SCAN,
-            ExternalScanPartitionPruneRule.DELTALAKE_SCAN,
-            ExternalScanPartitionPruneRule.FILE_SCAN,
-            ExternalScanPartitionPruneRule.ES_SCAN,
-            ExternalScanPartitionPruneRule.PAIMON_SCAN,
-            ExternalScanPartitionPruneRule.ODPS_SCAN,
-            ExternalScanPartitionPruneRule.KUDU_SCAN,
+            new ExternalScanPartitionPruneRule(),
             new LimitPruneTabletsRule()
     ));
 
@@ -271,22 +238,8 @@ public class RuleSet {
     ));
 
     public static final Rule PRUNE_COLUMNS_RULES = new CombinationRule(RuleType.GP_PRUNE_COLUMNS, ImmutableList.of(
-            PruneScanColumnRule.OLAP_SCAN,
-            PruneScanColumnRule.SCHEMA_SCAN,
-            PruneScanColumnRule.MYSQL_SCAN,
-            PruneScanColumnRule.ES_SCAN,
-            PruneHDFSScanColumnRule.HIVE_SCAN,
-            PruneHDFSScanColumnRule.ICEBERG_SCAN,
-            PruneHDFSScanColumnRule.DELTALAKE_SCAN,
-            PruneHDFSScanColumnRule.FILE_SCAN,
-            PruneHDFSScanColumnRule.HUDI_SCAN,
-            PruneHDFSScanColumnRule.TABLE_FUNCTION_TABLE_SCAN,
-            PruneHDFSScanColumnRule.ICEBERG_METADATA_SCAN,
-            PruneHDFSScanColumnRule.PAIMON_SCAN,
-            PruneHDFSScanColumnRule.ODPS_SCAN,
-            PruneScanColumnRule.KUDU_SCAN,
-            PruneScanColumnRule.JDBC_SCAN,
-            PruneScanColumnRule.BINLOG_SCAN,
+            new PruneScanColumnRule(),
+            new PruneHDFSScanColumnRule(),
             new PruneProjectColumnsRule(),
             new PruneFilterColumnsRule(),
             new PruneUKFKGroupByKeysRule(), // Put this before PruneAggregateColumnsRule
@@ -309,22 +262,7 @@ public class RuleSet {
                     new CastToEmptyRule(),
                     new PruneTrueFilterRule(),
                     new PushDownPredicateCTEAnchor(),
-                    PushDownPredicateScanRule.OLAP_SCAN,
-                    PushDownPredicateScanRule.HIVE_SCAN,
-                    PushDownPredicateScanRule.ICEBERG_SCAN,
-                    PushDownPredicateScanRule.HUDI_SCAN,
-                    PushDownPredicateScanRule.DELTALAKE_SCAN,
-                    PushDownPredicateScanRule.FILE_SCAN,
-                    PushDownPredicateScanRule.PAIMON_SCAN,
-                    PushDownPredicateScanRule.ICEBERG_METADATA_SCAN,
-                    PushDownPredicateScanRule.ICEBERG_EQUALITY_DELETE_SCAN,
-                    PushDownPredicateScanRule.KUDU_SCAN,
-                    PushDownPredicateScanRule.SCHEMA_SCAN,
-                    PushDownPredicateScanRule.ES_SCAN,
-                    PushDownPredicateScanRule.META_SCAN,
-                    PushDownPredicateScanRule.BINLOG_SCAN,
-                    PushDownPredicateScanRule.TABLE_FUNCTION_TABLE_SCAN,
-                    PushDownPredicateScanRule.VIEW_SCAN,
+                    new PushDownPredicateScanRule(),
                     new PushDownPredicateAggRule(),
                     new PushDownPredicateWindowRule(),
                     new PushDownPredicateJoinRule(),
@@ -336,9 +274,7 @@ public class RuleSet {
                     new PushDownPredicateTableFunctionRule(),
                     new PushDownPredicateRepeatRule(),
 
-                    PushDownPredicateToExternalTableScanRule.MYSQL_SCAN,
-                    PushDownPredicateToExternalTableScanRule.JDBC_SCAN,
-                    PushDownPredicateToExternalTableScanRule.ODPS_SCAN,
+                    new PushDownPredicateToExternalTableScanRule(),
                     new MergeTwoFiltersRule(),
                     new PushDownPredicateCTEConsumeRule()
             ));
@@ -439,13 +375,7 @@ public class RuleSet {
 
     public static final Rule PRUNE_EMPTY_OPERATOR_RULES =
             new CombinationRule(RuleType.GP_PRUNE_EMPTY_OPERATOR, ImmutableList.of(
-                    PruneEmptyScanRule.OLAP_SCAN,
-                    PruneEmptyScanRule.HIVE_SCAN,
-                    PruneEmptyScanRule.HUDI_SCAN,
-                    PruneEmptyScanRule.ICEBERG_SCAN,
-                    PruneEmptyScanRule.PAIMON_SCAN,
-                    PruneEmptyScanRule.ODPS_SCAN,
-                    PruneEmptyScanRule.KUDU_SCAN,
+                    new PruneEmptyScanRule(),
                     PruneEmptyJoinRule.JOIN_LEFT_EMPTY,
                     PruneEmptyJoinRule.JOIN_RIGHT_EMPTY,
                     new PruneEmptyDirectRule(),
@@ -459,10 +389,10 @@ public class RuleSet {
             new CombinationRule(RuleType.GP_SHORT_CIRCUIT_SET, ImmutableList.of(
                     new PruneTrueFilterRule(),
                     new PushDownPredicateProjectRule(),
-                    PushDownPredicateScanRule.OLAP_SCAN,
+                    new PushDownPredicateScanRule(),
                     new CastToEmptyRule(),
                     new PruneProjectColumnsRule(),
-                    PruneScanColumnRule.OLAP_SCAN,
+                    new PruneScanColumnRule(),
                     new PruneProjectEmptyRule(),
                     new MergeTwoProjectRule(),
                     new PruneProjectRule(),
@@ -486,9 +416,7 @@ public class RuleSet {
                     new PushDownAggToMetaScanRule(),
                     new PushDownFlatJsonMetaToMetaScanRule(),
                     new RewriteSimpleAggToMetaScanRule(),
-                    RewriteSimpleAggToHDFSScanRule.FILE_SCAN,
-                    RewriteSimpleAggToHDFSScanRule.HIVE_SCAN,
-                    RewriteSimpleAggToHDFSScanRule.ICEBERG_SCAN,
+                    new RewriteSimpleAggToHDFSScanRule(),
                     new MinMaxCountOptOnScanRule()
             ));
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java
index 74eba698a47f61..460b16e0ed64a3 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/CombinationRule.java
@@ -14,37 +14,38 @@
 
 package com.starrocks.sql.optimizer.rule.transformation;
 
+import com.google.common.collect.Sets;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
 import com.starrocks.sql.optimizer.operator.OperatorType;
+import com.starrocks.sql.optimizer.operator.pattern.OpPattern;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 import com.starrocks.sql.optimizer.rule.Rule;
 import com.starrocks.sql.optimizer.rule.RuleType;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 public class CombinationRule extends TransformationRule {
     private final List<Rule> rules;
-    private int patternHash = 0;
+    private final Set<OperatorType> ops = Sets.newHashSet();
 
     public CombinationRule(RuleType ruleType, List<Rule> rules) {
         super(ruleType, Pattern.create(OperatorType.PATTERN_LEAF));
         this.rules = rules;
 
-        if (rules.stream().anyMatch(rule -> rule.getPattern().getOpType().ordinal() > OperatorType.PATTERN.ordinal())) {
-            patternHash = Integer.MAX_VALUE;
-        } else {
+        if (rules.stream().allMatch(rule -> rule.getPattern().isFixedPattern())) {
             for (Rule rule : rules) {
-                OperatorType type = rule.getPattern().getOpType();
-                patternHash = patternHash | type.hashCode();
+                OperatorType type = ((OpPattern) rule.getPattern()).getOpType();
+                ops.add(type);
             }
         }
     }
 
     @Override
     public boolean check(OptExpression input, OptimizerContext context) {
-        return (patternHash & input.getOp().getOpType().hashCode()) != 0;
+        return ops.isEmpty() || ops.contains(input.getOp().getOpType());
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java
index b5a1570322c807..994fb17107298e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/ExternalScanPartitionPruneRule.java
@@ -15,16 +15,15 @@
 
 package com.starrocks.sql.optimizer.rule.transformation;
 
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
-import com.starrocks.sql.optimizer.operator.pattern.Pattern;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.rewrite.OptExternalPartitionPruner;
 import com.starrocks.sql.optimizer.rule.RuleType;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 
 import java.util.Collections;
 import java.util.List;
@@ -32,29 +31,21 @@
 import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED;
 
 public class ExternalScanPartitionPruneRule extends TransformationRule {
-    private static final Logger LOG = LogManager.getLogger(ExternalScanPartitionPruneRule.class);
 
-    public static final ExternalScanPartitionPruneRule HIVE_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_HIVE_SCAN);
-    public static final ExternalScanPartitionPruneRule HUDI_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_HUDI_SCAN);
-    public static final ExternalScanPartitionPruneRule ICEBERG_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_ICEBERG_SCAN);
-    public static final ExternalScanPartitionPruneRule DELTALAKE_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_DELTALAKE_SCAN);
-    public static final ExternalScanPartitionPruneRule FILE_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_FILE_SCAN);
-    public static final ExternalScanPartitionPruneRule ES_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_ES_SCAN);
-    public static final ExternalScanPartitionPruneRule PAIMON_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_PAIMON_SCAN);
-    public static final ExternalScanPartitionPruneRule ODPS_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_ODPS_SCAN);
-    public static final ExternalScanPartitionPruneRule KUDU_SCAN =
-            new ExternalScanPartitionPruneRule(OperatorType.LOGICAL_KUDU_SCAN);
-
-    public ExternalScanPartitionPruneRule(OperatorType logicalOperatorType) {
-        super(RuleType.TF_PARTITION_PRUNE, Pattern.create(logicalOperatorType));
+    private static final ImmutableSet<OperatorType> SUPPORT = ImmutableSet.of(
+            OperatorType.LOGICAL_HIVE_SCAN,
+            OperatorType.LOGICAL_HUDI_SCAN,
+            OperatorType.LOGICAL_ICEBERG_SCAN,
+            OperatorType.LOGICAL_DELTALAKE_SCAN,
+            OperatorType.LOGICAL_FILE_SCAN,
+            OperatorType.LOGICAL_ES_SCAN,
+            OperatorType.LOGICAL_PAIMON_SCAN,
+            OperatorType.LOGICAL_ODPS_SCAN,
+            OperatorType.LOGICAL_KUDU_SCAN
+    );
+
+    public ExternalScanPartitionPruneRule() {
+        super(RuleType.TF_PARTITION_PRUNE, MultiOpPattern.of(SUPPORT));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java
index 0c39a64ab1d418..cbbbcaba6a40ce 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitDirectRule.java
@@ -16,50 +16,52 @@
 package com.starrocks.sql.optimizer.rule.transformation;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalLimitOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOperator;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 import com.starrocks.sql.optimizer.rule.RuleType;
 
 import java.util.List;
+import java.util.Set;
 
 public class MergeLimitDirectRule extends TransformationRule {
-    public static final MergeLimitDirectRule AGGREGATE = new MergeLimitDirectRule(OperatorType.LOGICAL_AGGR);
-    public static final MergeLimitDirectRule OLAP_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_OLAP_SCAN);
-    public static final MergeLimitDirectRule VIEW_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_VIEW_SCAN);
-    public static final MergeLimitDirectRule HIVE_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_HIVE_SCAN);
-    public static final MergeLimitDirectRule ICEBERG_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_ICEBERG_SCAN);
-    public static final MergeLimitDirectRule HUDI_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_HUDI_SCAN);
-    public static final MergeLimitDirectRule DELTALAKE_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_DELTALAKE_SCAN);
-    public static final MergeLimitDirectRule FILE_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_FILE_SCAN);
-    public static final MergeLimitDirectRule PAIMON_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_PAIMON_SCAN);
-    public static final MergeLimitDirectRule ODPS_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_ODPS_SCAN);
-    public static final MergeLimitDirectRule KUDU_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_KUDU_SCAN);
-    public static final MergeLimitDirectRule SCHEMA_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_SCHEMA_SCAN);
-    public static final MergeLimitDirectRule MYSQL_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_MYSQL_SCAN);
-    public static final MergeLimitDirectRule ES_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_ES_SCAN);
-    public static final MergeLimitDirectRule JDBC_SCAN = new MergeLimitDirectRule(OperatorType.LOGICAL_JDBC_SCAN);
-    public static final MergeLimitDirectRule ICEBERG_METADATA_SCAN = new MergeLimitDirectRule(
-            OperatorType.LOGICAL_ICEBERG_METADATA_SCAN);
-    public static final MergeLimitDirectRule WINDOW = new MergeLimitDirectRule(OperatorType.LOGICAL_WINDOW);
-    public static final MergeLimitDirectRule INTERSECT = new MergeLimitDirectRule(OperatorType.LOGICAL_INTERSECT);
-    public static final MergeLimitDirectRule EXCEPT = new MergeLimitDirectRule(OperatorType.LOGICAL_EXCEPT);
-    public static final MergeLimitDirectRule VALUES = new MergeLimitDirectRule(OperatorType.LOGICAL_VALUES);
-    public static final MergeLimitDirectRule FILTER = new MergeLimitDirectRule(OperatorType.LOGICAL_FILTER);
-    public static final MergeLimitDirectRule TABLE_FUNCTION =
-            new MergeLimitDirectRule(OperatorType.LOGICAL_TABLE_FUNCTION);
-    public static final MergeLimitDirectRule TABLE_FUNCTION_TABLE_SCAN =
-            new MergeLimitDirectRule(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN);
-    public static final MergeLimitDirectRule CTE_CONSUMER =
-            new MergeLimitDirectRule(OperatorType.LOGICAL_CTE_CONSUME);
+    private static final Set<OperatorType> SUPPORT_OPERATOR = ImmutableSet.<OperatorType>builder()
+            .add(OperatorType.LOGICAL_OLAP_SCAN)
+            .add(OperatorType.LOGICAL_VIEW_SCAN)
+            .add(OperatorType.LOGICAL_HIVE_SCAN)
+            .add(OperatorType.LOGICAL_ICEBERG_SCAN)
+            .add(OperatorType.LOGICAL_HUDI_SCAN)
+            .add(OperatorType.LOGICAL_DELTALAKE_SCAN)
+            .add(OperatorType.LOGICAL_FILE_SCAN)
+            .add(OperatorType.LOGICAL_PAIMON_SCAN)
+            .add(OperatorType.LOGICAL_ODPS_SCAN)
+            .add(OperatorType.LOGICAL_KUDU_SCAN)
+            .add(OperatorType.LOGICAL_SCHEMA_SCAN)
+            .add(OperatorType.LOGICAL_MYSQL_SCAN)
+            .add(OperatorType.LOGICAL_ES_SCAN)
+            .add(OperatorType.LOGICAL_JDBC_SCAN)
+            .add(OperatorType.LOGICAL_ICEBERG_METADATA_SCAN)
+            .add(OperatorType.LOGICAL_AGGR)
+            .add(OperatorType.LOGICAL_WINDOW)
+            .add(OperatorType.LOGICAL_INTERSECT)
+            .add(OperatorType.LOGICAL_EXCEPT)
+            .add(OperatorType.LOGICAL_VALUES)
+            .add(OperatorType.LOGICAL_FILTER)
+            .add(OperatorType.LOGICAL_TABLE_FUNCTION)
+            .add(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN)
+            .add(OperatorType.LOGICAL_CTE_CONSUME)
+            .build();
 
-    private MergeLimitDirectRule(OperatorType logicalOperatorType) {
+    public MergeLimitDirectRule() {
         super(RuleType.TF_MERGE_LIMIT_DIRECT, Pattern.create(OperatorType.LOGICAL_LIMIT)
-                .addChildren(Pattern.create(logicalOperatorType, OperatorType.PATTERN_MULTI_LEAF)));
+                .addChildren(MultiOpPattern.of(SUPPORT_OPERATOR)
+                        .addChildren(Pattern.create(OperatorType.PATTERN_MULTI_LEAF))));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java
index ef11e121f516a2..f909a95d09c6d5 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneEmptyScanRule.java
@@ -21,23 +21,27 @@
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalValuesOperator;
-import com.starrocks.sql.optimizer.operator.pattern.Pattern;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.rule.RuleType;
 
 import java.util.List;
+import java.util.Set;
 
 // transform empty scan to empty values
 public class PruneEmptyScanRule extends TransformationRule {
-    public static final PruneEmptyScanRule OLAP_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_OLAP_SCAN);
-    public static final PruneEmptyScanRule HIVE_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_HIVE_SCAN);
-    public static final PruneEmptyScanRule HUDI_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_HUDI_SCAN);
-    public static final PruneEmptyScanRule ICEBERG_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_ICEBERG_SCAN);
-    public static final PruneEmptyScanRule PAIMON_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_PAIMON_SCAN);
-    public static final PruneEmptyScanRule ODPS_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_ODPS_SCAN);
-    public static final PruneEmptyScanRule KUDU_SCAN = new PruneEmptyScanRule(OperatorType.LOGICAL_KUDU_SCAN);
-    private PruneEmptyScanRule(OperatorType logicalOperatorType) {
-        super(RuleType.TF_PRUNE_EMPTY_SCAN, Pattern.create(logicalOperatorType));
+    private static final Set<OperatorType> SUPPORTED = Set.of(
+            OperatorType.LOGICAL_OLAP_SCAN,
+            OperatorType.LOGICAL_HIVE_SCAN,
+            OperatorType.LOGICAL_HUDI_SCAN,
+            OperatorType.LOGICAL_ICEBERG_SCAN,
+            OperatorType.LOGICAL_PAIMON_SCAN,
+            OperatorType.LOGICAL_ODPS_SCAN,
+            OperatorType.LOGICAL_KUDU_SCAN
+    );
+
+    public PruneEmptyScanRule() {
+        super(RuleType.TF_PRUNE_EMPTY_SCAN, MultiOpPattern.of(SUPPORTED));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java
index 8a896b31f70097..0c20071d9b6826 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRule.java
@@ -29,7 +29,7 @@
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
-import com.starrocks.sql.optimizer.operator.pattern.Pattern;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
 import com.starrocks.sql.optimizer.rule.RuleType;
@@ -45,27 +45,20 @@
 import static java.util.function.UnaryOperator.identity;
 
 public class PruneHDFSScanColumnRule extends TransformationRule {
-    public static final PruneHDFSScanColumnRule HIVE_SCAN = new PruneHDFSScanColumnRule(OperatorType.LOGICAL_HIVE_SCAN);
-    public static final PruneHDFSScanColumnRule ICEBERG_SCAN =
-            new PruneHDFSScanColumnRule(OperatorType.LOGICAL_ICEBERG_SCAN);
-    public static final PruneHDFSScanColumnRule HUDI_SCAN = new PruneHDFSScanColumnRule(OperatorType.LOGICAL_HUDI_SCAN);
-    public static final PruneHDFSScanColumnRule DELTALAKE_SCAN =
-            new PruneHDFSScanColumnRule(OperatorType.LOGICAL_DELTALAKE_SCAN);
-    public static final PruneHDFSScanColumnRule FILE_SCAN =
-            new PruneHDFSScanColumnRule(OperatorType.LOGICAL_FILE_SCAN);
-    public static final PruneHDFSScanColumnRule PAIMON_SCAN =
-            new PruneHDFSScanColumnRule(OperatorType.LOGICAL_PAIMON_SCAN);
-    public static final PruneHDFSScanColumnRule ODPS_SCAN =
-            new PruneHDFSScanColumnRule(OperatorType.LOGICAL_ODPS_SCAN);
-
-    public static final PruneHDFSScanColumnRule TABLE_FUNCTION_TABLE_SCAN =
-            new PruneHDFSScanColumnRule(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN);
-
-    public static final PruneHDFSScanColumnRule ICEBERG_METADATA_SCAN =
-            new PruneHDFSScanColumnRule(OperatorType.LOGICAL_ICEBERG_METADATA_SCAN);
-
-    public PruneHDFSScanColumnRule(OperatorType logicalOperatorType) {
-        super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, Pattern.create(logicalOperatorType));
+    private static final Set<OperatorType> SUPPORTED = Set.of(
+            OperatorType.LOGICAL_HIVE_SCAN,
+            OperatorType.LOGICAL_ICEBERG_SCAN,
+            OperatorType.LOGICAL_HUDI_SCAN,
+            OperatorType.LOGICAL_DELTALAKE_SCAN,
+            OperatorType.LOGICAL_FILE_SCAN,
+            OperatorType.LOGICAL_PAIMON_SCAN,
+            OperatorType.LOGICAL_ODPS_SCAN,
+            OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN,
+            OperatorType.LOGICAL_ICEBERG_METADATA_SCAN
+    );
+
+    public PruneHDFSScanColumnRule() {
+        super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, MultiOpPattern.of(SUPPORTED));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java
index b10ec6cb245a24..b872cf18184a0b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneScanColumnRule.java
@@ -25,7 +25,7 @@
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
-import com.starrocks.sql.optimizer.operator.pattern.Pattern;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
 import com.starrocks.sql.optimizer.rule.RuleType;
@@ -40,16 +40,18 @@
 import static java.util.function.Function.identity;
 
 public class PruneScanColumnRule extends TransformationRule {
-    public static final PruneScanColumnRule OLAP_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_OLAP_SCAN);
-    public static final PruneScanColumnRule SCHEMA_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_SCHEMA_SCAN);
-    public static final PruneScanColumnRule MYSQL_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_MYSQL_SCAN);
-    public static final PruneScanColumnRule ES_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_ES_SCAN);
-    public static final PruneScanColumnRule JDBC_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_JDBC_SCAN);
-    public static final PruneScanColumnRule BINLOG_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_BINLOG_SCAN);
-    public static final PruneScanColumnRule KUDU_SCAN = new PruneScanColumnRule(OperatorType.LOGICAL_KUDU_SCAN);
+    private static final Set<OperatorType> SUPPORTED = Set.of(
+            OperatorType.LOGICAL_OLAP_SCAN,
+            OperatorType.LOGICAL_SCHEMA_SCAN,
+            OperatorType.LOGICAL_MYSQL_SCAN,
+            OperatorType.LOGICAL_ES_SCAN,
+            OperatorType.LOGICAL_JDBC_SCAN,
+            OperatorType.LOGICAL_BINLOG_SCAN,
+            OperatorType.LOGICAL_KUDU_SCAN
+    );
 
-    public PruneScanColumnRule(OperatorType logicalOperatorType) {
-        super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, Pattern.create(logicalOperatorType));
+    public PruneScanColumnRule() {
+        super(RuleType.TF_PRUNE_OLAP_SCAN_COLUMNS, MultiOpPattern.of(SUPPORTED));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java
index 853fe6c286c632..76f91709e77da3 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownLimitDirectRule.java
@@ -22,21 +22,23 @@
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalLimitOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOperator;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 import com.starrocks.sql.optimizer.rule.RuleType;
 
 import java.util.List;
+import java.util.Set;
 
 public class PushDownLimitDirectRule extends TransformationRule {
-    public static final PushDownLimitDirectRule PROJECT = new PushDownLimitDirectRule(OperatorType.LOGICAL_PROJECT);
-    public static final PushDownLimitDirectRule ASSERT_ONE_ROW =
-            new PushDownLimitDirectRule(OperatorType.LOGICAL_ASSERT_ONE_ROW);
-    public static final PushDownLimitDirectRule CTE_CONSUME =
-            new PushDownLimitDirectRule(OperatorType.LOGICAL_CTE_CONSUME);
+    private static final Set<OperatorType> SUPPORTED_OPERATOR = Set.of(
+            OperatorType.LOGICAL_PROJECT,
+            OperatorType.LOGICAL_ASSERT_ONE_ROW,
+            OperatorType.LOGICAL_CTE_CONSUME
+    );
 
-    public PushDownLimitDirectRule(OperatorType logicalOperatorType) {
+    public PushDownLimitDirectRule() {
         super(RuleType.TF_PUSH_DOWN_LIMIT, Pattern.create(OperatorType.LOGICAL_LIMIT)
-                .addChildren(Pattern.create(logicalOperatorType, OperatorType.PATTERN_LEAF)));
+                .addChildren(MultiOpPattern.of(SUPPORTED_OPERATOR)));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java
index d699d810f573b8..4373e2d38b528b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateScanRule.java
@@ -15,6 +15,7 @@
 package com.starrocks.sql.optimizer.rule.transformation;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
@@ -25,6 +26,7 @@
 import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
@@ -38,44 +40,28 @@
 import java.util.stream.Collectors;
 
 public class PushDownPredicateScanRule extends TransformationRule {
-    public static final PushDownPredicateScanRule OLAP_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_OLAP_SCAN);
-    public static final PushDownPredicateScanRule HIVE_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_HIVE_SCAN);
-    public static final PushDownPredicateScanRule ICEBERG_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_ICEBERG_SCAN);
-    public static final PushDownPredicateScanRule HUDI_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_HUDI_SCAN);
-    public static final PushDownPredicateScanRule DELTALAKE_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_DELTALAKE_SCAN);
-    public static final PushDownPredicateScanRule FILE_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_FILE_SCAN);
-    public static final PushDownPredicateScanRule PAIMON_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_PAIMON_SCAN);
-    public static final PushDownPredicateScanRule ICEBERG_METADATA_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_ICEBERG_METADATA_SCAN);
+    private static final ImmutableSet<OperatorType> SUPPORT = ImmutableSet.of(
+            OperatorType.LOGICAL_OLAP_SCAN,
+            OperatorType.LOGICAL_HIVE_SCAN,
+            OperatorType.LOGICAL_ICEBERG_SCAN,
+            OperatorType.LOGICAL_HUDI_SCAN,
+            OperatorType.LOGICAL_DELTALAKE_SCAN,
+            OperatorType.LOGICAL_FILE_SCAN,
+            OperatorType.LOGICAL_PAIMON_SCAN,
+            OperatorType.LOGICAL_ICEBERG_METADATA_SCAN,
+            OperatorType.LOGICAL_ICEBERG_EQUALITY_DELETE_SCAN,
+            OperatorType.LOGICAL_KUDU_SCAN,
+            OperatorType.LOGICAL_SCHEMA_SCAN,
+            OperatorType.LOGICAL_ES_SCAN,
+            OperatorType.LOGICAL_META_SCAN,
+            OperatorType.LOGICAL_BINLOG_SCAN,
+            OperatorType.LOGICAL_VIEW_SCAN,
+            OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN
+    );
 
-    public static final PushDownPredicateScanRule ICEBERG_EQUALITY_DELETE_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_ICEBERG_EQUALITY_DELETE_SCAN);
-    public static final PushDownPredicateScanRule KUDU_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_KUDU_SCAN);
-    public static final PushDownPredicateScanRule SCHEMA_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_SCHEMA_SCAN);
-    public static final PushDownPredicateScanRule ES_SCAN = new PushDownPredicateScanRule(OperatorType.LOGICAL_ES_SCAN);
-    public static final PushDownPredicateScanRule META_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_META_SCAN);
-    public static final PushDownPredicateScanRule JDBC_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_JDBC_SCAN);
-    public static final PushDownPredicateScanRule BINLOG_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_BINLOG_SCAN);
-    public static final PushDownPredicateScanRule VIEW_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_VIEW_SCAN);
-
-    public static final PushDownPredicateScanRule TABLE_FUNCTION_TABLE_SCAN =
-            new PushDownPredicateScanRule(OperatorType.LOGICAL_TABLE_FUNCTION_TABLE_SCAN);
-
-    public PushDownPredicateScanRule(OperatorType type) {
-        super(RuleType.TF_PUSH_DOWN_PREDICATE_SCAN, Pattern.create(OperatorType.LOGICAL_FILTER, type));
+    public PushDownPredicateScanRule() {
+        super(RuleType.TF_PUSH_DOWN_PREDICATE_SCAN, Pattern.create(OperatorType.LOGICAL_FILTER).addChildren(
+                MultiOpPattern.of(SUPPORT)));
     }
 
     @Override
@@ -85,9 +71,6 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
         OptExpression scan = input.getInputs().get(0);
         LogicalScanOperator logicalScanOperator = (LogicalScanOperator) scan.getOp();
 
-        ScalarOperator scanPredicate = logicalScanOperator.getPredicate();
-        ScalarOperator filterPredicate = lfo.getPredicate();
-
         ScalarOperatorRewriter scalarOperatorRewriter = new ScalarOperatorRewriter();
         ScalarOperator predicates = Utils.compoundAnd(lfo.getPredicate(), logicalScanOperator.getPredicate());
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java
index 01d68e5586bdeb..a4d88404bb3f38 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PushDownPredicateToExternalTableScanRule.java
@@ -25,6 +25,7 @@
 import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
@@ -41,16 +42,12 @@
 // Because the external table may not support the functions in StarRocks,
 // to be on the safe side, we only push down partial predicates to the external table
 public class PushDownPredicateToExternalTableScanRule extends TransformationRule {
-    public static final PushDownPredicateToExternalTableScanRule MYSQL_SCAN =
-            new PushDownPredicateToExternalTableScanRule(OperatorType.LOGICAL_MYSQL_SCAN);
-    public static final PushDownPredicateToExternalTableScanRule JDBC_SCAN =
-            new PushDownPredicateToExternalTableScanRule(OperatorType.LOGICAL_JDBC_SCAN);
-    public static final PushDownPredicateToExternalTableScanRule ODPS_SCAN =
-            new PushDownPredicateToExternalTableScanRule(OperatorType.LOGICAL_ODPS_SCAN);
-
-    public PushDownPredicateToExternalTableScanRule(OperatorType type) {
+    public PushDownPredicateToExternalTableScanRule() {
         super(RuleType.TF_PUSH_DOWN_PREDICATE_TO_EXTERNAL_TABLE_SCAN,
-                Pattern.create(OperatorType.LOGICAL_FILTER, type));
+                Pattern.create(OperatorType.LOGICAL_FILTER)
+                        .addChildren(MultiOpPattern.of(OperatorType.LOGICAL_MYSQL_SCAN,
+                                OperatorType.LOGICAL_JDBC_SCAN,
+                                OperatorType.LOGICAL_ODPS_SCAN)));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java
index efca2737230d63..1617b54876a411 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteSimpleAggToHDFSScanRule.java
@@ -34,6 +34,7 @@
 import com.starrocks.sql.optimizer.operator.logical.LogicalHiveScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalIcebergScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
+import com.starrocks.sql.optimizer.operator.pattern.MultiOpPattern;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
@@ -45,40 +46,32 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 public class RewriteSimpleAggToHDFSScanRule extends TransformationRule {
     private static final Logger LOG = LogManager.getLogger(RewriteSimpleAggToHDFSScanRule.class);
 
-    public static final RewriteSimpleAggToHDFSScanRule HIVE_SCAN_NO_PROJECT =
-            new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_HIVE_SCAN, true);
-    public static final RewriteSimpleAggToHDFSScanRule ICEBERG_SCAN_NO_PROJECT =
-            new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_ICEBERG_SCAN, true);
-    public static final RewriteSimpleAggToHDFSScanRule FILE_SCAN_NO_PROJECT =
-            new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_FILE_SCAN, true);
+    private static final Set<OperatorType> SUPPORTED = Set.of(OperatorType.LOGICAL_HIVE_SCAN,
+            OperatorType.LOGICAL_ICEBERG_SCAN,
+            OperatorType.LOGICAL_FILE_SCAN
+    );
 
-    public static final RewriteSimpleAggToHDFSScanRule HIVE_SCAN =
-            new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_HIVE_SCAN);
-    public static final RewriteSimpleAggToHDFSScanRule ICEBERG_SCAN =
-            new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_ICEBERG_SCAN);
-    public static final RewriteSimpleAggToHDFSScanRule FILE_SCAN =
-            new RewriteSimpleAggToHDFSScanRule(OperatorType.LOGICAL_FILE_SCAN);
+    public static final RewriteSimpleAggToHDFSScanRule SCAN_NO_PROJECT =
+            new RewriteSimpleAggToHDFSScanRule(false);
 
-    final OperatorType scanOperatorType;
-    final boolean hasProjectOperator;
+    private final boolean hasProjectOperator;
 
-    private RewriteSimpleAggToHDFSScanRule(OperatorType logicalOperatorType, boolean withoutProject) {
+    private RewriteSimpleAggToHDFSScanRule(boolean withoutProject) {
         super(RuleType.TF_REWRITE_SIMPLE_AGG, Pattern.create(OperatorType.LOGICAL_AGGR)
-                .addChildren(Pattern.create(logicalOperatorType)));
-        hasProjectOperator = false;
-        scanOperatorType = logicalOperatorType;
+                .addChildren(MultiOpPattern.of(SUPPORTED)));
+        hasProjectOperator = withoutProject;
     }
 
-    private RewriteSimpleAggToHDFSScanRule(OperatorType logicalOperatorType) {
+    public RewriteSimpleAggToHDFSScanRule() {
         super(RuleType.TF_REWRITE_SIMPLE_AGG, Pattern.create(OperatorType.LOGICAL_AGGR)
-                .addChildren(Pattern.create(OperatorType.LOGICAL_PROJECT, logicalOperatorType)));
+                .addChildren(Pattern.create(OperatorType.LOGICAL_PROJECT).addChildren(MultiOpPattern.of(SUPPORTED))));
         hasProjectOperator = true;
-        scanOperatorType = logicalOperatorType;
     }
 
     private OptExpression buildAggScanOperator(LogicalAggregationOperator aggregationOperator,
@@ -97,7 +90,7 @@ private OptExpression buildAggScanOperator(LogicalAggregationOperator aggregatio
             if (tableRelationId == -1) {
                 tableRelationId = relationId;
             } else if (tableRelationId != relationId) {
-                LOG.warn("Table relationIds are different in columns, tableRelationId = %d, relationId = %d",
+                LOG.warn("Table relationIds are different in columns, tableRelationId = {}, relationId = {}",
                         tableRelationId, relationId);
                 return null;
             }
@@ -222,7 +215,7 @@ public boolean check(final OptExpression input, OptimizerContext context) {
             return false;
         }
 
-        if (scanOperatorType == OperatorType.LOGICAL_ICEBERG_SCAN) {
+        if (scanOperator.getOpType() == OperatorType.LOGICAL_ICEBERG_SCAN) {
             IcebergTable icebergTable = (IcebergTable) scanOperator.getTable();
             if (!icebergTable.isUnPartitioned() && !icebergTable.isAllPartitionColumnsAlwaysIdentity()) {
                 return false;
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java
index 0716f0bbceeadc..db4da33b23e8bc 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/SkewJoinOptimizeRule.java
@@ -51,6 +51,7 @@
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
 import com.starrocks.sql.optimizer.rewrite.ReplaceColumnRefRewriter;
 import com.starrocks.sql.optimizer.rewrite.ScalarOperatorRewriter;
+import com.starrocks.sql.optimizer.rule.Rule;
 import com.starrocks.sql.optimizer.rule.RuleType;
 import com.starrocks.sql.optimizer.statistics.ColumnStatistic;
 import com.starrocks.sql.optimizer.statistics.Statistics;
@@ -96,6 +97,12 @@ public SkewJoinOptimizeRule() {
                 Pattern.create(OperatorType.LOGICAL_JOIN, OperatorType.PATTERN_LEAF, OperatorType.PATTERN_LEAF));
     }
 
+    @Override
+    public List<Rule> successorRules() {
+        // skew join generate new join and on predicate, need to push down join on expression to child project again
+        return Lists.newArrayList(new PushDownJoinOnExpressionToChildProject());
+    }
+
     @Override
     public boolean check(OptExpression input, OptimizerContext context) {
         // respect the join hint
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java
index 7f50091d12eae9..17380d201a7ff9 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java
@@ -124,7 +124,7 @@ protected boolean match(Pattern pattern, OptExpression root) {
         }
 
         if (!pattern.children().isEmpty() && pattern.children().size() != root.getInputs().size() &&
-                pattern.children().stream().noneMatch(Pattern::isPatternMultiLeaf)) {
+                pattern.children().stream().noneMatch(p -> p.is(OperatorType.PATTERN_MULTI_LEAF))) {
             return false;
         }
         int patternIndex = 0;
@@ -138,7 +138,7 @@ protected boolean match(Pattern pattern, OptExpression root) {
                 return false;
             }
 
-            if (!(childPattern.isPatternMultiLeaf() && (root.getInputs().size() - childIndex) >
+            if (!(childPattern.is(OperatorType.PATTERN_MULTI_LEAF) && (root.getInputs().size() - childIndex) >
                     (pattern.children().size() - patternIndex))) {
                 patternIndex++;
             }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java
index a9a81a5f5a8e09..a5eb700b9648ab 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java
@@ -79,11 +79,10 @@ public void rewriteIterative(OptExpression tree, TaskContext rootTaskContext, Ru
         executeTasks(rootTaskContext);
     }
 
-    public boolean rewriteOnce(OptExpression tree, TaskContext rootTaskContext, Rule rule) {
+    public void rewriteOnce(OptExpression tree, TaskContext rootTaskContext, Rule rule) {
         RewriteTreeTask rewriteTreeTask = new RewriteTreeTask(rootTaskContext, tree, rule, true);
         pushTask(rewriteTreeTask);
         executeTasks(rootTaskContext);
-        return rewriteTreeTask.hasChange();
     }
 
     public void rewriteAtMostOnce(OptExpression tree, TaskContext rootTaskContext, Rule rule) {
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java
index 71d777ae4293f4..57372c09a3b3cf 100644
--- a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java
@@ -427,7 +427,7 @@ public long getTableCreateTime(String dbName, String tblName) {
 
         PaimonTable paimonTable = (PaimonTable) metadata.getTable("db1", "tbl1");
 
-        ExternalScanPartitionPruneRule rule0 = ExternalScanPartitionPruneRule.PAIMON_SCAN;
+        ExternalScanPartitionPruneRule rule0 = new ExternalScanPartitionPruneRule();
 
         ColumnRefOperator colRef1 = new ColumnRefOperator(1, Type.INT, "f2", true);
         Column col1 = new Column("f2", Type.INT, true);
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java
index 2166cbb8e47cd8..213248337ff167 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PruneHDFSScanColumnRuleTest.java
@@ -45,9 +45,9 @@
 import java.util.stream.Collectors;
 
 public class PruneHDFSScanColumnRuleTest {
-    private PruneHDFSScanColumnRule icebergRule = PruneHDFSScanColumnRule.ICEBERG_SCAN;
-    private PruneHDFSScanColumnRule hudiRule = PruneHDFSScanColumnRule.HUDI_SCAN;
-    private PruneHDFSScanColumnRule odpsRule = PruneHDFSScanColumnRule.ODPS_SCAN;
+    private PruneHDFSScanColumnRule icebergRule = new PruneHDFSScanColumnRule();
+    private PruneHDFSScanColumnRule hudiRule = new PruneHDFSScanColumnRule();
+    private PruneHDFSScanColumnRule odpsRule = new PruneHDFSScanColumnRule();
 
     ColumnRefOperator intColumnOperator = new ColumnRefOperator(1, Type.INT, "id", true);
     ColumnRefOperator strColumnOperator = new ColumnRefOperator(2, Type.STRING, "name", true);
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java
index 837baefd6ed72a..c56e0ca90bddcc 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java
@@ -42,7 +42,7 @@ public class PushDownScanRuleTest {
 
     @Test
     public void transform(@Mocked OlapTable table) {
-        PushDownPredicateScanRule rule = PushDownPredicateScanRule.OLAP_SCAN;
+        PushDownPredicateScanRule rule = new PushDownPredicateScanRule();
 
         OptExpression optExpression = new OptExpression(new LogicalFilterOperator(
                 new BinaryPredicateOperator(BinaryType.EQ,
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java
index e89844cc4b5069..064470f08ba213 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/SkewJoinTest.java
@@ -220,8 +220,8 @@ public void testSkewJoinWithStructType() throws Exception {
                 "  |  colocate: false, reason: \n" +
                 "  |  equal join conjunct: 10: rand_col = 17: rand_col\n" +
                 "  |  equal join conjunct: 9: cast = 5: v1",
-                "<slot 10> : CASE WHEN 2: c1.a[true] IS NULL THEN 24: round WHEN 2: c1.a[true] IN (1, 2) THEN 24: " +
-                        "round ELSE 0 END");
+                "<slot 10> : CASE WHEN 2: c1.a[true] IS NULL THEN 23: round WHEN 2: c1.a[true] IN (1, 2) THEN " +
+                        "23: round ELSE 0 END");
     }
 
     @Test

From 701d8ad0b573d904acf3f9a46d79dbfabc1d6dd7 Mon Sep 17 00:00:00 2001
From: Yixin Luo <18810541851@163.com>
Date: Thu, 16 Jan 2025 10:29:29 +0800
Subject: [PATCH 11/71] [Enhancement] support pk column lazy load (#54970)

Signed-off-by: luohaha <18810541851@163.com>
---
 be/src/common/config.h                        |   2 +
 be/src/storage/lake/rowset_update_state.cpp   | 120 ++++++++++++------
 be/src/storage/lake/rowset_update_state.h     |  48 ++++++-
 be/src/storage/lake/update_manager.cpp        |  10 +-
 be/src/storage/lake/update_manager.h          |   4 +-
 .../storage/lake/primary_key_publish_test.cpp |  37 +++++-
 be/test/storage/lake/test_util.h              |   2 +-
 7 files changed, 175 insertions(+), 48 deletions(-)

diff --git a/be/src/common/config.h b/be/src/common/config.h
index bd267e0e8b66ac..c7abe566a1b05f 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1545,6 +1545,8 @@ CONF_mInt64(load_spill_max_merge_bytes, "1073741824");
 CONF_mInt64(load_spill_merge_memory_limit_percent, "30");
 // Upper bound of spill merge thread count
 CONF_mInt64(load_spill_merge_max_thread, "16");
+// Do lazy load when PK column larger than this threshold. Default is 300MB.
+CONF_mInt64(pk_column_lazy_load_threshold_bytes, "314572800");
 
 // ignore union type tag in avro kafka routine load
 CONF_mBool(avro_ignore_union_type_tag, "false");
diff --git a/be/src/storage/lake/rowset_update_state.cpp b/be/src/storage/lake/rowset_update_state.cpp
index d9d2bbc96bea42..d49d7f7e5eb45a 100644
--- a/be/src/storage/lake/rowset_update_state.cpp
+++ b/be/src/storage/lake/rowset_update_state.cpp
@@ -34,6 +34,73 @@
 
 namespace starrocks::lake {
 
+Status SegmentPKEncodeResult::_load() {
+    pk_column->reset_column();
+    ChunkUniquePtr chunk_shared_ptr;
+    TRY_CATCH_BAD_ALLOC(chunk_shared_ptr = ChunkHelper::new_chunk(_pkey_schema, 4096));
+    auto chunk = chunk_shared_ptr.get();
+    if (_iter != nullptr) {
+        while (true) {
+            chunk->reset();
+            auto st = _iter->get_next(chunk);
+            if (st.is_end_of_file()) {
+                break;
+            } else if (!st.ok()) {
+                return st;
+            } else {
+                TRY_CATCH_BAD_ALLOC(
+                        PrimaryKeyEncoder::encode(_pkey_schema, *chunk, 0, chunk->num_rows(), pk_column.get()));
+                if (_lazy_load && pk_column->memory_usage() >= config::pk_column_lazy_load_threshold_bytes) {
+                    break;
+                }
+            }
+        }
+    }
+    if (pk_column->empty()) {
+        return Status::OK();
+    }
+    _current_rows += pk_column->size();
+    _begin_rowid_offsets.push_back(_current_rows);
+    return Status::OK();
+}
+
+Status SegmentPKEncodeResult::init(const ChunkIteratorPtr& iter, const Schema& pkey_schema, bool lazy_load) {
+    _iter = iter;
+    _pkey_schema = pkey_schema;
+    _lazy_load = lazy_load;
+    _begin_rowid_offsets.push_back(0);
+    RETURN_IF_ERROR(PrimaryKeyEncoder::create_column(_pkey_schema, &pk_column));
+    _status = _load();
+    if (_status.ok()) {
+        TRY_CATCH_BAD_ALLOC(pk_column->raw_data());
+        _memory_usage = pk_column->memory_usage();
+    }
+    return _status;
+}
+
+bool SegmentPKEncodeResult::done() {
+    return pk_column->empty() || !_status.ok();
+}
+
+Status SegmentPKEncodeResult::status() {
+    return _status;
+}
+
+void SegmentPKEncodeResult::next() {
+    _status = _load();
+    if (_status.ok()) {
+        _current_pk_column_idx++;
+    }
+}
+
+std::pair<Column*, size_t> SegmentPKEncodeResult::current() {
+    return std::make_pair(pk_column.get(), _begin_rowid_offsets[_current_pk_column_idx]);
+}
+
+void SegmentPKEncodeResult::close() {
+    _iter->close();
+}
+
 RowsetUpdateState::RowsetUpdateState() = default;
 
 RowsetUpdateState::~RowsetUpdateState() = default;
@@ -201,38 +268,17 @@ Status RowsetUpdateState::_do_load_upserts(uint32_t segment_id, const RowsetUpda
         pk_columns.push_back((uint32_t)i);
     }
     Schema pkey_schema = ChunkHelper::convert_schema(params.tablet_schema, pk_columns);
-    std::unique_ptr<Column> pk_column;
-    RETURN_IF_ERROR(PrimaryKeyEncoder::create_column(pkey_schema, &pk_column));
 
     if (_segment_iters.empty()) {
         ASSIGN_OR_RETURN(_segment_iters, _rowset_ptr->get_each_segment_iterator(pkey_schema, false, &_stats));
     }
     RETURN_ERROR_IF_FALSE(_segment_iters.size() == _rowset_ptr->num_segments());
-    // only hold pkey, so can use larger chunk size
-    ChunkUniquePtr chunk_shared_ptr;
-    TRY_CATCH_BAD_ALLOC(chunk_shared_ptr = ChunkHelper::new_chunk(pkey_schema, 4096));
-    auto chunk = chunk_shared_ptr.get();
-
-    auto itr = _segment_iters[segment_id].get();
-    auto& dest = _upserts[segment_id];
-    auto col = pk_column->clone();
-    if (itr != nullptr) {
-        while (true) {
-            chunk->reset();
-            auto st = itr->get_next(chunk);
-            if (st.is_end_of_file()) {
-                break;
-            } else if (!st.ok()) {
-                return st;
-            } else {
-                TRY_CATCH_BAD_ALLOC(PrimaryKeyEncoder::encode(pkey_schema, *chunk, 0, chunk->num_rows(), col.get()));
-            }
-        }
-        itr->close();
-    }
-    dest = std::move(col);
-    TRY_CATCH_BAD_ALLOC(dest->raw_data());
-    _memory_usage += dest->memory_usage();
+    auto& iter = _segment_iters[segment_id];
+    SegmentPKEncodeResultPtr result = std::make_unique<SegmentPKEncodeResult>();
+    // If this txn contains partial update or auto increment partial update, can't support lazy load now.
+    RETURN_IF_ERROR(result->init(iter, pkey_schema, !params.op_write.has_txn_meta()));
+    _upserts[segment_id] = std::move(result);
+    _memory_usage += _upserts[segment_id]->memory_usage();
 
     return Status::OK();
 }
@@ -288,14 +334,14 @@ Status RowsetUpdateState::_prepare_auto_increment_partial_update_states(uint32_t
 
     _auto_increment_partial_update_states[segment_id].init(
             modified_columns_schema, txn_meta.auto_increment_partial_update_column_id(), segment_id);
-    _auto_increment_partial_update_states[segment_id].src_rss_rowids.resize(_upserts[segment_id]->size());
+    _auto_increment_partial_update_states[segment_id].src_rss_rowids.resize(_upserts[segment_id]->pk_column->size());
     read_column.resize(1);
     read_column[0] = column->clone_empty();
     _auto_increment_partial_update_states[segment_id].write_column = column->clone_empty();
 
     // use upserts to get rowids in this segment
     RETURN_IF_ERROR(params.tablet->update_mgr()->get_rowids_from_pkindex(
-            params.tablet->id(), _base_versions[segment_id], _upserts[segment_id],
+            params.tablet->id(), _base_versions[segment_id], _upserts[segment_id]->pk_column,
             &(_auto_increment_partial_update_states[segment_id].src_rss_rowids), need_lock));
 
     std::vector<uint32_t> rowids;
@@ -351,7 +397,7 @@ Status RowsetUpdateState::_prepare_auto_increment_partial_update_states(uint32_t
         *    different segment file.
         */
     _auto_increment_delete_pks[segment_id].reset();
-    _auto_increment_delete_pks[segment_id] = _upserts[segment_id]->clone_empty();
+    _auto_increment_delete_pks[segment_id] = _upserts[segment_id]->pk_column->clone_empty();
     std::vector<uint32_t> delete_idxes;
     const int64* data = nullptr;
     TRY_CATCH_BAD_ALLOC(data = reinterpret_cast<const int64*>(
@@ -367,7 +413,7 @@ Status RowsetUpdateState::_prepare_auto_increment_partial_update_states(uint32_t
 
     if (delete_idxes.size() != 0) {
         TRY_CATCH_BAD_ALLOC(_auto_increment_delete_pks[segment_id]->append_selective(
-                *_upserts[segment_id], delete_idxes.data(), 0, delete_idxes.size()));
+                *(_upserts[segment_id]->pk_column), delete_idxes.data(), 0, delete_idxes.size()));
         _memory_usage += _auto_increment_delete_pks[segment_id]->memory_usage();
     }
     return Status::OK();
@@ -387,7 +433,7 @@ Status RowsetUpdateState::_prepare_partial_update_states(uint32_t segment_id, co
     std::vector<std::unique_ptr<Column>> read_columns;
     read_columns.resize(read_column_ids.size());
     _partial_update_states[segment_id].write_columns.resize(read_columns.size());
-    _partial_update_states[segment_id].src_rss_rowids.resize(_upserts[segment_id]->size());
+    _partial_update_states[segment_id].src_rss_rowids.resize(_upserts[segment_id]->pk_column->size());
     for (uint32_t j = 0; j < read_columns.size(); ++j) {
         auto column = ChunkHelper::column_from_field(*read_column_schema.field(j).get());
         read_columns[j] = column->clone_empty();
@@ -396,7 +442,7 @@ Status RowsetUpdateState::_prepare_partial_update_states(uint32_t segment_id, co
 
     // use upsert to get rowids for this segment
     RETURN_IF_ERROR(params.tablet->update_mgr()->get_rowids_from_pkindex(
-            params.tablet->id(), _base_versions[segment_id], _upserts[segment_id],
+            params.tablet->id(), _base_versions[segment_id], _upserts[segment_id]->pk_column,
             &(_partial_update_states[segment_id].src_rss_rowids), need_lock));
 
     size_t num_default = 0;
@@ -539,9 +585,9 @@ Status RowsetUpdateState::_resolve_conflict(uint32_t segment_id, const RowsetUpd
     }
 
     // use upserts to get rowids in this segment
-    std::vector<uint64_t> new_rss_rowids(_upserts[segment_id]->size());
+    std::vector<uint64_t> new_rss_rowids(_upserts[segment_id]->pk_column->size());
     RETURN_IF_ERROR(params.tablet->update_mgr()->get_rowids_from_pkindex(
-            params.tablet->id(), _base_versions[segment_id], _upserts[segment_id], &new_rss_rowids, false));
+            params.tablet->id(), _base_versions[segment_id], _upserts[segment_id]->pk_column, &new_rss_rowids, false));
 
     size_t total_conflicts = 0;
     std::shared_ptr<TabletSchema> tablet_schema = std::make_shared<TabletSchema>(params.metadata->schema());
@@ -691,7 +737,7 @@ Status RowsetUpdateState::_resolve_conflict_auto_increment(const RowsetUpdateSta
 
         // reslove delete-partial update conflict base on latest column values
         _auto_increment_delete_pks[segment_id].reset();
-        _auto_increment_delete_pks[segment_id] = _upserts[segment_id]->clone_empty();
+        _auto_increment_delete_pks[segment_id] = _upserts[segment_id]->pk_column->clone_empty();
         std::vector<uint32_t> delete_idxes;
         const int64* data = nullptr;
         TRY_CATCH_BAD_ALLOC(data = reinterpret_cast<const int64*>(
@@ -707,7 +753,7 @@ Status RowsetUpdateState::_resolve_conflict_auto_increment(const RowsetUpdateSta
 
         if (delete_idxes.size() != 0) {
             TRY_CATCH_BAD_ALLOC(_auto_increment_delete_pks[segment_id]->append_selective(
-                    *_upserts[segment_id], delete_idxes.data(), 0, delete_idxes.size()));
+                    *(_upserts[segment_id]->pk_column), delete_idxes.data(), 0, delete_idxes.size()));
         }
     }
     return Status::OK();
diff --git a/be/src/storage/lake/rowset_update_state.h b/be/src/storage/lake/rowset_update_state.h
index 15fc04c4e55170..89508d8f7ede47 100644
--- a/be/src/storage/lake/rowset_update_state.h
+++ b/be/src/storage/lake/rowset_update_state.h
@@ -82,6 +82,50 @@ struct RowsetUpdateStateParams {
     const RssidFileInfoContainer& container;
 };
 
+class SegmentPKEncodeResult {
+public:
+    SegmentPKEncodeResult() = default;
+    ~SegmentPKEncodeResult() { close(); }
+    Status init(const ChunkIteratorPtr& iter, const Schema& pkey_schema, bool load_whole);
+    void next();
+    bool done();
+    Status status();
+    void close();
+    // <Current pk column, begin rowid>
+    std::pair<Column*, size_t> current();
+
+    // Return the memory usage of this encode pk column.
+    // If _lazy_load is true, return 0, because memory allocation is lazy.
+    size_t memory_usage() const { return _memory_usage; }
+
+    // For large segment, we need to load segment file piece by piece.
+    ColumnUniquePtr pk_column;
+
+private:
+    Status _load();
+
+    // Iterator of this segment file.
+    ChunkIteratorPtr _iter;
+    // The PK schema of this segment file.
+    Schema _pkey_schema;
+    // status
+    Status _status = Status::OK();
+    // The current pk column index.
+    size_t _current_pk_column_idx = 0;
+    // The rowid offsets of each piece.
+    // E.g. if we have column vec : 100 rows, 101 rows, 200 rows,
+    // offset will be [0, 100, 201, 401]
+    std::vector<size_t> _begin_rowid_offsets;
+    // Current loaded row count of the segment.
+    size_t _current_rows = 0;
+    // If true, we will load segment peice by piece when needed.
+    bool _lazy_load = false;
+    // If enable lazy load, `_memory_usage` will record first piece of pk column memory usage.
+    size_t _memory_usage = 0;
+};
+
+using SegmentPKEncodeResultPtr = std::unique_ptr<SegmentPKEncodeResult>;
+
 class RowsetUpdateState {
 public:
     using ColumnUniquePtr = std::unique_ptr<Column>;
@@ -126,7 +170,7 @@ class RowsetUpdateState {
     // Release `del_id`-th delete file's state.
     void release_delete(uint32_t del_id);
 
-    const ColumnUniquePtr& upserts(uint32_t segment_id) const { return _upserts[segment_id]; }
+    const SegmentPKEncodeResultPtr& upserts(uint32_t segment_id) const { return _upserts[segment_id]; }
     const ColumnUniquePtr& deletes(uint32_t segment_id) const { return _deletes[segment_id]; }
 
     std::size_t memory_usage() const { return _memory_usage; }
@@ -167,7 +211,7 @@ class RowsetUpdateState {
     void _reset();
 
     // one for each segment file
-    std::vector<ColumnUniquePtr> _upserts;
+    std::vector<SegmentPKEncodeResultPtr> _upserts;
     // one for each delete file
     std::vector<ColumnUniquePtr> _deletes;
     size_t _memory_usage = 0;
diff --git a/be/src/storage/lake/update_manager.cpp b/be/src/storage/lake/update_manager.cpp
index 6fff4e404f752a..36e4fe41154fb0 100644
--- a/be/src/storage/lake/update_manager.cpp
+++ b/be/src/storage/lake/update_manager.cpp
@@ -257,7 +257,7 @@ Status UpdateManager::publish_primary_key_tablet(const TxnLogPB_OpWrite& op_writ
             RETURN_IF_ERROR(_do_update(rowset_id, segment_id, state.upserts(segment_id), index, &new_deletes));
         } else {
             RETURN_IF_ERROR(_do_update_with_condition(params, rowset_id, segment_id, condition_column,
-                                                      state.upserts(segment_id), index, &new_deletes));
+                                                      state.upserts(segment_id)->pk_column, index, &new_deletes));
         }
         // 2.3 handle auto increment deletes
         if (state.auto_increment_deletes(segment_id) != nullptr) {
@@ -375,10 +375,14 @@ Status UpdateManager::publish_column_mode_partial_update(const TxnLogPB_OpWrite&
     return Status::OK();
 }
 
-Status UpdateManager::_do_update(uint32_t rowset_id, int32_t upsert_idx, const ColumnUniquePtr& upsert,
+Status UpdateManager::_do_update(uint32_t rowset_id, int32_t upsert_idx, const SegmentPKEncodeResultPtr& upsert,
                                  PrimaryIndex& index, DeletesMap* new_deletes) {
     TRACE_COUNTER_SCOPE_LATENCY_US("do_update_latency_us");
-    return index.upsert(rowset_id + upsert_idx, 0, *upsert, new_deletes);
+    for (; !upsert->done(); upsert->next()) {
+        auto current = upsert->current();
+        RETURN_IF_ERROR(index.upsert(rowset_id + upsert_idx, current.second, *current.first, new_deletes));
+    }
+    return upsert->status();
 }
 
 Status UpdateManager::_do_update_with_condition(const RowsetUpdateStateParams& params, uint32_t rowset_id,
diff --git a/be/src/storage/lake/update_manager.h b/be/src/storage/lake/update_manager.h
index bcf1edee367fc2..7546288129588d 100644
--- a/be/src/storage/lake/update_manager.h
+++ b/be/src/storage/lake/update_manager.h
@@ -206,8 +206,8 @@ class UpdateManager {
 private:
     // print memory tracker state
     void _print_memory_stats();
-    Status _do_update(uint32_t rowset_id, int32_t upsert_idx, const ColumnUniquePtr& upsert, PrimaryIndex& index,
-                      DeletesMap* new_deletes);
+    Status _do_update(uint32_t rowset_id, int32_t upsert_idx, const SegmentPKEncodeResultPtr& upsert,
+                      PrimaryIndex& index, DeletesMap* new_deletes);
 
     Status _do_update_with_condition(const RowsetUpdateStateParams& params, uint32_t rowset_id, int32_t upsert_idx,
                                      int32_t condition_column, const ColumnUniquePtr& upsert, PrimaryIndex& index,
diff --git a/be/test/storage/lake/primary_key_publish_test.cpp b/be/test/storage/lake/primary_key_publish_test.cpp
index b4563238b53a71..3d20b2d8d57d6f 100644
--- a/be/test/storage/lake/primary_key_publish_test.cpp
+++ b/be/test/storage/lake/primary_key_publish_test.cpp
@@ -1237,10 +1237,10 @@ TEST_P(LakePrimaryKeyPublishTest, test_transform_batch_to_single) {
 }
 
 TEST_P(LakePrimaryKeyPublishTest, test_mem_tracker) {
-    EXPECT_EQ(1024 * 1024, _mem_tracker->limit());
-    EXPECT_EQ(1024 * 1024 * config::lake_pk_preload_memory_limit_percent / 100,
+    EXPECT_EQ(10 * 1024 * 1024, _mem_tracker->limit());
+    EXPECT_EQ(10 * 1024 * 1024 * config::lake_pk_preload_memory_limit_percent / 100,
               _update_mgr->compaction_state_mem_tracker()->limit());
-    EXPECT_EQ(1024 * 1024 * config::lake_pk_preload_memory_limit_percent / 100,
+    EXPECT_EQ(10 * 1024 * 1024 * config::lake_pk_preload_memory_limit_percent / 100,
               _update_mgr->update_state_mem_tracker()->limit());
 }
 
@@ -1890,6 +1890,37 @@ TEST_P(LakePrimaryKeyPublishTest, test_individual_index_compaction) {
     EXPECT_TRUE(new_tablet_metadata->orphan_files_size() >= (sst_cnt - 1));
 }
 
+TEST_P(LakePrimaryKeyPublishTest, test_publish_with_lazy_load) {
+    const size_t N = 40000;
+    auto [chunk0, indexes] = gen_data_and_index(N, 0, true, true);
+    auto version = 1;
+    auto tablet_id = _tablet_metadata->id();
+    int64_t old_val = config::pk_column_lazy_load_threshold_bytes;
+    config::pk_column_lazy_load_threshold_bytes = 1;
+    for (int i = 0; i < 3; i++) {
+        int64_t txn_id = next_id();
+        ASSIGN_OR_ABORT(auto delta_writer, DeltaWriterBuilder()
+                                                   .set_tablet_manager(_tablet_mgr.get())
+                                                   .set_tablet_id(tablet_id)
+                                                   .set_txn_id(txn_id)
+                                                   .set_partition_id(_partition_id)
+                                                   .set_mem_tracker(_mem_tracker.get())
+                                                   .set_schema_id(_tablet_schema->id())
+                                                   .build());
+        ASSERT_OK(delta_writer->open());
+        ASSERT_OK(delta_writer->write(*chunk0, indexes.data(), indexes.size()));
+        ASSERT_OK(delta_writer->finish_with_txnlog());
+        delta_writer->close();
+        ASSERT_OK(publish_single_version(tablet_id, version + 1, txn_id).status());
+        version++;
+    }
+    config::pk_column_lazy_load_threshold_bytes = old_val;
+    ASSERT_EQ(N, read_rows(tablet_id, version));
+    if (GetParam().enable_persistent_index && GetParam().persistent_index_type == PersistentIndexTypePB::LOCAL) {
+        check_local_persistent_index_meta(tablet_id, version);
+    }
+}
+
 INSTANTIATE_TEST_SUITE_P(LakePrimaryKeyPublishTest, LakePrimaryKeyPublishTest,
                          ::testing::Values(PrimaryKeyParam{true}, PrimaryKeyParam{false},
                                            PrimaryKeyParam{true, PersistentIndexTypePB::CLOUD_NATIVE},
diff --git a/be/test/storage/lake/test_util.h b/be/test/storage/lake/test_util.h
index 9a05f534a4d8b8..7bbdd83d20600f 100644
--- a/be/test/storage/lake/test_util.h
+++ b/be/test/storage/lake/test_util.h
@@ -71,7 +71,7 @@ class TestBase : public ::testing::Test {
     explicit TestBase(std::string test_dir, int64_t cache_limit = 1024 * 1024)
             : _test_dir(std::move(test_dir)),
               _parent_tracker(std::make_unique<MemTracker>(-1)),
-              _mem_tracker(std::make_unique<MemTracker>(1024 * 1024, "", _parent_tracker.get())),
+              _mem_tracker(std::make_unique<MemTracker>(10 * 1024 * 1024, "", _parent_tracker.get())),
               _lp(std::make_shared<FixedLocationProvider>(_test_dir)),
               _update_mgr(std::make_unique<UpdateManager>(_lp, _mem_tracker.get())),
               _tablet_mgr(std::make_unique<TabletManager>(_lp, _update_mgr.get(), cache_limit)) {}

From 47e386b47b1a10eb798352e48e60a526ead5308b Mon Sep 17 00:00:00 2001
From: PengFei Li <lpengfei2016@gmail.com>
Date: Thu, 16 Jan 2025 11:32:24 +0800
Subject: [PATCH 12/71] [BugFix] Fix set the gtid of null rowset when publish
 (#55024)

Signed-off-by: PengFei Li <lpengfei2016@gmail.com>
---
 be/src/agent/publish_version.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/be/src/agent/publish_version.cpp b/be/src/agent/publish_version.cpp
index b0b6010aa7a1be..5178218732f781 100644
--- a/be/src/agent/publish_version.cpp
+++ b/be/src/agent/publish_version.cpp
@@ -109,7 +109,10 @@ void run_publish_version_task(ThreadPoolToken* token, const TPublishVersionReque
                 task.tablet_id = itr.first.tablet_id;
                 task.version = publish_version_req.partition_version_infos[i].version;
                 task.rowset = std::move(itr.second);
-                task.rowset->rowset_meta()->set_gtid(publish_version_req.gtid);
+                // rowset can be nullptr if it just prepared but not committed
+                if (task.rowset != nullptr) {
+                    task.rowset->rowset_meta()->set_gtid(publish_version_req.gtid);
+                }
                 task.is_double_write = publish_version_req.partition_version_infos[i].__isset.is_double_write &&
                                        publish_version_req.partition_version_infos[i].is_double_write;
             }

From c42afe2439cbc43e7d2f543e85ac876d31ba4ecd Mon Sep 17 00:00:00 2001
From: before-Sunrise <71162020+before-Sunrise@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:40:07 +0800
Subject: [PATCH 13/71] [BugFix] make max_by/min_by's  output result column
 always nullable (#55116)

Signed-off-by: before-Sunrise <unclejyj@gmail.com>
---
 be/src/exec/aggregator.cpp                    |  4 +--
 .../R/test_max_min_by_with_empty_table        | 36 +++++++++++++++++++
 .../T/test_max_min_by_with_empty_table        | 32 +++++++++++++++++
 3 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 test/sql/test_max_min_by_not_filter_nulls_with_nulls/R/test_max_min_by_with_empty_table
 create mode 100644 test/sql/test_max_min_by_not_filter_nulls_with_nulls/T/test_max_min_by_with_empty_table

diff --git a/be/src/exec/aggregator.cpp b/be/src/exec/aggregator.cpp
index 6737b0703ee1fc..52e98e5afb8ed0 100644
--- a/be/src/exec/aggregator.cpp
+++ b/be/src/exec/aggregator.cpp
@@ -42,8 +42,8 @@
 
 namespace starrocks {
 
-static const std::unordered_set<std::string> ALWAYS_NULLABLE_RESULT_AGG_FUNCS = {"variance_samp", "var_samp",
-                                                                                 "stddev_samp", "covar_samp", "corr"};
+static const std::unordered_set<std::string> ALWAYS_NULLABLE_RESULT_AGG_FUNCS = {
+        "variance_samp", "var_samp", "stddev_samp", "covar_samp", "corr", "max_by_v2", "min_by_v2"};
 
 static const std::string AGG_STATE_UNION_SUFFIX = "_union";
 static const std::string AGG_STATE_MERGE_SUFFIX = "_merge";
diff --git a/test/sql/test_max_min_by_not_filter_nulls_with_nulls/R/test_max_min_by_with_empty_table b/test/sql/test_max_min_by_not_filter_nulls_with_nulls/R/test_max_min_by_with_empty_table
new file mode 100644
index 00000000000000..5b366f1c9f2862
--- /dev/null
+++ b/test/sql/test_max_min_by_not_filter_nulls_with_nulls/R/test_max_min_by_with_empty_table
@@ -0,0 +1,36 @@
+-- name: test_max_min_by_with_empty_table
+CREATE TABLE `primary_table_with_null_partition` (
+                `k1` date not null,
+                `k2` datetime not null,
+                `k3` varchar(20) not null,
+                `k4` varchar(20) not null,
+                `k5` boolean not null,
+                `v1` tinyint,
+                `v2` smallint,
+                `v3` int,
+                `v4` bigint,
+                `v5` largeint,
+                `v6` float,
+                `v7` double,
+                `v8` decimal(27,9)
+            ) 
+            PRIMARY KEY(`k1`, `k2`,`k3`)
+            COMMENT "OLAP"
+            PARTITION BY RANGE(`k1`)
+            (
+                PARTITION `p202006` VALUES LESS THAN ("2020-07-01"),
+                PARTITION `p202007` VALUES LESS THAN ("2020-08-01"),
+                PARTITION `p202008` VALUES LESS THAN ("2020-09-01")
+            )
+            DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 3
+            PROPERTIES (
+                "replication_num" = "1",
+                "enable_persistent_index" = "true",
+                "storage_format" = "v2" 
+            );
+-- result:
+-- !result
+select max_by(k1,k1) from primary_table_with_null_partition;
+-- result:
+None
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_max_min_by_not_filter_nulls_with_nulls/T/test_max_min_by_with_empty_table b/test/sql/test_max_min_by_not_filter_nulls_with_nulls/T/test_max_min_by_with_empty_table
new file mode 100644
index 00000000000000..e073158b6cb2c5
--- /dev/null
+++ b/test/sql/test_max_min_by_not_filter_nulls_with_nulls/T/test_max_min_by_with_empty_table
@@ -0,0 +1,32 @@
+-- name: test_max_min_by_with_empty_table
+CREATE TABLE `primary_table_with_null_partition` (
+                `k1` date not null,
+                `k2` datetime not null,
+                `k3` varchar(20) not null,
+                `k4` varchar(20) not null,
+                `k5` boolean not null,
+                `v1` tinyint,
+                `v2` smallint,
+                `v3` int,
+                `v4` bigint,
+                `v5` largeint,
+                `v6` float,
+                `v7` double,
+                `v8` decimal(27,9)
+            ) 
+            PRIMARY KEY(`k1`, `k2`,`k3`)
+            COMMENT "OLAP"
+            PARTITION BY RANGE(`k1`)
+            (
+                PARTITION `p202006` VALUES LESS THAN ("2020-07-01"),
+                PARTITION `p202007` VALUES LESS THAN ("2020-08-01"),
+                PARTITION `p202008` VALUES LESS THAN ("2020-09-01")
+            )
+            DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 3
+            PROPERTIES (
+                "replication_num" = "1",
+                "enable_persistent_index" = "true",
+                "storage_format" = "v2" 
+            );
+
+select max_by(k1,k1) from primary_table_with_null_partition;
\ No newline at end of file

From 4b791364b2b9704504ed50ecd35f65ff3f797bb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=B5=B5=E7=A9=BA=E4=BA=8B=E3=82=B9=E3=83=94=E3=83=AA?=
 =?UTF-8?q?=E3=83=83=E3=83=88?= <wanglichen@starrocks.com>
Date: Thu, 16 Jan 2025 15:37:10 +0800
Subject: [PATCH 14/71] [Doc] Remove CN deployment from shared-nothing (#55138)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 絵空事スピリット <wanglichen@starrocks.com>
---
 docs/en/deployment/deploy_manually.md | 122 +------------------------
 docs/zh/deployment/deploy_manually.md | 124 +-------------------------
 2 files changed, 8 insertions(+), 238 deletions(-)

diff --git a/docs/en/deployment/deploy_manually.md b/docs/en/deployment/deploy_manually.md
index 695f34737e14ac..abc14235a3ed54 100644
--- a/docs/en/deployment/deploy_manually.md
+++ b/docs/en/deployment/deploy_manually.md
@@ -172,65 +172,9 @@ The following procedures are performed on the BE instances.
       default_replication_num = 1
       ```
 
-## Step 3: (Optional) Start the CN service
+## Step 3: Set up the cluster
 
-A Compute Node (CN) is a stateless computing service that does not maintain data itself. You can optionally add CN nodes to your cluster to provide extra computing resources for queries. You can deploy CN nodes with the BE deployment files. Compute Nodes are supported since v2.4.
-
-1. Navigate to the directory that stores the [StarRocks BE deployment files](../deployment/prepare_deployment_files.md) you prepared earlier, and modify the CN configuration file **be/conf/cn.conf**.
-
-   a. If any of the CN ports mentioned in the [Environment Configuration Checklist](../deployment/environment_configurations.md) are occupied, you must assign valid alternatives in the CN configuration file.
-
-      ```YAML
-      be_port = vvvv                   # Default: 9060
-      be_http_port = xxxx              # Default: 8040
-      heartbeat_service_port = yyyy    # Default: 9050
-      brpc_port = zzzz                 # Default: 8060
-      ```
-
-   b. If you want to enable IP address access for your cluster, you must add the configuration item `priority_networks` in the configuration file and assign a dedicated IP address (in the CIDR format) to the CN node. You can ignore this configuration item if you want to enable FQDN access for your cluster.
-
-      ```YAML
-      priority_networks = x.x.x.x/x
-      ```
-
-      > **NOTE**
-      >
-      > - You can run `ifconfig` in your terminal to view the IP address(es) owned by the instance.
-      > - From v3.3.0, StarRocks supports deployment based on IPv6.
-
-   c. If you have multiple JDKs installed on the instance, and you want to use a specific JDK that is different from the one specified in the environment variable `JAVA_HOME`, you must specify the path where the chosen JDK is installed by adding the configuration item `JAVA_HOME` in the configuration file.
-
-      ```YAML
-      # Replace <path_to_JDK> with the path where the chosen JDK is installed.
-      JAVA_HOME = <path_to_JDK>
-      ```
-
-   For information about advanced configuration items, see [Parameter Configuration - BE configuration items](../administration/management/BE_configuration.md) because most of CN's parameters are inherited from BE.
-
-2. Start the CN node.
-
-   ```Bash
-   ./be/bin/start_cn.sh --daemon
-   ```
-
-   > **CAUTION**
-   >
-   > - Before starting the CN node with FQDN access enabled, make sure you have assigned hostnames for all instances in **/etc/hosts**. See [Environment Configuration Checklist - Hostnames](../deployment/environment_configurations.md#hostnames) for more information.
-   > - You do not need to specify the parameter `--host_type` when you start CN nodes.
-
-3. Check the CN logs to verify if the CN node is started successfully.
-
-   ```Bash
-   cat be/log/cn.INFO | grep heartbeat
-   ```
-
-   A record of log like "I0313 15:03:45.820030 412450 thrift_server.cpp:375] heartbeat has started listening port on 9050" suggests that the CN node is started properly.
-
-4. You can start new CN nodes by repeating the above procedures on other instances.
-
-## Step 4: Set up the cluster
-
-After all FE, BE nodes, and CN nodes are started properly, you can set up the StarRocks cluster.
+After all FE and BE nodes are started properly, you can set up the StarRocks cluster.
 
 The following procedures are performed on a MySQL client. You must have MySQL client 5.5.0 or later installed.
 
@@ -331,51 +275,7 @@ The following procedures are performed on a MySQL client. You must have MySQL cl
 
    If the field `Alive` is `true`, this BE node is properly started and added to the cluster.
 
-5. (Optional) Add a CN node to the cluster.
-
-   ```SQL
-   -- Replace <cn_address> with the IP address (priority_networks) 
-   -- or FQDN of the CN node, and replace <heartbeat_service_port> 
-   -- with the heartbeat_service_port (Default: 9050) you specified in cn.conf.
-   ALTER SYSTEM ADD COMPUTE NODE "<cn_address>:<heartbeat_service_port>";
-   ```
-
-   > **NOTE**
-   >
-   > You can add multiple CN nodes with one SQL. Each `<cn_address>:<heartbeat_service_port>` pair represents one CN node.
-
-6. (Optional) Check the status of the CN nodes by executing the following SQL.
-
-   ```SQL
-   SHOW PROC '/compute_nodes'\G
-   ```
-
-   Example:
-
-   ```Plain
-   MySQL [(none)]> SHOW PROC '/compute_nodes'\G
-   *************************** 1. row ***************************
-           ComputeNodeId: 10003
-                      IP: x.x.x.x
-           HeartbeatPort: 9050
-                  BePort: 9060
-                HttpPort: 8040
-                BrpcPort: 8060
-           LastStartTime: 2023-03-13 15:11:13
-           LastHeartbeat: 2023-03-13 15:11:13
-                   Alive: true
-    SystemDecommissioned: false
-   ClusterDecommissioned: false
-                  ErrMsg: 
-                 Version: 2.5.2-c3772fb
-   1 row in set (0.00 sec)
-   ```
-
-   If the field `Alive` is `true`, this CN node is properly started and added to the cluster.
-
-   After CNs are properly started and you want to use CNs during queries, set the system variables `SET prefer_compute_node = true;` and `SET use_compute_nodes = -1;`. For more information, see [System variables](../sql-reference/System_variable.md#descriptions-of-variables).
-
-## Step 5: (Optional) Deploy a high-availability FE cluster
+## Step 4: (Optional) Deploy a high-availability FE cluster
 
 A high-availability FE cluster requires at least THREE Follower FE nodes in the StarRocks cluster. After the Leader FE node is started successfully, you can then start two new FE nodes to deploy a high-availability FE cluster.
 
@@ -524,15 +424,9 @@ You can stop the StarRocks cluster by running the following commands on the corr
   ./be/bin/stop_be.sh --daemon
   ```
 
-- Stop a CN node.
-
-  ```Bash
-  ./be/bin/stop_cn.sh --daemon
-  ```
-
 ## Troubleshooting
 
-Try the following steps to identify the errors that occur when you start the FE, BE, or CN nodes:
+Try the following steps to identify the errors that occur when you start the FE or BE nodes:
 
 - If an FE node is not started properly, you can identify the problem by checking its log in **fe/log/fe.warn.log**.
 
@@ -550,14 +444,6 @@ Try the following steps to identify the errors that occur when you start the FE,
 
   Having identified and resolved the problem, you must first terminate the existing BE process, delete the existing **storage** directory, create a new data storage directory, and then restart the BE node with the correct configuration.
 
-- If a CN node is not started properly, you can identify the problem by checking its log in **be/log/cn.WARNING**.
-
-  ```Bash
-  cat be/log/cn.WARNING
-  ```
-
-  Having identified and resolved the problem, you must first terminate the existing CN process, and then restart the CN node with the correct configuration.
-
 ## What to do next
 
 Having deployed your StarRocks cluster, you can move on to [Post-deployment Setup](../deployment/post_deployment_setup.md) for instructions on initial management measures.
diff --git a/docs/zh/deployment/deploy_manually.md b/docs/zh/deployment/deploy_manually.md
index ca9930ce984043..3c2596e8a5986b 100644
--- a/docs/zh/deployment/deploy_manually.md
+++ b/docs/zh/deployment/deploy_manually.md
@@ -176,67 +176,9 @@ displayed_sidebar: docs
       default_replication_num = 1
       ```
 
-## 第三步：（可选）启动 CN 服务
+## 第三步：搭建集群
 
-Compute Node（CN）是一种无状态的计算服务，本身不存储数据。您可以通过添加 CN 节点为查询提供额外的计算资源。您可以使用 BE 部署文件部署 CN 节点。CN 节点自 v2.4 版本起支持。
-
-1. 进入先前准备好的 [StarRocks BE 部署文件](../deployment/prepare_deployment_files.md)所在路径，修改 CN 配置文件 **be/conf/cn.conf**。
-
-   a. 如果任何在 [环境配置清单](../deployment/environment_configurations.md) 中提到的 CN 端口被占用，您必须在 CN 配置文件中为其分配其他可用端口。.
-
-      ```YAML
-      be_port = vvvv                   # 默认值：9060
-      be_http_port = xxxx              # 默认值：8040
-      heartbeat_service_port = yyyy    # 默认值：9050
-      brpc_port = zzzz                 # 默认值：8060
-      ```
-
-   b. 如需为集群启用 IP 地址访问，您必须在配置文件中添加配置项 `priority_networks`，为 CN 节点分配一个专有的 IP 地址（CIDR格式）。如需为集群启用 FQDN 访问，则可以忽略该配置项。.
-
-      ```YAML
-      priority_networks = x.x.x.x/x
-      ```
-
-      > **说明**
-      >
-      > - 您可以在终端中运行 `ifconfig` 以查看当前实例拥有的 IP 地址。
-      > - 从 v3.3.0 开始，StarRocks 支持基于 IPv6 的部署。
-
-   c. 如果您的实例安装了多个 JDK，并且您要使用 JDK 与环境变量 `JAVA_HOME` 中指定的不同，则必须在配置文件中添加配置项 `JAVA_HOME` 来指定所选该 JDK 的安装路径。
-
-      ```YAML
-      # 将 <path_to_JDK> 替换为所选 JDK 的安装路径。
-      JAVA_HOME = <path_to_JDK>
-      ```
-
-   d.  由于大部分 CN 参数都继承自 BE 节点，您可以参考 [参数配置 - BE 配置项](../administration/management/BE_configuration.md) 了解更多 CN 高级配置项。
-
-2. 启动 CN 节点。
-
-   ```Bash
-   ./be/bin/start_cn.sh --daemon
-   ```
-
-   > **注意**
-   >
-   > - 如需启用 FQDN 访问，在启动 CN 节点之前，请确保您已经在 **/etc/hosts** 中为所有实例分配了主机名。有关详细信息，请参考 [环境配置清单 - 主机名](../deployment/environment_configurations.md#主机名)。
-   > - 启动 CN 节点时无需指定参数 `--host_type`。
-
-3. 查看 CN 日志，检查 CN 节点是否启动成功。
-
-   ```Bash
-   cat be/log/cn.INFO | grep heartbeat
-   ```
-
-   如果日志打印以下内容，则说明该 CN 节点启动成功：
-
-   "I0313 15:03:45.820030 412450 thrift_server.cpp:375] heartbeat has started listening port on 9050"
-
-4. 在其他实例上重复以上步骤，即可启动新的 CN 节点。
-
-## 第四步：搭建集群
-
-当所有 FE、BE、CN 节点启动成功后，即可搭建 StarRocks 集群。
+当所有 FE 和 BE 节点启动成功后，即可搭建 StarRocks 集群。
 
 以下过程在 MySQL 客户端实例上执行。您必须安装 MySQL 客户端（5.5.0 或更高版本）。
 
@@ -335,51 +277,7 @@ Compute Node（CN）是一种无状态的计算服务，本身不存储数据。
 
    如果字段 `Alive` 为 `true`，说明该 BE 节点正常启动并加入集群。
 
-5. （可选）添加 CN 节点至集群。
-
-   ```SQL
-   -- 将 <cn_address> 替换为 CN 节点的 IP 地址（priority_networks）或 FQDN，
-   -- 并将 <heartbeat_service_port>（默认：9050）替换为您在 cn.conf 中指定的 heartbeat_service_port。
-   ALTER SYSTEM ADD COMPUTE NODE "<cn_address>:<heartbeat_service_port>", "<cn2_address>:<heartbeat_service_port>", "<cn3_address>:<heartbeat_service_port>";
-   ```
-
-   > **说明**
-   >
-   > 您可以通过一条 SQL 添加多个 CN 节点。每对 `<cn_address>:<heartbeat_service_port>` 代表一个 CN 节点。
-
-6. （可选）执行以下 SQL 查看 CN 节点状态。
-
-   ```SQL
-   SHOW PROC '/compute_nodes'\G
-   ```
-
-   示例：
-
-   ```Plain
-   MySQL [(none)]> SHOW PROC '/compute_nodes'\G
-   *************************** 1. row ***************************
-           ComputeNodeId: 10003
-                      IP: x.x.x.x
-           HeartbeatPort: 9050
-                  BePort: 9060
-                HttpPort: 8040
-                BrpcPort: 8060
-           LastStartTime: 2023-03-13 15:11:13
-           LastHeartbeat: 2023-03-13 15:11:13
-                   Alive: true
-    SystemDecommissioned: false
-   ClusterDecommissioned: false
-                  ErrMsg: 
-                 Version: 2.5.2-c3772fb
-   1 row in set (0.00 sec)
-   ```
-
-   如果字段 `Alive` 为 `true`，说明该 CN 节点正常启动并加入集群。
-
-   如果执行查询时需要使用 CN 节点扩展算力，则需要设置系统变量 `SET
-prefer_compute_node = true;` 和 `SET use_compute_nodes = -1;`。系统变量的更多信息，请参见[系统变量](../sql-reference/System_variable.md#支持的变量)。
-
-## 第五步：（可选）部署高可用 FE 集群
+## 第四步：（可选）部署高可用 FE 集群
 
 高可用的 FE 集群需要在 StarRocks 集群中部署至少三个 Follower FE 节点。如需部署高可用的 FE 集群，您需要额外再启动两个新的 FE 节点。
 
@@ -527,15 +425,9 @@ prefer_compute_node = true;` 和 `SET use_compute_nodes = -1;`。系统变量的
   ./be/bin/stop_be.sh --daemon
   ```
 
-- 停止 CN 节点。
-
-  ```Bash
-  ./be/bin/stop_cn.sh --daemon
-  ```
-
 ## 故障排除
 
-如果启动 FE、BE 或 CN 节点失败，尝试以下步骤来发现问题：
+如果启动 FE 或 BE 节点失败，尝试以下步骤来发现问题：
 
 - 如果 FE 节点没有正常启动，您可以通过查看 **fe/log/fe.warn.log** 中的日志来确定问题所在。
 
@@ -553,14 +445,6 @@ prefer_compute_node = true;` 和 `SET use_compute_nodes = -1;`。系统变量的
 
   确定并解决问题后，您首先需要终止当前 BE 进程，删除现有的 **storage** 路径，新建数据存储路径，然后以正确的配置重启该 BE 节点。
 
-- 如果 CN 节点没有正常启动，您可以通过查看 **be/log/cn.WARNING** 中的日志来确定问题所在。
-
-  ```Bash
-  cat be/log/cn.WARNING
-  ```
-
-  确定并解决问题后，您首先需要终止当前 CN 进程，然后以正确的配置重启该 CN 节点。
-
 ## 下一步
 
 成功部署 StarRocks 集群后，您可以参考 [部署后设置](../deployment/post_deployment_setup.md) 以获取有关初始管理措施的说明。

From 19682206a63d504bccb76b3d4e6e2e11fe9dee87 Mon Sep 17 00:00:00 2001
From: gengjun-git <gengjun@starrocks.com>
Date: Thu, 16 Jan 2025 16:24:43 +0800
Subject: [PATCH 15/71] [BugFix] Fix NPE in HeartbeatMgr when FE node is not
 available (#54997)

Fixes
```
Cannot invoke "com.starrocks.http.rest.BootstrapFinishAction$BootstrapResult.getStatus()" because "result" is null
```

Signed-off-by: gengjun-git <gengjun@starrocks.com>
---
 .../java/com/starrocks/common/util/Util.java  |  4 +--
 .../com/starrocks/common/util/UtilTest.java   | 27 +++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/com/starrocks/common/util/UtilTest.java

diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/Util.java b/fe/fe-core/src/main/java/com/starrocks/common/util/Util.java
index 1689e8a33188cd..cc319892017b5a 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/util/Util.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/util/Util.java
@@ -356,7 +356,7 @@ public static boolean checkColumnSupported(List<Column> columns) {
     //      Base64.encodeBase64String("user:passwd".getBytes());
     // If no auth info, pass a null.
     public static String getResultForUrl(String urlStr, String encodedAuthInfo, int connectTimeoutMs,
-                                         int readTimeoutMs) {
+                                         int readTimeoutMs) throws Exception {
         StringBuilder sb = new StringBuilder();
         InputStream stream = null;
         String safeUrl = urlStr;
@@ -379,7 +379,7 @@ public static String getResultForUrl(String urlStr, String encodedAuthInfo, int
             }
         } catch (Exception e) {
             LOG.warn("failed to get result from url: {}. {}", safeUrl, e.getMessage());
-            return null;
+            throw e;
         } finally {
             if (stream != null) {
                 try {
diff --git a/fe/fe-core/src/test/java/com/starrocks/common/util/UtilTest.java b/fe/fe-core/src/test/java/com/starrocks/common/util/UtilTest.java
new file mode 100644
index 00000000000000..9c0403ab1d3d13
--- /dev/null
+++ b/fe/fe-core/src/test/java/com/starrocks/common/util/UtilTest.java
@@ -0,0 +1,27 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.common.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class UtilTest {
+
+    @Test
+    public void testGetResultForUrl() {
+        Assert.assertThrows(Exception.class,
+                () -> Util.getResultForUrl("http://127.0.0.1:23/invalid", null, 1000, 1000));
+    }
+}

From 5d145b050f5b71b5555f7cfc2181dadd51283891 Mon Sep 17 00:00:00 2001
From: PengFei Li <lpengfei2016@gmail.com>
Date: Thu, 16 Jan 2025 17:28:02 +0800
Subject: [PATCH 16/71] [Enhancement] Introduce TxnStateCache for merge commit
 sync mode (#55001)

Signed-off-by: PengFei Li <lpengfei2016@gmail.com>
---
 be/src/common/config.h                        |  22 +-
 be/src/http/action/update_config_action.cpp   |  10 +
 be/src/runtime/CMakeLists.txt                 |   1 +
 .../runtime/batch_write/batch_write_mgr.cpp   |  55 +-
 be/src/runtime/batch_write/batch_write_mgr.h  |  17 +-
 be/src/runtime/batch_write/batch_write_util.h |   2 +-
 .../batch_write/isomorphic_batch_write.cpp    | 105 +--
 .../batch_write/isomorphic_batch_write.h      |   7 +-
 .../runtime/batch_write/txn_state_cache.cpp   | 495 +++++++++++++
 be/src/runtime/batch_write/txn_state_cache.h  | 278 +++++++
 be/src/runtime/exec_env.cpp                   |   8 +-
 be/src/service/internal_service.cpp           |  11 +-
 be/src/service/internal_service.h             |   4 +
 be/src/util/dynamic_cache.h                   |  55 +-
 be/test/CMakeLists.txt                        |   1 +
 .../batch_write/batch_write_mgr_test.cpp      |  54 ++
 .../isomorphic_batch_write_test.cpp           | 103 ++-
 .../batch_write/txn_state_cache_test.cpp      | 694 ++++++++++++++++++
 be/test/util/dynamic_cache_test.cpp           |  15 +
 19 files changed, 1768 insertions(+), 169 deletions(-)
 create mode 100644 be/src/runtime/batch_write/txn_state_cache.cpp
 create mode 100644 be/src/runtime/batch_write/txn_state_cache.h
 create mode 100644 be/test/runtime/batch_write/txn_state_cache_test.cpp

diff --git a/be/src/common/config.h b/be/src/common/config.h
index c7abe566a1b05f..09e40a9a8b73db 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1526,15 +1526,19 @@ CONF_mInt32(apply_version_slow_log_sec, "30");
 CONF_mInt32(merge_commit_stream_load_pipe_block_wait_us, "500");
 // The maximum number of bytes that the merge commit stream load pipe can buffer.
 CONF_mInt64(merge_commit_stream_load_pipe_max_buffered_bytes, "1073741824");
-CONF_Int32(batch_write_thread_pool_num_min, "0");
-CONF_Int32(batch_write_thread_pool_num_max, "512");
-CONF_Int32(batch_write_thread_pool_queue_size, "4096");
-CONF_mInt32(batch_write_default_timeout_ms, "600000");
-CONF_mInt32(batch_write_rpc_request_retry_num, "10");
-CONF_mInt32(batch_write_rpc_request_retry_interval_ms, "500");
-CONF_mInt32(batch_write_rpc_reqeust_timeout_ms, "10000");
-CONF_mInt32(batch_write_poll_load_status_interval_ms, "200");
-CONF_mBool(batch_write_trace_log_enable, "false");
+CONF_Int32(merge_commit_thread_pool_num_min, "0");
+CONF_Int32(merge_commit_thread_pool_num_max, "512");
+CONF_Int32(merge_commit_thread_pool_queue_size, "4096");
+CONF_mInt32(merge_commit_default_timeout_ms, "600000");
+CONF_mInt32(merge_commit_rpc_request_retry_num, "10");
+CONF_mInt32(merge_commit_rpc_request_retry_interval_ms, "500");
+CONF_mInt32(merge_commit_rpc_reqeust_timeout_ms, "10000");
+CONF_mBool(merge_commit_trace_log_enable, "false");
+CONF_mInt32(merge_commit_txn_state_cache_capacity, "4096");
+CONF_mInt32(merge_commit_txn_state_clean_interval_sec, "300");
+CONF_mInt32(merge_commit_txn_state_expire_time_sec, "1800");
+CONF_mInt32(merge_commit_txn_state_poll_interval_ms, "2000");
+CONF_mInt32(merge_commit_txn_state_poll_max_fail_times, "2");
 
 CONF_mBool(enable_load_spill, "false");
 // Max chunk bytes which allow to spill per flush. Default is 10MB.
diff --git a/be/src/http/action/update_config_action.cpp b/be/src/http/action/update_config_action.cpp
index a3bfcad688f61d..aeadfc37da0ebb 100644
--- a/be/src/http/action/update_config_action.cpp
+++ b/be/src/http/action/update_config_action.cpp
@@ -52,6 +52,8 @@
 #include "http/http_headers.h"
 #include "http/http_request.h"
 #include "http/http_status.h"
+#include "runtime/batch_write/batch_write_mgr.h"
+#include "runtime/batch_write/txn_state_cache.h"
 #include "storage/compaction_manager.h"
 #include "storage/lake/compaction_scheduler.h"
 #include "storage/lake/load_spill_block_manager.h"
@@ -327,6 +329,14 @@ Status UpdateConfigAction::update_config(const std::string& name, const std::str
         _config_callback.emplace("load_spill_max_merge_bytes", [&]() -> Status {
             return StorageEngine::instance()->load_spill_block_merge_executor()->refresh_max_thread_num();
         });
+        _config_callback.emplace("merge_commit_txn_state_cache_capacity", [&]() -> Status {
+            LOG(INFO) << "set merge_commit_txn_state_cache_capacity: " << config::merge_commit_txn_state_cache_capacity;
+            auto batch_write_mgr = _exec_env->batch_write_mgr();
+            if (batch_write_mgr) {
+                batch_write_mgr->txn_state_cache()->set_capacity(config::merge_commit_txn_state_cache_capacity);
+            }
+            return Status::OK();
+        });
 
 #ifdef USE_STAROS
 #define UPDATE_STARLET_CONFIG(BE_CONFIG, STARLET_CONFIG)                                             \
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index 15e45ad435c555..88ff203f07730a 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -62,6 +62,7 @@ set(RUNTIME_FILES
     batch_write/isomorphic_batch_write.cpp
     batch_write/batch_write_mgr.cpp
     batch_write/batch_write_util.cpp
+    batch_write/txn_state_cache.cpp
     routine_load/data_consumer.cpp
     routine_load/data_consumer_group.cpp
     routine_load/data_consumer_pool.cpp
diff --git a/be/src/runtime/batch_write/batch_write_mgr.cpp b/be/src/runtime/batch_write/batch_write_mgr.cpp
index 9bbf9ebee286a5..2aa157f507bc25 100644
--- a/be/src/runtime/batch_write/batch_write_mgr.cpp
+++ b/be/src/runtime/batch_write/batch_write_mgr.cpp
@@ -25,6 +25,15 @@
 
 namespace starrocks {
 
+BatchWriteMgr::BatchWriteMgr(std::unique_ptr<bthreads::ThreadPoolExecutor> executor) : _executor(std::move(executor)) {}
+
+Status BatchWriteMgr::init() {
+    std::unique_ptr<ThreadPoolToken> token =
+            _executor->get_thread_pool()->new_token(ThreadPool::ExecutionMode::CONCURRENT);
+    _txn_state_cache = std::make_unique<TxnStateCache>(config::merge_commit_txn_state_cache_capacity, std::move(token));
+    return _txn_state_cache->init();
+}
+
 Status BatchWriteMgr::register_stream_load_pipe(StreamLoadContext* pipe_ctx) {
     BatchWriteId batch_write_id = {
             .db = pipe_ctx->db, .table = pipe_ctx->table, .load_params = pipe_ctx->load_parameters};
@@ -78,7 +87,7 @@ StatusOr<IsomorphicBatchWriteSharedPtr> BatchWriteMgr::_get_batch_write(const st
         return it->second;
     }
 
-    auto batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get());
+    auto batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get(), _txn_state_cache.get());
     Status st = batch_write->init();
     if (!st.ok()) {
         LOG(ERROR) << "Fail to init batch write, " << batch_write_id << ", status: " << st;
@@ -105,6 +114,10 @@ void BatchWriteMgr::stop() {
     for (auto& batch_write : stop_writes) {
         batch_write->stop();
     }
+    if (_txn_state_cache) {
+        _txn_state_cache->stop();
+    }
+    _executor->get_thread_pool()->shutdown();
 }
 
 StatusOr<StreamLoadContext*> BatchWriteMgr::create_and_register_pipe(
@@ -224,7 +237,45 @@ void BatchWriteMgr::receive_stream_load_rpc(ExecEnv* exec_env, brpc::Controller*
     ctx->buffer->flip();
     ctx->receive_bytes = io_buf.size();
     ctx->mc_read_data_cost_nanos = MonotonicNanos() - ctx->start_nanos;
-    ctx->status = exec_env->batch_write_mgr()->append_data(ctx);
+    ctx->status = append_data(ctx);
+}
+
+static TTransactionStatus::type to_thrift_txn_status(TransactionStatusPB status) {
+    switch (status) {
+    case TRANS_UNKNOWN:
+        return TTransactionStatus::UNKNOWN;
+    case TRANS_PREPARE:
+        return TTransactionStatus::PREPARE;
+    case TRANS_COMMITTED:
+        return TTransactionStatus::COMMITTED;
+    case TRANS_VISIBLE:
+        return TTransactionStatus::VISIBLE;
+    case TRANS_ABORTED:
+        return TTransactionStatus::ABORTED;
+    case TRANS_PREPARED:
+        return TTransactionStatus::PREPARED;
+    default:
+        return TTransactionStatus::UNKNOWN;
+    }
+}
+
+void BatchWriteMgr::update_transaction_state(const PUpdateTransactionStateRequest* request,
+                                             PUpdateTransactionStateResponse* response) {
+    for (int i = 0; i < request->states_size(); i++) {
+        auto& txn_state = request->states(i);
+        auto st = _txn_state_cache->push_state(txn_state.txn_id(), to_thrift_txn_status(txn_state.status()),
+                                               txn_state.reason());
+        if (!st.ok()) {
+            LOG(WARNING) << "Failed to update transaction state, txn_id: " << txn_state.txn_id()
+                         << ", txn status: " << TransactionStatusPB_Name(txn_state.status())
+                         << ", status reason: " << txn_state.reason() << ", update error: " << st;
+        } else {
+            TRACE_BATCH_WRITE << "Update transaction state, txn_id: " << txn_state.txn_id()
+                              << ", txn status: " << TransactionStatusPB_Name(txn_state.status())
+                              << ", status reason: " << txn_state.reason();
+        }
+        st.to_protobuf(response->add_results());
+    }
 }
 
 } // namespace starrocks
\ No newline at end of file
diff --git a/be/src/runtime/batch_write/batch_write_mgr.h b/be/src/runtime/batch_write/batch_write_mgr.h
index 0b860a27de01da..801bb013bfaec4 100644
--- a/be/src/runtime/batch_write/batch_write_mgr.h
+++ b/be/src/runtime/batch_write/batch_write_mgr.h
@@ -18,6 +18,7 @@
 
 #include "common/statusor.h"
 #include "runtime/batch_write/isomorphic_batch_write.h"
+#include "runtime/batch_write/txn_state_cache.h"
 #include "runtime/stream_load/stream_load_context.h"
 #include "util/bthreads/bthread_shared_mutex.h"
 #include "util/bthreads/executor.h"
@@ -32,10 +33,13 @@ class ExecEnv;
 class PStreamLoadRequest;
 class PStreamLoadResponse;
 class StreamLoadContext;
+class PUpdateTransactionStateRequest;
+class PUpdateTransactionStateResponse;
 
 class BatchWriteMgr {
 public:
-    BatchWriteMgr(std::unique_ptr<bthreads::ThreadPoolExecutor> executor) : _executor(std::move(executor)){};
+    BatchWriteMgr(std::unique_ptr<bthreads::ThreadPoolExecutor> executor);
+    Status init();
 
     Status register_stream_load_pipe(StreamLoadContext* pipe_ctx);
     void unregister_stream_load_pipe(StreamLoadContext* pipe_ctx);
@@ -45,19 +49,26 @@ class BatchWriteMgr {
 
     void stop();
 
+    bthreads::ThreadPoolExecutor* executor() { return _executor.get(); }
+    TxnStateCache* txn_state_cache() { return _txn_state_cache.get(); }
+
     static StatusOr<StreamLoadContext*> create_and_register_pipe(
             ExecEnv* exec_env, BatchWriteMgr* batch_write_mgr, const string& db, const string& table,
             const std::map<std::string, std::string>& load_parameters, const string& label, long txn_id,
             const TUniqueId& load_id, int32_t batch_write_interval_ms);
 
-    static void receive_stream_load_rpc(ExecEnv* exec_env, brpc::Controller* cntl, const PStreamLoadRequest* request,
-                                        PStreamLoadResponse* response);
+    void receive_stream_load_rpc(ExecEnv* exec_env, brpc::Controller* cntl, const PStreamLoadRequest* request,
+                                 PStreamLoadResponse* response);
+
+    void update_transaction_state(const PUpdateTransactionStateRequest* request,
+                                  PUpdateTransactionStateResponse* response);
 
 private:
     StatusOr<IsomorphicBatchWriteSharedPtr> _get_batch_write(const BatchWriteId& batch_write_id,
                                                              bool create_if_missing);
 
     std::unique_ptr<bthreads::ThreadPoolExecutor> _executor;
+    std::unique_ptr<TxnStateCache> _txn_state_cache;
     bthreads::BThreadSharedMutex _rw_mutex;
     std::unordered_map<BatchWriteId, IsomorphicBatchWriteSharedPtr, BatchWriteIdHash, BatchWriteIdEqual>
             _batch_write_map;
diff --git a/be/src/runtime/batch_write/batch_write_util.h b/be/src/runtime/batch_write/batch_write_util.h
index dee7c3c615aab4..bf6fb12324faf5 100644
--- a/be/src/runtime/batch_write/batch_write_util.h
+++ b/be/src/runtime/batch_write/batch_write_util.h
@@ -22,7 +22,7 @@
 
 namespace starrocks {
 
-#define TRACE_BATCH_WRITE LOG_IF(INFO, config::batch_write_trace_log_enable)
+#define TRACE_BATCH_WRITE LOG_IF(INFO, config::merge_commit_trace_log_enable)
 
 using BatchWriteLoadParams = std::map<std::string, std::string>;
 
diff --git a/be/src/runtime/batch_write/isomorphic_batch_write.cpp b/be/src/runtime/batch_write/isomorphic_batch_write.cpp
index 10fade35811d94..22b81b6bd47241 100644
--- a/be/src/runtime/batch_write/isomorphic_batch_write.cpp
+++ b/be/src/runtime/batch_write/isomorphic_batch_write.cpp
@@ -119,8 +119,9 @@ class AsyncAppendDataContext {
     std::atomic_int num_retries{-1};
 };
 
-IsomorphicBatchWrite::IsomorphicBatchWrite(BatchWriteId batch_write_id, bthreads::ThreadPoolExecutor* executor)
-        : _batch_write_id(std::move(batch_write_id)), _executor(executor) {}
+IsomorphicBatchWrite::IsomorphicBatchWrite(BatchWriteId batch_write_id, bthreads::ThreadPoolExecutor* executor,
+                                           TxnStateCache* txn_state_cache)
+        : _batch_write_id(std::move(batch_write_id)), _executor(executor), _txn_state_cache(txn_state_cache) {}
 
 Status IsomorphicBatchWrite::init() {
     TEST_ERROR_POINT("IsomorphicBatchWrite::init::error");
@@ -220,7 +221,6 @@ Status IsomorphicBatchWrite::append_data(StreamLoadContext* data_ctx) {
     if (_stopped.load(std::memory_order_acquire)) {
         return Status::ServiceUnavailable("Batch write is stopped");
     }
-    int64_t start_ts = MonotonicNanos();
     AsyncAppendDataContext* async_ctx = new AsyncAppendDataContext(data_ctx);
     async_ctx->ref();
     async_ctx->create_time_ts.store(MonotonicNanos());
@@ -258,10 +258,7 @@ Status IsomorphicBatchWrite::append_data(StreamLoadContext* data_ctx) {
     if (_batch_write_async) {
         return Status::OK();
     }
-    int64_t timeout_ms =
-            data_ctx->timeout_second > 0 ? data_ctx->timeout_second * 1000 : config::batch_write_default_timeout_ms;
-    int64_t left_timeout_ns = std::max((int64_t)0, timeout_ms * 1000 * 1000 - (MonotonicNanos() - start_ts));
-    return _wait_for_load_status(data_ctx, left_timeout_ns);
+    return _wait_for_load_finish(data_ctx);
 }
 
 int IsomorphicBatchWrite::_execute_tasks(void* meta, bthread::TaskIterator<Task>& iter) {
@@ -309,7 +306,7 @@ Status IsomorphicBatchWrite::_execute_write(AsyncAppendDataContext* async_ctx) {
             SCOPED_RAW_TIMER(&write_data_cost_ns);
             st = _write_data_to_pipe(async_ctx);
         }
-        if (st.ok() || num_retries >= config::batch_write_rpc_request_retry_num) {
+        if (st.ok() || num_retries >= config::merge_commit_rpc_request_retry_num) {
             break;
         }
         num_retries += 1;
@@ -324,7 +321,7 @@ Status IsomorphicBatchWrite::_execute_write(AsyncAppendDataContext* async_ctx) {
             SCOPED_RAW_TIMER(&wait_pipe_cost_ns);
             std::unique_lock<bthread::Mutex> lock(_mutex);
             if (_alive_stream_load_pipe_ctxs.empty()) {
-                _cv.wait_for(lock, config::batch_write_rpc_request_retry_interval_ms * 1000);
+                _cv.wait_for(lock, config::merge_commit_rpc_request_retry_interval_ms * 1000);
             }
         }
     }
@@ -404,7 +401,7 @@ Status IsomorphicBatchWrite::_send_rpc_request(StreamLoadContext* data_ctx) {
     st = ThriftRpcHelper::rpc<FrontendServiceClient>(
             master_addr.hostname, master_addr.port,
             [&request, &response](FrontendServiceConnection& client) { client->requestMergeCommit(response, request); },
-            config::batch_write_rpc_reqeust_timeout_ms);
+            config::merge_commit_rpc_reqeust_timeout_ms);
     TRACE_BATCH_WRITE << "receive requestBatchWrite response, " << _batch_write_id
                       << ", user label: " << data_ctx->label << ", master: " << master_addr
                       << ", cost: " << ((MonotonicNanos() - start_ts) / 1000) << "us, status: " << st
@@ -418,78 +415,42 @@ Status IsomorphicBatchWrite::_send_rpc_request(StreamLoadContext* data_ctx) {
     return st.ok() ? Status(response.status) : st;
 }
 
-bool is_final_load_status(const TTransactionStatus::type& status) {
-    switch (status) {
-    case TTransactionStatus::VISIBLE:
-    case TTransactionStatus::ABORTED:
-    case TTransactionStatus::UNKNOWN:
-        return true;
-    default:
-        return false;
+Status IsomorphicBatchWrite::_wait_for_load_finish(StreamLoadContext* data_ctx) {
+    int64_t total_timeout_ms =
+            data_ctx->timeout_second > 0 ? data_ctx->timeout_second * 1000 : config::merge_commit_default_timeout_ms;
+    int64_t left_timeout_ms =
+            std::max((int64_t)0, total_timeout_ms - (MonotonicNanos() - data_ctx->start_nanos) / 1000000);
+    StatusOr<TxnStateSubscriberPtr> subscriber_status = _txn_state_cache->subscribe_state(
+            data_ctx->txn_id, data_ctx->label, data_ctx->db, data_ctx->table, data_ctx->auth);
+    if (!subscriber_status.ok()) {
+        return Status::InternalError("Failed to create txn state subscriber, " +
+                                     subscriber_status.status().to_string());
     }
-}
-
-// TODO just poll the load status periodically. improve it later, such as cache the label, and FE notify the BE
-Status IsomorphicBatchWrite::_wait_for_load_status(StreamLoadContext* data_ctx, int64_t timeout_ns) {
+    TxnStateSubscriberPtr subscriber = std::move(subscriber_status.value());
     int64_t start_ts = MonotonicNanos();
-    int64_t wait_load_finish_ns = std::max((int64_t)0, data_ctx->mc_left_merge_time_nanos) + 1000000;
-    bthread_usleep(std::min(wait_load_finish_ns, timeout_ns) / 1000);
-    TGetLoadTxnStatusRequest request;
-    request.__set_db(_batch_write_id.db);
-    request.__set_tbl(_batch_write_id.table);
-    request.__set_txnId(data_ctx->txn_id);
-    set_request_auth(&request, data_ctx->auth);
-    TGetLoadTxnStatusResult response;
-    Status st;
-    do {
-        if (_stopped.load(std::memory_order_acquire)) {
-            return Status::ServiceUnavailable("Batch write is stopped");
-        }
-#ifndef BE_TEST
-        int64_t rpc_ts = MonotonicNanos();
-        TNetworkAddress master_addr = get_master_address();
-        st = ThriftRpcHelper::rpc<FrontendServiceClient>(
-                master_addr.hostname, master_addr.port,
-                [&request, &response](FrontendServiceConnection& client) {
-                    client->getLoadTxnStatus(response, request);
-                },
-                config::batch_write_rpc_reqeust_timeout_ms);
-        TRACE_BATCH_WRITE << "receive getLoadTxnStatus response, " << _batch_write_id
-                          << ", user label: " << data_ctx->label << ", txn_id: " << data_ctx->txn_id
-                          << ", label: " << data_ctx->batch_write_label << ", master: " << master_addr
-                          << ", cost: " << ((MonotonicNanos() - rpc_ts) / 1000) << "us, status: " << st
-                          << ", response: " << response;
-#else
-        TEST_SYNC_POINT_CALLBACK("IsomorphicBatchWrite::_wait_for_load_status::request", &request);
-        TEST_SYNC_POINT_CALLBACK("IsomorphicBatchWrite::_wait_for_load_status::status", &st);
-        TEST_SYNC_POINT_CALLBACK("IsomorphicBatchWrite::_wait_for_load_status::response", &response);
-#endif
-        if (st.ok() && is_final_load_status(response.status)) {
-            break;
-        }
-        int64_t left_timeout_ns = timeout_ns - (MonotonicNanos() - start_ts);
-        if (left_timeout_ns <= 0) {
-            break;
-        }
-        bthread_usleep(
-                std::min(config::batch_write_poll_load_status_interval_ms * (int64_t)1000, left_timeout_ns / 1000));
-    } while (true);
+    StatusOr<TxnState> status_or = subscriber->wait_finished_state(left_timeout_ms * 1000);
     data_ctx->mc_wait_finish_cost_nanos = MonotonicNanos() - start_ts;
-    if (!st.ok()) {
-        return Status::InternalError("Failed to get load status, " + st.to_string());
+    TRACE_BATCH_WRITE << "finish to wait load, " << _batch_write_id << ", user label: " << data_ctx->label
+                      << ", txn_id: " << data_ctx->txn_id << ", load label: " << data_ctx->batch_write_label
+                      << ", cost: " << (data_ctx->mc_wait_finish_cost_nanos / 1000)
+                      << "us, wait status: " << status_or.status() << ", "
+                      << (status_or.ok() ? status_or.value() : subscriber->current_state());
+    if (!status_or.ok()) {
+        TxnState current_state = subscriber->current_state();
+        return Status::InternalError(fmt::format("Failed to get load final status, current status: {}, error: {}",
+                                                 to_string(current_state.txn_status), status_or.status().to_string()));
     }
-    switch (response.status) {
-    case TTransactionStatus::PREPARE:
-    case TTransactionStatus::PREPARED:
-        return Status::TimedOut("load timeout, txn status: " + to_string(response.status));
+    switch (status_or.value().txn_status) {
     case TTransactionStatus::COMMITTED:
         return Status::PublishTimeout("Load has not been published before timeout");
     case TTransactionStatus::VISIBLE:
         return Status::OK();
     case TTransactionStatus::ABORTED:
-        return Status::InternalError("Load is aborted, reason: " + response.reason);
+        return Status::InternalError("Load is aborted, reason: " + status_or.value().reason);
+    case TTransactionStatus::UNKNOWN:
+        return Status::InternalError("Can't find the transaction, reason: " + status_or.value().reason);
     default:
-        return Status::InternalError("Load status is unknown: " + to_string(response.status));
+        return Status::InternalError("Load status is not final: " + to_string(status_or.value().txn_status));
     }
 }
 
diff --git a/be/src/runtime/batch_write/isomorphic_batch_write.h b/be/src/runtime/batch_write/isomorphic_batch_write.h
index c74b9f333ff7bb..c93bc176f0bbdb 100644
--- a/be/src/runtime/batch_write/isomorphic_batch_write.h
+++ b/be/src/runtime/batch_write/isomorphic_batch_write.h
@@ -25,6 +25,7 @@
 
 #include "common/statusor.h"
 #include "runtime/batch_write/batch_write_util.h"
+#include "runtime/batch_write/txn_state_cache.h"
 #include "util/countdown_latch.h"
 
 namespace starrocks {
@@ -44,7 +45,8 @@ struct Task {
 
 class IsomorphicBatchWrite {
 public:
-    explicit IsomorphicBatchWrite(BatchWriteId batch_write_id, bthreads::ThreadPoolExecutor* executor);
+    explicit IsomorphicBatchWrite(BatchWriteId batch_write_id, bthreads::ThreadPoolExecutor* executor,
+                                  TxnStateCache* txn_state_cache);
 
     Status init();
 
@@ -65,10 +67,11 @@ class IsomorphicBatchWrite {
     Status _execute_write(AsyncAppendDataContext* async_ctx);
     Status _write_data_to_pipe(AsyncAppendDataContext* data_ctx);
     Status _send_rpc_request(StreamLoadContext* data_ctx);
-    Status _wait_for_load_status(StreamLoadContext* data_ctx, int64_t timeout_ns);
+    Status _wait_for_load_finish(StreamLoadContext* data_ctx);
 
     BatchWriteId _batch_write_id;
     bthreads::ThreadPoolExecutor* _executor;
+    TxnStateCache* _txn_state_cache;
     bool _batch_write_async{false};
 
     bthread::Mutex _mutex;
diff --git a/be/src/runtime/batch_write/txn_state_cache.cpp b/be/src/runtime/batch_write/txn_state_cache.cpp
new file mode 100644
index 00000000000000..aa7a4fc5478a69
--- /dev/null
+++ b/be/src/runtime/batch_write/txn_state_cache.cpp
@@ -0,0 +1,495 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "runtime/batch_write/txn_state_cache.h"
+
+#include <utility>
+
+#include "agent/master_info.h"
+#include "gen_cpp/FrontendService.h"
+#include "runtime/batch_write/batch_write_util.h"
+#include "runtime/client_cache.h"
+#include "util/thread.h"
+#include "util/thrift_rpc_helper.h"
+
+namespace starrocks {
+
+TxnStateHandler::~TxnStateHandler() {
+    TEST_SYNC_POINT_CALLBACK("TxnStateHandler::destruct", this);
+    TRACE_BATCH_WRITE << "evict txn state, " << debug_string();
+}
+
+void TxnStateHandler::push_state(TTransactionStatus::type new_status, const std::string& reason) {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    if (_stopped) {
+        return;
+    }
+    _transition_txn_state(new_status, reason, true);
+    if (_is_finished_txn_state()) {
+        _cv.notify_all();
+    }
+}
+
+bool TxnStateHandler::poll_state(const StatusOr<TxnState>& result) {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    if (_stopped) {
+        return false;
+    }
+    if (!result.status().ok()) {
+        _num_poll_failure += 1;
+        TRACE_BATCH_WRITE << "handler poll failure, txn_id: " << _txn_id << ", num_poll_failure: " << _num_poll_failure
+                          << ", status: " << result.status();
+        // fast fail if there is failure between FE and BE
+        if (_num_poll_failure >= config::merge_commit_txn_state_poll_max_fail_times) {
+            _transition_txn_state(TTransactionStatus::UNKNOWN,
+                                  fmt::format("poll txn state failure exceeds max times {}, last error: {}",
+                                              _num_poll_failure, result.status().to_string(false)),
+                                  false);
+        }
+    } else {
+        TRACE_BATCH_WRITE << "handler poll success, txn_id: " << _txn_id << ", " << result.value();
+        _num_poll_failure = 0;
+        _transition_txn_state(result.value().txn_status, result.value().reason, false);
+    }
+    // stop polling if reach the finished state or there is no subscriber
+    if (_is_finished_txn_state()) {
+        _cv.notify_all();
+        return false;
+    } else {
+        return _num_subscriber > 0;
+    }
+}
+
+void TxnStateHandler::subscribe(bool& trigger_poll) {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    _num_subscriber++;
+    // trigger polling if this is the first subscriber and not in finished state
+    trigger_poll = _num_subscriber == 1 && !_is_finished_txn_state();
+}
+
+void TxnStateHandler::unsubscribe() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    _num_subscriber--;
+}
+
+StatusOr<TxnState> TxnStateHandler::wait_finished_state(const std::string& subscriber_name, int64_t timeout_us) {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    if (_is_finished_txn_state()) {
+        return _txn_state;
+    }
+    if (_stopped) {
+        return Status::ServiceUnavailable("Transaction state handler is stopped");
+    }
+    _num_waiting_finished_state++;
+    DeferOp defer([&] { _num_waiting_finished_state--; });
+
+    int64_t left_timeout_us = timeout_us;
+    while (left_timeout_us > 0) {
+        TRACE_BATCH_WRITE << "start to wait state, subscriber name: " << subscriber_name << ", txn_id: " << _txn_id
+                          << ", timeout_us: " << left_timeout_us;
+        auto start_us = MonotonicMicros();
+        int ret = _cv.wait_for(lock, left_timeout_us);
+        int64_t elapsed_us = MonotonicMicros() - start_us;
+        TRACE_BATCH_WRITE << "finish to wait state, subscriber name: " << subscriber_name << ", txn_id: " << _txn_id
+                          << ", elapsed: " << elapsed_us << " us, txn_status: " << to_string(_txn_state.txn_status)
+                          << ", reason: " << _txn_state.reason << ", stopped: " << _stopped;
+        if (_is_finished_txn_state()) {
+            return _txn_state;
+        } else if (_stopped) {
+            return Status::ServiceUnavailable("Transaction state handler is stopped");
+        } else if (ret == ETIMEDOUT) {
+            break;
+        }
+        left_timeout_us = std::max((int64_t)0, left_timeout_us - elapsed_us);
+    }
+    return Status::TimedOut(fmt::format("Wait txn state timeout {} us", timeout_us));
+}
+
+void TxnStateHandler::stop() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    if (_stopped) {
+        return;
+    }
+    _stopped = true;
+    _cv.notify_all();
+}
+
+std::string TxnStateHandler::debug_string() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    return fmt::format(
+            "txn_id: {}, txn_status: {}, reason: {}, num_subscriber: {}, num_waiting_finished_state: {}, stopped: {}",
+            _txn_id.load(), to_string(_txn_state.txn_status), _txn_state.reason, _num_subscriber,
+            _num_waiting_finished_state, _stopped);
+}
+
+void TxnStateHandler::_transition_txn_state(TTransactionStatus::type new_status, const std::string& reason,
+                                            bool from_fe) {
+    TTransactionStatus::type old_status = _txn_state.txn_status;
+    TRACE_BATCH_WRITE << "receive new txn state, txn_id: " << _txn_id
+                      << ", current status: " << to_string(_txn_state.txn_status)
+                      << ", current reason: " << _txn_state.reason << ", new status: " << new_status
+                      << ", new reason: " << reason << ", from_fe: " << from_fe;
+    // special case for COMMITTED status. If it's notified by FE, it means the load finished with
+    // publish timeout, _is_finished_txn_state() should return true, and notify subscribers.
+    if (new_status == TTransactionStatus::COMMITTED && from_fe) {
+        _committed_status_from_fe = from_fe;
+    }
+    if (old_status == TTransactionStatus::VISIBLE || old_status == TTransactionStatus::ABORTED ||
+        old_status == TTransactionStatus::UNKNOWN) {
+        return;
+    } else if (old_status == TTransactionStatus::PREPARED && new_status == TTransactionStatus::PREPARE) {
+        return;
+    } else if (old_status == TTransactionStatus::COMMITTED &&
+               (new_status != TTransactionStatus::VISIBLE && new_status != TTransactionStatus::UNKNOWN)) {
+        return;
+    }
+    _txn_state.txn_status = new_status;
+    _txn_state.reason = reason;
+}
+
+bool TxnStateHandler::_is_finished_txn_state() {
+    // The load can be successful or failed. When successful, the transaction status is VISIBLE.
+    // When failed, the transaction status can be COMMITTED, ABORTED, or UNKNOWN. COMMITTED is a
+    // special status
+    switch (_txn_state.txn_status) {
+    case TTransactionStatus::VISIBLE:
+    case TTransactionStatus::ABORTED:
+    case TTransactionStatus::UNKNOWN:
+        return true;
+    case TTransactionStatus::COMMITTED:
+        return _committed_status_from_fe;
+    default:
+        return false;
+    }
+}
+
+StatusOr<TxnState> TxnStateSubscriber::wait_finished_state(int64_t timeout_us) {
+    return _entry->value().wait_finished_state(_name, timeout_us);
+}
+
+TxnState TxnStateSubscriber::current_state() {
+    return _entry->value().txn_state();
+}
+
+inline int64_t get_current_ms() {
+    int64_t current_ts = MonotonicMillis();
+    TEST_SYNC_POINT_CALLBACK("TxnStatePoller::get_current_ms", &current_ts);
+    return current_ts;
+}
+
+Status TxnStatePoller::init() {
+    _schedule_thread = std::make_unique<std::thread>([this] { _schedule_func(); });
+    Thread::set_thread_name(*_schedule_thread.get(), "txn_state_sche");
+    return Status::OK();
+}
+
+void TxnStatePoller::submit(const TxnStatePollTask& task, int64_t delay_ms) {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    if (_stopped) {
+        return;
+    }
+    if (_pending_txn_ids.find(task.txn_id) != _pending_txn_ids.end()) {
+        return;
+    }
+    int64_t execute_time = get_current_ms() + delay_ms;
+    _pending_txn_ids.emplace(task.txn_id);
+    _pending_tasks.emplace(std::make_pair(execute_time, task));
+    _cv.notify_all();
+    TRACE_BATCH_WRITE << "submit poll task, txn_id: " << task.txn_id << ", db: " << task.db << ", tbl: " << task.tbl
+                      << ", delay_ms: " << delay_ms;
+}
+
+void TxnStatePoller::stop() {
+    {
+        std::unique_lock<bthread::Mutex> lock(_mutex);
+        if (_stopped) {
+            return;
+        }
+        _stopped = true;
+        _cv.notify_all();
+    }
+    if (_schedule_thread && _schedule_thread->joinable()) {
+        _schedule_thread->join();
+    }
+}
+
+void TxnStatePoller::_schedule_func() {
+    std::vector<TxnStatePollTask> poll_tasks;
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    _is_scheduling = true;
+    while (!_stopped) {
+        int64_t current_ts = get_current_ms();
+        auto it = _pending_tasks.begin();
+        while (it != _pending_tasks.end()) {
+            if (it->first <= current_ts) {
+                _pending_txn_ids.erase(it->second.txn_id);
+                poll_tasks.emplace_back(it->second);
+                it = _pending_tasks.erase(it);
+            } else {
+                break;
+            }
+        }
+        if (!poll_tasks.empty()) {
+            lock.unlock();
+            _schedule_poll_tasks(poll_tasks);
+            poll_tasks.clear();
+            lock.lock();
+        }
+        if (_stopped) {
+            break;
+        }
+        if (_pending_tasks.empty()) {
+            _cv.wait(lock);
+        } else {
+            // at least wait 50 ms to avoid busy loop
+            int64_t wait_time_ms = std::max((int64_t)50, _pending_tasks.begin()->first - get_current_ms());
+            _cv.wait_for(lock, wait_time_ms * 1000);
+        }
+    }
+    _is_scheduling = false;
+}
+
+void TxnStatePoller::_schedule_poll_tasks(const std::vector<TxnStatePollTask>& poll_tasks) {
+    for (const auto& task : poll_tasks) {
+        // check current state of the txn, if it's not in cache or is final, skip poll
+        auto current_state = _txn_state_cache->get_state(task.txn_id);
+        if (!current_state.ok()) {
+            TRACE_BATCH_WRITE << "skip poll task because fail to get txn state, txn_id: " << task.txn_id
+                              << ", db: " << task.db << ", tbl: " << task.tbl << ", error: " << current_state.status();
+            continue;
+        }
+        TTransactionStatus::type txn_status = current_state.value().txn_status;
+        if (txn_status == TTransactionStatus::VISIBLE || txn_status == TTransactionStatus::ABORTED ||
+            txn_status == TTransactionStatus::UNKNOWN) {
+            TRACE_BATCH_WRITE << "skip poll task because txn state is final, txn_id: " << task.txn_id
+                              << ", db: " << task.db << ", tbl: " << task.tbl << ", state status: " << txn_status
+                              << ", reason: " << current_state.value().reason;
+            continue;
+        }
+        Status status = _poll_token->submit_func([this, task] { _execute_poll(task); }, ThreadPool::HIGH_PRIORITY);
+        if (!status.ok()) {
+            _txn_state_cache->_notify_poll_result(
+                    task, Status::InternalError("failed to submit poll txn state task, error: " + status.to_string()));
+        } else {
+            TRACE_BATCH_WRITE << "schedule poll task, txn_id: " << task.txn_id << ", db: " << task.db
+                              << ", tbl: " << task.tbl;
+        }
+    }
+}
+
+void TxnStatePoller::_execute_poll(const TxnStatePollTask& task) {
+    int64_t start_ts = MonotonicMicros();
+    TGetLoadTxnStatusRequest request;
+    request.__set_db(task.db);
+    request.__set_tbl(task.tbl);
+    request.__set_txnId(task.txn_id);
+    set_request_auth(&request, task.auth);
+    TGetLoadTxnStatusResult response;
+    Status status;
+#ifndef BE_TEST
+    TNetworkAddress master_addr = get_master_address();
+    status = ThriftRpcHelper::rpc<FrontendServiceClient>(
+            master_addr.hostname, master_addr.port,
+            [&request, &response](FrontendServiceConnection& client) { client->getLoadTxnStatus(response, request); });
+#else
+    TEST_SYNC_POINT_CALLBACK("TxnStatePoller::_execute_poll::request", &request);
+    TEST_SYNC_POINT_CALLBACK("TxnStatePoller::_execute_poll::status", &status);
+    TEST_SYNC_POINT_CALLBACK("TxnStatePoller::_execute_poll::response", &response);
+#endif
+    TRACE_BATCH_WRITE << "execute poll task, txn_id: " << task.txn_id << ", db: " << task.db << ", tbl: " << task.tbl
+                      << ", cost: " << (MonotonicMicros() - start_ts) << " us, rpc status: " << status
+                      << ", response: " << response;
+    if (status.ok()) {
+        _txn_state_cache->_notify_poll_result(task, TxnState{response.status, response.reason});
+    } else {
+        _txn_state_cache->_notify_poll_result(
+                task, Status::InternalError("poll txn state failed, error: " + status.to_string()));
+    }
+}
+
+bool TxnStatePoller::TEST_is_txn_pending(int64_t txn_id) {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    return _pending_txn_ids.find(txn_id) != _pending_txn_ids.end();
+}
+
+StatusOr<int64_t> TxnStatePoller::TEST_pending_execution_time(int64_t txn_id) {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    auto it = _pending_tasks.begin();
+    while (it != _pending_tasks.end()) {
+        if (it->second.txn_id == txn_id) {
+            return it->first;
+        }
+        ++it;
+    }
+    return Status::NotFound("no task found");
+}
+
+bool TxnStatePoller::TEST_is_scheduling() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    return _is_scheduling;
+}
+
+TxnStateCache::TxnStateCache(size_t capacity, std::unique_ptr<ThreadPoolToken> poller_token)
+        : _capacity(capacity), _poll_state_token(std::move(poller_token)) {
+    size_t capacity_per_shard = (_capacity + (kNumShards - 1)) / kNumShards;
+    for (int32_t i = 0; i < kNumShards; i++) {
+        _shards[i] = std::make_unique<TxnStateDynamicCache>(capacity_per_shard);
+    }
+}
+
+Status TxnStateCache::init() {
+    _txn_state_poller = std::make_unique<TxnStatePoller>(this, _poll_state_token.get());
+    RETURN_IF_ERROR(_txn_state_poller->init());
+    _txn_state_clean_thread = std::make_unique<std::thread>([this] { _txn_state_clean_func(); });
+    Thread::set_thread_name(*_txn_state_clean_thread.get(), "txn_state_clean");
+    return Status::OK();
+}
+
+Status TxnStateCache::push_state(int64_t txn_id, TTransactionStatus::type status, const std::string& reason) {
+    auto cache = _get_txn_cache(txn_id);
+    ASSIGN_OR_RETURN(auto entry, _get_txn_entry(cache, txn_id, true));
+    DCHECK(entry != nullptr);
+    entry->value().push_state(status, reason);
+    cache->release(entry);
+    return Status::OK();
+}
+
+StatusOr<TxnState> TxnStateCache::get_state(int64_t txn_id) {
+    auto cache = _get_txn_cache(txn_id);
+    ASSIGN_OR_RETURN(auto entry, _get_txn_entry(cache, txn_id, false));
+    if (entry == nullptr) {
+        return Status::NotFound("Transaction state not found");
+    }
+    auto txn_state = entry->value().txn_state();
+    cache->release(entry);
+    return txn_state;
+}
+
+StatusOr<TxnStateSubscriberPtr> TxnStateCache::subscribe_state(int64_t txn_id, const std::string& subscriber_name,
+                                                               const std::string& db, const std::string& tbl,
+                                                               const AuthInfo& auth) {
+    auto cache = _get_txn_cache(txn_id);
+    ASSIGN_OR_RETURN(auto entry, _get_txn_entry(cache, txn_id, true));
+    DCHECK(entry != nullptr);
+    bool trigger_poll = false;
+    entry->value().subscribe(trigger_poll);
+    TRACE_BATCH_WRITE << "create subscriber, txn_id: " << txn_id << ", name: " << subscriber_name << ", db: " << db
+                      << ", tbl: " << tbl << ", trigger_poll: " << trigger_poll;
+    if (trigger_poll) {
+        _txn_state_poller->submit({txn_id, db, tbl, auth}, config::merge_commit_txn_state_poll_interval_ms);
+    }
+    return std::make_unique<TxnStateSubscriber>(cache, entry, subscriber_name);
+}
+
+void TxnStateCache::set_capacity(size_t new_capacity) {
+    std::unique_lock<bthreads::BThreadSharedMutex> lock;
+    if (_stopped) {
+        return;
+    }
+    const size_t capacity_per_shard = (new_capacity + (kNumShards - 1)) / kNumShards;
+    for (auto& _shard : _shards) {
+        _shard->set_capacity(capacity_per_shard);
+    }
+    _capacity = new_capacity;
+}
+
+void TxnStateCache::stop() {
+    {
+        std::unique_lock<bthreads::BThreadSharedMutex> lock;
+        if (_stopped) {
+            return;
+        }
+        _stopped = true;
+    }
+    for (auto& cache : _shards) {
+        auto entries = cache->get_all_entries();
+        for (auto entry : entries) {
+            entry->value().stop();
+            cache->release(entry);
+        }
+    }
+    if (_txn_state_poller) {
+        _txn_state_poller->stop();
+    }
+    _poll_state_token->shutdown();
+    _txn_state_clean_stop_latch.count_down();
+    if (_txn_state_clean_thread && _txn_state_clean_thread->joinable()) {
+        _txn_state_clean_thread->join();
+    }
+}
+
+int32_t TxnStateCache::size() {
+    int32_t size = 0;
+    for (auto& cache : _shards) {
+        size += cache->size();
+    }
+    return size;
+}
+
+StatusOr<TxnStateDynamicCacheEntry*> TxnStateCache::_get_txn_entry(TxnStateDynamicCache* cache, int64_t txn_id,
+                                                                   bool create_if_not_exist) {
+    // use lock to avoid creating new entry after stopped
+    std::shared_lock<bthreads::BThreadSharedMutex> lock;
+    if (_stopped) {
+        return Status::ServiceUnavailable("Transaction state cache is stopped");
+    }
+    TxnStateDynamicCacheEntry* entry = nullptr;
+    if (create_if_not_exist) {
+        entry = cache->get_or_create(txn_id, 1);
+        DCHECK(entry != nullptr);
+        // initialize txn_id
+        entry->value().set_txn_id(txn_id);
+    } else {
+        entry = cache->get(txn_id);
+    }
+    if (entry) {
+        // expire time does not need very accurate and monotonic, so do not protect it from concurrent update
+        entry->update_expire_time(get_current_ms() + config::merge_commit_txn_state_expire_time_sec * 1000);
+    }
+    return entry;
+}
+
+void TxnStateCache::_notify_poll_result(const TxnStatePollTask& task, const StatusOr<TxnState>& result) {
+    auto cache = _get_txn_cache(task.txn_id);
+    auto entry_st = _get_txn_entry(cache, task.txn_id, false);
+    if (!entry_st.ok() || entry_st.value() == nullptr) {
+        TRACE_BATCH_WRITE << "skip notify poll result, txn_id: " << task.txn_id << ", db: " << task.db
+                          << ", tbl: " << task.tbl << ", entry status: "
+                          << (entry_st.ok() ? Status::NotFound("not in cache") : entry_st.status());
+        return;
+    }
+    auto entry = entry_st.value();
+    DeferOp defer([&] { cache->release(entry); });
+    bool continue_poll = entry->value().poll_state(result);
+    TRACE_BATCH_WRITE << "notify poll result, txn_id: " << task.txn_id << ", db: " << task.db << ", tbl: " << task.tbl
+                      << ", continue_poll: " << continue_poll;
+    if (continue_poll) {
+        _txn_state_poller->submit(task, config::merge_commit_txn_state_poll_interval_ms);
+    }
+}
+
+void TxnStateCache::_txn_state_clean_func() {
+    while (!_stopped) {
+        int32_t clean_interval_sec = config::merge_commit_txn_state_clean_interval_sec;
+        _txn_state_clean_stop_latch.wait_for(std::chrono::seconds(clean_interval_sec));
+        if (_stopped) {
+            break;
+        }
+        for (auto& cache : _shards) {
+            cache->clear_expired();
+        }
+    }
+}
+
+} // namespace starrocks
\ No newline at end of file
diff --git a/be/src/runtime/batch_write/txn_state_cache.h b/be/src/runtime/batch_write/txn_state_cache.h
new file mode 100644
index 00000000000000..079d538fc7a67d
--- /dev/null
+++ b/be/src/runtime/batch_write/txn_state_cache.h
@@ -0,0 +1,278 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <bthread/mutex.h>
+
+#include <map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "common/utils.h"
+#include "testutil/sync_point.h"
+#include "util/bthreads/bthread_shared_mutex.h"
+#include "util/countdown_latch.h"
+#include "util/dynamic_cache.h"
+#include "util/threadpool.h"
+#include "util/thrift_rpc_helper.h"
+
+namespace starrocks {
+
+class ThreadPoolToken;
+class TxnStateHandler;
+class TxnStateSubscriber;
+class TxnStateCache;
+using TxnStateDynamicCache = DynamicCache<int64_t, TxnStateHandler, bthread::Mutex>;
+using TxnStateDynamicCachePtr = std::unique_ptr<TxnStateDynamicCache>;
+using TxnStateDynamicCacheEntry = TxnStateDynamicCache::Entry;
+
+struct TxnState {
+    TTransactionStatus::type txn_status{TTransactionStatus::PREPARE};
+    std::string reason;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const TxnState& txn_state) {
+    os << "txn_status: " << to_string(txn_state.txn_status) << ", reason: " << txn_state.reason;
+    return os;
+}
+
+// Handle the transaction state. It's the value of the DynamicCache entry
+// 1. transmit the txn state according to the new state pushed by FE or polled from FE
+// 2. notify txn state subscribers whether it reaches the finished state
+// 3. control the behaviour of txn state poll. The poll task starts to schedule when the
+//    first subscriber comes(see subscribe()), and continue to schedule (see poll_state())
+//    until the txn state reaches the finished state, and there is no subscriber
+class TxnStateHandler {
+public:
+    ~TxnStateHandler();
+
+    // update the txn state pushed by FE
+    void push_state(TTransactionStatus::type new_status, const std::string& reason);
+    // update the txn state polled by the cache. The returned value tell the caller to
+    // continue or stop polling according the current result and the current txn state
+    bool poll_state(const StatusOr<TxnState>& result);
+
+    // Add a subscriber for the finished state. Handler will set 'trigger_poll'
+    // to tell whether the caller should submit a txn state poll task
+    void subscribe(bool& trigger_poll);
+    // Remove a subscriber which has called subscribe() before
+    void unsubscribe();
+    // A subscriber calls this function to wait for the txn state to reach
+    // a finished state or error happens.
+    StatusOr<TxnState> wait_finished_state(const std::string& subscriber_name, int64_t timeout_us);
+    int32_t num_waiting_finished_state();
+
+    void set_txn_id(int64_t txn_id) { _txn_id.store(txn_id); }
+    int64_t txn_id() { return _txn_id.load(); }
+    TxnState txn_state();
+    bool committed_status_from_fe();
+    int32_t num_poll_failure();
+    std::string debug_string();
+
+    void stop();
+
+    // For testing
+    int32_t TEST_num_subscriber() { return _num_subscriber; }
+
+private:
+    void _transition_txn_state(TTransactionStatus::type new_status, const std::string& reason, bool from_fe);
+    // Whether the current status indicate the load is finished
+    bool _is_finished_txn_state();
+
+    // lazy initialized
+    std::atomic<int64_t> _txn_id{-1};
+    bthread::Mutex _mutex;
+    bthread::ConditionVariable _cv;
+    TxnState _txn_state;
+    // whether COMMITTED status is notified by FE. Only valid if txn status is COMMITTED.
+    // If true, means publish timeout happens, and should notify subscribers. If false,
+    // means its from poll should continue to wait.
+    bool _committed_status_from_fe{false};
+    int32_t _num_subscriber{0};
+    int32_t _num_waiting_finished_state{0};
+    int32_t _num_poll_failure{0};
+    bool _stopped{false};
+};
+
+inline int32_t TxnStateHandler::num_waiting_finished_state() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    return _num_waiting_finished_state;
+}
+
+inline TxnState TxnStateHandler::txn_state() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    return _txn_state;
+}
+
+inline bool TxnStateHandler::committed_status_from_fe() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    return _committed_status_from_fe;
+}
+
+inline int32_t TxnStateHandler::num_poll_failure() {
+    std::unique_lock<bthread::Mutex> lock(_mutex);
+    return _num_poll_failure;
+}
+
+inline std::ostream& operator<<(std::ostream& os, TxnStateHandler& holder) {
+    os << holder.debug_string();
+    return os;
+}
+
+// A subscriber which will wait for the finished txn state. It holds a reference
+// to the entry of DynamicCache so that the cache will not evict the txn state.
+// The subscriber can call wait_finished_state() to wait for the finished state.
+class TxnStateSubscriber {
+public:
+    TxnStateSubscriber(TxnStateDynamicCache* cache, TxnStateDynamicCacheEntry* entry, std::string name)
+            : _cache(cache), _entry(entry), _name(std::move(name)){};
+
+    ~TxnStateSubscriber() {
+        _entry->value().unsubscribe();
+        _cache->release(_entry);
+    }
+
+    const std::string& name() const { return _name; }
+    StatusOr<TxnState> wait_finished_state(int64_t timeout_us);
+    TxnState current_state();
+    TxnStateDynamicCacheEntry* entry() { return _entry; }
+
+private:
+    TxnStateDynamicCache* _cache;
+    TxnStateDynamicCacheEntry* _entry;
+    std::string _name;
+};
+using TxnStateSubscriberPtr = std::unique_ptr<TxnStateSubscriber>;
+
+struct TxnStatePollTask {
+    int64_t txn_id;
+    std::string db;
+    std::string tbl;
+    AuthInfo auth;
+};
+
+// Schedule and execute txn state poll tasks. The poller uses a single thread
+// to schedule tasks according to their execution time, and submit them to the
+// thread pool to run which will send rpc to FE to get txn state.
+class TxnStatePoller {
+public:
+    TxnStatePoller(TxnStateCache* txn_state_cache, ThreadPoolToken* poll_token)
+            : _txn_state_cache(txn_state_cache), _poll_token(poll_token) {}
+    Status init();
+    // submit a task which should be executed after the delay time
+    void submit(const TxnStatePollTask& task, int64_t delay_ms);
+    void stop();
+
+    // For testing
+    bool TEST_is_txn_pending(int64_t txn_id);
+    StatusOr<int64_t> TEST_pending_execution_time(int64_t txn_id);
+    bool TEST_is_scheduling();
+
+private:
+    void _schedule_func();
+    void _schedule_poll_tasks(const std::vector<TxnStatePollTask>& poll_tasks);
+    void _execute_poll(const TxnStatePollTask& task);
+
+    TxnStateCache* _txn_state_cache;
+    ThreadPoolToken* _poll_token;
+    std::unique_ptr<std::thread> _schedule_thread;
+    bthread::Mutex _mutex;
+    bthread::ConditionVariable _cv;
+    // txn ids to schedule, used to duplicate tasks for the same txn
+    std::unordered_set<int64_t> _pending_txn_ids;
+    // sorted execution time (milliseconds) -> task
+    std::multimap<int64_t, TxnStatePollTask> _pending_tasks;
+    bool _is_scheduling{false};
+    bool _stopped{false};
+};
+
+// A cache for txn states. It can receive txn state in two ways: pushed by FE and polled from FE by itself.
+// When the load finishes, FE will try to push the txn state to BE which is more efficient and realtime,
+// but it does not always work because the push may fail for some reason, such as FE leader switch or crash.
+// So BE will poll the txn state from FE periodically in a low frequency to detect those bad cases rather
+// than just waiting until timeout. Apart from maintaining the txn state, the cache also provides a subscribe
+// mechanism to notify the subscriber when the txn state reaches the finished state.
+// The poll state task starts to schedule when the first subscriber comes, and continue to schedule when the
+// last poll finishes. The schedule will end when the txn reaches the finished state or there is no subscriber.
+class TxnStateCache {
+public:
+    TxnStateCache(size_t capacity, std::unique_ptr<ThreadPoolToken> poller_token);
+    Status init();
+
+    // update the txn state which is pushed by FE. It will create an entry
+    // in the DynamicCache it the txn does not in the cache before.
+    Status push_state(int64_t txn_id, TTransactionStatus::type status, const std::string& reason);
+
+    // get the current state of txn_id. A TxnState will return if the txn is in the cache.
+    // Status::NotFound will return if the txn is not in the cache. Other status will return
+    // if error happens.
+    StatusOr<TxnState> get_state(int64_t txn_id);
+
+    // create a TxnStateSubscriber to subscribe the finished txn state. It will create an entry
+    // in the DynamicCache it the txn does not in the cache before. The subscriber will hold a
+    // reference to the entry, so the entry will not be evicted if any subscriber is using it.
+    // The db/tbl/auth may be used to poll txn state.
+    StatusOr<TxnStateSubscriberPtr> subscribe_state(int64_t txn_id, const std::string& subscriber_name,
+                                                    const std::string& db, const std::string& tbl,
+                                                    const AuthInfo& auth);
+
+    void set_capacity(size_t new_capacity);
+    int32_t size();
+
+    void stop();
+
+    // For testing
+    std::vector<TxnStateDynamicCache*> get_cache_shards() {
+        std::vector<TxnStateDynamicCache*> ret;
+        for (auto& shard : _shards) {
+            ret.push_back(shard.get());
+        }
+        return ret;
+    }
+    TxnStatePoller* txn_state_poller() { return _txn_state_poller.get(); }
+
+private:
+    static const int kNumShardBits = 5;
+    static const int kNumShards = 1 << kNumShardBits;
+
+    friend class TxnStatePoller;
+
+    TxnStateDynamicCache* _get_txn_cache(int64_t txn_id);
+    // if create_if_not_exist is true, must return non nullptr entry if status is ok.
+    // if create_if_not_exist is false, return nullptr if txn is not in cache.
+    // Return not ok status if error happens.
+    StatusOr<TxnStateDynamicCacheEntry*> _get_txn_entry(TxnStateDynamicCache* cache, int64_t txn_id,
+                                                        bool create_if_not_exist);
+    void _notify_poll_result(const TxnStatePollTask& task, const StatusOr<TxnState>& result);
+    void _txn_state_clean_func();
+
+    size_t _capacity;
+    std::unique_ptr<ThreadPoolToken> _poll_state_token;
+    TxnStateDynamicCachePtr _shards[kNumShards];
+    std::unique_ptr<TxnStatePoller> _txn_state_poller;
+    // protect the cache from being accessed after it is stopped
+    bthreads::BThreadSharedMutex _rw_mutex;
+    std::atomic<bool> _stopped{false};
+
+    std::unique_ptr<std::thread> _txn_state_clean_thread;
+    // used to notify the clean thread to stop
+    CountDownLatch _txn_state_clean_stop_latch{1};
+};
+
+inline TxnStateDynamicCache* TxnStateCache::_get_txn_cache(int64_t txn_id) {
+    return _shards[txn_id & (kNumShards - 1)].get();
+}
+} // namespace starrocks
diff --git a/be/src/runtime/exec_env.cpp b/be/src/runtime/exec_env.cpp
index c0a4dc1269023e..a1fd4098cc16d0 100644
--- a/be/src/runtime/exec_env.cpp
+++ b/be/src/runtime/exec_env.cpp
@@ -513,14 +513,15 @@ Status ExecEnv::init(const std::vector<StorePath>& store_paths, bool as_cn) {
 
     std::unique_ptr<ThreadPool> batch_write_thread_pool;
     RETURN_IF_ERROR(ThreadPoolBuilder("batch_write")
-                            .set_min_threads(config::batch_write_thread_pool_num_min)
-                            .set_max_threads(config::batch_write_thread_pool_num_max)
-                            .set_max_queue_size(config::batch_write_thread_pool_queue_size)
+                            .set_min_threads(config::merge_commit_thread_pool_num_min)
+                            .set_max_threads(config::merge_commit_thread_pool_num_max)
+                            .set_max_queue_size(config::merge_commit_thread_pool_queue_size)
                             .set_idle_timeout(MonoDelta::FromMilliseconds(10000))
                             .build(&batch_write_thread_pool));
     auto batch_write_executor =
             std::make_unique<bthreads::ThreadPoolExecutor>(batch_write_thread_pool.release(), kTakesOwnership);
     _batch_write_mgr = new BatchWriteMgr(std::move(batch_write_executor));
+    RETURN_IF_ERROR(_batch_write_mgr->init());
 
     _routine_load_task_executor = new RoutineLoadTaskExecutor(this);
     RETURN_IF_ERROR(_routine_load_task_executor->init());
@@ -745,6 +746,7 @@ void ExecEnv::destroy() {
     SAFE_DELETE(_result_queue_mgr);
     SAFE_DELETE(_result_mgr);
     SAFE_DELETE(_stream_mgr);
+    SAFE_DELETE(_batch_write_mgr);
     SAFE_DELETE(_external_scan_context_mgr);
     SAFE_DELETE(_lake_tablet_manager);
     SAFE_DELETE(_lake_update_manager);
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 821fabbed5bf35..5898072676137b 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -1279,7 +1279,16 @@ void PInternalServiceImplBase<T>::stream_load(google::protobuf::RpcController* c
                                               google::protobuf::Closure* done) {
     ClosureGuard closure_guard(done);
     auto* cntl = static_cast<brpc::Controller*>(cntl_base);
-    BatchWriteMgr::receive_stream_load_rpc(_exec_env, cntl, request, response);
+    _exec_env->batch_write_mgr()->receive_stream_load_rpc(_exec_env, cntl, request, response);
+}
+
+template <typename T>
+void PInternalServiceImplBase<T>::update_transaction_state(google::protobuf::RpcController* cntl_base,
+                                                           const PUpdateTransactionStateRequest* request,
+                                                           PUpdateTransactionStateResponse* response,
+                                                           google::protobuf::Closure* done) {
+    ClosureGuard closure_guard(done);
+    _exec_env->batch_write_mgr()->update_transaction_state(request, response);
 }
 
 template class PInternalServiceImplBase<PInternalService>;
diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h
index 7fc6d3a637e70d..ff33b1e92a1eef 100644
--- a/be/src/service/internal_service.h
+++ b/be/src/service/internal_service.h
@@ -185,6 +185,10 @@ class PInternalServiceImplBase : public T {
     void stream_load(google::protobuf::RpcController* controller, const PStreamLoadRequest* request,
                      PStreamLoadResponse* response, google::protobuf::Closure* done) override;
 
+    void update_transaction_state(google::protobuf::RpcController* controller,
+                                  const PUpdateTransactionStateRequest* request,
+                                  PUpdateTransactionStateResponse* response, google::protobuf::Closure* done) override;
+
 private:
     void _transmit_chunk(::google::protobuf::RpcController* controller,
                          const ::starrocks::PTransmitChunkParams* request, ::starrocks::PTransmitChunkResult* response,
diff --git a/be/src/util/dynamic_cache.h b/be/src/util/dynamic_cache.h
index 6b7837eef0a020..121e399d7c6fad 100644
--- a/be/src/util/dynamic_cache.h
+++ b/be/src/util/dynamic_cache.h
@@ -37,12 +37,12 @@ namespace starrocks {
 // Note: the capacity is a soft limit, it will only free unused objects
 // to reduce memory usage, but if currently used(pinned) objects' memory
 // exceeds capacity, that's allowed.
-template <class Key, class T>
+template <class Key, class T, class Lock = std::mutex>
 class DynamicCache {
 public:
     struct Entry {
     public:
-        Entry(DynamicCache<Key, T>& cache, Key key) : _cache(cache), _key(std::move(key)), _ref(1) {}
+        Entry(DynamicCache<Key, T, Lock>& cache, Key key) : _cache(cache), _key(std::move(key)), _ref(1) {}
 
         const Key& key() const { return _key; }
         T& value() { return _value; }
@@ -54,10 +54,10 @@ class DynamicCache {
         uint32_t get_ref() const { return _ref.load(); }
 
     protected:
-        friend class DynamicCache<Key, T>;
+        friend class DynamicCache<Key, T, Lock>;
         typedef typename std::list<Entry*>::const_iterator Handle;
 
-        DynamicCache<Key, T>& _cache;
+        DynamicCache<Key, T, Lock>& _cache;
         Handle _handle;
         Key _key;
         size_t _size = 0;
@@ -72,7 +72,7 @@ class DynamicCache {
 
     DynamicCache(size_t capacity) : _size(0), _capacity(capacity) {}
     ~DynamicCache() {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         _object_size = 0;
         _size = 0;
         auto itr = _list.begin();
@@ -98,7 +98,7 @@ class DynamicCache {
 
     // get or return null
     Entry* get(const Key& key) {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         auto itr = _map.find(key);
         if (itr == _map.end()) {
             return nullptr;
@@ -111,8 +111,8 @@ class DynamicCache {
 
     // atomic get_or_create operation, to prevent loading
     // same resource multiple times
-    Entry* get_or_create(const Key& key) {
-        std::lock_guard<std::mutex> lg(_lock);
+    Entry* get_or_create(const Key& key, size_t init_size = 0) {
+        std::lock_guard<Lock> lg(_lock);
         auto itr = _map.find(key);
         if (itr == _map.end()) {
             // at first all created object is with size 0
@@ -123,6 +123,7 @@ class DynamicCache {
             insert->_handle = ret;
             _map[key] = ret;
             (*ret)->_ref++;
+            (*ret)->_size = init_size;
             _object_size++;
             if (insert->_size > 0) {
                 _size += insert->_size;
@@ -140,7 +141,7 @@ class DynamicCache {
 
     // release(unuse) an object get/get_or_create'ed earlier
     void release(Entry* entry) {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         // CHECK _ref > 1
         entry->_ref--;
         if (entry->_ref > 0) {
@@ -159,7 +160,7 @@ class DynamicCache {
 
     // remove an object get/get_or_create'ed earlier
     bool remove(Entry* entry) {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         entry->_ref--;
         if (entry->_ref != 1) {
             return false;
@@ -179,7 +180,7 @@ class DynamicCache {
     // if no one use this object, object will be removed
     // otherwise do not remove the object, return false
     bool try_remove_by_key(const Key& key) {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         auto itr = _map.find(key);
         if (itr == _map.end()) {
             return true;
@@ -203,7 +204,7 @@ class DynamicCache {
     // remove object by key
     // return true if object exist and is removed
     bool remove_by_key(const Key& key) {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         auto itr = _map.find(key);
         if (itr == _map.end()) {
             return false;
@@ -228,7 +229,7 @@ class DynamicCache {
     // track size changes and evict objects accordingly
     // return false if actual memory usage is larger than capacity
     bool update_object_size(Entry* entry, size_t new_size) {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         _size += new_size - entry->_size;
         if (_mem_tracker) _mem_tracker->consume(new_size - entry->_size);
         entry->_size = new_size;
@@ -240,7 +241,7 @@ class DynamicCache {
         std::vector<Entry*> entry_list;
         {
             int64_t now = MonotonicMillis();
-            std::lock_guard<std::mutex> lg(_lock);
+            std::lock_guard<Lock> lg(_lock);
             auto itr = _list.begin();
             while (itr != _list.end()) {
                 Entry* entry = (*itr);
@@ -266,7 +267,7 @@ class DynamicCache {
     void clear() {
         std::vector<Entry*> entry_list;
         {
-            std::lock_guard<std::mutex> lg(_lock);
+            std::lock_guard<Lock> lg(_lock);
             auto itr = _list.begin();
             while (itr != _list.end()) {
                 Entry* entry = (*itr);
@@ -296,13 +297,13 @@ class DynamicCache {
     // adjust capacity
     // return false if actual memory usage is larger than capacity
     bool set_capacity(size_t capacity) {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         _capacity = capacity;
         return _evict();
     }
 
     std::vector<std::pair<Key, size_t>> get_entry_sizes() const {
-        std::lock_guard<std::mutex> lg(_lock);
+        std::lock_guard<Lock> lg(_lock);
         std::vector<std::pair<Key, size_t>> ret;
         ret.reserve(_map.size());
         auto itr = _list.begin();
@@ -317,7 +318,7 @@ class DynamicCache {
     void try_evict(size_t target_capacity) {
         std::vector<Entry*> entry_list;
         {
-            std::lock_guard<std::mutex> lg(_lock);
+            std::lock_guard<Lock> lg(_lock);
             _evict(target_capacity, &entry_list);
         }
         for (Entry* entry : entry_list) {
@@ -326,6 +327,22 @@ class DynamicCache {
         return;
     }
 
+    std::vector<Entry*> get_all_entries() {
+        std::vector<Entry*> entry_list;
+        {
+            std::lock_guard<Lock> lg(_lock);
+            entry_list.reserve(_list.size());
+            auto itr = _list.begin();
+            while (itr != _list.end()) {
+                Entry* entry = (*itr);
+                entry->_ref++;
+                entry_list.push_back(entry);
+                itr++;
+            }
+        }
+        return entry_list;
+    }
+
     bool TEST_evict(size_t target_capacity, std::vector<Entry*>* entry_list) {
         return _evict(target_capacity, entry_list);
     }
@@ -360,7 +377,7 @@ class DynamicCache {
         return ret;
     }
 
-    mutable std::mutex _lock;
+    mutable Lock _lock;
     List _list;
     Map _map;
     size_t _object_size{0};
diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt
index b462d5bbe83227..cb14ffdfaae8a5 100644
--- a/be/test/CMakeLists.txt
+++ b/be/test/CMakeLists.txt
@@ -391,6 +391,7 @@ set(EXEC_FILES
         ./runtime/batch_write/batch_write_mgr_test.cpp
         ./runtime/batch_write/batch_write_util_test.cpp
         ./runtime/batch_write/isomorphic_batch_write_test.cpp
+        ./runtime/batch_write/txn_state_cache_test.cpp
         #./runtime/routine_load_task_executor_test.cpp
         ./runtime/routine_load/data_consumer_test.cpp
         ./runtime/small_file_mgr_test.cpp
diff --git a/be/test/runtime/batch_write/batch_write_mgr_test.cpp b/be/test/runtime/batch_write/batch_write_mgr_test.cpp
index 21d2b34d435b78..9917d65f05d36d 100644
--- a/be/test/runtime/batch_write/batch_write_mgr_test.cpp
+++ b/be/test/runtime/batch_write/batch_write_mgr_test.cpp
@@ -36,6 +36,7 @@ class BatchWriteMgrTest : public testing::Test {
     BatchWriteMgrTest() = default;
     ~BatchWriteMgrTest() override = default;
     void SetUp() override {
+        config::merge_commit_trace_log_enable = true;
         _exec_env = ExecEnv::GetInstance();
         std::unique_ptr<ThreadPool> thread_pool;
         ASSERT_OK(ThreadPoolBuilder("BatchWriteMgrTest")
@@ -46,6 +47,7 @@ class BatchWriteMgrTest : public testing::Test {
                           .build(&thread_pool));
         auto executor = std::make_unique<bthreads::ThreadPoolExecutor>(thread_pool.release(), kTakesOwnership);
         _batch_write_mgr = std::make_unique<BatchWriteMgr>(std::move(executor));
+        ASSERT_OK(_batch_write_mgr->init());
     }
 
     void TearDown() override {
@@ -373,4 +375,56 @@ TEST_F(BatchWriteMgrTest, stream_load_rpc_fail) {
     }
 }
 
+TEST_F(BatchWriteMgrTest, update_transaction_state) {
+    PUpdateTransactionStateRequest request;
+    std::vector<TxnState> expected_cache_state;
+
+    auto prepare_state = request.add_states();
+    prepare_state->set_txn_id(1);
+    prepare_state->set_status(TransactionStatusPB::TRANS_PREPARE);
+    prepare_state->set_reason("");
+    expected_cache_state.push_back({TTransactionStatus::PREPARE, ""});
+
+    auto prepared_state = request.add_states();
+    prepared_state->set_txn_id(2);
+    prepared_state->set_status(TransactionStatusPB::TRANS_PREPARED);
+    prepared_state->set_reason("");
+    expected_cache_state.push_back({TTransactionStatus::PREPARED, ""});
+
+    auto commited_state = request.add_states();
+    commited_state->set_txn_id(3);
+    commited_state->set_status(TransactionStatusPB::TRANS_COMMITTED);
+    commited_state->set_reason("");
+    expected_cache_state.push_back({TTransactionStatus::COMMITTED, ""});
+
+    auto visible_state = request.add_states();
+    visible_state->set_txn_id(4);
+    visible_state->set_status(TransactionStatusPB::TRANS_VISIBLE);
+    visible_state->set_reason("");
+    expected_cache_state.push_back({TTransactionStatus::VISIBLE, ""});
+
+    auto aborted_state = request.add_states();
+    aborted_state->set_txn_id(5);
+    aborted_state->set_status(TransactionStatusPB::TRANS_ABORTED);
+    aborted_state->set_reason("artificial failure");
+    expected_cache_state.push_back({TTransactionStatus::ABORTED, "artificial failure"});
+
+    auto unknown_state = request.add_states();
+    unknown_state->set_txn_id(6);
+    unknown_state->set_status(TransactionStatusPB::TRANS_UNKNOWN);
+    unknown_state->set_reason("");
+    expected_cache_state.push_back({TTransactionStatus::UNKNOWN, ""});
+
+    PUpdateTransactionStateResponse response;
+    _batch_write_mgr->update_transaction_state(&request, &response);
+    ASSERT_EQ(request.states_size(), response.results_size());
+    for (int i = 1; i <= expected_cache_state.size(); ++i) {
+        ASSERT_EQ(TStatusCode::OK, response.results(i - 1).status_code());
+        auto actual_state = _batch_write_mgr->txn_state_cache()->get_state(i);
+        ASSERT_OK(actual_state.status());
+        ASSERT_EQ(expected_cache_state[i - 1].txn_status, actual_state.value().txn_status);
+        ASSERT_EQ(expected_cache_state[i - 1].reason, actual_state.value().reason);
+    }
+}
+
 } // namespace starrocks
\ No newline at end of file
diff --git a/be/test/runtime/batch_write/isomorphic_batch_write_test.cpp b/be/test/runtime/batch_write/isomorphic_batch_write_test.cpp
index 92e24a21b9ae04..d7dffa44682e8f 100644
--- a/be/test/runtime/batch_write/isomorphic_batch_write_test.cpp
+++ b/be/test/runtime/batch_write/isomorphic_batch_write_test.cpp
@@ -33,6 +33,7 @@ class IsomorphicBatchWriteTest : public testing::Test {
     IsomorphicBatchWriteTest() = default;
     ~IsomorphicBatchWriteTest() override = default;
     void SetUp() override {
+        config::merge_commit_trace_log_enable = true;
         _exec_env = ExecEnv::GetInstance();
         std::unique_ptr<ThreadPool> thread_pool;
         ASSERT_OK(ThreadPoolBuilder("IsomorphicBatchWriteTest")
@@ -42,12 +43,22 @@ class IsomorphicBatchWriteTest : public testing::Test {
                           .set_idle_timeout(MonoDelta::FromMilliseconds(10000))
                           .build(&thread_pool));
         _executor = std::make_unique<bthreads::ThreadPoolExecutor>(thread_pool.release(), kTakesOwnership);
+        std::unique_ptr<ThreadPoolToken> token =
+                _executor->get_thread_pool()->new_token(ThreadPool::ExecutionMode::CONCURRENT);
+        _txn_state_cache = std::make_unique<TxnStateCache>(2048, std::move(token));
+        ASSERT_OK(_txn_state_cache->init());
     }
 
     void TearDown() override {
         for (auto* ctx : _to_release_contexts) {
             StreamLoadContext::release(ctx);
         }
+        if (_txn_state_cache) {
+            _txn_state_cache->stop();
+        }
+        if (_executor) {
+            _executor->get_thread_pool()->shutdown();
+        }
     }
 
     StreamLoadContext* build_pipe_context(const std::string& label, int64_t txn_id, const BatchWriteId& batch_write_id,
@@ -84,12 +95,13 @@ class IsomorphicBatchWriteTest : public testing::Test {
         return ctx;
     }
 
-    void test_append_data_sync_base(const Status& rpc_status, const TGetLoadTxnStatusResult& expect_result,
+    void test_append_data_sync_base(int64_t txn_id, std::string label, const TxnState& txn_state,
                                     const Status& expect_st);
 
 protected:
     ExecEnv* _exec_env;
     std::unique_ptr<bthreads::ThreadPoolExecutor> _executor;
+    std::unique_ptr<TxnStateCache> _txn_state_cache;
     std::unordered_set<StreamLoadContext*> _to_release_contexts;
 };
 
@@ -102,7 +114,8 @@ void verify_data(std::string expected, ByteBufferPtr actual) {
 
 TEST_F(IsomorphicBatchWriteTest, register_and_unregister_pipe) {
     BatchWriteId batch_write_id{.db = "db", .table = "table", .load_params = {}};
-    IsomorphicBatchWriteSharedPtr batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get());
+    IsomorphicBatchWriteSharedPtr batch_write =
+            std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get(), _txn_state_cache.get());
     ASSERT_OK(batch_write->init());
     DeferOp defer_writer([&] { batch_write->stop(); });
 
@@ -134,7 +147,8 @@ TEST_F(IsomorphicBatchWriteTest, register_and_unregister_pipe) {
 
 TEST_F(IsomorphicBatchWriteTest, append_data_async) {
     BatchWriteId batch_write_id{.db = "db", .table = "table", .load_params = {{HTTP_MERGE_COMMIT_ASYNC, "true"}}};
-    IsomorphicBatchWriteSharedPtr batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get());
+    IsomorphicBatchWriteSharedPtr batch_write =
+            std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get(), _txn_state_cache.get());
     ASSERT_OK(batch_write->init());
     DeferOp defer_writer([&] { batch_write->stop(); });
 
@@ -244,34 +258,21 @@ TEST_F(IsomorphicBatchWriteTest, append_data_async) {
 }
 
 TEST_F(IsomorphicBatchWriteTest, append_data_sync) {
-    TGetLoadTxnStatusResult expect_result;
-    expect_result.__set_status(TTransactionStatus::UNKNOWN);
-    test_append_data_sync_base(Status::InternalError("Artificial failure"), expect_result,
-                               Status::InternalError("Failed to get load status, Internal error: Artificial failure"));
-    expect_result.__set_status(TTransactionStatus::PREPARE);
-    test_append_data_sync_base(Status::OK(), expect_result, Status::TimedOut("load timeout, txn status: PREPARE"));
-    expect_result.__set_status(TTransactionStatus::PREPARED);
-    test_append_data_sync_base(Status::OK(), expect_result, Status::TimedOut("load timeout, txn status: PREPARED"));
-    expect_result.__set_status(TTransactionStatus::COMMITTED);
-    test_append_data_sync_base(Status::OK(), expect_result,
+    test_append_data_sync_base(1, "label1", {TTransactionStatus::UNKNOWN, ""},
+                               Status::InternalError("Can't find the transaction, reason: "));
+    test_append_data_sync_base(2, "label2", {TTransactionStatus::COMMITTED, ""},
                                Status::PublishTimeout("Load has not been published before timeout"));
-    expect_result.__set_status(TTransactionStatus::VISIBLE);
-    test_append_data_sync_base(Status::OK(), expect_result, Status::OK());
-    expect_result.__set_status(TTransactionStatus::ABORTED);
-    expect_result.__set_reason("artificial failure");
-    test_append_data_sync_base(Status::OK(), expect_result,
+    test_append_data_sync_base(3, "label3", {TTransactionStatus::VISIBLE, ""}, Status::OK());
+    test_append_data_sync_base(4, "label4", {TTransactionStatus::ABORTED, "artificial failure"},
                                Status::InternalError("Load is aborted, reason: artificial failure"));
-    expect_result.__set_status(TTransactionStatus::UNKNOWN);
-    expect_result.__set_reason("");
-    test_append_data_sync_base(Status::OK(), expect_result, Status::InternalError("Load status is unknown: UNKNOWN"));
 }
 
-void IsomorphicBatchWriteTest::test_append_data_sync_base(const Status& rpc_status,
-                                                          const TGetLoadTxnStatusResult& expect_result,
+void IsomorphicBatchWriteTest::test_append_data_sync_base(int64_t txn_id, std::string label, const TxnState& txn_state,
                                                           const Status& expect_st) {
     BatchWriteId batch_write_id{
             .db = "db", .table = "table", .load_params = {{HTTP_MERGE_COMMIT_ASYNC, "false"}, {HTTP_TIMEOUT, "1"}}};
-    IsomorphicBatchWriteSharedPtr batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get());
+    IsomorphicBatchWriteSharedPtr batch_write =
+            std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get(), _txn_state_cache.get());
     ASSERT_OK(batch_write->init());
     DeferOp defer_writer([&] { batch_write->stop(); });
 
@@ -281,9 +282,6 @@ void IsomorphicBatchWriteTest::test_append_data_sync_base(const Status& rpc_stat
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::request");
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::status");
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::response");
-        SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::_wait_for_load_status::request");
-        SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::_wait_for_load_status::status");
-        SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::_wait_for_load_status::response");
         SyncPoint::GetInstance()->DisableProcessing();
     });
 
@@ -293,7 +291,7 @@ void IsomorphicBatchWriteTest::test_append_data_sync_base(const Status& rpc_stat
     SyncPoint::GetInstance()->SetCallBack("TimeBoundedStreamLoadPipe::get_current_ns",
                                           [&](void* arg) { *((int64_t*)arg) = 0; });
     StreamLoadContext* pipe_ctx1 =
-            build_pipe_context("label1", 1, batch_write_id, std::make_shared<TimeBoundedStreamLoadPipe>("p1", 1000));
+            build_pipe_context(label, txn_id, batch_write_id, std::make_shared<TimeBoundedStreamLoadPipe>("p1", 1000));
     SyncPoint::GetInstance()->SetCallBack("IsomorphicBatchWrite::send_rpc_request::status",
                                           [&](void* arg) { *((Status*)arg) = Status::OK(); });
     SyncPoint::GetInstance()->SetCallBack("IsomorphicBatchWrite::send_rpc_request::response", [&](void* arg) {
@@ -301,26 +299,14 @@ void IsomorphicBatchWriteTest::test_append_data_sync_base(const Status& rpc_stat
         TStatus status;
         status.__set_status_code(TStatusCode::OK);
         result->__set_status(status);
-        result->__set_label("label1");
+        result->__set_label(label);
         ASSERT_OK(batch_write->register_stream_load_pipe(pipe_ctx1));
     });
 
     // stream pipe left time is 100ms
     SyncPoint::GetInstance()->SetCallBack("TimeBoundedStreamLoadPipe::get_current_ns",
                                           [&](void* arg) { *((int64_t*)arg) = 900000000; });
-    SyncPoint::GetInstance()->SetCallBack("IsomorphicBatchWrite::_wait_for_load_status::request", [&](void* arg) {
-        TGetLoadTxnStatusRequest* request = (TGetLoadTxnStatusRequest*)arg;
-        EXPECT_EQ(batch_write_id.db, request->db);
-        EXPECT_EQ(batch_write_id.table, request->tbl);
-        EXPECT_EQ(1, request->txnId);
-    });
-    SyncPoint::GetInstance()->SetCallBack("IsomorphicBatchWrite::_wait_for_load_status::status",
-                                          [&](void* arg) { *((Status*)arg) = rpc_status; });
-    SyncPoint::GetInstance()->SetCallBack("IsomorphicBatchWrite::_wait_for_load_status::response", [&](void* arg) {
-        TGetLoadTxnStatusResult* result = (TGetLoadTxnStatusResult*)arg;
-        result->__set_status(expect_result.status);
-        result->__set_reason(expect_result.reason);
-    });
+    ASSERT_OK(_txn_state_cache->push_state(txn_id, txn_state.txn_status, txn_state.reason));
     StreamLoadContext* data_ctx1 = build_data_context(batch_write_id, "data1");
     Status result = batch_write->append_data(data_ctx1);
     ASSERT_EQ(1, num_rpc_request);
@@ -332,7 +318,8 @@ void IsomorphicBatchWriteTest::test_append_data_sync_base(const Status& rpc_stat
 
 TEST_F(IsomorphicBatchWriteTest, stop_write) {
     BatchWriteId batch_write_id{.db = "db", .table = "table", .load_params = {}};
-    IsomorphicBatchWriteSharedPtr batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get());
+    IsomorphicBatchWriteSharedPtr batch_write =
+            std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get(), _txn_state_cache.get());
     ASSERT_OK(batch_write->init());
     DeferOp defer_writer([&] { batch_write->stop(); });
 
@@ -362,22 +349,23 @@ TEST_F(IsomorphicBatchWriteTest, stop_write) {
 
 TEST_F(IsomorphicBatchWriteTest, reach_max_rpc_retry) {
     BatchWriteId batch_write_id{.db = "db", .table = "table", .load_params = {{HTTP_MERGE_COMMIT_ASYNC, "true"}}};
-    IsomorphicBatchWriteSharedPtr batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get());
+    IsomorphicBatchWriteSharedPtr batch_write =
+            std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get(), _txn_state_cache.get());
     ASSERT_OK(batch_write->init());
     DeferOp defer_writer([&] { batch_write->stop(); });
 
-    auto old_retry_num = config::batch_write_rpc_request_retry_num;
-    auto old_retry_interval = config::batch_write_rpc_request_retry_interval_ms;
-    config::batch_write_rpc_request_retry_num = 5;
-    config::batch_write_rpc_request_retry_interval_ms = 10;
+    auto old_retry_num = config::merge_commit_rpc_request_retry_num;
+    auto old_retry_interval = config::merge_commit_rpc_request_retry_interval_ms;
+    config::merge_commit_rpc_request_retry_num = 5;
+    config::merge_commit_rpc_request_retry_interval_ms = 10;
     SyncPoint::GetInstance()->EnableProcessing();
     DeferOp defer([&]() {
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::request");
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::status");
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::response");
         SyncPoint::GetInstance()->DisableProcessing();
-        config::batch_write_rpc_request_retry_num = old_retry_num;
-        config::batch_write_rpc_request_retry_interval_ms = old_retry_interval;
+        config::merge_commit_rpc_request_retry_num = old_retry_num;
+        config::merge_commit_rpc_request_retry_interval_ms = old_retry_interval;
     });
 
     int num_rpc_request = 0;
@@ -402,22 +390,23 @@ TEST_F(IsomorphicBatchWriteTest, reach_max_rpc_retry) {
 
 TEST_F(IsomorphicBatchWriteTest, stop_retry_if_rpc_failed) {
     BatchWriteId batch_write_id{.db = "db", .table = "table", .load_params = {{HTTP_MERGE_COMMIT_ASYNC, "true"}}};
-    IsomorphicBatchWriteSharedPtr batch_write = std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get());
+    IsomorphicBatchWriteSharedPtr batch_write =
+            std::make_shared<IsomorphicBatchWrite>(batch_write_id, _executor.get(), _txn_state_cache.get());
     ASSERT_OK(batch_write->init());
     DeferOp defer_writer([&] { batch_write->stop(); });
 
-    auto old_retry_num = config::batch_write_rpc_request_retry_num;
-    auto old_retry_interval = config::batch_write_rpc_request_retry_interval_ms;
-    config::batch_write_rpc_request_retry_num = 5;
-    config::batch_write_rpc_request_retry_interval_ms = 10;
+    auto old_retry_num = config::merge_commit_rpc_request_retry_num;
+    auto old_retry_interval = config::merge_commit_rpc_request_retry_interval_ms;
+    config::merge_commit_rpc_request_retry_num = 5;
+    config::merge_commit_rpc_request_retry_interval_ms = 10;
     SyncPoint::GetInstance()->EnableProcessing();
     DeferOp defer([&]() {
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::request");
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::status");
         SyncPoint::GetInstance()->ClearCallBack("IsomorphicBatchWrite::send_rpc_request::response");
         SyncPoint::GetInstance()->DisableProcessing();
-        config::batch_write_rpc_request_retry_num = old_retry_num;
-        config::batch_write_rpc_request_retry_interval_ms = old_retry_interval;
+        config::merge_commit_rpc_request_retry_num = old_retry_num;
+        config::merge_commit_rpc_request_retry_interval_ms = old_retry_interval;
     });
 
     // rpc failed
diff --git a/be/test/runtime/batch_write/txn_state_cache_test.cpp b/be/test/runtime/batch_write/txn_state_cache_test.cpp
new file mode 100644
index 00000000000000..cd8b83793a3298
--- /dev/null
+++ b/be/test/runtime/batch_write/txn_state_cache_test.cpp
@@ -0,0 +1,694 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "runtime/batch_write/txn_state_cache.h"
+
+#include "testutil/assert.h"
+#include "util/await.h"
+
+namespace starrocks {
+
+class TxnStateCacheTest : public testing::Test {
+public:
+    TxnStateCacheTest() = default;
+    ~TxnStateCacheTest() override = default;
+
+    void SetUp() override {
+        config::merge_commit_trace_log_enable = true;
+        _db = "test_db";
+        _tbl = "test_tbl";
+        _auth = {"test_user", "test_password"};
+        ASSERT_OK(ThreadPoolBuilder("IsomorphicBatchWriteTest")
+                          .set_min_threads(0)
+                          .set_max_threads(1)
+                          .set_max_queue_size(2048)
+                          .set_idle_timeout(MonoDelta::FromMilliseconds(10000))
+                          .build(&_thread_pool));
+    }
+
+    void TearDown() override {
+        if (_thread_pool) {
+            _thread_pool->shutdown();
+        }
+    }
+
+    std::unique_ptr<TxnStateCache> create_cache(int32_t capacity) {
+        // use serial mode to run poll task in the same order of their execution time, this is just used for testing
+        std::unique_ptr<ThreadPoolToken> token = _thread_pool->new_token(ThreadPool::ExecutionMode::SERIAL);
+        std::unique_ptr<TxnStateCache> cache = std::make_unique<TxnStateCache>(capacity, std::move(token));
+        EXPECT_OK(cache->init());
+        return cache;
+    }
+
+protected:
+    std::string _db;
+    std::string _tbl;
+    AuthInfo _auth;
+
+private:
+    std::unique_ptr<ThreadPool> _thread_pool;
+};
+
+void assert_txn_state_eq(const TxnState& expected, const TxnState& actual) {
+    ASSERT_EQ(expected.txn_status, actual.txn_status);
+    ASSERT_EQ(expected.reason, actual.reason);
+}
+
+TEST_F(TxnStateCacheTest, handler_push_state) {
+    // PREPARE -> COMMITTED -> VISIBLE
+    {
+        TxnStateHandler handler;
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        handler.push_state(TTransactionStatus::COMMITTED, "");
+        assert_txn_state_eq({TTransactionStatus::COMMITTED, ""}, handler.txn_state());
+        ASSERT_TRUE(handler.committed_status_from_fe());
+        handler.push_state(TTransactionStatus::VISIBLE, "");
+        assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, handler.txn_state());
+    }
+
+    // PREPARE -> ABORTED
+    {
+        TxnStateHandler handler;
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        handler.push_state(TTransactionStatus::ABORTED, "manual failure");
+        assert_txn_state_eq({TTransactionStatus::ABORTED, "manual failure"}, handler.txn_state());
+    }
+
+    // PREPARE -> UNKNOWN
+    {
+        TxnStateHandler handler;
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        handler.push_state(TTransactionStatus::UNKNOWN, "");
+        assert_txn_state_eq({TTransactionStatus::UNKNOWN, ""}, handler.txn_state());
+    }
+}
+
+TEST_F(TxnStateCacheTest, handler_poll_state) {
+    bool trigger_poll = false;
+    // PREPARE -> PREPARED -> COMMITTED -> VISIBLE
+    {
+        TxnStateHandler handler;
+        handler.subscribe(trigger_poll);
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        ASSERT_TRUE(handler.poll_state(TxnState{TTransactionStatus::PREPARED, ""}));
+        assert_txn_state_eq({TTransactionStatus::PREPARED, ""}, handler.txn_state());
+        ASSERT_TRUE(handler.poll_state(TxnState{TTransactionStatus::COMMITTED, ""}));
+        assert_txn_state_eq({TTransactionStatus::COMMITTED, ""}, handler.txn_state());
+        ASSERT_FALSE(handler.committed_status_from_fe());
+        ASSERT_FALSE(handler.poll_state(TxnState{TTransactionStatus::VISIBLE, ""}));
+        assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, handler.txn_state());
+    }
+
+    // PREPARE -> ABORTED
+    {
+        TxnStateHandler handler;
+        handler.subscribe(trigger_poll);
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        ASSERT_FALSE(handler.poll_state(TxnState{TTransactionStatus::ABORTED, "manual failure"}));
+        assert_txn_state_eq({TTransactionStatus::ABORTED, "manual failure"}, handler.txn_state());
+    }
+
+    // PREPARE -> UNKNOWN
+    {
+        TxnStateHandler handler;
+        handler.subscribe(trigger_poll);
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        ASSERT_FALSE(handler.poll_state(TxnState{TTransactionStatus::UNKNOWN, ""}));
+        assert_txn_state_eq({TTransactionStatus::UNKNOWN, ""}, handler.txn_state());
+    }
+
+    // max failure (merge_commit_txn_state_poll_max_fail_times) = 2, and only one failure
+    {
+        TxnStateHandler handler;
+        handler.subscribe(trigger_poll);
+        ASSERT_EQ(0, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        ASSERT_TRUE(handler.poll_state(Status::InternalError("artificial failure")));
+        ASSERT_EQ(1, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        ASSERT_TRUE(handler.poll_state(TxnState{TTransactionStatus::COMMITTED, ""}));
+        ASSERT_EQ(0, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::COMMITTED, ""}, handler.txn_state());
+        ASSERT_FALSE(handler.committed_status_from_fe());
+        ASSERT_TRUE(handler.poll_state(Status::InternalError("artificial failure")));
+        ASSERT_EQ(1, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::COMMITTED, ""}, handler.txn_state());
+        ASSERT_FALSE(handler.poll_state(TxnState{TTransactionStatus::VISIBLE, ""}));
+        ASSERT_EQ(0, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, handler.txn_state());
+    }
+
+    // max failure (merge_commit_txn_state_poll_max_fail_times) = 2, and reach max failure
+    {
+        TxnStateHandler handler;
+        handler.subscribe(trigger_poll);
+        ASSERT_EQ(0, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        ASSERT_TRUE(handler.poll_state(TxnState{TTransactionStatus::COMMITTED, ""}));
+        ASSERT_EQ(0, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::COMMITTED, ""}, handler.txn_state());
+        ASSERT_FALSE(handler.committed_status_from_fe());
+        ASSERT_TRUE(handler.poll_state(Status::InternalError("artificial failure")));
+        ASSERT_EQ(1, handler.num_poll_failure());
+        assert_txn_state_eq({TTransactionStatus::COMMITTED, ""}, handler.txn_state());
+        ASSERT_FALSE(handler.poll_state(Status::InternalError("artificial failure")));
+        ASSERT_EQ(2, handler.num_poll_failure());
+        assert_txn_state_eq(
+                {TTransactionStatus::UNKNOWN, "poll txn state failure exceeds max times 2, last error: " +
+                                                      Status::InternalError("artificial failure").to_string(false)},
+                handler.txn_state());
+    }
+
+    // no subscriber
+    {
+        TxnStateHandler handler;
+        handler.subscribe(trigger_poll);
+        assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, handler.txn_state());
+        ASSERT_TRUE(handler.poll_state(TxnState{TTransactionStatus::PREPARED, ""}));
+        assert_txn_state_eq({TTransactionStatus::PREPARED, ""}, handler.txn_state());
+        handler.unsubscribe();
+        ASSERT_FALSE(handler.poll_state(TxnState{TTransactionStatus::COMMITTED, ""}));
+    }
+}
+
+TEST_F(TxnStateCacheTest, handler_subscriber) {
+    TxnStateHandler handler;
+    bool trigger_poll = false;
+    handler.subscribe(trigger_poll);
+    ASSERT_EQ(1, handler.TEST_num_subscriber());
+    ASSERT_TRUE(trigger_poll);
+    handler.subscribe(trigger_poll);
+    ASSERT_EQ(2, handler.TEST_num_subscriber());
+    ASSERT_FALSE(trigger_poll);
+    handler.push_state(TTransactionStatus::VISIBLE, "");
+    handler.unsubscribe();
+    ASSERT_EQ(1, handler.TEST_num_subscriber());
+    handler.unsubscribe();
+    ASSERT_EQ(0, handler.TEST_num_subscriber());
+    handler.subscribe(trigger_poll);
+    ASSERT_EQ(1, handler.TEST_num_subscriber());
+    ASSERT_FALSE(trigger_poll);
+}
+
+TEST_F(TxnStateCacheTest, handler_push_state_notify_subscriber) {
+    TxnStateHandler handler;
+    StatusOr<TxnState> expected_status;
+    auto wait_func = [&](const std::string& name, int64_t timeout_us) {
+        bool trigger_poll = false;
+        handler.subscribe(trigger_poll);
+        auto st = handler.wait_finished_state(name, timeout_us);
+        ASSERT_EQ(expected_status.status().to_string(), st.status().to_string());
+        if (st.ok()) {
+            assert_txn_state_eq(expected_status.value(), st.value());
+        }
+    };
+
+    // wait timeout
+    expected_status = Status::TimedOut("Wait txn state timeout 10000 us");
+    auto t0 = std::thread([&]() { wait_func("t0", 10000); });
+    t0.join();
+    ASSERT_EQ(0, handler.num_waiting_finished_state());
+
+    // wait until final state
+    auto t1 = std::thread([&]() { wait_func("t1", 60000000); });
+    auto t2 = std::thread([&]() { wait_func("t2", 60000000); });
+    ASSERT_TRUE(Awaitility().timeout(5000000).until([&] { return handler.num_waiting_finished_state() == 2; }));
+    expected_status = {TTransactionStatus::VISIBLE, ""};
+    handler.push_state(TTransactionStatus::VISIBLE, "");
+    t1.join();
+    t2.join();
+    ASSERT_EQ(0, handler.num_waiting_finished_state());
+
+    // already in final state
+    auto t3 = std::thread([&]() { wait_func("t3", 60000000); });
+    t3.join();
+    ASSERT_EQ(0, handler.num_waiting_finished_state());
+}
+
+TEST_F(TxnStateCacheTest, handler_poll_state_notify_subscriber) {
+    TxnStateHandler handler;
+    StatusOr<TxnState> expected_status;
+    auto wait_func = [&](const std::string& name, int64_t timeout_us) {
+        bool trigger_poll = false;
+        handler.subscribe(trigger_poll);
+        auto st = handler.wait_finished_state(name, timeout_us);
+        ASSERT_EQ(expected_status.status().to_string(), st.status().to_string());
+        if (st.ok()) {
+            assert_txn_state_eq(expected_status.value(), st.value());
+        }
+    };
+
+    auto t1 = std::thread([&]() { wait_func("t1", 60000000); });
+    auto t2 = std::thread([&]() { wait_func("t2", 60000000); });
+    ASSERT_TRUE(Awaitility().timeout(5000000).until([&] { return handler.num_waiting_finished_state() == 2; }));
+    expected_status = {TTransactionStatus::VISIBLE, ""};
+    handler.poll_state(TxnState{TTransactionStatus::VISIBLE, ""});
+    t1.join();
+    t2.join();
+    ASSERT_EQ(0, handler.num_waiting_finished_state());
+}
+
+TEST_F(TxnStateCacheTest, handler_stop) {
+    TxnStateHandler handler;
+    StatusOr<TxnState> expected_status;
+    auto wait_func = [&](const std::string& name, int64_t timeout_us) {
+        bool trigger_poll = false;
+        handler.subscribe(trigger_poll);
+        auto st = handler.wait_finished_state(name, timeout_us);
+        ASSERT_EQ(expected_status.status().to_string(), st.status().to_string());
+        if (st.ok()) {
+            assert_txn_state_eq(expected_status.value(), st.value());
+        }
+    };
+
+    // wait when stopped
+    auto t1 = std::thread([&]() { wait_func("t1", 60000000); });
+    auto t2 = std::thread([&]() { wait_func("t2", 60000000); });
+    ASSERT_TRUE(Awaitility().timeout(60000000).until([&] { return handler.num_waiting_finished_state() == 2; }));
+    expected_status = Status::ServiceUnavailable("Transaction state handler is stopped");
+    handler.stop();
+    t1.join();
+    t2.join();
+
+    // already stopped
+    auto t3 = std::thread([&]() { wait_func("t3", 60000000); });
+    t3.join();
+}
+
+TEST_F(TxnStateCacheTest, poller_skip_schedule) {
+    auto cache = create_cache(2048);
+    auto old_poll_interval_ms = config::merge_commit_txn_state_poll_interval_ms;
+    config::merge_commit_txn_state_poll_interval_ms = 100;
+    SyncPoint::GetInstance()->EnableProcessing();
+    DeferOp defer([&] {
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::get_current_ms");
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::_execute_poll::request");
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::_execute_poll::status");
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::_execute_poll::response");
+        SyncPoint::GetInstance()->DisableProcessing();
+        cache->stop();
+        config::merge_commit_txn_state_poll_interval_ms = old_poll_interval_ms;
+    });
+    std::atomic<int32_t> num_rpc = 0;
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::request",
+                                          [&](void* arg) { num_rpc.fetch_add(1); });
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::status",
+                                          [&](void* arg) { *((Status*)arg) = Status::OK(); });
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::response", [&](void* arg) {
+        TGetLoadTxnStatusResult* result = (TGetLoadTxnStatusResult*)arg;
+        result->__set_status(TTransactionStatus::VISIBLE);
+        result->__set_reason("");
+    });
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 0; });
+
+    TxnStatePoller* poller = cache->txn_state_poller();
+    ASSERT_TRUE(cache->get_state(1).status().is_not_found());
+    // create a subscriber to trigger the poll scheduling
+    auto s1 = cache->subscribe_state(1, "s1", _db, _tbl, _auth);
+    ASSERT_OK(s1.status());
+    ASSERT_TRUE(poller->TEST_is_txn_pending(1));
+
+    // transit the state to the final before scheduling the poll, and the poll should be skipped
+    ASSERT_OK(cache->push_state(1, TTransactionStatus::VISIBLE, ""));
+    assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, cache->get_state(1).value());
+    // advance the time to schedule the poll for txn 1
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 200; });
+    // create subscriber for txn 2 to trigger the poll scheduling
+    auto s2 = cache->subscribe_state(2, "s2", _db, _tbl, _auth);
+    ASSERT_OK(s2.status());
+    ASSERT_TRUE(poller->TEST_is_txn_pending(2));
+    // advance the time to schedule the poll for txn 2
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 400; });
+    // when state for txn 2 becomes visible, it means poll task for txn 2 has been finished. The thread pool token
+    // is serial, so the poll task (if not skipped) for txn 1 also should be finished
+    ASSERT_TRUE(Awaitility().timeout(5000000).until(
+            [&] { return s2.value()->current_state().txn_status == TTransactionStatus::VISIBLE; }));
+    ASSERT_EQ(1, num_rpc.load());
+}
+
+TEST_F(TxnStateCacheTest, cache_push_state) {
+    auto cache = create_cache(2048);
+    DeferOp defer([&] { cache->stop(); });
+
+    ASSERT_TRUE(cache->get_state(1).status().is_not_found());
+    ASSERT_OK(cache->push_state(1, TTransactionStatus::PREPARE, ""));
+    auto st_1 = cache->get_state(1);
+    ASSERT_OK(st_1.status());
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, st_1.value());
+    ASSERT_OK(cache->push_state(1, TTransactionStatus::VISIBLE, ""));
+    st_1 = cache->get_state(1);
+    assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, st_1.value());
+
+    ASSERT_TRUE(cache->get_state(2).status().is_not_found());
+    ASSERT_OK(cache->push_state(2, TTransactionStatus::ABORTED, "artificial failure"));
+    auto st_2 = cache->get_state(2);
+    ASSERT_OK(st_2.status());
+    assert_txn_state_eq({TTransactionStatus::ABORTED, "artificial failure"}, st_2.value());
+
+    ASSERT_TRUE(cache->get_state(3).status().is_not_found());
+    ASSERT_OK(cache->push_state(3, TTransactionStatus::UNKNOWN, ""));
+    auto st_3 = cache->get_state(3);
+    ASSERT_OK(st_3.status());
+    assert_txn_state_eq({TTransactionStatus::UNKNOWN, ""}, st_3.value());
+}
+
+TEST_F(TxnStateCacheTest, cache_push_state_notify_subscriber) {
+    auto cache = create_cache(2048);
+    SyncPoint::GetInstance()->EnableProcessing();
+    DeferOp defer([&] {
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::get_current_ms");
+        SyncPoint::GetInstance()->DisableProcessing();
+        cache->stop();
+    });
+    // disable poller to avoid unexpected txn state update
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 0; });
+
+    auto wait_func = [&](TxnStateSubscriber* subscriber, int64_t timeout_us, StatusOr<TxnState> expected) {
+        auto st = subscriber->wait_finished_state(timeout_us);
+        ASSERT_EQ(expected.status().to_string(), st.status().to_string());
+        if (st.ok()) {
+            assert_txn_state_eq(expected.value(), st.value());
+        }
+    };
+
+    ASSERT_TRUE(cache->get_state(1).status().is_not_found());
+    auto s1_1 = cache->subscribe_state(1, "s1_1", _db, _tbl, _auth);
+    ASSERT_OK(s1_1.status());
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s1_1.value()->current_state());
+    auto s1_2 = cache->subscribe_state(1, "s1_2", _db, _tbl, _auth);
+    ASSERT_OK(s1_2.status());
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s1_2.value()->current_state());
+
+    ASSERT_TRUE(cache->get_state(2).status().is_not_found());
+    ASSERT_OK(cache->push_state(2, TTransactionStatus::ABORTED, "artificial failure"));
+    auto s2_1 = cache->subscribe_state(2, "s2_1", _db, _tbl, _auth);
+    ASSERT_OK(s2_1.status());
+    assert_txn_state_eq({TTransactionStatus::ABORTED, "artificial failure"}, s2_1.value()->current_state());
+
+    auto t1_1 = std::thread([&]() {
+        wait_func(s1_1.value().get(), 60000000, StatusOr<TxnState>({TTransactionStatus::VISIBLE, ""}));
+    });
+    ASSERT_OK(cache->push_state(1, TTransactionStatus::VISIBLE, ""));
+    t1_1.join();
+
+    auto t1_2 = std::thread([&]() {
+        wait_func(s1_2.value().get(), 60000000, StatusOr<TxnState>({TTransactionStatus::VISIBLE, ""}));
+    });
+    t1_2.join();
+
+    auto t2_1 = std::thread([&]() {
+        wait_func(s2_1.value().get(), 60000000,
+                  StatusOr<TxnState>({TTransactionStatus::ABORTED, "artificial failure"}));
+    });
+    t2_1.join();
+
+    ASSERT_TRUE(cache->get_state(3).status().is_not_found());
+    auto s3_1 = cache->subscribe_state(3, "s3_1", _db, _tbl, _auth);
+    ASSERT_OK(s3_1.status());
+    auto t3_1 = std::thread([&]() {
+        wait_func(s3_1.value().get(), 10000, StatusOr<TxnState>(Status::TimedOut("Wait txn state timeout 10000 us")));
+    });
+    t3_1.join();
+}
+
+TEST_F(TxnStateCacheTest, cache_poll_state_notify_subscriber) {
+    auto cache = create_cache(2048);
+    auto old_poll_interval_ms = config::merge_commit_txn_state_poll_interval_ms;
+    config::merge_commit_txn_state_poll_interval_ms = 100;
+    SyncPoint::GetInstance()->EnableProcessing();
+    DeferOp defer([&] {
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::get_current_ms");
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::_execute_poll::request");
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::_execute_poll::status");
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::_execute_poll::response");
+        SyncPoint::GetInstance()->DisableProcessing();
+        cache->stop();
+        config::merge_commit_txn_state_poll_interval_ms = old_poll_interval_ms;
+    });
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 0; });
+
+    auto wait_func = [&](TxnStateSubscriber* subscriber, int64_t timeout_us, StatusOr<TxnState> expected) {
+        auto st = subscriber->wait_finished_state(timeout_us);
+        ASSERT_EQ(expected.status().to_string(), st.status().to_string());
+        if (st.ok()) {
+            assert_txn_state_eq(expected.value(), st.value());
+        }
+    };
+
+    std::atomic<int32_t> num_rpc = 0;
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::request",
+                                          [&](void* arg) { num_rpc.fetch_add(1); });
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::status",
+                                          [&](void* arg) { *((Status*)arg) = Status::OK(); });
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::response", [&](void* arg) {
+        TGetLoadTxnStatusResult* result = (TGetLoadTxnStatusResult*)arg;
+        result->__set_status(TTransactionStatus::VISIBLE);
+        result->__set_reason("");
+    });
+
+    // txn 1 and 2 should be scheduled at time 100, current is 0
+    TxnStatePoller* poller = cache->txn_state_poller();
+    ASSERT_TRUE(cache->get_state(1).status().is_not_found());
+    auto s1_1 = cache->subscribe_state(1, "s1_1", _db, _tbl, _auth);
+    ASSERT_OK(s1_1.status());
+    ASSERT_TRUE(poller->TEST_is_txn_pending(1));
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s1_1.value()->current_state());
+    auto s1_2 = cache->subscribe_state(1, "s1_2", _db, _tbl, _auth);
+    ASSERT_OK(s1_2.status());
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s1_2.value()->current_state());
+
+    ASSERT_TRUE(cache->get_state(2).status().is_not_found());
+    auto s2_1 = cache->subscribe_state(2, "s2_1", _db, _tbl, _auth);
+    ASSERT_OK(s2_1.status());
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s2_1.value()->current_state());
+    ASSERT_TRUE(poller->TEST_is_txn_pending(2));
+
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 50; });
+    // txn 3 should be scheduled at time 150, current is 50
+    ASSERT_TRUE(cache->get_state(3).status().is_not_found());
+    auto s3_1 = cache->subscribe_state(3, "s3_1", _db, _tbl, _auth);
+    ASSERT_OK(s3_1.status());
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s3_1.value()->current_state());
+    ASSERT_TRUE(poller->TEST_is_txn_pending(1));
+    ASSERT_TRUE(poller->TEST_is_txn_pending(2));
+    ASSERT_TRUE(poller->TEST_is_txn_pending(3));
+
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 80; });
+    // txn 4 should be scheduled at time 180, current is 80
+    ASSERT_TRUE(cache->get_state(4).status().is_not_found());
+    auto s4_1 = cache->subscribe_state(4, "s4_1", _db, _tbl, _auth);
+    ASSERT_OK(s4_1.status());
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s4_1.value()->current_state());
+    ASSERT_TRUE(poller->TEST_is_txn_pending(1));
+    ASSERT_TRUE(poller->TEST_is_txn_pending(2));
+    ASSERT_TRUE(poller->TEST_is_txn_pending(3));
+    ASSERT_TRUE(poller->TEST_is_txn_pending(4));
+
+    auto t1_1 = std::thread([&]() {
+        wait_func(s1_1.value().get(), 60000000, StatusOr<TxnState>({TTransactionStatus::VISIBLE, ""}));
+    });
+    auto t1_2 = std::thread([&]() {
+        wait_func(s1_2.value().get(), 60000000, StatusOr<TxnState>({TTransactionStatus::VISIBLE, ""}));
+    });
+    auto t2_1 = std::thread([&]() {
+        wait_func(s2_1.value().get(), 60000000, StatusOr<TxnState>({TTransactionStatus::VISIBLE, ""}));
+    });
+    auto t3_1 = std::thread([&]() {
+        wait_func(s3_1.value().get(), 60000000, StatusOr<TxnState>({TTransactionStatus::VISIBLE, ""}));
+    });
+
+    // advance time and should trigger txn 1, 2 and 3 to poll
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 160; });
+    t1_1.join();
+    t1_2.join();
+    t2_1.join();
+    t3_1.join();
+
+    assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, s1_1.value()->current_state());
+    assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, s2_1.value()->current_state());
+    assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, s3_1.value()->current_state());
+    ASSERT_FALSE(poller->TEST_is_txn_pending(1));
+    ASSERT_FALSE(poller->TEST_is_txn_pending(2));
+    ASSERT_FALSE(poller->TEST_is_txn_pending(3));
+    ASSERT_TRUE(poller->TEST_is_txn_pending(4));
+    ASSERT_EQ(3, num_rpc.load());
+
+    assert_txn_state_eq({TTransactionStatus::PREPARE, ""}, s4_1.value()->current_state());
+    auto t4_1 = std::thread([&]() {
+        wait_func(s4_1.value().get(), 60000000, StatusOr<TxnState>({TTransactionStatus::VISIBLE, ""}));
+    });
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::response", [&](void* arg) {
+        TGetLoadTxnStatusResult* result = (TGetLoadTxnStatusResult*)arg;
+        result->__set_status(TTransactionStatus::COMMITTED);
+        result->__set_reason("");
+    });
+    // advance time and trigger txn 4 to poll. it should continue polling because
+    // the state is COMMITTED which is not final. next poll time is 300
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 200; });
+    ASSERT_TRUE(Awaitility().timeout(5000000).until([&] {
+        auto st = poller->TEST_pending_execution_time(4);
+        return st.ok() && st.value() == 300;
+    }));
+    assert_txn_state_eq({TTransactionStatus::COMMITTED, ""}, s4_1.value()->current_state());
+    ASSERT_EQ(4, num_rpc.load());
+
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::_execute_poll::response", [&](void* arg) {
+        TGetLoadTxnStatusResult* result = (TGetLoadTxnStatusResult*)arg;
+        result->__set_status(TTransactionStatus::VISIBLE);
+        result->__set_reason("");
+    });
+    // advance time and trigger txn 4 to poll again. This time it should reach the finished state
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 400; });
+    t4_1.join();
+    ASSERT_EQ(5, num_rpc.load());
+    assert_txn_state_eq({TTransactionStatus::VISIBLE, ""}, s4_1.value()->current_state());
+}
+
+TEST_F(TxnStateCacheTest, cache_eviction) {
+    int32_t numShards = 1 << 5;
+    auto cache = create_cache(numShards * 3);
+    DeferOp defer_stop([&] { cache->stop(); });
+
+    int64_t evict_txn_id = -1;
+    int32_t num_evict = 0;
+    SyncPoint::GetInstance()->EnableProcessing();
+    DeferOp defer([]() {
+        SyncPoint::GetInstance()->ClearCallBack("TxnStateHandler::destruct");
+        SyncPoint::GetInstance()->DisableProcessing();
+    });
+
+    SyncPoint::GetInstance()->SetCallBack("TxnStateHandler::destruct", [&](void* arg) {
+        TxnStateHandler* handler = (TxnStateHandler*)arg;
+        evict_txn_id = handler->txn_id();
+        num_evict += 1;
+    });
+    ASSERT_OK(cache->push_state(1 << 5, TTransactionStatus::VISIBLE, ""));
+    ASSERT_EQ(1, cache->size());
+    ASSERT_EQ(0, num_evict);
+    auto s1 = cache->subscribe_state(2 << 5, "s1", _db, _tbl, _auth);
+    ASSERT_OK(s1.status());
+    ASSERT_EQ(2, cache->size());
+    ASSERT_EQ(0, num_evict);
+    ASSERT_OK(cache->push_state(3 << 5, TTransactionStatus::VISIBLE, ""));
+    ASSERT_EQ(3, cache->size());
+    ASSERT_EQ(0, num_evict);
+
+    ASSERT_OK(cache->push_state(4 << 5, TTransactionStatus::VISIBLE, ""));
+    ASSERT_EQ(3, cache->size());
+    ASSERT_EQ(1 << 5, evict_txn_id);
+    ASSERT_EQ(1, num_evict);
+
+    auto s2 = cache->subscribe_state(5 << 5, "s2", _db, _tbl, _auth);
+    ASSERT_OK(s2.status());
+    ASSERT_EQ(3, cache->size());
+    ASSERT_EQ(3 << 5, evict_txn_id);
+    ASSERT_EQ(2, num_evict);
+
+    auto s3 = cache->subscribe_state(6 << 5, "s3", _db, _tbl, _auth);
+    ASSERT_OK(s3.status());
+    ASSERT_EQ(3, cache->size());
+    ASSERT_EQ(4 << 5, evict_txn_id);
+    ASSERT_EQ(3, num_evict);
+
+    s1.value().reset();
+    ASSERT_OK(cache->push_state(7 << 5, TTransactionStatus::VISIBLE, ""));
+    ASSERT_EQ(3, cache->size());
+    ASSERT_EQ(2 << 5, evict_txn_id);
+    ASSERT_EQ(4, num_evict);
+}
+
+TEST_F(TxnStateCacheTest, cache_set_capacity) {
+    auto cache = create_cache(2048);
+    DeferOp defer([&] { cache->stop(); });
+    auto shards = cache->get_cache_shards();
+    for (auto shard : shards) {
+        ASSERT_EQ(2048 / 32, shard->capacity());
+    }
+    cache->set_capacity(4096);
+    for (auto shard : shards) {
+        ASSERT_EQ(4096 / 32, shard->capacity());
+    }
+}
+
+TEST_F(TxnStateCacheTest, cache_clean_txn_state) {
+    auto old_clean_interval_sec = config::merge_commit_txn_state_clean_interval_sec;
+    auto old_expire_time_sec = config::merge_commit_txn_state_expire_time_sec;
+    config::merge_commit_txn_state_clean_interval_sec = 1;
+    config::merge_commit_txn_state_expire_time_sec = 1;
+    auto cache = create_cache(2048);
+    SyncPoint::GetInstance()->EnableProcessing();
+    DeferOp defer([&] {
+        SyncPoint::GetInstance()->ClearCallBack("TxnStateHandler::destruct");
+        SyncPoint::GetInstance()->DisableProcessing();
+        cache->stop();
+        config::merge_commit_txn_state_clean_interval_sec = old_clean_interval_sec;
+        config::merge_commit_txn_state_expire_time_sec = old_expire_time_sec;
+    });
+    std::atomic<int32_t> num_evict = 0;
+    SyncPoint::GetInstance()->SetCallBack("TxnStateHandler::destruct", [&](void* arg) { num_evict += 1; });
+    ASSERT_OK(cache->push_state(1, TTransactionStatus::VISIBLE, ""));
+    ASSERT_OK(cache->push_state(2, TTransactionStatus::VISIBLE, ""));
+    ASSERT_OK(cache->push_state(3, TTransactionStatus::VISIBLE, ""));
+    ASSERT_TRUE(Awaitility().timeout(10000000).until([&] { return num_evict == 3; }));
+    for (auto shard : cache->get_cache_shards()) {
+        ASSERT_EQ(0, shard->object_size());
+    }
+}
+
+TEST_F(TxnStateCacheTest, cache_stop) {
+    auto cache = create_cache(2048);
+    SyncPoint::GetInstance()->EnableProcessing();
+    DeferOp defer([&] {
+        SyncPoint::GetInstance()->ClearCallBack("TxnStatePoller::get_current_ms");
+        SyncPoint::GetInstance()->DisableProcessing();
+        cache->stop();
+    });
+    // disable poller to avoid unexpected txn state update
+    SyncPoint::GetInstance()->SetCallBack("TxnStatePoller::get_current_ms", [&](void* arg) { *((int64_t*)arg) = 0; });
+
+    Status expected_status = Status::ServiceUnavailable("Transaction state handler is stopped");
+    auto wait_func = [&](TxnStateSubscriber* subscriber, int64_t timeout_us, StatusOr<TxnState> expected) {
+        auto st = subscriber->wait_finished_state(timeout_us);
+        ASSERT_EQ(expected.status().to_string(), st.status().to_string());
+        if (st.ok()) {
+            assert_txn_state_eq(expected.value(), st.value());
+        }
+    };
+
+    auto s1 = cache->subscribe_state(1, "s1", _db, _tbl, _auth);
+    ASSERT_OK(s1.status());
+    auto t1 = std::thread([&]() { wait_func(s1.value().get(), 60000000, StatusOr<TxnState>(expected_status)); });
+    auto s2 = cache->subscribe_state(1, "s2", _db, _tbl, _auth);
+    ASSERT_OK(s2.status());
+    auto s3 = cache->subscribe_state(2, "s3", _db, _tbl, _auth);
+    ASSERT_OK(s3.status());
+    auto t3 = std::thread([&]() { wait_func(s3.value().get(), 60000000, StatusOr<TxnState>(expected_status)); });
+
+    ASSERT_TRUE(Awaitility().timeout(5000000).until(
+            [&] { return s1.value()->entry()->value().num_waiting_finished_state() == 1; }));
+    ASSERT_TRUE(Awaitility().timeout(5000000).until(
+            [&] { return s3.value()->entry()->value().num_waiting_finished_state() == 1; }));
+    cache->stop();
+    t1.join();
+    t3.join();
+    auto wait_st = s2.value()->wait_finished_state(10000);
+    ASSERT_EQ(expected_status.to_string(), wait_st.status().to_string());
+
+    expected_status = Status::ServiceUnavailable("Transaction state cache is stopped");
+    ASSERT_EQ(expected_status.to_string(), cache->push_state(3, TTransactionStatus::VISIBLE, "").to_string(false));
+    ASSERT_EQ(expected_status.to_string(), cache->subscribe_state(3, "s4", _db, _tbl, _auth).status().to_string(false));
+    ASSERT_FALSE(cache->txn_state_poller()->TEST_is_scheduling());
+}
+
+} // namespace starrocks
diff --git a/be/test/util/dynamic_cache_test.cpp b/be/test/util/dynamic_cache_test.cpp
index b6ec91056cf380..c7a43b6f55e3bc 100644
--- a/be/test/util/dynamic_cache_test.cpp
+++ b/be/test/util/dynamic_cache_test.cpp
@@ -86,4 +86,19 @@ TEST(DynamicCacheTest, cache2) {
     }
 }
 
+TEST(DynamicCacheTest, get_all_entries) {
+    DynamicCache<int32_t, int64_t> cache(100);
+    for (int i = 0; i < 20; i++) {
+        auto e = cache.get_or_create(i, 1);
+        cache.release(e);
+    }
+    std::vector<DynamicCache<int32_t, int64_t>::Entry*> entries = cache.get_all_entries();
+    ASSERT_EQ(20, entries.size());
+    for (int i = 0; i < 20; i++) {
+        auto e = entries[i];
+        ASSERT_EQ(i, e->key());
+        cache.release(e);
+    }
+}
+
 } // namespace starrocks

From 957ecfe9c037de21fdf9ce6420cf9cbe7c891369 Mon Sep 17 00:00:00 2001
From: satanson <ranpanf@gmail.com>
Date: Thu, 16 Jan 2025 18:56:51 +0800
Subject: [PATCH 17/71] [BugFix] destruct workgroup's MemTracker prematurely
 (#55134)

Signed-off-by: satanson <ranpanf@gmail.com>
---
 be/src/exec/pipeline/query_context.cpp | 12 +++++++++++-
 be/src/exec/pipeline/query_context.h   |  8 ++++++--
 be/src/exec/workgroup/work_group.h     |  2 ++
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/be/src/exec/pipeline/query_context.cpp b/be/src/exec/pipeline/query_context.cpp
index 09b940cf88bfd7..3a0f2331ef6303 100644
--- a/be/src/exec/pipeline/query_context.cpp
+++ b/be/src/exec/pipeline/query_context.cpp
@@ -190,10 +190,20 @@ Status QueryContext::init_query_once(workgroup::WorkGroup* wg, bool enable_group
 void QueryContext::release_workgroup_token_once() {
     auto* old = _wg_running_query_token_atomic_ptr.load();
     if (old != nullptr && _wg_running_query_token_atomic_ptr.compare_exchange_strong(old, nullptr)) {
+        // The release_workgroup_token_once function is called by FragmentContext::cancel
+        // to detach the QueryContext from the workgroup.
+        // When the workgroup undergoes a configuration change, the old version of the workgroup is released,
+        // and a new version is created. The old workgroup will only be physically destroyed once no
+        // QueryContext is attached to it.
+        // However, the MemTracker of the old workgroup outlives the workgroup itself because
+        // it is accessed during the destruction of the QueryContext through its MemTracker
+        // (the workgroup's MemTracker serves as the parent of the QueryContext's MemTracker).
+        // To prevent the MemTracker from being released prematurely, it must be explicitly retained
+        // to ensure it remains valid until it is no longer needed.
+        _wg_mem_tracker = _wg_running_query_token_ptr->get_wg()->grab_mem_tracker();
         _wg_running_query_token_ptr.reset();
     }
 }
-
 void QueryContext::set_query_trace(std::shared_ptr<starrocks::debug::QueryTrace> query_trace) {
     std::call_once(_query_trace_init_flag, [this, &query_trace]() { _query_trace = std::move(query_trace); });
 }
diff --git a/be/src/exec/pipeline/query_context.h b/be/src/exec/pipeline/query_context.h
index 639790acfd822e..6f96baf0b80aa5 100644
--- a/be/src/exec/pipeline/query_context.h
+++ b/be/src/exec/pipeline/query_context.h
@@ -305,8 +305,6 @@ class QueryContext : public std::enable_shared_from_this<QueryContext> {
     int64_t _big_query_profile_threshold_ns = 0;
     int64_t _runtime_profile_report_interval_ns = std::numeric_limits<int64_t>::max();
     TPipelineProfileLevel::type _profile_level;
-    std::shared_ptr<MemTracker> _mem_tracker;
-    std::shared_ptr<MemTracker> _connector_scan_mem_tracker;
     ObjectPool _object_pool;
     DescriptorTbl* _desc_tbl = nullptr;
     std::once_flag _query_trace_init_flag;
@@ -354,8 +352,14 @@ class QueryContext : public std::enable_shared_from_this<QueryContext> {
     std::shared_ptr<QueryStatisticsRecvr> _sub_plan_query_statistics_recvr; // For receive
 
     int64_t _scan_limit = 0;
+    // _wg_mem_tracker is used to grab mem_tracker in workgroup to prevent it from
+    // being released prematurely in FragmentContext::cancel, otherwise accessing
+    // workgroup's mem_tracker in QueryContext's dtor shall cause segmentation fault.
+    std::shared_ptr<MemTracker> _wg_mem_tracker = nullptr;
     workgroup::RunningQueryTokenPtr _wg_running_query_token_ptr;
     std::atomic<workgroup::RunningQueryToken*> _wg_running_query_token_atomic_ptr = nullptr;
+    std::shared_ptr<MemTracker> _mem_tracker;
+    std::shared_ptr<MemTracker> _connector_scan_mem_tracker;
 
     // STREAM MV
     std::shared_ptr<StreamEpochManager> _stream_epoch_manager;
diff --git a/be/src/exec/workgroup/work_group.h b/be/src/exec/workgroup/work_group.h
index 9fa55a1f480664..800069e005320e 100644
--- a/be/src/exec/workgroup/work_group.h
+++ b/be/src/exec/workgroup/work_group.h
@@ -101,6 +101,7 @@ using WorkGroupScanSchedEntity = WorkGroupSchedEntity<ScanTaskQueue>;
 struct RunningQueryToken {
     explicit RunningQueryToken(WorkGroupPtr wg) : wg(std::move(wg)) {}
     ~RunningQueryToken();
+    WorkGroupPtr get_wg() { return wg; }
 
 private:
     WorkGroupPtr wg;
@@ -127,6 +128,7 @@ class WorkGroup : public std::enable_shared_from_this<WorkGroup> {
     void copy_metrics(const WorkGroup& rhs);
 
     MemTracker* mem_tracker() { return _mem_tracker.get(); }
+    std::shared_ptr<MemTracker> grab_mem_tracker() { return _mem_tracker; }
     const MemTracker* mem_tracker() const { return _mem_tracker.get(); }
     MemTracker* connector_scan_mem_tracker() { return _connector_scan_mem_tracker.get(); }
 

From 4ff8cd14bb9ca9ef9c5f2dffcd6dc527d21fc178 Mon Sep 17 00:00:00 2001
From: Evgeniy Zuikin <eugenzuy@gmail.com>
Date: Thu, 16 Jan 2025 18:31:13 +0700
Subject: [PATCH 18/71] [Feature] Make blacklist persistent (#55014)

Signed-off-by: Evgeniy Zuykin <evgeniy.zuykin@agoda.com>
---
 .../java/com/starrocks/meta/SqlBlackList.java | 109 +++++++--
 .../persist/DeleteSqlBlackLists.java          |  29 +++
 .../java/com/starrocks/persist/EditLog.java   |  21 ++
 .../persist/EditLogDeserializer.java          |   2 +
 .../com/starrocks/persist/OperationType.java  |   5 +
 .../persist/SqlBlackListPersistInfo.java      |  31 +++
 .../persist/metablock/SRMetaBlockID.java      |   2 +
 .../java/com/starrocks/qe/ShowExecutor.java   |   7 +-
 .../java/com/starrocks/qe/StmtExecutor.java   |  13 +-
 .../com/starrocks/server/GlobalStateMgr.java  |  10 +
 .../sql/ast/AddSqlBlackListStmt.java          |   2 +-
 .../sql/ast/DelSqlBlackListStmt.java          |   2 +-
 .../SqlBlacklistAndWhitelistTest.java         |   2 +-
 .../com/starrocks/backup/RestoreJobTest.java  |   2 +-
 .../starrocks/persist/OperationTypeTest.java  |   2 +
 .../starrocks/planner/QueryPlannerTest.java   |  41 ++--
 .../starrocks/server/SqlBlacklistTest.java    | 228 ++++++++++++++++++
 17 files changed, 457 insertions(+), 51 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/persist/DeleteSqlBlackLists.java
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/persist/SqlBlackListPersistInfo.java
 create mode 100644 fe/fe-core/src/test/java/com/starrocks/server/SqlBlacklistTest.java

diff --git a/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java b/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java
index d8d84003b136f3..92e7cf61b5883e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java
+++ b/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java
@@ -15,57 +15,130 @@
 
 package com.starrocks.meta;
 
+import com.staros.util.LockCloseable;
 import com.starrocks.common.AnalysisException;
 import com.starrocks.common.ErrorCode;
 import com.starrocks.common.ErrorReport;
+import com.starrocks.persist.ImageWriter;
+import com.starrocks.persist.SqlBlackListPersistInfo;
+import com.starrocks.persist.metablock.SRMetaBlockEOFException;
+import com.starrocks.persist.metablock.SRMetaBlockException;
+import com.starrocks.persist.metablock.SRMetaBlockID;
+import com.starrocks.persist.metablock.SRMetaBlockReader;
+import com.starrocks.persist.metablock.SRMetaBlockWriter;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 // Used by sql's blacklist
 public class SqlBlackList {
-    private static final SqlBlackList INSTANCE = new SqlBlackList();
 
-    public static SqlBlackList getInstance() {
-        return INSTANCE;
-    }
+    private static final Logger LOG = LogManager.getLogger(SqlBlackList.class);
 
-    public static void verifying(String sql) throws AnalysisException {
+    public void verifying(String sql) throws AnalysisException {
         String formatSql = sql.replace("\r", " ").replace("\n", " ").replaceAll("\\s+", " ");
-        for (BlackListSql patternAndId : getInstance().sqlBlackListMap.values()) {
-            Matcher m = patternAndId.pattern.matcher(formatSql);
-            if (m.find()) {
-                ErrorReport.reportAnalysisException(ErrorCode.ERR_SQL_IN_BLACKLIST_ERROR);
+        try (LockCloseable ignored = new LockCloseable(rwLock.readLock())) {
+            for (BlackListSql patternAndId : sqlBlackListMap.values()) {
+                Matcher m = patternAndId.pattern.matcher(formatSql);
+                if (m.find()) {
+                    ErrorReport.reportAnalysisException(ErrorCode.ERR_SQL_IN_BLACKLIST_ERROR);
+                }
+            }
+        }
+    }
+
+    public void load(SRMetaBlockReader reader) throws IOException, SRMetaBlockException, SRMetaBlockEOFException {
+        try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) {
+            int cnt = reader.readInt();
+            for (int i = 0; i < cnt; i++) {
+                SqlBlackListPersistInfo sqlBlackListPersistInfo = reader.readJson(SqlBlackListPersistInfo.class);
+                put(sqlBlackListPersistInfo.id, Pattern.compile(sqlBlackListPersistInfo.pattern));
             }
+            LOG.info("loaded {} SQL blacklist patterns", sqlBlackListMap.size());
         }
     }
 
     // we use string of sql as key, and (pattern, id) as value.
-    public void put(Pattern pattern) {
-        if (!sqlBlackListMap.containsKey(pattern.toString())) {
-            long id = ids.getAndIncrement();
-            sqlBlackListMap.putIfAbsent(pattern.toString(), new BlackListSql(pattern, id));
+    public long put(Pattern pattern) {
+        try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) {
+            BlackListSql blackListSql = sqlBlackListMap.get(pattern.toString());
+            if (blackListSql == null) {
+                long id = ids.getAndIncrement();
+                sqlBlackListMap.put(pattern.toString(), new BlackListSql(pattern, id));
+                return id;
+            } else {
+                return blackListSql.id;
+            }
+        }
+    }
+
+    public void put(long id, Pattern pattern) {
+        try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) {
+            BlackListSql blackListSql = sqlBlackListMap.get(pattern.toString());
+            if (blackListSql == null) {
+                ids.set(Math.max(ids.get(), id + 1));
+                sqlBlackListMap.put(pattern.toString(), new BlackListSql(pattern, id));
+            }
         }
     }
 
     // we delete sql's regular expression use id, so we iterate this map.
     public void delete(long id) {
-        for (Map.Entry<String, BlackListSql> entry : sqlBlackListMap.entrySet()) {
-            if (entry.getValue().id == id) {
-                sqlBlackListMap.remove(entry.getKey());
+        try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) {
+            for (Map.Entry<String, BlackListSql> entry : sqlBlackListMap.entrySet()) {
+                if (entry.getValue().id == id) {
+                    sqlBlackListMap.remove(entry.getKey());
+                }
+            }
+        }
+    }
+
+    public void delete(List<Long> ids) {
+        for (Long id : ids) {
+            this.delete(id);
+        }
+    }
+
+    public void save(ImageWriter imageWriter) throws IOException, SRMetaBlockException {
+        try (LockCloseable ignored = new LockCloseable(rwLock.readLock())) {
+            // one for self and N for patterns
+            final int cnt = 1 + sqlBlackListMap.size();
+            SRMetaBlockWriter writer = imageWriter.getBlockWriter(SRMetaBlockID.BLACKLIST_MGR, cnt);
+
+            // write patterns
+            writer.writeInt(sqlBlackListMap.size());
+            for (BlackListSql p : sqlBlackListMap.values()) {
+                writer.writeJson(new SqlBlackListPersistInfo(p.id, p.pattern.pattern()));
             }
+            writer.close();
         }
     }
 
+    public List<BlackListSql> getBlackLists() {
+        try (LockCloseable ignored = new LockCloseable(rwLock.readLock())) {
+            return this.sqlBlackListMap.values().stream().sorted(Comparator.comparing(x -> x.id)).collect(Collectors.toList());
+        }
+    }
+
+    private final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+
     // sqlBlackListMap: key is String(sql), value is BlackListSql.
     // BlackListSql is (Pattern, id). Pattern is the regular expression, id marks this sql, and is show with "show sqlblacklist";
-    public ConcurrentMap<String, BlackListSql> sqlBlackListMap = new ConcurrentHashMap<>();
+    private final ConcurrentMap<String, BlackListSql> sqlBlackListMap = new ConcurrentHashMap<>();
 
     // ids used in sql blacklist
-    public AtomicLong ids = new AtomicLong();
+    private final AtomicLong ids = new AtomicLong();
 }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/DeleteSqlBlackLists.java b/fe/fe-core/src/main/java/com/starrocks/persist/DeleteSqlBlackLists.java
new file mode 100644
index 00000000000000..b30c4fe554ec4f
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/persist/DeleteSqlBlackLists.java
@@ -0,0 +1,29 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.persist;
+
+import com.google.gson.annotations.SerializedName;
+import com.starrocks.common.io.JsonWriter;
+
+import java.util.List;
+
+public class DeleteSqlBlackLists extends JsonWriter {
+    public DeleteSqlBlackLists(List<Long> ids) {
+        this.ids = ids;
+    }
+
+    @SerializedName("ids")
+    public final List<Long> ids;
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java
index 4f627fb8ec9a28..e8230bf7219325 100644
--- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java
@@ -124,6 +124,7 @@
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ExecutionException;
+import java.util.regex.Pattern;
 
 /**
  * EditLog maintains a log of the memory modifications.
@@ -1111,6 +1112,17 @@ public void loadJournal(GlobalStateMgr globalStateMgr, JournalEntity journal)
                     globalStateMgr.getClusterSnapshotMgr().replayLog(log);
                     break;
                 }
+                case OperationType.OP_ADD_SQL_QUERY_BLACK_LIST: {
+                    SqlBlackListPersistInfo addBlacklistRequest = (SqlBlackListPersistInfo) journal.data();
+                    GlobalStateMgr.getCurrentState().getSqlBlackList()
+                            .put(addBlacklistRequest.id, Pattern.compile(addBlacklistRequest.pattern));
+                    break;
+                }
+                case OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST: {
+                    DeleteSqlBlackLists deleteBlackListsRequest = (DeleteSqlBlackLists) journal.data();
+                    GlobalStateMgr.getCurrentState().getSqlBlackList().delete(deleteBlackListsRequest.ids);
+                    break;
+                }
                 default: {
                     if (Config.metadata_ignore_unknown_operation_type) {
                         LOG.warn("UNKNOWN Operation Type {}", opCode);
@@ -1795,6 +1807,15 @@ public void logAlterMaterializedViewProperties(ModifyTablePropertyOperationLog l
         logEdit(OperationType.OP_ALTER_MATERIALIZED_VIEW_PROPERTIES, log);
     }
 
+    public void logAddSQLBlackList(SqlBlackListPersistInfo addBlackList) {
+        logEdit(OperationType.OP_ADD_SQL_QUERY_BLACK_LIST, addBlackList);
+    }
+
+    public void logDeleteSQLBlackList(DeleteSqlBlackLists deleteBlacklists) {
+        logEdit(OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST, deleteBlacklists);
+    }
+
+
     public void logStarMgrOperation(StarMgrJournal journal) {
         logEdit(OperationType.OP_STARMGR, journal);
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java
index 598255874ea454..40359c2a9b056f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java
@@ -243,6 +243,8 @@ public class EditLogDeserializer {
             .put(OperationType.OP_ALTER_WAREHOUSE, Warehouse.class)
             .put(OperationType.OP_DROP_WAREHOUSE, DropWarehouseLog.class)
             .put(OperationType.OP_CLUSTER_SNAPSHOT_LOG, ClusterSnapshotLog.class)
+            .put(OperationType.OP_ADD_SQL_QUERY_BLACK_LIST, SqlBlackListPersistInfo.class)
+            .put(OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST, DeleteSqlBlackLists.class)
             .build();
 
     public static Writable deserialize(Short opCode, DataInput in) throws IOException {
diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java
index 892150ae323d5b..b64b4600f5d591 100644
--- a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java
@@ -591,6 +591,11 @@ public class OperationType {
     @IgnorableOnReplayFailed
     public static final short OP_CLUSTER_SNAPSHOT_LOG = 13513;
 
+    @IgnorableOnReplayFailed
+    public static final short OP_ADD_SQL_QUERY_BLACK_LIST = 13520;
+    @IgnorableOnReplayFailed
+    public static final short OP_DELETE_SQL_QUERY_BLACK_LIST = 13521;
+
     /**
      * NOTICE: OperationType cannot use a value exceeding 20000, please follow the above sequence number
      */
diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/SqlBlackListPersistInfo.java b/fe/fe-core/src/main/java/com/starrocks/persist/SqlBlackListPersistInfo.java
new file mode 100644
index 00000000000000..7bb7c0b783dda7
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/persist/SqlBlackListPersistInfo.java
@@ -0,0 +1,31 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.persist;
+
+import com.google.gson.annotations.SerializedName;
+import com.starrocks.common.io.JsonWriter;
+
+public class SqlBlackListPersistInfo extends JsonWriter {
+    public SqlBlackListPersistInfo(long id, String pattern) {
+        this.id = id;
+        this.pattern = pattern;
+    }
+
+    @SerializedName("id")
+    public final long id;
+
+    @SerializedName("pattern")
+    public final String pattern;
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java b/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java
index f9a7505b9f31a1..7a8f87c720330b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java
+++ b/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java
@@ -99,6 +99,8 @@ public int getId() {
 
     public static final SRMetaBlockID CLUSTER_SNAPSHOT_MGR = new SRMetaBlockID(33);
 
+    public static final SRMetaBlockID BLACKLIST_MGR = new SRMetaBlockID(34);
+
     /**
      * NOTICE: SRMetaBlockID cannot use a value exceeding 20000, please follow the above sequence number
      */
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
index 04d58ac7ec5faf..1fa5c5e05c26b5 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
@@ -129,7 +129,6 @@
 import com.starrocks.load.streamload.StreamLoadFunctionalExprProvider;
 import com.starrocks.load.streamload.StreamLoadTask;
 import com.starrocks.meta.BlackListSql;
-import com.starrocks.meta.SqlBlackList;
 import com.starrocks.proto.FailPointTriggerModeType;
 import com.starrocks.proto.PFailPointInfo;
 import com.starrocks.proto.PFailPointTriggerMode;
@@ -2234,10 +2233,10 @@ public ShowResultSet visitShowPluginsStatement(ShowPluginsStmt statement, Connec
         @Override
         public ShowResultSet visitShowSqlBlackListStatement(ShowSqlBlackListStmt statement, ConnectContext context) {
             List<List<String>> rows = new ArrayList<>();
-            for (Map.Entry<String, BlackListSql> entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) {
+            for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) {
                 List<String> oneSql = new ArrayList<>();
-                oneSql.add(String.valueOf(entry.getValue().id));
-                oneSql.add(entry.getKey());
+                oneSql.add(String.valueOf(entry.id));
+                oneSql.add(entry.pattern.toString());
                 rows.add(oneSql);
             }
             return new ShowResultSet(statement.getMetaData(), rows);
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
index 76763ce6713b9e..b20c5fabf0ac93 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
@@ -99,7 +99,6 @@
 import com.starrocks.load.InsertOverwriteJobMgr;
 import com.starrocks.load.loadv2.InsertLoadJob;
 import com.starrocks.load.loadv2.LoadJob;
-import com.starrocks.meta.SqlBlackList;
 import com.starrocks.metric.MetricRepo;
 import com.starrocks.metric.TableMetricsEntity;
 import com.starrocks.metric.TableMetricsRegistry;
@@ -108,6 +107,8 @@
 import com.starrocks.mysql.MysqlEofPacket;
 import com.starrocks.mysql.MysqlSerializer;
 import com.starrocks.persist.CreateInsertOverwriteJobLog;
+import com.starrocks.persist.DeleteSqlBlackLists;
+import com.starrocks.persist.SqlBlackListPersistInfo;
 import com.starrocks.persist.gson.GsonUtils;
 import com.starrocks.planner.FileScanNode;
 import com.starrocks.planner.HiveTableSink;
@@ -631,7 +632,7 @@ public void execute() throws Exception {
                     String originSql = origStmt.originStmt.trim()
                             .toLowerCase().replaceAll(" +", " ");
                     // If this sql is in blacklist, show message.
-                    SqlBlackList.verifying(originSql);
+                    GlobalStateMgr.getCurrentState().getSqlBlackList().verifying(originSql);
                 }
             }
 
@@ -1691,7 +1692,9 @@ public void checkPrivilegeForKillAnalyzeStmt(ConnectContext context, long analyz
 
     private void handleAddSqlBlackListStmt() {
         AddSqlBlackListStmt addSqlBlackListStmt = (AddSqlBlackListStmt) parsedStmt;
-        SqlBlackList.getInstance().put(addSqlBlackListStmt.getSqlPattern());
+        long id = GlobalStateMgr.getCurrentState().getSqlBlackList().put(addSqlBlackListStmt.getSqlPattern());
+        GlobalStateMgr.getCurrentState().getEditLog()
+                .logAddSQLBlackList(new SqlBlackListPersistInfo(id, addSqlBlackListStmt.getSqlPattern().pattern()));
     }
 
     private void handleDelSqlBlackListStmt() {
@@ -1699,8 +1702,10 @@ private void handleDelSqlBlackListStmt() {
         List<Long> indexs = delSqlBlackListStmt.getIndexs();
         if (indexs != null) {
             for (long id : indexs) {
-                SqlBlackList.getInstance().delete(id);
+                GlobalStateMgr.getCurrentState().getSqlBlackList().delete(id);
             }
+            GlobalStateMgr.getCurrentState().getEditLog()
+                    .logDeleteSQLBlackList(new DeleteSqlBlackLists(indexs));
         }
     }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
index 4b74586c334a53..1738230bacf2f4 100644
--- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
@@ -166,6 +166,7 @@
 import com.starrocks.load.streamload.StreamLoadMgr;
 import com.starrocks.memory.MemoryUsageTracker;
 import com.starrocks.memory.ProcProfileCollector;
+import com.starrocks.meta.SqlBlackList;
 import com.starrocks.metric.MetricRepo;
 import com.starrocks.persist.BackendIdsUpdateInfo;
 import com.starrocks.persist.EditLog;
@@ -521,6 +522,8 @@ public class GlobalStateMgr {
 
     private final ClusterSnapshotMgr clusterSnapshotMgr;
 
+    private final SqlBlackList sqlBlackList;
+
     public NodeMgr getNodeMgr() {
         return nodeMgr;
     }
@@ -819,6 +822,7 @@ public void transferToNonLeader(FrontendNodeType newType) {
         this.authorizer = new Authorizer(accessControlProvider);
         this.ddlStmtExecutor = new DDLStmtExecutor(DDLStmtExecutor.StmtExecutorVisitor.getInstance());
         this.showExecutor = new ShowExecutor(ShowExecutor.ShowExecutorVisitor.getInstance());
+        this.sqlBlackList = new SqlBlackList();
         this.temporaryTableCleaner = new TemporaryTableCleaner();
         this.queryDeployExecutor =
                 ThreadPoolManager.newDaemonFixedThreadPool(Config.query_deploy_threadpool_size, Integer.MAX_VALUE,
@@ -1563,6 +1567,7 @@ public void loadImage(String imageDir) throws IOException {
                 .put(SRMetaBlockID.PIPE_MGR, pipeManager.getRepo()::load)
                 .put(SRMetaBlockID.WAREHOUSE_MGR, warehouseMgr::load)
                 .put(SRMetaBlockID.CLUSTER_SNAPSHOT_MGR, clusterSnapshotMgr::load)
+                .put(SRMetaBlockID.BLACKLIST_MGR, sqlBlackList::load)
                 .build();
 
         Set<SRMetaBlockID> metaMgrMustExists = new HashSet<>(loadImages.keySet());
@@ -1762,6 +1767,7 @@ public void saveImage(ImageWriter imageWriter, File curFile) throws IOException
                 keyMgr.save(imageWriter);
                 pipeManager.getRepo().save(imageWriter);
                 warehouseMgr.save(imageWriter);
+                sqlBlackList.save(imageWriter);
             } catch (SRMetaBlockException e) {
                 LOG.error("Save meta block failed ", e);
                 throw new IOException("Save meta block failed ", e);
@@ -2182,6 +2188,10 @@ public ExportMgr getExportMgr() {
         return this.exportMgr;
     }
 
+    public SqlBlackList getSqlBlackList() {
+        return this.sqlBlackList;
+    }
+
     public MaterializedViewMgr getMaterializedViewMgr() {
         return this.materializedViewMgr;
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java
index 5b9720b885164a..078a31cb58ad64 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java
@@ -65,7 +65,7 @@ public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
 
     @Override
     public RedirectStatus getRedirectStatus() {
-        return RedirectStatus.NO_FORWARD;
+        return RedirectStatus.FORWARD_NO_SYNC;
     }
 }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java
index 51c3b04e218f30..34d4c367b57b5c 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java
@@ -46,7 +46,7 @@ public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
 
     @Override
     public RedirectStatus getRedirectStatus() {
-        return RedirectStatus.NO_FORWARD;
+        return RedirectStatus.FORWARD_NO_SYNC;
     }
 }
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java
index f4e308c87d43ff..93c1fc1f95ad54 100644
--- a/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java
@@ -39,7 +39,7 @@ public void testAddSqlBlacklist() {
     @Test
     public void testDelSqlBlacklist() {
         DelSqlBlackListStmt stmt = (DelSqlBlackListStmt) analyzeSuccess("delete sqlblacklist  2, 6;");
-        Assert.assertEquals(Lists.asList(2L, new Long[] {6L}), stmt.getIndexs());
+        Assert.assertEquals(Lists.asList(2L, new Long[]{6L}), stmt.getIndexs());
         Assert.assertNotNull(stmt.getRedirectStatus());
         // bad cases
         analyzeFail("DELETE SQLBLACKLIST");
diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java
index daf600a4ab8a0c..02302cc293378b 100644
--- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java
@@ -557,7 +557,7 @@ public void testRunBackupListTable() {
                 minTimes = 0;
                 result = id.incrementAndGet();
 
-                GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo();
+                globalStateMgr.getNodeMgr().getClusterInfo();
                 minTimes = 0;
                 result = systemInfoService;
             }
diff --git a/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java b/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java
index 01f6e411a2a214..1da0c84a9a509d 100644
--- a/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java
@@ -151,6 +151,8 @@ public void testRecoverableOperations() {
         Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_DELETE_REPLICATION_JOB));
         Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_RESET_FRONTENDS));
         Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_CLUSTER_SNAPSHOT_LOG));
+        Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_ADD_SQL_QUERY_BLACK_LIST));
+        Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST));
     }
 
     @Test
diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java
index 816e93f56a1853..41da313ebd57b8 100644
--- a/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java
@@ -38,7 +38,6 @@
 import com.starrocks.common.util.UUIDUtil;
 import com.starrocks.ha.FrontendNodeType;
 import com.starrocks.meta.BlackListSql;
-import com.starrocks.meta.SqlBlackList;
 import com.starrocks.qe.ConnectContext;
 import com.starrocks.qe.QueryState;
 import com.starrocks.qe.StmtExecutor;
@@ -161,11 +160,11 @@ public void testSqlBlackList() throws Exception {
         StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement);
         stmtExecutor1.execute();
 
-        Assert.assertEquals(SqlBlackList.getInstance().sqlBlackListMap.entrySet().size(), 1);
+        Assert.assertEquals(GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size(), 1);
         long id = -1;
-        for (Map.Entry<String, BlackListSql> entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) {
-            id = entry.getValue().id;
-            Assert.assertEquals("select k1 from .+", entry.getKey());
+        for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) {
+            id = entry.id;
+            Assert.assertEquals("select k1 from .+", entry.pattern.pattern());
         }
 
         String sql = "select k1 from test.baseall";
@@ -193,7 +192,7 @@ public void testSqlBlackList() throws Exception {
                 connectContext.getSessionVariable().getSqlMode());
         StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement);
         stmtExecutor3.execute();
-        Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size());
+        Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size());
     }
 
     @Test
@@ -210,11 +209,11 @@ public void testSqlBlackListWithLineSeparators() throws Exception {
         StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement);
         stmtExecutor1.execute();
 
-        Assert.assertEquals(1, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size());
+        Assert.assertEquals(1, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size());
         long id = -1;
-        for (Map.Entry<String, BlackListSql> entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) {
-            id = entry.getValue().id;
-            Assert.assertEquals("select k1 from .+", entry.getKey());
+        for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) {
+            id = entry.id;
+            Assert.assertEquals("select k1 from .+", entry.pattern.pattern());
         }
 
         String sql = "select k1 from test.baseall";
@@ -241,7 +240,7 @@ public void testSqlBlackListWithLineSeparators() throws Exception {
                 connectContext.getSessionVariable().getSqlMode());
         StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement);
         stmtExecutor3.execute();
-        Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size());
+        Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size());
     }
 
     @Test
@@ -258,11 +257,11 @@ public void testSqlBlackListUseWhere() throws Exception {
         StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement);
         stmtExecutor1.execute();
 
-        Assert.assertEquals(SqlBlackList.getInstance().sqlBlackListMap.entrySet().size(), 1);
+        Assert.assertEquals(GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size(), 1);
         long id = -1;
-        for (Map.Entry<String, BlackListSql> entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) {
-            id = entry.getValue().id;
-            Assert.assertEquals("( where )", entry.getKey());
+        for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) {
+            id = entry.id;
+            Assert.assertEquals("( where )", entry.pattern.pattern());
         }
 
         String sql4 = "select k1 as awhere from test.baseall";
@@ -283,7 +282,7 @@ public void testSqlBlackListUseWhere() throws Exception {
                 connectContext.getSessionVariable().getSqlMode());
         StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement);
         stmtExecutor3.execute();
-        Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size());
+        Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size());
     }
     @Test
     public void testSqlBlackListWithInsert() throws Exception {
@@ -299,11 +298,11 @@ public void testSqlBlackListWithInsert() throws Exception {
         StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement);
         stmtExecutor1.execute();
 
-        Assert.assertEquals(SqlBlackList.getInstance().sqlBlackListMap.entrySet().size(), 1);
+        Assert.assertEquals(GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size(), 1);
         long id = -1;
-        for (Map.Entry<String, BlackListSql> entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) {
-            id = entry.getValue().id;
-            Assert.assertEquals("insert into .+ values.+", entry.getKey());
+        for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) {
+            id = entry.id;
+            Assert.assertEquals("insert into .+ values.+", entry.pattern.pattern());
         }
 
         String sql =
@@ -323,7 +322,7 @@ public void testSqlBlackListWithInsert() throws Exception {
                 connectContext.getSessionVariable().getSqlMode());
         StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement);
         stmtExecutor3.execute();
-        Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size());
+        Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size());
     }
 
     @Test
diff --git a/fe/fe-core/src/test/java/com/starrocks/server/SqlBlacklistTest.java b/fe/fe-core/src/test/java/com/starrocks/server/SqlBlacklistTest.java
new file mode 100644
index 00000000000000..12244274f7d3dc
--- /dev/null
+++ b/fe/fe-core/src/test/java/com/starrocks/server/SqlBlacklistTest.java
@@ -0,0 +1,228 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.server;
+
+import com.starrocks.analysis.RedirectStatus;
+import com.starrocks.common.jmockit.Deencapsulation;
+import com.starrocks.meta.BlackListSql;
+import com.starrocks.meta.SqlBlackList;
+import com.starrocks.persist.DeleteSqlBlackLists;
+import com.starrocks.persist.EditLog;
+import com.starrocks.persist.SqlBlackListPersistInfo;
+import com.starrocks.qe.ConnectContext;
+import com.starrocks.qe.ShowExecutor;
+import com.starrocks.qe.ShowResultSet;
+import com.starrocks.qe.StmtExecutor;
+import com.starrocks.sql.analyzer.AnalyzeTestUtil;
+import com.starrocks.sql.ast.AddSqlBlackListStmt;
+import com.starrocks.sql.ast.DelSqlBlackListStmt;
+import com.starrocks.sql.ast.ShowSqlBlackListStmt;
+import com.starrocks.utframe.UtFrameUtils;
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.util.List;
+import java.util.UUID;
+import java.util.regex.Pattern;
+
+import static com.starrocks.sql.analyzer.AnalyzeTestUtil.parseSql;
+
+public class SqlBlacklistTest {
+    GlobalStateMgr state;
+    SqlBlackList sqlBlackList;
+    EditLog editLog;
+    ConnectContext connectContext;
+
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        AnalyzeTestUtil.init();
+    }
+
+    @Before
+    public void beforeEach() {
+        state = Deencapsulation.newInstance(GlobalStateMgr.class);
+        sqlBlackList = new SqlBlackList();
+        connectContext = UtFrameUtils.createDefaultCtx();
+        editLog = Mockito.mock(EditLog.class);
+        connectContext.setQueryId(UUID.randomUUID());
+    }
+
+    @Test
+    public void testAddSQLBlacklist() throws Exception {
+        mockupGlobalState();
+
+        ArgumentCaptor<SqlBlackListPersistInfo> addBlacklistEditLogArgument = ArgumentCaptor
+                .forClass(SqlBlackListPersistInfo.class);
+
+        AddSqlBlackListStmt addStatement = (AddSqlBlackListStmt) parseSql("ADD SQLBLACKLIST \".+\";");
+        Assert.assertEquals(addStatement.getSql(), ".+");
+
+        StmtExecutor addStatementExecutor = new StmtExecutor(connectContext, addStatement);
+        addStatementExecutor.execute();
+        List<BlackListSql> blackLists = sqlBlackList.getBlackLists();
+        Assert.assertEquals(1, blackLists.size());
+        Assert.assertEquals(0, blackLists.get(0).id);
+        Assert.assertEquals(".+", blackLists.get(0).pattern.pattern());
+
+        Mockito.verify(editLog).logAddSQLBlackList(addBlacklistEditLogArgument.capture());
+
+        Assert.assertEquals(0, addBlacklistEditLogArgument.getValue().id);
+        Assert.assertEquals(".+", addBlacklistEditLogArgument.getValue().pattern);
+    }
+
+    @Test
+    public void testShowBlacklist() {
+        mockupGlobalState();
+        sqlBlackList.put(Pattern.compile("qwert"));
+        sqlBlackList.put(Pattern.compile("abcde"));
+
+        ShowSqlBlackListStmt showSqlStatement = (ShowSqlBlackListStmt) parseSql("SHOW SQLBLACKLIST");
+
+        ShowResultSet resultSet = ShowExecutor.execute(showSqlStatement, connectContext);
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(0L, resultSet.getLong(0));
+        Assert.assertEquals("qwert", resultSet.getString(1));
+        Assert.assertTrue(resultSet.next());
+        Assert.assertEquals(1L, resultSet.getLong(0));
+        Assert.assertEquals("abcde", resultSet.getString(1));
+        Assert.assertFalse(resultSet.next());
+    }
+
+    @Test
+    public void testBlackListReturnsSameIdIfPatternAlreadyExists() {
+        mockupGlobalState();
+        Pattern p = Pattern.compile("qwert");
+        long id = sqlBlackList.put(p);
+
+        Assert.assertEquals(id, sqlBlackList.put(p));
+    }
+
+    @Test
+    public void testDeleteSqlBlacklist() throws Exception {
+        mockupGlobalState();
+        long id1 = sqlBlackList.put(Pattern.compile("qwert"));
+        long id2 = sqlBlackList.put(Pattern.compile("abcde"));
+
+        ArgumentCaptor<DeleteSqlBlackLists> deleteBlacklistsEditLogArgument =
+                ArgumentCaptor.forClass(DeleteSqlBlackLists.class);
+
+        StmtExecutor deleteStatementExecutor = new StmtExecutor(connectContext, new DelSqlBlackListStmt(List.of(id1, id2)));
+        deleteStatementExecutor.execute();
+        Assert.assertTrue(sqlBlackList
+                .getBlackLists().stream().noneMatch(x -> x.id == id1 || x.id != id2));
+
+        Mockito.verify(editLog).logDeleteSQLBlackList(deleteBlacklistsEditLogArgument.capture());
+
+        Assert.assertEquals(List.of(id1, id2), deleteBlacklistsEditLogArgument.getValue().ids);
+    }
+
+    @Test
+    public void testRedirectStatus() {
+        Assert.assertEquals(
+                new AddSqlBlackListStmt("ADD SQLBLACKLIST \".+\";").getRedirectStatus(),
+                RedirectStatus.FORWARD_NO_SYNC
+        );
+        Assert.assertEquals(
+                new DelSqlBlackListStmt(List.of(1L, 2L)).getRedirectStatus(),
+                RedirectStatus.FORWARD_NO_SYNC
+        );
+    }
+
+    @Test
+    public void testSaveLoadBlackListImage() throws Exception {
+        SqlBlackList originalBlacklist = new SqlBlackList();
+        originalBlacklist.put(Pattern.compile("zxcvbqwert"));
+        originalBlacklist.put(Pattern.compile("qwdsad"));
+
+        UtFrameUtils.PseudoImage testImage = new UtFrameUtils.PseudoImage();
+        originalBlacklist.save(testImage.getImageWriter());
+
+        SqlBlackList recoveredBlackList = new SqlBlackList();
+        recoveredBlackList.load(testImage.getMetaBlockReader());
+
+        Assert.assertEquals(originalBlacklist.getBlackLists().size(), recoveredBlackList.getBlackLists().size());
+        Assert.assertEquals(originalBlacklist.getBlackLists().get(0).id, recoveredBlackList.getBlackLists().get(0).id);
+        Assert.assertEquals(
+                originalBlacklist.getBlackLists().get(0).pattern.pattern(),
+                recoveredBlackList.getBlackLists().get(0).pattern.pattern()
+        );
+        Assert.assertEquals(originalBlacklist.getBlackLists().get(1).id, recoveredBlackList.getBlackLists().get(1).id);
+        Assert.assertEquals(
+                originalBlacklist.getBlackLists().get(1).pattern.pattern(),
+                recoveredBlackList.getBlackLists().get(1).pattern.pattern()
+        );
+    }
+
+    @Test
+    public void testSqlBlacklistJournalOperations() throws Exception {
+        UtFrameUtils.createMinStarRocksCluster();
+        UtFrameUtils.setUpForPersistTest();
+        UtFrameUtils.PseudoJournalReplayer.resetFollowerJournalQueue();
+
+        // add blacklists
+
+        GlobalStateMgr.getCurrentState().getEditLog().logAddSQLBlackList(new SqlBlackListPersistInfo(123, "p1"));
+        GlobalStateMgr.getCurrentState().getEditLog().logAddSQLBlackList(new SqlBlackListPersistInfo(1234, "p2"));
+        UtFrameUtils.PseudoJournalReplayer.replayJournalToEnd();
+
+        List<BlackListSql> resultBlackLists = GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists();
+        Assert.assertEquals(2, resultBlackLists.size());
+        Assert.assertEquals(123L, resultBlackLists.get(0).id);
+        Assert.assertEquals("p1", resultBlackLists.get(0).pattern.pattern());
+        Assert.assertEquals(1234L, resultBlackLists.get(1).id);
+        Assert.assertEquals("p2", resultBlackLists.get(1).pattern.pattern());
+
+        // delete blacklists
+
+        GlobalStateMgr.getCurrentState().getEditLog().logDeleteSQLBlackList(new DeleteSqlBlackLists(List.of(123L, 1234L)));
+        UtFrameUtils.PseudoJournalReplayer.replayJournalToEnd();
+
+        Assert.assertTrue(
+                sqlBlackList.getBlackLists().stream()
+                        .noneMatch(x -> x.id == 123L || x.id == 1234L)
+        );
+
+    }
+
+    private void mockupGlobalState() {
+        MockUp<GlobalStateMgr> mockUp = new MockUp<GlobalStateMgr>() {
+            @Mock
+            GlobalStateMgr getCurrentState() {
+                return state;
+            }
+
+            @Mock
+            public SqlBlackList getSqlBlackList() {
+                return sqlBlackList;
+            }
+
+            @Mock
+            public boolean isLeader() {
+                return true;
+            }
+
+            @Mock
+            public EditLog getEditLog() {
+                return editLog;
+            }
+        };
+    }
+}

From 37ed8644bb36fbe6b82c0ad4e520cf603f96a69b Mon Sep 17 00:00:00 2001
From: stdpain <34912776+stdpain@users.noreply.github.com>
Date: Thu, 16 Jan 2025 19:49:29 +0800
Subject: [PATCH 19/71] [Enhancement] add  session variable
 parallel_merge_late_materialization_mode control the parallel merge behaviour
 (#55082)

Signed-off-by: stdpain <drfeng08@gmail.com>
---
 be/src/exec/exchange_node.cpp                 |   4 +
 ...xchange_parallel_merge_source_operator.cpp |   3 +-
 .../exchange_parallel_merge_source_operator.h |   2 +
 ...al_parallel_merge_sort_source_operator.cpp |   2 +-
 ...ocal_parallel_merge_sort_source_operator.h |   2 +
 be/src/exec/sorting/merge_path.cpp            |  14 +-
 be/src/exec/sorting/merge_path.h              |   5 +-
 be/src/exec/topn_node.cpp                     |   4 +
 .../com/starrocks/planner/ExchangeNode.java   |   7 +-
 .../java/com/starrocks/planner/SortNode.java  |   3 +
 .../com/starrocks/qe/SessionVariable.java     |   9 ++
 .../qe/SessionVariableConstants.java          |   4 +
 gensrc/thrift/PlanNodes.thrift                |  20 +--
 .../R/test_parallel_merge_lazy_materialize    | 151 ++++++++++++++++++
 .../T/test_parallel_merge_lazy_materialize    |  47 ++++++
 15 files changed, 257 insertions(+), 20 deletions(-)
 create mode 100644 test/sql/test_sort/R/test_parallel_merge_lazy_materialize
 create mode 100644 test/sql/test_sort/T/test_parallel_merge_lazy_materialize

diff --git a/be/src/exec/exchange_node.cpp b/be/src/exec/exchange_node.cpp
index 4466a86bc50973..2fb1d07c15bc0d 100644
--- a/be/src/exec/exchange_node.cpp
+++ b/be/src/exec/exchange_node.cpp
@@ -259,6 +259,10 @@ pipeline::OpFactories ExchangeNode::decompose_to_pipeline(pipeline::PipelineBuil
             auto exchange_merge_sort_source_operator = std::make_shared<ExchangeParallelMergeSourceOperatorFactory>(
                     context->next_operator_id(), id(), _num_senders, _input_row_desc, &_sort_exec_exprs, _is_asc_order,
                     _nulls_first, _offset, _limit);
+            if (_texchange_node.__isset.parallel_merge_late_materialize_mode) {
+                exchange_merge_sort_source_operator->set_materialized_mode(
+                        _texchange_node.parallel_merge_late_materialize_mode);
+            }
             exchange_merge_sort_source_operator->set_degree_of_parallelism(context->degree_of_parallelism());
             operators.emplace_back(std::move(exchange_merge_sort_source_operator));
             // This particular exchange source will be executed in a concurrent way, and finally we need to gather them into one
diff --git a/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.cpp b/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.cpp
index 1c4b620486717a..53597022c0e02c 100644
--- a/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.cpp
+++ b/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.cpp
@@ -108,7 +108,8 @@ merge_path::MergePathCascadeMerger* ExchangeParallelMergeSourceOperatorFactory::
         SortDescs sort_descs(_is_asc_order, _nulls_first);
         _merger = std::make_unique<merge_path::MergePathCascadeMerger>(
                 state->chunk_size(), degree_of_parallelism(), _sort_exec_exprs->lhs_ordering_expr_ctxs(), sort_descs,
-                _row_desc.tuple_descriptors()[0], TTopNType::ROW_NUMBER, _offset, _limit, chunk_providers);
+                _row_desc.tuple_descriptors()[0], TTopNType::ROW_NUMBER, _offset, _limit, chunk_providers,
+                _late_materialize_mode);
     }
     return _merger.get();
 }
diff --git a/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.h b/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.h
index 0d7adfc563fd70..b6e30b5924073c 100644
--- a/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.h
+++ b/be/src/exec/pipeline/exchange/exchange_parallel_merge_source_operator.h
@@ -108,6 +108,7 @@ class ExchangeParallelMergeSourceOperatorFactory final : public SourceOperatorFa
     void close_stream_recvr();
 
     SourceOperatorFactory::AdaptiveState adaptive_initial_state() const override { return AdaptiveState::ACTIVE; }
+    void set_materialized_mode(TLateMaterializeMode::type mode) { _late_materialize_mode = mode; }
 
 private:
     const int32_t _num_sender;
@@ -117,6 +118,7 @@ class ExchangeParallelMergeSourceOperatorFactory final : public SourceOperatorFa
     const std::vector<bool>& _nulls_first;
     const int64_t _offset;
     const int64_t _limit;
+    TLateMaterializeMode::type _late_materialize_mode = TLateMaterializeMode::AUTO;
 
     std::shared_ptr<DataStreamRecvr> _stream_recvr;
     std::atomic<int64_t> _stream_recvr_cnt = 0;
diff --git a/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.cpp b/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.cpp
index cd8199e7d70bd4..5115c50d40d909 100644
--- a/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.cpp
+++ b/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.cpp
@@ -105,7 +105,7 @@ OperatorPtr LocalParallelMergeSortSourceOperatorFactory::create(int32_t degree_o
             _mergers.push_back(std::make_unique<merge_path::MergePathCascadeMerger>(
                     _state->chunk_size(), degree_of_parallelism, sort_context->sort_exprs(), sort_context->sort_descs(),
                     _tuple_desc, sort_context->topn_type(), sort_context->offset(), sort_context->limit(),
-                    chunk_providers));
+                    chunk_providers, _late_materialize_mode));
         }
         return std::make_shared<LocalParallelMergeSortSourceOperator>(
                 this, _id, _plan_node_id, driver_sequence, sort_context.get(), _is_gathered, _mergers[0].get());
diff --git a/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.h b/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.h
index bd9a5aecad0927..019471bf563c90 100644
--- a/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.h
+++ b/be/src/exec/pipeline/sort/local_parallel_merge_sort_source_operator.h
@@ -94,11 +94,13 @@ class LocalParallelMergeSortSourceOperatorFactory final : public SourceOperatorF
 
     void set_tuple_desc(const TupleDescriptor* tuple_desc) { _tuple_desc = tuple_desc; }
     void set_is_gathered(const bool is_gathered) { _is_gathered = is_gathered; }
+    void set_materialized_mode(TLateMaterializeMode::type mode) { _late_materialize_mode = mode; }
 
 private:
     const TupleDescriptor* _tuple_desc;
     bool _is_gathered = true;
     RuntimeState* _state;
+    TLateMaterializeMode::type _late_materialize_mode = TLateMaterializeMode::AUTO;
 
     // share data with multiple partition sort sink opeartor through _sort_context.
     std::shared_ptr<SortContextFactory> _sort_context_factory;
diff --git a/be/src/exec/sorting/merge_path.cpp b/be/src/exec/sorting/merge_path.cpp
index 9d92fc65b79ba5..b77252541d527e 100644
--- a/be/src/exec/sorting/merge_path.cpp
+++ b/be/src/exec/sorting/merge_path.cpp
@@ -645,7 +645,8 @@ MergePathCascadeMerger::MergePathCascadeMerger(const size_t chunk_size, const in
                                                std::vector<ExprContext*> sort_exprs, const SortDescs& sort_descs,
                                                const TupleDescriptor* tuple_desc, const TTopNType::type topn_type,
                                                const int64_t offset, const int64_t limit,
-                                               std::vector<MergePathChunkProvider> chunk_providers)
+                                               std::vector<MergePathChunkProvider> chunk_providers,
+                                               TLateMaterializeMode::type mode)
         : _chunk_size(chunk_size > MAX_CHUNK_SIZE ? MAX_CHUNK_SIZE : chunk_size),
           _streaming_batch_size(4 * chunk_size * degree_of_parallelism),
           _degree_of_parallelism(degree_of_parallelism),
@@ -657,7 +658,8 @@ MergePathCascadeMerger::MergePathCascadeMerger(const size_t chunk_size, const in
           _limit(limit),
           _chunk_providers(std::move(chunk_providers)),
           _process_cnts(degree_of_parallelism),
-          _output_chunks(degree_of_parallelism) {
+          _output_chunks(degree_of_parallelism),
+          _late_materialization_mode(mode) {
     _working_nodes.resize(_degree_of_parallelism);
     _metrics.resize(_degree_of_parallelism);
 
@@ -1133,11 +1135,17 @@ void MergePathCascadeMerger::_init_late_materialization() {
             metrics.profile->add_info_string("LateMaterialization", _late_materialization ? "True" : "False");
         });
     });
-
     if (_chunk_providers.size() <= 2) {
         _late_materialization = false;
         return;
     }
+    if (_late_materialization_mode == TLateMaterializeMode::ALWAYS) {
+        _late_materialization = true;
+        return;
+    } else if (_late_materialization_mode == TLateMaterializeMode::NEVER) {
+        _late_materialization = false;
+        return;
+    }
 
     const auto level_size = static_cast<size_t>(std::ceil(std::log2(_chunk_providers.size())));
     std::unordered_set<SlotId> early_materialized_slots;
diff --git a/be/src/exec/sorting/merge_path.h b/be/src/exec/sorting/merge_path.h
index 66bf89797cf08d..08b8dc9110c1d9 100644
--- a/be/src/exec/sorting/merge_path.h
+++ b/be/src/exec/sorting/merge_path.h
@@ -373,7 +373,8 @@ class MergePathCascadeMerger {
     MergePathCascadeMerger(const size_t chunk_size, const int32_t degree_of_parallelism,
                            std::vector<ExprContext*> sort_exprs, const SortDescs& sort_descs,
                            const TupleDescriptor* tuple_desc, const TTopNType::type topn_type, const int64_t offset,
-                           const int64_t limit, std::vector<MergePathChunkProvider> chunk_providers);
+                           const int64_t limit, std::vector<MergePathChunkProvider> chunk_providers,
+                           TLateMaterializeMode::type mode = TLateMaterializeMode::AUTO);
     const std::vector<ExprContext*>& sort_exprs() const { return _sort_exprs; }
     const SortDescs& sort_descs() const { return _sort_descs; }
 
@@ -505,6 +506,8 @@ class MergePathCascadeMerger {
     // First pending should not be recorded, because it all comes from the operator dependency
     bool _is_first_pending = true;
 
+    TLateMaterializeMode::type _late_materialization_mode;
+
     starrocks::pipeline::Observable _observable;
 };
 
diff --git a/be/src/exec/topn_node.cpp b/be/src/exec/topn_node.cpp
index 12b047e49ded63..8c20e022051180 100644
--- a/be/src/exec/topn_node.cpp
+++ b/be/src/exec/topn_node.cpp
@@ -346,6 +346,10 @@ std::vector<std::shared_ptr<pipeline::OperatorFactory>> TopNNode::_decompose_to_
                 ->set_tuple_desc(_materialized_tuple_desc);
         down_cast<LocalParallelMergeSortSourceOperatorFactory*>(source_operator.get())->set_is_gathered(need_merge);
     }
+    if (enable_parallel_merge && _tnode.sort_node.__isset.parallel_merge_late_materialize_mode) {
+        down_cast<LocalParallelMergeSortSourceOperatorFactory*>(source_operator.get())
+                ->set_materialized_mode(_tnode.sort_node.parallel_merge_late_materialize_mode);
+    }
 
     ops_sink_with_sort.emplace_back(std::move(sink_operator));
     context->add_pipeline(ops_sink_with_sort);
diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/ExchangeNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/ExchangeNode.java
index c3daf74b2621a7..df790ee360ee5d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/planner/ExchangeNode.java
+++ b/fe/fe-core/src/main/java/com/starrocks/planner/ExchangeNode.java
@@ -50,6 +50,7 @@
 import com.starrocks.sql.optimizer.operator.TopNType;
 import com.starrocks.thrift.TExchangeNode;
 import com.starrocks.thrift.TExplainLevel;
+import com.starrocks.thrift.TLateMaterializeMode;
 import com.starrocks.thrift.TNormalExchangeNode;
 import com.starrocks.thrift.TNormalPlanNode;
 import com.starrocks.thrift.TNormalSortInfo;
@@ -198,8 +199,10 @@ protected void toThrift(TPlanNode msg) {
         if (partitionType != null) {
             msg.exchange_node.setPartition_type(partitionType);
         }
-        SessionVariable sessionVariable = ConnectContext.get().getSessionVariable();
-        msg.exchange_node.setEnable_parallel_merge(sessionVariable.isEnableParallelMerge());
+        SessionVariable sv = ConnectContext.get().getSessionVariable();
+        msg.exchange_node.setEnable_parallel_merge(sv.isEnableParallelMerge());
+        TLateMaterializeMode mode = TLateMaterializeMode.valueOf(sv.getParallelMergeLateMaterializationMode().toUpperCase());
+        msg.exchange_node.setParallel_merge_late_materialize_mode(mode);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/SortNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/SortNode.java
index d90c794744fb78..721f4197580768 100644
--- a/fe/fe-core/src/main/java/com/starrocks/planner/SortNode.java
+++ b/fe/fe-core/src/main/java/com/starrocks/planner/SortNode.java
@@ -53,6 +53,7 @@
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.sql.optimizer.operator.TopNType;
 import com.starrocks.thrift.TExplainLevel;
+import com.starrocks.thrift.TLateMaterializeMode;
 import com.starrocks.thrift.TNormalPlanNode;
 import com.starrocks.thrift.TNormalSortInfo;
 import com.starrocks.thrift.TNormalSortNode;
@@ -231,6 +232,8 @@ protected void toThrift(TPlanNode msg) {
 
         msg.sort_node.setLate_materialization(sessionVariable.isFullSortLateMaterialization());
         msg.sort_node.setEnable_parallel_merge(sessionVariable.isEnableParallelMerge());
+        TLateMaterializeMode mode = TLateMaterializeMode.valueOf(sessionVariable.getParallelMergeLateMaterializationMode().toUpperCase());
+        msg.sort_node.setParallel_merge_late_materialize_mode(mode);
 
         if (info.getPartitionExprs() != null) {
             msg.sort_node.setPartition_exprs(Expr.treesToThrift(info.getPartitionExprs()));
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
index 7910f5ee8f706d..80d1fc3c95a0a6 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
@@ -498,6 +498,7 @@ public class SessionVariable implements Serializable, Writable, Cloneable {
     public static final String GROUP_EXECUTION_MIN_SCAN_ROWS = "group_execution_min_scan_rows";
 
     public static final String ENABLE_PARALLEL_MERGE = "enable_parallel_merge";
+    public static final String PARALLEL_MERGE_LATE_MATERIALIZATION_MODE = "parallel_merge_late_materialization_mode";
     public static final String ENABLE_QUERY_QUEUE = "enable_query_queue";
 
     public static final String WINDOW_PARTITION_MODE = "window_partition_mode";
@@ -1588,6 +1589,10 @@ public static MaterializedViewRewriteMode parse(String str) {
     @VarAttr(name = ENABLE_PARALLEL_MERGE)
     private boolean enableParallelMerge = true;
 
+    // AUTO/ALWAYS/NEVER
+    @VarAttr(name = PARALLEL_MERGE_LATE_MATERIALIZATION_MODE)
+    private String parallelMergeLateMaterializationMode = SessionVariableConstants.AUTO;
+
     @VarAttr(name = ENABLE_QUERY_QUEUE, flag = VariableMgr.INVISIBLE)
     private boolean enableQueryQueue = true;
 
@@ -1781,6 +1786,10 @@ public boolean isEnableParallelMerge() {
         return enableParallelMerge;
     }
 
+    public String getParallelMergeLateMaterializationMode() {
+        return parallelMergeLateMaterializationMode;
+    }
+
     public void setEnableParallelMerge(boolean enableParallelMerge) {
         this.enableParallelMerge = enableParallelMerge;
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariableConstants.java b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariableConstants.java
index 5feab63a7cb3b9..4b07f11a2aad50 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariableConstants.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariableConstants.java
@@ -36,6 +36,10 @@ private SessionVariableConstants() {}
 
     public static final String VARCHAR = "varchar";
 
+    public static final String ALWAYS = "always";
+
+    public static final String NEVER = "never";
+
     public enum ChooseInstancesMode {
 
         // the number of chosen instances is the same as the max number of instances from its children fragments
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 17d90fe4e48dc0..bcb5d1f95dbf73 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -802,18 +802,6 @@ enum TAggregationOp {
   PERCENT_RANK
 }
 
-//struct TAggregateFunctionCall {
-  // The aggregate function to call.
-//  1: required Types.TFunction fn
-
-  // The input exprs to this aggregate function
-//  2: required list<Exprs.TExpr> input_exprs
-
-  // If set, this aggregate function udf has varargs and this is the index for the
-  // first variable argument.
-//  3: optional i32 vararg_start_idx
-//}
-
 struct TAggregationNode {
   1: optional list<Exprs.TExpr> grouping_exprs
   // aggregate exprs. The root of each expr is the aggregate function. The
@@ -887,6 +875,12 @@ enum TTopNType {
   DENSE_RANK
 }
 
+enum TLateMaterializeMode {
+  AUTO,
+  ALWAYS,
+  NEVER,
+}
+
 struct TSortNode {
   1: required TSortInfo sort_info
   // Indicates whether the backend service should use topn vs. sorting
@@ -926,6 +920,7 @@ struct TSortNode {
   32: optional list<Exprs.TExpr> pre_agg_exprs;
   33: optional list<Types.TSlotId> pre_agg_output_slot_id;
   34: optional bool pre_agg_insert_local_shuffle;
+  40: optional TLateMaterializeMode parallel_merge_late_materialize_mode;
 }
 
 enum TAnalyticWindowType {
@@ -1084,6 +1079,7 @@ struct TExchangeNode {
   // Sender's partition type
   4: optional Partitions.TPartitionType partition_type;
   5: optional bool enable_parallel_merge
+  6: optional TLateMaterializeMode parallel_merge_late_materialize_mode;
 }
 
 // This contains all of the information computed by the plan as part of the resource
diff --git a/test/sql/test_sort/R/test_parallel_merge_lazy_materialize b/test/sql/test_sort/R/test_parallel_merge_lazy_materialize
new file mode 100644
index 00000000000000..3cddfd1963f696
--- /dev/null
+++ b/test/sql/test_sort/R/test_parallel_merge_lazy_materialize
@@ -0,0 +1,151 @@
+-- name: test_parallel_merge_lazy_materialize
+CREATE TABLE `t0` (
+  `c0` int(11) NULL COMMENT "",
+  `c1` varchar(20) NULL COMMENT "",
+  `c2` varchar(200) NULL COMMENT "",
+  `c3` int(11) NULL COMMENT "",
+  `c4` int(11) NULL COMMENT "",
+  `c5` int(11) NULL COMMENT "",
+  `c6` int(11) NULL COMMENT "",
+  `c7` int(11) NULL COMMENT ""
+) ENGINE=OLAP
+DUPLICATE KEY(`c0`, `c1`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`c0`, `c1`) BUCKETS 48
+PROPERTIES (
+"replication_num" = "1",
+"in_memory" = "false",
+"storage_format" = "DEFAULT",
+"enable_persistent_index" = "false",
+"replicated_storage" = "true",
+"compression" = "LZ4"
+);
+-- result:
+-- !result
+insert into t0 SELECT generate_series, generate_series, generate_series, generate_series, generate_series, generate_series, generate_series, generate_series FROM TABLE(generate_series(1,  40960));
+-- result:
+-- !result
+select count(*) from t0;
+-- result:
+40960
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(ORDER BY c5) AS wv FROM t0) a;
+-- result:
+838881280	20480.5	1	40960
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+-- result:
+40960	1.0	1	1
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv), SUM(c1), SUM(c2), SUM(c3), SUM(c4), SUM(c5), SUM(c6) FROM (SELECT c1,c2,c3,c4,c5,c6, COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+-- result:
+40960	1.0	1	1	838881280.0	838881280.0	838881280	838881280	838881280	838881280
+-- !result
+select * from t0 order by 1,2,3,4,5,6 limit 10;
+-- result:
+1	1	1	1	1	1	1	1
+2	2	2	2	2	2	2	2
+3	3	3	3	3	3	3	3
+4	4	4	4	4	4	4	4
+5	5	5	5	5	5	5	5
+6	6	6	6	6	6	6	6
+7	7	7	7	7	7	7	7
+8	8	8	8	8	8	8	8
+9	9	9	9	9	9	9	9
+10	10	10	10	10	10	10	10
+-- !result
+select * from t0 order by 1,2,3,4,5,6 desc limit 10;
+-- result:
+1	1	1	1	1	1	1	1
+2	2	2	2	2	2	2	2
+3	3	3	3	3	3	3	3
+4	4	4	4	4	4	4	4
+5	5	5	5	5	5	5	5
+6	6	6	6	6	6	6	6
+7	7	7	7	7	7	7	7
+8	8	8	8	8	8	8	8
+9	9	9	9	9	9	9	9
+10	10	10	10	10	10	10	10
+-- !result
+set parallel_merge_late_materialization_mode="always";
+-- result:
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(ORDER BY c5) AS wv FROM t0) a;
+-- result:
+838881280	20480.5	1	40960
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+-- result:
+40960	1.0	1	1
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv), SUM(c1), SUM(c2), SUM(c3), SUM(c4), SUM(c5), SUM(c6) FROM (SELECT c1,c2,c3,c4,c5,c6, COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+-- result:
+40960	1.0	1	1	838881280.0	838881280.0	838881280	838881280	838881280	838881280
+-- !result
+select * from t0 order by 1,2,3,4,5,6 limit 10;
+-- result:
+1	1	1	1	1	1	1	1
+2	2	2	2	2	2	2	2
+3	3	3	3	3	3	3	3
+4	4	4	4	4	4	4	4
+5	5	5	5	5	5	5	5
+6	6	6	6	6	6	6	6
+7	7	7	7	7	7	7	7
+8	8	8	8	8	8	8	8
+9	9	9	9	9	9	9	9
+10	10	10	10	10	10	10	10
+-- !result
+select * from t0 order by 1,2,3,4,5,6 desc limit 10;
+-- result:
+1	1	1	1	1	1	1	1
+2	2	2	2	2	2	2	2
+3	3	3	3	3	3	3	3
+4	4	4	4	4	4	4	4
+5	5	5	5	5	5	5	5
+6	6	6	6	6	6	6	6
+7	7	7	7	7	7	7	7
+8	8	8	8	8	8	8	8
+9	9	9	9	9	9	9	9
+10	10	10	10	10	10	10	10
+-- !result
+set parallel_merge_late_materialization_mode="never";
+-- result:
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(ORDER BY c5) AS wv FROM t0) a;
+-- result:
+838881280	20480.5	1	40960
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+-- result:
+40960	1.0	1	1
+-- !result
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv), SUM(c1), SUM(c2), SUM(c3), SUM(c4), SUM(c5), SUM(c6) FROM (SELECT c1,c2,c3,c4,c5,c6, COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+-- result:
+40960	1.0	1	1	838881280.0	838881280.0	838881280	838881280	838881280	838881280
+-- !result
+select * from t0 order by 1,2,3,4,5,6 limit 10;
+-- result:
+1	1	1	1	1	1	1	1
+2	2	2	2	2	2	2	2
+3	3	3	3	3	3	3	3
+4	4	4	4	4	4	4	4
+5	5	5	5	5	5	5	5
+6	6	6	6	6	6	6	6
+7	7	7	7	7	7	7	7
+8	8	8	8	8	8	8	8
+9	9	9	9	9	9	9	9
+10	10	10	10	10	10	10	10
+-- !result
+select * from t0 order by 1,2,3,4,5,6 desc limit 10;
+-- result:
+1	1	1	1	1	1	1	1
+2	2	2	2	2	2	2	2
+3	3	3	3	3	3	3	3
+4	4	4	4	4	4	4	4
+5	5	5	5	5	5	5	5
+6	6	6	6	6	6	6	6
+7	7	7	7	7	7	7	7
+8	8	8	8	8	8	8	8
+9	9	9	9	9	9	9	9
+10	10	10	10	10	10	10	10
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_sort/T/test_parallel_merge_lazy_materialize b/test/sql/test_sort/T/test_parallel_merge_lazy_materialize
new file mode 100644
index 00000000000000..79c50c22a66c21
--- /dev/null
+++ b/test/sql/test_sort/T/test_parallel_merge_lazy_materialize
@@ -0,0 +1,47 @@
+-- name: test_parallel_merge_lazy_materialize
+
+CREATE TABLE `t0` (
+  `c0` int(11) NULL COMMENT "",
+  `c1` varchar(20) NULL COMMENT "",
+  `c2` varchar(200) NULL COMMENT "",
+  `c3` int(11) NULL COMMENT "",
+  `c4` int(11) NULL COMMENT "",
+  `c5` int(11) NULL COMMENT "",
+  `c6` int(11) NULL COMMENT "",
+  `c7` int(11) NULL COMMENT ""
+) ENGINE=OLAP
+DUPLICATE KEY(`c0`, `c1`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`c0`, `c1`) BUCKETS 48
+PROPERTIES (
+"replication_num" = "1",
+"in_memory" = "false",
+"storage_format" = "DEFAULT",
+"enable_persistent_index" = "false",
+"replicated_storage" = "true",
+"compression" = "LZ4"
+);
+
+insert into t0 SELECT generate_series, generate_series, generate_series, generate_series, generate_series, generate_series, generate_series, generate_series FROM TABLE(generate_series(1,  40960));
+
+select count(*) from t0;
+-- auto
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(ORDER BY c5) AS wv FROM t0) a;
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv), SUM(c1), SUM(c2), SUM(c3), SUM(c4), SUM(c5), SUM(c6) FROM (SELECT c1,c2,c3,c4,c5,c6, COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+select * from t0 order by 1,2,3,4,5,6 limit 10;
+select * from t0 order by 1,2,3,4,5,6 desc limit 10;
+
+set parallel_merge_late_materialization_mode="always";
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(ORDER BY c5) AS wv FROM t0) a;
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv), SUM(c1), SUM(c2), SUM(c3), SUM(c4), SUM(c5), SUM(c6) FROM (SELECT c1,c2,c3,c4,c5,c6, COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+select * from t0 order by 1,2,3,4,5,6 limit 10;
+select * from t0 order by 1,2,3,4,5,6 desc limit 10;
+
+set parallel_merge_late_materialization_mode="never";
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(ORDER BY c5) AS wv FROM t0) a;
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv) FROM (SELECT COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+SELECT SUM(wv), AVG(wv), MIN(wv), MAX(wv), SUM(c1), SUM(c2), SUM(c3), SUM(c4), SUM(c5), SUM(c6) FROM (SELECT c1,c2,c3,c4,c5,c6, COUNT(c2) OVER(PARTITION BY c1 ORDER BY c5) AS wv FROM t0) a;
+select * from t0 order by 1,2,3,4,5,6 limit 10;
+select * from t0 order by 1,2,3,4,5,6 desc limit 10;

From 72e2b99ff76f9643f4f7a6c55bc31e67a2dcb83a Mon Sep 17 00:00:00 2001
From: trueeyu <lxhhust350@qq.com>
Date: Thu, 16 Jan 2025 20:18:20 +0800
Subject: [PATCH 20/71] [BugFix] Fix the bug of SpillMemTableSink checking keys
 type (#55153)

Signed-off-by: trueeyu <lxhhust350@qq.com>
---
 be/src/storage/lake/spill_mem_table_sink.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/storage/lake/spill_mem_table_sink.cpp b/be/src/storage/lake/spill_mem_table_sink.cpp
index 69a5a4b8d447af..fbb859036bcffa 100644
--- a/be/src/storage/lake/spill_mem_table_sink.cpp
+++ b/be/src/storage/lake/spill_mem_table_sink.cpp
@@ -218,7 +218,7 @@ Status SpillMemTableSink::merge_blocks_to_segments() {
         total_merges++;
         // PK shouldn't do agg because pk support order key different from primary key,
         // in that case, data is sorted by order key and cannot be aggregated by primary key
-        bool do_agg = _schema->keys_type() == TKeysType::AGG_KEYS || _schema->keys_type() == TKeysType::UNIQUE_KEYS;
+        bool do_agg = _schema->keys_type() == KeysType::AGG_KEYS || _schema->keys_type() == KeysType::UNIQUE_KEYS;
         auto tmp_itr = new_heap_merge_iterator(merge_inputs);
         auto merge_itr = do_agg ? new_aggregate_iterator(tmp_itr) : tmp_itr;
         RETURN_IF_ERROR(merge_itr->init_encoded_schema(EMPTY_GLOBAL_DICTMAPS));

From c2c766692c5d6028950ce42fcf15537fdc63a23e Mon Sep 17 00:00:00 2001
From: Drake Wang <wxl24life@gmail.com>
Date: Thu, 16 Jan 2025 21:14:22 +0800
Subject: [PATCH 21/71] [BugFix] Fix warehouse property might not take effect
 when executing load stmts (#49684)

Signed-off-by: drake_wang <wxl250059@alibaba-inc.com>
---
 .../TransactionWithChannelHandler.java        |   8 +-
 .../com/starrocks/load/loadv2/LoadJob.java    |  31 ++-
 .../starrocks/http/StarRocksHttpTestCase.java |   2 +
 .../http/TransactionLoadActionTest.java       |  74 ++++++-
 .../starrocks/load/loadv2/LoadJobTest.java    | 203 ++++++++++++++++++
 .../load/loadv2/SparkLoadJobTest.java         |  16 ++
 .../streamload/StreamLoadManagerTest.java     |  14 +-
 7 files changed, 328 insertions(+), 20 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithChannelHandler.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithChannelHandler.java
index 081edaf363858b..b742bcda4572f1 100644
--- a/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithChannelHandler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/transaction/TransactionWithChannelHandler.java
@@ -22,6 +22,7 @@
 import com.starrocks.http.rest.transaction.TransactionOperationParams.Channel;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.thrift.TNetworkAddress;
+import com.starrocks.warehouse.Warehouse;
 import org.apache.commons.lang3.Validate;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -57,9 +58,12 @@ public ResultWrapper handle(BaseRequest request, BaseResponse response) throws S
                     throw new DdlException(String.format(
                             "Channel ID should be between [0, %d].", (channel.getNum() - 1)));
                 }
-
+                String warehouseName = txnOperationParams.getWarehouseName();
+                Warehouse warehouse =
+                        GlobalStateMgr.getCurrentState().getWarehouseMgr().getWarehouse(warehouseName);
                 GlobalStateMgr.getCurrentState().getStreamLoadMgr().beginLoadTaskFromFrontend(
-                        dbName, tableName, label, "", "", timeoutMillis, channel.getNum(), channel.getId(), result);
+                        dbName, tableName, label, "", "", timeoutMillis, channel.getNum(), channel.getId(), result,
+                        warehouse.getId());
                 return new ResultWrapper(result);
             case TXN_PREPARE:
                 GlobalStateMgr.getCurrentState().getStreamLoadMgr().prepareLoadTask(
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java
index a9ee610897d7a3..9d7a8b3983f95f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadJob.java
@@ -71,6 +71,7 @@
 import com.starrocks.qe.QeProcessorImpl;
 import com.starrocks.qe.scheduler.Coordinator;
 import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.server.RunMode;
 import com.starrocks.server.WarehouseManager;
 import com.starrocks.sql.ast.AlterLoadStmt;
 import com.starrocks.sql.ast.LoadStmt;
@@ -464,13 +465,15 @@ public void setJobProperties(Map<String, String> properties) throws DdlException
                 logRejectedRecordNum = Long.parseLong(properties.get(LoadStmt.LOG_REJECTED_RECORD_NUM));
             }
 
-            if (properties.containsKey(PropertyAnalyzer.PROPERTIES_WAREHOUSE)) {
-                String warehouseName = properties.get(PropertyAnalyzer.PROPERTIES_WAREHOUSE);
-                Warehouse warehouse = GlobalStateMgr.getCurrentState().getWarehouseMgr().getWarehouse(warehouseName);
-                if (warehouse == null) {
-                    throw new DdlException("Warehouse " + warehouseName + " not exists.");
+            if (RunMode.isSharedDataMode()) {
+                if (properties.containsKey(PropertyAnalyzer.PROPERTIES_WAREHOUSE)) {
+                    String warehouseName = properties.get(PropertyAnalyzer.PROPERTIES_WAREHOUSE);
+                    Warehouse warehouse =
+                            GlobalStateMgr.getCurrentState().getWarehouseMgr().getWarehouse(warehouseName);
+                    warehouseId = warehouse.getId();
+                } else {
+                    warehouseId = ConnectContext.get().getCurrentWarehouseId();
                 }
-                warehouseId = warehouse.getId();
             }
 
             if (properties.containsKey(LoadStmt.STRIP_OUTER_ARRAY)) {
@@ -484,6 +487,11 @@ public void setJobProperties(Map<String, String> properties) throws DdlException
             if (properties.containsKey(LoadStmt.JSONROOT)) {
                 jsonOptions.jsonRoot = properties.get(LoadStmt.JSONROOT);
             }
+        } else {
+            if (RunMode.isSharedDataMode()) {
+                // if no properties set, we should still set warehouse here
+                warehouseId = ConnectContext.get().getCurrentWarehouseId();
+            }
         }
     }
 
@@ -895,8 +903,8 @@ public List<Comparable> getShowInfo() throws DdlException {
             }
             jobInfo.add(loadingStatus.getLoadStatistic().toShowInfoStr());
             // warehouse
-            Warehouse warehouse = GlobalStateMgr.getCurrentState().getWarehouseMgr().getWarehouse(warehouseId);
-            if (warehouse != null) {
+            if (RunMode.isSharedDataMode()) {
+                Warehouse warehouse = GlobalStateMgr.getCurrentState().getWarehouseMgr().getWarehouse(warehouseId);
                 jobInfo.add(warehouse.getName());
             } else {
                 jobInfo.add("");
@@ -1049,6 +1057,13 @@ public TLoadInfo toThrift() {
             info.setNum_scan_rows(loadingStatus.getLoadStatistic().totalSourceLoadRows());
             info.setNum_sink_rows(loadingStatus.getLoadStatistic().totalSinkLoadRows());
             info.setNum_scan_bytes(loadingStatus.getLoadStatistic().sourceScanBytes());
+            // warehouse
+            if (RunMode.getCurrentRunMode() == RunMode.SHARED_DATA) {
+                Warehouse warehouse = GlobalStateMgr.getCurrentState().getWarehouseMgr().getWarehouse(warehouseId);
+                info.setWarehouse(warehouse.getName());
+            } else {
+                info.setWarehouse("");
+            }
             return info;
         } finally {
             readUnlock();
diff --git a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java
index 2e4e77ddcae6e2..7904da87b73e4c 100644
--- a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java
+++ b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java
@@ -543,6 +543,8 @@ MaterializedViewHandler getRollupHandler() {
             }
         };
 
+        // init default warehouse
+        globalStateMgr.getWarehouseMgr().initDefaultWarehouse();
         assignBackends();
         doSetUp();
     }
diff --git a/fe/fe-core/src/test/java/com/starrocks/http/TransactionLoadActionTest.java b/fe/fe-core/src/test/java/com/starrocks/http/TransactionLoadActionTest.java
index dcb94e5ad61b5e..6602665e5017d2 100644
--- a/fe/fe-core/src/test/java/com/starrocks/http/TransactionLoadActionTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/http/TransactionLoadActionTest.java
@@ -88,6 +88,7 @@ public class TransactionLoadActionTest extends StarRocksHttpTestCase {
     private static final String CHANNEL_NUM_STR = "channel_num";
     private static final String CHANNEL_ID_STR = "channel_id";
     private static final String SOURCE_TYPE = "source_type";
+    private static final String WAREHOUSE_KEY = "warehouse";
 
     private static HttpServer beServer;
     private static int TEST_HTTP_PORT = 0;
@@ -246,7 +247,7 @@ public void beginTransactionWithChannelInfoTest() throws Exception {
                 {
                     streamLoadMgr.beginLoadTaskFromFrontend(
                             anyString, anyString, anyString, anyString, anyString,
-                            anyLong, anyInt, anyInt, (TransactionResult) any);
+                            anyLong, anyInt, anyInt, (TransactionResult) any, anyLong);
                     times = 1;
                     result = new Delegate<Void>() {
 
@@ -258,7 +259,8 @@ public void beginLoadTaskFromFrontend(String dbName,
                                                   long timeoutMillis,
                                                   int channelNum,
                                                   int channelId,
-                                                  TransactionResult resp) {
+                                                  TransactionResult resp,
+                                                  long warehouseId) {
                             resp.addResultEntry(TransactionResult.LABEL_KEY, label);
                         }
 
@@ -286,7 +288,7 @@ public void beginLoadTaskFromFrontend(String dbName,
                 {
                     streamLoadMgr.beginLoadTaskFromFrontend(
                             anyString, anyString, anyString, anyString, anyString,
-                            anyLong, anyInt, anyInt, (TransactionResult) any);
+                            anyLong, anyInt, anyInt, (TransactionResult) any, anyLong);
                     times = 1;
                     result = new StarRocksException("begin load task error");
                 }
@@ -323,6 +325,72 @@ public void beginTransactionWithoutChannelInfoTest() throws Exception {
         }
     }
 
+    @Test
+    public void beginTransactionWithWarehouseTest() throws Exception {
+        {
+            new Expectations() {
+                {
+                    streamLoadMgr.beginLoadTaskFromFrontend(
+                            anyString, anyString, anyString, anyString, anyString,
+                            anyLong, anyInt, anyInt, (TransactionResult) any, anyLong);
+                    times = 1;
+                    result = new Delegate<Void>() {
+
+                        public void beginLoadTaskFromFrontend(String dbName,
+                                                              String tableName,
+                                                              String label,
+                                                              String user,
+                                                              String clientIp,
+                                                              long timeoutMillis,
+                                                              int channelNum,
+                                                              int channelId,
+                                                              TransactionResult resp,
+                                                              long warehouseId) {
+                            resp.addResultEntry(TransactionResult.LABEL_KEY, label);
+                        }
+
+                    };
+                }
+            };
+
+            String label = RandomStringUtils.randomAlphanumeric(32);
+            Request request = newRequest(TransactionOperation.TXN_BEGIN, (uriBuilder, reqBuilder) -> {
+                reqBuilder.addHeader(DB_KEY, DB_NAME);
+                reqBuilder.addHeader(TABLE_KEY, TABLE_NAME);
+                reqBuilder.addHeader(LABEL_KEY, label);
+                reqBuilder.addHeader(CHANNEL_ID_STR, "0");
+                reqBuilder.addHeader(CHANNEL_NUM_STR, "2");
+                // no warehouse set here
+            });
+            try (Response response = networkClient.newCall(request).execute()) {
+                Map<String, Object> body = parseResponseBody(response);
+                assertEquals(OK, body.get(TransactionResult.STATUS_KEY));
+                assertEquals(label, Objects.toString(body.get(TransactionResult.LABEL_KEY)));
+            }
+        }
+    }
+
+    @Test
+    public void beginTransactionWithNonExistentWarehouseTest() throws Exception {
+        {
+            String label = RandomStringUtils.randomAlphanumeric(32);
+            Request request = newRequest(TransactionOperation.TXN_BEGIN, (uriBuilder, reqBuilder) -> {
+                reqBuilder.addHeader(DB_KEY, DB_NAME);
+                reqBuilder.addHeader(TABLE_KEY, TABLE_NAME);
+                reqBuilder.addHeader(LABEL_KEY, label);
+                reqBuilder.addHeader(CHANNEL_ID_STR, "0");
+                reqBuilder.addHeader(CHANNEL_NUM_STR, "2");
+                reqBuilder.addHeader(WAREHOUSE_KEY, "non_exist_warehouse");
+            });
+            try (Response response = networkClient.newCall(request).execute()) {
+                Map<String, Object> body = parseResponseBody(response);
+                assertEquals(FAILED, body.get(TransactionResult.STATUS_KEY));
+                assertTrue(Objects.toString(body.get(TransactionResult.MESSAGE_KEY))
+                        .contains("Warehouse name: non_exist_warehouse not exist"));
+            }
+        }
+    }
+
     @Test
     public void beginTransactionForBypassWriteTest() throws Exception {
         {
diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java
index 64f9baf3d15489..f6ce088c33dbdd 100644
--- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/LoadJobTest.java
@@ -44,17 +44,24 @@
 import com.starrocks.common.LoadException;
 import com.starrocks.common.MetaNotFoundException;
 import com.starrocks.common.jmockit.Deencapsulation;
+import com.starrocks.common.util.PropertyAnalyzer;
+import com.starrocks.common.util.TimeUtils;
 import com.starrocks.metric.LongCounterMetric;
 import com.starrocks.metric.MetricRepo;
 import com.starrocks.persist.EditLog;
+import com.starrocks.qe.ConnectContext;
 import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.server.RunMode;
+import com.starrocks.server.WarehouseManager;
 import com.starrocks.sql.ast.LoadStmt;
 import com.starrocks.task.LeaderTask;
 import com.starrocks.task.LeaderTaskExecutor;
+import com.starrocks.thrift.TLoadInfo;
 import com.starrocks.thrift.TUniqueId;
 import com.starrocks.transaction.GlobalTransactionMgr;
 import com.starrocks.transaction.RunningTxnExceedException;
 import com.starrocks.transaction.TransactionState;
+import com.starrocks.warehouse.Warehouse;
 import mockit.Expectations;
 import mockit.Injectable;
 import mockit.Mock;
@@ -64,11 +71,23 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.time.ZoneId;
+import java.util.List;
 import java.util.Map;
+import java.util.TimeZone;
 import java.util.concurrent.ArrayBlockingQueue;
 
 public class LoadJobTest {
 
+    @Mocked
+    private GlobalStateMgr globalStateMgr;
+
+    @Mocked
+    private WarehouseManager warehouseManager;
+
+    @Mocked
+    private Warehouse warehouse;
+
     @BeforeClass
     public static void start() {
         MetricRepo.init();
@@ -107,6 +126,26 @@ public void testSetJobPropertiesWithErrorTimeout() {
 
     @Test
     public void testSetJobProperties() {
+        new Expectations() {
+            {
+                GlobalStateMgr.getCurrentState();
+                minTimes = 0;
+                result = globalStateMgr;
+
+                globalStateMgr.getWarehouseMgr();
+                minTimes = 0;
+                result = warehouseManager;
+
+                warehouseManager.getWarehouse(anyLong);
+                minTimes = 0;
+                result = warehouse;
+
+                warehouse.getId();
+                minTimes = 0;
+                result = 1001L;
+            }
+        };
+
         Map<String, String> jobProperties = Maps.newHashMap();
         jobProperties.put(LoadStmt.TIMEOUT_PROPERTY, "1000");
         jobProperties.put(LoadStmt.MAX_FILTER_RATIO_PROPERTY, "0.1");
@@ -125,6 +164,69 @@ public void testSetJobProperties() {
         }
     }
 
+    @Test
+    public void testSetJobPropertiesForWarehouse() {
+        new MockUp<RunMode>() {
+            @Mock
+            public RunMode getCurrentRunMode() {
+                return RunMode.SHARED_DATA;
+            }
+        };
+
+        new Expectations() {
+            {
+                GlobalStateMgr.getCurrentState();
+                minTimes = 0;
+                result = globalStateMgr;
+
+                globalStateMgr.getWarehouseMgr();
+                minTimes = 0;
+                result = warehouseManager;
+
+                warehouseManager.getWarehouse(anyLong);
+                minTimes = 0;
+                result = warehouse;
+
+                warehouse.getId();
+                minTimes = 0;
+                result = 1001L;
+            }
+        };
+
+        ConnectContext context = new ConnectContext(null);
+        new Expectations(context) {
+            {
+                ConnectContext.get();
+                result = context;
+
+                context.getCurrentWarehouseId();
+                result = 1000L;
+            }
+        };
+
+        try {
+            // normal, jobProperties set
+            LoadJob loadJob1 = new BrokerLoadJob();
+            Map<String, String> jobProperties1 = Maps.newHashMap();
+            jobProperties1.put(PropertyAnalyzer.PROPERTIES_WAREHOUSE, "test_warehouse");
+            loadJob1.setJobProperties(jobProperties1);
+            Assert.assertEquals(1001L, (long) Deencapsulation.getField(loadJob1, "warehouseId"));
+
+            // with jobProperties set, but no warehouse property,
+            LoadJob loadJob2 = new BrokerLoadJob();
+            Map<String, String> jobProperties2 = Maps.newHashMap();
+            loadJob2.setJobProperties(jobProperties2);
+            Assert.assertEquals(1000L, (long) Deencapsulation.getField(loadJob2, "warehouseId"));
+
+            // no jobProperties provided
+            LoadJob loadJob3 = new BrokerLoadJob();
+            loadJob3.setJobProperties(null);
+            Assert.assertEquals(1000L, (long) Deencapsulation.getField(loadJob3, "warehouseId"));
+        } catch (DdlException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+
     @Test
     public void testExecute(@Mocked GlobalTransactionMgr globalTransactionMgr,
                             @Mocked LeaderTaskExecutor leaderTaskExecutor)
@@ -233,4 +335,105 @@ public void testUpdateStateToFinished(@Mocked MetricRepo metricRepo,
         Assert.assertEquals(100, (int) Deencapsulation.getField(loadJob, "progress"));
         Assert.assertEquals(0, loadJob.idToTasks.size());
     }
+
+    @Test
+    public void testGetShowInfo() throws DdlException {
+        TimeZone tz = TimeZone.getTimeZone(ZoneId.of("Asia/Shanghai"));
+        new MockUp<TimeUtils>() {
+            @Mock
+            public TimeZone getTimeZone() {
+                return tz;
+            }
+        };
+
+        new MockUp<RunMode>() {
+            @Mock
+            public RunMode getCurrentRunMode() {
+                return RunMode.SHARED_DATA;
+            }
+        };
+
+        new Expectations() {
+            {
+                GlobalStateMgr.getCurrentState();
+                minTimes = 0;
+                result = globalStateMgr;
+
+                globalStateMgr.getWarehouseMgr();
+                minTimes = 0;
+                result = warehouseManager;
+
+                warehouseManager.getWarehouse(anyLong);
+                minTimes = 0;
+                result = warehouse;
+
+                warehouse.getName();
+                minTimes = 0;
+                result = "test_wh";
+            }
+        };
+
+        LoadJob loadJob = new BrokerLoadJob();
+        List<Comparable> showInfo = loadJob.getShowInfo();
+        Assert.assertNotNull(showInfo);
+        Comparable result = showInfo.get(showInfo.size() - 1);
+        Assert.assertEquals("test_wh", result);
+
+        new MockUp<RunMode>() {
+            @Mock
+            public RunMode getCurrentRunMode() {
+                return RunMode.SHARED_NOTHING;
+            }
+        };
+
+        showInfo = loadJob.getShowInfo();
+        Assert.assertNotNull(showInfo);
+        result = showInfo.get(showInfo.size() - 1);
+        Assert.assertEquals("", result);
+    }
+
+    @Test
+    public void testToThrift() {
+        LoadJob loadJob = new BrokerLoadJob();
+
+        new MockUp<RunMode>() {
+            @Mock
+            public RunMode getCurrentRunMode() {
+                return RunMode.SHARED_DATA;
+            }
+        };
+
+        new Expectations() {
+            {
+                GlobalStateMgr.getCurrentState();
+                minTimes = 0;
+                result = globalStateMgr;
+
+                globalStateMgr.getWarehouseMgr();
+                minTimes = 0;
+                result = warehouseManager;
+
+                warehouseManager.getWarehouse(anyLong);
+                minTimes = 0;
+                result = warehouse;
+
+                warehouse.getName();
+                minTimes = 0;
+                result = "test_wh";
+            }
+        };
+
+        TLoadInfo loadInfo = loadJob.toThrift();
+        Assert.assertEquals("test_wh", loadInfo.getWarehouse());
+
+        new MockUp<RunMode>() {
+            @Mock
+            public RunMode getCurrentRunMode() {
+                return RunMode.SHARED_NOTHING;
+            }
+        };
+
+        loadInfo = loadJob.toThrift();
+        Assert.assertEquals("", loadInfo.getWarehouse());
+    }
 }
diff --git a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java
index eb5aac0fd5e7f4..bf85ad1c2ddefb 100644
--- a/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/load/loadv2/SparkLoadJobTest.java
@@ -66,11 +66,13 @@
 import com.starrocks.load.loadv2.LoadJob.LoadJobStateUpdateInfo;
 import com.starrocks.load.loadv2.SparkLoadJob.SparkLoadJobStateUpdateInfo;
 import com.starrocks.load.loadv2.etl.EtlJobConfig;
+import com.starrocks.qe.ConnectContext;
 import com.starrocks.qe.OriginStatement;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.sql.ast.DataDescription;
 import com.starrocks.sql.ast.LoadStmt;
 import com.starrocks.sql.ast.ResourceDesc;
+import com.starrocks.sql.ast.UserIdentity;
 import com.starrocks.task.AgentBatchTask;
 import com.starrocks.task.AgentTaskExecutor;
 import com.starrocks.task.LeaderTaskExecutor;
@@ -119,6 +121,8 @@ public class SparkLoadJobTest {
     private long backendId;
     private int schemaHash;
 
+    private ConnectContext ctx;
+
     @Before
     public void setUp() {
         dbId = 1L;
@@ -140,6 +144,11 @@ public void setUp() {
         backendId = 15L;
         physicalPartitionId = 16L;
         schemaHash = 146886;
+
+        // for warehouse property
+        ctx = new ConnectContext(null);
+        ctx.setCurrentUserIdentity(new UserIdentity("testUser", "%"));
+        ctx.setQualifiedUser("testCluster:testUser");
     }
 
     @Test
@@ -191,6 +200,13 @@ public void testCreateFromLoadStmt(@Mocked GlobalStateMgr globalStateMgr, @Injec
             }
         };
 
+        new Expectations(ctx) {
+            {
+                ConnectContext.get();
+                result = ctx;
+            }
+        };
+
         try {
             Assert.assertTrue(resource.getSparkConfigs().isEmpty());
             resourceDesc.analyze();
diff --git a/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java b/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java
index b0f35cebd838ea..b0b6d87b3a93d0 100644
--- a/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/load/streamload/StreamLoadManagerTest.java
@@ -62,7 +62,7 @@ public class StreamLoadManagerTest {
     public void setUp() {
         globalTransactionMgr = new GlobalTransactionMgr(globalStateMgr);
         FeConstants.runningUnitTest = true;
-    
+
         try {
             db = CatalogMocker.mockDb();
         } catch (AnalysisException e) {
@@ -146,14 +146,14 @@ public long getDataQuota() {
     @Test
     public void testBeginStreamLoadTask() throws StarRocksException {
         StreamLoadMgr streamLoadManager = new StreamLoadMgr();
-        
+
         String dbName = "test_db";
         String tableName = "test_tbl";
         String labelName = "label1";
         long timeoutMillis = 100000;
         int channelNum = 1;
         int channelId = 0;
-        
+
         TransactionResult resp = new TransactionResult();
         streamLoadManager.beginLoadTaskFromFrontend(
                 dbName, tableName, labelName, "", "", timeoutMillis, channelNum, channelId, resp);
@@ -170,20 +170,20 @@ public void testBeginStreamLoadTask() throws StarRocksException {
         Map<String, StreamLoadTask> dbToLabelToStreamLoadTask =
                 Deencapsulation.getField(streamLoadManager, "dbToLabelToStreamLoadTask");
         Assert.assertEquals(1, idToStreamLoadTask.size());
-        
+
     }
 
     @Test
     public void testChannelIdEqualChannelNum() throws StarRocksException {
         StreamLoadMgr streamLoadManager = new StreamLoadMgr();
-        
+
         String dbName = "test_db";
         String tableName = "test_tbl";
         String labelName = "label1";
         long timeoutMillis = 100000;
         int channelNum = 1;
         int channelId = 1;
-        
+
         TransactionResult resp = new TransactionResult();
         streamLoadManager.beginLoadTaskFromFrontend(
                 dbName, tableName, labelName, "", "", timeoutMillis, channelNum, channelId, resp);
@@ -204,7 +204,7 @@ public void testGetTaskByName() throws StarRocksException {
         long timeoutMillis = 100000;
         int channelNum = 5;
         int channelId = 0;
-        
+
         TransactionResult resp = new TransactionResult();
         streamLoadManager.beginLoadTaskFromFrontend(
                 dbName, tableName, labelName, "", "", timeoutMillis, channelNum, channelId, resp);

From 3f85637a8dd6c3fa3f4d8026c3a03a321951fbb8 Mon Sep 17 00:00:00 2001
From: trueeyu <lxhhust350@qq.com>
Date: Fri, 17 Jan 2025 02:37:23 +0800
Subject: [PATCH 22/71] [Refactor] Remove the compatile code for RF_VERSION
 (#55148)

Signed-off-by: trueeyu <lxhhust350@qq.com>
---
 be/src/exprs/runtime_filter.cpp      |  25 -----
 be/src/exprs/runtime_filter.h        | 132 ++-------------------------
 be/src/exprs/runtime_filter_bank.cpp |  16 +---
 3 files changed, 14 insertions(+), 159 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 6045fdffbdd257..e6f5f7c9040b9a 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -18,31 +18,6 @@
 #include "util/compression/stream_compression.h"
 
 namespace starrocks {
-// TODO: remove it
-LogicalType RuntimeFilterSerializeType::from_serialize_type(RuntimeFilterSerializeType::PrimitiveType ptype) {
-    switch (ptype) {
-#define CONVERT_PTYPE(type_name)                       \
-    case RuntimeFilterSerializeType::TYPE_##type_name: \
-        return LogicalType::TYPE_##type_name;
-        APPLY_FOR_SCALAR_THRIFT_TYPE(CONVERT_PTYPE);
-#undef CONVERT_PTYPE
-    default:
-        return TYPE_UNKNOWN;
-    }
-}
-
-RuntimeFilterSerializeType::PrimitiveType RuntimeFilterSerializeType::to_serialize_type(LogicalType type) {
-    switch (type) {
-#define CONVERT_TYPE(type_name)         \
-    case LogicalType::TYPE_##type_name: \
-        return RuntimeFilterSerializeType::TYPE_##type_name;
-        APPLY_FOR_SCALAR_THRIFT_TYPE(CONVERT_TYPE);
-#undef CONVERT_TYPE
-    default:
-        return RuntimeFilterSerializeType::TYPE_NULL;
-    }
-}
-
 void SimdBlockFilter::init(size_t nums) {
     nums = std::max(MINIMUM_ELEMENT_NUM, nums);
     int log_heap_space = std::ceil(std::log2(nums));
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index b9a74d83ca93ed..50210778078f7d 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -35,61 +35,11 @@ namespace starrocks {
 // 0x1. initial global runtime filter impl
 // 0x2. change simd-block-filter hash function.
 // 0x3. Fix serialize problem
-inline const constexpr uint8_t RF_VERSION = 0x2;
+inline const constexpr uint8_t RF_VERSION = 0x2; // deprecated
 inline const constexpr uint8_t RF_VERSION_V2 = 0x3;
 static_assert(sizeof(RF_VERSION_V2) == sizeof(RF_VERSION));
 inline const constexpr int32_t RF_VERSION_SZ = sizeof(RF_VERSION_V2);
 
-// compatible code from 2.5 to 3.0
-// TODO: remove it
-class RuntimeFilterSerializeType {
-public:
-    enum PrimitiveType {
-        INVALID_TYPE = 0,
-        TYPE_NULL,     /* 1 */
-        TYPE_BOOLEAN,  /* 2 */
-        TYPE_TINYINT,  /* 3 */
-        TYPE_SMALLINT, /* 4 */
-        TYPE_INT,      /* 5 */
-        TYPE_BIGINT,   /* 6 */
-        TYPE_LARGEINT, /* 7 */
-        TYPE_FLOAT,    /* 8 */
-        TYPE_DOUBLE,   /* 9 */
-        TYPE_VARCHAR,  /* 10 */
-        TYPE_DATE,     /* 11 */
-        TYPE_DATETIME, /* 12 */
-        TYPE_BINARY,
-        /* 13 */      // Not implemented
-        TYPE_DECIMAL, /* 14 */
-        TYPE_CHAR,    /* 15 */
-
-        TYPE_STRUCT,    /* 16 */
-        TYPE_ARRAY,     /* 17 */
-        TYPE_MAP,       /* 18 */
-        TYPE_HLL,       /* 19 */
-        TYPE_DECIMALV2, /* 20 */
-
-        TYPE_TIME,       /* 21 */
-        TYPE_OBJECT,     /* 22 */
-        TYPE_PERCENTILE, /* 23 */
-        TYPE_DECIMAL32,  /* 24 */
-        TYPE_DECIMAL64,  /* 25 */
-        TYPE_DECIMAL128, /* 26 */
-
-        TYPE_JSON,      /* 27 */
-        TYPE_FUNCTION,  /* 28 */
-        TYPE_VARBINARY, /* 28 */
-    };
-
-    static_assert(sizeof(PrimitiveType) == sizeof(int32_t));
-    static_assert(sizeof(PrimitiveType) == sizeof(LogicalType));
-    static_assert(sizeof(TPrimitiveType::type) == sizeof(LogicalType));
-
-    static PrimitiveType to_serialize_type(LogicalType type);
-
-    static LogicalType from_serialize_type(PrimitiveType type);
-};
-
 static constexpr uint32_t SALT[8] = {0x47b6137b, 0x44974d91, 0x8824ad5b, 0xa2b7289d,
                                      0x705495c7, 0x2df1424b, 0x9efc4947, 0x5c6bfb31};
 
@@ -238,60 +188,6 @@ class SimdBlockFilter {
     Bucket* _directory = nullptr;
 };
 
-// If size is very small(< 1000), SmallHashSet is faster than SimdBlockFilter
-// This fast bloom filter is inspired by parallel-hashmap row_hash_set
-class SmallHashSet {
-public:
-    using ctrl_t = int8_t;
-
-    ~SmallHashSet() { free(_ctrl); }
-
-    size_t grow_to_lower_bound_capacity(size_t growth) {
-        return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7);
-    }
-
-    size_t normalize_capacity(size_t n) { return n ? ~size_t{} >> __builtin_clzll(n) : 1; }
-
-    static constexpr ctrl_t KEMPTY = -128;
-    void init(size_t size) {
-        _capacity = normalize_capacity(grow_to_lower_bound_capacity(size * 2));
-        posix_memalign(reinterpret_cast<void**>(&_ctrl), 16, _capacity + 17);
-        memset(_ctrl, -128, _capacity + 17);
-    }
-
-    void insert_hash(size_t hash) {
-        size_t h1_hash = hash >> 7;
-        size_t offset_ = h1_hash & _capacity;
-        ctrl_t h2_hash = hash & 0x7F;
-        while ((_ctrl[offset_] != KEMPTY) & (_ctrl[offset_] != h2_hash)) {
-            offset_++;
-        }
-        _ctrl[offset_] = h2_hash;
-    }
-
-    bool test_hash(size_t hash) {
-        size_t h1_hash = hash >> 7;
-        size_t offset_ = h1_hash & _capacity;
-        char h2_hash = hash & 0x7F;
-#ifdef __SSE2__
-        __m128i ctrl = _mm_loadu_si128(reinterpret_cast<__m128i*>(_ctrl + offset_));
-        auto match = _mm_set1_epi8(h2_hash);
-        return _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl));
-#else
-        for (size_t i = 0; i < 16; ++i) {
-            if (_ctrl[offset_] == h2_hash) {
-                return true;
-            }
-        }
-        return false;
-#endif
-    }
-
-private:
-    ctrl_t* _ctrl = nullptr;
-    size_t _capacity = 0;
-};
-
 // The runtime filter generated by join right small table
 class JoinRuntimeFilter;
 using JoinRuntimeFilterPtr = std::shared_ptr<const JoinRuntimeFilter>;
@@ -670,15 +566,10 @@ class RuntimeBloomFilter final : public JoinRuntimeFilter {
 
     size_t serialize(int serialize_version, uint8_t* data) const override {
         size_t offset = 0;
-        if (serialize_version == RF_VERSION) {
-            auto ltype = RuntimeFilterSerializeType::to_serialize_type(Type);
-            memcpy(data + offset, &ltype, sizeof(ltype));
-            offset += sizeof(ltype);
-        } else {
-            auto ltype = to_thrift(Type);
-            memcpy(data + offset, &ltype, sizeof(ltype));
-            offset += sizeof(ltype);
-        }
+        DCHECK(serialize_version != RF_VERSION);
+        auto ltype = to_thrift(Type);
+        memcpy(data + offset, &ltype, sizeof(ltype));
+        offset += sizeof(ltype);
 
         offset += JoinRuntimeFilter::serialize(serialize_version, data + offset);
         memcpy(data + offset, &_has_min_max, sizeof(_has_min_max));
@@ -710,15 +601,10 @@ class RuntimeBloomFilter final : public JoinRuntimeFilter {
 
     size_t deserialize(int serialize_version, const uint8_t* data) override {
         size_t offset = 0;
-        if (serialize_version == RF_VERSION) {
-            RuntimeFilterSerializeType::PrimitiveType ltype = RuntimeFilterSerializeType::to_serialize_type(Type);
-            memcpy(&ltype, data + offset, sizeof(ltype));
-            offset += sizeof(ltype);
-        } else {
-            auto ltype = to_thrift(Type);
-            memcpy(&ltype, data + offset, sizeof(ltype));
-            offset += sizeof(ltype);
-        }
+        DCHECK(serialize_version != RF_VERSION);
+        auto ltype = to_thrift(Type);
+        memcpy(&ltype, data + offset, sizeof(ltype));
+        offset += sizeof(ltype);
 
         offset += JoinRuntimeFilter::deserialize(serialize_version, data + offset);
 
diff --git a/be/src/exprs/runtime_filter_bank.cpp b/be/src/exprs/runtime_filter_bank.cpp
index 842ea9df480836..2c5c21301bcdf9 100644
--- a/be/src/exprs/runtime_filter_bank.cpp
+++ b/be/src/exprs/runtime_filter_bank.cpp
@@ -87,23 +87,17 @@ int RuntimeFilterHelper::deserialize_runtime_filter(ObjectPool* pool, JoinRuntim
     uint8_t version = 0;
     memcpy(&version, data, sizeof(version));
     offset += sizeof(version);
-    if (version != RF_VERSION && version != RF_VERSION_V2) {
+    if (version != RF_VERSION_V2) {
         // version mismatch and skip this chunk.
         LOG(WARNING) << "unrecognized version:" << version;
         return 0;
     }
 
     // peek logical type.
-    LogicalType ltype = TYPE_UNKNOWN;
-    if (version == RF_VERSION) {
-        RuntimeFilterSerializeType::PrimitiveType type;
-        memcpy(&type, data + offset, sizeof(type));
-        ltype = RuntimeFilterSerializeType::from_serialize_type(type);
-    } else {
-        TPrimitiveType::type type;
-        memcpy(&type, data + offset, sizeof(type));
-        ltype = thrift_to_type(type);
-    }
+    TPrimitiveType::type type;
+    memcpy(&type, data + offset, sizeof(type));
+    LogicalType ltype = thrift_to_type(type);
+
     JoinRuntimeFilter* filter = create_join_runtime_filter(pool, ltype);
     DCHECK(filter != nullptr);
     if (filter != nullptr) {

From 55b704f911cdbc076620a88b032338778b38c3c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=B5=B5=E7=A9=BA=E4=BA=8B=E3=82=B9=E3=83=94=E3=83=AA?=
 =?UTF-8?q?=E3=83=83=E3=83=88?= <wanglichen@starrocks.com>
Date: Fri, 17 Jan 2025 09:43:35 +0800
Subject: [PATCH 23/71] [Doc] Update Match Column By Name (#55127)

---
 docs/en/loading/InsertInto.md                 | 22 ++++++++-----------
 .../loading_unloading/INSERT.md               | 15 ++++---------
 docs/zh/loading/InsertInto.md                 | 22 ++++++++-----------
 .../loading_unloading/INSERT.md               | 13 +++--------
 4 files changed, 25 insertions(+), 47 deletions(-)

diff --git a/docs/en/loading/InsertInto.md b/docs/en/loading/InsertInto.md
index d033fba500bb92..9ef4aa37ced350 100644
--- a/docs/en/loading/InsertInto.md
+++ b/docs/en/loading/InsertInto.md
@@ -502,7 +502,7 @@ INSERT INTO insert_wiki_edit (
 SELECT event_time, channel, user FROM source_wiki_edit;
 ```
 
-The column mapping will change if you changed the order of `channel` and `user` in either the column clause or the SELECT statement.
+The column mapping will change if you changed the order of `channel` and `user` in either the column list or the SELECT statement.
 
 ```SQL
 INSERT INTO insert_wiki_edit (
@@ -515,23 +515,19 @@ SELECT event_time, user, channel FROM source_wiki_edit;
 
 Here, the ingested data are probably not what you want, because `channel` in the target table `insert_wiki_edit` will be filled with data from `user` in the source table `source_wiki_edit`.
 
-By setting the property `match_column_by` to `name` in the INSERT statement, the system will detect the column names in the source and the target tables, and match the columns with the same name.
+By adding `BY NAME` clause in the INSERT statement, the system will detect the column names in the source and the target tables, and match the columns with the same name.
 
-`match_column_by`: The mode how the system matches the columns in the source and target tables. Valid values:
-- `position` (Default): The system matches the columns by the position of the columns in the column clause and the SELECT statement.
-- `name`: The system matches the columns with the same name.
+:::note
+
+- You cannot specify the column list if `BY NAME` is specified.
+- If `BY NAME` is not specified, the system matches the columns by the position of the columns in the column list and the SELECT statement.
+
+:::
 
 The following example matches each column in the source and target tables by their names:
 
 ```SQL
-INSERT INTO insert_wiki_edit (
-    event_time,
-    channel,
-    user
-)
-PROPERTIES(
-    "match_column_by" = "name"
-)
+INSERT INTO insert_wiki_edit BY NAME
 SELECT event_time, user, channel FROM source_wiki_edit;
 ```
 
diff --git a/docs/en/sql-reference/sql-statements/loading_unloading/INSERT.md b/docs/en/sql-reference/sql-statements/loading_unloading/INSERT.md
index a328d40a4a8a30..10b3d836e13454 100644
--- a/docs/en/sql-reference/sql-statements/loading_unloading/INSERT.md
+++ b/docs/en/sql-reference/sql-statements/loading_unloading/INSERT.md
@@ -19,7 +19,7 @@ You can submit an asynchronous INSERT task using [SUBMIT TASK](ETL/SUBMIT_TASK.m
   [ PARTITION (<partition_name> [, ...] ) ]
   [ TEMPORARY PARTITION (<temporary_partition_name> [, ...] ) ]
   [ WITH LABEL <label>]
-  [ (<column_name>[, ...]) ]
+  [ (<column_name>[, ...]) | BY NAME ]
   [ PROPERTIES ("key"="value", ...) ]
   { VALUES ( { <expression> | DEFAULT } [, ...] ) | <query> }
   ```
@@ -42,7 +42,8 @@ You can submit an asynchronous INSERT task using [SUBMIT TASK](ETL/SUBMIT_TASK.m
 | PARTITION    |  The partitions into which you want to load the data. You can specify multiple partitions, which must be separated by commas (,). It must be set to partitions that exist in the destination table. If you specify this parameter, the data will be inserted only into the specified partitions. If you do not specify this parameter, the data will be inserted into all partitions. |
 | TEMPORARY PARTITION |The name of the [temporary partition](../../../table_design/data_distribution/Temporary_partition.md) into which you want to load data. You can specify multiple temporary partitions, which must be separated by commas (,).|
 | label         | The unique identification label for each data load transaction within the database. If it is not specified, the system automatically generates one for the transaction. We recommend you specify the label for the transaction. Otherwise, you cannot check the transaction status if a connection error occurs and no result is returned. You can check the transaction status via `SHOW LOAD WHERE label="label"` statement. For the naming conventions of labels, see [System Limits](../../System_limit.md). |
-| column_name   | The name of the destination column(s) to load data in. It must be set as columns that exist in the destination table. <ul><li>If the property `match_column_by` is not set or set to `position` (default), the destination columns you specify are mapped one on one in sequence to the columns of the source table, regardless of what the destination column names are. </li><li>If `match_column_by` is set to `name`, the destination columns are mapped to columns with the same names in the source table, regardless of the column orders in destination and source tables.</li><li>If no destination column is specified, the default value is all columns in the destination table.</li><li>If the specified column in the source table does not exist in the destination column, the default value will be written to this column. </li><li>If the specified column does not have a default value, the transaction will fail.</li><li>If the column type of the source table is inconsistent with that of the destination table, the system will perform an implicit conversion on the mismatched column.</li><li>If the conversion fails, a syntax parsing error will be returned.</li></ul>**NOTE**<br />From v3.3.1, specifying a column list in the INSERT INTO statement on a Primary Key table will perform Partial Updates (instead of Full Upsert in earlier versions). If the column list is not specified, the system will perform Full Upsert. |
+| column_name   | The name of the destination column(s) to load data in. It must be set as columns that exist in the destination table. **You cannot specify both `column_name` and `BY NAME`.**<ul><li>If `BY NAME` is not specified, the destination columns you specified are mapped one on one in sequence to the source columns, regardless of what the destination column names are. </li><li>If `BY NAME` is specified, the destination columns are mapped to the source columns with the same names, regardless of the column orders in destination and source tables.</li><li>If no destination column is specified, the default value is all columns in the destination table.</li><li>If the specified column in the source table does not exist in the destination column, the default value will be written to this column. </li><li>If the specified column does not have a default value, the transaction will fail.</li><li>If the column type of the source table is inconsistent with that of the destination table, the system will perform an implicit conversion on the mismatched column.</li><li>If the conversion fails, a syntax parsing error will be returned.</li></ul>**NOTE**<br />From v3.3.1, specifying a column list in the INSERT INTO statement on a Primary Key table will perform Partial Updates (instead of Full Upsert in earlier versions). If the column list is not specified, the system will perform Full Upsert. |
+| BY NAME       | To match the source and destination columns by their names. **You cannot specify both `column_name` and `BY NAME`.** If it is not specified, the destination columns are mapped one on one in sequence to the source columns, regardless of what the destination column names are.  |
 | PROPERTIES    | Properties of the INSERT job. Each property must be a key-value pair. For supported properties, see [PROPERTIES](#properties). |
 | expression    | Expression that assigns values to the column.                |
 | DEFAULT       | Assigns default value to the column.                         |
@@ -58,7 +59,6 @@ INSERT statements support configuring PROPERTIES from v3.4.0 onwards.
 | timeout          | The timeout duration of the INSERT job. Unit: Seconds. You can also set the timeout duration for INSERT within the session or globally using the variable `insert_timeout`. |
 | strict_mode      | Whether to enable strict mode while loading data using INSERT from FILES(). Valid values: `true` (Default) and `false`. When strict mode is enabled, the system loads only qualified rows. It filters out unqualified rows and returns details about the unqualified rows. For more information, see [Strict mode](../../../loading/load_concept/strict_mode.md). You can also enable strict mode for INSERT from FILES() within the session or globally using the variable `enable_insert_strict`. |
 | max_filter_ratio | The maximum error tolerance of INSERT from FILES(). It's the maximum ratio of data records that can be filtered out due to inadequate data quality. When the ratio of unqualified data records reaches this threshold, the job fails. Default value: `0`. Range: [0, 1]. You can also set the maximum error tolerance for INSERT from FILES() within the session or globally using the variable `insert_max_filter_ratio`. |
-| match_column_by  | The mode how the system matches the columns in the source and target tables. Valid values:<ul><li>`position` (Default): The system matches the columns by the position of the columns in the column clause and the SELECT statement.</li><li>`name`: The system matches the columns with the same name.</li></ul> |
 
 :::note
 
@@ -223,14 +223,7 @@ SELECT * FROM FILES(
 The following example matches each column in the source and target tables by their names:
 
 ```SQL
-INSERT INTO insert_wiki_edit (
-    event_time,
-    channel,
-    user
-)
-PROPERTIES(
-    "match_column_by" = "name"
-)
+INSERT INTO insert_wiki_edit BY NAME
 SELECT event_time, user, channel FROM source_wiki_edit;
 ```
 
diff --git a/docs/zh/loading/InsertInto.md b/docs/zh/loading/InsertInto.md
index bf6fd85714f86a..f8d3ba3c126881 100644
--- a/docs/zh/loading/InsertInto.md
+++ b/docs/zh/loading/InsertInto.md
@@ -484,7 +484,7 @@ INSERT INTO insert_wiki_edit (
 SELECT event_time, channel, user FROM source_wiki_edit;
 ```
 
-如果您在 Column 子句或 SELECT 语句中改变了 `channel` 和 `user` 的顺序，列的映射关系将发生变化。
+如果您在 Column List 或 SELECT 语句中改变了 `channel` 和 `user` 的顺序，列的映射关系将发生变化。
 
 ```SQL
 INSERT INTO insert_wiki_edit (
@@ -497,23 +497,19 @@ SELECT event_time, user, channel FROM source_wiki_edit;
 
 此处，由于目标表 `insert_wiki_edit` 中的 `channel` 列被源表 `source_wiki_edit` 中的 `user` 的数据所填满，导入的数据可能并不是所需的结果。
 
-通过在 INSERT 语句中将属性 match_column_by 设置为 name，系统将根据源表和目标表中的列名进行匹配，匹配同名的列。
+通过在 INSERT 语句中添加 `BY NAME` 子句，系统将根据检查源表和目标表中的列名，匹配同名的列。
 
-`match_column_by`：系统匹配源表和目标表中的列的方式。有效值：
-- `position`（默认值）：系统根据 Column 子句和 SELECT 语句中列的位置来匹配列。
-- `name`：系统根据列名匹配相同名称的列。
+:::note
+
+- 如果指定了 `BY NAME`，则不能指定 Column List。
+- 如果未指定 `BY NAME`，系统将根据 Column List 和 SELECT 语句中列的位置来匹配列。
+
+:::
 
 以下示例通过列名匹配源表和目标表中的列：
 
 ```SQL
-INSERT INTO insert_wiki_edit (
-    event_time,
-    channel,
-    user
-)
-PROPERTIES(
-    "match_column_by" = "name"
-)
+INSERT INTO insert_wiki_edit BY NAME
 SELECT event_time, user, channel FROM source_wiki_edit;
 ```
 
diff --git a/docs/zh/sql-reference/sql-statements/loading_unloading/INSERT.md b/docs/zh/sql-reference/sql-statements/loading_unloading/INSERT.md
index 127d771328c652..1567e688335721 100644
--- a/docs/zh/sql-reference/sql-statements/loading_unloading/INSERT.md
+++ b/docs/zh/sql-reference/sql-statements/loading_unloading/INSERT.md
@@ -42,7 +42,8 @@ displayed_sidebar: docs
 | PARTITION  | 导入的目标分区。此参数必须是目标表中存在的分区，多个分区名称用逗号（,）分隔。如果指定该参数，数据只会被导入相应分区内。如果未指定，则默认将数据导入至目标表的所有分区。 |
 | TEMPORARY PARTITION | 指定要把数据导入哪些[临时分区](../../../table_design/data_distribution/Temporary_partition.md)。|
 | label       | 导入作业的标识，数据库内唯一。如果未指定，StarRocks 会自动为作业生成一个 Label。建议您指定 Label。否则，如果当前导入作业因网络错误无法返回结果，您将无法得知该导入操作是否成功。如果指定了 Label，可以通过 SQL 命令 `SHOW LOAD WHERE label="label";` 查看任务结果。关于 Label 命名要求，参见[系统限制](../../System_limit.md)。 |
-| column_name | 导入的目标列，必须是目标表中存在的列。<ul><li>如果未设置属性 `match_column_by` 或设置为 `position` (默认值)，则目标列的对应关系与列名无关，但与其顺序一一对应。</li><li>如果将 `match_column_by` 设置为 `name`，则目标列的对应关系与顺序无关，系统将根据同名列匹配。</li><li>如果不指定目标列，默认为目标表中的所有列。</li><li>如果源表中的某个列在目标列不存在，则写入默认值。</li><li>如果当前列没有默认值，导入作业会失败。</li><li>如果查询语句的结果列类型与目标列的类型不一致，会进行隐式转化，如果不能进行转化，那么 INSERT INTO 语句会报语法解析错误。</li></ul>**说明**<br />自 v3.3.1 起，INSERT INTO 导入主键表时指定 Column List 会执行部分列更新（而在先前版本中，指定 Column List 仍然导致 Full Upsert）。如不指定 Column List，系统执行 Full Upsert。 |
+| column_name | 导入的目标列，必须是目标表中存在的列。**不可同时指定 `column_name` 和 `BY NAME`。**<ul><li>如果未指定 `BY NAME`，则目标列的对应关系与列名无关，但与其顺序一一对应。</li><li>如果指定了 `BY NAME`，则目标列的对应关系与顺序无关，系统将根据同名列匹配。</li><li>如果不指定目标列，默认为目标表中的所有列。</li><li>如果源表中的某个列在目标列不存在，则写入默认值。</li><li>如果当前列没有默认值，导入作业会失败。</li><li>如果查询语句的结果列类型与目标列的类型不一致，会进行隐式转化，如果不能进行转化，那么 INSERT INTO 语句会报语法解析错误。</li></ul>**说明**<br />自 v3.3.1 起，INSERT INTO 导入主键表时指定 Column List 会执行部分列更新（而在先前版本中，指定 Column List 仍然导致 Full Upsert）。如不指定 Column List，系统执行 Full Upsert。 |
+| BY NAME     | 系统根据列名匹配相同名称的列。**不可同时指定 `column_name` 和 `BY NAME`。**如果未指定 `BY NAME`，则目标列的对应关系与列名无关，但与其顺序一一对应。|
 | PROPERTIES  | INSERT 作业的属性Properties of the INSERT job. 每个属性必须为一对键值。关于支持的属性，参考 [PROPERTIES](#properties)。 |
 | expression  | 表达式，用以为对应列赋值。                                   |
 | DEFAULT     | 为对应列赋予默认值。                                         |
@@ -58,7 +59,6 @@ INSERT statements support configuring PROPERTIES from v3.4.0 onwards.
 | timeout          | INSERT 作业的超时时间。单位：秒。您也可以通过变量 `insert_timeout` 在当前 Session 中或全局设置 INSERT 的超时时间。 |
 | strict_mode      | 是否在使用 INSERT from FILES() 导入数据时启用严格模式。有效值：`true` 和 `false`（默认值）。启用严格模式时，系统仅导入合格的数据行，过滤掉不合格的行，并返回不合格行的详细信息。更多信息请参见 [严格模式](../../../loading/load_concept/strict_mode.md)。您也可以通过变量 `enable_insert_strict` 在当前 Session 中或全局启用 INSERT 的严格模式。 |
 | max_filter_ratio | INSERT from FILES() 导入作业的最大容忍率，即导入作业能够容忍的因数据质量不合格而过滤掉的数据行所占的最大比例。当不合格行数比例超过该限制时，导入作业失败。默认值：`0`。范围：[0, 1]。您也可以通过变量 `insert_max_filter_ratio` 在当前 Session 中或全局设置 INSERT 的最大错误容忍度。 |
-| match_column_by  | 系统匹配源表和目标表中的列的方式。有效值：<ul><li>`position`（默认值）：系统根据 Column 子句和 SELECT 语句中列的位置来匹配列。</li><li>`name`：系统根据列名匹配相同名称的列。</li></ul> |
 
 :::note
 
@@ -209,14 +209,7 @@ SELECT * FROM FILES(
 以下示例通过列名匹配源表和目标表中的列：
 
 ```SQL
-INSERT INTO insert_wiki_edit (
-    event_time,
-    channel,
-    user
-)
-PROPERTIES(
-    "match_column_by" = "name"
-)
+INSERT INTO insert_wiki_edit BY NAME
 SELECT event_time, user, channel FROM source_wiki_edit;
 ```
 

From 737c9ec919bf546d6fc65b721c375bfc779b1f4e Mon Sep 17 00:00:00 2001
From: "shuming.li" <ming.moriarty@gmail.com>
Date: Fri, 17 Jan 2025 10:16:42 +0800
Subject: [PATCH 24/71] [BugFix] Fix foriegn key constraints for primary/unique
 tables (#55125)

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
---
 .../java/com/starrocks/catalog/OlapTable.java |  2 +-
 .../common/util/PropertyAnalyzer.java         |  9 +-
 .../planner/MaterializedViewManyJoinTest.java |  2 +-
 .../starrocks/planner/TablePruningTest.java   | 89 +++++++++++++++++++
 4 files changed, 92 insertions(+), 10 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
index 07a89b0b65f3fa..ba084e0886014d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
@@ -3069,7 +3069,7 @@ public List<UniqueConstraint> getUniqueConstraints() {
         }
         if (keysType == KeysType.UNIQUE_KEYS || keysType == KeysType.PRIMARY_KEYS) {
             uniqueConstraints.add(
-                    new UniqueConstraint(null, null, null, getKeyColumns()
+                    new UniqueConstraint(null, null, getName(), getKeyColumns()
                             .stream().map(Column::getColumnId).collect(Collectors.toList())));
         }
         if (tableProperty != null && tableProperty.getUniqueConstraints() != null) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java
index f1a902dd7c5b16..d9b996a2eb02f4 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java
@@ -1235,7 +1235,7 @@ private static void analyzeForeignKeyUniqueConstraint(Table parentTable, List<St
         if (parentTableKeyType == KeysType.AGG_KEYS) {
             throw new SemanticException(
                     String.format("do not support reference agg table:%s", parentTable.getName()));
-        } else if (parentTableKeyType == KeysType.DUP_KEYS) {
+        } else {
             // for DUP_KEYS type olap table or external table
             if (!parentTable.hasUniqueConstraints() && mvUniqueConstraints.isEmpty()) {
                 throw new SemanticException(
@@ -1260,13 +1260,6 @@ private static void analyzeForeignKeyUniqueConstraint(Table parentTable, List<St
                                     parentTable.getName()));
                 }
             }
-        } else {
-            // for PRIMARY_KEYS and UNIQUE_KEYS type table
-            // parent columns should be keys
-            if (!((OlapTable) parentTable).isKeySet(Sets.newHashSet(parentColumns))) {
-                throw new SemanticException(String.format("columns:%s are not key columns of table:%s",
-                        parentColumns, parentTable.getName()));
-            }
         }
     }
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManyJoinTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManyJoinTest.java
index 28a284b716daa1..65866de918e060 100644
--- a/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManyJoinTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/planner/MaterializedViewManyJoinTest.java
@@ -138,7 +138,7 @@ public static void beforeClass() throws Exception {
     public void testManyJoins(String name, String query, boolean expectHitMv) throws Exception {
         Stopwatch watch = Stopwatch.createStarted();
         // Make sure it's not empty
-        String plan = getFragmentPlan(query, "MV");
+        String plan = getFragmentPlan(query);
         PlanTestBase.assertContains(plan, "OlapScanNode");
         if (expectHitMv) {
             PlanTestBase.assertContains(plan, "MaterializedView: true");
diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/TablePruningTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/TablePruningTest.java
index e57221f475bcb0..a87e69b01831ac 100644
--- a/fe/fe-core/src/test/java/com/starrocks/planner/TablePruningTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/planner/TablePruningTest.java
@@ -16,6 +16,7 @@
 
 import com.google.common.collect.Lists;
 import com.starrocks.common.FeConstants;
+import com.starrocks.sql.plan.PlanTestBase;
 import com.starrocks.statistic.StatsConstants;
 import com.starrocks.utframe.StarRocksAssert;
 import com.starrocks.utframe.UtFrameUtils;
@@ -797,6 +798,94 @@ public void testExplainLogicalCloneOperator() throws Exception {
         ctx.getSessionVariable().setEnableCboTablePrune(true);
         String plan = UtFrameUtils.explainLogicalPlan(ctx, sql);
         Assert.assertTrue(plan, plan.contains("CLONE"));
+        starRocksAssert.dropTable("AA");
+        starRocksAssert.dropTable("BB");
+    }
+
+    @Test
+    public void testPruneWithPKUKTable1() throws Exception {
+        ctx.getSessionVariable().setEnableCboTablePrune(true);
+
+        final String tabAA = "CREATE TABLE `AA` (\n" +
+                "    `id` int(11) NOT NULL,\n" +
+                "    `id2` int(11) NOT NULL,\n" +
+                "    `name` varchar(25) NOT NULL\n" +
+                "    ) ENGINE=OLAP\n" +
+                "PRIMARY KEY(`id`)\n" +
+                "DISTRIBUTED BY HASH(`id`) BUCKETS 10 PROPERTIES (\"replication_num\" = \"1\");\n";
+        final String tabBB = "CREATE TABLE `BB` (\n" +
+                "      `id` int(11) NOT NULL,\n" +
+                "      `id2` int(11) NOT NULL,\n" +
+                "      `name` varchar(25) NOT NULL,\n" +
+                "      `age` varchar(25)\n" +
+                "      ) ENGINE=OLAP\n" +
+                "UNIQUE KEY(`id`)\n" +
+                "DISTRIBUTED BY HASH(`id`) BUCKETS 10  PROPERTIES (\"replication_num\" = \"1\");";
+        starRocksAssert.withTable(tabAA);
+        starRocksAssert.withTable(tabBB);
+        starRocksAssert.alterTableProperties(
+                "alter table AA set(\"foreign_key_constraints\" = \"AA(id2) REFERENCES BB(id)\");");
+        final String sql = "select AA.id2, BB.id from AA inner join BB on AA.id2 = BB.id";
+        final String plan = UtFrameUtils.explainLogicalPlan(ctx, sql);
+
+        PlanTestBase.assertNotContains(plan, "BB");
+        PlanTestBase.assertContains(plan, "CLONE");
+        starRocksAssert.dropTable("AA");
+        starRocksAssert.dropTable("BB");
+    }
+
+    @Test
+    public void testPruneWithPKUKTable2() throws Exception {
+        ctx.getSessionVariable().setEnableCboTablePrune(true);
+
+        final String tabAA = "CREATE TABLE `AA` (\n" +
+                "    `id` int(11) NOT NULL,\n" +
+                "    `id2` int(11) NOT NULL,\n" +
+                "    `id3` int(11) NOT NULL,\n" +
+                "    `name` varchar(25) NOT NULL\n" +
+                "    ) ENGINE=OLAP\n" +
+                "PRIMARY KEY(`id`)\n" +
+                "DISTRIBUTED BY HASH(`id`) BUCKETS 10 PROPERTIES (" +
+                "   \"replication_num\" = \"1\"," +
+                "   \"unique_constraints\" = \"id3\"" +
+                ");\n";
+        final String tabBB = "CREATE TABLE `BB` (\n" +
+                "      `id` int(11) NOT NULL,\n" +
+                "      `id2` int(11) NOT NULL,\n" +
+                "      `name` varchar(25) NOT NULL,\n" +
+                "      `age` varchar(25)\n" +
+                "      ) ENGINE=OLAP\n" +
+                "PRIMARY KEY(`id`)\n" +
+                "DISTRIBUTED BY HASH(`id`) BUCKETS 10  PROPERTIES " +
+                "(" +
+                "   \"replication_num\" = \"1\"," +
+                "   \"unique_constraints\" = \"id2\"" +
+                ");";
+        starRocksAssert.withTable(tabAA);
+        starRocksAssert.withTable(tabBB);
+        // add non primary key foreign key constraints should be ok
+        starRocksAssert.alterTableProperties(
+                "alter table AA set(\"unique_constraints\" = \"name\");");
+        // add non primary key foreign key constraints should be ok
+        starRocksAssert.alterTableProperties(
+                "alter table AA set(\"foreign_key_constraints\" = \"AA(id2) REFERENCES BB(id2)\");");
+        // test table prune with non-primary keys
+        {
+            final String sql = "select AA.id2, BB.id2 from AA inner join BB on AA.id2 = BB.id2";
+            String plan = UtFrameUtils.explainLogicalPlan(ctx, sql);
+            PlanTestBase.assertNotContains(plan, "BB");
+            PlanTestBase.assertContains(plan, "CLONE");
+        }
+
+        // test not table prune with non-fks
+        {
+            String sql = "select AA.id, BB.id from AA inner join BB on AA.id = BB.id";
+            String plan = UtFrameUtils.explainLogicalPlan(ctx, sql);
+            PlanTestBase.assertContains(plan, "BB");
+            PlanTestBase.assertNotContains(plan, "CLONE");
+        }
+        starRocksAssert.dropTable("AA");
+        starRocksAssert.dropTable("BB");
     }
 
     @Test

From 5aff9f22c2c1c1e6d3add76050ca77c0d9719010 Mon Sep 17 00:00:00 2001
From: RyanZ <dirtysalt1987@gmail.com>
Date: Thu, 16 Jan 2025 18:19:02 -0800
Subject: [PATCH 25/71] [BugFix] fix fragment stance state transition in
 incremental scan ranges case (#55174)

Signed-off-by: yanz <dirtysalt1987@gmail.com>
---
 .../starrocks/qe/scheduler/dag/FragmentInstanceExecState.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/scheduler/dag/FragmentInstanceExecState.java b/fe/fe-core/src/main/java/com/starrocks/qe/scheduler/dag/FragmentInstanceExecState.java
index 3137c41ac70b78..762ab5408f9dad 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/scheduler/dag/FragmentInstanceExecState.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/scheduler/dag/FragmentInstanceExecState.java
@@ -175,7 +175,7 @@ public void serializeRequest() {
      * The state transitions to DEPLOYING.
      */
     public void deployAsync() {
-        transitionState(State.DEPLOYING);
+        transitionState(State.CREATED, State.DEPLOYING);
 
         TNetworkAddress brpcAddress = worker.getBrpcAddress();
         try {

From e3f5de00916df7bdc66612200b817208c4f98f47 Mon Sep 17 00:00:00 2001
From: trueeyu <lxhhust350@qq.com>
Date: Fri, 17 Jan 2025 10:26:26 +0800
Subject: [PATCH 26/71] [Refactor] Rename OlapRuntimeScanRangePruner to
 RuntimeScanRangePruner (#55154)

Signed-off-by: trueeyu <lxhhust350@qq.com>
---
 be/src/connector/lake_connector.cpp                |  4 ++--
 be/src/exec/hdfs_scanner.cpp                       |  4 ++--
 be/src/exec/hdfs_scanner.h                         |  2 +-
 be/src/exec/olap_scan_prepare.cpp                  |  4 ++--
 be/src/exec/olap_scan_prepare.h                    |  2 +-
 be/src/exec/pipeline/scan/olap_chunk_source.cpp    |  4 ++--
 be/src/formats/parquet/file_reader.cpp             |  2 +-
 be/src/formats/parquet/file_reader.h               |  4 ++--
 be/src/storage/rowset/rowset_options.h             |  4 ++--
 be/src/storage/rowset/segment_iterator.cpp         |  4 ++--
 be/src/storage/rowset/segment_options.h            |  4 ++--
 ...ntime_range_pruner.h => runtime_range_pruner.h} |  6 +++---
 ...e_range_pruner.hpp => runtime_range_pruner.hpp} | 14 +++++++-------
 be/src/storage/tablet_reader_params.h              |  4 ++--
 be/test/formats/parquet/file_reader_test.cpp       |  2 +-
 be/test/storage/olap_runtime_range_pruner_test.cpp |  7 +++----
 16 files changed, 35 insertions(+), 36 deletions(-)
 rename be/src/storage/{olap_runtime_range_pruner.h => runtime_range_pruner.h} (94%)
 rename be/src/storage/{olap_runtime_range_pruner.hpp => runtime_range_pruner.hpp} (94%)

diff --git a/be/src/connector/lake_connector.cpp b/be/src/connector/lake_connector.cpp
index 7b35f1572afb7d..c81b26a0695fb8 100644
--- a/be/src/connector/lake_connector.cpp
+++ b/be/src/connector/lake_connector.cpp
@@ -23,11 +23,11 @@
 #include "runtime/global_dict/parser.h"
 #include "storage/column_predicate_rewriter.h"
 #include "storage/lake/tablet.h"
-#include "storage/olap_runtime_range_pruner.hpp"
 #include "storage/predicate_parser.h"
 #include "storage/predicate_tree/predicate_tree.hpp"
 #include "storage/projection_iterator.h"
 #include "storage/rowset/short_key_range_option.h"
+#include "storage/runtime_range_pruner.hpp"
 #include "util/starrocks_metrics.h"
 
 namespace starrocks::connector {
@@ -260,7 +260,7 @@ Status LakeDataSource::init_reader_params(const std::vector<OlapScanRange*>& key
             !config::disable_storage_page_cache && _scan_range.fill_data_cache && !_scan_range.skip_page_cache;
     _params.lake_io_opts.fill_data_cache = _scan_range.fill_data_cache;
     _params.lake_io_opts.skip_disk_cache = _scan_range.skip_disk_cache;
-    _params.runtime_range_pruner = OlapRuntimeScanRangePruner(parser, _conjuncts_manager->unarrived_runtime_filters());
+    _params.runtime_range_pruner = RuntimeScanRangePruner(parser, _conjuncts_manager->unarrived_runtime_filters());
     _params.splitted_scan_rows = _provider->get_splitted_scan_rows();
     _params.scan_dop = _provider->get_scan_dop();
 
diff --git a/be/src/exec/hdfs_scanner.cpp b/be/src/exec/hdfs_scanner.cpp
index 00e22b72321060..31e630baae1b44 100644
--- a/be/src/exec/hdfs_scanner.cpp
+++ b/be/src/exec/hdfs_scanner.cpp
@@ -23,8 +23,8 @@
 #include "io/compressed_input_stream.h"
 #include "io/shared_buffered_input_stream.h"
 #include "pipeline/fragment_context.h"
-#include "storage/olap_runtime_range_pruner.hpp"
 #include "storage/predicate_parser.h"
+#include "storage/runtime_range_pruner.hpp"
 #include "util/compression/compression_utils.h"
 #include "util/compression/stream_compression.h"
 
@@ -189,7 +189,7 @@ Status HdfsScanner::_build_scanner_context() {
         ASSIGN_OR_RETURN(ctx.predicate_tree,
                          ctx.conjuncts_manager->get_predicate_tree(predicate_parser, ctx.predicate_free_pool));
         ctx.rf_scan_range_pruner = opts.obj_pool->add(
-                new OlapRuntimeScanRangePruner(predicate_parser, ctx.conjuncts_manager->unarrived_runtime_filters()));
+                new RuntimeScanRangePruner(predicate_parser, ctx.conjuncts_manager->unarrived_runtime_filters()));
     }
     return Status::OK();
 }
diff --git a/be/src/exec/hdfs_scanner.h b/be/src/exec/hdfs_scanner.h
index 7f1e786111c504..dc0bcca90ee2da 100644
--- a/be/src/exec/hdfs_scanner.h
+++ b/be/src/exec/hdfs_scanner.h
@@ -324,7 +324,7 @@ struct HdfsScannerContext {
 
     int64_t connector_max_split_size = 0;
 
-    OlapRuntimeScanRangePruner* rf_scan_range_pruner = nullptr;
+    RuntimeScanRangePruner* rf_scan_range_pruner = nullptr;
 
     // update none_existed_slot
     // update conjunct
diff --git a/be/src/exec/olap_scan_prepare.cpp b/be/src/exec/olap_scan_prepare.cpp
index 3a39e77d93fb23..1b78017e033e10 100644
--- a/be/src/exec/olap_scan_prepare.cpp
+++ b/be/src/exec/olap_scan_prepare.cpp
@@ -26,10 +26,10 @@
 #include "gutil/map_util.h"
 #include "runtime/descriptors.h"
 #include "storage/column_predicate.h"
-#include "storage/olap_runtime_range_pruner.h"
-#include "storage/olap_runtime_range_pruner.hpp"
 #include "storage/predicate_parser.h"
 #include "storage/predicate_tree/predicate_tree.hpp"
+#include "storage/runtime_range_pruner.h"
+#include "storage/runtime_range_pruner.hpp"
 #include "types/date_value.hpp"
 #include "types/logical_type.h"
 #include "types/logical_type_infra.h"
diff --git a/be/src/exec/olap_scan_prepare.h b/be/src/exec/olap_scan_prepare.h
index a241ff77ea227b..82dc2aced53b00 100644
--- a/be/src/exec/olap_scan_prepare.h
+++ b/be/src/exec/olap_scan_prepare.h
@@ -18,9 +18,9 @@
 #include "exec/olap_common.h"
 #include "exprs/expr.h"
 #include "exprs/expr_context.h"
-#include "storage/olap_runtime_range_pruner.h"
 #include "storage/predicate_tree/predicate_tree_fwd.h"
 #include "storage/predicate_tree_params.h"
+#include "storage/runtime_range_pruner.h"
 
 namespace starrocks {
 
diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.cpp b/be/src/exec/pipeline/scan/olap_chunk_source.cpp
index d180bae61525ed..44c93d42def643 100644
--- a/be/src/exec/pipeline/scan/olap_chunk_source.cpp
+++ b/be/src/exec/pipeline/scan/olap_chunk_source.cpp
@@ -37,9 +37,9 @@
 #include "storage/chunk_helper.h"
 #include "storage/column_predicate_rewriter.h"
 #include "storage/index/vector/vector_search_option.h"
-#include "storage/olap_runtime_range_pruner.hpp"
 #include "storage/predicate_parser.h"
 #include "storage/projection_iterator.h"
+#include "storage/runtime_range_pruner.hpp"
 #include "storage/storage_engine.h"
 #include "storage/tablet_index.h"
 #include "types/logical_type.h"
@@ -265,7 +265,7 @@ Status OlapChunkSource::_init_reader_params(const std::vector<std::unique_ptr<Ol
         _params.sorted_by_keys_per_tablet = thrift_olap_scan_node.sorted_by_keys_per_tablet;
     }
     _params.runtime_range_pruner =
-            OlapRuntimeScanRangePruner(parser, _scan_ctx->conjuncts_manager().unarrived_runtime_filters());
+            RuntimeScanRangePruner(parser, _scan_ctx->conjuncts_manager().unarrived_runtime_filters());
     _morsel->init_tablet_reader_params(&_params);
 
     ASSIGN_OR_RETURN(auto pred_tree, _scan_ctx->conjuncts_manager().get_predicate_tree(parser, _predicate_free_pool));
diff --git a/be/src/formats/parquet/file_reader.cpp b/be/src/formats/parquet/file_reader.cpp
index 60a10fae4404ae..38b7072ac5c1de 100644
--- a/be/src/formats/parquet/file_reader.cpp
+++ b/be/src/formats/parquet/file_reader.cpp
@@ -350,7 +350,7 @@ bool FileReader::_filter_group_with_more_filter(const GroupReaderPtr& group_read
 bool FileReader::_filter_group(const GroupReaderPtr& group_reader) {
     if (config::parquet_advance_zonemap_filter) {
         if (_scanner_ctx->rf_scan_range_pruner != nullptr) {
-            _rf_scan_range_pruner = std::make_shared<OlapRuntimeScanRangePruner>(*_scanner_ctx->rf_scan_range_pruner);
+            _rf_scan_range_pruner = std::make_shared<RuntimeScanRangePruner>(*_scanner_ctx->rf_scan_range_pruner);
         }
         auto res = _scanner_ctx->predicate_tree.visit(
                 ZoneMapEvaluator<FilterLevel::ROW_GROUP>{_scanner_ctx->predicate_tree, group_reader.get()});
diff --git a/be/src/formats/parquet/file_reader.h b/be/src/formats/parquet/file_reader.h
index 0584e6585e7e26..05f16b81dcbb3f 100644
--- a/be/src/formats/parquet/file_reader.h
+++ b/be/src/formats/parquet/file_reader.h
@@ -30,7 +30,7 @@
 #include "gen_cpp/parquet_types.h"
 #include "io/shared_buffered_input_stream.h"
 #include "runtime/runtime_state.h"
-#include "storage/olap_runtime_range_pruner.hpp"
+#include "storage/runtime_range_pruner.hpp"
 
 namespace tparquet {
 class ColumnMetaData;
@@ -148,7 +148,7 @@ class FileReader {
     GroupReaderParam _group_reader_param;
     std::shared_ptr<MetaHelper> _meta_helper = nullptr;
     SkipRowsContextPtr _skip_rows_ctx = nullptr;
-    std::shared_ptr<OlapRuntimeScanRangePruner> _rf_scan_range_pruner;
+    std::shared_ptr<RuntimeScanRangePruner> _rf_scan_range_pruner;
 };
 
 } // namespace starrocks::parquet
diff --git a/be/src/storage/rowset/rowset_options.h b/be/src/storage/rowset/rowset_options.h
index 138d065d204310..1da1cfa23f0733 100644
--- a/be/src/storage/rowset/rowset_options.h
+++ b/be/src/storage/rowset/rowset_options.h
@@ -21,9 +21,9 @@
 #include "column/column_access_path.h"
 #include "runtime/global_dict/types.h"
 #include "storage/olap_common.h"
-#include "storage/olap_runtime_range_pruner.h"
 #include "storage/options.h"
 #include "storage/predicate_tree/predicate_tree.hpp"
+#include "storage/runtime_range_pruner.h"
 #include "storage/seek_range.h"
 #include "storage/tablet_schema.h"
 
@@ -81,7 +81,7 @@ class RowsetReadOptions {
     RowidRangeOptionPtr rowid_range_option = nullptr;
     ShortKeyRangesOptionPtr short_key_ranges_option = nullptr;
 
-    OlapRuntimeScanRangePruner runtime_range_pruner;
+    RuntimeScanRangePruner runtime_range_pruner;
 
     std::vector<ColumnAccessPathPtr>* column_access_paths = nullptr;
 
diff --git a/be/src/storage/rowset/segment_iterator.cpp b/be/src/storage/rowset/segment_iterator.cpp
index c4bc486b9ab15b..06ec797a80ffc2 100644
--- a/be/src/storage/rowset/segment_iterator.cpp
+++ b/be/src/storage/rowset/segment_iterator.cpp
@@ -45,8 +45,6 @@
 #include "storage/index/vector/vector_index_reader_factory.h"
 #include "storage/index/vector/vector_search_option.h"
 #include "storage/lake/update_manager.h"
-#include "storage/olap_runtime_range_pruner.h"
-#include "storage/olap_runtime_range_pruner.hpp"
 #include "storage/projection_iterator.h"
 #include "storage/range.h"
 #include "storage/roaring2range.h"
@@ -61,6 +59,8 @@
 #include "storage/rowset/rowid_column_iterator.h"
 #include "storage/rowset/segment.h"
 #include "storage/rowset/short_key_range_option.h"
+#include "storage/runtime_range_pruner.h"
+#include "storage/runtime_range_pruner.hpp"
 #include "storage/types.h"
 #include "storage/update_manager.h"
 #include "types/array_type_info.h"
diff --git a/be/src/storage/rowset/segment_options.h b/be/src/storage/rowset/segment_options.h
index 8130781f9b253b..e633b5e7962517 100644
--- a/be/src/storage/rowset/segment_options.h
+++ b/be/src/storage/rowset/segment_options.h
@@ -23,9 +23,9 @@
 #include "runtime/global_dict/types.h"
 #include "storage/del_vector.h"
 #include "storage/disjunctive_predicates.h"
-#include "storage/olap_runtime_range_pruner.h"
 #include "storage/options.h"
 #include "storage/predicate_tree/predicate_tree.hpp"
+#include "storage/runtime_range_pruner.h"
 #include "storage/seek_range.h"
 #include "storage/tablet_schema.h"
 
@@ -92,7 +92,7 @@ class SegmentReadOptions {
     SparseRangePtr rowid_range_option = nullptr;
     std::vector<ShortKeyRangeOptionPtr> short_key_ranges;
 
-    OlapRuntimeScanRangePruner runtime_range_pruner;
+    RuntimeScanRangePruner runtime_range_pruner;
 
     const std::atomic<bool>* is_cancelled = nullptr;
 
diff --git a/be/src/storage/olap_runtime_range_pruner.h b/be/src/storage/runtime_range_pruner.h
similarity index 94%
rename from be/src/storage/olap_runtime_range_pruner.h
rename to be/src/storage/runtime_range_pruner.h
index 05830eaf3b4d2f..98d841200f64fa 100644
--- a/be/src/storage/olap_runtime_range_pruner.h
+++ b/be/src/storage/runtime_range_pruner.h
@@ -42,14 +42,14 @@ struct UnarrivedRuntimeFilterList {
     }
 };
 
-class OlapRuntimeScanRangePruner {
+class RuntimeScanRangePruner {
 public:
     using PredicatesRawPtrs = std::vector<const ColumnPredicate*>;
     using RuntimeFilterArrivedCallBack = std::function<Status(int, const PredicatesRawPtrs&)>;
     static constexpr auto rf_update_threshold = 4096 * 10;
 
-    OlapRuntimeScanRangePruner() = default;
-    OlapRuntimeScanRangePruner(PredicateParser* parser, const UnarrivedRuntimeFilterList& params) {
+    RuntimeScanRangePruner() = default;
+    RuntimeScanRangePruner(PredicateParser* parser, const UnarrivedRuntimeFilterList& params) {
         _parser = parser;
         _init(params);
     }
diff --git a/be/src/storage/olap_runtime_range_pruner.hpp b/be/src/storage/runtime_range_pruner.hpp
similarity index 94%
rename from be/src/storage/olap_runtime_range_pruner.hpp
rename to be/src/storage/runtime_range_pruner.hpp
index d293f7bc157e52..147628faa163be 100644
--- a/be/src/storage/olap_runtime_range_pruner.hpp
+++ b/be/src/storage/runtime_range_pruner.hpp
@@ -25,8 +25,8 @@
 #include "storage/column_and_predicate.h"
 #include "storage/column_or_predicate.h"
 #include "storage/column_predicate.h"
-#include "storage/olap_runtime_range_pruner.h"
 #include "storage/predicate_parser.h"
+#include "storage/runtime_range_pruner.h"
 
 namespace starrocks {
 namespace detail {
@@ -222,9 +222,9 @@ struct RuntimeColumnPredicateBuilder {
 };
 } // namespace detail
 
-inline Status OlapRuntimeScanRangePruner::_update(const ColumnIdToGlobalDictMap* global_dictmaps,
-                                                  RuntimeFilterArrivedCallBack&& updater, bool force,
-                                                  size_t raw_read_rows) {
+inline Status RuntimeScanRangePruner::_update(const ColumnIdToGlobalDictMap* global_dictmaps,
+                                              RuntimeFilterArrivedCallBack&& updater, bool force,
+                                              size_t raw_read_rows) {
     if (_arrived_runtime_filters_masks.empty()) {
         return Status::OK();
     }
@@ -252,8 +252,8 @@ inline Status OlapRuntimeScanRangePruner::_update(const ColumnIdToGlobalDictMap*
     return Status::OK();
 }
 
-inline auto OlapRuntimeScanRangePruner::_get_predicates(const ColumnIdToGlobalDictMap* global_dictmaps, size_t idx,
-                                                        ObjectPool* pool) -> StatusOr<PredicatesRawPtrs> {
+inline auto RuntimeScanRangePruner::_get_predicates(const ColumnIdToGlobalDictMap* global_dictmaps, size_t idx,
+                                                    ObjectPool* pool) -> StatusOr<PredicatesRawPtrs> {
     // convert to olap filter
     auto slot_desc = _slot_descs[idx];
     return type_dispatch_predicate<StatusOr<PredicatesRawPtrs>>(
@@ -261,7 +261,7 @@ inline auto OlapRuntimeScanRangePruner::_get_predicates(const ColumnIdToGlobalDi
             _unarrived_runtime_filters[idx], slot_desc, _driver_sequence, pool);
 }
 
-inline void OlapRuntimeScanRangePruner::_init(const UnarrivedRuntimeFilterList& params) {
+inline void RuntimeScanRangePruner::_init(const UnarrivedRuntimeFilterList& params) {
     for (size_t i = 0; i < params.slot_descs.size(); ++i) {
         if (_parser->can_pushdown(params.slot_descs[i])) {
             _unarrived_runtime_filters.emplace_back(params.unarrived_runtime_filters[i]);
diff --git a/be/src/storage/tablet_reader_params.h b/be/src/storage/tablet_reader_params.h
index 0d8ba9a7a135ae..8b3163e8f7e200 100644
--- a/be/src/storage/tablet_reader_params.h
+++ b/be/src/storage/tablet_reader_params.h
@@ -23,8 +23,8 @@
 #include "runtime/global_dict/types.h"
 #include "storage/chunk_iterator.h"
 #include "storage/olap_common.h"
-#include "storage/olap_runtime_range_pruner.h"
 #include "storage/predicate_tree/predicate_tree.hpp"
+#include "storage/runtime_range_pruner.h"
 #include "storage/tuple.h"
 
 namespace starrocks {
@@ -86,7 +86,7 @@ struct TabletReaderParams {
     ShortKeyRangesOptionPtr short_key_ranges_option = nullptr;
 
     bool sorted_by_keys_per_tablet = false;
-    OlapRuntimeScanRangePruner runtime_range_pruner;
+    RuntimeScanRangePruner runtime_range_pruner;
 
     std::vector<ColumnAccessPathPtr>* column_access_paths = nullptr;
     bool use_pk_index = false;
diff --git a/be/test/formats/parquet/file_reader_test.cpp b/be/test/formats/parquet/file_reader_test.cpp
index 5accc045f9b967..f9d9ba11f4a42c 100644
--- a/be/test/formats/parquet/file_reader_test.cpp
+++ b/be/test/formats/parquet/file_reader_test.cpp
@@ -742,7 +742,7 @@ StatusOr<HdfsScannerContext*> FileReaderTest::_create_context_for_filter_row_gro
     rf_list->driver_sequence = 1;
     rf_list->unarrived_runtime_filters.emplace_back(rf_desc);
     rf_list->slot_descs.emplace_back(ctx->slot_descs[0]);
-    ctx->rf_scan_range_pruner = _pool.add(new OlapRuntimeScanRangePruner(pred_parser, *rf_list));
+    ctx->rf_scan_range_pruner = _pool.add(new RuntimeScanRangePruner(pred_parser, *rf_list));
 
     return ctx;
 }
diff --git a/be/test/storage/olap_runtime_range_pruner_test.cpp b/be/test/storage/olap_runtime_range_pruner_test.cpp
index 7b29b16b96e72d..317f0753f4bdfb 100644
--- a/be/test/storage/olap_runtime_range_pruner_test.cpp
+++ b/be/test/storage/olap_runtime_range_pruner_test.cpp
@@ -12,8 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "storage/olap_runtime_range_pruner.hpp"
-
 #include <gtest/gtest.h>
 
 #include "exprs/runtime_filter_bank.h"
@@ -22,6 +20,7 @@
 #include "runtime/runtime_state.h"
 #include "storage/column_predicate.h"
 #include "storage/predicate_parser.h"
+#include "storage/runtime_range_pruner.hpp"
 #include "testutil/exprs_test_helper.h"
 #include "testutil/schema_test_helper.h"
 
@@ -103,7 +102,7 @@ TEST_F(OlapRuntimeRangePrunerTest, update_1) {
 
     UnarrivedRuntimeFilterList unarrivedRuntimeFilterList;
     unarrivedRuntimeFilterList.add_unarrived_rf(runtime_filter_desc.get(), &slot, 1);
-    OlapRuntimeScanRangePruner pruner(_predicate_parser.get(), unarrivedRuntimeFilterList);
+    RuntimeScanRangePruner pruner(_predicate_parser.get(), unarrivedRuntimeFilterList);
 
     size_t pred_size = 0;
     std::string pred_1;
@@ -162,7 +161,7 @@ TEST_F(OlapRuntimeRangePrunerTest, update_has_null) {
 
     UnarrivedRuntimeFilterList unarrivedRuntimeFilterList;
     unarrivedRuntimeFilterList.add_unarrived_rf(runtime_filter_desc.get(), &slot, 1);
-    OlapRuntimeScanRangePruner pruner(_predicate_parser.get(), unarrivedRuntimeFilterList);
+    RuntimeScanRangePruner pruner(_predicate_parser.get(), unarrivedRuntimeFilterList);
 
     size_t pred_size = 0;
     std::string pred;

From 23e6b791483a7cf6295f5ed46d43e7485fcd548b Mon Sep 17 00:00:00 2001
From: stdpain <34912776+stdpain@users.noreply.github.com>
Date: Fri, 17 Jan 2025 10:36:04 +0800
Subject: [PATCH 27/71] [Enhancement] Optimize the performance of reading
 dictionary pages (#55137)

Signed-off-by: stdpain <drfeng08@gmail.com>
---
 be/src/common/config.h                      |  2 +
 be/src/storage/rowset/binary_dict_page.cpp  |  4 +-
 be/src/storage/rowset/binary_plain_page.cpp | 45 +++++++++++++++++++++
 be/src/storage/rowset/binary_plain_page.h   | 15 +++++++
 4 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/be/src/common/config.h b/be/src/common/config.h
index 09e40a9a8b73db..7f2a6a3ea6eb2d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -428,6 +428,8 @@ CONF_Double(dictionary_encoding_ratio, "0.7");
 // performance degradation.
 CONF_Int32(dictionary_page_size, "1048576");
 
+CONF_Int32(small_dictionary_page_size, "4096");
+
 // Just like dictionary_encoding_ratio, dictionary_encoding_ratio_for_non_string_column is used for
 // no-string column.
 CONF_Double(dictionary_encoding_ratio_for_non_string_column, "0");
diff --git a/be/src/storage/rowset/binary_dict_page.cpp b/be/src/storage/rowset/binary_dict_page.cpp
index d54d8c959903f6..0bbf43af2ed09d 100644
--- a/be/src/storage/rowset/binary_dict_page.cpp
+++ b/be/src/storage/rowset/binary_dict_page.cpp
@@ -256,9 +256,7 @@ Status BinaryDictPageDecoder<Type>::next_batch(const SparseRange<>& range, Colum
             slices[i] = element;
         }
     } else {
-        for (int i = 0; i < nread; ++i) {
-            slices[i] = _dict_decoder->string_at_index(codewords[i]);
-        }
+        _dict_decoder->batch_string_at_index(slices.data(), codewords, nread);
     }
 
     bool ok = dst->append_strings_overflow(slices, _max_value_legth);
diff --git a/be/src/storage/rowset/binary_plain_page.cpp b/be/src/storage/rowset/binary_plain_page.cpp
index b48ce94b241f91..d54669167ed674 100644
--- a/be/src/storage/rowset/binary_plain_page.cpp
+++ b/be/src/storage/rowset/binary_plain_page.cpp
@@ -1,5 +1,23 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 #include "storage/rowset/binary_plain_page.h"
 
+#ifdef __SSE4_2__
+#include <emmintrin.h>
+#endif
+
 #include <cstring>
 
 #include "column/binary_column.h"
@@ -135,6 +153,33 @@ bool BinaryPlainPageDecoder<Type>::append_range(uint32_t idx, uint32_t end, Colu
     }
 }
 
+template <LogicalType Type>
+void BinaryPlainPageDecoder<Type>::batch_string_at_index(Slice* dst, const int32_t* idx, size_t size) const {
+    if (_parsed_datas.has_value()) {
+        const std::vector<Slice>& parsed_data = *_parsed_datas;
+        const Slice* parsed_data_ptr = parsed_data.data();
+        static_assert(sizeof(Slice) == sizeof(__int128));
+#ifdef __SSE4_2__
+#pragma GCC unroll 2
+        for (int i = 0; i < size; ++i) {
+            DCHECK_LT(idx[i], parsed_data.size());
+            __m128i slice = _mm_loadu_si128((__m128i_u*)(parsed_data_ptr + idx[i]));
+            _mm_storeu_si128((__m128i_u*)(dst + i), slice);
+        }
+#else
+        for (int i = 0; i < size; ++i) {
+            DCHECK_LT(idx[i], parsed_data.size());
+            dst[i] = parsed_data[idx[i]];
+        }
+#endif
+
+    } else {
+        for (int i = 0; i < size; ++i) {
+            dst[i] = string_at_index(idx[i]);
+        }
+    }
+}
+
 template class BinaryPlainPageDecoder<TYPE_CHAR>;
 template class BinaryPlainPageDecoder<TYPE_VARCHAR>;
 template class BinaryPlainPageDecoder<TYPE_HLL>;
diff --git a/be/src/storage/rowset/binary_plain_page.h b/be/src/storage/rowset/binary_plain_page.h
index 3a84072fa679a9..2fbbff0bb8109f 100644
--- a/be/src/storage/rowset/binary_plain_page.h
+++ b/be/src/storage/rowset/binary_plain_page.h
@@ -201,6 +201,17 @@ class BinaryPlainPageDecoder final : public PageDecoder {
                 static_cast<uint32_t>(_data.get_size()) - (_num_elems + 1) * static_cast<uint32_t>(sizeof(uint32_t));
         _offsets_ptr = reinterpret_cast<uint32_t*>(_data.data + _offsets_pos);
 
+        if (_data.size < config::small_dictionary_page_size) {
+            _parsed_datas = std::vector<Slice>();
+            _parsed_datas->reserve(_num_elems);
+            for (uint32_t i = 0; i < _num_elems; i++) {
+                const uint32_t off1 = offset_uncheck(i);
+                const uint32_t off2 = offset(i + 1);
+                Slice s(&_data[off1], off2 - off1);
+                _parsed_datas->emplace_back(s);
+            }
+        }
+
         _parsed = true;
 
         return Status::OK();
@@ -236,6 +247,8 @@ class BinaryPlainPageDecoder final : public PageDecoder {
         return {&_data[start_offset], len};
     }
 
+    void batch_string_at_index(Slice* dst, const int32_t* idx, size_t size) const;
+
     int find(const Slice& word) const {
         DCHECK(_parsed);
         for (uint32_t i = 0; i < _num_elems; i++) {
@@ -285,6 +298,8 @@ class BinaryPlainPageDecoder final : public PageDecoder {
 
     // Index of the currently seeked element in the page.
     uint32_t _cur_idx;
+
+    std::optional<std::vector<Slice>> _parsed_datas;
 };
 
 } // namespace starrocks

From 6e86e4563f4b9259fe87df1982c2a6fdccd0b791 Mon Sep 17 00:00:00 2001
From: xiangguangyxg <110401425+xiangguangyxg@users.noreply.github.com>
Date: Fri, 17 Jan 2025 10:54:01 +0800
Subject: [PATCH 28/71] [Enhancement] Use cluster_snapshot_path to specify the
 cluster snapshot to be restored (#55147)

Signed-off-by: xiangguangyxg <xiangguangyxg@gmail.com>
---
 conf/cluster_snapshot.yaml                    |  3 +-
 .../lake/snapshot/ClusterSnapshotConfig.java  | 29 ++++++-------------
 .../snapshot/RestoreClusterSnapshotMgr.java   | 14 ++++-----
 .../snapshot/ClusterSnapshotConfigTest.java   | 10 +++----
 .../RestoreClusterSnapshotMgrTest.java        |  2 +-
 .../test/resources/conf/cluster_snapshot.yaml |  3 +-
 .../resources/conf/cluster_snapshot2.yaml     |  3 +-
 7 files changed, 22 insertions(+), 42 deletions(-)

diff --git a/conf/cluster_snapshot.yaml b/conf/cluster_snapshot.yaml
index 868c025d0da0c5..490697b5f20e9a 100644
--- a/conf/cluster_snapshot.yaml
+++ b/conf/cluster_snapshot.yaml
@@ -2,9 +2,8 @@
 
 # information about the cluster snapshot to be downloaded and restored
 #cluster_snapshot:
+#    cluster_snapshot_path: s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000
 #    storage_volume_name: my_s3_volume #defined in storage_volumes
-#    cluster_service_id: f7265e80-631c-44d3-a8ac-cf7cdc7adec811019
-#    cluster_snapshot_name: automated_cluster_snapshot_1704038400000
 
 # do not include leader fe
 #frontends:
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotConfig.java b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotConfig.java
index 8ebc3e6d8dddd9..7b35818adb6fc6 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotConfig.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotConfig.java
@@ -42,16 +42,21 @@ public class ClusterSnapshotConfig {
     private static final Logger LOG = LogManager.getLogger(ClusterSnapshotConfig.class);
 
     public static class ClusterSnapshot {
+        @JsonProperty("cluster_snapshot_path")
+        private String clusterSnapshotPath;
+
         @JsonProperty("storage_volume_name")
         private String storageVolumeName;
 
         private StorageVolume storageVolume;
 
-        @JsonProperty("cluster_service_id")
-        private String clusterServiceId;
+        public String getClusterSnapshotPath() {
+            return clusterSnapshotPath;
+        }
 
-        @JsonProperty("cluster_snapshot_name")
-        private String clusterSnapshotName;
+        public void setClusterSnapshotPath(String clusterSnapshotPath) {
+            this.clusterSnapshotPath = clusterSnapshotPath;
+        }
 
         public String getStorageVolumeName() {
             return storageVolumeName;
@@ -68,22 +73,6 @@ public StorageVolume getStorageVolume() {
         public void setStorageVolume(StorageVolume storageVolume) {
             this.storageVolume = storageVolume;
         }
-
-        public String getClusterServiceId() {
-            return clusterServiceId;
-        }
-
-        public void setClusterServiceId(String clusterServiceId) {
-            this.clusterServiceId = clusterServiceId;
-        }
-
-        public String getClusterSnapshotName() {
-            return clusterSnapshotName;
-        }
-
-        public void setClusterSnapshotName(String clusterSnapshotName) {
-            this.clusterSnapshotName = clusterSnapshotName;
-        }
     }
 
     public static class Frontend {
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgr.java b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgr.java
index 062646cc037bb1..2d61aaa8f01a46 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgr.java
@@ -34,7 +34,6 @@
 import java.io.File;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
 public class RestoreClusterSnapshotMgr {
     private static final Logger LOG = LogManager.getLogger(RestoreClusterSnapshotMgr.class);
@@ -86,6 +85,7 @@ public static void finishRestoring() throws StarRocksException {
         } finally {
             self.rollbackConfig();
             instance = null;
+            LOG.info("FE finished to restore from a cluster snapshot");
         }
     }
 
@@ -118,17 +118,13 @@ private void downloadSnapshot() throws StarRocksException {
             LOG.info("Deleted image dir {}", localImagePath);
         }
         if (FileUtils.deleteQuietly(new File(localBdbPath))) {
-            LOG.info("Deleted bdb {}", localBdbPath);
+            LOG.info("Deleted bdb dir {}", localBdbPath);
         }
 
-        ClusterSnapshotConfig.StorageVolume storageVolume = clusterSnapshot.getStorageVolume();
-        // TODO: use constant and support no snapshot name
-        String snapshotImagePath = String.join("/", storageVolume.getLocation(), clusterSnapshot.getClusterServiceId(),
-                "meta/image", clusterSnapshot.getClusterSnapshotName());
-        Map<String, String> properties = storageVolume.getProperties();
+        String snapshotImagePath = clusterSnapshot.getClusterSnapshotPath();
 
-        LOG.info("Copy snapshot image {} to local dir {}", snapshotImagePath, localImagePath);
-        HdfsUtil.copyToLocal(snapshotImagePath, localImagePath, properties);
+        LOG.info("Download cluster snapshot {} to local dir {}", snapshotImagePath, localImagePath);
+        HdfsUtil.copyToLocal(snapshotImagePath, localImagePath, clusterSnapshot.getStorageVolume().getProperties());
     }
 
     private void updateFrontends() throws StarRocksException {
diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/ClusterSnapshotConfigTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/ClusterSnapshotConfigTest.java
index 1d0aef05b089ad..f710d408e3c28a 100644
--- a/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/ClusterSnapshotConfigTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/ClusterSnapshotConfigTest.java
@@ -28,16 +28,14 @@ public void testLoadFromFile() {
 
         ClusterSnapshotConfig.ClusterSnapshot clusterSnapshot = config.getClusterSnapshot();
         Assert.assertNotNull(clusterSnapshot);
+        Assert.assertEquals(
+                "s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000",
+                clusterSnapshot.getClusterSnapshotPath());
         Assert.assertEquals("my_s3_volume", clusterSnapshot.getStorageVolumeName());
         Assert.assertNotNull(clusterSnapshot.getStorageVolume());
-        Assert.assertEquals("f7265e80-631c-44d3-a8ac-cf7cdc7adec811019",
-                clusterSnapshot.getClusterServiceId());
-        Assert.assertEquals("automated_cluster_snapshot_1704038400000",
-                clusterSnapshot.getClusterSnapshotName());
 
+        clusterSnapshot.setClusterSnapshotPath(clusterSnapshot.getClusterSnapshotPath());
         clusterSnapshot.setStorageVolumeName(clusterSnapshot.getStorageVolumeName());
-        clusterSnapshot.setClusterServiceId(clusterSnapshot.getClusterServiceId());
-        clusterSnapshot.setClusterSnapshotName(clusterSnapshot.getClusterSnapshotName());
 
         Assert.assertEquals(2, config.getFrontends().size());
         Assert.assertEquals(2, config.getComputeNodes().size());
diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgrTest.java
index 5379d513b3ca16..017bc4a496cce1 100644
--- a/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgrTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/lake/snapshot/RestoreClusterSnapshotMgrTest.java
@@ -31,7 +31,7 @@ public static void beforeClass() throws Exception {
     }
 
     @Test
-    public void testDownloadSnapshotFaied() throws Exception {
+    public void testDownloadSnapshotFailed() throws Exception {
         Assert.assertThrows(StarRocksException.class, () -> {
             RestoreClusterSnapshotMgr.init("src/test/resources/conf/cluster_snapshot.yaml",
                     new String[] { "-cluster_snapshot" });
diff --git a/fe/fe-core/src/test/resources/conf/cluster_snapshot.yaml b/fe/fe-core/src/test/resources/conf/cluster_snapshot.yaml
index 9406ed31f9d1fe..757d0ecaeb6d66 100644
--- a/fe/fe-core/src/test/resources/conf/cluster_snapshot.yaml
+++ b/fe/fe-core/src/test/resources/conf/cluster_snapshot.yaml
@@ -2,9 +2,8 @@
 
 # information about the cluster snapshot to be downloaded and restored
 cluster_snapshot:
+    cluster_snapshot_path: s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000
     storage_volume_name: my_s3_volume #defined in storage_volumes
-    cluster_service_id: f7265e80-631c-44d3-a8ac-cf7cdc7adec811019
-    cluster_snapshot_name: automated_cluster_snapshot_1704038400000
 
 # do not include leader fe
 frontends:
diff --git a/fe/fe-core/src/test/resources/conf/cluster_snapshot2.yaml b/fe/fe-core/src/test/resources/conf/cluster_snapshot2.yaml
index 3586b0a6e4728b..960882dcdcb4ea 100644
--- a/fe/fe-core/src/test/resources/conf/cluster_snapshot2.yaml
+++ b/fe/fe-core/src/test/resources/conf/cluster_snapshot2.yaml
@@ -2,9 +2,8 @@
 
 # information about the cluster snapshot to be downloaded and restored
 # cluster_snapshot:
+#     cluster_snapshot_path: s3://defaultbucket/test/f7265e80-631c-44d3-a8ac-cf7cdc7adec811019/meta/image/automated_cluster_snapshot_1704038400000
 #     storage_volume_name: my_s3_volume #defined in storage_volumes
-#     cluster_service_id: f7265e80-631c-44d3-a8ac-cf7cdc7adec811019
-#     cluster_snapshot_name: automated_cluster_snapshot_1704038400000
 
 # do not include leader fe
 frontends:

From 4fbcc67db7ac7e503211d158871c3e95c6d5ee1b Mon Sep 17 00:00:00 2001
From: gengjun-git <gengjun@starrocks.com>
Date: Fri, 17 Jan 2025 11:06:06 +0800
Subject: [PATCH 29/71] [Enhancement] Support mysql_clear_password pulgin for
 native user (#55145)

Signed-off-by: gengjun-git <gengjun@starrocks.com>
---
 .../authentication/PlainPasswordAuthenticationProvider.java  | 4 +++-
 .../main/java/com/starrocks/mysql/MysqlHandshakePacket.java  | 4 ++--
 fe/fe-core/src/main/java/com/starrocks/mysql/MysqlProto.java | 5 ++++-
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/authentication/PlainPasswordAuthenticationProvider.java b/fe/fe-core/src/main/java/com/starrocks/authentication/PlainPasswordAuthenticationProvider.java
index 4055882acb58ba..961da0e2a4bea9 100644
--- a/fe/fe-core/src/main/java/com/starrocks/authentication/PlainPasswordAuthenticationProvider.java
+++ b/fe/fe-core/src/main/java/com/starrocks/authentication/PlainPasswordAuthenticationProvider.java
@@ -21,6 +21,7 @@
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.sql.ast.UserAuthOption;
 import com.starrocks.sql.ast.UserIdentity;
+import org.apache.commons.lang3.StringUtils;
 
 import java.nio.charset.StandardCharsets;
 
@@ -111,7 +112,8 @@ public void authenticate(
         } else {
             // Plain remote password, scramble it first.
             byte[] scrambledRemotePass =
-                    MysqlPassword.makeScrambledPassword(new String(remotePassword, StandardCharsets.UTF_8));
+                    MysqlPassword.makeScrambledPassword((StringUtils.stripEnd(
+                            new String(remotePassword, StandardCharsets.UTF_8), "\0")));
             if (!MysqlPassword.checkScrambledPlainPass(authenticationInfo.getPassword(), scrambledRemotePass)) {
                 throw new AuthenticationException("password mismatch!");
             }
diff --git a/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlHandshakePacket.java b/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlHandshakePacket.java
index e30d70823dcb2b..87808ec6ebd685 100644
--- a/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlHandshakePacket.java
+++ b/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlHandshakePacket.java
@@ -59,8 +59,8 @@ public class MysqlHandshakePacket extends MysqlPacket {
     private static final MysqlCapability CAPABILITY = MysqlCapability.DEFAULT_CAPABILITY;
     // status flags not supported in StarRocks
     private static final int STATUS_FLAGS = 0;
-    private static final String NATIVE_AUTH_PLUGIN_NAME = "mysql_native_password";
-    private static final String CLEAR_PASSWORD_PLUGIN_NAME = "mysql_clear_password";
+    public static final String NATIVE_AUTH_PLUGIN_NAME = "mysql_native_password";
+    public static final String CLEAR_PASSWORD_PLUGIN_NAME = "mysql_clear_password";
     public static final String AUTHENTICATION_KERBEROS_CLIENT = "authentication_kerberos_client";
 
     private static final ImmutableMap<String, Boolean> SUPPORTED_PLUGINS = new ImmutableMap.Builder<String, Boolean>()
diff --git a/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlProto.java b/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlProto.java
index 69f1a00b42a39a..dd009ade031484 100644
--- a/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlProto.java
+++ b/fe/fe-core/src/main/java/com/starrocks/mysql/MysqlProto.java
@@ -51,6 +51,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static com.starrocks.mysql.MysqlHandshakePacket.AUTHENTICATION_KERBEROS_CLIENT;
@@ -217,7 +218,9 @@ public static NegotiateResult negotiate(ConnectContext context) throws IOExcepti
         serializer.setCapability(context.getCapability());
 
         // NOTE: when we behind proxy, we need random string sent by proxy.
-        byte[] randomString = handshakePacket.getAuthPluginData();
+        byte[] randomString =
+                Objects.equals(authPluginName, MysqlHandshakePacket.CLEAR_PASSWORD_PLUGIN_NAME) ?
+                        null : handshakePacket.getAuthPluginData();
         // check authenticate
         if (!authenticate(context, authPacket.getAuthResponse(), randomString, authPacket.getUser())) {
             sendResponsePacket(context);

From fe95ca66cbcb9d4eb2fd31b49fff50fdd62050c3 Mon Sep 17 00:00:00 2001
From: wyb <wybb86@gmail.com>
Date: Fri, 17 Jan 2025 11:08:47 +0800
Subject: [PATCH 30/71] [BugFix] Fix promote ip string as double type in csv
 files() (#55173)

Signed-off-by: wyb <wybb86@gmail.com>
---
 be/src/util/string_parser.hpp         |  8 ++++++++
 be/test/util/string_parser_test.cpp   |  2 ++
 test/sql/test_files/R/test_csv_ip     | 28 +++++++++++++++++++++++++++
 test/sql/test_files/T/test_csv_ip     | 12 ++++++++++++
 test/sql/test_files/csv_format/ip.csv |  1 +
 5 files changed, 51 insertions(+)
 create mode 100644 test/sql/test_files/R/test_csv_ip
 create mode 100644 test/sql/test_files/T/test_csv_ip
 create mode 100644 test/sql/test_files/csv_format/ip.csv

diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp
index 951b690fa6eedf..21a7b78e12b785 100644
--- a/be/src/util/string_parser.hpp
+++ b/be/src/util/string_parser.hpp
@@ -522,6 +522,14 @@ inline T StringParser::string_to_float_internal(const char* s, int len, ParseRes
     auto res = fast_float::from_chars(s + i, s + j + 1, val);
 
     if (LIKELY(res.ec == std::errc())) {
+        // 'res.ptr' is set to point right after the parsed number.
+        // if there are some chars left, treate it as failure.
+        // for example,
+        // '10.11.12.13' is parsed as 10.11, res.ptr is '.12.13', so it is invalid.
+        if (res.ptr != s + j + 1) {
+            *result = PARSE_FAILURE;
+            return 0;
+        }
         if (UNLIKELY(val == std::numeric_limits<T>::infinity())) {
             *result = PARSE_OVERFLOW;
         } else {
diff --git a/be/test/util/string_parser_test.cpp b/be/test/util/string_parser_test.cpp
index 77d0550edb5a14..e7c52dbe1dc1f5 100644
--- a/be/test/util/string_parser_test.cpp
+++ b/be/test/util/string_parser_test.cpp
@@ -540,6 +540,8 @@ TEST(StringToFloat, Basic) {
     test_all_float_variants("in finity", StringParser::PARSE_FAILURE);
     test_all_float_variants("na", StringParser::PARSE_FAILURE);
     test_all_float_variants("ThisIsANaN", StringParser::PARSE_FAILURE);
+    test_all_float_variants("10.1.2.3", StringParser::PARSE_FAILURE);
+    test_all_float_variants("10.1 max", StringParser::PARSE_FAILURE);
 }
 
 TEST(StringToFloat, InvalidLeadingTrailing) {
diff --git a/test/sql/test_files/R/test_csv_ip b/test/sql/test_files/R/test_csv_ip
new file mode 100644
index 00000000000000..8d0b9ac0c73e33
--- /dev/null
+++ b/test/sql/test_files/R/test_csv_ip
@@ -0,0 +1,28 @@
+-- name: test_csv_ip
+
+create database db_${uuid0};
+use db_${uuid0};
+
+shell: ossutil64 mkdir oss://${oss_bucket}/test_files/csv_format/${uuid0} >/dev/null || echo "exit 0" >/dev/null
+
+shell: ossutil64 cp --force ./sql/test_files/csv_format/ip.csv oss://${oss_bucket}/test_files/csv_format/${uuid0}/ | grep -Pv "(average|elapsed)"
+-- result:
+0
+
+Succeed: Total num: 1, size: 14. OK num: 1(upload 1 files).
+-- !result
+
+
+select * from files("path" = "oss://${oss_bucket}/test_files/csv_format/${uuid0}/*", "format" = "csv", "csv.column_separator" = "|", "csv.row_delimiter" = "\n");
+-- result:
+1	10.11.12.13
+-- !result
+
+desc files("path" = "oss://${oss_bucket}/test_files/csv_format/${uuid0}/*", "format" = "csv", "csv.column_separator" = "|", "csv.row_delimiter" = "\n");
+-- result:
+$1	bigint	YES
+$2	varchar(1048576)	YES
+-- !result
+
+
+shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null
diff --git a/test/sql/test_files/T/test_csv_ip b/test/sql/test_files/T/test_csv_ip
new file mode 100644
index 00000000000000..b62ffaa4e2d944
--- /dev/null
+++ b/test/sql/test_files/T/test_csv_ip
@@ -0,0 +1,12 @@
+-- name: test_csv_ip
+
+create database db_${uuid0};
+use db_${uuid0};
+
+shell: ossutil64 mkdir oss://${oss_bucket}/test_files/csv_format/${uuid0} >/dev/null || echo "exit 0" >/dev/null
+shell: ossutil64 cp --force ./sql/test_files/csv_format/ip.csv oss://${oss_bucket}/test_files/csv_format/${uuid0}/ | grep -Pv "(average|elapsed)"
+
+select * from files("path" = "oss://${oss_bucket}/test_files/csv_format/${uuid0}/*", "format" = "csv", "csv.column_separator" = "|", "csv.row_delimiter" = "\n");
+desc files("path" = "oss://${oss_bucket}/test_files/csv_format/${uuid0}/*", "format" = "csv", "csv.column_separator" = "|", "csv.row_delimiter" = "\n");
+
+shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null
diff --git a/test/sql/test_files/csv_format/ip.csv b/test/sql/test_files/csv_format/ip.csv
new file mode 100644
index 00000000000000..419eb79e83ba9a
--- /dev/null
+++ b/test/sql/test_files/csv_format/ip.csv
@@ -0,0 +1 @@
+1|10.11.12.13

From 6909ac7d68641c1453565e4c72832b78086e5cb7 Mon Sep 17 00:00:00 2001
From: "shuming.li" <ming.moriarty@gmail.com>
Date: Fri, 17 Jan 2025 11:21:38 +0800
Subject: [PATCH 31/71] [BugFix] Fix RewriteUnnestBitmapRule rewrite bug
 (#55168)

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
---
 .../RewriteUnnestBitmapRule.java              | 20 +++++---
 .../starrocks/sql/plan/TableFunctionTest.java | 13 +++++
 .../R/test_unnest_bitmap                      | 50 +++++++++++++++++++
 .../T/test_unnest_bitmap                      | 29 +++++++++++
 4 files changed, 105 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteUnnestBitmapRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteUnnestBitmapRule.java
index c845530cca61d5..4d5d95c28b4e18 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteUnnestBitmapRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteUnnestBitmapRule.java
@@ -13,6 +13,7 @@
 // limitations under the License.
 package com.starrocks.sql.optimizer.rule.transformation;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.starrocks.analysis.Expr;
 import com.starrocks.catalog.Function;
@@ -85,14 +86,22 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
             return false;
         }).map(entry -> new Pair<>(entry.getKey(), entry.getValue())).findFirst().get();
 
-        // if bitmap_to_array's output will be used by upper nodes, it's not safe to rewrite
+        // if bitmap_to_array's output are used by upper nodes, it's not safe to rewrite
         if (originalTableFunctionOperator.getOuterColRefs().contains(bitmapToArray.first)) {
             return Lists.newArrayList();
         }
 
-        ColumnRefOperator bitmapColumn = bitmapToArray.second.getColumnRefs().get(0);
-        columnRefMap.remove(bitmapToArray.first);
-        columnRefMap.putIfAbsent(bitmapColumn, bitmapColumn);
+        Preconditions.checkArgument(bitmapToArray.second.getChildren().size() == 1);
+        // use bitmapToArrayArg to replace bitmapColumn
+        // eg:
+        //  with r1 as (select b1 as b2 from test_agg),
+        //       r2 as (select sub_bitmap(b1, 0, 10) as b2 from test_agg),
+        //       r3 as (select bitmap_and(t0.b2, t1.b2) as b2 from r1 t0 join r2 t1)
+        //  select unnest as r1 from test_agg, unnest(bitmap_to_array(b2)) order by r1;
+        //  use bitmap_and(t0.b2, t1.b2) as b2 to replace bitmap_to_array(b2)
+        final ScalarOperator bitmapToArrayArg = bitmapToArray.second.getChild(0);
+        final ColumnRefOperator bitmapColumn = bitmapToArray.first;
+        columnRefMap.put(bitmapColumn, bitmapToArrayArg);
 
         TableFunction unnestBitmapFn =
                 (TableFunction) Expr.getBuiltinFunction(FunctionSet.UNNEST_BITMAP, new Type[] {Type.BITMAP},
@@ -107,6 +116,3 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
         return Lists.newArrayList(OptExpression.create(newTableFunctionOperator, input.inputAt(0)));
     }
 }
-
-
-
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/TableFunctionTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/TableFunctionTest.java
index 66fb8cb6f0f988..63aef555095a0b 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/TableFunctionTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/TableFunctionTest.java
@@ -277,4 +277,17 @@ public void testRewrite() throws Exception {
         assertContains(plan, "tableFunctionName: unnest_bitmap");
         assertNotContains(plan, "bitmap_to_array");
     }
+
+    @Test
+    public void testUnnesetBitmapToArrayToUnnestBitmapRewrite() throws Exception {
+        String sql = "with r1 as (select b1 as b2 from test_agg),\n" +
+                "\t r2 as (select sub_bitmap(b1, 0, 10) as b2 from test_agg),\n" +
+                "\t r3 as (select bitmap_and(t0.b2, t1.b2) as b2 from r1 t0 join r2 t1)\n" +
+                "select unnest as r1 from r3, unnest(bitmap_to_array(b2)) order by r1;";
+        String plan = getFragmentPlan(sql);
+        PlanTestBase.assertContains(plan, "5:Project\n" +
+                "  |  <slot 28> : bitmap_and(10: b1, 25: sub_bitmap)");
+        PlanTestBase.assertContains(plan, "tableFunctionName: unnest_bitmap");
+        PlanTestBase.assertNotContains(plan, "bitmap_to_array");
+    }
 }
diff --git a/test/sql/test_bitmap_functions/R/test_unnest_bitmap b/test/sql/test_bitmap_functions/R/test_unnest_bitmap
index 5a60b0970b620c..7aa8506b083b2f 100644
--- a/test/sql/test_bitmap_functions/R/test_unnest_bitmap
+++ b/test/sql/test_bitmap_functions/R/test_unnest_bitmap
@@ -42,4 +42,54 @@ select c1, unnest_bitmap as c3 from t1, unnest_bitmap(c2) order by c1 desc, c3 d
 4	18
 4	17
 4	16
+-- !result
+set enable_rewrite_unnest_bitmap_to_array=true;
+-- result:
+-- !result
+CREATE TABLE test_tags (
+	c1 varchar(65533) NOT NULL,
+	tag_name varchar(65533) NOT NULL,
+	tag_value varchar(65533) NOT NULL,
+	rb bitmap NOT NULL
+) ENGINE=OLAP
+    PRIMARY KEY(c1, tag_name, tag_value)
+PARTITION BY (c1)
+DISTRIBUTED BY HASH(tag_name, tag_value)
+PROPERTIES (
+"replication_num" = "1"
+);
+-- result:
+-- !result
+insert into test_tags(c1, tag_name, tag_value, rb) SELECT '20250114050000','a','57',bitmap_from_string("57");
+-- result:
+-- !result
+insert into test_tags(c1, tag_name, tag_value, rb) SELECT '20250114050000','a','a',bitmap_from_string("57,22253296,29101576,43027104");
+-- result:
+-- !result
+WITH result AS (SELECT rb FROM test_tags),
+     page_result AS (SELECT sub_bitmap(bitmap_union ( rb ), 0, 20 ) rb FROM result),
+     all_tag AS (SELECT tag_name, tag_value, c1, bitmap_and (t0.rb, t1.rb) rb2 FROM test_tags t0 JOIN page_result t1 WHERE bitmap_has_any ( t0.rb, t1.rb))
+SELECT tag_name, bitmap_to_string(rb2) as row_id_ FROM all_tag order by 1, 2;
+-- result:
+a	57
+a	57,22253296,29101576,43027104
+-- !result
+WITH result AS (SELECT rb FROM test_tags),
+     page_result AS (SELECT sub_bitmap(bitmap_union ( rb ), 0, 20 ) rb FROM result),
+     all_tag AS (SELECT tag_name, tag_value, c1, bitmap_and (t0.rb, t1.rb) rb2 FROM test_tags t0 JOIN page_result t1 WHERE bitmap_has_any ( t0.rb, t1.rb))
+SELECT tag_name, bitmap_to_array(rb2) as row_id_ FROM all_tag order by 1, 2;
+-- result:
+a	[57]
+a	[57,22253296,29101576,43027104]
+-- !result
+WITH result AS (SELECT rb FROM test_tags),
+     page_result AS (SELECT sub_bitmap(bitmap_union ( rb ), 0, 20 ) rb FROM result),
+     all_tag AS (SELECT tag_name, tag_value, c1, bitmap_and (t0.rb, t1.rb) rb2 FROM test_tags t0 JOIN page_result t1 WHERE bitmap_has_any ( t0.rb, t1.rb))
+SELECT tag_name, unnest as row_id_ FROM all_tag, unnest (bitmap_to_array(rb2)) order by 1, 2;
+-- result:
+a	57
+a	57
+a	22253296
+a	29101576
+a	43027104
 -- !result
\ No newline at end of file
diff --git a/test/sql/test_bitmap_functions/T/test_unnest_bitmap b/test/sql/test_bitmap_functions/T/test_unnest_bitmap
index c6dbfb37e929b8..b322e85a484449 100644
--- a/test/sql/test_bitmap_functions/T/test_unnest_bitmap
+++ b/test/sql/test_bitmap_functions/T/test_unnest_bitmap
@@ -16,3 +16,32 @@ insert into t1 select 5, null;
 select sum(c1), sum(unnest_bitmap) from t1, unnest_bitmap(c2);
 select c1, unnest_bitmap as c3 from t1, unnest_bitmap(c2) order by c1 asc, c3 asc limit 5;
 select c1, unnest_bitmap as c3 from t1, unnest_bitmap(c2) order by c1 desc, c3 desc limit 5;
+
+set enable_rewrite_unnest_bitmap_to_array=true;
+CREATE TABLE test_tags (
+	c1 varchar(65533) NOT NULL,
+	tag_name varchar(65533) NOT NULL,
+	tag_value varchar(65533) NOT NULL,
+	rb bitmap NOT NULL
+) ENGINE=OLAP
+    PRIMARY KEY(c1, tag_name, tag_value)
+PARTITION BY (c1)
+DISTRIBUTED BY HASH(tag_name, tag_value)
+PROPERTIES (
+"replication_num" = "1"
+);
+insert into test_tags(c1, tag_name, tag_value, rb) SELECT '20250114050000','a','57',bitmap_from_string("57");
+insert into test_tags(c1, tag_name, tag_value, rb) SELECT '20250114050000','a','a',bitmap_from_string("57,22253296,29101576,43027104");
+
+WITH result AS (SELECT rb FROM test_tags),
+     page_result AS (SELECT sub_bitmap(bitmap_union ( rb ), 0, 20 ) rb FROM result),
+     all_tag AS (SELECT tag_name, tag_value, c1, bitmap_and (t0.rb, t1.rb) rb2 FROM test_tags t0 JOIN page_result t1 WHERE bitmap_has_any ( t0.rb, t1.rb))
+SELECT tag_name, bitmap_to_string(rb2) as row_id_ FROM all_tag order by 1, 2;
+WITH result AS (SELECT rb FROM test_tags),
+     page_result AS (SELECT sub_bitmap(bitmap_union ( rb ), 0, 20 ) rb FROM result),
+     all_tag AS (SELECT tag_name, tag_value, c1, bitmap_and (t0.rb, t1.rb) rb2 FROM test_tags t0 JOIN page_result t1 WHERE bitmap_has_any ( t0.rb, t1.rb))
+SELECT tag_name, bitmap_to_array(rb2) as row_id_ FROM all_tag order by 1, 2;
+WITH result AS (SELECT rb FROM test_tags),
+     page_result AS (SELECT sub_bitmap(bitmap_union ( rb ), 0, 20 ) rb FROM result),
+     all_tag AS (SELECT tag_name, tag_value, c1, bitmap_and (t0.rb, t1.rb) rb2 FROM test_tags t0 JOIN page_result t1 WHERE bitmap_has_any ( t0.rb, t1.rb))
+SELECT tag_name, unnest as row_id_ FROM all_tag, unnest (bitmap_to_array(rb2)) order by 1, 2;
\ No newline at end of file

From 52aa95f6c6336f624617e23ddb846087f4d05196 Mon Sep 17 00:00:00 2001
From: trueeyu <lxhhust350@qq.com>
Date: Fri, 17 Jan 2025 14:02:34 +0800
Subject: [PATCH 32/71] [Enhancement] TopN runtime filter for string support
 update (#55122)

Signed-off-by: trueeyu <lxhhust350@qq.com>
---
 be/src/exec/olap_scan_prepare.cpp             |  12 +-
 be/src/exprs/min_max_predicate.h              |   4 +-
 be/src/exprs/runtime_filter.h                 |  66 +++++++--
 be/src/formats/orc/orc_chunk_reader.cpp       | 110 +++++++--------
 be/src/storage/runtime_range_pruner.hpp       |  33 +++--
 be/test/exprs/runtime_filter_test.cpp         | 128 ++++++++++--------
 .../olap_runtime_range_pruner_test.cpp        |   8 +-
 7 files changed, 211 insertions(+), 150 deletions(-)

diff --git a/be/src/exec/olap_scan_prepare.cpp b/be/src/exec/olap_scan_prepare.cpp
index 1b78017e033e10..a6d744d52f1474 100644
--- a/be/src/exec/olap_scan_prepare.cpp
+++ b/be/src/exec/olap_scan_prepare.cpp
@@ -590,8 +590,8 @@ void ChunkPredicateBuilder<E, Type>::normalized_rf_with_null(const JoinRuntimeFi
                                                                                                              &decoder);
     const TypeDescriptor& col_type = col_ref->type();
 
-    ColumnPtr const_min_col = parser.template min_const_column<SlotType>(col_type);
-    ColumnPtr const_max_col = parser.template max_const_column<SlotType>(col_type);
+    ColumnPtr const_min_col = parser.template min_const_column<SlotType>(col_type, pool);
+    ColumnPtr const_max_col = parser.template max_const_column<SlotType>(col_type, pool);
     VectorizedLiteral* min_literal = pool->add(new VectorizedLiteral(std::move(const_min_col), col_type));
     VectorizedLiteral* max_literal = pool->add(new VectorizedLiteral(std::move(const_max_col), col_type));
 
@@ -719,7 +719,7 @@ Status ChunkPredicateBuilder<E, Type>::normalize_join_runtime_filter(const SlotD
                 } else {
                     detail::RuntimeColumnPredicateBuilder::build_minmax_range<
                             RangeType, SlotType, LowCardDictType,
-                            detail::RuntimeColumnPredicateBuilder::GlobalDictCodeDecoder>(*range, rf,
+                            detail::RuntimeColumnPredicateBuilder::GlobalDictCodeDecoder>(*range, rf, _opts.obj_pool,
                                                                                           &iter->second.first);
                 }
             } else {
@@ -729,7 +729,7 @@ Status ChunkPredicateBuilder<E, Type>::normalize_join_runtime_filter(const SlotD
                 } else {
                     detail::RuntimeColumnPredicateBuilder::build_minmax_range<
                             RangeType, SlotType, SlotType, detail::RuntimeColumnPredicateBuilder::DummyDecoder>(
-                            *range, rf, nullptr);
+                            *range, rf, _opts.obj_pool, nullptr);
                 }
             }
         } else {
@@ -738,8 +738,8 @@ Status ChunkPredicateBuilder<E, Type>::normalize_join_runtime_filter(const SlotD
                         rf, desc->probe_expr_ctx()->root(), nullptr);
             } else {
                 detail::RuntimeColumnPredicateBuilder::build_minmax_range<
-                        RangeType, SlotType, SlotType, detail::RuntimeColumnPredicateBuilder::DummyDecoder>(*range, rf,
-                                                                                                            nullptr);
+                        RangeType, SlotType, SlotType, detail::RuntimeColumnPredicateBuilder::DummyDecoder>(
+                        *range, rf, _opts.obj_pool, nullptr);
             }
         }
     }
diff --git a/be/src/exprs/min_max_predicate.h b/be/src/exprs/min_max_predicate.h
index bf5a517be9a853..5d0ae3088f6887 100644
--- a/be/src/exprs/min_max_predicate.h
+++ b/be/src/exprs/min_max_predicate.h
@@ -140,8 +140,8 @@ class MinMaxPredicateBuilder {
     template <LogicalType ltype>
     Expr* operator()() {
         auto* bloom_filter = (RuntimeBloomFilter<ltype>*)(_filter);
-        return _pool->add(new MinMaxPredicate<ltype>(_slot_id, bloom_filter->min_value(), bloom_filter->max_value(),
-                                                     bloom_filter->has_null()));
+        return _pool->add(new MinMaxPredicate<ltype>(_slot_id, bloom_filter->min_value(_pool),
+                                                     bloom_filter->max_value(_pool), bloom_filter->has_null()));
     }
 
 private:
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index 50210778078f7d..771db1905331f2 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -401,8 +401,13 @@ class RuntimeBloomFilter final : public JoinRuntimeFilter {
         p->init(1);
 
         if constexpr (IsSlice<CppType>) {
-            p->_slice_min = val.to_string();
-            val = Slice(p->_slice_min.data(), val.get_size());
+            if constexpr (is_min) {
+                p->_slice_min = val.to_string();
+                val = Slice(p->_slice_min.data(), p->_slice_min.size());
+            } else {
+                p->_slice_max = val.to_string();
+                val = Slice(p->_slice_max.data(), p->_slice_max.size());
+            }
         }
 
         if constexpr (is_min) {
@@ -431,18 +436,33 @@ class RuntimeBloomFilter final : public JoinRuntimeFilter {
     void update_min_max(CppType val) {
         // now slice have not support update min/max
         if constexpr (IsSlice<CppType>) {
-            return;
-        }
-
-        if constexpr (is_min) {
-            if (_min < val) {
-                _min = val;
-                _update_version();
+            std::lock_guard<std::mutex> lk(_slice_mutex);
+            if constexpr (is_min) {
+                if (_min < val) {
+                    _slice_min = val.to_string();
+                    _min.data = _slice_min.data();
+                    _min.size = _slice_min.size();
+                    _update_version();
+                }
+            } else {
+                if (_max > val) {
+                    _slice_max = val.to_string();
+                    _max.data = _slice_max.data();
+                    _max.size = _slice_max.size();
+                    _update_version();
+                }
             }
         } else {
-            if (_max > val) {
-                _max = val;
-                _update_version();
+            if constexpr (is_min) {
+                if (_min < val) {
+                    _min = val;
+                    _update_version();
+                }
+            } else {
+                if (_max > val) {
+                    _max = val;
+                    _update_version();
+                }
             }
         }
     }
@@ -490,9 +510,25 @@ class RuntimeBloomFilter final : public JoinRuntimeFilter {
 
     void insert_null() { _has_null = true; }
 
-    CppType min_value() const { return _min; }
+    CppType min_value(ObjectPool* pool) const {
+        if constexpr (IsSlice<CppType>) {
+            std::lock_guard<std::mutex> lk(_slice_mutex);
+            auto* str = pool->template add(new std::string(_min.get_data(), _min.get_size()));
+            return Slice(*str);
+        } else {
+            return _min;
+        }
+    }
 
-    CppType max_value() const { return _max; }
+    CppType max_value(ObjectPool* pool) const {
+        if constexpr (IsSlice<CppType>) {
+            std::lock_guard<std::mutex> lk(_slice_mutex);
+            auto* str = pool->template add(new std::string(_max.get_data(), _max.get_size()));
+            return Slice(*str);
+        } else {
+            return _max;
+        }
+    }
 
     void set_left_close_interval(bool close_interval) { _left_close_interval = close_interval; }
     void set_right_close_interval(bool close_interval) { _right_close_interval = close_interval; }
@@ -808,6 +844,7 @@ class RuntimeBloomFilter final : public JoinRuntimeFilter {
             _max = std::max(_max, bf->_max);
 
             if constexpr (IsSlice<CppType>) {
+                std::lock_guard<std::mutex> lk(_slice_mutex);
                 // maybe we are refering to another runtime filter instance
                 // for security we have to copy that back to our instance.
                 if (_min.size != 0 && _min.data != _slice_min.data()) {
@@ -922,6 +959,7 @@ class RuntimeBloomFilter final : public JoinRuntimeFilter {
     CppType _max;
     std::string _slice_min;
     std::string _slice_max;
+    mutable std::mutex _slice_mutex;
     bool _has_min_max = true;
     bool _left_close_interval = true;
     bool _right_close_interval = true;
diff --git a/be/src/formats/orc/orc_chunk_reader.cpp b/be/src/formats/orc/orc_chunk_reader.cpp
index 0d5d8c9234cd05..4a66f2362999a2 100644
--- a/be/src/formats/orc/orc_chunk_reader.cpp
+++ b/be/src/formats/orc/orc_chunk_reader.cpp
@@ -1051,8 +1051,8 @@ Status OrcChunkReader::_add_conjunct(const Expr* conjunct,
     case type: {                                                       \
         auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf); \
         if (xrf == nullptr) return false;                              \
-        auto lower = orc::Literal(bool(xrf->min_value()));             \
-        auto upper = orc::Literal(bool(xrf->max_value()));             \
+        auto lower = orc::Literal(bool(xrf->min_value(&_pool)));       \
+        auto upper = orc::Literal(bool(xrf->max_value(&_pool)));       \
         ADD_RF_TO_BUILDER                                              \
     }
 
@@ -1060,8 +1060,8 @@ Status OrcChunkReader::_add_conjunct(const Expr* conjunct,
     case type: {                                                       \
         auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf); \
         if (xrf == nullptr) return false;                              \
-        auto lower = orc::Literal(int64_t(xrf->min_value()));          \
-        auto upper = orc::Literal(int64_t(xrf->max_value()));          \
+        auto lower = orc::Literal(int64_t(xrf->min_value(&_pool)));    \
+        auto upper = orc::Literal(int64_t(xrf->max_value(&_pool)));    \
         ADD_RF_TO_BUILDER                                              \
     }
 
@@ -1069,60 +1069,60 @@ Status OrcChunkReader::_add_conjunct(const Expr* conjunct,
     case type: {                                                       \
         auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf); \
         if (xrf == nullptr) return false;                              \
-        auto lower = orc::Literal(double(xrf->min_value()));           \
-        auto upper = orc::Literal(double(xrf->max_value()));           \
+        auto lower = orc::Literal(double(xrf->min_value(&_pool)));     \
+        auto upper = orc::Literal(double(xrf->max_value(&_pool)));     \
         ADD_RF_TO_BUILDER                                              \
     }
 
-#define ADD_RF_STRING_TYPE(type)                                                 \
-    case type: {                                                                 \
-        auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf);           \
-        if (xrf == nullptr) return false;                                        \
-        auto lower = orc::Literal(xrf->min_value().data, xrf->min_value().size); \
-        auto upper = orc::Literal(xrf->max_value().data, xrf->max_value().size); \
-        ADD_RF_TO_BUILDER                                                        \
-    }
-
-#define ADD_RF_DATE_TYPE(type)                                                                                        \
-    case type: {                                                                                                      \
-        auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf);                                                \
-        if (xrf == nullptr) return false;                                                                             \
-        auto lower =                                                                                                  \
-                orc::Literal(orc::PredicateDataType::DATE, OrcDateHelper::native_date_to_orc_date(xrf->min_value())); \
-        auto upper =                                                                                                  \
-                orc::Literal(orc::PredicateDataType::DATE, OrcDateHelper::native_date_to_orc_date(xrf->max_value())); \
-        ADD_RF_TO_BUILDER                                                                                             \
-    }
-
-#define ADD_RF_DECIMALV2_TYPE(type)                                                                                    \
-    case type: {                                                                                                       \
-        auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf);                                                 \
-        if (xrf == nullptr) return false;                                                                              \
-        auto lower =                                                                                                   \
-                orc::Literal(to_orc128(xrf->min_value().value()), xrf->min_value().PRECISION, xrf->min_value().SCALE); \
-        auto upper =                                                                                                   \
-                orc::Literal(to_orc128(xrf->max_value().value()), xrf->max_value().PRECISION, xrf->max_value().SCALE); \
-        ADD_RF_TO_BUILDER                                                                                              \
-    }
-
-#define ADD_RF_DECIMALV3_TYPE(xtype)                                                                          \
-    case xtype: {                                                                                             \
-        auto* xrf = dynamic_cast<const RuntimeBloomFilter<xtype>*>(rf);                                       \
-        if (xrf == nullptr) return false;                                                                     \
-        auto lower = orc::Literal(orc::Int128(xrf->min_value()), slot->type().precision, slot->type().scale); \
-        auto upper = orc::Literal(orc::Int128(xrf->max_value()), slot->type().precision, slot->type().scale); \
-        ADD_RF_TO_BUILDER                                                                                     \
-    }
-
-#define ADD_RF_DECIMAL128_TYPE(xtype)                                                                            \
-    case xtype: {                                                                                                \
-        auto* xrf = dynamic_cast<const RuntimeBloomFilter<xtype>*>(rf);                                          \
-        if (xrf == nullptr) return false;                                                                        \
-        auto lower = orc::Literal(orc::Int128(xrf->min_value() >> 64, xrf->min_value()), slot->type().precision, \
-                                  slot->type().scale);                                                           \
-        auto upper = orc::Literal(orc::Int128(xrf->max_value() >> 64, xrf->max_value()), slot->type().precision, \
-                                  slot->type().scale);                                                           \
-        ADD_RF_TO_BUILDER                                                                                        \
+#define ADD_RF_STRING_TYPE(type)                                                             \
+    case type: {                                                                             \
+        auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf);                       \
+        if (xrf == nullptr) return false;                                                    \
+        auto lower = orc::Literal(xrf->min_value(&_pool).data, xrf->min_value(&_pool).size); \
+        auto upper = orc::Literal(xrf->max_value(&_pool).data, xrf->max_value(&_pool).size); \
+        ADD_RF_TO_BUILDER                                                                    \
+    }
+
+#define ADD_RF_DATE_TYPE(type)                                                                     \
+    case type: {                                                                                   \
+        auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf);                             \
+        if (xrf == nullptr) return false;                                                          \
+        auto lower = orc::Literal(orc::PredicateDataType::DATE,                                    \
+                                  OrcDateHelper::native_date_to_orc_date(xrf->min_value(&_pool))); \
+        auto upper = orc::Literal(orc::PredicateDataType::DATE,                                    \
+                                  OrcDateHelper::native_date_to_orc_date(xrf->max_value(&_pool))); \
+        ADD_RF_TO_BUILDER                                                                          \
+    }
+
+#define ADD_RF_DECIMALV2_TYPE(type)                                                                            \
+    case type: {                                                                                               \
+        auto* xrf = dynamic_cast<const RuntimeBloomFilter<type>*>(rf);                                         \
+        if (xrf == nullptr) return false;                                                                      \
+        auto lower = orc::Literal(to_orc128(xrf->min_value(&_pool).value()), xrf->min_value(&_pool).PRECISION, \
+                                  xrf->min_value(&_pool).SCALE);                                               \
+        auto upper = orc::Literal(to_orc128(xrf->max_value(&_pool).value()), xrf->max_value(&_pool).PRECISION, \
+                                  xrf->max_value(&_pool).SCALE);                                               \
+        ADD_RF_TO_BUILDER                                                                                      \
+    }
+
+#define ADD_RF_DECIMALV3_TYPE(xtype)                                                                                \
+    case xtype: {                                                                                                   \
+        auto* xrf = dynamic_cast<const RuntimeBloomFilter<xtype>*>(rf);                                             \
+        if (xrf == nullptr) return false;                                                                           \
+        auto lower = orc::Literal(orc::Int128(xrf->min_value(&_pool)), slot->type().precision, slot->type().scale); \
+        auto upper = orc::Literal(orc::Int128(xrf->max_value(&_pool)), slot->type().precision, slot->type().scale); \
+        ADD_RF_TO_BUILDER                                                                                           \
+    }
+
+#define ADD_RF_DECIMAL128_TYPE(xtype)                                                                \
+    case xtype: {                                                                                    \
+        auto* xrf = dynamic_cast<const RuntimeBloomFilter<xtype>*>(rf);                              \
+        if (xrf == nullptr) return false;                                                            \
+        auto lower = orc::Literal(orc::Int128(xrf->min_value(&_pool) >> 64, xrf->min_value(&_pool)), \
+                                  slot->type().precision, slot->type().scale);                       \
+        auto upper = orc::Literal(orc::Int128(xrf->max_value(&_pool) >> 64, xrf->max_value(&_pool)), \
+                                  slot->type().precision, slot->type().scale);                       \
+        ADD_RF_TO_BUILDER                                                                            \
     }
 
 bool OrcChunkReader::_add_runtime_filter(const uint64_t column_id, const SlotDescriptor* slot,
diff --git a/be/src/storage/runtime_range_pruner.hpp b/be/src/storage/runtime_range_pruner.hpp
index 147628faa163be..3c93cd25c50025 100644
--- a/be/src/storage/runtime_range_pruner.hpp
+++ b/be/src/storage/runtime_range_pruner.hpp
@@ -70,13 +70,13 @@ struct RuntimeColumnPredicateBuilder {
             if constexpr (ltype == TYPE_VARCHAR) {
                 auto cid = parser->column_id(*slot);
                 if (auto iter = global_dictmaps->find(cid); iter != global_dictmaps->end()) {
-                    build_minmax_range<RangeType, limit_type, LowCardDictType, GlobalDictCodeDecoder>(range, rf,
+                    build_minmax_range<RangeType, limit_type, LowCardDictType, GlobalDictCodeDecoder>(range, rf, pool,
                                                                                                       iter->second);
                 } else {
-                    build_minmax_range<RangeType, limit_type, mapping_type, DummyDecoder>(range, rf, nullptr);
+                    build_minmax_range<RangeType, limit_type, mapping_type, DummyDecoder>(range, rf, pool, nullptr);
                 }
             } else {
-                build_minmax_range<RangeType, limit_type, mapping_type, DummyDecoder>(range, rf, nullptr);
+                build_minmax_range<RangeType, limit_type, mapping_type, DummyDecoder>(range, rf, pool, nullptr);
             }
 
             std::vector<TCondition> filters;
@@ -155,18 +155,18 @@ struct RuntimeColumnPredicateBuilder {
     struct MinMaxParser {
         MinMaxParser(const RuntimeFilter* runtime_filter_, Decoder* decoder)
                 : runtime_filter(runtime_filter_), decoder(decoder) {}
-        auto min_value() {
-            auto code = runtime_filter->min_value();
+        auto min_value(ObjectPool* pool) {
+            auto code = runtime_filter->min_value(pool);
             return decoder->decode(code);
         }
-        auto max_value() {
-            auto code = runtime_filter->max_value();
+        auto max_value(ObjectPool* pool) {
+            auto code = runtime_filter->max_value(pool);
             return decoder->decode(code);
         }
 
         template <LogicalType Type>
-        ColumnPtr min_const_column(const TypeDescriptor& col_type) {
-            auto min_decode_value = min_value();
+        ColumnPtr min_const_column(const TypeDescriptor& col_type, ObjectPool* pool) {
+            auto min_decode_value = min_value(pool);
             if constexpr (lt_is_decimal<Type>) {
                 return ColumnHelper::create_const_decimal_column<Type>(min_decode_value, col_type.precision,
                                                                        col_type.scale, 1);
@@ -176,8 +176,8 @@ struct RuntimeColumnPredicateBuilder {
         }
 
         template <LogicalType Type>
-        ColumnPtr max_const_column(const TypeDescriptor& col_type) {
-            auto max_decode_value = max_value();
+        ColumnPtr max_const_column(const TypeDescriptor& col_type, ObjectPool* pool) {
+            auto max_decode_value = max_value(pool);
             if constexpr (lt_is_decimal<Type>) {
                 return ColumnHelper::create_const_decimal_column<Type>(max_decode_value, col_type.precision,
                                                                        col_type.scale, 1);
@@ -193,20 +193,25 @@ struct RuntimeColumnPredicateBuilder {
 
     template <class Range, LogicalType SlotType, LogicalType mapping_type, template <class> class Decoder,
               class... Args>
-    static void build_minmax_range(Range& range, const JoinRuntimeFilter* rf, Args&&... args) {
+    static void build_minmax_range(Range& range, const JoinRuntimeFilter* rf, ObjectPool* pool, Args&&... args) {
         using ValueType = typename RunTimeTypeTraits<SlotType>::CppType;
 
         const RuntimeBloomFilter<mapping_type>* filter = down_cast<const RuntimeBloomFilter<mapping_type>*>(rf);
         using DecoderType = Decoder<typename RunTimeTypeTraits<mapping_type>::CppType>;
         DecoderType decoder(std::forward<Args>(args)...);
         MinMaxParser<RuntimeBloomFilter<mapping_type>, DecoderType> parser(filter, &decoder);
+        if (filter->is_empty_range()) {
+            range.clear_to_empty();
+            return;
+        }
+
         SQLFilterOp min_op;
         if (filter->left_close_interval()) {
             min_op = to_olap_filter_type(TExprOpcode::GE, false);
         } else {
             min_op = to_olap_filter_type(TExprOpcode::GT, false);
         }
-        auto min_value = parser.min_value();
+        auto min_value = parser.min_value(pool);
         (void)range.add_range(min_op, static_cast<ValueType>(min_value));
 
         SQLFilterOp max_op;
@@ -216,7 +221,7 @@ struct RuntimeColumnPredicateBuilder {
             max_op = to_olap_filter_type(TExprOpcode::LT, false);
         }
 
-        auto max_value = parser.max_value();
+        auto max_value = parser.max_value(pool);
         (void)range.add_range(max_op, static_cast<ValueType>(max_value));
     }
 };
diff --git a/be/test/exprs/runtime_filter_test.cpp b/be/test/exprs/runtime_filter_test.cpp
index 1b204e77b1d639..6e85e9761327cc 100644
--- a/be/test/exprs/runtime_filter_test.cpp
+++ b/be/test/exprs/runtime_filter_test.cpp
@@ -42,6 +42,7 @@ class RuntimeBloomFilterTest : public ::testing::Test {
     void _check_equal(const Filter& real, const std::vector<uint8_t>& expect);
 
     using Int32RF = RuntimeBloomFilter<TYPE_INT>;
+    using StringRF = RuntimeBloomFilter<TYPE_VARCHAR>;
     ObjectPool _pool;
 };
 
@@ -52,6 +53,24 @@ void RuntimeBloomFilterTest::_check_equal(const Filter& real, const std::vector<
     }
 }
 
+TEST_F(RuntimeBloomFilterTest, create_with_range) {
+    auto* rf = StringRF::create_with_range<true>(&_pool, "00001", true);
+    ASSERT_EQ(rf->min_value(&_pool), Slice("00001"));
+    ASSERT_TRUE(rf->left_close_interval());
+
+    rf = StringRF::create_with_range<false>(&_pool, "00009", true);
+    ASSERT_EQ(rf->max_value(&_pool), Slice("00009"));
+    ASSERT_TRUE(rf->right_close_interval());
+
+    auto* int_rf = Int32RF::create_with_range<true>(&_pool, 1, true);
+    ASSERT_EQ(int_rf->min_value(&_pool), 1);
+    ASSERT_TRUE(int_rf->left_close_interval());
+
+    int_rf = Int32RF::create_with_range<false>(&_pool, 9, true);
+    ASSERT_EQ(int_rf->max_value(&_pool), 9);
+    ASSERT_TRUE(int_rf->right_close_interval());
+}
+
 TEST_F(RuntimeBloomFilterTest, evaluate_with_min_max) {
     // [10, 20]
     auto* rf = _pool.add(new Int32RF());
@@ -140,10 +159,10 @@ TEST_F(RuntimeBloomFilterTest, create_with_full_range_without_null) {
     ASSERT_FALSE(rf->has_null());
 }
 
-TEST_F(RuntimeBloomFilterTest, create_with_range) {
+TEST_F(RuntimeBloomFilterTest, create_with_range_nullable) {
     auto* rf = Int32RF::create_with_range<true>(&_pool, 10, true, true);
-    ASSERT_EQ(rf->min_value(), 10);
-    ASSERT_EQ(rf->max_value(), std::numeric_limits<int32_t>::max());
+    ASSERT_EQ(rf->min_value(&_pool), 10);
+    ASSERT_EQ(rf->max_value(&_pool), std::numeric_limits<int32_t>::max());
     ASSERT_TRUE(rf->has_null());
 }
 
@@ -248,15 +267,15 @@ static std::shared_ptr<BinaryColumn> gen_random_binary_column(const std::string&
     return col;
 }
 
-TEST_F(RuntimeFilterTest, TestJoinRuntimeFilter) {
-    RuntimeBloomFilter<TYPE_INT> bf;
+TEST_F(RuntimeBloomFilterTest, TestJoinRuntimeFilter) {
+    Int32RF bf;
     JoinRuntimeFilter* rf = &bf;
     bf.init(100);
     for (int i = 0; i <= 200; i += 17) {
         bf.insert(i);
     }
-    EXPECT_EQ(bf.min_value(), 0);
-    EXPECT_EQ(bf.max_value(), 187);
+    EXPECT_EQ(bf.min_value(&_pool), 0);
+    EXPECT_EQ(bf.max_value(&_pool), 187);
     for (int i = 0; i <= 200; i += 17) {
         EXPECT_TRUE(bf.test_data(i));
         EXPECT_FALSE(bf.test_data(i + 1));
@@ -264,12 +283,11 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilter) {
     EXPECT_FALSE(rf->has_null());
     bf.insert_null();
     EXPECT_TRUE(rf->has_null());
-    EXPECT_EQ(bf.min_value(), 0);
-    EXPECT_EQ(bf.max_value(), 187);
+    EXPECT_EQ(bf.min_value(&_pool), 0);
+    EXPECT_EQ(bf.max_value(&_pool), 187);
 
     // test evaluate.
-    TypeDescriptor type_desc(TYPE_INT);
-    ColumnPtr column = ColumnHelper::create_column(type_desc, false);
+    ColumnPtr column = ColumnHelper::create_column(TYPE_INT_DESC, false);
     auto* col = ColumnHelper::as_raw_column<RunTimeTypeTraits<TYPE_INT>::ColumnType>(column);
     for (int i = 0; i <= 200; i += 1) {
         col->append(i);
@@ -289,15 +307,15 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilter) {
     EXPECT_EQ(chunk.num_rows(), 12);
 }
 
-TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterSlice) {
-    RuntimeBloomFilter<TYPE_VARCHAR> bf;
+TEST_F(RuntimeBloomFilterTest, TestJoinRuntimeFilterSlice) {
+    StringRF bf;
     bf.init(100);
     std::vector<Slice> values{"aa", "bb", "cc", "d"};
     for (auto& s : values) {
         bf.insert(s);
     }
-    EXPECT_EQ(bf.min_value(), values[0]);
-    EXPECT_EQ(bf.max_value(), values[values.size() - 1]);
+    EXPECT_EQ(bf.min_value(&_pool), values[0]);
+    EXPECT_EQ(bf.max_value(&_pool), values[values.size() - 1]);
     for (auto& s : values) {
         EXPECT_TRUE(bf.test_data(s));
     }
@@ -327,17 +345,17 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterSerialize) {
     EXPECT_TRUE(rf1->check_equal(*rf0));
 }
 
-TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterSerialize2) {
-    RuntimeBloomFilter<TYPE_INT> bf0;
+TEST_F(RuntimeBloomFilterTest, TestJoinRuntimeFilterSerialize2) {
+    Int32RF bf0;
     JoinRuntimeFilter* rf0 = &bf0;
     bf0.init(100);
     for (int i = 0; i <= 200; i += 17) {
         bf0.insert(i);
     }
-    EXPECT_EQ(bf0.min_value(), 0);
-    EXPECT_EQ(bf0.max_value(), 187);
+    EXPECT_EQ(bf0.min_value(&_pool), 0);
+    EXPECT_EQ(bf0.max_value(&_pool), 187);
 
-    RuntimeBloomFilter<TYPE_VARCHAR> bf1;
+    StringRF bf1;
     JoinRuntimeFilter* rf1 = &bf1;
     std::vector<std::string> data = {"aa", "bb", "cc", "dd"};
     std::vector<Slice> values;
@@ -348,8 +366,8 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterSerialize2) {
     for (auto& s : values) {
         bf1.insert(s);
     }
-    EXPECT_EQ(bf1.min_value(), values[0]);
-    EXPECT_EQ(bf1.max_value(), values[values.size() - 1]);
+    EXPECT_EQ(bf1.min_value(&_pool), values[0]);
+    EXPECT_EQ(bf1.max_value(&_pool), values[values.size() - 1]);
 
     int rf_version = RF_VERSION_V2;
 
@@ -372,15 +390,15 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterSerialize2) {
     EXPECT_TRUE(rf3->check_equal(*rf1));
 }
 
-TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge) {
-    RuntimeBloomFilter<TYPE_INT> bf0;
+TEST_F(RuntimeBloomFilterTest, TestJoinRuntimeFilterMerge) {
+    Int32RF bf0;
     JoinRuntimeFilter* rf0 = &bf0;
     bf0.init(100);
     for (int i = 0; i <= 200; i += 17) {
         bf0.insert(i);
     }
-    EXPECT_EQ(bf0.min_value(), 0);
-    EXPECT_EQ(bf0.max_value(), 187);
+    EXPECT_EQ(bf0.min_value(&_pool), 0);
+    EXPECT_EQ(bf0.max_value(&_pool), 187);
 
     RuntimeBloomFilter<TYPE_INT> bf1;
     JoinRuntimeFilter* rf1 = &bf1;
@@ -388,8 +406,8 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge) {
     for (int i = 1; i <= 200; i += 17) {
         bf1.insert(i);
     }
-    EXPECT_EQ(bf1.min_value(), 1);
-    EXPECT_EQ(bf1.max_value(), 188);
+    EXPECT_EQ(bf1.min_value(&_pool), 1);
+    EXPECT_EQ(bf1.max_value(&_pool), 188);
 
     RuntimeBloomFilter<TYPE_INT> bf2;
     bf2.init(100);
@@ -400,12 +418,12 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge) {
         EXPECT_TRUE(bf2.test_data(i + 1));
         EXPECT_FALSE(bf2.test_data(i + 2));
     }
-    EXPECT_EQ(bf2.min_value(), 0);
-    EXPECT_EQ(bf2.max_value(), 188);
+    EXPECT_EQ(bf2.min_value(&_pool), 0);
+    EXPECT_EQ(bf2.max_value(&_pool), 188);
 }
 
-TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge2) {
-    RuntimeBloomFilter<TYPE_VARCHAR> bf0;
+TEST_F(RuntimeBloomFilterTest, TestJoinRuntimeFilterMerge2) {
+    StringRF bf0;
     JoinRuntimeFilter* rf0 = &bf0;
     std::vector<std::string> data = {"bb", "cc", "dd"};
     {
@@ -418,11 +436,11 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge2) {
             bf0.insert(s);
         }
         // bb - dd
-        EXPECT_EQ(bf0.min_value(), values[0]);
-        EXPECT_EQ(bf0.max_value(), values[values.size() - 1]);
+        EXPECT_EQ(bf0.min_value(&_pool), values[0]);
+        EXPECT_EQ(bf0.max_value(&_pool), values[values.size() - 1]);
     }
 
-    RuntimeBloomFilter<TYPE_VARCHAR> bf1;
+    StringRF bf1;
     JoinRuntimeFilter* rf1 = &bf1;
     std::vector<std::string> data2 = {"aa", "bb", "cc", "dc"};
 
@@ -436,18 +454,18 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge2) {
             bf1.insert(s);
         }
         // aa - dc
-        EXPECT_EQ(bf1.min_value(), values[0]);
-        EXPECT_EQ(bf1.max_value(), values[values.size() - 1]);
+        EXPECT_EQ(bf1.min_value(&_pool), values[0]);
+        EXPECT_EQ(bf1.max_value(&_pool), values[values.size() - 1]);
     }
 
     // range aa - dd
     rf0->merge(rf1);
-    EXPECT_EQ(bf0.min_value(), Slice("aa", 2));
-    EXPECT_EQ(bf0.max_value(), Slice("dd", 2));
+    EXPECT_EQ(bf0.min_value(&_pool), Slice("aa", 2));
+    EXPECT_EQ(bf0.max_value(&_pool), Slice("dd", 2));
 }
 
-TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge3) {
-    RuntimeBloomFilter<TYPE_VARCHAR> bf0;
+TEST_F(RuntimeBloomFilterTest, TestJoinRuntimeFilterMerge3) {
+    StringRF bf0;
     JoinRuntimeFilter* rf0 = &bf0;
     ObjectPool pool;
     int rf_version = RF_VERSION_V2;
@@ -470,9 +488,9 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge3) {
         RuntimeFilterHelper::deserialize_runtime_filter(&pool, &rf0, (const uint8_t*)buf.data(), actual_size);
     }
 
-    auto* pbf0 = static_cast<RuntimeBloomFilter<TYPE_VARCHAR>*>(rf0);
-    EXPECT_EQ(pbf0->min_value(), Slice("bb", 2));
-    EXPECT_EQ(pbf0->max_value(), Slice("dd", 2));
+    auto* pbf0 = static_cast<StringRF*>(rf0);
+    EXPECT_EQ(pbf0->min_value(&_pool), Slice("bb", 2));
+    EXPECT_EQ(pbf0->max_value(&_pool), Slice("dd", 2));
 
     RuntimeBloomFilter<TYPE_VARCHAR> bf1;
     JoinRuntimeFilter* rf1 = &bf1;
@@ -494,15 +512,15 @@ TEST_F(RuntimeFilterTest, TestJoinRuntimeFilterMerge3) {
         RuntimeFilterHelper::deserialize_runtime_filter(&pool, &rf1, (const uint8_t*)buf.data(), actual_size);
     }
 
-    auto* pbf1 = static_cast<RuntimeBloomFilter<TYPE_VARCHAR>*>(rf1);
-    EXPECT_EQ(pbf1->min_value(), Slice("aa", 2));
-    EXPECT_EQ(pbf1->max_value(), Slice("dc", 2));
+    auto* pbf1 = static_cast<StringRF*>(rf1);
+    EXPECT_EQ(pbf1->min_value(&_pool), Slice("aa", 2));
+    EXPECT_EQ(pbf1->max_value(&_pool), Slice("dc", 2));
 
     // range aa - dd
     rf0->merge(rf1);
     // out of scope, we expect aa and dd would be still alive.
-    EXPECT_EQ(pbf0->min_value(), Slice("aa", 2));
-    EXPECT_EQ(pbf0->max_value(), Slice("dd", 2));
+    EXPECT_EQ(pbf0->min_value(&_pool), Slice("aa", 2));
+    EXPECT_EQ(pbf0->max_value(&_pool), Slice("dd", 2));
 }
 
 typedef std::function<void(BinaryColumn*, std::vector<uint32_t>&, std::vector<size_t>&)> PartitionByFunc;
@@ -914,13 +932,13 @@ TEST_F(RuntimeFilterTest, TestLocalHashBucketRuntimeFilterWithBucketAbsent2) {
     test_bucket_shuffle_grf_helper(3, 3, 4, {0, 1, 2, 0});
 }
 
-TEST_F(RuntimeFilterTest, TestGlobalRuntimeFilterMinMax) {
-    RuntimeBloomFilter<TYPE_INT> prototype;
+TEST_F(RuntimeBloomFilterTest, TestGlobalRuntimeFilterMinMax) {
+    Int32RF prototype;
     ObjectPool pool;
 
-    RuntimeBloomFilter<TYPE_INT>* global = prototype.create_empty(&pool);
+    auto* global = prototype.create_empty(&pool);
     for (int i = 0; i < 3; i++) {
-        RuntimeBloomFilter<TYPE_INT> local;
+        Int32RF local;
         local.init(10);
         for (int j = 0; j < 4; j++) {
             int value = (i + 1) * 10 + j;
@@ -928,8 +946,8 @@ TEST_F(RuntimeFilterTest, TestGlobalRuntimeFilterMinMax) {
         }
         global->concat(&local);
     }
-    EXPECT_EQ(global->min_value(), 10);
-    EXPECT_EQ(global->max_value(), 33);
+    EXPECT_EQ(global->min_value(&_pool), 10);
+    EXPECT_EQ(global->max_value(&_pool), 33);
 }
 
 void test_pipeline_level_helper(TRuntimeFilterBuildJoinMode::type join_mode, const RuntimeFilterLayout& layout,
diff --git a/be/test/storage/olap_runtime_range_pruner_test.cpp b/be/test/storage/olap_runtime_range_pruner_test.cpp
index 317f0753f4bdfb..6a967aa0359dab 100644
--- a/be/test/storage/olap_runtime_range_pruner_test.cpp
+++ b/be/test/storage/olap_runtime_range_pruner_test.cpp
@@ -75,8 +75,8 @@ TEST_F(OlapRuntimeRangePrunerTest, min_max_parser) {
     rf.insert(20);
 
     detail::RuntimeColumnPredicateBuilder::MinMaxParser<Int32RuntimeFilter, Int32Decoder> parser(&rf, &decoder);
-    ColumnPtr min_column = parser.min_const_column<TYPE_INT>(TYPE_INT_DESC);
-    ColumnPtr max_column = parser.max_const_column<TYPE_INT>(TYPE_INT_DESC);
+    ColumnPtr min_column = parser.min_const_column<TYPE_INT>(TYPE_INT_DESC, &_pool);
+    ColumnPtr max_column = parser.max_const_column<TYPE_INT>(TYPE_INT_DESC, &_pool);
     ASSERT_EQ(min_column->debug_string(), "CONST: 10 Size : 1");
     ASSERT_EQ(max_column->debug_string(), "CONST: 20 Size : 1");
 }
@@ -89,8 +89,8 @@ TEST_F(OlapRuntimeRangePrunerTest, min_max_parser_for_decimal) {
     rf.insert(20);
 
     detail::RuntimeColumnPredicateBuilder::MinMaxParser<Decimal32RuntimeFilter, Int32Decoder> parser(&rf, &decoder);
-    ColumnPtr min_column = parser.min_const_column<TYPE_DECIMAL32>(TYPE_DECIMAL32_DESC);
-    ColumnPtr max_column = parser.max_const_column<TYPE_DECIMAL32>(TYPE_DECIMAL32_DESC);
+    ColumnPtr min_column = parser.min_const_column<TYPE_DECIMAL32>(TYPE_DECIMAL32_DESC, &_pool);
+    ColumnPtr max_column = parser.max_const_column<TYPE_DECIMAL32>(TYPE_DECIMAL32_DESC, &_pool);
     ASSERT_EQ(min_column->debug_string(), "CONST: 0.0010 Size : 1");
     ASSERT_EQ(max_column->debug_string(), "CONST: 0.0020 Size : 1");
 }

From e798bdf8ff316f644df102fb53b9541e56a24a84 Mon Sep 17 00:00:00 2001
From: IHEII <jervisli@tencent.com>
Date: Fri, 17 Jan 2025 14:04:06 +0800
Subject: [PATCH 33/71] [Refactor] remove unused member in CascadeChunkMerger
 (#54791)

Signed-off-by: IHEII <jervisli@tencent.com>
---
 be/src/runtime/sorted_chunks_merger.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/be/src/runtime/sorted_chunks_merger.h b/be/src/runtime/sorted_chunks_merger.h
index 6fa54be2db8f04..780b9709ee93a0 100644
--- a/be/src/runtime/sorted_chunks_merger.h
+++ b/be/src/runtime/sorted_chunks_merger.h
@@ -127,7 +127,6 @@ class CascadeChunkMerger : public ChunkMerger {
 private:
     const std::vector<ExprContext*>* _sort_exprs;
     SortDescs _sort_desc;
-    std::vector<std::unique_ptr<SimpleChunkSortCursor>> _cursors;
 
     std::unique_ptr<MergeCursorsCascade> _merger;
     ChunkSlice _current_chunk;

From 8b30bbed9621dfbc1b7534d045d9a4526dcd9b82 Mon Sep 17 00:00:00 2001
From: Rohit Satardekar <rohitrs1983@gmail.com>
Date: Fri, 17 Jan 2025 12:05:39 +0530
Subject: [PATCH 34/71] [Enhancement] support like in show catalogs stmt
 (#53698)

Signed-off-by: Rohit Satardekar <rohitrs1983@gmail.com>
---
 .../main/java/com/starrocks/qe/ShowExecutor.java    |  9 ++++++++-
 .../com/starrocks/sql/ast/ShowCatalogsStmt.java     | 13 ++++++++++---
 .../java/com/starrocks/sql/parser/AstBuilder.java   |  7 ++++++-
 .../main/java/com/starrocks/sql/parser/StarRocks.g4 |  2 +-
 .../starrocks/analysis/ShowCatalogsStmtTest.java    |  2 +-
 5 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
index 1fa5c5e05c26b5..a121f085a93667 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
@@ -2379,7 +2379,14 @@ public ShowResultSet visitShowUserStatement(ShowUserStmt statement, ConnectConte
         public ShowResultSet visitShowCatalogsStatement(ShowCatalogsStmt statement, ConnectContext context) {
             GlobalStateMgr globalStateMgr = GlobalStateMgr.getCurrentState();
             CatalogMgr catalogMgr = globalStateMgr.getCatalogMgr();
+            PatternMatcher matcher = null;
+            if (statement.getPattern() != null) {
+                matcher = PatternMatcher.createMysqlPattern(statement.getPattern(),
+                        CaseSensibility.CONFIG.getCaseSensibility());
+            }
+            PatternMatcher finalMatcher = matcher;
             List<List<String>> rowSet = catalogMgr.getCatalogsInfo().stream()
+                    .filter(rowMatch -> finalMatcher == null || finalMatcher.match(rowMatch.get(0)))
                     .filter(row -> {
                                 if (!InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME.equals(row.get(0))) {
 
@@ -2394,7 +2401,7 @@ public ShowResultSet visitShowCatalogsStatement(ShowCatalogsStmt statement, Conn
                                     return true;
                                 }
                                 return true;
-                            }
+                    }
                     )
                     .sorted(Comparator.comparing(o -> o.get(0))).collect(Collectors.toList());
             return new ShowResultSet(statement.getMetaData(), rowSet);
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowCatalogsStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowCatalogsStmt.java
index d7659e5089e511..a81fbc1858933e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowCatalogsStmt.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ShowCatalogsStmt.java
@@ -29,12 +29,19 @@ public class ShowCatalogsStmt extends ShowStmt {
                     .addColumn(new Column("Comment", ScalarType.createVarchar(30)))
                     .build();
 
-    public ShowCatalogsStmt() {
-        this(NodePosition.ZERO);
+    private final String pattern;
+
+    public ShowCatalogsStmt(String pattern) {
+        this(pattern, NodePosition.ZERO);
     }
 
-    public ShowCatalogsStmt(NodePosition pos) {
+    public ShowCatalogsStmt(String pattern, NodePosition pos) {
         super(pos);
+        this.pattern = pattern;
+    }
+
+    public String getPattern() {
+        return pattern;
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java
index 333fb4c1ad1e70..4bb009d6fa4cdc 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java
@@ -2197,7 +2197,12 @@ public ParseNode visitShowCreateExternalCatalogStatement(
 
     @Override
     public ParseNode visitShowCatalogsStatement(StarRocksParser.ShowCatalogsStatementContext context) {
-        return new ShowCatalogsStmt(createPos(context));
+        NodePosition pos = createPos(context);
+        if (context.pattern != null) {
+            StringLiteral stringLiteral = (StringLiteral) visit(context.pattern);
+            return new ShowCatalogsStmt(stringLiteral.getValue(), pos);
+        }
+        return new ShowCatalogsStmt(null, createPos(context));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4
index 7e4d9cf83608e6..5670d248bba1d0 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4
@@ -776,7 +776,7 @@ dropExternalCatalogStatement
     ;
 
 showCatalogsStatement
-    : SHOW CATALOGS
+    : SHOW CATALOGS (LIKE pattern=string)?
     ;
 
 alterCatalogStatement
diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCatalogsStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCatalogsStmtTest.java
index 7f6fc165475cd0..5e3696b5b7a671 100644
--- a/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCatalogsStmtTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/analysis/ShowCatalogsStmtTest.java
@@ -67,7 +67,7 @@ public static void beforeClass() throws Exception {
 
     @Test
     public void testShowCatalogsNormal() throws AnalysisException, DdlException {
-        ShowCatalogsStmt stmt = new ShowCatalogsStmt();
+        ShowCatalogsStmt stmt = new ShowCatalogsStmt(null);
         ShowResultSet resultSet = ShowExecutor.execute(stmt, ctx);
         ShowResultSetMetaData metaData = resultSet.getMetaData();
         Assert.assertEquals("Catalog", metaData.getColumn(0).getName());

From 5cc4e6a02a68f21963e3fe3302216282cbc9464b Mon Sep 17 00:00:00 2001
From: srlch <111035020+srlch@users.noreply.github.com>
Date: Fri, 17 Jan 2025 16:10:17 +0800
Subject: [PATCH 35/71] [Feature] Support Cluster Snapshot Backup: deletion
 control (part4) (#54980)

Signed-off-by: srlch <linzichao@starrocks.com>
---
 .../com/starrocks/alter/AlterHandler.java     |  4 +-
 .../starrocks/catalog/CatalogRecycleBin.java  | 59 +++++++++++---
 .../com/starrocks/lake/StarMgrMetaSyncer.java |  4 +-
 .../lake/snapshot/ClusterSnapshot.java        | 32 ++++----
 .../lake/snapshot/ClusterSnapshotJob.java     | 22 +++---
 .../lake/snapshot/ClusterSnapshotMgr.java     | 51 ++++++++++--
 .../lake/vacuum/AutovacuumDaemon.java         |  3 +
 .../catalog/CatalogRecycleBinTest.java        | 49 +++++++++++-
 .../starrocks/clone/TabletSchedulerTest.java  |  5 ++
 .../starrocks/lake/ClusterSnapshotTest.java   | 77 ++++++++++++++++++-
 .../starrocks/lake/StarMgrMetaSyncerTest.java |  5 ++
 11 files changed, 264 insertions(+), 47 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterHandler.java
index 0cf0f581f41ce6..de4dfb8c399545 100644
--- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterHandler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterHandler.java
@@ -127,7 +127,9 @@ private void clearExpireFinishedOrCancelledAlterJobsV2() {
         Iterator<Map.Entry<Long, AlterJobV2>> iterator = alterJobsV2.entrySet().iterator();
         while (iterator.hasNext()) {
             AlterJobV2 alterJobV2 = iterator.next().getValue();
-            if (alterJobV2.isExpire()) {
+            long validDeletionTimeMs = GlobalStateMgr.getCurrentState().getClusterSnapshotMgr()
+                                                                       .getValidDeletionTimeMsByAutomatedSnapshot();
+            if (alterJobV2.isExpire() && alterJobV2.getFinishedTimeMs() < validDeletionTimeMs) {
                 iterator.remove();
                 RemoveAlterJobV2OperationLog log =
                         new RemoveAlterJobV2OperationLog(alterJobV2.getJobId(), alterJobV2.getType());
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java
index 5e2904826e4c90..2735d3e056d34e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/CatalogRecycleBin.java
@@ -181,12 +181,12 @@ public synchronized void recycleTable(long dbId, Table table, boolean recoverabl
             nameToTableInfo.remove(dbId, table.getName());
             nameToTableInfo.put(dbId, oldTable.getName(), oldTableInfo);
             // Speed up the deletion of this renamed table by modifying its recycle time to zero
-            idToRecycleTime.put(oldTable.getId(), 0L);
+            idToRecycleTime.put(oldTable.getId(), System.currentTimeMillis());
         }
 
         // If the table was force dropped, set recycle time to zero so that this table will be deleted immediately
         // in the next cleanup round.
-        idToRecycleTime.put(table.getId(), !recoverable ? 0 : System.currentTimeMillis());
+        idToRecycleTime.put(table.getId(), System.currentTimeMillis());
         idToTableInfo.put(dbId, table.getId(), newTableInfo);
         nameToTableInfo.put(dbId, table.getName(), newTableInfo);
 
@@ -228,8 +228,7 @@ public synchronized void recyclePartition(RecyclePartitionInfo recyclePartitionI
 
         disableRecoverPartitionWithSameName(dbId, tableId, partitionName);
 
-        long recycleTime = recyclePartitionInfo.isRecoverable() ? System.currentTimeMillis() : 0;
-        idToRecycleTime.put(partitionId, recycleTime);
+        idToRecycleTime.put(partitionId, System.currentTimeMillis());
         idToPartition.put(partitionId, recyclePartitionInfo);
         LOG.info("Finished put partition '{}' to recycle bin. dbId: {} tableId: {} partitionId: {} recoverable: {}",
                 partitionName, dbId, tableId, partitionId, recyclePartitionInfo.isRecoverable());
@@ -300,12 +299,42 @@ public synchronized List<PhysicalPartition> getPhysicalPartitions(long tableId)
                 .collect(Collectors.toList());
     }
 
+    private synchronized boolean checkValidDeletionByClusterSnapshot(long id) {
+        Long originalRecycleTime = idToRecycleTime.get(id);
+        if (originalRecycleTime == null) {
+            return true;
+        }
+        return originalRecycleTime < GlobalStateMgr.getCurrentState()
+                              .getClusterSnapshotMgr().getValidDeletionTimeMsByAutomatedSnapshot();
+    }
+
+    private synchronized long getAdjustedRecycleTimestamp(long id) {
+        Map<Long, RecycleTableInfo> idToRecycleTableInfo =  Maps.newHashMap();
+        for (Map<Long, RecycleTableInfo> tableEntry : idToTableInfo.rowMap().values()) {
+            for (Map.Entry<Long, RecycleTableInfo> entry : tableEntry.entrySet()) {
+                idToRecycleTableInfo.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        RecycleTableInfo tableInfo = idToRecycleTableInfo.get(id);
+        if (tableInfo != null && !tableInfo.isRecoverable()) {
+            return 0;
+        }
+
+        RecyclePartitionInfo partitionInfo = idToPartition.get(id);
+        if (partitionInfo != null && !partitionInfo.isRecoverable()) {
+            return 0;
+        }
+
+        return idToRecycleTime.get(id);
+    }
+
     /**
      * if we can erase this instance, we should check if anyone enable erase later.
      * Only used by main loop.
      */
     private synchronized boolean timeExpired(long id, long currentTimeMs) {
-        long latencyMs = currentTimeMs - idToRecycleTime.get(id);
+        long latencyMs = currentTimeMs - getAdjustedRecycleTimestamp(id);
         long expireMs = max(Config.catalog_trash_expire_second * 1000L, MIN_ERASE_LATENCY);
         if (enableEraseLater.contains(id)) {
             // if enableEraseLater is set, extend the timeout by LATE_RECYCLE_INTERVAL_SECONDS
@@ -315,6 +344,10 @@ private synchronized boolean timeExpired(long id, long currentTimeMs) {
     }
 
     private synchronized boolean canEraseTable(RecycleTableInfo tableInfo, long currentTimeMs) {
+        if (!checkValidDeletionByClusterSnapshot(tableInfo.getTable().getId())) {
+            return false;
+        }
+
         if (timeExpired(tableInfo.getTable().getId(), currentTimeMs)) {
             return true;
         }
@@ -327,6 +360,10 @@ private synchronized boolean canEraseTable(RecycleTableInfo tableInfo, long curr
     }
 
     private synchronized boolean canErasePartition(RecyclePartitionInfo partitionInfo, long currentTimeMs) {
+        if (!checkValidDeletionByClusterSnapshot(partitionInfo.getPartition().getId())) {
+            return false;
+        }
+
         if (timeExpired(partitionInfo.getPartition().getId(), currentTimeMs)) {
             return true;
         }
@@ -355,11 +392,15 @@ public synchronized boolean ensureEraseLater(long id, long currentTimeMs) {
             return false;
         }
         // 2. will expire after quite a long time, don't worry
-        long latency = currentTimeMs - idToRecycleTime.get(id);
+        long latency = currentTimeMs - getAdjustedRecycleTimestamp(id);
         if (latency < (Config.catalog_trash_expire_second - LATE_RECYCLE_INTERVAL_SECONDS) * 1000L) {
             return true;
         }
-        // 3. already expired, sorry.
+        // 3. check valid by cluster snapshot
+        if (!checkValidDeletionByClusterSnapshot(id)) {
+            return true;
+        } 
+        // 4. already expired, sorry.
         if (latency > Config.catalog_trash_expire_second * 1000L) {
             return false;
         }
@@ -374,7 +415,7 @@ protected synchronized void eraseDatabase(long currentTimeMs) {
             Map.Entry<Long, RecycleDatabaseInfo> entry = dbIter.next();
             RecycleDatabaseInfo dbInfo = entry.getValue();
             Database db = dbInfo.getDb();
-            if (timeExpired(db.getId(), currentTimeMs)) {
+            if (timeExpired(db.getId(), currentTimeMs) && checkValidDeletionByClusterSnapshot(db.getId())) {
                 // erase db
                 dbIter.remove();
                 removeRecycleMarkers(entry.getKey());
@@ -635,7 +676,7 @@ private synchronized void disableRecoverPartitionWithSameName(long dbId, long ta
                 continue;
             }
             partitionInfo.setRecoverable(false);
-            idToRecycleTime.replace(partitionInfo.getPartition().getId(), 0L);
+            idToRecycleTime.replace(partitionInfo.getPartition().getId(), System.currentTimeMillis());
             break;
         }
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java b/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java
index 8476b8c5632e4a..c245cb7d05da45 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/StarMgrMetaSyncer.java
@@ -79,7 +79,9 @@ private List<Long> getAllPartitionShardGroupId() {
             locker.lockDatabase(db.getId(), LockType.READ);
             try {
                 for (Table table : GlobalStateMgr.getCurrentState().getLocalMetastore().getTablesIncludeRecycleBin(db)) {
-                    if (table.isCloudNativeTableOrMaterializedView()) {
+                    if (table.isCloudNativeTableOrMaterializedView() &&
+                            GlobalStateMgr.getCurrentState().getClusterSnapshotMgr()
+                                                            .checkValidDeletionForTableFromAlterJob(table.getId())) {
                         GlobalStateMgr.getCurrentState().getLocalMetastore()
                                 .getAllPartitionsIncludeRecycleBin((OlapTable) table)
                                 .stream()
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshot.java b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshot.java
index 62667abc28b9da..e71c3dd29fd0fd 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshot.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshot.java
@@ -31,10 +31,10 @@ public enum ClusterSnapshotType { AUTOMATED, MANUAL, INCREMENTAL }
     private ClusterSnapshotType type;
     @SerializedName(value = "storageVolumeName")
     private String storageVolumeName;
-    @SerializedName(value = "createdTime")
-    private long createdTime;
-    @SerializedName(value = "finishedTime")
-    private long finishedTime;
+    @SerializedName(value = "createdTimeMs")
+    private long createdTimeMs;
+    @SerializedName(value = "finishedTimeMs")
+    private long finishedTimeMs;
     @SerializedName(value = "feJournalId")
     private long feJournalId;
     @SerializedName(value = "starMgrJournal")
@@ -42,14 +42,14 @@ public enum ClusterSnapshotType { AUTOMATED, MANUAL, INCREMENTAL }
 
     public ClusterSnapshot() {}
 
-    public ClusterSnapshot(long id, String snapshotName, String storageVolumeName, long createdTime,
-                           long finishedTime, long feJournalId, long starMgrJournalId) {
+    public ClusterSnapshot(long id, String snapshotName, String storageVolumeName, long createdTimeMs,
+                           long finishedTimeMs, long feJournalId, long starMgrJournalId) {
         this.id = id;
         this.snapshotName = snapshotName;
         this.type = ClusterSnapshotType.AUTOMATED;
         this.storageVolumeName = storageVolumeName;
-        this.createdTime = createdTime;
-        this.finishedTime = finishedTime;
+        this.createdTimeMs = createdTimeMs;
+        this.finishedTimeMs = finishedTimeMs;
         this.feJournalId = feJournalId;
         this.starMgrJournalId = starMgrJournalId;
     }
@@ -59,8 +59,8 @@ public void setJournalIds(long feJournalId, long starMgrJournalId) {
         this.starMgrJournalId = starMgrJournalId;
     }
 
-    public void setFinishedTime(long finishedTime) {
-        this.finishedTime = finishedTime;
+    public void setFinishedTimeMs(long finishedTimeMs) {
+        this.finishedTimeMs = finishedTimeMs;
     }
 
     public String getSnapshotName() {
@@ -71,12 +71,12 @@ public String getStorageVolumeName() {
         return storageVolumeName;
     }
 
-    public long getCreatedTime() {
-        return createdTime;
+    public long getCreatedTimeMs() {
+        return createdTimeMs;
     }
 
-    public long getFinishedTime() {
-        return finishedTime;
+    public long getFinishedTimeMs() {
+        return finishedTimeMs;
     }
 
     public long getFeJournalId() {
@@ -95,8 +95,8 @@ public TClusterSnapshotsItem getInfo() {
         TClusterSnapshotsItem item = new TClusterSnapshotsItem();
         item.setSnapshot_name(snapshotName);
         item.setSnapshot_type(type.name());
-        item.setCreated_time(createdTime / 1000);
-        item.setFinished_time(finishedTime / 1000);
+        item.setCreated_time(createdTimeMs / 1000);
+        item.setFinished_time(finishedTimeMs / 1000);
         item.setFe_jouranl_id(feJournalId);
         item.setStarmgr_jouranl_id(starMgrJournalId);
         item.setProperties("");
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotJob.java b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotJob.java
index b6bacf6ba70af0..884be585355a0c 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotJob.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotJob.java
@@ -46,8 +46,8 @@ public enum ClusterSnapshotJobState { INITIALIZING, SNAPSHOTING, UPLOADING, FINI
     @SerializedName(value = "errMsg")
     private String errMsg;
 
-    public ClusterSnapshotJob(long id, String snapshotName, String storageVolumeName, long createdTime) {
-        this.snapshot = new ClusterSnapshot(id, snapshotName, storageVolumeName, createdTime, -1, 0, 0);
+    public ClusterSnapshotJob(long id, String snapshotName, String storageVolumeName, long createdTimeMs) {
+        this.snapshot = new ClusterSnapshot(id, snapshotName, storageVolumeName, createdTimeMs, -1, 0, 0);
         this.state = ClusterSnapshotJobState.INITIALIZING;
         this.errMsg = "";
     }
@@ -55,7 +55,7 @@ public ClusterSnapshotJob(long id, String snapshotName, String storageVolumeName
     public void setState(ClusterSnapshotJobState state) {
         this.state = state;
         if (state == ClusterSnapshotJobState.FINISHED) {
-            snapshot.setFinishedTime(System.currentTimeMillis());
+            snapshot.setFinishedTimeMs(System.currentTimeMillis());
         }
     }
 
@@ -75,12 +75,12 @@ public String getStorageVolumeName() {
         return snapshot.getStorageVolumeName();
     }
 
-    public long getCreatedTime() {
-        return snapshot.getCreatedTime();
+    public long getCreatedTimeMs() {
+        return snapshot.getCreatedTimeMs();
     }
 
-    public long getFinishedTime() {
-        return snapshot.getFinishedTime();
+    public long getFinishedTimeMs() {
+        return snapshot.getFinishedTimeMs();
     }
 
     public long getFeJournalId() {
@@ -105,6 +105,10 @@ public boolean isUnFinishedState() {
                state == ClusterSnapshotJobState.FINISHED;
     }
 
+    public boolean isFinished() {
+        return state == ClusterSnapshotJobState.FINISHED;
+    }
+
     public void logJob() {
         ClusterSnapshotLog log = new ClusterSnapshotLog();
         log.setSnapshotJob(this);
@@ -123,8 +127,8 @@ public TClusterSnapshotJobsItem getInfo() {
         TClusterSnapshotJobsItem item = new TClusterSnapshotJobsItem();
         item.setSnapshot_name(getSnapshotName());
         item.setJob_id(getId());
-        item.setCreated_time(getCreatedTime() / 1000);
-        item.setFinished_time(getFinishedTime() / 1000);
+        item.setCreated_time(getCreatedTimeMs() / 1000);
+        item.setFinished_time(getFinishedTimeMs() / 1000);
         item.setState(state.name());
         item.setDetail_info("");
         item.setError_message(errMsg);
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java
index b5a42fb44ebe79..63f4776551a54b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/snapshot/ClusterSnapshotMgr.java
@@ -15,6 +15,7 @@
 package com.starrocks.lake.snapshot;
 
 import com.google.gson.annotations.SerializedName;
+import com.starrocks.alter.AlterJobV2;
 import com.starrocks.common.Config;
 import com.starrocks.common.StarRocksException;
 import com.starrocks.lake.snapshot.ClusterSnapshotJob.ClusterSnapshotJobState;
@@ -117,11 +118,11 @@ protected void addAutomatedClusterSnapshot(ClusterSnapshot newAutomatedClusterSn
     }
 
     public ClusterSnapshotJob createAutomatedSnapshotJob() {
-        long createTime = System.currentTimeMillis();
+        long createTimeMs = System.currentTimeMillis();
         long id = GlobalStateMgr.getCurrentState().getNextId();
-        String snapshotName = AUTOMATED_NAME_PREFIX + '_' + String.valueOf(createTime);
+        String snapshotName = AUTOMATED_NAME_PREFIX + '_' + String.valueOf(createTimeMs);
         String storageVolumeName = automatedSnapshotSvName;
-        ClusterSnapshotJob job = new ClusterSnapshotJob(id, snapshotName, storageVolumeName, createTime);
+        ClusterSnapshotJob job = new ClusterSnapshotJob(id, snapshotName, storageVolumeName, createTimeMs);
         job.logJob();
 
         addJob(job);
@@ -148,13 +149,53 @@ public boolean containsAutomatedSnapshot() {
     }
 
     public synchronized void addJob(ClusterSnapshotJob job) {
-        if (Config.max_historical_automated_cluster_snapshot_jobs >= 1 &&
-                historyAutomatedSnapshotJobs.size() == Config.max_historical_automated_cluster_snapshot_jobs) {
+        int maxSize = Math.max(Config.max_historical_automated_cluster_snapshot_jobs, 2);
+        if (historyAutomatedSnapshotJobs.size() == maxSize) {
             historyAutomatedSnapshotJobs.pollFirstEntry();
         }
         historyAutomatedSnapshotJobs.put(job.getId(), job);
     }
 
+    public synchronized long getValidDeletionTimeMsByAutomatedSnapshot() {
+        if (!isAutomatedSnapshotOn()) {
+            return Long.MAX_VALUE;
+        }
+
+        boolean findLastSuccess = false;
+        long previousAutomatedSnapshotCreatedTimsMs = 0;
+        for (Map.Entry<Long, ClusterSnapshotJob> entry : historyAutomatedSnapshotJobs.descendingMap().entrySet()) {
+            ClusterSnapshotJob job = entry.getValue();
+            if (job.isFinished()) {
+                if (findLastSuccess) {
+                    previousAutomatedSnapshotCreatedTimsMs = job.getCreatedTimeMs();
+                    break;
+                }
+
+                findLastSuccess = true;
+            }
+        }
+
+        return previousAutomatedSnapshotCreatedTimsMs;
+    }
+
+    public synchronized boolean checkValidDeletionForTableFromAlterJob(long tableId) {
+        if (!isAutomatedSnapshotOn()) {
+            return true;
+        }
+
+        boolean valid = true;
+        Map<Long, AlterJobV2> alterJobs = GlobalStateMgr.getCurrentState().getRollupHandler().getAlterJobsV2();
+        alterJobs.putAll(GlobalStateMgr.getCurrentState().getSchemaChangeHandler().getAlterJobsV2());
+        for (Map.Entry<Long, AlterJobV2> entry : alterJobs.entrySet()) {
+            AlterJobV2 alterJob = entry.getValue();
+            if (alterJob.getTableId() == tableId) {
+                valid = (alterJob.getFinishedTimeMs() < getValidDeletionTimeMsByAutomatedSnapshot());
+                break;
+            }
+        }
+        return valid;
+    }
+
     public TClusterSnapshotJobsResponse getAllJobsInfo() {
         TClusterSnapshotJobsResponse response = new TClusterSnapshotJobsResponse();
         for (Map.Entry<Long, ClusterSnapshotJob> entry : historyAutomatedSnapshotJobs.entrySet()) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java b/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java
index 1f501d4aae8e91..0a8063b3bae3a1 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/vacuum/AutovacuumDaemon.java
@@ -166,6 +166,9 @@ private void vacuumPartitionImpl(Database db, OlapTable table, PhysicalPartition
             vacuumRequest.minRetainVersion = minRetainVersion;
             vacuumRequest.graceTimestamp =
                     startTime / MILLISECONDS_PER_SECOND - Config.lake_autovacuum_grace_period_minutes * 60;
+            vacuumRequest.graceTimestamp = Math.min(vacuumRequest.graceTimestamp,
+                            Math.max(GlobalStateMgr.getCurrentState().getClusterSnapshotMgr()
+                                                   .getValidDeletionTimeMsByAutomatedSnapshot() / MILLISECONDS_PER_SECOND, 1));
             vacuumRequest.minActiveTxnId = minActiveTxnId;
             vacuumRequest.partitionId = partition.getId();
             vacuumRequest.deleteTxnLog = needDeleteTxnLog;
diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java
index 3baaf9688f6961..6f4e7c549c9adc 100644
--- a/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/catalog/CatalogRecycleBinTest.java
@@ -21,6 +21,7 @@
 import com.google.common.collect.Sets;
 import com.starrocks.common.Config;
 import com.starrocks.common.jmockit.Deencapsulation;
+import com.starrocks.lake.snapshot.ClusterSnapshotMgr;
 import com.starrocks.persist.EditLog;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.sql.ast.PartitionValue;
@@ -186,15 +187,19 @@ public void testReplayEraseTable() {
     @Test
     public void testReplayEraseTableEx(@Mocked GlobalStateMgr globalStateMgr) {
 
+        ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr();
         new Expectations() {
             {
                 GlobalStateMgr.getCurrentState();
                 result = globalStateMgr;
 
-                globalStateMgr.getEditLog().logEraseMultiTables((List<Long>) any);
+                globalStateMgr.getCurrentState().getEditLog().logEraseMultiTables((List<Long>) any);
                 minTimes = 1;
                 maxTimes = 1;
                 result = null;
+
+                globalStateMgr.getCurrentState().getClusterSnapshotMgr();
+                result = clusterSnapshotMgr;
             }
         };
 
@@ -207,9 +212,9 @@ public void testReplayEraseTableEx(@Mocked GlobalStateMgr globalStateMgr) {
         bin.recycleTable(13, table3, true);
 
         bin.eraseTable(System.currentTimeMillis() + Config.catalog_trash_expire_second * 1000L + 10000);
-        waitPartitionClearFinished(bin, 11L, System.currentTimeMillis() + Config.catalog_trash_expire_second * 1000L + 10000);
-        waitPartitionClearFinished(bin, 12L, System.currentTimeMillis() + Config.catalog_trash_expire_second * 1000L + 10000);
-        waitPartitionClearFinished(bin, 13L, System.currentTimeMillis() + Config.catalog_trash_expire_second * 1000L + 10000);
+        waitTableClearFinished(bin, 1L, System.currentTimeMillis() + Config.catalog_trash_expire_second * 1000L + 10000);
+        waitTableClearFinished(bin, 2L, System.currentTimeMillis() + Config.catalog_trash_expire_second * 1000L + 10000);
+        waitTableClearFinished(bin, 3L, System.currentTimeMillis() + Config.catalog_trash_expire_second * 1000L + 10000);
 
         Assert.assertEquals(0, bin.getTables(11L).size());
         Assert.assertEquals(0, bin.getTables(12L).size());
@@ -357,6 +362,14 @@ public void testAddTabletToInvertedIndexWithLocalTabletError(@Mocked GlobalState
 
     @Test
     public void testEnsureEraseLater() {
+        ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr();
+        new Expectations() {
+            {
+                GlobalStateMgr.getCurrentState().getClusterSnapshotMgr();
+                result = clusterSnapshotMgr;
+            }
+        };
+
         Config.catalog_trash_expire_second = 600; // set expire in 10 minutes
         CatalogRecycleBin recycleBin = new CatalogRecycleBin();
         Database db = new Database(111, "uno");
@@ -428,6 +441,13 @@ public void testRecycleDb(@Mocked GlobalStateMgr globalStateMgr, @Mocked EditLog
                 minTimes = 0;
             }
         };
+        ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr();
+        new Expectations() {
+            {
+                globalStateMgr.getCurrentState().getClusterSnapshotMgr();
+                result = clusterSnapshotMgr;
+            }
+        };
 
         recycleBin.eraseDatabase(now);
 
@@ -486,6 +506,13 @@ public void testRecycleTableMaxBatchSize(@Mocked GlobalStateMgr globalStateMgr,
                 result = null;
             }
         };
+        ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr();
+        new Expectations() {
+            {
+                globalStateMgr.getCurrentState().getClusterSnapshotMgr();
+                result = clusterSnapshotMgr;
+            }
+        };
         CatalogRecycleBin recycleBin = new CatalogRecycleBin();
         for (int i = 0; i < CatalogRecycleBin.getMaxEraseOperationsPerCycle() + 1; i++) {
             Table t = new Table(i, String.format("t%d", i), Table.TableType.VIEW, null);
@@ -523,6 +550,13 @@ public void testRecycleTable(@Mocked GlobalStateMgr globalStateMgr, @Mocked Edit
                 result = null;
             }
         };
+        ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr();
+        new Expectations() {
+            {
+                globalStateMgr.getCurrentState().getClusterSnapshotMgr();
+                result = clusterSnapshotMgr;
+            }
+        };
 
         // 1. add 2 tables
         long dbId = 1;
@@ -605,6 +639,13 @@ public void testRecyclePartition(@Mocked GlobalStateMgr globalStateMgr, @Mocked
                 minTimes = 0;
             }
         };
+        ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr();
+        new Expectations() {
+            {
+                globalStateMgr.getCurrentState().getClusterSnapshotMgr();
+                result = clusterSnapshotMgr;
+            }
+        };
 
         // 1. add 2 partitions
         long dbId = 1;
diff --git a/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedulerTest.java b/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedulerTest.java
index 288bfe2bf0a44a..5a1f00419c041a 100644
--- a/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedulerTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/clone/TabletSchedulerTest.java
@@ -37,6 +37,7 @@
 import com.starrocks.common.util.concurrent.lock.LockManager;
 import com.starrocks.common.util.concurrent.lock.LockType;
 import com.starrocks.common.util.concurrent.lock.Locker;
+import com.starrocks.lake.snapshot.ClusterSnapshotMgr;
 import com.starrocks.persist.EditLog;
 import com.starrocks.qe.VariableMgr;
 import com.starrocks.server.GlobalStateMgr;
@@ -138,6 +139,10 @@ public void setup() throws Exception {
                 globalStateMgr.getVariableMgr();
                 minTimes = 0;
                 result = variableMgr;
+
+                globalStateMgr.getClusterSnapshotMgr();
+                minTimes = 0;
+                result = new ClusterSnapshotMgr();
             }
         };
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java
index 8f40015f38543b..ee3db9dbf49989 100644
--- a/fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/lake/ClusterSnapshotTest.java
@@ -14,7 +14,11 @@
 
 package com.starrocks.lake.snapshot;
 
+import com.starrocks.alter.AlterJobV2;
 import com.starrocks.alter.AlterTest;
+import com.starrocks.alter.MaterializedViewHandler;
+import com.starrocks.alter.SchemaChangeHandler;
+import com.starrocks.alter.SchemaChangeJobV2;
 import com.starrocks.common.AlreadyExistsException;
 import com.starrocks.common.Config;
 import com.starrocks.common.DdlException;
@@ -66,6 +70,8 @@ public class ClusterSnapshotTest {
     private ClusterSnapshotMgr clusterSnapshotMgr = new ClusterSnapshotMgr();
     private boolean initSv = false;
 
+    private long nextId = 0;
+
     @BeforeClass
     public static void beforeClass() throws Exception {
         AlterTest.beforeClass();
@@ -103,7 +109,8 @@ public ClusterSnapshotMgr getClusterSnapshotMgr() {
 
             @Mock
             public long getNextId() {
-                return 0L;
+                nextId = nextId + 1;
+                return nextId;
             }
         };
 
@@ -310,4 +317,70 @@ public long getMaxJournalId() {
         }
         setAutomatedSnapshotOff(false);
     }
-}
\ No newline at end of file
+
+    @Test
+    public void testDeletionControl() {
+        new MockUp<RunMode>() {
+            @Mock
+            public boolean isSharedDataMode() {
+                return true;
+            }
+        };
+
+        ClusterSnapshotMgr localClusterSnapshotMgr = new ClusterSnapshotMgr();
+        Assert.assertTrue(localClusterSnapshotMgr.getValidDeletionTimeMsByAutomatedSnapshot() == Long.MAX_VALUE);
+        localClusterSnapshotMgr.setAutomatedSnapshotOn(storageVolumeName);
+        Assert.assertEquals(localClusterSnapshotMgr.getValidDeletionTimeMsByAutomatedSnapshot(), 0L);
+
+        ClusterSnapshotJob job1 = localClusterSnapshotMgr.createAutomatedSnapshotJob();
+        job1.setState(ClusterSnapshotJobState.FINISHED);
+        Assert.assertEquals(localClusterSnapshotMgr.getValidDeletionTimeMsByAutomatedSnapshot(), 0L);
+        ClusterSnapshotJob job2 = localClusterSnapshotMgr.createAutomatedSnapshotJob();
+        job2.setState(ClusterSnapshotJobState.FINISHED);
+        Assert.assertEquals(localClusterSnapshotMgr.getValidDeletionTimeMsByAutomatedSnapshot(), job1.getCreatedTimeMs());
+        localClusterSnapshotMgr.setAutomatedSnapshotOff();
+
+        localClusterSnapshotMgr = new ClusterSnapshotMgr();
+        Assert.assertTrue(localClusterSnapshotMgr.checkValidDeletionForTableFromAlterJob(10));
+        AlterJobV2 alterjob1 = new SchemaChangeJobV2(1, 2, 10, "table1", 100000);
+        AlterJobV2 alterjob2 = new SchemaChangeJobV2(2, 2, 11, "table2", 100000);
+        alterjob1.setJobState(AlterJobV2.JobState.FINISHED);
+        alterjob1.setFinishedTimeMs(1000);
+        alterjob2.setJobState(AlterJobV2.JobState.FINISHED);
+        alterjob2.setFinishedTimeMs(1000);
+
+
+        MaterializedViewHandler rollupHandler = new MaterializedViewHandler();
+        SchemaChangeHandler schemaChangeHandler = new SchemaChangeHandler();
+        schemaChangeHandler.addAlterJobV2(alterjob1);
+        schemaChangeHandler.addAlterJobV2(alterjob2);
+
+        new MockUp<GlobalStateMgr>() {
+            @Mock
+            public SchemaChangeHandler getSchemaChangeHandler() {
+                return schemaChangeHandler;
+            }
+
+            @Mock
+            public MaterializedViewHandler getRollupHandler() {
+                return rollupHandler;
+            }
+        };
+
+        localClusterSnapshotMgr.setAutomatedSnapshotOn(storageVolumeName);
+        Assert.assertTrue(!localClusterSnapshotMgr.checkValidDeletionForTableFromAlterJob(10));
+        Assert.assertTrue(!localClusterSnapshotMgr.checkValidDeletionForTableFromAlterJob(11));
+        ClusterSnapshotJob j1 = localClusterSnapshotMgr.createAutomatedSnapshotJob();
+        j1.setState(ClusterSnapshotJobState.FINISHED);
+
+        Assert.assertTrue(!localClusterSnapshotMgr.checkValidDeletionForTableFromAlterJob(10));
+        Assert.assertTrue(!localClusterSnapshotMgr.checkValidDeletionForTableFromAlterJob(11));
+
+        ClusterSnapshotJob j2 = localClusterSnapshotMgr.createAutomatedSnapshotJob();
+        j2.setState(ClusterSnapshotJobState.FINISHED);
+
+        Assert.assertTrue(localClusterSnapshotMgr.checkValidDeletionForTableFromAlterJob(10));
+        Assert.assertTrue(localClusterSnapshotMgr.checkValidDeletionForTableFromAlterJob(11));
+        localClusterSnapshotMgr.setAutomatedSnapshotOff();
+    }
+}
diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/StarMgrMetaSyncerTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/StarMgrMetaSyncerTest.java
index 043a6791972b7c..3e28382702fb62 100644
--- a/fe/fe-core/src/test/java/com/starrocks/lake/StarMgrMetaSyncerTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/lake/StarMgrMetaSyncerTest.java
@@ -43,6 +43,7 @@
 import com.starrocks.common.DdlException;
 import com.starrocks.common.jmockit.Deencapsulation;
 import com.starrocks.common.util.concurrent.lock.LockManager;
+import com.starrocks.lake.snapshot.ClusterSnapshotMgr;
 import com.starrocks.proto.DeleteTabletRequest;
 import com.starrocks.proto.DeleteTabletResponse;
 import com.starrocks.proto.StatusPB;
@@ -148,6 +149,10 @@ public void setUp() throws Exception {
                 globalStateMgr.getGtidGenerator();
                 minTimes = 0;
                 result = new GtidGenerator();
+
+                globalStateMgr.getClusterSnapshotMgr();
+                minTimes = 0;
+                result = new ClusterSnapshotMgr();
             }
         };
 

From 50c805e58a3ab9077b3dcf674af5ead2fc0320ec Mon Sep 17 00:00:00 2001
From: Murphy <96611012+murphyatwork@users.noreply.github.com>
Date: Fri, 17 Jan 2025 16:18:12 +0800
Subject: [PATCH 36/71] [BugFix] fix reload FK of table failure (#55126)

Signed-off-by: Murphy <mofei@starrocks.com>
---
 .../java/com/starrocks/catalog/OlapTable.java |  5 +--
 .../com/starrocks/server/GlobalStateMgr.java  | 28 ++++++++++++
 .../com/starrocks/server/LocalMetastore.java  | 12 -----
 .../starrocks/server/GlobalStateMgrTest.java  | 45 +++++++++++++++++++
 4 files changed, 75 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
index ba084e0886014d..5e8e181dcf8d3e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java
@@ -3105,8 +3105,7 @@ public List<ForeignKeyConstraint> getForeignKeyConstraints() {
      */
     @Override
     public boolean hasForeignKeyConstraints() {
-        return tableProperty != null && tableProperty.getForeignKeyConstraints() != null &&
-                !tableProperty.getForeignKeyConstraints().isEmpty();
+        return tableProperty != null && CollectionUtils.isNotEmpty(getForeignKeyConstraints());
     }
 
     @Override
@@ -3451,7 +3450,7 @@ public Map<String, String> getCommonProperties() {
 
         // foreign key constraint
         String foreignKeyConstraint = tableProperties.get(PropertyAnalyzer.PROPERTIES_FOREIGN_KEY_CONSTRAINT);
-        if (!Strings.isNullOrEmpty(foreignKeyConstraint)) {
+        if (!Strings.isNullOrEmpty(foreignKeyConstraint) && hasForeignKeyConstraints()) {
             properties.put(PropertyAnalyzer.PROPERTIES_FOREIGN_KEY_CONSTRAINT,
                     ForeignKeyConstraint.getShowCreateTableConstraintDesc(this, getForeignKeyConstraints()));
         }
diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
index 1738230bacf2f4..bdfa794dd53774 100644
--- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
@@ -101,6 +101,7 @@
 import com.starrocks.common.util.LogUtil;
 import com.starrocks.common.util.PropertyAnalyzer;
 import com.starrocks.common.util.SmallFileMgr;
+import com.starrocks.common.util.UUIDUtil;
 import com.starrocks.common.util.Util;
 import com.starrocks.common.util.concurrent.QueryableReentrantLock;
 import com.starrocks.common.util.concurrent.lock.LockManager;
@@ -1641,9 +1642,36 @@ public void loadImage(String imageDir) throws IOException {
     }
 
     private void postLoadImage() {
+        onReloadTables();
         processMvRelatedMeta();
     }
 
+    /**
+     * Call Table::onReload after load all tables, some properties like FK may depend on other databases/catalogs
+     */
+    private void onReloadTables() {
+        TemporaryTableMgr temporaryTableMgr = GlobalStateMgr.getCurrentState().getTemporaryTableMgr();
+        List<String> dbNames = metadataMgr.listDbNames(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME);
+        for (String dbName : dbNames) {
+            Database db = metadataMgr.getDb(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME, dbName);
+            if (db == null) {
+                continue;
+            }
+            for (Table table : db.getTables()) {
+                try {
+                    table.onReload();
+
+                    if (table.isTemporaryTable()) {
+                        temporaryTableMgr.addTemporaryTable(UUIDUtil.genUUID(), db.getId(), table.getName(),
+                                table.getId());
+                    }
+                } catch (Throwable e) {
+                    LOG.error("reload table failed: {}", table, e);
+                }
+            }
+        }
+    }
+
     private void processMvRelatedMeta() {
         List<String> dbNames = metadataMgr.listDbNames(InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME);
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java
index 94725d80032f9c..dc194192180162 100644
--- a/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java
+++ b/fe/fe-core/src/main/java/com/starrocks/server/LocalMetastore.java
@@ -116,7 +116,6 @@
 import com.starrocks.common.util.DynamicPartitionUtil;
 import com.starrocks.common.util.PropertyAnalyzer;
 import com.starrocks.common.util.TimeUtils;
-import com.starrocks.common.util.UUIDUtil;
 import com.starrocks.common.util.Util;
 import com.starrocks.common.util.concurrent.CountingLatch;
 import com.starrocks.common.util.concurrent.lock.LockType;
@@ -5069,17 +5068,6 @@ public void load(SRMetaBlockReader reader) throws IOException, SRMetaBlockExcept
             idToDb.put(db.getId(), db);
             fullNameToDb.put(db.getFullName(), db);
             stateMgr.getGlobalTransactionMgr().addDatabaseTransactionMgr(db.getId());
-            db.getTables().stream().filter(tbl -> !tbl.isMaterializedView()).forEach(tbl -> {
-                try {
-                    tbl.onReload();
-                    if (tbl.isTemporaryTable()) {
-                        TemporaryTableMgr temporaryTableMgr = GlobalStateMgr.getCurrentState().getTemporaryTableMgr();
-                        temporaryTableMgr.addTemporaryTable(UUIDUtil.genUUID(), db.getId(), tbl.getName(), tbl.getId());
-                    }
-                } catch (Throwable e) {
-                    LOG.error("reload table failed: {}", tbl, e);
-                }
-            });
         });
 
         AutoIncrementInfo autoIncrementInfo = reader.readJson(AutoIncrementInfo.class);
diff --git a/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java
index b2d7d60711d9a2..a616b1ba437cf9 100644
--- a/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java
@@ -40,6 +40,7 @@
 import com.sleepycat.je.rep.ReplicaStateException;
 import com.sleepycat.je.rep.UnknownMasterException;
 import com.sleepycat.je.rep.util.ReplicationGroupAdmin;
+import com.starrocks.catalog.Table;
 import com.starrocks.common.Config;
 import com.starrocks.common.DdlException;
 import com.starrocks.common.Pair;
@@ -52,8 +53,11 @@
 import com.starrocks.persist.ImageFormatVersion;
 import com.starrocks.persist.ImageWriter;
 import com.starrocks.persist.OperationType;
+import com.starrocks.qe.ConnectContext;
 import com.starrocks.sql.ast.ModifyFrontendAddressClause;
+import com.starrocks.sql.ast.UserIdentity;
 import com.starrocks.system.Frontend;
+import com.starrocks.utframe.StarRocksAssert;
 import com.starrocks.utframe.UtFrameUtils;
 import mockit.Expectations;
 import mockit.Mock;
@@ -65,6 +69,8 @@
 import org.mockito.Mockito;
 
 import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
@@ -310,4 +316,43 @@ public void testErrorOccursWhileRemovingClusterIdAndRoleWhenStartAtFirstTime() {
             Assert.assertEquals(removeFileErrorMessage, suppressedExceptions[0].getMessage());
         }
     }
+
+    @Test
+    public void testReloadTables() throws Exception {
+        ConnectContext ctx = UtFrameUtils.initCtxForNewPrivilege(UserIdentity.ROOT);
+        UtFrameUtils.createMinStarRocksCluster();
+        UtFrameUtils.setUpForPersistTest();
+        GlobalStateMgr currentState = GlobalStateMgr.getCurrentState();
+        StarRocksAssert starRocksAssert = new StarRocksAssert();
+
+        currentState.getLocalMetastore().createDb("db1");
+        currentState.getLocalMetastore().createDb("db2");
+        {
+            String sql = "create table db1.t1(c1 int not null, c2 int) " +
+                    "properties('replication_num'='1', 'unique_constraints'='c1') ";
+            starRocksAssert.withTable(sql);
+        }
+        {
+            String sql = "create table db2.t1(c1 int, c2 int) properties('replication_num'='1'," +
+                    "'foreign_key_constraints'='(c1) REFERENCES db1.t1(c1)')";
+            starRocksAssert.withTable(sql);
+        }
+
+        // move image file
+        String imagePath = currentState.dumpImage();
+        Path targetPath = Path.of(Config.meta_dir, GlobalStateMgr.IMAGE_DIR, "/v2",
+                Path.of(imagePath).getFileName().toString());
+        Files.move(Path.of(imagePath), targetPath);
+        // move checksum file
+        Path checksumPath = Path.of(Config.meta_dir, "checksum.0");
+        Path checksumTarget = Path.of(Config.meta_dir, GlobalStateMgr.IMAGE_DIR, "/v2", "checksum.0");
+        Files.move(checksumPath, checksumTarget);
+
+        GlobalStateMgr newState = new MyGlobalStateMgr(false);
+        newState.loadImage(Config.meta_dir + GlobalStateMgr.IMAGE_DIR);
+        Table table = newState.getLocalMetastore().getTable("db1", "t1");
+        Assert.assertNotNull(table);
+        table = newState.getLocalMetastore().getTable("db2", "t1");
+        Assert.assertEquals(1, table.getForeignKeyConstraints().size());
+    }
 }

From d4cf6fdd382e08be6237253b5cde3db613fb2dfe Mon Sep 17 00:00:00 2001
From: stdpain <34912776+stdpain@users.noreply.github.com>
Date: Fri, 17 Jan 2025 16:24:47 +0800
Subject: [PATCH 37/71] [BugFix] Fix TopN RuntimeFilter with nulls first crash
 with lowcardinality optimize (#55163)

Signed-off-by: stdpain <drfeng08@gmail.com>
---
 be/src/exec/olap_scan_prepare.cpp | 19 +++++------
 be/src/exec/olap_scan_prepare.h   |  3 +-
 test/sql/test_sort/R/test_topn    | 54 +++++++++++++++++++++++++++++++
 test/sql/test_sort/T/test_topn    | 34 ++++++++++++++++++-
 4 files changed, 99 insertions(+), 11 deletions(-)

diff --git a/be/src/exec/olap_scan_prepare.cpp b/be/src/exec/olap_scan_prepare.cpp
index a6d744d52f1474..ec8459f4983e9b 100644
--- a/be/src/exec/olap_scan_prepare.cpp
+++ b/be/src/exec/olap_scan_prepare.cpp
@@ -20,6 +20,7 @@
 #include "exprs/binary_predicate.h"
 #include "exprs/compound_predicate.h"
 #include "exprs/dictmapping_expr.h"
+#include "exprs/expr.h"
 #include "exprs/expr_context.h"
 #include "exprs/in_const_predicate.hpp"
 #include "exprs/is_null_predicate.h"
@@ -577,19 +578,19 @@ Status ChunkPredicateBuilder<E, Type>::normalize_binary_predicate(const SlotDesc
 
 template <BoxedExprType E, CompoundNodeType Type>
 template <LogicalType SlotType, LogicalType MappingType, template <class> class Decoder, class... Args>
-void ChunkPredicateBuilder<E, Type>::normalized_rf_with_null(const JoinRuntimeFilter* rf, Expr* col_ref,
-                                                             Args&&... args) {
+void ChunkPredicateBuilder<E, Type>::normalized_rf_with_null(const JoinRuntimeFilter* rf,
+                                                             const SlotDescriptor* slot_desc, Args&&... args) {
     DCHECK(Type == CompoundNodeType::AND);
-
+    using RFColumnPredicateBuilder = detail::RuntimeColumnPredicateBuilder;
     ObjectPool* pool = _opts.obj_pool;
 
     const auto* filter = down_cast<const RuntimeBloomFilter<MappingType>*>(rf);
     using DecoderType = Decoder<typename RunTimeTypeTraits<MappingType>::CppType>;
     DecoderType decoder(std::forward<Args>(args)...);
-    detail::RuntimeColumnPredicateBuilder::MinMaxParser<RuntimeBloomFilter<MappingType>, DecoderType> parser(filter,
-                                                                                                             &decoder);
-    const TypeDescriptor& col_type = col_ref->type();
+    RFColumnPredicateBuilder::MinMaxParser<RuntimeBloomFilter<MappingType>, DecoderType> parser(filter, &decoder);
 
+    const TypeDescriptor& col_type = slot_desc->type();
+    ColumnRef* col_ref = pool->add(new ColumnRef(slot_desc));
     ColumnPtr const_min_col = parser.template min_const_column<SlotType>(col_type, pool);
     ColumnPtr const_max_col = parser.template max_const_column<SlotType>(col_type, pool);
     VectorizedLiteral* min_literal = pool->add(new VectorizedLiteral(std::move(const_min_col), col_type));
@@ -715,7 +716,7 @@ Status ChunkPredicateBuilder<E, Type>::normalize_join_runtime_filter(const SlotD
                 if (rf->has_null()) {
                     normalized_rf_with_null<SlotType, LowCardDictType,
                                             detail::RuntimeColumnPredicateBuilder::GlobalDictCodeDecoder>(
-                            rf, desc->probe_expr_ctx()->root(), &iter->second.first);
+                            rf, &slot, &iter->second.first);
                 } else {
                     detail::RuntimeColumnPredicateBuilder::build_minmax_range<
                             RangeType, SlotType, LowCardDictType,
@@ -725,7 +726,7 @@ Status ChunkPredicateBuilder<E, Type>::normalize_join_runtime_filter(const SlotD
             } else {
                 if (rf->has_null()) {
                     normalized_rf_with_null<SlotType, SlotType, detail::RuntimeColumnPredicateBuilder::DummyDecoder>(
-                            rf, desc->probe_expr_ctx()->root(), nullptr);
+                            rf, &slot, nullptr);
                 } else {
                     detail::RuntimeColumnPredicateBuilder::build_minmax_range<
                             RangeType, SlotType, SlotType, detail::RuntimeColumnPredicateBuilder::DummyDecoder>(
@@ -735,7 +736,7 @@ Status ChunkPredicateBuilder<E, Type>::normalize_join_runtime_filter(const SlotD
         } else {
             if (rf->has_null()) {
                 normalized_rf_with_null<SlotType, SlotType, detail::RuntimeColumnPredicateBuilder::DummyDecoder>(
-                        rf, desc->probe_expr_ctx()->root(), nullptr);
+                        rf, &slot, nullptr);
             } else {
                 detail::RuntimeColumnPredicateBuilder::build_minmax_range<
                         RangeType, SlotType, SlotType, detail::RuntimeColumnPredicateBuilder::DummyDecoder>(
diff --git a/be/src/exec/olap_scan_prepare.h b/be/src/exec/olap_scan_prepare.h
index 82dc2aced53b00..625c87f6678120 100644
--- a/be/src/exec/olap_scan_prepare.h
+++ b/be/src/exec/olap_scan_prepare.h
@@ -18,6 +18,7 @@
 #include "exec/olap_common.h"
 #include "exprs/expr.h"
 #include "exprs/expr_context.h"
+#include "runtime/descriptors.h"
 #include "storage/predicate_tree/predicate_tree_fwd.h"
 #include "storage/predicate_tree_params.h"
 #include "storage/runtime_range_pruner.h"
@@ -91,7 +92,7 @@ class ChunkPredicateBuilder {
     const UnarrivedRuntimeFilterList& unarrived_runtime_filters() { return rt_ranger_params; }
 
     template <LogicalType SlotType, LogicalType MappingType, template <class> class Decoder, class... Args>
-    void normalized_rf_with_null(const JoinRuntimeFilter* rf, Expr* col_ref, Args&&... args);
+    void normalized_rf_with_null(const JoinRuntimeFilter* rf, const SlotDescriptor* slot_desc, Args&&... args);
 
 private:
     const ScanConjunctsManagerOptions& _opts;
diff --git a/test/sql/test_sort/R/test_topn b/test/sql/test_sort/R/test_topn
index 5cbaf3bdabbee5..0f93a4be1eb1b9 100644
--- a/test/sql/test_sort/R/test_topn
+++ b/test/sql/test_sort/R/test_topn
@@ -484,4 +484,58 @@ select c1,c0 from t5 order by c1,c0 desc limit 10;
 11	8
 12	9
 13	10
+-- !result
+CREATE TABLE `tlow` (
+  `c0` int(11) NULL COMMENT "",
+  `c1` varchar(20) NULL COMMENT "",
+  `c2` varchar(200) NULL COMMENT "",
+  `c3` int(11) NULL COMMENT ""
+) ENGINE=OLAP
+DUPLICATE KEY(`c0`, `c1`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`c0`, `c1`) BUCKETS 64
+PROPERTIES (
+"colocate_with" = "${uuid0}",
+"replication_num" = "1",
+"in_memory" = "false",
+"storage_format" = "DEFAULT",
+"enable_persistent_index" = "false",
+"replicated_storage" = "true",
+"compression" = "LZ4"
+);
+-- result:
+-- !result
+set io_tasks_per_scan_operator=1;
+-- result:
+-- !result
+set group_execution_min_scan_rows=1;
+-- result:
+-- !result
+insert into tlow SELECT generate_series, if (generate_series<10, null, generate_series%10), generate_series, generate_series FROM TABLE(generate_series(1,  40960));
+-- result:
+-- !result
+analyze full table tl;
+-- result:
+E: (1064, 'Getting analyzing error. Detail message: Table tl is not found.')
+-- !result
+select count(distinct c1) from tlow;
+-- result:
+10
+-- !result
+function: wait_global_dict_ready('c1', 'tlow')
+-- result:
+
+-- !result
+select c1 from tlow group by c0, c1 order by c1 nulls first limit 10;
+-- result:
+None
+None
+None
+None
+None
+None
+None
+None
+None
+0
 -- !result
\ No newline at end of file
diff --git a/test/sql/test_sort/T/test_topn b/test/sql/test_sort/T/test_topn
index 55fbe635ef53d7..2036c1a89ec7a4 100644
--- a/test/sql/test_sort/T/test_topn
+++ b/test/sql/test_sort/T/test_topn
@@ -100,4 +100,36 @@ select c1 from t5 order by c1 asc limit 10;
 select c1,c0 from t5 where c1 < 10 order by c1,c0 asc limit 10;
 select c0,c1 from t5 order by c0 desc limit 10;
 select c1 from t5 order by c1 desc limit 10;
-select c1,c0 from t5 order by c1,c0 desc limit 10;
\ No newline at end of file
+select c1,c0 from t5 order by c1,c0 desc limit 10;
+
+-- test has null topn rf with lowcardinality 
+CREATE TABLE `tlow` (
+  `c0` int(11) NULL COMMENT "",
+  `c1` varchar(20) NULL COMMENT "",
+  `c2` varchar(200) NULL COMMENT "",
+  `c3` int(11) NULL COMMENT ""
+) ENGINE=OLAP
+DUPLICATE KEY(`c0`, `c1`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`c0`, `c1`) BUCKETS 64
+PROPERTIES (
+"colocate_with" = "${uuid0}",
+"replication_num" = "1",
+"in_memory" = "false",
+"storage_format" = "DEFAULT",
+"enable_persistent_index" = "false",
+"replicated_storage" = "true",
+"compression" = "LZ4"
+);
+
+set io_tasks_per_scan_operator=1;
+set group_execution_min_scan_rows=1;
+
+insert into tlow SELECT generate_series, if (generate_series<10, null, generate_series%10), generate_series, generate_series FROM TABLE(generate_series(1,  40960));
+analyze full table tl;
+select count(distinct c1) from tlow;
+function: wait_global_dict_ready('c1', 'tlow')
+
+select c1 from tlow group by c0, c1 order by c1 nulls first limit 10;
+
+

From eb1cb80b1a41b6f54f13dec81545a4cd87d62ac9 Mon Sep 17 00:00:00 2001
From: stephen <91597003+stephen-shelby@users.noreply.github.com>
Date: Fri, 17 Jan 2025 16:28:29 +0800
Subject: [PATCH 38/71] [BugFix] fix samples rows calculation error when
 sampling to collect table statistics (#55201)

Signed-off-by: stephen <stephen5217@163.com>
---
 .../statistic/base/PartitionSampler.java      | 20 +++++++++++++++++--
 .../statistic/base/TabletSampler.java         |  4 ----
 .../statistic/hyper/HyperJobTest.java         | 15 ++++++++++++++
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java b/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java
index 72c86c13ba24b3..dd7b216f63f1ec 100644
--- a/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java
@@ -116,14 +116,26 @@ public void classifyPartitions(Table table, List<Long> partitions) {
 
             long totalRows = high.getTotalRows() + mediumHigh.getTotalRows() + mediumLow.getTotalRows()
                     + low.getTotalRows();
-            long sampleRows = high.getSampleRows() + mediumHigh.getSampleRows() + mediumLow.getSampleRows()
-                    + low.getSampleRows();
 
             List<TabletStats> highSampleTablets = high.sample();
             List<TabletStats> mediumHighSampleTablets = mediumHigh.sample();
             List<TabletStats> mediumLowSampleTablets = mediumLow.sample();
             List<TabletStats> lowSampleTablets = low.sample();
 
+            long sampleRows = Math.min(sampleRowsLimit, highSampleTablets.stream()
+                    .mapToLong(e -> getReadRowCount(e.getRowCount(), highRatio))
+                    .sum());
+            sampleRows += Math.min(sampleRowsLimit, mediumHighSampleTablets.stream()
+                    .mapToLong(e -> getReadRowCount(e.getRowCount(), mediumHighRatio))
+                    .sum());
+            sampleRows += Math.min(sampleRowsLimit, mediumLowSampleTablets.stream()
+                    .mapToLong(e -> getReadRowCount(e.getRowCount(), mediumLowRatio))
+                    .sum());
+            sampleRows += Math.min(sampleRowsLimit, lowSampleTablets.stream()
+                    .mapToLong(e -> getReadRowCount(e.getRowCount(), lowRatio))
+                    .sum());
+            sampleRows = Math.max(1, sampleRows);
+
             long totalTablets = high.getTotalTablets() + mediumHigh.getTotalTablets() + mediumLow.getTotalTablets() +
                     low.getTotalTablets();
             long sampleTablets = highSampleTablets.size() + mediumHighSampleTablets.size()
@@ -157,4 +169,8 @@ public static PartitionSampler create(Table table, List<Long> partitions, Map<St
         sampler.classifyPartitions(table, partitions);
         return sampler;
     }
+
+    private long getReadRowCount(long totalRowCount, double readRatio) {
+        return (long) Math.max(totalRowCount * readRatio, 1L);
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/base/TabletSampler.java b/fe/fe-core/src/main/java/com/starrocks/statistic/base/TabletSampler.java
index b0610126927f32..e9a9e83e687a8d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/statistic/base/TabletSampler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/statistic/base/TabletSampler.java
@@ -57,10 +57,6 @@ public long getTotalRows() {
         return totalRows;
     }
 
-    public long getSampleRows() {
-        return (long) Math.min(sampleRowsLimit, Math.max(totalRows * tabletReadRatio, 1L));
-    }
-
     public long getTotalTablets() {
         return tablets.size();
     }
diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java
index bd97dc273b2408..67231af387816a 100644
--- a/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java
@@ -43,6 +43,8 @@
 import com.starrocks.statistic.base.PartitionSampler;
 import com.starrocks.statistic.base.PrimitiveTypeColumnStats;
 import com.starrocks.statistic.base.SubFieldColumnStats;
+import com.starrocks.statistic.base.TabletSampler;
+import com.starrocks.statistic.sample.TabletStats;
 import com.starrocks.utframe.StarRocksAssert;
 import mockit.Mock;
 import mockit.MockUp;
@@ -231,6 +233,19 @@ public void testSubfieldSampleJobs() {
                 "IFNULL(MAX(`c6`.`c`.`b`), ''), IFNULL(MIN(`c6`.`c`.`b`), ''), cast(-1.0 as BIGINT) FROM base_cte_table");
     }
 
+    @Test
+    public void testSampleRows() {
+        new MockUp<TabletSampler>() {
+            @Mock
+            public List<TabletStats> sample() {
+                return List.of(new TabletStats(1, pid, 5000000));
+            }
+
+        };
+        PartitionSampler sampler = PartitionSampler.create(table, List.of(pid), Maps.newHashMap());
+        Assert.assertEquals(800000, sampler.getSampleInfo(pid).getSampleRowCount());
+    }
+
     @AfterClass
     public static void afterClass() {
         FeConstants.runningUnitTest = false;

From 688a26df7fcf4f3dfbef240c0113c73a81a7c300 Mon Sep 17 00:00:00 2001
From: stephen <91597003+stephen-shelby@users.noreply.github.com>
Date: Fri, 17 Jan 2025 17:21:55 +0800
Subject: [PATCH 39/71] [BugFix] fix analyze table with sample ratio (#55202)

Signed-off-by: stephen <stephen5217@163.com>
---
 .../com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java   | 5 +++++
 .../java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java  | 8 ++++++++
 2 files changed, 13 insertions(+)

diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java
index 0bf46f90cdd93e..e205e666553256 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java
@@ -80,6 +80,11 @@ public static void analyze(StatementBase statement, ConnectContext session) {
             StatsConstants.STATISTIC_SAMPLE_COLLECT_ROWS,
             StatsConstants.STATISTIC_EXCLUDE_PATTERN,
 
+            StatsConstants.HIGH_WEIGHT_SAMPLE_RATIO,
+            StatsConstants.MEDIUM_HIGH_WEIGHT_SAMPLE_RATIO,
+            StatsConstants.MEDIUM_LOW_WEIGHT_SAMPLE_RATIO,
+            StatsConstants.LOW_WEIGHT_SAMPLE_RATIO,
+
             StatsConstants.HISTOGRAM_BUCKET_NUM,
             StatsConstants.HISTOGRAM_MCV_SIZE,
             StatsConstants.HISTOGRAM_SAMPLE_RATIO,
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java
index f232a7e00bd021..6b8d70158e4f90 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeStmtTest.java
@@ -491,4 +491,12 @@ public void testAnalyzePredicateColumns() {
         stmt = (AnalyzeStmt) analyzeSuccess("analyze table db.tbl predicate columns");
         Assert.assertTrue(stmt.isUsePredicateColumns());
     }
+
+    @Test
+    public void testAnalyzeTableWithSampleRatio() {
+        analyzeSuccess("analyze sample table db.tbl properties(\"high_weight_sample_ratio\" = \"0.6\")");
+        analyzeSuccess("analyze sample table db.tbl properties(\"medium_high_weight_sample_ratio\" = \"0.6\")");
+        analyzeSuccess("analyze sample table db.tbl properties(\"medium_low_weight_sample_ratio\" = \"0.6\")");
+        analyzeSuccess("analyze sample table db.tbl properties(\"low_weight_sample_ratio\" = \"0.6\")");
+    }
 }

From cbb7eb1d20b3974cb902119dc16442b566e524ec Mon Sep 17 00:00:00 2001
From: tracymacding <tracymacding@gmail.com>
Date: Fri, 17 Jan 2025 21:12:48 +0800
Subject: [PATCH 40/71] [BugFix] fix thrift compatibility issue with v3.3
 (#55214)

Signed-off-by: tracymacding <tracymacding@gmail.com>
---
 gensrc/thrift/PlanNodes.thrift | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index bcb5d1f95dbf73..2c6372d654e4a0 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -133,11 +133,11 @@ struct TInternalScanRange {
   12: optional bool fill_data_cache = true;
   // used for per-bucket compute optimize
   13: optional i32 bucket_sequence
-  14: optional i64 gtid
   // skip page cache when access page data
-  15: optional bool skip_page_cache = false;
+  14: optional bool skip_page_cache = false;
   // skip local disk data cache when access page data
-  16: optional bool skip_disk_cache = false;
+  15: optional bool skip_disk_cache = false;
+  16: optional i64 gtid
 }
 
 enum TFileFormatType {

From 952799a2994c8042f7f79d07cd5668220fdbce24 Mon Sep 17 00:00:00 2001
From: "shuming.li" <ming.moriarty@gmail.com>
Date: Fri, 17 Jan 2025 23:53:38 +0800
Subject: [PATCH 41/71] [Doc] Add ds_hll_count_distinct doc (#54745)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
Signed-off-by: 絵空事スピリット <wanglichen@starrocks.com>
Co-authored-by: 絵空事スピリット <wanglichen@starrocks.com>
---
 .../approx_count_distinct_hll_sketch.md       | 30 ---------
 .../ds_hll_count_distinct.md                  | 63 +++++++++++++++++++
 .../ds_hll_count_distinct.md                  | 56 +++++++++++++++++
 3 files changed, 119 insertions(+), 30 deletions(-)
 delete mode 100644 docs/en/sql-reference/sql-functions/aggregate-functions/approx_count_distinct_hll_sketch.md
 create mode 100644 docs/en/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md
 create mode 100644 docs/zh/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md

diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/approx_count_distinct_hll_sketch.md b/docs/en/sql-reference/sql-functions/aggregate-functions/approx_count_distinct_hll_sketch.md
deleted file mode 100644
index c08f23275fbbc0..00000000000000
--- a/docs/en/sql-reference/sql-functions/aggregate-functions/approx_count_distinct_hll_sketch.md
+++ /dev/null
@@ -1,30 +0,0 @@
-# APPROX_COUNT_DISTINCT_HLL_SKETCH
-
-
-
-Returns the approximate value of aggregate function similar to the result of COUNT(DISTINCT col). Like APPROX_COUNT_DISTINCT(expr).
-
-It is faster than the COUNT and DISTINCT combination and uses a fixed-size memory, so less memory is used for columns of high cardinality.
-
-It is slower than APPROX_COUNT_DISTINCT(expr) but with higher precision. Which takes advantages of Apache Datasketches.
-
-## Syntax
-
-```Haskell
-APPROX_COUNT_DISTINCT_HLL_SKETCH(expr)
-```
-
-## Examples
-
-```plain text
-MySQL > select APPROX_COUNT_DISTINCT_HLL_SKETCH(query_id) from log_statis group by datetime;
-+-----------------------------------+
-| approx_count_distinct_hll_sketch(`query_id`) |
-+-----------------------------------+
-| 17721                             |
-+-----------------------------------+
-```
-
-## keyword
-
-APPROX_COUNT_DISTINCT_HLL_SKETCH,APPROX_COUNT_DISTINCT
diff --git a/docs/en/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md b/docs/en/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md
new file mode 100644
index 00000000000000..84bb46182f68c8
--- /dev/null
+++ b/docs/en/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md
@@ -0,0 +1,63 @@
+# ds_hll_count_distinct
+
+Returns the approximate value of aggregate function similar to the result of COUNT(DISTINCT col). APPROX_COUNT_DISTINCT(expr) is similar function.
+
+ds_hll_count_distinct is faster than the COUNT and DISTINCT combination and uses a fixed-size memory, so less memory is required for columns of high cardinality.
+
+It is slower than APPROX_COUNT_DISTINCT(expr) but with higher precision because it adopts of Apache Datasketches. For more information, see [HyperLogLog Sketches](https://datasketches.apache.org/docs/HLL/HllSketches.html).
+
+## Syntax
+
+```Haskell
+ds_hll_count_distinct(expr, [log_k], [tgt_type])
+```
+- `log_k`: Integer. Range [4, 21]. Default: 17.
+- `tgt_type`: Valid values are `HLL_4`, `HLL_6` (default) and `HLL_8`.
+
+## Examples
+
+```plain text
+mysql> CREATE TABLE t1 (
+    ->   id BIGINT NOT NULL,
+    ->   province VARCHAR(64),
+    ->   age SMALLINT,
+    ->   dt VARCHAR(10) NOT NULL
+    -> )
+    -> DUPLICATE KEY(id)
+    -> DISTRIBUTED BY HASH(id) BUCKETS 4;
+Query OK, 0 rows affected (0.02 sec)
+
+mysql> insert into t1 select generate_series, generate_series, generate_series % 100, "2024-07-24" from table(generate_series(1, 100000));
+
+Query OK, 100000 rows affected (0.29 sec)
+
+mysql> select ds_hll_count_distinct(id), ds_hll_count_distinct(province), ds_hll_count_distinct(age), ds_hll_count_distinct(dt) from t1 order by 1, 2;
++---------------------------+---------------------------------+----------------------------+---------------------------+
+| ds_hll_count_distinct(id) | ds_hll_count_distinct(province) | ds_hll_count_distinct(age) | ds_hll_count_distinct(dt) |
++---------------------------+---------------------------------+----------------------------+---------------------------+
+|                    100090 |                          100140 |                        100 |                         1 |
++---------------------------+---------------------------------+----------------------------+---------------------------+
+1 row in set (0.07 sec)
+
+mysql> select ds_hll_count_distinct(id, 21), ds_hll_count_distinct(province, 21), ds_hll_count_distinct(age, 21), ds_hll_count_distinct(dt, 21) from t1 order by 1, 2;
++-------------------------------+-------------------------------------+--------------------------------+-------------------------------+
+| ds_hll_count_distinct(id, 21) | ds_hll_count_distinct(province, 21) | ds_hll_count_distinct(age, 21) | ds_hll_count_distinct(dt, 21) |
++-------------------------------+-------------------------------------+--------------------------------+-------------------------------+
+|                         99995 |                              100001 |                            100 |                             1 |
++-------------------------------+-------------------------------------+--------------------------------+-------------------------------+
+1 row in set (0.07 sec)
+
+
+mysql> select ds_hll_count_distinct(id, 10, "HLL_8"), ds_hll_count_distinct(province, 10, "HLL_8"), ds_hll_count_distinct(age, 10, "HLL_8"), ds_hll_count_distinct(dt, 10, "HLL_8") from t1 order by 1, 2;
++----------------------------------------+----------------------------------------------+-----------------------------------------+----------------------------------------+
+| ds_hll_count_distinct(id, 10, 'HLL_8') | ds_hll_count_distinct(province, 10, 'HLL_8') | ds_hll_count_distinct(age, 10, 'HLL_8') | ds_hll_count_distinct(dt, 10, 'HLL_8') |
++----------------------------------------+----------------------------------------------+-----------------------------------------+----------------------------------------+
+|                                  99844 |                                       101905 |                                      96 |                                      1 |
++----------------------------------------+----------------------------------------------+-----------------------------------------+----------------------------------------+
+1 row in set (0.09 sec)
+
+```
+
+## Keywords
+
+DS_HLL_COUNT_DISTINCT,APPROX_COUNT_DISTINCT
diff --git a/docs/zh/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md b/docs/zh/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md
new file mode 100644
index 00000000000000..7cb8d85c136b13
--- /dev/null
+++ b/docs/zh/sql-reference/sql-functions/aggregate-functions/ds_hll_count_distinct.md
@@ -0,0 +1,56 @@
+# ds_hll_count_distinct
+
+返回聚合函数的近似值，结果类似于 COUNT(DISTINCT col)。相似函数为 APPROX_COUNT_DISTINCT(expr)。
+
+相较于 COUNT DISTINCT 速度更快，并且使用固定大小的内存，因此基于高基数列使用时内存占用更少。
+
+相较于 APPROX_COUNT_DISTINCT(expr) 速度更慢，但由于 Apache Datasketches 的优势，导致其精度更高。更多信息，参考 [HyperLogLog Sketches](https://datasketches.apache.org/docs/HLL/HllSketches.html)。
+
+## 语法
+
+```Haskell
+ds_hll_count_distinct(expr, [log_k], [tgt_type])
+```
+- `log_k`：必须为整数。范围：[4, 21]。默认值：17。
+- `tgt_type`：有效值为 `HLL_4`、`HLL_6`（默认）以及 `HLL_8`。
+
+## 示例
+
+```plain text
+mysql> CREATE TABLE t1 (
+    ->   id BIGINT NOT NULL,
+    ->   province VARCHAR(64),
+    ->   age SMALLINT,
+    ->   dt VARCHAR(10) NOT NULL
+    -> )
+    -> DUPLICATE KEY(id)
+    -> DISTRIBUTED BY HASH(id) BUCKETS 4;
+Query OK, 0 rows affected (0.02 sec)
+mysql> insert into t1 select generate_series, generate_series, generate_series % 100, "2024-07-24" from table(generate_series(1, 100000));
+Query OK, 100000 rows affected (0.29 sec)
+mysql> select ds_hll_count_distinct(id), ds_hll_count_distinct(province), ds_hll_count_distinct(age), ds_hll_count_distinct(dt) from t1 order by 1, 2;
++---------------------------+---------------------------------+----------------------------+---------------------------+
+| ds_hll_count_distinct(id) | ds_hll_count_distinct(province) | ds_hll_count_distinct(age) | ds_hll_count_distinct(dt) |
++---------------------------+---------------------------------+----------------------------+---------------------------+
+|                    100090 |                          100140 |                        100 |                         1 |
++---------------------------+---------------------------------+----------------------------+---------------------------+
+1 row in set (0.07 sec)
+mysql> select ds_hll_count_distinct(id, 21), ds_hll_count_distinct(province, 21), ds_hll_count_distinct(age, 21), ds_hll_count_distinct(dt, 21) from t1 order by 1, 2;
++-------------------------------+-------------------------------------+--------------------------------+-------------------------------+
+| ds_hll_count_distinct(id, 21) | ds_hll_count_distinct(province, 21) | ds_hll_count_distinct(age, 21) | ds_hll_count_distinct(dt, 21) |
++-------------------------------+-------------------------------------+--------------------------------+-------------------------------+
+|                         99995 |                              100001 |                            100 |                             1 |
++-------------------------------+-------------------------------------+--------------------------------+-------------------------------+
+1 row in set (0.07 sec)
+mysql> select ds_hll_count_distinct(id, 10, "HLL_8"), ds_hll_count_distinct(province, 10, "HLL_8"), ds_hll_count_distinct(age, 10, "HLL_8"), ds_hll_count_distinct(dt, 10, "HLL_8") from t1 order by 1, 2;
++----------------------------------------+----------------------------------------------+-----------------------------------------+----------------------------------------+
+| ds_hll_count_distinct(id, 10, 'HLL_8') | ds_hll_count_distinct(province, 10, 'HLL_8') | ds_hll_count_distinct(age, 10, 'HLL_8') | ds_hll_count_distinct(dt, 10, 'HLL_8') |
++----------------------------------------+----------------------------------------------+-----------------------------------------+----------------------------------------+
+|                                  99844 |                                       101905 |                                      96 |                                      1 |
++----------------------------------------+----------------------------------------------+-----------------------------------------+----------------------------------------+
+1 row in set (0.09 sec)
+```
+
+## Keywords
+
+DS_HLL_COUNT_DISTINCT,APPROX_COUNT_DISTINCT

From dae0946a90209fbcc8163a49bf878b050ce5b289 Mon Sep 17 00:00:00 2001
From: Connor Brennan <cbrennan@pinterest.com>
Date: Sun, 19 Jan 2025 20:11:52 -0500
Subject: [PATCH 42/71] [BugFix] Call StarOsAgent.prepare wherever serviceId is
 required (#54047)

---
 .../src/main/java/com/starrocks/lake/StarOSAgent.java    | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/StarOSAgent.java b/fe/fe-core/src/main/java/com/starrocks/lake/StarOSAgent.java
index b7efd13a9b250f..588a96de0f7d3c 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/StarOSAgent.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/StarOSAgent.java
@@ -145,6 +145,7 @@ public String getRawServiceId() {
     }
 
     public String addFileStore(FileStoreInfo fsInfo) throws DdlException {
+        prepare();
         try {
             return client.addFileStore(fsInfo, serviceId);
         } catch (StarClientException e) {
@@ -153,6 +154,7 @@ public String addFileStore(FileStoreInfo fsInfo) throws DdlException {
     }
 
     public void removeFileStoreByName(String fsName) throws DdlException {
+        prepare();
         try {
             client.removeFileStoreByName(fsName, serviceId);
         } catch (StarClientException e) {
@@ -161,6 +163,7 @@ public void removeFileStoreByName(String fsName) throws DdlException {
     }
 
     public void updateFileStore(FileStoreInfo fsInfo) throws DdlException {
+        prepare();
         try {
             client.updateFileStore(fsInfo, serviceId);
         } catch (StarClientException e) {
@@ -169,6 +172,7 @@ public void updateFileStore(FileStoreInfo fsInfo) throws DdlException {
     }
 
     public FileStoreInfo getFileStoreByName(String fsName) throws DdlException {
+        prepare();
         try {
             return client.getFileStoreByName(fsName, serviceId);
         } catch (StarClientException e) {
@@ -180,6 +184,7 @@ public FileStoreInfo getFileStoreByName(String fsName) throws DdlException {
     }
 
     public FileStoreInfo getFileStore(String fsKey) throws DdlException {
+        prepare();
         try {
             return client.getFileStore(fsKey, serviceId);
         } catch (StarClientException e) {
@@ -191,6 +196,7 @@ public FileStoreInfo getFileStore(String fsKey) throws DdlException {
     }
 
     public List<FileStoreInfo> listFileStore() throws DdlException {
+        prepare();
         try {
             return client.listFileStore(serviceId);
         } catch (StarClientException e) {
@@ -215,6 +221,7 @@ private static String constructTablePath(long dbId, long tableId) {
     }
 
     public FilePathInfo allocateFilePath(long dbId, long tableId) throws DdlException {
+        prepare();
         try {
             FileStoreType fsType = getFileStoreType(Config.cloud_native_storage_type);
             if (fsType == null || fsType == FileStoreType.INVALID) {
@@ -230,6 +237,7 @@ public FilePathInfo allocateFilePath(long dbId, long tableId) throws DdlExceptio
     }
 
     public FilePathInfo allocateFilePath(String storageVolumeId, long dbId, long tableId) throws DdlException {
+        prepare();
         try {
             String suffix = constructTablePath(dbId, tableId);
             FilePathInfo pathInfo = client.allocateFilePath(serviceId, storageVolumeId, suffix);
@@ -300,6 +308,7 @@ private long getWorker(String workerIpPort) throws DdlException {
     }
 
     public long getWorkerTabletNum(String workerIpPort) {
+        prepare();
         try {
             WorkerInfo workerInfo = client.getWorkerInfo(serviceId, workerIpPort);
             return workerInfo.getTabletNum();

From f451b656fc5b9903b5acc78a8ced140fe9897720 Mon Sep 17 00:00:00 2001
From: wangshisan <ganggewang@tencent.com>
Date: Mon, 20 Jan 2025 10:55:11 +0800
Subject: [PATCH 43/71] [BugFix] Fix shared UDF concurrent load issue (#54779)

Signed-off-by: ganggewang <ganggewang@tencent.com>
---
 be/src/runtime/user_function_cache.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp
index f89012addd2539..22fe0d9874bc62 100644
--- a/be/src/runtime/user_function_cache.cpp
+++ b/be/src/runtime/user_function_cache.cpp
@@ -262,7 +262,9 @@ Status UserFunctionCache::_load_cache_entry(const std::string& url, UserFunction
         RETURN_IF_ERROR(_download_lib(url, entry));
     }
 
-    RETURN_IF_ERROR(_load_cache_entry_internal(url, entry, loader));
+    if (!entry->is_loaded.load()) {
+        RETURN_IF_ERROR(_load_cache_entry_internal(url, entry, loader));
+    }
     return Status::OK();
 }
 

From ecfeb0e3b42a5f39579e84426d399d51ebb6f54e Mon Sep 17 00:00:00 2001
From: RyanZ <dirtysalt1987@gmail.com>
Date: Sun, 19 Jan 2025 19:23:34 -0800
Subject: [PATCH 44/71] [Enhancement] support all tables using pk/fk
 optimization (#55220)

Signed-off-by: yanz <dirtysalt1987@gmail.com>
---
 .../com/starrocks/catalog/BaseTableInfo.java  |  14 +-
 .../java/com/starrocks/catalog/HiveTable.java |  12 --
 .../java/com/starrocks/catalog/Table.java     |  38 +++++-
 .../optimizer/UKFKConstraintsCollector.java   |  34 ++---
 .../JoinReorderCardinalityPreserving.java     |  11 +-
 .../rule/transformation/pruner/CPBiRel.java   |  18 +--
 .../transformation/pruner/CPJoinGardener.java |  18 +--
 .../pruner/CboTablePruneRule.java             |  41 +++---
 .../pruner/UniquenessBasedTablePruneRule.java |  15 +--
 .../starrocks/catalog/IcebergTableTest.java   |  34 +++++
 test/sql/test_iceberg/R/test_pkfk_property    | 120 ++++++++++++++++++
 test/sql/test_iceberg/T/test_pkfk_property    |  76 +++++++++++
 12 files changed, 337 insertions(+), 94 deletions(-)
 create mode 100644 test/sql/test_iceberg/R/test_pkfk_property
 create mode 100644 test/sql/test_iceberg/T/test_pkfk_property

diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/BaseTableInfo.java b/fe/fe-core/src/main/java/com/starrocks/catalog/BaseTableInfo.java
index 9f7ff9807a9cfe..e9ad627433290d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/BaseTableInfo.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/BaseTableInfo.java
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-
 package com.starrocks.catalog;
 
 import com.google.common.base.Joiner;
@@ -21,6 +20,7 @@
 import com.google.common.base.Strings;
 import com.google.gson.annotations.SerializedName;
 import com.starrocks.server.CatalogMgr;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * BaseTableInfo is used for MaterializedView persisted as a base table's meta info which can be an olap
@@ -112,6 +112,7 @@ public long getTableId() {
 
     /**
      * Called when a table is renamed.
+     *
      * @param newTable the new table with the new table name
      */
     public void onTableRename(Table newTable, String oldTableName) {
@@ -164,4 +165,15 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hashCode(catalogName, dbId, tableId, dbName, tableIdentifier, tableName);
     }
+
+    public boolean matchTable(Table t) {
+        if (isInternalCatalog()) {
+            return tableId == t.getId();
+        } else {
+            return StringUtils.equals(catalogName, t.getCatalogName()) &&
+                    StringUtils.equals(dbName, t.getCatalogDBName()) &&
+                    StringUtils.equals(tableName, t.getCatalogTableName()) &&
+                    StringUtils.equals(tableIdentifier, t.getTableIdentifier());
+        }
+    }
 }
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java
index 62a7aa2fea1a20..1c0e1341749a06 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/HiveTable.java
@@ -45,8 +45,6 @@
 import com.starrocks.analysis.DescriptorTable.ReferencedPartitionInfo;
 import com.starrocks.analysis.Expr;
 import com.starrocks.analysis.LiteralExpr;
-import com.starrocks.catalog.constraint.ForeignKeyConstraint;
-import com.starrocks.catalog.constraint.UniqueConstraint;
 import com.starrocks.common.Config;
 import com.starrocks.common.util.TimeUtils;
 import com.starrocks.common.util.concurrent.lock.LockType;
@@ -410,16 +408,6 @@ public String toString() {
         return sb.toString();
     }
 
-    @Override
-    public List<UniqueConstraint> getUniqueConstraints() {
-        return uniqueConstraints;
-    }
-
-    @Override
-    public List<ForeignKeyConstraint> getForeignKeyConstraints() {
-        return foreignKeyConstraints;
-    }
-
     @Override
     public boolean supportInsert() {
         return true;
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java
index 5c3f59c75ab697..b6054fd6bcbf1d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Table.java
@@ -34,6 +34,7 @@
 
 package com.starrocks.catalog;
 
+import com.google.common.base.Function;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
@@ -491,6 +492,10 @@ public List<Column> getColumns() {
         return new ArrayList<>(nameToColumn.values());
     }
 
+    public List<Column> getKeyColumns() {
+        return null;
+    }
+
     public void addColumn(Column column) {
         fullSchema.add(column);
         nameToColumn.put(column.getName(), column);
@@ -512,8 +517,6 @@ public TTableDescriptor toThrift(List<ReferencedPartitionInfo> partitions) {
         return null;
     }
 
-
-
     @Override
     public void gsonPostProcess() throws IOException {
         updateSchemaIndex();
@@ -828,4 +831,35 @@ public boolean isTable() {
     public boolean isSupportBackupRestore() {
         return isOlapTableOrMaterializedView() || isOlapView();
     }
+
+    // Sometimes when we prune a table but want to preserve a single column
+    // for olap table, we could use first key column
+    // for other tables, probably we could choose the most narrow column for performance.
+    // but theoretically we can choose any column.
+    public Column getPresentivateColumn() {
+        List<Column> keyColumns = getKeyColumns();
+        if (keyColumns != null && keyColumns.size() > 0) {
+            return keyColumns.get(0);
+        }
+        List<UniqueConstraint> uniqueConstraintList = getUniqueConstraints();
+        if (uniqueConstraintList != null) {
+            for (UniqueConstraint uc : uniqueConstraintList) {
+                for (ColumnId id : uc.getUniqueColumns()) {
+                    Column c = getColumn(id);
+                    if (c != null) {
+                        return c;
+                    }
+                }
+            }
+        }
+        return getColumns().stream().min((c1, c2) -> {
+            Function<Column, Integer> ff = c -> {
+                if (c.getType().isScalarType()) {
+                    return c.getType().getTypeSize();
+                }
+                return Integer.MAX_VALUE;
+            };
+            return ff.apply(c1) - ff.apply(c2);
+        }).get();
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/UKFKConstraintsCollector.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/UKFKConstraintsCollector.java
index eb34216133fd01..d135ae32d57ab1 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/UKFKConstraintsCollector.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/UKFKConstraintsCollector.java
@@ -18,7 +18,7 @@
 import com.starrocks.analysis.JoinOperator;
 import com.starrocks.catalog.BaseTableInfo;
 import com.starrocks.catalog.Column;
-import com.starrocks.catalog.OlapTable;
+import com.starrocks.catalog.Table;
 import com.starrocks.catalog.constraint.ForeignKeyConstraint;
 import com.starrocks.catalog.constraint.UniqueConstraint;
 import com.starrocks.common.Pair;
@@ -28,9 +28,9 @@
 import com.starrocks.sql.optimizer.operator.UKFKConstraints;
 import com.starrocks.sql.optimizer.operator.logical.LogicalAggregationOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
-import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
+import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
 import com.starrocks.sql.optimizer.operator.physical.PhysicalJoinOperator;
-import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator;
+import com.starrocks.sql.optimizer.operator.physical.PhysicalScanOperator;
 import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
@@ -128,45 +128,37 @@ public Void visitLogicalTableScan(OptExpression optExpression, Void context) {
         if (!visitChildren(optExpression, context)) {
             return null;
         }
-        if (!(optExpression.getOp() instanceof LogicalOlapScanOperator)) {
-            optExpression.setConstraints(new UKFKConstraints());
-            return null;
-        }
-        LogicalOlapScanOperator scanOperator = optExpression.getOp().cast();
+        LogicalScanOperator scanOperator = optExpression.getOp().cast();
         ColumnRefSet usedColumns = new ColumnRefSet();
         if (scanOperator.getPredicate() != null) {
             usedColumns.union(scanOperator.getPredicate().getUsedColumns());
         }
-        OlapTable table = (OlapTable) scanOperator.getTable();
+        Table table = scanOperator.getTable();
         Map<String, ColumnRefOperator> columnNameToColRefMap = scanOperator.getColumnNameToColRefMap();
 
-        visitOlapTable(optExpression, table, columnNameToColRefMap, usedColumns);
+        visitTable(optExpression, table, columnNameToColRefMap, usedColumns);
 
         return null;
     }
 
     @Override
-    public Void visitPhysicalOlapScan(OptExpression optExpression, Void context) {
+    public Void visitPhysicalScan(OptExpression optExpression, Void context) {
         if (!visitChildren(optExpression, context)) {
             return null;
         }
-        if (!(optExpression.getOp() instanceof PhysicalOlapScanOperator)) {
-            optExpression.setConstraints(new UKFKConstraints());
-            return null;
-        }
-        PhysicalOlapScanOperator scanOperator = optExpression.getOp().cast();
+        PhysicalScanOperator scanOperator = optExpression.getOp().cast();
         ColumnRefSet usedColumns = scanOperator.getUsedColumns();
-        OlapTable table = (OlapTable) scanOperator.getTable();
+        Table table = scanOperator.getTable();
         Map<String, ColumnRefOperator> columnNameToColRefMap = scanOperator.getColRefToColumnMetaMap().entrySet()
                 .stream().collect(Collectors.toMap(entry -> entry.getValue().getName(), Map.Entry::getKey));
 
-        visitOlapTable(optExpression, table, columnNameToColRefMap, usedColumns);
+        visitTable(optExpression, table, columnNameToColRefMap, usedColumns);
 
         return null;
     }
 
-    private void visitOlapTable(OptExpression optExpression, OlapTable table,
-                                Map<String, ColumnRefOperator> columnNameToColRefMap, ColumnRefSet usedColumns) {
+    private void visitTable(OptExpression optExpression, Table table,
+                            Map<String, ColumnRefOperator> columnNameToColRefMap, ColumnRefSet usedColumns) {
         ColumnRefSet outputColumns = optExpression.getRowOutputInfo().getOutputColumnRefSet();
         UKFKConstraints constraint = new UKFKConstraints();
         if (table.hasUniqueConstraints()) {
@@ -205,7 +197,7 @@ private void visitOlapTable(OptExpression optExpression, OlapTable table,
         }
 
         if (table.hasForeignKeyConstraints()) {
-            Column firstKeyColumn = table.getKeyColumns().get(0);
+            Column firstKeyColumn = table.getPresentivateColumn();
             ColumnRefOperator firstKeyColumnRef = columnNameToColRefMap.get(firstKeyColumn.getName());
             List<ForeignKeyConstraint> fkConstraints = table.getForeignKeyConstraints();
             for (ForeignKeyConstraint fkConstraint : fkConstraints) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/join/JoinReorderCardinalityPreserving.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/join/JoinReorderCardinalityPreserving.java
index 9e66f82639109f..ac61e7a621a371 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/join/JoinReorderCardinalityPreserving.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/join/JoinReorderCardinalityPreserving.java
@@ -19,7 +19,7 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.starrocks.analysis.BinaryType;
-import com.starrocks.catalog.OlapTable;
+import com.starrocks.catalog.Table;
 import com.starrocks.common.Pair;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
@@ -283,16 +283,13 @@ protected void enumerate() {
             return;
         }
         // We only try to reorder atoms which are LogicalScanOperator, since at present,
-        // we can extract cardinality-preserving relation from a pair of OlapTable.
+        // we can extract cardinality-preserving relation from a pair of table.
         List<OptExpression> scanOps = atomOptExprs.stream().filter(opt -> {
             if (!(opt.getOp() instanceof LogicalScanOperator)) {
                 return false;
             }
             LogicalScanOperator scanOp = opt.getOp().cast();
-            if (!(scanOp.getTable() instanceof OlapTable)) {
-                return false;
-            }
-            OlapTable table = ((OlapTable) scanOp.getTable());
+            Table table = scanOp.getTable();
             return table.hasUniqueConstraints() || table.hasForeignKeyConstraints();
         }).collect(Collectors.toList());
 
@@ -302,7 +299,7 @@ protected void enumerate() {
 
         // Construct a mapping from ColumnRefOperator to OptExpression, later we
         // only try matches equality predicate that references ColumnRefOperators backed
-        // by real columns of OlapTables to pairs of ColumnRefOperator of cardinality-preserving
+        // by real columns of tables to pairs of ColumnRefOperator of cardinality-preserving
         // relation of two OptExpression.
         Map<ColumnRefOperator, OptExpression> colRefToScanNodes = Maps.newHashMap();
         for (OptExpression scanOp : scanOps) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPBiRel.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPBiRel.java
index fb636d18216a32..d5f95f937fbda5 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPBiRel.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPBiRel.java
@@ -15,14 +15,13 @@
 package com.starrocks.sql.optimizer.rule.transformation.pruner;
 
 import com.google.common.collect.Lists;
-import com.starrocks.catalog.OlapTable;
+import com.starrocks.catalog.Table;
 import com.starrocks.catalog.constraint.ForeignKeyConstraint;
 import com.starrocks.common.Pair;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -30,7 +29,7 @@
 import java.util.stream.Collectors;
 
 // Cardinality-preserving bi-relation, two scan operator form a CPBiRel instance
-// when their underlying OlapTable has {foreign,primary,unique} key constraints and
+// when their underlying Table has {foreign,primary,unique} key constraints and
 // the join equality predicates can match these constraints.
 public class CPBiRel {
     // CPBiRel is a bi-relation consisting OptExpression
@@ -85,10 +84,10 @@ public Set<Pair<ColumnRefOperator, ColumnRefOperator>> getPairs() {
 
     // check if referencing table's foreign key constraint aims at referenced Table
     public static boolean isForeignKeyConstraintReferenceToUniqueKey(
-            OlapTable baseTable,
+            Table baseTable,
             ForeignKeyConstraint foreignKeyConstraint,
-            OlapTable referencedTable) {
-        if (foreignKeyConstraint.getParentTableInfo().getTableId() != referencedTable.getId()) {
+            Table referencedTable) {
+        if (!foreignKeyConstraint.getParentTableInfo().matchTable(referencedTable)) {
             return false;
         }
         Set<String> referencedColumnNames =
@@ -101,11 +100,8 @@ public static List<CPBiRel> extractCPBiRels(OptExpression lhs, OptExpression rhs
                                                 boolean leftToRight) {
         LogicalScanOperator lhsScanOp = lhs.getOp().cast();
         LogicalScanOperator rhsScanOp = rhs.getOp().cast();
-        if (!(lhsScanOp.getTable() instanceof OlapTable) || !(rhsScanOp.getTable() instanceof OlapTable)) {
-            return Collections.emptyList();
-        }
-        OlapTable lhsTable = (OlapTable) lhsScanOp.getTable();
-        OlapTable rhsTable = (OlapTable) rhsScanOp.getTable();
+        Table lhsTable = lhsScanOp.getTable();
+        Table rhsTable = rhsScanOp.getTable();
         Map<String, ColumnRefOperator> lhsColumnName2ColRef =
                 lhsScanOp.getColumnMetaToColRefMap().entrySet().stream()
                         .collect(Collectors.toMap(e -> e.getKey().getName(), Map.Entry::getValue));
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPJoinGardener.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPJoinGardener.java
index 841441d83a362a..beb83a57664d7a 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPJoinGardener.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CPJoinGardener.java
@@ -21,7 +21,7 @@
 import com.google.common.collect.Sets;
 import com.starrocks.analysis.JoinOperator;
 import com.starrocks.catalog.Column;
-import com.starrocks.catalog.OlapTable;
+import com.starrocks.catalog.Table;
 import com.starrocks.catalog.Type;
 import com.starrocks.common.Pair;
 import com.starrocks.common.util.UnionFind;
@@ -33,7 +33,6 @@
 import com.starrocks.sql.optimizer.operator.Operator;
 import com.starrocks.sql.optimizer.operator.OperatorBuilderFactory;
 import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
-import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
@@ -271,11 +270,8 @@ public Boolean visitLogicalTableScan(OptExpression optExpression, Void context)
         if (scanOp.hasLimit()) {
             return visit(optExpression, context);
         }
-        //TODO(by satanson): non-OlapTable will be supported in future
-        if (!(scanOp.getTable() instanceof OlapTable)) {
-            return visit(optExpression, context);
-        }
-        OlapTable table = ((OlapTable) scanOp.getTable());
+
+        Table table = scanOp.getTable();
         // A Table that has no PK/UK/FK can not associate with other tables via
         // cardinality-preserving relations.
         if (!table.hasUniqueConstraints() && !table.hasForeignKeyConstraints()) {
@@ -652,7 +648,7 @@ private PruneContext markPrunedTableOfHubCPNode(CPNode root, ColumnRefSet origin
 
         Map<Long, List<CPNode>> tableIdToChildGroups = Maps.newHashMap();
         for (CPNode child : children) {
-            LogicalOlapScanOperator scanOp = child.getValue().getOp().cast();
+            LogicalScanOperator scanOp = child.getValue().getOp().cast();
             Long tableId = scanOp.getTable().getId();
             tableIdToChildGroups.computeIfAbsent(tableId, k -> Lists.newArrayList()).add(child);
         }
@@ -1020,10 +1016,8 @@ public Pair<Operator, ColumnRefSet> visitLogicalTableScan(OptExpression optExpre
                 }
             });
             if (newColRefToColMetaMap.isEmpty()) {
-                Preconditions.checkArgument(scanOperator.getTable() instanceof OlapTable);
-                OlapTable table = (OlapTable) scanOperator.getTable();
-                Preconditions.checkArgument(!table.getKeyColumns().isEmpty());
-                Column firstKeyColumn = table.getKeyColumns().get(0);
+                Table table = scanOperator.getTable();
+                Column firstKeyColumn = table.getPresentivateColumn();
                 ColumnRefOperator firstKeyColRef = scanOperator.getColumnMetaToColRefMap().get(firstKeyColumn);
                 newColRefToColMetaMap.put(firstKeyColRef, firstKeyColumn);
             }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CboTablePruneRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CboTablePruneRule.java
index 65564a9ee46e7f..7e0ee121397463 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CboTablePruneRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/CboTablePruneRule.java
@@ -22,7 +22,7 @@
 import com.google.common.collect.Sets;
 import com.starrocks.analysis.JoinOperator;
 import com.starrocks.catalog.Column;
-import com.starrocks.catalog.OlapTable;
+import com.starrocks.catalog.Table;
 import com.starrocks.common.Pair;
 import com.starrocks.sql.optimizer.JoinHelper;
 import com.starrocks.sql.optimizer.OptExpression;
@@ -34,7 +34,7 @@
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.Projection;
 import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
-import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
+import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
 import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
@@ -55,10 +55,9 @@
 import java.util.stream.Collectors;
 
 public class CboTablePruneRule extends TransformationRule {
-    public CboTablePruneRule() {
+    private CboTablePruneRule() {
         super(RuleType.TF_CBO_TABLE_PRUNE_RULE,
-                Pattern.create(OperatorType.LOGICAL_JOIN, OperatorType.LOGICAL_OLAP_SCAN,
-                        OperatorType.LOGICAL_OLAP_SCAN));
+                Pattern.create(OperatorType.LOGICAL_JOIN, OperatorType.PATTERN_SCAN, OperatorType.PATTERN_SCAN));
     }
 
     // the count of joins of these types exceeds certain threshold, this Rule would be time-consuming
@@ -78,9 +77,13 @@ public static CboTablePruneRule getInstance() {
     @Override
     public boolean check(OptExpression input, OptimizerContext context) {
         LogicalJoinOperator joinOp = input.getOp().cast();
-        return joinOp.getJoinType() == JoinOperator.INNER_JOIN ||
+        boolean supportedJoinType = (joinOp.getJoinType() == JoinOperator.INNER_JOIN ||
                 joinOp.getJoinType() == JoinOperator.LEFT_OUTER_JOIN ||
-                joinOp.getJoinType() == JoinOperator.RIGHT_OUTER_JOIN;
+                joinOp.getJoinType() == JoinOperator.RIGHT_OUTER_JOIN);
+        if (!supportedJoinType) {
+            return false;
+        }
+        return true;
     }
 
     @Override
@@ -89,8 +92,8 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
         JoinOperator joinType = joinOp.getJoinType();
         OptExpression lhs = input.inputAt(0);
         OptExpression rhs = input.inputAt(1);
-        LogicalOlapScanOperator lhsScanOp = input.inputAt(0).getOp().cast();
-        LogicalOlapScanOperator rhsScanOp = input.inputAt(1).getOp().cast();
+        LogicalScanOperator lhsScanOp = input.inputAt(0).getOp().cast();
+        LogicalScanOperator rhsScanOp = input.inputAt(1).getOp().cast();
         Pair<List<BinaryPredicateOperator>, List<ScalarOperator>> onPredicates =
                 JoinHelper.separateEqualPredicatesFromOthers(input);
         List<BinaryPredicateOperator> eqOnPredicates = onPredicates.first;
@@ -137,8 +140,8 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
         return Collections.emptyList();
     }
 
-    private boolean matchUniqueConstraints(LogicalOlapScanOperator scanOp, List<ColumnRefOperator> colRefs) {
-        OlapTable table = (OlapTable) scanOp.getTable();
+    private boolean matchUniqueConstraints(LogicalScanOperator scanOp, List<ColumnRefOperator> colRefs) {
+        Table table = scanOp.getTable();
         if (!table.hasUniqueConstraints()) {
             return false;
         }
@@ -219,8 +222,8 @@ private List<OptExpression> tryToHandleLeftOrRightJoinOnSameTableAsInnerJoin(
             Optional<ScalarOperator> optOtherJoinOnPredicate) {
         OptExpression lhs = joinOptExpression.inputAt(0);
         OptExpression rhs = joinOptExpression.inputAt(1);
-        LogicalOlapScanOperator lhsScanOp = lhs.getOp().cast();
-        LogicalOlapScanOperator rhsScanOp = rhs.getOp().cast();
+        LogicalScanOperator lhsScanOp = lhs.getOp().cast();
+        LogicalScanOperator rhsScanOp = rhs.getOp().cast();
         LogicalJoinOperator joinOp = joinOptExpression.getOp().cast();
         JoinOperator joinType = joinOp.getJoinType();
         // If there exist other predicates in on-clause except equality predicates, left/right join
@@ -230,7 +233,7 @@ private List<OptExpression> tryToHandleLeftOrRightJoinOnSameTableAsInnerJoin(
         }
 
         // must be the same table
-        if (lhsScanOp.getTable().getId() != rhsScanOp.getTable().getId()) {
+        if (!lhsScanOp.getTable().equals(rhsScanOp.getTable())) {
             return Collections.emptyList();
         }
 
@@ -238,7 +241,7 @@ private List<OptExpression> tryToHandleLeftOrRightJoinOnSameTableAsInnerJoin(
         if (eqColRefPairs.stream().anyMatch(p -> p.first.isNullable() || p.second.isNullable())) {
             return Collections.emptyList();
         }
-        // rhs of LEFT JOIN or lhs of RIGHT JOIN must output all rows of the OlapTable
+        // rhs of LEFT JOIN or lhs of RIGHT JOIN must output all rows of the table
         if ((joinType.isLeftOuterJoin() && (rhsScanOp.getPredicate() != null || rhsScanOp.hasLimit())) ||
                 (joinType.isRightOuterJoin() && (lhsScanOp.getPredicate() != null || lhsScanOp.hasLimit()))) {
             return Collections.emptyList();
@@ -256,8 +259,8 @@ List<OptExpression> handleSameTableInnerJoin(OptExpression joinOptExpression,
                                                  Optional<ScalarOperator> optOtherJoinOnPredicate) {
         OptExpression lhs = joinOptExpression.inputAt(0);
         OptExpression rhs = joinOptExpression.inputAt(1);
-        LogicalOlapScanOperator lhsScanOp = lhs.getOp().cast();
-        LogicalOlapScanOperator rhsScanOp = rhs.getOp().cast();
+        LogicalScanOperator lhsScanOp = lhs.getOp().cast();
+        LogicalScanOperator rhsScanOp = rhs.getOp().cast();
         Map<Column, ColumnRefOperator> lhsColToColRef = lhsScanOp.getColumnMetaToColRefMap();
         Map<Column, ColumnRefOperator> rhsColToColRef = rhsScanOp.getColumnMetaToColRefMap();
         Map<ColumnRefOperator, ScalarOperator> rewriteMapping = Maps.newHashMap();
@@ -334,8 +337,8 @@ List<OptExpression> pruneInnerJoin(OptExpression joinOp, OptExpression retainOp,
 
         // create a new ScanOperator who unifies join operator, retain-side scan operator and
         // prune-side scan operators.
-        LogicalOlapScanOperator.Builder newOpBuilder =
-                (LogicalOlapScanOperator.Builder) OperatorBuilderFactory.build(retainOp.getOp())
+        LogicalScanOperator.Builder newOpBuilder =
+                (LogicalScanOperator.Builder) OperatorBuilderFactory.build(retainOp.getOp())
                         .withOperator(retainOp.getOp())
                         .setPredicate(newPredicate)
                         .setProjection(new Projection(newColRefMap));
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/UniquenessBasedTablePruneRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/UniquenessBasedTablePruneRule.java
index 0fe7d4e92e6961..58c5bc5fdeed3d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/UniquenessBasedTablePruneRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/pruner/UniquenessBasedTablePruneRule.java
@@ -19,7 +19,7 @@
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.starrocks.analysis.JoinOperator;
-import com.starrocks.catalog.OlapTable;
+import com.starrocks.catalog.Table;
 import com.starrocks.common.Pair;
 import com.starrocks.sql.optimizer.JoinHelper;
 import com.starrocks.sql.optimizer.OptExpression;
@@ -31,8 +31,8 @@
 import com.starrocks.sql.optimizer.operator.Projection;
 import com.starrocks.sql.optimizer.operator.logical.LogicalAggregationOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
-import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalProjectOperator;
+import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
 import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
@@ -98,16 +98,13 @@ public Boolean visit(OptExpression optExpression, Void context) {
 
         @Override
         public Boolean visitLogicalTableScan(OptExpression optExpression, Void context) {
-            if (!(optExpression.getOp() instanceof LogicalOlapScanOperator)) {
-                return visit(optExpression, context);
-            }
-            LogicalOlapScanOperator scanOp = optExpression.getOp().cast();
-            OlapTable table = (OlapTable) scanOp.getTable();
+            LogicalScanOperator scanOp = optExpression.getOp().cast();
+            Table table = scanOp.getTable();
             if (!table.hasUniqueConstraints()) {
                 return visit(optExpression, context);
             }
             Map<String, ColumnRefOperator> nameToColRefMap = scanOp.getColumnNameToColRefMap();
-            List<ColumnRefSet> uniqueKeys = table.getUniqueConstraints().stream().map(uc ->
+            List<ColumnRefSet> uniqueKeys = table.  getUniqueConstraints().stream().map(uc ->
                     new ColumnRefSet(uc.getUniqueColumnNames(table).stream().map(nameToColRefMap::get)
                             .collect(Collectors.toList()))).collect(Collectors.toList());
             uniqueKeys = propagateThroughProjection(optExpression, uniqueKeys);
@@ -277,7 +274,7 @@ public static class Pruner extends OptExpressionVisitor<Optional<OptExpression>,
         public Optional<OptExpression> visit(OptExpression optExpression, ColumnRefSet context) {
             return Optional.empty();
         }
-        
+
         private Optional<OptExpression> pruneJoin(OptExpression optExpression, OptExpression retainedChd,
                                                   OptExpression prunedChd) {
             Operator joinOp = optExpression.getOp();
diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/IcebergTableTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/IcebergTableTest.java
index f2b7c4b1c6bd03..3a0d1122bc5de2 100644
--- a/fe/fe-core/src/test/java/com/starrocks/catalog/IcebergTableTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/catalog/IcebergTableTest.java
@@ -16,6 +16,7 @@
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.starrocks.catalog.constraint.UniqueConstraint;
 import com.starrocks.common.DdlException;
 import com.starrocks.connector.iceberg.TableTestBase;
 import com.starrocks.server.IcebergTableFactory;
@@ -29,7 +30,9 @@
 import java.util.List;
 import java.util.Map;
 
+import static com.starrocks.catalog.Type.ARRAY_BIGINT;
 import static com.starrocks.catalog.Type.INT;
+import static com.starrocks.catalog.Type.STRING;
 import static com.starrocks.server.ExternalTableFactory.RESOURCE;
 
 public class IcebergTableTest extends TableTestBase {
@@ -69,4 +72,35 @@ public void testCreateTableResourceName(@Mocked Table icebergNativeTable) throws
         IcebergTable table = newBuilder.build();
         Assert.assertEquals(table.getResourceName(), resourceName);
     }
+
+    @Test
+    public void testIcebergTableRepresentativeColumn() {
+        List<Column> columns = Lists.newArrayList(
+                new Column("k1", INT),
+                new Column("k2", STRING),
+                new Column("k3", ARRAY_BIGINT));
+        IcebergTable.Builder tableBuilder = IcebergTable.builder()
+                .setId(1000)
+                .setSrTableName("supplier")
+                .setCatalogName("iceberg_catalog")
+                .setCatalogDBName("iceberg_oss_tpch_1g_parquet_gzip")
+                .setCatalogTableName("supplier")
+                .setFullSchema(columns)
+                .setNativeTable(null)
+                .setIcebergProperties(new HashMap<>());
+        // by default use k1 as column
+        IcebergTable table = tableBuilder.build();
+        {
+            Column c = table.getPresentivateColumn();
+            Assert.assertEquals(c.getName(), "k1");
+        }
+
+        // use k3 as unique column
+        List<ColumnId> uniqueColumns = Lists.newArrayList(columns.get(2).getColumnId());
+        table.setUniqueConstraints(Lists.newArrayList(new UniqueConstraint("cat", "db", "tbl", uniqueColumns)));
+        {
+            Column c = table.getPresentivateColumn();
+            Assert.assertEquals(c.getName(), "k3");
+        }
+    }
 }
diff --git a/test/sql/test_iceberg/R/test_pkfk_property b/test/sql/test_iceberg/R/test_pkfk_property
new file mode 100644
index 00000000000000..cc8e10248fb957
--- /dev/null
+++ b/test/sql/test_iceberg/R/test_pkfk_property
@@ -0,0 +1,120 @@
+-- name: test_pkfk_property
+create external catalog iceberg_sql_test_${uuid0} PROPERTIES ("type"="iceberg", "iceberg.catalog.type"="hive", "iceberg.catalog.hive.metastore.uris"="${iceberg_catalog_hive_metastore_uris}","enable_iceberg_metadata_cache"="true","aws.s3.access_key" = "${oss_ak}","aws.s3.secret_key" = "${oss_sk}","aws.s3.endpoint" = "${oss_endpoint}");
+-- result:
+-- !result
+create database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+-- result:
+-- !result
+use iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+-- result:
+-- !result
+create external table payment (id int, created_at date, status string);
+-- result:
+-- !result
+create external table txn (id int, created_at date, payment_id int);
+-- result:
+-- !result
+INSERT INTO payment (id, created_at, status) VALUES
+(1, '2025-01-01', 'COMPLETED'),
+(2, '2025-01-02', 'PENDING'),
+(3, '2025-01-03', 'FAILED'),
+(4, '2025-01-04', 'COMPLETED'),
+(5, '2025-01-05', 'REFUNDED');
+-- result:
+-- !result
+INSERT INTO txn (id, created_at, payment_id) VALUES
+(101, '2025-01-01', 1),
+(102, '2025-01-02', 2),
+(103, '2025-01-03', 3),
+(104, '2025-01-04', 4),
+(105, '2025-01-05', 5),
+(106, '2025-01-06', 1);
+-- result:
+-- !result
+alter table payment set  ("unique_constraints" = "id");
+-- result:
+-- !result
+ALTER TABLE txn SET ("foreign_key_constraints" = "(payment_id) REFERENCES payment(id)");
+-- result:
+-- !result
+set enable_ukfk_opt = false;
+-- result:
+-- !result
+SET enable_rbo_table_prune=true;
+-- result:
+-- !result
+SET enable_cbo_table_prune=true; 
+SET enable_table_prune_on_update = true;
+-- result:
+-- !result
+function: assert_explain_not_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+-- result:
+None
+-- !result
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn")
+-- result:
+None
+-- !result
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+-- result:
+None
+-- !result
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn")
+-- result:
+None
+-- !result
+set enable_ukfk_opt = true;
+-- result:
+-- !result
+SET enable_rbo_table_prune=false;
+-- result:
+-- !result
+SET enable_cbo_table_prune=false; 
+SET enable_table_prune_on_update = false;
+-- result:
+-- !result
+function: assert_explain_not_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+-- result:
+None
+-- !result
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn")
+-- result:
+None
+-- !result
+function: assert_explain_not_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+-- result:
+None
+-- !result
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn", "payment_id IS NOT NULL")
+-- result:
+None
+-- !result
+set enable_eliminate_agg = false;
+-- result:
+-- !result
+function: assert_explain_contains("select count(1) as cnt, payment.id from payment group by payment.id;", "AGGREGATE")
+-- result:
+None
+-- !result
+set enable_eliminate_agg = true;
+-- result:
+-- !result
+function: assert_explain_not_contains("select count(1) as cnt, payment.id from payment group by payment.id;", "AGGREGATE")
+-- result:
+None
+-- !result
+drop table txn force;
+-- result:
+-- !result
+drop table payment force;
+-- result:
+-- !result
+drop database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+-- result:
+-- !result
+drop catalog iceberg_sql_test_${uuid0};
+-- result:
+-- !result
+set catalog default_catalog;
+-- result:
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_iceberg/T/test_pkfk_property b/test/sql/test_iceberg/T/test_pkfk_property
new file mode 100644
index 00000000000000..01ea237b4ff5dc
--- /dev/null
+++ b/test/sql/test_iceberg/T/test_pkfk_property
@@ -0,0 +1,76 @@
+-- name: test_pkfk_property
+
+create external catalog iceberg_sql_test_${uuid0} PROPERTIES ("type"="iceberg", "iceberg.catalog.type"="hive", "iceberg.catalog.hive.metastore.uris"="${iceberg_catalog_hive_metastore_uris}","enable_iceberg_metadata_cache"="true","aws.s3.access_key" = "${oss_ak}","aws.s3.secret_key" = "${oss_sk}","aws.s3.endpoint" = "${oss_endpoint}");
+
+create database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+use iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+
+create external table payment (id int, created_at date, status string);
+create external table txn (id int, created_at date, payment_id int);
+
+
+INSERT INTO payment (id, created_at, status) VALUES
+(1, '2025-01-01', 'COMPLETED'),
+(2, '2025-01-02', 'PENDING'),
+(3, '2025-01-03', 'FAILED'),
+(4, '2025-01-04', 'COMPLETED'),
+(5, '2025-01-05', 'REFUNDED');
+
+INSERT INTO txn (id, created_at, payment_id) VALUES
+(101, '2025-01-01', 1),
+(102, '2025-01-02', 2),
+(103, '2025-01-03', 3),
+(104, '2025-01-04', 4),
+(105, '2025-01-05', 5),
+(106, '2025-01-06', 1);
+
+-- add table properties
+alter table payment set  ("unique_constraints" = "id");
+ALTER TABLE txn SET ("foreign_key_constraints" = "(payment_id) REFERENCES payment(id)");
+
+-- because two groups of session variables can both work. here we can test one group of session variables.
+set enable_ukfk_opt = false;
+SET enable_rbo_table_prune=true;
+SET enable_cbo_table_prune=true; 
+SET enable_table_prune_on_update = true;
+
+-- payment can be elimiated
+function: assert_explain_not_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn")
+
+-- payment can not be elimiated because txn.id is nullable.
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn")
+
+
+-- switch to another group of session variables
+set enable_ukfk_opt = true;
+SET enable_rbo_table_prune=false;
+SET enable_cbo_table_prune=false; 
+SET enable_table_prune_on_update = false;
+
+-- payment can be elimiated
+function: assert_explain_not_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn left join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn")
+
+-- payment can be elimiated with a extra predicate: payment_id is not null.
+function: assert_explain_not_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.payment")
+function: assert_explain_contains("select txn.id, txn.created_at, txn.payment_id from  txn inner join payment on payment.id = txn.payment_id", "TABLE: iceberg_db_${uuid0}.txn", "payment_id IS NOT NULL")
+
+
+-- test elimiate agg on primary key
+set enable_eliminate_agg = false;
+function: assert_explain_contains("select count(1) as cnt, payment.id from payment group by payment.id;", "AGGREGATE")
+
+set enable_eliminate_agg = true;
+function: assert_explain_not_contains("select count(1) as cnt, payment.id from payment group by payment.id;", "AGGREGATE")
+
+
+-- order matters. if drop table payment first, when drop table txn, it will fail because payment is not found.
+drop table txn force;
+drop table payment force;
+
+drop database iceberg_sql_test_${uuid0}.iceberg_db_${uuid0};
+drop catalog iceberg_sql_test_${uuid0};
+
+set catalog default_catalog;

From 834532c76bbbed754e96540943d83776344af526 Mon Sep 17 00:00:00 2001
From: Yixin Luo <18810541851@163.com>
Date: Mon, 20 Jan 2025 11:38:53 +0800
Subject: [PATCH 45/71] [BugFix] fix ingestion hang because of alter job
 timeout (#55207)

---
 .../java/com/starrocks/alter/AlterJobV2.java   | 18 ++++++++++++------
 .../alter/LakeTableSchemaChangeJob.java        |  4 +---
 .../java/com/starrocks/alter/RollupJobV2.java  |  4 +---
 .../starrocks/alter/SchemaChangeHandler.java   |  2 +-
 .../com/starrocks/alter/SchemaChangeJobV2.java |  4 +---
 5 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java
index e10426c0ad5895..1b238e9c2202a6 100644
--- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java
+++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java
@@ -223,8 +223,10 @@ public long getWarehouseId() {
      */
     public synchronized void run() {
         if (isTimeout()) {
-            cancelHook(cancelImpl("Timeout"));
-            return;
+            if (cancelInternal("Timeout")) {
+                // If this job can't be cancelled, we should execute it.
+                return;
+            }
         }
 
         // create connectcontext
@@ -255,15 +257,19 @@ public synchronized void run() {
                 } // else: handle the new state
             }
         } catch (AlterCancelException e) {
-            cancelHook(cancelImpl(e.getMessage()));
+            cancelInternal(e.getMessage());
         }
     }
 
+    protected boolean cancelInternal(String errMsg) {
+        boolean cancelled = cancelImpl(errMsg);
+        cancelHook(cancelled);
+        return cancelled;
+    }
+
     public boolean cancel(String errMsg) {
         synchronized (this) {
-            boolean cancelled = cancelImpl(errMsg);
-            cancelHook(cancelled);
-            return cancelled;
+            return cancelInternal(errMsg);
         }
     }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java
index 21a6e5e9c0afc0..0cef06b5d7b610 100644
--- a/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java
+++ b/fe/fe-core/src/main/java/com/starrocks/alter/LakeTableSchemaChangeJob.java
@@ -1080,9 +1080,7 @@ public final boolean cancel(String errMsg) {
                 createReplicaLatch.countDownToZero(new Status(TStatusCode.OK, ""));
             }
             synchronized (this) {
-                boolean cancelled = cancelImpl(errMsg);
-                cancelHook(cancelled);
-                return cancelled;
+                return cancelInternal(errMsg);
             }
         } finally {
             isCancelling.set(false);
diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java
index d9538eb4fe84b1..17dda7b2bc508d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java
@@ -770,9 +770,7 @@ public final boolean cancel(String errMsg) {
                 createReplicaLatch.countDownToZero(new Status(TStatusCode.OK, ""));
             }
             synchronized (this) {
-                boolean cancelled = cancelImpl(errMsg);
-                cancelHook(cancelled);
-                return cancelled;
+                return cancelInternal(errMsg);
             }
         } finally {
             isCancelling.set(false);
diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java
index 2ff33e541d59c6..0e62624e58f75f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java
@@ -2110,7 +2110,7 @@ public AlterJobV2 createAlterMetaJob(AlterClause alterClause, Database db, OlapT
             long timeoutSecond = PropertyAnalyzer.analyzeTimeout(properties, Config.alter_table_timeout_second);
             alterMetaJob = new LakeTableAlterMetaJob(GlobalStateMgr.getCurrentState().getNextId(),
                     db.getId(),
-                    olapTable.getId(), olapTable.getName(), timeoutSecond,
+                    olapTable.getId(), olapTable.getName(), timeoutSecond * 1000 /* should be ms*/,
                     TTabletMetaType.ENABLE_PERSISTENT_INDEX, enablePersistentIndex, persistentIndexType);
         } else {
             // shouldn't happen
diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java
index 19c32b871839eb..5545a0ef9ed5a2 100644
--- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java
@@ -946,9 +946,7 @@ public final boolean cancel(String errMsg) {
                 createReplicaLatch.countDownToZero(new Status(TStatusCode.OK, ""));
             }
             synchronized (this) {
-                boolean cancelled = cancelImpl(errMsg);
-                cancelHook(cancelled);
-                return cancelled;
+                return cancelInternal(errMsg);
             }
         } finally {
             isCancelling.set(false);

From 1e977bc243154e8dba259ab23f023bb2bfdf3d97 Mon Sep 17 00:00:00 2001
From: "zihe.liu" <ziheliu1024@gmail.com>
Date: Mon, 20 Jan 2025 13:14:24 +0800
Subject: [PATCH 46/71] [BugFix] Fix bugs of vector index (#55123)

Signed-off-by: zihe.liu <ziheliu1024@gmail.com>
---
 be/src/common/config.h                        |   2 +-
 .../exec/pipeline/scan/olap_chunk_source.cpp  |  22 +-
 be/src/exec/pipeline/scan/olap_chunk_source.h |   3 +-
 .../index/vector/tenann/del_id_filter.h       |  11 +-
 .../vector/tenann/tenann_index_builder.cpp    | 143 ++--
 .../vector/tenann/tenann_index_builder.h      |  29 +-
 .../vector/tenann/tenann_index_utils.cpp      |   7 +-
 .../index/vector/vector_index_builder.h       |  22 +-
 .../vector/vector_index_builder_factory.cpp   |   4 +-
 .../vector/vector_index_builder_factory.h     |   2 +-
 .../index/vector/vector_index_writer.cpp      |  45 +-
 .../index/vector/vector_index_writer.h        |  18 +-
 be/src/storage/rowset/array_column_writer.cpp |   5 +-
 be/src/storage/rowset/segment_iterator.cpp    |   4 +-
 .../starrocks/alter/SchemaChangeHandler.java  |  26 +-
 .../starrocks/analysis/VectorIndexUtil.java   |  41 +-
 .../com/starrocks/catalog/FunctionSet.java    |   2 -
 .../com/starrocks/catalog/IndexParams.java    |  13 +-
 .../starrocks/common/VectorIndexParams.java   | 137 +++-
 .../starrocks/common/VectorSearchOptions.java |  96 ++-
 .../com/starrocks/planner/OlapScanNode.java   |  26 +-
 .../com/starrocks/qe/SessionVariable.java     |  17 +-
 .../com/starrocks/sql/StatementPlanner.java   |  59 +-
 .../sql/analyzer/CreateTableAnalyzer.java     |  12 +-
 .../sql/analyzer/SetStmtAnalyzer.java         |  21 +
 .../starrocks/sql/optimizer/Optimizer.java    |   8 +-
 .../sql/optimizer/OptimizerContext.java       |  13 +-
 .../sql/optimizer/operator/Projection.java    |   1 +
 .../RewriteToVectorPlanRule.java              | 498 +++++++++-----
 .../starrocks/analysis/VectorIndexTest.java   |   5 +-
 .../starrocks/planner/VectorIndexTest.java    | 468 ++++++++++++-
 .../AnalyzeAlterTableStatementTest.java       |   3 +-
 .../sql/analyzer/AnalyzeSetVariableTest.java  |  44 ++
 .../analyzer/AnalyzeVectorIndexDMLTest.java   | 625 ++++++++++++++++++
 gensrc/thrift/PlanNodes.thrift                |   1 +
 .../sql/test_vector_index/R/test_vector_index |  27 +-
 .../R/test_vector_index_hnsw                  | 484 ++++++++++++++
 .../R/test_vector_index_insert                | 170 +++++
 .../R/test_vector_index_ivfpq                 | 536 +++++++++++++++
 .../sql/test_vector_index/T/test_vector_index |  22 +-
 .../T/test_vector_index_hnsw                  | 249 +++++++
 .../T/test_vector_index_insert                | 120 ++++
 .../T/test_vector_index_ivfpq                 | 271 ++++++++
 43 files changed, 3726 insertions(+), 586 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeVectorIndexDMLTest.java
 create mode 100644 test/sql/test_vector_index/R/test_vector_index_hnsw
 create mode 100644 test/sql/test_vector_index/R/test_vector_index_insert
 create mode 100644 test/sql/test_vector_index/R/test_vector_index_ivfpq
 create mode 100644 test/sql/test_vector_index/T/test_vector_index_hnsw
 create mode 100644 test/sql/test_vector_index/T/test_vector_index_insert
 create mode 100644 test/sql/test_vector_index/T/test_vector_index_ivfpq

diff --git a/be/src/common/config.h b/be/src/common/config.h
index 7f2a6a3ea6eb2d..7ee9ae95ad8a09 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1489,7 +1489,7 @@ CONF_mBool(enable_vector_index_block_cache, "true");
 CONF_mInt32(config_vector_index_build_concurrency, "8");
 
 // default not to build the empty index
-CONF_mInt32(config_vector_index_default_build_threshold, "0");
+CONF_mInt32(config_vector_index_default_build_threshold, "100");
 
 // When upgrade thrift to 0.20.0, the MaxMessageSize member defines the maximum size of a (received) message, in bytes.
 // The default value is represented by a constant named DEFAULT_MAX_MESSAGE_SIZE, whose value is 100 * 1024 * 1024 bytes.
diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.cpp b/be/src/exec/pipeline/scan/olap_chunk_source.cpp
index 44c93d42def643..f80ddb12d6a97f 100644
--- a/be/src/exec/pipeline/scan/olap_chunk_source.cpp
+++ b/be/src/exec/pipeline/scan/olap_chunk_source.cpp
@@ -82,6 +82,7 @@ Status OlapChunkSource::prepare(RuntimeState* state) {
     if (_use_vector_index) {
         _use_ivfpq = vector_search_options.use_ivfpq;
         _vector_distance_column_name = vector_search_options.vector_distance_column_name;
+        _vector_slot_id = vector_search_options.vector_slot_id;
         _params.vector_search_option = std::make_shared<VectorSearchOption>();
     }
     const TupleDescriptor* tuple_desc = state->desc_tbl().get_tuple_descriptor(thrift_olap_scan_node.tuple_id);
@@ -320,12 +321,10 @@ Status OlapChunkSource::_init_scanner_columns(std::vector<uint32_t>& scanner_col
     for (auto slot : *_slots) {
         DCHECK(slot->is_materialized());
         int32_t index;
-        if (_use_vector_index && !_use_ivfpq) {
-            index = _tablet_schema->field_index(slot->col_name(), _vector_distance_column_name);
-            if (slot->col_name() == _vector_distance_column_name) {
-                _params.vector_search_option->vector_column_id = index;
-                _params.vector_search_option->vector_slot_id = slot->id();
-            }
+        if (_use_vector_index && !_use_ivfpq && slot->id() == _vector_slot_id) {
+            index = _tablet_schema->num_columns();
+            _params.vector_search_option->vector_column_id = index;
+            _params.vector_search_option->vector_slot_id = slot->id();
         } else {
             index = _tablet_schema->field_index(slot->col_name());
         }
@@ -352,12 +351,7 @@ Status OlapChunkSource::_init_scanner_columns(std::vector<uint32_t>& scanner_col
 
 Status OlapChunkSource::_init_unused_output_columns(const std::vector<std::string>& unused_output_columns) {
     for (const auto& col_name : unused_output_columns) {
-        int32_t index;
-        if (_use_vector_index && !_use_ivfpq) {
-            index = _tablet_schema->field_index(col_name, _vector_distance_column_name);
-        } else {
-            index = _tablet_schema->field_index(col_name);
-        }
+        int32_t index = _tablet_schema->field_index(col_name);
         if (index < 0) {
             std::stringstream ss;
             ss << "invalid field name: " << col_name;
@@ -562,8 +556,8 @@ Status OlapChunkSource::_init_global_dicts(TabletReaderParams* params) {
         if (iter != global_dict_map.end()) {
             auto& dict_map = iter->second.first;
             int32_t index;
-            if (_use_vector_index && !_use_ivfpq) {
-                index = _tablet_schema->field_index(slot->col_name(), _vector_distance_column_name);
+            if (_use_vector_index && !_use_ivfpq && slot->id() == _vector_slot_id) {
+                index = _tablet_schema->num_columns();
             } else {
                 index = _tablet_schema->field_index(slot->col_name());
             }
diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.h b/be/src/exec/pipeline/scan/olap_chunk_source.h
index 8186a1ac095944..6057bbd17e2536 100644
--- a/be/src/exec/pipeline/scan/olap_chunk_source.h
+++ b/be/src/exec/pipeline/scan/olap_chunk_source.h
@@ -106,10 +106,9 @@ class OlapChunkSource final : public ChunkSource {
     std::vector<ColumnAccessPathPtr> _column_access_paths;
 
     bool _use_vector_index = false;
-
     bool _use_ivfpq = false;
-
     std::string _vector_distance_column_name;
+    SlotId _vector_slot_id;
 
     // The following are profile meatures
     int64_t _num_rows_read = 0;
diff --git a/be/src/storage/index/vector/tenann/del_id_filter.h b/be/src/storage/index/vector/tenann/del_id_filter.h
index b23b3e245fdd50..d8b36a96a5abae 100644
--- a/be/src/storage/index/vector/tenann/del_id_filter.h
+++ b/be/src/storage/index/vector/tenann/del_id_filter.h
@@ -31,6 +31,9 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
+
+#pragma once
+
 #ifdef WITH_TENANN
 #include "storage/del_vector.h"
 #include "storage/range.h"
@@ -39,10 +42,10 @@
 
 namespace starrocks {
 
-class DelIdFilter : public tenann::IdFilter {
+class DelIdFilter final : public tenann::IdFilter {
 public:
-    DelIdFilter(const SparseRange<>& scan_range);
-    ~DelIdFilter() = default;
+    explicit DelIdFilter(const SparseRange<>& scan_range);
+    ~DelIdFilter() override = default;
 
     bool IsMember(tenann::idx_t id) const override;
 
@@ -51,4 +54,4 @@ class DelIdFilter : public tenann::IdFilter {
 };
 
 } // namespace starrocks
-#endif
\ No newline at end of file
+#endif
diff --git a/be/src/storage/index/vector/tenann/tenann_index_builder.cpp b/be/src/storage/index/vector/tenann/tenann_index_builder.cpp
index 991a64767563fe..84b07655ac4c29 100644
--- a/be/src/storage/index/vector/tenann/tenann_index_builder.cpp
+++ b/be/src/storage/index/vector/tenann/tenann_index_builder.cpp
@@ -25,6 +25,7 @@
 #include "tenann/factory/index_factory.h"
 
 namespace starrocks {
+
 // =============== TenAnnIndexBuilderProxy =============
 
 Status TenAnnIndexBuilderProxy::init() {
@@ -35,10 +36,19 @@ Status TenAnnIndexBuilderProxy::init() {
                         return Status::OK();
                     }).status());
 
-    if (!meta.common_params().contains("dim")) {
-        return Status::InvalidArgument("Dim is needed because it's a critical common param");
+    const auto& params = meta.common_params();
+
+    if (!params.contains(index::vector::DIM)) {
+        return Status::InvalidArgument("dim is needed because it's a critical common param");
+    }
+    _dim = params[index::vector::DIM];
+
+    if (!params.contains(index::vector::METRIC_TYPE)) {
+        return Status::InvalidArgument("metric_type is needed because it's a critical common param");
     }
-    _dim = meta.common_params()["dim"];
+    _is_input_normalized = params.contains(index::vector::IS_VECTOR_NORMED) &&
+                           params[index::vector::IS_VECTOR_NORMED] &&
+                           params[index::vector::METRIC_TYPE] == tenann::MetricType::kCosineSimilarity;
 
     auto meta_copy = meta;
     if (meta.index_type() == tenann::IndexType::kFaissIvfPq && config::enable_vector_index_block_cache) {
@@ -51,7 +61,7 @@ Status TenAnnIndexBuilderProxy::init() {
         // build and write index
         _index_builder = tenann::IndexFactory::CreateBuilderFromMeta(meta_copy);
         _index_builder->index_writer()->SetIndexCache(tenann::IndexCache::GetGlobalInstance());
-        if (_src_is_nullable) {
+        if (_is_element_nullable) {
             _index_builder->EnableCustomRowId();
         }
         _index_builder->Open(_index_path);
@@ -66,98 +76,64 @@ Status TenAnnIndexBuilderProxy::init() {
     return Status::OK();
 }
 
-Status TenAnnIndexBuilderProxy::add(const Column& data) {
-    try {
-        auto vector_view = tenann::ArraySeqView{.data = const_cast<uint8_t*>(data.raw_data()),
-                                                .dim = _dim,
-                                                .size = static_cast<uint32_t>(data.size()),
-                                                .elem_type = tenann::kFloatType};
-        if (data.is_array() && data.size() != 0) {
-            const auto& cur_array = down_cast<const ArrayColumn&>(data);
-            auto offsets = cur_array.offsets();
-            size_t last_offset = 0;
-            auto* offsets_data = reinterpret_cast<uint32_t*>(offsets.mutable_raw_data());
-            for (size_t i = 1; i < offsets.size(); i++) {
-                size_t dim = offsets_data[i] - last_offset;
-                if (dim > 0 && _dim != dim) {
-                    LOG(WARNING) << "index dim: " << _dim << ", written dim: " << dim;
-                    return Status::InvalidArgument(
-                            strings::Substitute("The dimensions of the vector written are inconsistent, index dim is "
-                                                "$0 but data dim is $1, vector data is ",
-                                                _dim, dim));
-                }
-                last_offset = offsets_data[i];
-            }
-        }
-
-        _index_builder->Add({vector_view});
-    } catch (tenann::Error& e) {
-        LOG(WARNING) << e.what();
-        return Status::InternalError(e.what());
+template <bool is_input_normalized>
+static Status valid_input_vector(const ArrayColumn& input_column, const size_t index_dim) {
+    if (input_column.empty()) {
+        return Status::OK();
     }
-    return Status::OK();
-}
 
-Status TenAnnIndexBuilderProxy::add(const Column& data, const Column& null_map, const size_t offset) {
-    try {
-        auto vector_view = tenann::ArraySeqView{.data = const_cast<uint8_t*>(data.raw_data()),
-                                                .dim = _dim,
-                                                .size = static_cast<uint32_t>(data.size()),
-                                                .elem_type = tenann::kFloatType};
-        if (data.is_array() && data.size() != 0) {
-            const auto& cur_array = down_cast<const ArrayColumn&>(data);
-            auto offsets = cur_array.offsets();
-            size_t last_offset = 0;
-            auto* offsets_data = reinterpret_cast<uint32_t*>(offsets.mutable_raw_data());
-            for (size_t i = 1; i < offsets.size(); i++) {
-                size_t dim = offsets_data[i] - last_offset;
-                if (dim > 0 && _dim != dim) {
-                    LOG(WARNING) << "index dim: " << _dim << ", written dim: " << dim;
-                    return Status::InvalidArgument(
-                            strings::Substitute("The dimensions of the vector written are inconsistent, index dim is "
-                                                "$0 but data dim is $1, vector data is ",
-                                                _dim, dim));
-                }
-                last_offset = offsets_data[i];
-            }
+    const size_t num_rows = input_column.size();
+    const auto* offsets = reinterpret_cast<const uint32_t*>(input_column.offsets().raw_data());
+    const auto* nums = reinterpret_cast<const float*>(input_column.elements().raw_data());
+
+    for (size_t i = 0; i < num_rows; i++) {
+        const size_t input_dim = offsets[i + 1] - offsets[i];
+
+        if (input_dim != index_dim) {
+            return Status::InvalidArgument(
+                    strings::Substitute("The dimensions of the vector written are inconsistent, index dim is "
+                                        "$0 but data dim is $1",
+                                        index_dim, input_dim));
         }
-        std::vector<int64_t> row_ids(data.size());
-        std::iota(row_ids.begin(), row_ids.end(), offset);
-        _index_builder->Add({vector_view}, row_ids.data(), null_map.raw_data());
 
-    } catch (tenann::Error& e) {
-        LOG(WARNING) << e.what();
-        return Status::InternalError(e.what());
+        if constexpr (is_input_normalized) {
+            double sum = 0;
+            for (int j = 0; j < input_dim; j++) {
+                sum += nums[offsets[i] + j] * nums[offsets[i] + j];
+            }
+            if (std::abs(sum - 1) > 1e-6) {
+                return Status::InvalidArgument(
+                        "The input vector is not normalized but `metric_type` is cosine_similarity and "
+                        "`is_vector_normed` is true");
+            }
+        }
     }
+
     return Status::OK();
 }
 
-Status TenAnnIndexBuilderProxy::write(const Column& data) {
-    try {
-        auto vector_view = tenann::ArraySeqView{.data = const_cast<uint8_t*>(data.raw_data()),
-                                                .dim = _dim,
-                                                .size = static_cast<uint32_t>(data.size()),
-                                                .elem_type = tenann::kFloatType};
+Status TenAnnIndexBuilderProxy::add(const Column& array_column, const size_t offset) {
+    DCHECK(array_column.is_array());
+    const auto& array_col = down_cast<const ArrayColumn&>(array_column);
 
-        _index_builder->Add({vector_view});
-    } catch (tenann::Error& e) {
-        LOG(WARNING) << e.what();
-        return Status::InternalError(e.what());
+    DCHECK(array_col.elements_column()->is_nullable());
+    const auto& nullable_elements = down_cast<const NullableColumn&>(array_col.elements());
+    const auto& is_element_nulls = nullable_elements.null_column_ref();
+
+    if (_is_input_normalized) {
+        RETURN_IF_ERROR(valid_input_vector<true>(array_col, _dim));
+    } else {
+        RETURN_IF_ERROR(valid_input_vector<false>(array_col, _dim));
     }
-    return Status::OK();
-}
 
-Status TenAnnIndexBuilderProxy::write(const Column& data, const Column& null_map) {
     try {
-        auto vector_view = tenann::ArraySeqView{.data = const_cast<uint8_t*>(data.raw_data()),
+        auto vector_view = tenann::ArraySeqView{.data = const_cast<uint8_t*>(array_col.raw_data()),
                                                 .dim = _dim,
-                                                .size = static_cast<uint32_t>(data.size()),
+                                                .size = static_cast<uint32_t>(array_col.size()),
                                                 .elem_type = tenann::kFloatType};
-
-        std::vector<int64_t> row_ids(data.size());
-        std::iota(row_ids.begin(), row_ids.end(), 0);
-        _index_builder->Add({vector_view}, row_ids.data(), null_map.raw_data());
-
+        std::vector<int64_t> row_ids(array_col.size());
+        std::iota(row_ids.begin(), row_ids.end(), offset);
+        _index_builder->Add({vector_view}, row_ids.data(), is_element_nulls.raw_data());
     } catch (tenann::Error& e) {
         LOG(WARNING) << e.what();
         return Status::InternalError(e.what());
@@ -175,10 +151,11 @@ Status TenAnnIndexBuilderProxy::flush() {
     return Status::OK();
 }
 
-void TenAnnIndexBuilderProxy::close() {
+void TenAnnIndexBuilderProxy::close() const {
     if (_index_builder && !_index_builder->is_closed()) {
         _index_builder->Close();
     }
 }
+
 } // namespace starrocks
 #endif
diff --git a/be/src/storage/index/vector/tenann/tenann_index_builder.h b/be/src/storage/index/vector/tenann/tenann_index_builder.h
index 3f53affa555e2b..7d29e11f1ad3cd 100644
--- a/be/src/storage/index/vector/tenann/tenann_index_builder.h
+++ b/be/src/storage/index/vector/tenann/tenann_index_builder.h
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#pragma once
+
 #ifdef WITH_TENANN
 
 #include <memory>
@@ -24,35 +26,34 @@
 namespace starrocks {
 
 // A proxy to real Ten ANN index builder
-class TenAnnIndexBuilderProxy : public VectorIndexBuilder {
+class TenAnnIndexBuilderProxy final : public VectorIndexBuilder {
 public:
     TenAnnIndexBuilderProxy(std::shared_ptr<TabletIndex> tablet_index, std::string segment_index_path,
-                            bool src_is_nullable)
+                            bool is_element_nullable)
             : VectorIndexBuilder(std::move(tablet_index), std::move(segment_index_path)),
-              _src_is_nullable(src_is_nullable){};
+              _is_element_nullable(is_element_nullable) {}
 
     // proxy should not clean index builder resource
     ~TenAnnIndexBuilderProxy() override { close(); };
 
     Status init() override;
 
-    Status add(const Column& data) override;
-
-    Status add(const Column& data, const Column& null_map, const size_t offset) override;
-
-    Status write(const Column& data) override;
-
-    Status write(const Column& data, const Column& null_map) override;
+    Status add(const Column& array_column, const size_t offset) override;
 
     Status flush() override;
 
-    void close();
+    void close() const;
 
 private:
-    std::shared_ptr<tenann::IndexBuilder> _index_builder;
-    uint32_t _dim = 0;
     OnceFlag _init_once;
-    bool _src_is_nullable;
+    std::shared_ptr<tenann::IndexBuilder> _index_builder = nullptr;
+    uint32_t _dim = 0;
+    // This will be true when `metric_type` is cosine_similarity and `is_vector_normed` is true.
+    // When it is true, the vector (a row of the array column) is either null or the sum of the squares of all elements
+    // equals 1.
+    bool _is_input_normalized = false;
+
+    const bool _is_element_nullable;
 };
 
 } // namespace starrocks
diff --git a/be/src/storage/index/vector/tenann/tenann_index_utils.cpp b/be/src/storage/index/vector/tenann/tenann_index_utils.cpp
index 18568650d31a71..35ccb41eb7c624 100644
--- a/be/src/storage/index/vector/tenann/tenann_index_utils.cpp
+++ b/be/src/storage/index/vector/tenann/tenann_index_utils.cpp
@@ -80,17 +80,16 @@ StatusOr<tenann::IndexMeta> get_vector_meta(const std::shared_ptr<TabletIndex>&
     if (meta.index_type() == tenann::IndexType::kFaissIvfPq) {
         meta.index_params()[starrocks::index::vector::NLIST] = int(4 * sqrt(starrocks::index::vector::nb_));
 
-        CRITICAL_CHECK_AND_GET(tablet_index, index_properties, M, param_value)
-        meta.index_params()[starrocks::index::vector::M] = std::atoi(param_value.c_str());
-
         CRITICAL_CHECK_AND_GET(tablet_index, index_properties, nbits, param_value)
         meta.index_params()[starrocks::index::vector::NBITS] = std::atoi(param_value.c_str());
 
+        CRITICAL_CHECK_AND_GET(tablet_index, index_properties, m_ivfpq, param_value)
+        meta.index_params()[starrocks::index::vector::M] = std::atoi(param_value.c_str());
     } else if (meta.index_type() == tenann::IndexType::kFaissHnsw) {
         CRITICAL_CHECK_AND_GET(tablet_index, index_properties, efconstruction, param_value)
         meta.index_params()[starrocks::index::vector::EF_CONSTRUCTION] = std::atoi(param_value.c_str());
 
-        CRITICAL_CHECK_AND_GET(tablet_index, index_properties, M, param_value)
+        CRITICAL_CHECK_AND_GET(tablet_index, index_properties, m, param_value)
         meta.index_params()[starrocks::index::vector::M] = std::atoi(param_value.c_str());
 
         GET_OR_DEFAULT(tablet_index, search_properties, efsearch, param_value, "40")
diff --git a/be/src/storage/index/vector/vector_index_builder.h b/be/src/storage/index/vector/vector_index_builder.h
index 1975551862249c..60fa6458558e40 100644
--- a/be/src/storage/index/vector/vector_index_builder.h
+++ b/be/src/storage/index/vector/vector_index_builder.h
@@ -29,17 +29,7 @@ class VectorIndexBuilder {
     // init from builder meta
     virtual Status init() = 0;
 
-    // add not null data
-    virtual Status add(const Column& data) = 0;
-
-    // add data contains null
-    virtual Status add(const Column& data, const Column& null_map, const size_t offset) = 0;
-
-    // write not null data
-    virtual Status write(const Column& data) = 0;
-
-    // write data contains nulls
-    virtual Status write(const Column& data, const Column& null_map) = 0;
+    virtual Status add(const Column& array_column, const size_t offset) = 0;
 
     // flush data into disk
     virtual Status flush() = 0;
@@ -56,20 +46,14 @@ class VectorIndexBuilder {
     std::string _index_path;
 };
 
-class EmptyVectorIndexBuilder : public VectorIndexBuilder {
+class EmptyVectorIndexBuilder final : public VectorIndexBuilder {
 public:
     EmptyVectorIndexBuilder(std::shared_ptr<TabletIndex> tablet_index, std::string segment_index_path)
             : VectorIndexBuilder(std::move(tablet_index), std::move(segment_index_path)){};
 
     Status init() override { return Status::OK(); }
 
-    Status add(const Column& data) override { return Status::OK(); }
-
-    Status add(const Column& data, const Column& null_map, const size_t offset) override { return Status::OK(); }
-
-    Status write(const Column& data) override { return Status::OK(); }
-
-    Status write(const Column& data, const Column& null_map) override { return Status::OK(); }
+    Status add(const Column& array_column, const size_t offset) override { return Status::OK(); }
 
     Status flush() override {
         RETURN_IF_ERROR(VectorIndexBuilder::flush_empty(_index_path));
diff --git a/be/src/storage/index/vector/vector_index_builder_factory.cpp b/be/src/storage/index/vector/vector_index_builder_factory.cpp
index 6ef43054bc6133..3dc3ed7ef83cdc 100644
--- a/be/src/storage/index/vector/vector_index_builder_factory.cpp
+++ b/be/src/storage/index/vector/vector_index_builder_factory.cpp
@@ -21,11 +21,11 @@ namespace starrocks {
 // =============== IndexBuilderFactory =============
 StatusOr<std::unique_ptr<VectorIndexBuilder>> VectorIndexBuilderFactory::create_index_builder(
         const std::shared_ptr<TabletIndex>& tablet_index, const std::string& segment_index_path,
-        const IndexBuilderType index_builder_type, const bool src_is_nullable) {
+        const IndexBuilderType index_builder_type, const bool is_element_nullable) {
     switch (index_builder_type) {
     case TEN_ANN:
 #ifdef WITH_TENANN
-        return std::make_unique<TenAnnIndexBuilderProxy>(tablet_index, segment_index_path, src_is_nullable);
+        return std::make_unique<TenAnnIndexBuilderProxy>(tablet_index, segment_index_path, is_element_nullable);
 #else
         return std::make_unique<EmptyVectorIndexBuilder>(tablet_index, segment_index_path);
 #endif
diff --git a/be/src/storage/index/vector/vector_index_builder_factory.h b/be/src/storage/index/vector/vector_index_builder_factory.h
index f2328b7f5dd43b..53a94a9b892b3c 100644
--- a/be/src/storage/index/vector/vector_index_builder_factory.h
+++ b/be/src/storage/index/vector/vector_index_builder_factory.h
@@ -28,7 +28,7 @@ class VectorIndexBuilderFactory {
 public:
     static StatusOr<std::unique_ptr<VectorIndexBuilder>> create_index_builder(
             const std::shared_ptr<TabletIndex>& tablet_index, const std::string& segment_index_path,
-            const IndexBuilderType index_builder_type, const bool src_is_nullable);
+            const IndexBuilderType index_builder_type, const bool is_element_nullable);
 
     static StatusOr<IndexBuilderType> get_index_builder_type_from_config(std::shared_ptr<TabletIndex> _tablet_index) {
         return IndexBuilderType::TEN_ANN;
diff --git a/be/src/storage/index/vector/vector_index_writer.cpp b/be/src/storage/index/vector/vector_index_writer.cpp
index 64c6a0c1c0ad1c..6ca49599b16116 100644
--- a/be/src/storage/index/vector/vector_index_writer.cpp
+++ b/be/src/storage/index/vector/vector_index_writer.cpp
@@ -21,9 +21,9 @@
 namespace starrocks {
 
 void VectorIndexWriter::create(const std::shared_ptr<TabletIndex>& tablet_index,
-                               const std::string& vector_index_file_path, bool is_nullable,
+                               const std::string& vector_index_file_path, bool is_element_nullable,
                                std::unique_ptr<VectorIndexWriter>* res) {
-    (*res) = std::make_unique<VectorIndexWriter>(tablet_index, vector_index_file_path, is_nullable);
+    *res = std::make_unique<VectorIndexWriter>(tablet_index, vector_index_file_path, is_element_nullable);
 }
 
 Status VectorIndexWriter::init() {
@@ -44,29 +44,24 @@ Status VectorIndexWriter::append(const Column& src) {
     int64_t duration = 0;
     {
         SCOPED_RAW_TIMER(&duration);
-        if (_index_builder.get() == nullptr) {
+
+        if (_index_builder == nullptr) {
             if (_row_size + src.size() >= _start_vector_index_build_threshold) {
                 RETURN_IF_ERROR(_prepare_index_builder());
             } else {
-                if (!_buffer_column.get()) {
-                    if (is_nullable()) {
-                        if (src.is_nullable()) {
-                            _buffer_column = std::make_unique<NullableColumn>(down_cast<const NullableColumn&>(src));
-                        } else {
-                            _buffer_column = NullableColumn::wrap_if_necessary(src.clone_shared());
-                        }
-                    } else {
-                        _buffer_column = std::make_unique<ArrayColumn>(down_cast<const ArrayColumn&>(src));
-                    }
+                if (_buffer_column == nullptr) {
+                    _buffer_column = src.clone_shared();
                 } else {
                     _buffer_column->append(src, 0, src.size());
                 }
             }
         }
-        if (_index_builder.get() != nullptr) {
+
+        if (_index_builder != nullptr) {
             RETURN_IF_ERROR(_append_data(src, _next_row_id));
         }
     }
+
     _next_row_id += src.size();
     _buffer_size += src.byte_size();
     _row_size += src.size();
@@ -111,11 +106,12 @@ uint64_t VectorIndexWriter::estimate_buffer_size() const {
 Status VectorIndexWriter::_prepare_index_builder() {
     ASSIGN_OR_RETURN(auto index_builder_type,
                      VectorIndexBuilderFactory::get_index_builder_type_from_config(_tablet_index))
-    ASSIGN_OR_RETURN(_index_builder, VectorIndexBuilderFactory::create_index_builder(
-                                             _tablet_index, _vector_index_file_path, index_builder_type, _is_nullable))
+    ASSIGN_OR_RETURN(_index_builder,
+                     VectorIndexBuilderFactory::create_index_builder(_tablet_index, _vector_index_file_path,
+                                                                     index_builder_type, _is_element_nullable));
     RETURN_IF_ERROR(_index_builder->init());
 
-    if (_buffer_column.get()) {
+    if (_buffer_column != nullptr) {
         RETURN_IF_ERROR(_append_data(*_buffer_column, 0));
         _buffer_column.reset();
     }
@@ -123,19 +119,8 @@ Status VectorIndexWriter::_prepare_index_builder() {
 }
 
 Status VectorIndexWriter::_append_data(const Column& src, size_t offset) {
-    if (is_nullable()) {
-        if (src.is_nullable()) {
-            auto nullable_column = down_cast<const NullableColumn&>(src);
-            const auto& data_column_ref = nullable_column.data_column_ref();
-            const auto& null_column_ref = nullable_column.null_column_ref();
-            RETURN_IF_ERROR(_index_builder->add(data_column_ref, null_column_ref, offset));
-        } else {
-            auto empty_null_ptr = NullColumn::create(src.size(), 0);
-            RETURN_IF_ERROR(_index_builder->add(src, *empty_null_ptr, offset));
-        }
-    } else {
-        RETURN_IF_ERROR(_index_builder->add(src));
-    }
+    DCHECK(src.is_array());
+    RETURN_IF_ERROR(_index_builder->add(src, offset));
     return Status::OK();
 }
 
diff --git a/be/src/storage/index/vector/vector_index_writer.h b/be/src/storage/index/vector/vector_index_writer.h
index 20b4556e7a8a11..73bd050f60cc95 100644
--- a/be/src/storage/index/vector/vector_index_writer.h
+++ b/be/src/storage/index/vector/vector_index_writer.h
@@ -27,20 +27,22 @@
 #include "types/bitmap_value.h"
 
 namespace starrocks {
+
 class ArrayColumn;
+
 class VectorIndexWriter {
 public:
     static void create(const std::shared_ptr<TabletIndex>& tablet_index, const std::string& vector_index_file_path,
-                       bool is_nullable, std::unique_ptr<VectorIndexWriter>* res);
-
-    VectorIndexWriter() = default;
-    ~VectorIndexWriter() = default;
+                       bool is_element_nullable, std::unique_ptr<VectorIndexWriter>* res);
 
     VectorIndexWriter(const std::shared_ptr<TabletIndex>& tablet_index, std::string vector_index_file_path,
-                      bool is_nullable)
+                      bool is_element_nullable)
             : _tablet_index(tablet_index),
               _vector_index_file_path(std::move(vector_index_file_path)),
-              _is_nullable(is_nullable){};
+              _is_element_nullable(is_element_nullable) {
+        // Element of array column must be nullable.
+        DCHECK(_is_element_nullable);
+    }
 
     Status init();
 
@@ -54,8 +56,6 @@ class VectorIndexWriter {
 
     uint64_t total_mem_footprint() const { return estimate_buffer_size(); }
 
-    bool is_nullable() const { return _is_nullable; }
-
 private:
     std::shared_ptr<TabletIndex> _tablet_index;
     std::string _vector_index_file_path;
@@ -68,7 +68,7 @@ class VectorIndexWriter {
 
     // size of null_bit column is the same size with buffer_column
     // e.g. buffer_column: [1, NULL, 3, NULL, 4], null_column: [0, 1, 0, 1, 0]
-    bool _is_nullable;
+    const bool _is_element_nullable;
     size_t _next_row_id = 0;
     size_t _row_size = 0;
     size_t _buffer_size = 0;
diff --git a/be/src/storage/rowset/array_column_writer.cpp b/be/src/storage/rowset/array_column_writer.cpp
index 6fae8c973da00c..57ec1265b38bef 100644
--- a/be/src/storage/rowset/array_column_writer.cpp
+++ b/be/src/storage/rowset/array_column_writer.cpp
@@ -133,7 +133,8 @@ ArrayColumnWriter::ArrayColumnWriter(const ColumnWriterOptions& opts, TypeInfoPt
         DCHECK(_opts.tablet_index.count(IndexType::VECTOR) > 0);
         auto tablet_index = std::make_shared<TabletIndex>(_opts.tablet_index.at(IndexType::VECTOR));
         std::string index_path = _opts.standalone_index_file_paths.at(IndexType::VECTOR);
-        VectorIndexWriter::create(tablet_index, index_path, is_nullable(), &_vector_index_writer);
+        // Element column of array column MUST BE nullable.
+        VectorIndexWriter::create(tablet_index, index_path, true, &_vector_index_writer);
     }
 }
 
@@ -174,6 +175,8 @@ Status ArrayColumnWriter::append(const Column& column) {
 
     // 4. write vector index
     if (_vector_index_writer.get()) {
+        // Vector index only support non-nullable array column.
+        DCHECK(!is_nullable());
         RETURN_IF_ERROR(_vector_index_writer->append(*array_column));
     }
 
diff --git a/be/src/storage/rowset/segment_iterator.cpp b/be/src/storage/rowset/segment_iterator.cpp
index 06ec797a80ffc2..8ab3957cbe697a 100644
--- a/be/src/storage/rowset/segment_iterator.cpp
+++ b/be/src/storage/rowset/segment_iterator.cpp
@@ -413,7 +413,7 @@ SegmentIterator::SegmentIterator(std::shared_ptr<Segment> segment, Schema schema
         _result_order = _opts.vector_search_option->result_order;
         _use_ivfpq = _opts.vector_search_option->use_ivfpq;
         _query_params = _opts.vector_search_option->query_params;
-        if (_vector_range > 0 && _use_ivfpq) {
+        if (_vector_range >= 0 && _use_ivfpq) {
             _k = _opts.vector_search_option->k * _opts.vector_search_option->pq_refine_factor *
                  _opts.vector_search_option->k_factor;
         } else {
@@ -611,7 +611,7 @@ Status SegmentIterator::_get_row_ranges_by_vector_index() {
 
     {
         SCOPED_RAW_TIMER(&_opts.stats->vector_search_timer);
-        if (_vector_range > 0) {
+        if (_vector_range >= 0) {
             st = _ann_reader->range_search(_query_view, _k, &result_ids, &result_distances, &del_id_filter,
                                            static_cast<float>(_vector_range), _result_order);
         } else {
diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java
index 0e62624e58f75f..fb33f839625514 100644
--- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java
@@ -2080,7 +2080,7 @@ public AlterJobV2 createAlterMetaJob(AlterClause alterClause, Database db, OlapT
                 if (oldEnablePersistentIndex == enablePersistentIndex
                         && persistentIndexType == oldPersistentIndexType) {
                     LOG.info(String.format("table: %s enable_persistent_index is %s persistent_index_type is %s, "
-                            +  "nothing need to do", olapTable.getName(), enablePersistentIndex, persistentIndexType));
+                            + "nothing need to do", olapTable.getName(), enablePersistentIndex, persistentIndexType));
                     return null;
                 }
                 if (properties.containsKey(PropertyAnalyzer.PROPERTIES_PERSISTENT_INDEX_TYPE)
@@ -2140,10 +2140,10 @@ public ShowResultSet processLakeTableAlterMeta(AlterClause alterClause, Database
 
     public void processLakeTableDropPersistentIndex(AlterClause alterClause, Database db, OlapTable olapTable)
             throws StarRocksException {
-        if (!olapTable.enablePersistentIndex() || 
+        if (!olapTable.enablePersistentIndex() ||
                 olapTable.getPersistentIndexType() != TPersistentIndexType.CLOUD_NATIVE) {
             LOG.warn(String.format("drop persistent index on table %s failed, it must be" +
-                        " cloud_native persistent index", olapTable.getName()));
+                    " cloud_native persistent index", olapTable.getName()));
             throw new DdlException("drop persistent index only support cloud native index");
         }
         Set<Long> dropPindexTablets = ((DropPersistentIndexClause) alterClause).getTabletIds();
@@ -2165,7 +2165,7 @@ public void processLakeTableDropPersistentIndex(AlterClause alterClause, Databas
                 LOG.warn(String.format("drop persistent index on tablet %d failed, error: %s",
                         tabletId, e.getMessage()));
                 throw new DdlException(String.format("drop persistent index on tablet %d failed, error: %s",
-                    tabletId, e.getMessage()));
+                        tabletId, e.getMessage()));
             }
         }
     }
@@ -2175,7 +2175,7 @@ public void updateTableMeta(Database db, String tableName, Map<String, String> p
             throws DdlException {
         List<Partition> partitions = Lists.newArrayList();
         OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore()
-                    .getTable(db.getFullName(), tableName);
+                .getTable(db.getFullName(), tableName);
 
         Locker locker = new Locker();
         locker.lockTablesWithIntensiveDbLock(db.getId(), Lists.newArrayList(olapTable.getId()), LockType.READ);
@@ -2370,7 +2370,7 @@ public void updatePartitionsInMemoryMeta(Database db,
                                              List<String> partitionNames,
                                              Map<String, String> properties) throws DdlException {
         OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore()
-                    .getTable(db.getFullName(), tableName);
+                .getTable(db.getFullName(), tableName);
         Locker locker = new Locker();
         locker.lockTablesWithIntensiveDbLock(db.getId(), Lists.newArrayList(olapTable.getId()), LockType.READ);
         try {
@@ -2414,7 +2414,7 @@ public void updateBinlogPartitionTabletMeta(Database db,
         // be id -> Set<tablet id>
         Map<Long, Set<Long>> beIdToTabletId = Maps.newHashMap();
         OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore()
-                    .getTable(db.getFullName(), tableName);
+                .getTable(db.getFullName(), tableName);
 
         Locker locker = new Locker();
         locker.lockTablesWithIntensiveDbLock(db.getId(), Lists.newArrayList(olapTable.getId()), LockType.READ);
@@ -2501,7 +2501,7 @@ public void updatePartitionTabletMeta(Database db,
         // be id -> <tablet id>
         Map<Long, Set<Long>> beIdToTabletSet = Maps.newHashMap();
         OlapTable olapTable = (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore()
-                    .getTable(db.getFullName(), tableName);
+                .getTable(db.getFullName(), tableName);
 
         Locker locker = new Locker();
         locker.lockTablesWithIntensiveDbLock(db.getId(), Lists.newArrayList(olapTable.getId()), LockType.READ);
@@ -2720,6 +2720,16 @@ private void processAddIndex(CreateIndexClause alterClause, OlapTable olapTable,
             throw new SemanticException("GIN does not support replicated mode");
         }
 
+        if (newIndex.getIndexType() == IndexType.VECTOR) {
+            Optional<Index> oldVectorIndex =
+                    newIndexes.stream().filter(index -> index.getIndexType() == IndexType.VECTOR).findFirst();
+            if (oldVectorIndex.isPresent()) {
+                throw new SemanticException(
+                        String.format("At most one vector index is allowed for a table, but there is already a vector index [%s]",
+                                oldVectorIndex.get().getIndexName()));
+            }
+        }
+
         List<Index> existedIndexes = olapTable.getIndexes();
         IndexDef indexDef = alterClause.getIndexDef();
         Set<String> newColset = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/VectorIndexUtil.java b/fe/fe-core/src/main/java/com/starrocks/analysis/VectorIndexUtil.java
index f8f79fafc3bd76..b0d436e2b35b62 100644
--- a/fe/fe-core/src/main/java/com/starrocks/analysis/VectorIndexUtil.java
+++ b/fe/fe-core/src/main/java/com/starrocks/analysis/VectorIndexUtil.java
@@ -47,8 +47,14 @@ public static void checkVectorIndexValid(Column column, Map<String, String> prop
             throw new SemanticException("The vector index does not support shared data mode");
         }
         if (!Config.enable_experimental_vector) {
-            throw new SemanticException("The vector index is disabled, enable it by setting FE config `enable_experimental_vector` to true");
+            throw new SemanticException(
+                    "The vector index is disabled, enable it by setting FE config `enable_experimental_vector` to true");
         }
+
+        if (column.isAllowNull()) {
+            throw new SemanticException("The vector index can only build on non-nullable column");
+        }
+
         // Only support create vector index on DUPLICATE/PRIMARY table or key columns of UNIQUE/AGGREGATE table.
         if (keysType != KeysType.DUP_KEYS && keysType != KeysType.PRIMARY_KEYS) {
             throw new SemanticException("The vector index can only build on DUPLICATE or PRIMARY table");
@@ -67,8 +73,10 @@ public static void checkVectorIndexValid(Column column, Map<String, String> prop
         // check param keys which must not be null
         Map<String, IndexParamItem> mustNotNullParams = IndexParams.getInstance().getMustNotNullParams(IndexType.VECTOR);
 
-        Map<String, IndexParamItem> indexIndexParams = IndexParams.getInstance().getKeySetByIndexTypeAndParamType(IndexType.VECTOR, IndexParamType.INDEX);
-        Map<String, IndexParamItem> searchIndexParams = IndexParams.getInstance().getKeySetByIndexTypeAndParamType(IndexType.VECTOR, IndexParamType.SEARCH);
+        Map<String, IndexParamItem> indexIndexParams =
+                IndexParams.getInstance().getKeySetByIndexTypeAndParamType(IndexType.VECTOR, IndexParamType.INDEX);
+        Map<String, IndexParamItem> searchIndexParams =
+                IndexParams.getInstance().getKeySetByIndexTypeAndParamType(IndexType.VECTOR, IndexParamType.SEARCH);
 
         Map<VectorIndexType, Set<String>> indexParamsGroupByType =
                 Arrays.stream(IndexParamsKey.values()).filter(belong -> belong.getBelongVectorIndexType() != null)
@@ -76,7 +84,8 @@ public static void checkVectorIndexValid(Column column, Map<String, String> prop
                                 Collectors.mapping(Enum::name, Collectors.toSet())));
 
         Map<VectorIndexType, Set<String>> searchParamsGroupByType =
-                Arrays.stream(VectorIndexParams.SearchParamsKey.values()).filter(belong -> belong.getBelongVectorIndexType() != null)
+                Arrays.stream(VectorIndexParams.SearchParamsKey.values())
+                        .filter(belong -> belong.getBelongVectorIndexType() != null)
                         .collect(Collectors.groupingBy(SearchParamsKey::getBelongVectorIndexType,
                                 Collectors.mapping(Enum::name, Collectors.toSet())));
 
@@ -119,7 +128,6 @@ public static void checkVectorIndexValid(Column column, Map<String, String> prop
         configSearchParams.removeAll(Optional.ofNullable(searchParamsGroupByType.get(vectorIndexType))
                 .orElse(Collections.emptySet()));
 
-
         if (!configIndexParams.isEmpty()) {
             throw new SemanticException(String.format("Index params %s should not define with %s", configIndexParams,
                     vectorIndexType));
@@ -130,10 +138,33 @@ public static void checkVectorIndexValid(Column column, Map<String, String> prop
                     vectorIndexType));
         }
 
+        if (vectorIndexType == VectorIndexType.IVFPQ) {
+            String m = properties.get(IndexParamsKey.M_IVFPQ.name().toUpperCase());
+            if (m == null) {
+                throw new SemanticException("`M_IVFPQ` is required for IVFPQ index");
+            }
+            // m is a valid integer which is guaranteed by checkParams.
+            int mValue = Integer.parseInt(m);
+
+            String dim = properties.get(CommonIndexParamKey.DIM.name().toUpperCase());
+            int dimValue = Integer.parseInt(dim);
+            if (dimValue % mValue != 0) {
+                throw new SemanticException("`DIM` should be a multiple of `M_IVFPQ` for IVFPQ index");
+            }
+        }
+
         // add default properties
+        Set<String> indexParams = indexParamsGroupByType.get(vectorIndexType);
+        paramsNeedDefault.keySet().removeIf(key -> !indexParams.contains(key));
         if (!paramsNeedDefault.isEmpty()) {
             addDefaultProperties(properties, paramsNeedDefault);
         }
+
+        // Lower all the keys and values of properties.
+        Map<String, String> lowerProperties = properties.entrySet().stream()
+                .collect(Collectors.toMap(entry -> entry.getKey().toLowerCase(), entry -> entry.getValue().toLowerCase()));
+        properties.clear();
+        properties.putAll(lowerProperties);
     }
 
     private static void addDefaultProperties(Map<String, String> properties, Map<String, IndexParamItem> paramsNeedDefault) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java b/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java
index 60810d6ec29dc7..6fbf0c0d87e8ef 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java
@@ -157,7 +157,6 @@ public class FunctionSet {
 
     // Vector Index functions:
     public static final String APPROX_COSINE_SIMILARITY = "approx_cosine_similarity";
-    public static final String APPROX_COSINE_SIMILARITY_NORM = "approx_cosine_similarity_norm";
     public static final String APPROX_L2_DISTANCE = "approx_l2_distance";
 
     // Geo functions:
@@ -652,7 +651,6 @@ public class FunctionSet {
     public static final Set<String> VECTOR_COMPUTE_FUNCTIONS =
             ImmutableSet.<String>builder()
                     .add(APPROX_COSINE_SIMILARITY)
-                    .add(APPROX_COSINE_SIMILARITY_NORM)
                     .add(APPROX_L2_DISTANCE)
                     .build();
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/IndexParams.java b/fe/fe-core/src/main/java/com/starrocks/catalog/IndexParams.java
index 2adaf37945d46a..0dbe8e65dc5130 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/IndexParams.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/IndexParams.java
@@ -30,7 +30,6 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.stream.Collectors;
 
 public class IndexParams {
@@ -62,9 +61,11 @@ private IndexParams() {
         register(builder, IndexType.VECTOR, IndexParamType.INDEX, VectorIndexParams.IndexParamsKey.M, false, true, "16", null);
         register(builder, IndexType.VECTOR, IndexParamType.INDEX, VectorIndexParams.IndexParamsKey.EFCONSTRUCTION, false, true,
                 "40", null);
-        register(builder, IndexType.VECTOR, IndexParamType.INDEX, VectorIndexParams.IndexParamsKey.NBITS, false, false, "8",
+        register(builder, IndexType.VECTOR, IndexParamType.INDEX, VectorIndexParams.IndexParamsKey.NBITS, false, true, "8",
                 null);
-        register(builder, IndexType.VECTOR, IndexParamType.INDEX, VectorIndexParams.IndexParamsKey.NLIST, false, false, null,
+        register(builder, IndexType.VECTOR, IndexParamType.INDEX, VectorIndexParams.IndexParamsKey.NLIST, false, true, "16",
+                null);
+        register(builder, IndexType.VECTOR, IndexParamType.INDEX, VectorIndexParams.IndexParamsKey.M_IVFPQ, false, false, null,
                 null);
 
         // search
@@ -168,7 +169,11 @@ public Map<String, IndexParamItem> getMustNotNullParams(IndexType indexType) {
     }
 
     public void checkParams(String key, String value) throws SemanticException {
-        Optional.ofNullable(paramsHolder.get(key)).ifPresent(p -> p.checkValue(value));
+        IndexParamItem item = paramsHolder.get(key);
+        if (item == null) {
+            throw new SemanticException("Unknown index param: `" + key + "`");
+        }
+        item.checkValue(value);
     }
 
     public enum IndexParamType {
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java b/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java
index d874d86f2d2dde..41f73c2279ccaa 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java
@@ -23,7 +23,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-
 public class VectorIndexParams {
 
     public enum CommonIndexParamKey implements ParamsKey {
@@ -44,13 +43,7 @@ public void check(String value) {
         DIM {
             @Override
             public void check(String value) {
-                if (!StringUtils.isNumeric(value)) {
-                    throw new SemanticException("Value of `DIM` must be a number");
-                }
-                int dim = Integer.parseInt(value);
-                if (dim <= 0) {
-                    throw new SemanticException("Value of `DIM` must greater then 0");
-                }
+                validateInteger(value, "DIM", 1);
             }
         },
         // Vector space metrics method, the enumeration of values refer to MetricsType
@@ -67,7 +60,14 @@ public void check(String value) {
         },
 
         // Whether vector should be normed
-        IS_VECTOR_NORMED,
+        IS_VECTOR_NORMED {
+            @Override
+            public void check(String value) {
+                if (!StringUtils.equalsIgnoreCase(value, "true") && !StringUtils.equalsIgnoreCase(value, "false")) {
+                    throw new SemanticException("Value of `IS_VECTOR_NORMED` must be `true` or `false`");
+                }
+            }
+        },
 
         // Threshold of row number to build index file
         INDEX_BUILD_THRESHOLD
@@ -82,9 +82,7 @@ public enum VectorIndexType {
     }
 
     public enum MetricsType {
-        COSINE_DISTANCE,
         COSINE_SIMILARITY,
-        INNER_PRODUCT,
         L2_DISTANCE,
     }
 
@@ -93,30 +91,65 @@ public enum IndexParamsKey implements ParamsKey {
 
         // the parameter "M" is a crucial construction parameter that refers to the maximum number of neighbors each node can have at the base layer,
         // which is the bottommost layer of the graph.
-        M(VectorIndexType.HNSW),
+        M(VectorIndexType.HNSW) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "M", 2);
+            }
+        },
 
         // EF_CONSTRUCTION is an important parameter that stands for the construction-time expansion factor.
         // This parameter controls the depth of the neighbor search for each data point during the index construction process.
         // Specifically, EF_CONSTRUCTION determines the size of the candidate list for nearest neighbors when inserting new nodes into the HNSW graph.
-        EFCONSTRUCTION(VectorIndexType.HNSW),
+        EFCONSTRUCTION(VectorIndexType.HNSW) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "EFCONSTRUCTION", 1);
+            }
+        },
 
         // For IVFPQ
 
         // NBITS is a key parameter that refers to the number of bits used to quantize each sub-vector. Within the context of Product Quantization (PQ),
         // NBITS determines the size of the quantization codebook, which is the number of cluster centers in each quantized subspace.
-        NBITS(VectorIndexType.IVFPQ),
-
+        NBITS(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                try {
+                    double num = Double.parseDouble(value);
+                    if (num != 8) {
+                        throw new SemanticException(String.format("Value of `%s` must be 8", "NBITS"));
+                    }
+                } catch (NumberFormatException e) {
+                    throw new SemanticException(String.format("Value of `%s` must be a integer", "NBITS"));
+                }
+            }
+        },
 
         // NLIST is a parameter in the IVF (Inverted File) indexing structure that represents the number of inverted lists, or equivalently,
         // the number of cluster centers (also known as visual words).
         // This parameter is set when constructing the index using the k-means clustering algorithm, with the purpose of grouping the vectors in the dataset
         // around these cluster centers.
-        NLIST(VectorIndexType.IVFPQ);
+        NLIST(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "NLIST", 1);
+            }
+        },
+
+        M_IVFPQ(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "M_IVFPQ", 2);
+            }
+        };
+
+        private final VectorIndexType belongVectorIndexType;
 
-        private VectorIndexType belongVectorIndexType = null;
         IndexParamsKey(VectorIndexType vectorIndexType) {
             belongVectorIndexType = vectorIndexType;
         }
+
         public VectorIndexType getBelongVectorIndexType() {
             return belongVectorIndexType;
         }
@@ -127,7 +160,12 @@ public enum SearchParamsKey implements ParamsKey {
         // The EF_SEARCH parameter represents the size of the dynamic candidate list during the search process, meaning that during the search phase,
         // the algorithm maintains a priority queue of size ef_search. This queue is used to store the current nearest neighbor candidates and graph
         // nodes for further exploration.
-        EFSEARCH(VectorIndexType.HNSW),
+        EFSEARCH(VectorIndexType.HNSW) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "EFSEARCH", 1);
+            }
+        },
 
         // For IVFPG
         // NPROBE determines the number of Voronoi cells (or inverted lists)
@@ -135,21 +173,36 @@ public enum SearchParamsKey implements ParamsKey {
         // In IVFPQ, the dataset is first divided into multiple Voronoi cells, and an inverted list is established for each cell. Then, for each query,
         // we first find the nearest NPROBE Voronoi cells, and then only search in the inverted lists of these cells.
         // The value of NPROBE affects the accuracy and efficiency of the search.
-        NPROBE(VectorIndexType.IVFPQ),
+        NPROBE(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "NPROBE", 1);
+            }
+        },
 
         // MAX_CODES determines the maximum number of codes to be inspected during the search phase.
         // In IVFPQ, the dataset is divided into multiple Voronoi cells, and each cell has an inverted list that contains the codes of all points in the cell.
         // For each query, we first find the nearest NPROBE Voronoi cells, and then search in the inverted lists of these cells.
         // However, to control the computational complexity of the search, we usually do not inspect all codes, but only inspect the first maxCodes codes in
         // each inverted list. The value of maxCodes affects the accuracy and efficiency of the search
-        MAX_CODES(VectorIndexType.IVFPQ),
+        MAX_CODES(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "MAX_CODES", 0);
+            }
+        },
 
         // SCAN_TABLE_THRESHOLD parameter is used to control the number of entries that are scanned in the lookup table during the search process.
         // The lookup table is a data structure that stores precomputed distances between the query and the centroids of the quantization cells.
         // During the search process, the algorithm scans the lookup table to find the nearest centroids to the query.
         // The SCAN_TABLE_THRESHOLD parameter determines the maximum number of entries that the algorithm will scan in the lookup table.
         // By adjusting this parameter, one can control the balance between search accuracy and efficiency.
-        SCAN_TABLE_THRESHOLD(VectorIndexType.IVFPQ),
+        SCAN_TABLE_THRESHOLD(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "SCAN_TABLE_THRESHOLD", 0);
+            }
+        },
 
         // POLYSEMOUS_HT is a parameter related to the Polysemous Coding technique.
         // Polysemous Coding is a technique used to balance the trade-off between recall and precision in large-scale search problems.
@@ -159,19 +212,57 @@ public enum SearchParamsKey implements ParamsKey {
         // The POLYSEMOUS_HT parameter is a threshold used in the Polysemous Coding technique. It determines the number of "hamming thresholds"
         // to be used in the search process. The hamming threshold is a measure of similarity between two binary codes: the lower the hamming distance,
         // the more similar the codes are.
-        POLYSEMOUS_HT(VectorIndexType.IVFPQ),
+        POLYSEMOUS_HT(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                validateInteger(value, "POLYSEMOUS_HT", 0);
+            }
+        },
 
         // The RANGE_SEARCH_CONFIDENCE parameter determines the confidence level of the range search in IVFPQ (Inverted File with Product Quantization).
         // We have developed our own index search algorithm based on the triangle inequality. Adjusting this parameter allows us to control the
         // performance and accuracy of the range search.
-        RANGE_SEARCH_CONFIDENCE(VectorIndexType.IVFPQ);
+        RANGE_SEARCH_CONFIDENCE(VectorIndexType.IVFPQ) {
+            @Override
+            public void check(String value) {
+                validateDouble(value, "RANGE_SEARCH_CONFIDENCE", 0.0, 1.0);
+            }
+        };
+
         private VectorIndexType belongVectorIndexType = null;
+
         SearchParamsKey(VectorIndexType vectorIndexType) {
             belongVectorIndexType = vectorIndexType;
         }
+
         public VectorIndexType getBelongVectorIndexType() {
             return belongVectorIndexType;
         }
     }
 
+    private static void validateInteger(String value, String key, Integer min) {
+        try {
+            int num = Integer.parseInt(value);
+            if (min != null && num < min) {
+                throw new SemanticException(String.format("Value of `%s` must be >= %d", key, min));
+            }
+        } catch (NumberFormatException e) {
+            throw new SemanticException(String.format("Value of `%s` must be a integer", key));
+        }
+    }
+
+    private static void validateDouble(String value, String key, Double min, Double max) {
+        try {
+            double num = Double.parseDouble(value);
+            if (min != null && num < min) {
+                throw new SemanticException(String.format("Value of `%s` must be >= %f", key, min));
+            }
+            if (max != null && num > max) {
+                throw new SemanticException(String.format("Value of `%s` must be <= %f", key, max));
+            }
+        } catch (NumberFormatException e) {
+            throw new SemanticException(String.format("Value of `%s` must be a double", key));
+        }
+    }
+
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/VectorSearchOptions.java b/fe/fe-core/src/main/java/com/starrocks/common/VectorSearchOptions.java
index 7f141a1ed3b421..f0c97523f7b24d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/VectorSearchOptions.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/VectorSearchOptions.java
@@ -14,40 +14,27 @@
 
 package com.starrocks.common;
 
-import com.google.gson.annotations.SerializedName;
-import com.google.gson.reflect.TypeToken;
-import com.starrocks.persist.gson.GsonUtils;
+import com.starrocks.thrift.TVectorSearchOptions;
 
-import java.lang.reflect.Type;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 public class VectorSearchOptions {
+    private static final int RESULT_ORDER_ASC = 0;
+    private static final int RESULT_ORDER_DESC = 1;
 
-    public VectorSearchOptions() {}
-
-    @SerializedName(value = "enableUseANN")
     private boolean enableUseANN = false;
-
-    @SerializedName(value = "useIVFPQ")
     private boolean useIVFPQ = false;
 
-    @SerializedName(value = "vectorDistanceColumnName")
-    private String vectorDistanceColumnName = "vector_distance";
+    private String distanceColumnName = "";
+    private int distanceSlotId = 0;
 
-    @SerializedName(value = "vectorLimitK")
-    private long vectorLimitK;
+    private long limitK = 0;
+    private int resultOrder = 0;
 
-    @SerializedName(value = "queryVector")
+    private double predicateRange = -1;
     private List<String> queryVector = new ArrayList<>();
 
-    @SerializedName(value = "vectorRange")
-    private double vectorRange = -1;
-
-    @SerializedName(value = "resultOrder")
-    private int resultOrder = 0;
-
     public boolean isEnableUseANN() {
         return enableUseANN;
     }
@@ -64,57 +51,56 @@ public void setUseIVFPQ(boolean useIVFPQ) {
         this.useIVFPQ = useIVFPQ;
     }
 
-    public String getVectorDistanceColumnName() {
-        return vectorDistanceColumnName;
-    }
-
-    public void setVectorDistanceColumnName(String vectorDistanceColumnName) {
-        this.vectorDistanceColumnName = vectorDistanceColumnName;
+    public String getDistanceColumnName() {
+        return distanceColumnName;
     }
 
-    public long getVectorLimitK() {
-        return vectorLimitK;
+    public void setDistanceColumnName(String distanceColumnName) {
+        this.distanceColumnName = distanceColumnName;
     }
 
-    public void setVectorLimitK(long vectorLimitK) {
-        this.vectorLimitK = vectorLimitK;
+    public void setDistanceSlotId(int distanceSlotId) {
+        this.distanceSlotId = distanceSlotId;
     }
 
-    public List<String> getQueryVector() {
-        return queryVector;
+    public void setLimitK(long limitK) {
+        this.limitK = limitK;
     }
 
     public void setQueryVector(List<String> queryVector) {
         this.queryVector = queryVector;
     }
 
-    public double getVectorRange() {
-        return vectorRange;
-    }
-
-    public void setVectorRange(double vectorRange) {
-        this.vectorRange = vectorRange;
-    }
-
-    public int getResultOrder() {
-        return resultOrder;
-    }
-
-    public void setResultOrder(int resultOrder) {
-        this.resultOrder = resultOrder;
+    public void setPredicateRange(double predicateRange) {
+        this.predicateRange = predicateRange;
     }
 
-    public static VectorSearchOptions read(String json) {
-        return GsonUtils.GSON.fromJson(json, VectorSearchOptions.class);
+    public void setResultOrder(boolean isAsc) {
+        this.resultOrder = isAsc ? RESULT_ORDER_ASC : RESULT_ORDER_DESC;
     }
 
-    public static Map<String, String> readAnnParams(String json) {
-        Type type = new TypeToken<Map<String, String>>() {}.getType();
-        return GsonUtils.GSON.fromJson(json, type);
+    public TVectorSearchOptions toThrift() {
+        TVectorSearchOptions opts = new TVectorSearchOptions();
+        opts.setEnable_use_ann(true);
+        opts.setVector_limit_k(limitK);
+        opts.setVector_distance_column_name(distanceColumnName);
+        opts.setVector_slot_id(distanceSlotId);
+        opts.setQuery_vector(queryVector);
+        opts.setVector_range(predicateRange);
+        opts.setResult_order(resultOrder);
+        opts.setUse_ivfpq(useIVFPQ);
+        return opts;
     }
 
-    @Override
-    public String toString() {
-        return GsonUtils.GSON.toJson(this);
+    public String getExplainString(String prefix) {
+        return prefix + "VECTORINDEX: ON" + "\n" +
+                prefix + prefix +
+                "IVFPQ: " + (useIVFPQ ? "ON" : "OFF") + ", " +
+                "Distance Column: <" + distanceSlotId + ":" + distanceColumnName + ">, " +
+                "LimitK: " + limitK + ", " +
+                "Order: " + (resultOrder == RESULT_ORDER_ASC ? "ASC" : "DESC") + ", " +
+                "Query Vector: " + queryVector + ", " +
+                "Predicate Range: " + predicateRange +
+                "\n";
     }
 }
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java
index 2cd4dce5ee23c8..85dda6399c879c 100644
--- a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java
@@ -103,7 +103,6 @@
 import com.starrocks.thrift.TScanRangeLocation;
 import com.starrocks.thrift.TScanRangeLocations;
 import com.starrocks.thrift.TTableSampleOptions;
-import com.starrocks.thrift.TVectorSearchOptions;
 import com.starrocks.warehouse.Warehouse;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
@@ -818,6 +817,14 @@ protected String getNodeExplainString(String prefix, TExplainLevel detailLevel)
             output.append(prefix).append("SORT COLUMN: ").append(sortColumn).append("\n");
         }
 
+        if (Config.enable_experimental_vector) {
+            if (vectorSearchOptions != null && vectorSearchOptions.isEnableUseANN()) {
+                output.append(vectorSearchOptions.getExplainString(prefix));
+            } else {
+                output.append(prefix).append("VECTORINDEX: OFF").append("\n");
+            }
+        }
+
         if (detailLevel != TExplainLevel.VERBOSE) {
             if (isPreAggregation) {
                 output.append(prefix).append("PREAGGREGATION: ON").append("\n");
@@ -825,13 +832,6 @@ protected String getNodeExplainString(String prefix, TExplainLevel detailLevel)
                 output.append(prefix).append("PREAGGREGATION: OFF. Reason: ").append(reasonOfPreAggregation)
                         .append("\n");
             }
-            if (ConnectContext.get() != null && Config.enable_experimental_vector == true) {
-                if (vectorSearchOptions != null && vectorSearchOptions.isEnableUseANN()) {
-                    output.append(prefix).append("VECTORINDEX: ON").append("\n");
-                } else {
-                    output.append(prefix).append("VECTORINDEX: OFF").append("\n");
-                }
-            }
             if (!conjuncts.isEmpty()) {
                 output.append(prefix).append("PREDICATES: ").append(
                         getExplainString(conjuncts)).append("\n");
@@ -1103,15 +1103,7 @@ protected void toThrift(TPlanNode msg) {
             }
 
             if (vectorSearchOptions != null && vectorSearchOptions.isEnableUseANN()) {
-                TVectorSearchOptions tVectorSearchOptions = new TVectorSearchOptions();
-                tVectorSearchOptions.setEnable_use_ann(true);
-                tVectorSearchOptions.setVector_limit_k(vectorSearchOptions.getVectorLimitK());
-                tVectorSearchOptions.setVector_distance_column_name(vectorSearchOptions.getVectorDistanceColumnName());
-                tVectorSearchOptions.setQuery_vector(vectorSearchOptions.getQueryVector());
-                tVectorSearchOptions.setVector_range(vectorSearchOptions.getVectorRange());
-                tVectorSearchOptions.setResult_order(vectorSearchOptions.getResultOrder());
-                tVectorSearchOptions.setUse_ivfpq(vectorSearchOptions.isUseIVFPQ());
-                msg.olap_scan_node.setVector_search_options(tVectorSearchOptions);
+                msg.olap_scan_node.setVector_search_options(vectorSearchOptions.toThrift());
             }
 
             msg.olap_scan_node.setUse_pk_index(usePkIndex);
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
index 80d1fc3c95a0a6..504bdce4b93943 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
@@ -46,7 +46,6 @@
 import com.starrocks.catalog.InternalCatalog;
 import com.starrocks.common.ErrorCode;
 import com.starrocks.common.ErrorReport;
-import com.starrocks.common.VectorSearchOptions;
 import com.starrocks.common.io.Text;
 import com.starrocks.common.io.Writable;
 import com.starrocks.common.util.CompressionUtils;
@@ -54,6 +53,7 @@
 import com.starrocks.connector.PlanMode;
 import com.starrocks.datacache.DataCachePopulateMode;
 import com.starrocks.monitor.unit.TimeValue;
+import com.starrocks.persist.gson.GsonUtils;
 import com.starrocks.qe.VariableMgr.VarAttr;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.server.RunMode;
@@ -85,6 +85,7 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.lang.reflect.Field;
+import java.lang.reflect.Type;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -2217,6 +2218,18 @@ public void setEnableParallelPrepareMetadata(boolean enableParallelPrepareMetada
         this.enableParallelPrepareMetadata = enableParallelPrepareMetadata;
     }
 
+    public void setAnnParams(String annParams) {
+        this.annParams = annParams;
+    }
+
+    public Map<String, String> getAnnParams() {
+        if (Strings.isNullOrEmpty(annParams)) {
+            return Maps.newHashMap();
+        }
+        Type type = new com.google.gson.reflect.TypeToken<Map<String, String>>() {}.getType();
+        return GsonUtils.GSON.fromJson(annParams, type);
+    }
+
     public String getHiveTempStagingDir() {
         return hiveTempStagingDir;
     }
@@ -4664,7 +4677,7 @@ public TQueryOptions toThrift() {
         tResult.setConnector_io_tasks_slow_io_latency_ms(connectorIoTasksSlowIoLatency);
         tResult.setConnector_scan_use_query_mem_ratio(connectorScanUseQueryMemRatio);
         tResult.setScan_use_query_mem_ratio(scanUseQueryMemRatio);
-        tResult.setAnn_params(VectorSearchOptions.readAnnParams(annParams));
+        tResult.setAnn_params(getAnnParams());
         tResult.setPq_refine_factor(pqRefineFactor);
         tResult.setK_factor(kFactor);
         tResult.setEnable_collect_table_level_scan_stats(enableCollectTableLevelScanStats);
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
index e74fa33b1d9f40..ababf5b585c2c5 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
@@ -20,7 +20,6 @@
 import com.google.common.collect.Sets;
 import com.starrocks.catalog.Database;
 import com.starrocks.catalog.ExternalOlapTable;
-import com.starrocks.catalog.Index;
 import com.starrocks.catalog.KeysType;
 import com.starrocks.catalog.OlapTable;
 import com.starrocks.catalog.Table;
@@ -31,9 +30,6 @@
 import com.starrocks.common.ErrorCode;
 import com.starrocks.common.ErrorReport;
 import com.starrocks.common.LabelAlreadyUsedException;
-import com.starrocks.common.VectorIndexParams.CommonIndexParamKey;
-import com.starrocks.common.VectorIndexParams.VectorIndexType;
-import com.starrocks.common.VectorSearchOptions;
 import com.starrocks.common.profile.Timer;
 import com.starrocks.common.profile.Tracers;
 import com.starrocks.http.HttpConnectContext;
@@ -52,7 +48,6 @@
 import com.starrocks.sql.analyzer.SemanticException;
 import com.starrocks.sql.ast.DeleteStmt;
 import com.starrocks.sql.ast.DmlStmt;
-import com.starrocks.sql.ast.IndexDef;
 import com.starrocks.sql.ast.InsertStmt;
 import com.starrocks.sql.ast.QueryRelation;
 import com.starrocks.sql.ast.QueryStatement;
@@ -87,7 +82,6 @@
 
 import java.util.Collections;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -137,14 +131,12 @@ public static ExecPlan plan(StatementBase stmt, ConnectContext session,
                 boolean areTablesCopySafe = AnalyzerUtils.areTablesCopySafe(queryStmt);
                 needWholePhaseLock = isLockFree(areTablesCopySafe, session) ? false : true;
                 ExecPlan plan;
-                VectorSearchOptions vectorSearchOptions = new VectorSearchOptions();
                 if (needWholePhaseLock) {
-                    plan = createQueryPlan(queryStmt, session, resultSinkType, vectorSearchOptions);
+                    plan = createQueryPlan(queryStmt, session, resultSinkType);
                 } else {
                     long planStartTime = OptimisticVersion.generate();
                     unLock(plannerMetaLocker);
-                    plan = createQueryPlanWithReTry(queryStmt, session, resultSinkType, plannerMetaLocker,
-                                                    planStartTime, vectorSearchOptions);
+                    plan = createQueryPlanWithReTry(queryStmt, session, resultSinkType, plannerMetaLocker, planStartTime);
                 }
                 setOutfileSink(queryStmt, plan);
                 return plan;
@@ -240,16 +232,14 @@ public static MVTransformerContext makeMVTransformerContext(SessionVariable sess
 
     private static ExecPlan createQueryPlan(StatementBase stmt,
                                             ConnectContext session,
-                                            TResultSinkType resultSinkType,
-                                            VectorSearchOptions vectorSearchOptions) {
+                                            TResultSinkType resultSinkType) {
         QueryStatement queryStmt = (QueryStatement) stmt;
-        checkVectorIndex(queryStmt, vectorSearchOptions);
         QueryRelation query = (QueryRelation) queryStmt.getQueryRelation();
         List<String> colNames = query.getColumnOutputNames();
         // 1. Build Logical plan
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
         LogicalPlan logicalPlan;
-        MVTransformerContext mvTransformerContext  = makeMVTransformerContext(session.getSessionVariable());
+        MVTransformerContext mvTransformerContext = makeMVTransformerContext(session.getSessionVariable());
 
         try (Timer ignored = Tracers.watchScope("Transformer")) {
             // get a logicalPlan without inlining views
@@ -270,8 +260,7 @@ private static ExecPlan createQueryPlan(StatementBase stmt,
                     stmt,
                     new PhysicalPropertySet(),
                     new ColumnRefSet(logicalPlan.getOutputColumn()),
-                    columnRefFactory,
-                    vectorSearchOptions);
+                    columnRefFactory);
         }
 
         try (Timer ignored = Tracers.watchScope("ExecPlanBuild")) {
@@ -295,13 +284,10 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt,
                                                     ConnectContext session,
                                                     TResultSinkType resultSinkType,
                                                     PlannerMetaLocker plannerMetaLocker,
-                                                    long planStartTime,
-                                                    VectorSearchOptions vectorSearchOptions) {
+                                                    long planStartTime) {
         QueryRelation query = queryStmt.getQueryRelation();
         List<String> colNames = query.getColumnOutputNames();
 
-        checkVectorIndex(queryStmt, vectorSearchOptions);
-
         // 1. Build Logical plan
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
         boolean isSchemaValid = true;
@@ -344,8 +330,7 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt,
                         queryStmt,
                         new PhysicalPropertySet(),
                         new ColumnRefSet(logicalPlan.getOutputColumn()),
-                        columnRefFactory,
-                        vectorSearchOptions);
+                        columnRefFactory);
             }
 
             try (Timer ignored = Tracers.watchScope("ExecPlanBuild")) {
@@ -378,36 +363,6 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt,
                 "schema of %s had been updated frequently during the plan generation", updatedTables);
     }
 
-    private static boolean checkAndSetVectorIndex(OlapTable olapTable, VectorSearchOptions vectorSearchOptions) {
-        for (Index index : olapTable.getIndexes()) {
-            if (index.getIndexType() == IndexDef.IndexType.VECTOR) {
-                Map<String, String> indexProperties = index.getProperties();
-                String indexType = indexProperties.get(CommonIndexParamKey.INDEX_TYPE.name().toLowerCase(Locale.ROOT));
-
-                if (VectorIndexType.IVFPQ.name().equalsIgnoreCase(indexType)) {
-                    vectorSearchOptions.setUseIVFPQ(true);
-                }
-
-                vectorSearchOptions.setEnableUseANN(true);
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private static void checkVectorIndex(QueryStatement queryStmt, VectorSearchOptions vectorSearchOptions) {
-        Set<OlapTable> olapTables = Sets.newHashSet();
-        AnalyzerUtils.copyOlapTable(queryStmt, olapTables);
-        boolean hasVectorIndex = false;
-        for (OlapTable olapTable : olapTables) {
-            if (checkAndSetVectorIndex(olapTable, vectorSearchOptions)) {
-                hasVectorIndex = true;
-                break;
-            }
-        }
-        vectorSearchOptions.setEnableUseANN(hasVectorIndex);
-    }
-
     public static Set<OlapTable> collectOriginalOlapTables(ConnectContext session, StatementBase queryStmt) {
         Set<OlapTable> olapTables = Sets.newHashSet();
         PlannerMetaLocker locker = new PlannerMetaLocker(session, queryStmt);
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java
index c048f282491a9d..0831477e5795c8 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/CreateTableAnalyzer.java
@@ -97,7 +97,7 @@ public static void analyze(CreateTableStmt statement, ConnectContext context) {
             analyzeTemporaryTable(statement, context, catalogName, db, tableName);
         } else {
             if (GlobalStateMgr.getCurrentState().getMetadataMgr()
-                        .tableExists(catalogName, tableNameObject.getDb(), tableName) && !statement.isSetIfNotExists()) {
+                    .tableExists(catalogName, tableNameObject.getDb(), tableName) && !statement.isSetIfNotExists()) {
                 ErrorReport.reportSemanticException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
             }
         }
@@ -734,6 +734,16 @@ public static void analyzeIndexDefs(CreateTableStmt statement) {
 
         List<Index> indexes = new ArrayList<>();
         if (CollectionUtils.isNotEmpty(indexDefs)) {
+            List<String> vectorIndexNames = indexDefs.stream()
+                    .filter(indexDef -> indexDef.getIndexType() == IndexDef.IndexType.VECTOR)
+                    .map(IndexDef::getIndexName)
+                    .toList();
+            if (vectorIndexNames.size() > 1) {
+                throw new SemanticException(
+                        String.format("At most one vector index is allowed for a table, but %d were found: %s",
+                                vectorIndexNames.size(), vectorIndexNames));
+            }
+
             Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
             Set<List<String>> distinctCol = new HashSet<>();
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SetStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SetStmtAnalyzer.java
index 6e26967907bc1e..678c6132d2e442 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SetStmtAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SetStmtAnalyzer.java
@@ -25,6 +25,7 @@
 import com.starrocks.analysis.StringLiteral;
 import com.starrocks.analysis.Subquery;
 import com.starrocks.catalog.ArrayType;
+import com.starrocks.catalog.IndexParams;
 import com.starrocks.catalog.PrimitiveType;
 import com.starrocks.catalog.Type;
 import com.starrocks.common.ErrorCode;
@@ -37,6 +38,7 @@
 import com.starrocks.datacache.DataCachePopulateMode;
 import com.starrocks.monitor.unit.TimeValue;
 import com.starrocks.mysql.MysqlPassword;
+import com.starrocks.persist.gson.GsonUtils;
 import com.starrocks.qe.ConnectContext;
 import com.starrocks.qe.GlobalVariable;
 import com.starrocks.qe.SessionVariable;
@@ -65,6 +67,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 public class SetStmtAnalyzer {
     public static void analyze(SetStmt setStmt, ConnectContext session) {
@@ -333,6 +336,24 @@ private static void analyzeSystemVariable(SystemVariable var) {
             }
         }
 
+        if (variable.equalsIgnoreCase(SessionVariable.ANN_PARAMS)) {
+            String annParams = resolvedExpression.getStringValue();
+            if (!Strings.isNullOrEmpty(annParams)) {
+                Map<String, String> annParamMap = null;
+                try {
+                    java.lang.reflect.Type type = new com.google.gson.reflect.TypeToken<Map<String, String>>() {}.getType();
+                    annParamMap = GsonUtils.GSON.fromJson(annParams, type);
+                } catch (Exception e) {
+                    throw new SemanticException(String.format("Unsupported ann_params: %s, " +
+                            "It should be a Dict JSON string, each key and value of which is string", annParams));
+                }
+
+                for (Map.Entry<String, String> entry : annParamMap.entrySet()) {
+                    IndexParams.getInstance().checkParams(entry.getKey().toUpperCase(), entry.getValue());
+                }
+            }
+        }
+
         var.setResolvedExpression(resolvedExpression);
     }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
index eaf325e43c288a..003e5fd4a4be30 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
@@ -20,7 +20,6 @@
 import com.starrocks.analysis.JoinOperator;
 import com.starrocks.catalog.MaterializedView;
 import com.starrocks.catalog.OlapTable;
-import com.starrocks.common.VectorSearchOptions;
 import com.starrocks.common.profile.Timer;
 import com.starrocks.common.profile.Tracers;
 import com.starrocks.qe.ConnectContext;
@@ -181,7 +180,7 @@ public OptExpression optimize(ConnectContext connectContext,
                                   ColumnRefSet requiredColumns,
                                   ColumnRefFactory columnRefFactory) {
         return optimize(connectContext, logicOperatorTree, null, null, requiredProperty,
-                requiredColumns, columnRefFactory, new VectorSearchOptions());
+                requiredColumns, columnRefFactory);
     }
 
     public OptExpression optimize(ConnectContext connectContext,
@@ -190,14 +189,11 @@ public OptExpression optimize(ConnectContext connectContext,
                                   StatementBase stmt,
                                   PhysicalPropertySet requiredProperty,
                                   ColumnRefSet requiredColumns,
-                                  ColumnRefFactory columnRefFactory,
-                                  VectorSearchOptions vectorSearchOptions) {
+                                  ColumnRefFactory columnRefFactory) {
         try {
             // prepare for optimizer
             prepare(connectContext, columnRefFactory, logicOperatorTree);
 
-            context.setVectorSearchOptions(vectorSearchOptions);
-
             // prepare for mv rewrite
             prepareMvRewrite(connectContext, logicOperatorTree, columnRefFactory, requiredColumns);
             try (Timer ignored = Tracers.watchScope("MVTextRewrite")) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java
index 81e934efd7d855..7b7c5d77e7723e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java
@@ -19,7 +19,6 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.starrocks.catalog.OlapTable;
-import com.starrocks.common.VectorSearchOptions;
 import com.starrocks.qe.ConnectContext;
 import com.starrocks.qe.SessionVariable;
 import com.starrocks.server.GlobalStateMgr;
@@ -56,7 +55,7 @@ public class OptimizerContext {
     private TaskContext currentTaskContext;
     private final OptimizerConfig optimizerConfig;
 
-    private Set<OlapTable>  queryTables;
+    private Set<OlapTable> queryTables;
 
     private long updateTableId = -1;
 
@@ -84,8 +83,6 @@ public class OptimizerContext {
     // collect all LogicalOlapScanOperators in the query before any optimization
     private List<LogicalOlapScanOperator> allLogicalOlapScanOperators;
 
-    private VectorSearchOptions vectorSearchOptions = new VectorSearchOptions();
-
     @VisibleForTesting
     public OptimizerContext(Memo memo, ColumnRefFactory columnRefFactory) {
         this.memo = memo;
@@ -315,12 +312,4 @@ public void setAllLogicalOlapScanOperators(List<LogicalOlapScanOperator> allScan
     public List<LogicalOlapScanOperator> getAllLogicalOlapScanOperators() {
         return allLogicalOlapScanOperators;
     }
-
-    public void setVectorSearchOptions(VectorSearchOptions vectorSearchOptions) {
-        this.vectorSearchOptions = vectorSearchOptions;
-    }
-
-    public VectorSearchOptions getVectorSearchOptions() {
-        return vectorSearchOptions;
-    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Projection.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Projection.java
index 30bb60c7dde892..7272d642c0cfe7 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Projection.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/operator/Projection.java
@@ -27,6 +27,7 @@
 import java.util.Set;
 
 public class Projection {
+    // output column ref -> expression
     private final Map<ColumnRefOperator, ScalarOperator> columnRefMap;
     // Used for common operator compute result reuse, we need to compute
     // common sub operators firstly in BE
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteToVectorPlanRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteToVectorPlanRule.java
index 1e4a60c409d455..6ebe37c66149fb 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteToVectorPlanRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/RewriteToVectorPlanRule.java
@@ -13,30 +13,37 @@
 // limitations under the License.
 package com.starrocks.sql.optimizer.rule.transformation;
 
-import com.google.common.collect.Lists;
+import com.google.common.base.Enums;
+import com.google.common.base.Preconditions;
 import com.starrocks.analysis.BinaryType;
+import com.starrocks.catalog.ArrayType;
 import com.starrocks.catalog.Column;
-import com.starrocks.catalog.FunctionSet;
+import com.starrocks.catalog.ColumnId;
+import com.starrocks.catalog.Index;
+import com.starrocks.catalog.OlapTable;
 import com.starrocks.catalog.Type;
 import com.starrocks.common.Config;
+import com.starrocks.common.VectorIndexParams;
 import com.starrocks.common.VectorSearchOptions;
+import com.starrocks.sql.analyzer.SemanticException;
+import com.starrocks.sql.ast.IndexDef;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
-import com.starrocks.sql.optimizer.base.ColumnRefFactory;
-import com.starrocks.sql.optimizer.base.Ordering;
-import com.starrocks.sql.optimizer.operator.Operator;
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.Projection;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalTopNOperator;
 import com.starrocks.sql.optimizer.operator.pattern.Pattern;
+import com.starrocks.sql.optimizer.operator.scalar.ArrayOperator;
 import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
 import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
+import com.starrocks.sql.optimizer.operator.scalar.CastOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.operator.scalar.CompoundPredicateOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
 import com.starrocks.sql.optimizer.rule.RuleType;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -46,9 +53,7 @@
 import java.util.stream.Collectors;
 
 import static com.starrocks.analysis.BinaryType.GE;
-import static com.starrocks.analysis.BinaryType.GT;
 import static com.starrocks.analysis.BinaryType.LE;
-import static com.starrocks.analysis.BinaryType.LT;
 import static com.starrocks.catalog.FunctionSet.APPROX_COSINE_SIMILARITY;
 import static com.starrocks.catalog.FunctionSet.APPROX_L2_DISTANCE;
 
@@ -62,224 +67,328 @@ public RewriteToVectorPlanRule() {
 
     @Override
     public boolean check(OptExpression input, OptimizerContext context) {
-        LogicalTopNOperator topNOperator = (LogicalTopNOperator) input.getOp();
-        LogicalOlapScanOperator scanOperator = (LogicalOlapScanOperator) input.getInputs().get(0).getOp();
-
-        scanOperator.setVectorSearchOptions(context.getVectorSearchOptions());
-        VectorSearchOptions vectorSearchOptions = scanOperator.getVectorSearchOptions();
-
-        if (!vectorSearchOptions.isEnableUseANN() || Config.enable_experimental_vector != true) {
+        if (!Config.enable_experimental_vector) {
             return false;
         }
 
-        Map<ColumnRefOperator, ScalarOperator> columnRefMap = scanOperator.getProjection().getColumnRefMap();
+        LogicalTopNOperator topNOp = (LogicalTopNOperator) input.getOp();
+        LogicalOlapScanOperator scanOp = (LogicalOlapScanOperator) input.getInputs().get(0).getOp();
 
-        boolean isEnableUseANN = false;
-        for (Map.Entry<ColumnRefOperator, ScalarOperator> entry : columnRefMap.entrySet()) {
-            if (FunctionSet.VECTOR_COMPUTE_FUNCTIONS.contains(entry.getKey().getName())
-                    && entry.getValue() instanceof CallOperator) {
-                CallOperator callOperator = (CallOperator) entry.getValue();
-                vectorSearchOptions.setQueryVector(collectVectorQuery(callOperator));
-                isEnableUseANN = true;
-                break;
-            }
+        if (scanOp.getProjection() == null) {
+            return false;
         }
 
-        if (!isEnableUseANN) {
-            vectorSearchOptions.setEnableUseANN(false);
+        if (topNOp.getLimit() <= 0 || topNOp.getOrderByElements().size() != 1) {
             return false;
         }
 
-        if (!topNOperator.getOrderByElements().isEmpty() &&
-                FunctionSet.VECTOR_COMPUTE_FUNCTIONS.contains(
-                        topNOperator.getOrderByElements().get(0).getColumnRef().getName())) {
-            return topNOperator.getLimit() != Operator.DEFAULT_LIMIT &&
-                    columnRefMap.entrySet().stream()
-                    .filter(entry -> FunctionSet.VECTOR_COMPUTE_FUNCTIONS.contains(entry.getKey().getName()))
-                    .anyMatch(entry -> entry.getValue() instanceof CallOperator);
-        }
-        return false;
+        return true;
     }
 
     @Override
     public List<OptExpression> transform(OptExpression input, OptimizerContext context) {
-        LogicalTopNOperator topNOperator = (LogicalTopNOperator) input.getOp();
-        LogicalOlapScanOperator scanOperator = (LogicalOlapScanOperator) input.getInputs().get(0).getOp();
-
-        VectorSearchOptions options = scanOperator.getVectorSearchOptions();
-        // set limit_K for ann searcher
-        options.setVectorLimitK(topNOperator.getLimit());
-        ScalarOperator predicate = scanOperator.getPredicate();
-        Optional<ScalarOperator> newPredicate = Optional.empty();
-        boolean isAscending = topNOperator.getOrderByElements().get(0).isAscending();
+        LogicalTopNOperator topNOp = (LogicalTopNOperator) input.getOp();
+        LogicalOlapScanOperator scanOp = (LogicalOlapScanOperator) input.getInputs().get(0).getOp();
+        VectorSearchOptions opts = scanOp.getVectorSearchOptions();
+
+        Optional<VectorFuncInfo> optionalInfo = extractOrderByVectorFuncInfo(topNOp, scanOp);
+        if (optionalInfo.isEmpty()) {
+            return List.of();
+        }
+        VectorFuncInfo info = optionalInfo.get();
+
+        int dim =
+                Integer.parseInt(info.index.getProperties().get(VectorIndexParams.CommonIndexParamKey.DIM.name().toLowerCase()));
+        if (info.vectorQuery.size() != dim) {
+            throw new SemanticException(
+                    String.format("The vector query size (%s) is not equal to the vector index dimension (%d)",
+                            info.vectorQuery, dim));
+        }
+
+        ScalarOperator predicate = scanOp.getPredicate();
         if (predicate != null) {
-            newPredicate = findAndSetVectorRange(predicate, isAscending, options);
-            if (!options.isEnableUseANN()) {
-                return Lists.newArrayList(input);
+            Optional<Double> value = extractVectorRange(predicate, info);
+            // If some predicates cannot be parsed to vector range, vector index cannot be used.
+            if (value.isEmpty()) {
+                return List.of();
             }
+            // All the predicates are parsed to vector range, so remove predicates from scan operator.
+            predicate = null;
+            opts.setPredicateRange(value.get());
         }
-        options.setResultOrder(isAscending ? 0 : 1);
-        String functionName = topNOperator.getOrderByElements().get(0).getColumnRef().getName();
-
-        if (functionName.equalsIgnoreCase(APPROX_L2_DISTANCE) && !isAscending ||
-                functionName.equalsIgnoreCase(APPROX_COSINE_SIMILARITY) && isAscending ||
-                    !options.isEnableUseANN()) {
-            options.setEnableUseANN(false);
-            return Lists.newArrayList(input);
-        }
-        if (options.isUseIVFPQ()) {
+
+        opts.setEnableUseANN(true);
+        String indexType = info.index.getProperties().get(VectorIndexParams.CommonIndexParamKey.INDEX_TYPE.name().toLowerCase());
+        opts.setUseIVFPQ(VectorIndexParams.VectorIndexType.IVFPQ.name().equalsIgnoreCase(indexType));
+        opts.setLimitK(topNOp.getLimit());
+        opts.setResultOrder(info.isAscending);
+        opts.setDistanceColumnName("__vector_" + info.outColumnRef.getName());
+        opts.setQueryVector(info.vectorQuery);
+
+        if (opts.isUseIVFPQ()) {
             // Skip rewrite because IVFPQ is inaccurate and requires a brute force search after the ANN index search
-            input.getInputs().get(0).getOp()
-                    .setPredicate(newPredicate.isPresent() ? newPredicate.get() : null);
-            return Lists.newArrayList(input);
+            LogicalOlapScanOperator newScanOp = LogicalOlapScanOperator.builder()
+                    .withOperator(scanOp)
+                    .setPredicate(predicate)
+                    .build();
+            return List.of(OptExpression.create(topNOp, OptExpression.create(newScanOp)));
         }
 
-        Optional<OptExpression> result = buildVectorSortScanOperator(topNOperator,
-                scanOperator, context, newPredicate, options);
-        return result.isPresent() ? Lists.newArrayList(result.get()) : Lists.newArrayList(input);
+        return List.of(rewriteOptByDistanceColumn(topNOp, scanOp, context, predicate, info, opts));
     }
 
-    public Optional<OptExpression> buildVectorSortScanOperator(LogicalTopNOperator topNOperator,
-            LogicalOlapScanOperator scanOperator, OptimizerContext context,
-            Optional<ScalarOperator> newPredicate, VectorSearchOptions vectorSearchOptions) {
-        // bottom-up
-        String distanceColumnName = scanOperator.getVectorSearchOptions().getVectorDistanceColumnName();
+    private OptExpression rewriteOptByDistanceColumn(LogicalTopNOperator topNOp,
+                                                     LogicalOlapScanOperator scanOp,
+                                                     OptimizerContext context,
+                                                     ScalarOperator newPredicate,
+                                                     VectorFuncInfo info,
+                                                     VectorSearchOptions opts) {
+        // Add index distanceColumn to the scan operator, including table, colRefToColumnMetaMap, and columnMetaToColRefMap.
+        String distanceColumnName = scanOp.getVectorSearchOptions().getDistanceColumnName();
         Column distanceColumn = new Column(distanceColumnName, Type.FLOAT);
-        scanOperator.getTable().addColumn(distanceColumn);
-
-        ColumnRefFactory columnRefFactory = context.getColumnRefFactory();
-        ColumnRefOperator distanceColumnRefOperator = columnRefFactory.create(distanceColumnName, Type.FLOAT, false);
-
-        Map<ColumnRefOperator, Column> colRefToColumnMetaMap = new HashMap<>(scanOperator.getColRefToColumnMetaMap());
-        colRefToColumnMetaMap.put(distanceColumnRefOperator, distanceColumn);
-
-        Map<Column, ColumnRefOperator> columnMetaToColRefMap = new HashMap<>(scanOperator.getColumnMetaToColRefMap());
-        columnMetaToColRefMap.put(distanceColumn, distanceColumnRefOperator);
-
-        // new Scan operator
-        LogicalOlapScanOperator newScanOperator = new LogicalOlapScanOperator(scanOperator.getTable(),
-                colRefToColumnMetaMap, columnMetaToColRefMap, scanOperator.getDistributionSpec(),
-                scanOperator.getLimit(), newPredicate.isPresent() ? newPredicate.get() : null,
-                scanOperator.getSelectedIndexId(), scanOperator.getSelectedPartitionId(),
-                scanOperator.getPartitionNames(), scanOperator.hasTableHints(),
-                scanOperator.getSelectedTabletId(), scanOperator.getHintsTabletIds(),
-                scanOperator.getHintsReplicaIds(), scanOperator.isUsePkIndex());
-
-        newScanOperator.setVectorSearchOptions(vectorSearchOptions);
-        Map<ColumnRefOperator, ScalarOperator> scanProjectMap = new HashMap<>();
-        Map<ColumnRefOperator, ScalarOperator> topNProjectMap = new HashMap<>();
-        // find original column and project it onto the topN
-        Optional<ColumnRefOperator> originalColRef = scanOperator.getProjection().getColumnRefMap()
-                .entrySet().stream().filter(entry -> FunctionSet.VECTOR_COMPUTE_FUNCTIONS
-                        .contains(entry.getKey().getName())).map(entry -> entry.getKey())
-                                .findFirst();
-        if (originalColRef.isEmpty()) {
-            return Optional.empty();
+        scanOp.getTable().addColumn(distanceColumn);
+
+        ColumnRefOperator distanceColRef = context.getColumnRefFactory().create(distanceColumnName, Type.FLOAT, false);
+        Map<ColumnRefOperator, Column> newColRefToColumnMetaMap = new HashMap<>(scanOp.getColRefToColumnMetaMap());
+        newColRefToColumnMetaMap.put(distanceColRef, distanceColumn);
+
+        Map<Column, ColumnRefOperator> newColumnMetaToColRefMap = new HashMap<>(scanOp.getColumnMetaToColRefMap());
+        newColumnMetaToColRefMap.put(distanceColumn, distanceColRef);
+
+        opts.setDistanceSlotId(distanceColRef.getId());
+
+        // Replace the original function call by the distance column ref.
+        Map<ColumnRefOperator, ScalarOperator> newScanProjectMap = scanOp.getProjection().getColumnRefMap().entrySet().stream()
+                .collect(Collectors.toMap(
+                        Map.Entry::getKey,
+                        entry -> rewriteScalarOperatorByDistanceColumn(entry.getValue(), info, distanceColRef)
+                ));
+
+        LogicalOlapScanOperator newScanOp = LogicalOlapScanOperator.builder()
+                .withOperator(scanOp)
+                .setProjection(new Projection(newScanProjectMap))
+                .setPredicate(newPredicate)
+                .setColRefToColumnMetaMap(newColRefToColumnMetaMap)
+                .setColumnMetaToColRefMap(newColumnMetaToColRefMap)
+                .build();
+
+        return OptExpression.create(topNOp, OptExpression.create(newScanOp));
+    }
+
+    ScalarOperator rewriteScalarOperatorByDistanceColumn(ScalarOperator scalarOperator, VectorFuncInfo info,
+                                                         ColumnRefOperator distanceColRef) {
+        if (scalarOperator.equals(info.vectorFuncCallOperator)) {
+            return distanceColRef;
+        }
+
+        for (int i = 0; i < scalarOperator.getChildren().size(); i++) {
+            ScalarOperator child = scalarOperator.getChild(i);
+            scalarOperator.setChild(i, rewriteScalarOperatorByDistanceColumn(child, info, distanceColRef));
+        }
+
+        return scalarOperator;
+    }
+
+    /**
+     * Check if the operator matches the specific vector function call.
+     *
+     * <p> For example, assume that `vectorFuncCallOperator` is `approx_l2_distance(v1, [1,2,3])`,
+     * then the following operators match:
+     * - `approx_l2_distance(v1, [1,2,3])`
+     * - `cast(approx_l2_distance(v1, [1,2,3]) as float)`
+     * - `cast(approx_l2_distance(v1, [1,2,3]) as double)`
+     */
+    private boolean matchesVectorFuncCall(CallOperator vectorFuncCallOperator, ScalarOperator operator) {
+        if (operator instanceof CastOperator) {
+            CastOperator castOperator = (CastOperator) operator;
+            return castOperator.getType().isFloatingPointType() &&
+                    matchesVectorFuncCall(vectorFuncCallOperator, castOperator.getChild(0));
+        }
+
+        if (operator instanceof CallOperator) {
+            return vectorFuncCallOperator.equals(operator);
         }
 
-        scanOperator.getProjection().getColumnRefMap().entrySet().stream()
-                .forEach(entry -> {
-                    if (FunctionSet.VECTOR_COMPUTE_FUNCTIONS.contains(entry.getKey().getName())
-                            && entry.getValue() instanceof CallOperator) {
-                        scanProjectMap.put(distanceColumnRefOperator, distanceColumnRefOperator);
+        return false;
+    }
+
+    /**
+     * Extract the vector range from the predicates.
+     *
+     * <p> Only the predicates in the following format can be parsed to vector range:
+     * - req1: <=, >=, and one side is constant, the other side is the vector index column.
+     * - req2: AND, and each child predicate meets req1.
+     *
+     * <p> For example, suppose v1 is the vector index column and isAscending=true, then:
+     * - v1 <= 10: 10
+     * - v1 <= 10 AND v1 < 20: 10
+     * - v1 >= 10: cannot be parsed
+     * - c1 <= 10: cannot be parsed
+     * - v1 <= 10 and c1 < 10: cannot be parsed
+     *
+     * @return the vector range value if the predicate can be parsed to vector range, otherwise empty.
+     */
+    private Optional<Double> extractVectorRange(ScalarOperator predicate, VectorFuncInfo info) {
+        if (predicate instanceof BinaryPredicateOperator) {
+            return parseVectorRangeFromBinaryPredicate(predicate, info);
+        } else if (predicate instanceof CompoundPredicateOperator) {
+            CompoundPredicateOperator compoundPredicate = (CompoundPredicateOperator) predicate;
+            if (!compoundPredicate.isAnd()) {
+                return Optional.empty();
+            }
+            Optional<Double> value = Optional.empty();
+            for (ScalarOperator child : predicate.getChildren()) {
+                Optional<Double> childValue = parseVectorRangeFromBinaryPredicate(child, info);
+                if (childValue.isEmpty()) {
+                    return Optional.empty();
+                }
+                if (value.isEmpty()) {
+                    value = childValue;
+                } else {
+                    if (info.isAscending) {
+                        value = Optional.of(Math.min(value.get(), childValue.get()));
                     } else {
-                        scanProjectMap.put(entry.getKey(), entry.getValue());
-                        topNProjectMap.put(entry.getKey(), entry.getValue());
+                        value = Optional.of(Math.max(value.get(), childValue.get()));
                     }
-                });
-        newScanOperator.setProjection(new Projection(scanProjectMap));
-
-        List<Ordering> orderByElements = topNOperator.getOrderByElements().stream().map(ordering ->
-                FunctionSet.VECTOR_COMPUTE_FUNCTIONS.contains(ordering.getColumnRef().getName()) ?
-                    new Ordering(distanceColumnRefOperator, ordering.isAscending(), ordering.isNullsFirst()) : ordering)
-                .collect(Collectors.toList());
-
-        boolean hasProjection = topNOperator.getProjection() != null;
-        Map<ColumnRefOperator, ScalarOperator> newTopNProjectMap = new HashMap<>();
-        if (hasProjection) {
-            topNOperator.getProjection().getColumnRefMap().entrySet().stream()
-                    .forEach(entry -> {
-                        if (FunctionSet.VECTOR_COMPUTE_FUNCTIONS.contains(entry.getKey().getName())) {
-                            newTopNProjectMap.put(originalColRef.get(), distanceColumnRefOperator);
-                        } else {
-                            newTopNProjectMap.put(entry.getKey(), entry.getValue());
-                        }
-                    });
-        } else {
-            topNProjectMap.put(originalColRef.get(), distanceColumnRefOperator);
+                }
+            }
+            return value;
         }
 
-        // new TopN operator
-        LogicalTopNOperator newTopNOperator = new LogicalTopNOperator(topNOperator.getLimit(),
-                topNOperator.getPredicate(),
-                hasProjection ? new Projection(newTopNProjectMap) : new Projection(topNProjectMap),
-                topNOperator.getPartitionByColumns(), topNOperator.getPartitionLimit(), orderByElements,
-                topNOperator.getOffset(), topNOperator.getSortPhase(), topNOperator.getTopNType(), topNOperator.isSplit());
+        return Optional.empty();
+    }
 
-        OptExpression topNExpression = OptExpression.create(newTopNOperator);
-        topNExpression.getInputs().clear();
-        topNExpression.getInputs().add(OptExpression.create(newScanOperator));
+    private Optional<Double> parseVectorRangeFromBinaryPredicate(ScalarOperator predicate, VectorFuncInfo info) {
+        if (predicate instanceof BinaryPredicateOperator) {
+            BinaryType binaryType = ((BinaryPredicateOperator) predicate).getBinaryType();
+            ScalarOperator lhs = predicate.getChild(0);
+            ScalarOperator rhs = predicate.getChild(1);
+
+            if (rhs instanceof ConstantOperator && matchesVectorFuncCall(info.vectorFuncCallOperator, lhs) &&
+                    (((binaryType.equals(LE)) && info.isAscending) || ((binaryType.equals(GE)) && !info.isAscending))) {
+                return Optional.of((double) ((ConstantOperator) rhs).getValue());
+            } else if (lhs instanceof ConstantOperator && matchesVectorFuncCall(info.vectorFuncCallOperator, rhs) &&
+                    (((binaryType.equals(GE)) && info.isAscending) || ((binaryType.equals(LE)) && !info.isAscending))) {
+                return Optional.of((double) ((ConstantOperator) lhs).getValue());
+            }
+        }
 
-        return Optional.of(topNExpression);
+        return Optional.empty();
     }
 
-    public Optional<ScalarOperator> findAndSetVectorRange(ScalarOperator operator,
-                boolean isAscending, VectorSearchOptions options) {
-        if (!options.isEnableUseANN()) {
+    /**
+     * Extract the vector function information. If the vector index can be used, the following requirements need to be met:
+     * 1. The first column of the ordering is the <approx_distance> function.
+     * 2. The <approx_distance> function needs to match the metric_type and isAscending of the vector index.
+     * - If the metric_type is L2_DISTANCE, then the <approx_distance> function is approx_l2_distance, and the order is ASC.
+     * - If the metric_type is COSINE_SIMILARITY, then the <approx_distance> function is cosine_similarity, and the order is DESC.
+     * 3. The arguments of the <approx_distance> function are the vector index column and a constant array.
+     *
+     * @return the vector function information if the ordering column is matched, otherwise empty.
+     */
+    private Optional<VectorFuncInfo> extractOrderByVectorFuncInfo(LogicalTopNOperator topNOp, LogicalOlapScanOperator scanOp) {
+        OlapTable table = (OlapTable) scanOp.getTable();
+        Index index = table.getIndexes().stream()
+                .filter(i -> i.getIndexType() == IndexDef.IndexType.VECTOR)
+                .findFirst()
+                .orElse(null);
+        if (index == null) {
             return Optional.empty();
         }
 
-        if (operator instanceof BinaryPredicateOperator && operator.getChild(1) instanceof ConstantOperator &&
-                (isVectorCallOperator(operator.getChild(0)))) {
-            BinaryType binaryType = ((BinaryPredicateOperator) operator).getBinaryType();
-            if (((binaryType.equals(LE) || binaryType.equals(LT)) && !isAscending) ||
-                    ((binaryType.equals(GE) || binaryType.equals(GT)) && isAscending)) {
-                options.setEnableUseANN(false);
-                return Optional.empty();
-            }
-            options.setVectorRange((double) (((ConstantOperator) operator.getChild(1)).getValue()));
+        ColumnRefOperator outColRef = topNOp.getOrderByElements().get(0).getColumnRef();
+        final boolean isAscending = topNOp.getOrderByElements().get(0).isAscending();
+
+        String rawMetricType = index.getProperties().get(VectorIndexParams.CommonIndexParamKey.METRIC_TYPE.name().toLowerCase());
+        VectorIndexParams.MetricsType metricType =
+                Enums.getIfPresent(VectorIndexParams.MetricsType.class, StringUtils.upperCase(rawMetricType)).orNull();
+        Preconditions.checkNotNull(metricType, "Invalid metric type [" + rawMetricType + "] for vector index");
+
+        // 1. Check: it is a matched vector function.
+        ScalarOperator inOperator = scanOp.getProjection().getColumnRefMap().get(outColRef);
+        if (!(inOperator instanceof CallOperator)) {
             return Optional.empty();
-        } else if (operator instanceof CompoundPredicateOperator) {
-            List<ScalarOperator> newOperators = new ArrayList<>();
-            for (ScalarOperator child : operator.getChildren()) {
-                Optional<ScalarOperator> newChild = findAndSetVectorRange(child, isAscending, options);
-                if (newChild.isPresent()) {
-                    newOperators.add(newChild.get());
-                }
-            }
-            if (newOperators.size() > 1) {
-                return Optional.of(new CompoundPredicateOperator(((CompoundPredicateOperator) operator).getCompoundType(),
-                        newOperators));
-            } else if (newOperators.size() == 1) {
-                return Optional.of(newOperators.get(0));
-            } else {
-                return Optional.empty();
-            }
+        }
+        CallOperator inCallOperator = (CallOperator) inOperator;
+
+        boolean matchedFunc;
+        switch (metricType) {
+            case L2_DISTANCE:
+                matchedFunc = inCallOperator.getFnName().equalsIgnoreCase(APPROX_L2_DISTANCE) && isAscending;
+                break;
+            case COSINE_SIMILARITY:
+                matchedFunc = inCallOperator.getFnName().equalsIgnoreCase(APPROX_COSINE_SIMILARITY) && !isAscending;
+                break;
+            default:
+                matchedFunc = false;
+        }
+        if (!matchedFunc) {
+            return Optional.empty();
+        }
+
+        // 2. Check: the vector function's arguments are column ref and constant.
+        ScalarOperator lhs = inCallOperator.getChild(0);
+        ScalarOperator rhs = inCallOperator.getChild(1);
+        ColumnRefOperator colRefArgument;
+        if (isConstantArrayFloat(lhs) && rhs.isColumnRef()) {
+            colRefArgument = (ColumnRefOperator) rhs;
+        } else if (isConstantArrayFloat(rhs) && lhs.isColumnRef()) {
+            colRefArgument = (ColumnRefOperator) lhs;
         } else {
-            options.setEnableUseANN(false);
-            return Optional.of(operator.clone());
+            return Optional.empty();
         }
-    }
 
-    public boolean isVectorCallOperator(ScalarOperator scalarOperator) {
-        if (scalarOperator instanceof CallOperator &&
-                FunctionSet.VECTOR_COMPUTE_FUNCTIONS.contains(((CallOperator) scalarOperator).getFnName())) {
-            return true;
+        // 3. Check: the column ref argument of the vector function matches the index column.
+        Column column = scanOp.getColRefToColumnMetaMap().get(colRefArgument);
+        if (column == null) {
+            return Optional.empty();
         }
-        if (scalarOperator.getChildren().size() == 0) {
-            return false;
+
+        ColumnId indexColumnId = index.getColumns().get(0);
+        if (!column.getColumnId().equals(indexColumnId)) {
+            return Optional.empty();
         }
-        return isVectorCallOperator(scalarOperator.getChild(0));
-    }
 
-    public List<String> collectVectorQuery(CallOperator callOperator) {
-        // suppose it's a standard vector query
+        // 4. Parse query vector values.
         List<String> vectorQuery = new ArrayList<>();
-        collectVector(callOperator, vectorQuery);
-        return vectorQuery;
+        extractValuesFromConstantArray(inCallOperator, vectorQuery);
+
+        return Optional.of(
+                new VectorFuncInfo(index, colRefArgument, outColRef, inCallOperator, metricType, vectorQuery, isAscending));
+    }
+
+    /**
+     * Whether the scalar operator is a constant array of float, which is represented as
+     * `ArrayOperator(type=ArrayType(float))` or
+     * `CastOperator(child=ArrayOperator(type=ArrayType(numeric_type)), type=ArrayType(float))`.
+     */
+    private boolean isConstantArrayFloat(ScalarOperator scalarOperator) {
+        if (!scalarOperator.isConstant()) {
+            return false;
+        }
+
+        if (scalarOperator instanceof CastOperator) {
+            if (!scalarOperator.getType().isArrayType()) {
+                return false;
+            }
+            ArrayType arrayType = (ArrayType) scalarOperator.getType();
+            if (!arrayType.getItemType().isFloatingPointType()) {
+                return false;
+            }
+
+            return scalarOperator.getChildren().stream().allMatch(this::isConstantArrayFloat);
+        } else if (scalarOperator instanceof ArrayOperator) {
+            if (!scalarOperator.getType().isArrayType()) {
+                return false;
+            }
+            ArrayType innerArrayType = (ArrayType) scalarOperator.getType();
+            return innerArrayType.getItemType().isNumericType();
+        } else {
+            return false;
+        }
     }
 
-    public void collectVector(ScalarOperator scalarOperator, List<String> vectorQuery) {
+    private void extractValuesFromConstantArray(ScalarOperator scalarOperator, List<String> vectorQuery) {
         if (scalarOperator instanceof ColumnRefOperator) {
             return;
         }
@@ -290,7 +399,34 @@ public void collectVector(ScalarOperator scalarOperator, List<String> vectorQuer
         }
 
         for (ScalarOperator child : scalarOperator.getChildren()) {
-            collectVector(child, vectorQuery);
+            extractValuesFromConstantArray(child, vectorQuery);
+        }
+    }
+
+    private static class VectorFuncInfo {
+        private final Index index;
+        // vector index column
+        private final ColumnRefOperator inColumnRef;
+        // The column ref of the first ordering column, which is obtained by vectorFuncCallOperator `<approx_distance>(inColumnRef, vectorQuery)`.
+        // - If metricType is L2_DISTANCE, then <approx_distance> function is `approx_l2_distance`, and the order is ASC.
+        // - If metricType is COSINE_SIMILARITY, then <approx_distance> function `is cosine_similarity`, and the order is DESC.
+        private final ColumnRefOperator outColumnRef;
+        private final CallOperator vectorFuncCallOperator;
+        private final VectorIndexParams.MetricsType metricType;
+        // The constant vector argument value of the <approx_distance> function
+        private final List<String> vectorQuery;
+        private final boolean isAscending;
+
+        public VectorFuncInfo(Index index, ColumnRefOperator inColumnRef, ColumnRefOperator outColumnRef,
+                              CallOperator vectorFuncCallOperator,
+                              VectorIndexParams.MetricsType metricType, List<String> vectorQuery, boolean isAscending) {
+            this.index = index;
+            this.inColumnRef = inColumnRef;
+            this.outColumnRef = outColumnRef;
+            this.vectorFuncCallOperator = vectorFuncCallOperator;
+            this.metricType = metricType;
+            this.vectorQuery = vectorQuery;
+            this.isAscending = isAscending;
         }
     }
 }
diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/VectorIndexTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/VectorIndexTest.java
index 43e4513c67f599..7f6ce33ca06846 100644
--- a/fe/fe-core/src/test/java/com/starrocks/analysis/VectorIndexTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/analysis/VectorIndexTest.java
@@ -86,7 +86,7 @@ public void testCheckVectorIndex() {
                 () -> VectorIndexUtil.checkVectorIndexValid(c3, Collections.emptyMap(), KeysType.DUP_KEYS),
                 "You should set index_type at least to add a vector index.");
 
-        Column c4 = new Column("f4", Type.ARRAY_FLOAT, true);
+        Column c4 = new Column("f4", Type.ARRAY_FLOAT, false);
         Assertions.assertThrows(
                 SemanticException.class,
                 () -> VectorIndexUtil.checkVectorIndexValid(c4, new HashMap<>() {{
@@ -148,8 +148,7 @@ public void testCheckVectorIndex() {
                     put(VectorIndexParams.IndexParamsKey.M.name(), "10");
                     put(VectorIndexParams.IndexParamsKey.EFCONSTRUCTION.name(), "10");
                     put(VectorIndexParams.SearchParamsKey.EFSEARCH.name(), "10");
-                }}, KeysType.DUP_KEYS),
-                "Params HNSW should not define with NBITS"
+                }}, KeysType.DUP_KEYS)
         );
 
         Map<String, String> paramItemMap = new HashMap<>(){{
diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/VectorIndexTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/VectorIndexTest.java
index 152504daebda7e..135df9e5407a8d 100644
--- a/fe/fe-core/src/test/java/com/starrocks/planner/VectorIndexTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/planner/VectorIndexTest.java
@@ -36,10 +36,13 @@
 
 import com.starrocks.common.Config;
 import com.starrocks.common.FeConstants;
+import com.starrocks.sql.analyzer.SemanticException;
 import com.starrocks.sql.plan.PlanTestBase;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
 public class VectorIndexTest extends PlanTestBase {
 
     @BeforeClass
@@ -49,7 +52,9 @@ public static void beforeClass() throws Exception {
         FeConstants.enablePruneEmptyOutputScan = false;
         starRocksAssert.withTable("CREATE TABLE test.test_cosine ("
                 + " c0 INT,"
-                + " c1 array<float>,"
+                + " c1 array<float> NOT NULL,"
+                + " c2 array<float>,"
+                + " vector_distance float,"
                 + " INDEX index_vector1 (c1) USING VECTOR ('metric_type' = 'cosine_similarity', "
                 + "'is_vector_normed' = 'false', 'M' = '512', 'index_type' = 'hnsw', 'dim'='5') "
                 + ") "
@@ -59,7 +64,8 @@ public static void beforeClass() throws Exception {
 
         starRocksAssert.withTable("CREATE TABLE test.test_l2 ("
                 + " c0 INT,"
-                + " c1 array<float>,"
+                + " c1 array<float> NOT NULL,"
+                + " c2 array<float>,"
                 + " INDEX index_vector1 (c1) USING VECTOR ('metric_type' = 'l2_distance', "
                 + "'is_vector_normed' = 'false', 'M' = '512', 'index_type' = 'hnsw', 'dim'='5') "
                 + ") "
@@ -69,53 +75,481 @@ public static void beforeClass() throws Exception {
 
         starRocksAssert.withTable("CREATE TABLE test.test_ivfpq ("
                 + " c0 INT,"
-                + " c1 array<float>,"
+                + " c1 array<float> NOT NULL,"
+                + " c2 array<float>,"
                 + " INDEX index_vector1 (c1) USING VECTOR ('metric_type' = 'l2_distance', "
-                + "'is_vector_normed' = 'false', 'nbits' = '1', 'index_type' = 'ivfpq', 'dim'='5') "
+                + "'is_vector_normed' = 'false', 'nbits' = '8', 'index_type' = 'ivfpq', 'dim'='4', 'm_ivfpq'='2') "
+                + ") "
+                + "DUPLICATE KEY(c0) "
+                + "DISTRIBUTED BY HASH(c0) BUCKETS 1 "
+                + "PROPERTIES ('replication_num'='1');");
+
+        starRocksAssert.withTable("CREATE TABLE test.test_no_vector_index ("
+                + " c0 INT,"
+                + " c1 array<float>,"
+                + " c2 array<float>"
                 + ") "
                 + "DUPLICATE KEY(c0) "
                 + "DISTRIBUTED BY HASH(c0) BUCKETS 1 "
                 + "PROPERTIES ('replication_num'='1');");
     }
 
+    @Test
+    public void testMeetOrderByRequirement() throws Exception {
+        String sql;
+        String plan;
+
+        // Basic cases.
+        sql = "select c1 from test_cosine " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "  2:TOP-N\n" +
+                "  |  order by: [5, FLOAT, false] DESC\n" +
+                "  |  build runtime filters:\n" +
+                "  |  - filter_id = 0, build_expr = (<slot 5> 5: approx_cosine_similarity), remote = false\n" +
+                "  |  offset: 0\n" +
+                "  |  limit: 10\n" +
+                "  |  cardinality: 1\n" +
+                "  |  \n" +
+                "  1:Project\n" +
+                "  |  output columns:\n" +
+                "  |  2 <-> [2: c1, ARRAY<FLOAT>, false]\n" +
+                "  |  5 <-> [7: __vector_approx_cosine_similarity, FLOAT, false]\n" +
+                "  |  cardinality: 1\n" +
+                "  |  \n" +
+                "  0:OlapScanNode\n" +
+                "     table: test_cosine, rollup: test_cosine\n" +
+                "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: -1.0");
+
+        sql = "select c1 from test_l2 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: -1.0");
+
+        // Constant vector with cast.
+        sql = "select c1 from test_cosine " +
+                "order by approx_cosine_similarity([cast(1.1 as double),cast(2.1 as double)," +
+                "cast(3.1 as double),cast(4.1 as double),cast(5.1 as double)], c1) desc " +
+                "limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.1, 3.1, 4.1, 5.1], Predicate Range: -1.0");
+
+        sql = "select c1 from test_cosine " +
+                "order by approx_cosine_similarity([cast(1.1 as float),cast(2.1 as float),cast(3.1 as float)" +
+                ",cast(4.1 as float),cast(5.1 as float)], c1) desc " +
+                "limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.1, 3.1, 4.1, 5.1], Predicate Range: -1.0");
+
+        sql = "select c1 from test_cosine " +
+                "order by approx_cosine_similarity([cast(1.1 as int),cast(2.1 as int),cast(3.1 as int)" +
+                ",cast(4.1 as int),cast(5.1 as int)], c1) desc " +
+                "limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.1, 3.1, 4.1, 5.1], Predicate Range: -1.0");
+    }
+
+    @Test
+    public void testNotMeetOrderByRequirement() throws Exception {
+        String sql;
+        String plan;
+
+        // Wrong function name.
+        sql = "select c1 from test_l2 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        sql = "select c1 from test_cosine " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // Wrong column ref.
+        sql = "select c1 from test_l2 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c2) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // Wrong constant vector
+        sql = "select c1 from test_l2 " +
+                "order by approx_l2_distance(['a', 'b', 'c'], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        sql = "select c1 from test_l2 " +
+                "order by approx_l2_distance(c2, c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // Wrong ASC/DESC
+        sql = "select c1 from test_l2 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) DESC limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        sql = "select c1 from test_cosine " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // No limit.
+        sql = "select c1 from test_cosine " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) DESC";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+    }
+
+    @Test
+    public void testMeetPredicateRequirement() throws Exception {
+        String sql;
+        String plan;
+
+        // Basic cases.
+        sql = "select c1 from test_cosine " +
+                "where approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) >= 100 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        sql = "select c1 from test_cosine " +
+                "where approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) >= 100 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        // Cast
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= cast(100 as double) " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= cast(100 as int) " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= cast(100 as float) " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        // AND
+        sql = "select c1 from test_cosine " +
+                "where approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) >= 1000 " +
+                "and approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) >= 100 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 1000.0");
+
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 100 and approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 1000 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+    }
+
+    @Test
+    public void testNotMeetPredicateRequirement() throws Exception {
+        String sql;
+        String plan;
+
+        // Predicate direction wrong.
+        sql = "select c1 from test_cosine " +
+                "where approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) <= 100 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) >= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // Must >=, <=, not >, <.
+        sql = "select c1 from test_cosine " +
+                "where approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) > 100 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) < 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // Column ref is not vector column.
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c2) <= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // constant vector is not the same.
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([10,2.2,3.3], c2) <= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // Cannot deal with approx_l2_distance with other functions.
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) * 2 <= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // Cannot deal with approx_l2_distance with other predicates.
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 100 and c0 < 10 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+
+        // OR
+        sql = "select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 100 or approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 1000 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "VECTORINDEX: OFF");
+    }
+
+    @Test
+    public void testRewrite() throws Exception {
+        String sql;
+        String plan;
+
+        sql = "select c1, " +
+                "approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1)+1, " +
+                "approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1)+2, " +
+                "cast(approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) as string), " +
+                "approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c2)+2 " +
+                "from test_cosine " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "  2:TOP-N\n" +
+                "  |  order by: [9, FLOAT, false] DESC\n" +
+                "  |  build runtime filters:\n" +
+                "  |  - filter_id = 0, build_expr = (<slot 9> 9: approx_cosine_similarity), remote = false\n" +
+                "  |  offset: 0\n" +
+                "  |  limit: 10\n" +
+                "  |  cardinality: 1\n" +
+                "  |  \n" +
+                "  1:Project\n" +
+                "  |  output columns:\n" +
+                "  |  2 <-> [2: c1, ARRAY<FLOAT>, false]\n" +
+                "  |  5 <-> [13: cast, DOUBLE, true] + 1.0\n" +
+                "  |  6 <-> [13: cast, DOUBLE, true] + 2.0\n" +
+                "  |  7 <-> cast([12: __vector_approx_cosine_similarity, FLOAT, false] as VARCHAR(65533))\n" +
+                "  |  8 <-> cast(approx_cosine_similarity[(cast([1.1,2.2,3.3,4.4,5.5] as ARRAY<FLOAT>), [3: c2, ARRAY<FLOAT>, true]); " +
+                "args: INVALID_TYPE,INVALID_TYPE; result: FLOAT; args nullable: true; result nullable: true] as DOUBLE) + 2.0\n" +
+                "  |  9 <-> [12: __vector_approx_cosine_similarity, FLOAT, false]\n" +
+                "  |  common expressions:\n" +
+                "  |  13 <-> cast([12: __vector_approx_cosine_similarity, FLOAT, false] as DOUBLE)\n" +
+                "  |  cardinality: 1\n" +
+                "  |  \n" +
+                "  0:OlapScanNode\n" +
+                "     table: test_cosine, rollup: test_cosine\n" +
+                "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <12:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: -1.0");
+    }
+
+    @Test
+    public void testArgumentOrder() throws Exception {
+        String sql;
+        String plan;
+
+        // Vector function argument order doesn't matter.
+        sql = "select c1 from test_cosine " +
+                "order by approx_cosine_similarity(c1, [1.1,2.2,3.3,4.4,5.5]) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: -1.0");
+
+        sql = "select c1 from test_l2 " +
+                "order by approx_l2_distance(c1, [1.1,2.2,3.3,4.4,5.5]) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: -1.0");
+
+        // Predicate argument order doesn't matter.
+        sql = "select c1 from test_cosine " +
+                "where 100 <= approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <7:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+
+        sql = "select c1 from test_l2 " +
+                "where 100 >= approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
+        plan = getVerboseExplain(sql);
+        assertContains(plan, "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <6:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+    }
+
+    @Test
+    public void testMultipleTables() throws Exception {
+        String sql;
+        String plan;
+
+        sql = "(select c1 from test_cosine " +
+                "where approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) >= 100 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10) " +
+                "UNION ALL " +
+                "(select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10) " +
+                "UNION ALL " +
+                "(select c1 from test_cosine " +
+                "where approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) >= 100 " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) limit 10) " +
+                "UNION ALL " +
+                "(select c1 from test_l2 " +
+                "where approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) <= 100 " +
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) DESC limit 10) " +
+                "UNION ALL " +
+                "(select c1 from test_no_vector_index)";
+        plan = getVerboseExplain(sql);
+        System.out.println(plan);
+        assertContains(plan, "  1:OlapScanNode\n" +
+                "     table: test_cosine, rollup: test_cosine\n" +
+                "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <24:__vector_approx_cosine_similarity>, LimitK: 10, Order: DESC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+        assertContains(plan, "  7:OlapScanNode\n" +
+                "     table: test_l2, rollup: test_l2\n" +
+                "     VECTORINDEX: ON\n" +
+                "          IVFPQ: OFF, Distance Column: <23:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, " +
+                "Query Vector: [1.1, 2.2, 3.3, 4.4, 5.5], Predicate Range: 100.0");
+        assertContains(plan, "  13:OlapScanNode\n" +
+                "     table: test_cosine, rollup: test_cosine\n" +
+                "     VECTORINDEX: OFF");
+        assertContains(plan, "  25:OlapScanNode\n" +
+                "     table: test_no_vector_index, rollup: test_no_vector_index\n" +
+                "     VECTORINDEX: OFF");
+    }
+
+    @Test
+    public void testQueryVectorDimNotMatch() throws Exception {
+        String sql = "select c1 from test.test_cosine " +
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4], c1) desc limit 10";
+        assertThatThrownBy(() -> getVerboseExplain(sql))
+                .isInstanceOf(SemanticException.class)
+                .hasMessageContaining(
+                        "The vector query size ([1.1, 2.2, 3.3, 4.4]) is not equal to the vector index dimension (5)");
+    }
+
+    @Test
+    public void testIvfpq() throws Exception {
+        String sql = "select c1, approx_l2_distance([1.1,2.2,3.3,4.4], c1) as score"
+                + " from test_ivfpq order by score limit 10";
+        String plan = getVerboseExplain(sql);
+        assertContains(plan, "  2:TOP-N\n" +
+                "  |  order by: [4, FLOAT, true] ASC\n" +
+                "  |  build runtime filters:\n" +
+                "  |  - filter_id = 0, build_expr = (<slot 4> 4: approx_l2_distance), remote = false\n" +
+                "  |  offset: 0\n" +
+                "  |  limit: 10\n" +
+                "  |  cardinality: 1\n" +
+                "  |  \n" +
+                "  1:Project\n" +
+                "  |  output columns:\n" +
+                "  |  2 <-> [2: c1, ARRAY<FLOAT>, false]\n" +
+                "  |  4 <-> approx_l2_distance[(cast([1.1,2.2,3.3,4.4] as ARRAY<FLOAT>), [2: c1, ARRAY<FLOAT>, false]); args: INVALID_TYPE,INVALID_TYPE; result: FLOAT; args nullable: true; result nullable: true]\n" +
+                "  |  cardinality: 1\n" +
+                "  |  \n" +
+                "  0:OlapScanNode\n" +
+                "     table: test_ivfpq, rollup: test_ivfpq\n" +
+                "     VECTORINDEX: ON\n" +
+                "          IVFPQ: ON, Distance Column: <0:__vector_approx_l2_distance>, LimitK: 10, Order: ASC, Query Vector: [1.1, 2.2, 3.3, 4.4], Predicate Range: -1.0");
+    }
+
     @Test
     public void testVectorIndexSyntax() throws Exception {
         String sql1 = "select c1 from test.test_cosine " +
-                "order by approx_cosine_similarity([1.1,2.2,3.3], c1) desc limit 10";
+                "order by approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
         assertPlanContains(sql1, "VECTORINDEX: ON");
 
         String sql2 = "select c1 from test.test_l2 " +
-                "order by approx_l2_distance([1.1,2.2,3.3], c1) limit 10";
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) limit 10";
         assertPlanContains(sql2, "VECTORINDEX: ON");
 
         // Sorting in desc order doesn't make sense in l2_distance,
         // which won't trigger the vector retrieval logic.
         String sql3 = "select c1 from test.test_l2 " +
-                "order by approx_l2_distance([1.1,2.2,3.3], c1) desc limit 10";
+                "order by approx_l2_distance([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
         assertPlanContains(sql3, "VECTORINDEX: OFF");
 
         String sql4 = "select c1 from test.test_cosine " +
-                "order by cosine_similarity([1.1,2.2,3.3], c1) desc limit 10";
+                "order by cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) desc limit 10";
         assertPlanContains(sql4, "VECTORINDEX: OFF");
 
-        String sql5 = "select c1, approx_l2_distance([1.1,2.2,3.3], c1) as score"
+        String sql5 = "select c1, approx_l2_distance([1.1,2.2,3.3,4.4], c1) as score"
                 + " from test.test_ivfpq order by score limit 10";
         assertPlanContains(sql5, "VECTORINDEX: ON");
 
-        String sql6 = "select c1, approx_cosine_similarity([1.1,2.2,3.3], c1) as score"
+        String sql6 = "select c1, approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) as score"
                 + " from test.test_cosine order by score desc limit 10";
         assertPlanContains(sql6, "VECTORINDEX: ON");
 
-        String sql7 = "select c1, approx_cosine_similarity([1.1,2.2,3.3], c1) as score"
+        String sql7 = "select c1, approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) as score"
                 + " from test.test_cosine where c0 = 1 order by score desc limit 10";
         assertPlanContains(sql7, "VECTORINDEX: OFF");
 
-        String sql8 = "select c1, approx_cosine_similarity([1.1,2.2,3.3], c1) as score"
-                + " from test.test_cosine having score > 0.8 order by score desc limit 10";
+        String sql8 = "select c1, approx_cosine_similarity([1.1,2.2,3.3,4.4,5.5], c1) as score"
+                + " from test.test_cosine having score >= cast(0.8 as float) order by score desc limit 10";
         assertPlanContains(sql8, "VECTORINDEX: ON");
-
-        String sql9 = "select c1, approx_cosine_similarity([1.1,2.2,3.3], c1) as score"
-                + " from test.test_cosine having score < 0.8 order by score desc limit 10";
-        assertPlanContains(sql9, "VECTORINDEX: OFF");
     }
+
 }
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeAlterTableStatementTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeAlterTableStatementTest.java
index 0162e6273151d5..81b1ed9167a319 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeAlterTableStatementTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeAlterTableStatementTest.java
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-
 package com.starrocks.sql.analyzer;
 
 import com.google.common.collect.Lists;
@@ -76,7 +75,7 @@ public void testNoClause() {
     }
 
     @Test(expected = SemanticException.class)
-    public void testCompactionClause()  {
+    public void testCompactionClause() {
         new MockUp<RunMode>() {
             @Mock
             public RunMode getCurrentRunMode() {
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeSetVariableTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeSetVariableTest.java
index 6a33cf94302982..cb60ce2971990c 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeSetVariableTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeSetVariableTest.java
@@ -18,6 +18,7 @@
 import com.starrocks.analysis.Subquery;
 import com.starrocks.catalog.ResourceGroupMgr;
 import com.starrocks.qe.ConnectContext;
+import com.starrocks.qe.SessionVariable;
 import com.starrocks.qe.SetExecutor;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.sql.ast.SetPassVar;
@@ -27,6 +28,7 @@
 import com.starrocks.thrift.TWorkGroup;
 import com.starrocks.utframe.StarRocksAssert;
 import com.starrocks.utframe.UtFrameUtils;
+import com.uber.m3.util.ImmutableMap;
 import mockit.Expectations;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -35,9 +37,12 @@
 import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeFail;
 import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeSetUserVariableFail;
 import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeSuccess;
+import static com.starrocks.sql.analyzer.AnalyzeTestUtil.connectContext;
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class AnalyzeSetVariableTest {
     private static StarRocksAssert starRocksAssert;
+
     @BeforeClass
     public static void beforeClass() throws Exception {
         UtFrameUtils.createMinStarRocksCluster();
@@ -212,6 +217,7 @@ public void testSetResourceGroupName() {
                 mgr.chooseResourceGroupByName(rg1Name);
                 result = rg1;
             }
+
             {
                 mgr.chooseResourceGroupByName(anyString);
                 result = null;
@@ -241,6 +247,7 @@ public void testSetResourceGroupID() {
                 mgr.chooseResourceGroupByID(rg1ID);
                 result = rg1;
             }
+
             {
                 mgr.chooseResourceGroupByID(anyLong);
                 result = null;
@@ -310,4 +317,41 @@ public void testComputationFragmentSchedulingPolicy() {
         sql = "SET computation_fragment_scheduling_policy = compute_nodes";
         analyzeFail(sql);
     }
+
+    @Test
+    public void testSetAnnParams() {
+        SessionVariable sv = connectContext.getSessionVariable();
+        String sql;
+
+        sql = "set ann_params='invalid-format'";
+        analyzeFail(sql,
+                "Unsupported ann_params: invalid-format, " +
+                        "It should be a Dict JSON string, each key and value of which is string");
+
+        sql = "set ann_params='{\"Efsearch\": [1,2,3]}'";
+        analyzeFail(sql,
+                "Unsupported ann_params: {\"Efsearch\": [1,2,3]}, " +
+                        "It should be a Dict JSON string, each key and value of which is string");
+
+        sql = "set ann_params='{\"invalid-key\":\"abc\"}'";
+        analyzeFail(sql, "Unknown index param: `INVALID-KEY");
+
+        sql = "set ann_params='{\"Efsearch\": 0}'";
+        analyzeFail(sql, "Value of `EFSEARCH` must be >= 1");
+
+        sql = "set ann_params='{}'";
+        analyzeSuccess(sql);
+        sv.setAnnParams("{}");
+        assertThat(connectContext.getSessionVariable().getAnnParams()).isEmpty();
+
+        sql = "set ann_params=''";
+        analyzeSuccess(sql);
+        sv.setAnnParams("");
+        assertThat(connectContext.getSessionVariable().getAnnParams()).isEmpty();
+
+        sql = "set ann_params='{\"Efsearch\": 1}'";
+        analyzeSuccess(sql);
+        sv.setAnnParams("{\"Efsearch\": 1}");
+        assertThat(connectContext.getSessionVariable().getAnnParams()).containsExactlyEntriesOf(ImmutableMap.of("Efsearch", "1"));
+    }
 }
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeVectorIndexDMLTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeVectorIndexDMLTest.java
new file mode 100644
index 00000000000000..42fe06aaab5e59
--- /dev/null
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeVectorIndexDMLTest.java
@@ -0,0 +1,625 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.analyzer;
+
+import com.starrocks.common.Config;
+import com.starrocks.qe.ConnectContext;
+import com.starrocks.qe.QueryState;
+import com.starrocks.qe.StmtExecutor;
+import com.starrocks.sql.ast.StatementBase;
+import com.starrocks.sql.parser.SqlParser;
+import com.starrocks.utframe.UtFrameUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.List;
+
+import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeFail;
+import static com.starrocks.sql.analyzer.AnalyzeTestUtil.analyzeSuccess;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class AnalyzeVectorIndexDMLTest {
+    private static ConnectContext connectContext;
+
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        UtFrameUtils.createMinStarRocksCluster();
+        AnalyzeTestUtil.init();
+        UtFrameUtils.addMockBackend(10002);
+        UtFrameUtils.addMockBackend(10003);
+        connectContext = AnalyzeTestUtil.getConnectContext();
+
+        Config.enable_experimental_vector = true;
+    }
+
+    @Test
+    public void testValidateParamsForCreateTable() {
+        String sql;
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeSuccess(sql);
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '2', \n" +
+                "        'efconstruction' = '1'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeSuccess(sql);
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'invalid-index-type', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '16', \n" +
+                "        'efconstruction' = '40'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `index_type` must in (IVFPQ,HNSW)");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='invalid-dim', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '16', \n" +
+                "        'efconstruction' = '40'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `DIM` must be a integer");
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='0', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '16', \n" +
+                "        'efconstruction' = '40'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `DIM` must be >= 1");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'invalid-metric-type', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '16', \n" +
+                "        'efconstruction' = '40'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `METRIC_TYPE` must be in [l2_distance, cosine_similarity]");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'invalid-is-vector-normed', \n" +
+                "        'M' = '16', \n" +
+                "        'efconstruction' = '40'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `IS_VECTOR_NORMED` must be `true` or `false`");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = 'invalid-M', \n" +
+                "        'efconstruction' = '40'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `M` must be a integer");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '1', \n" +
+                "        'efconstruction' = '40'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `M` must be >= 2");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '1', \n" +
+                "        'efconstruction' = 'invalid-efconstruction'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `EFCONSTRUCTION` must be a integer");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '1', \n" +
+                "        'efconstruction' = '0'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `EFCONSTRUCTION` must be >= 1");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '2', \n" +
+                "        'efconstruction' = '1',\n" +
+                "        'Nbits' = '8'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Index params [NBITS] should not define with HNSW");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = 'invalid-Nbits',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `NBITS` must be a integer");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '2',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `NBITS` must be 8");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = 'invalid-Nlist', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `NLIST` must be a integer");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '0',\n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Value of `NLIST` must be >= 1");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16', 'invalid-key'='10', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "Unknown index param: `INVALID-KEY`");
+    }
+
+    @Test
+    public void testValidateParamsForAlterTable() throws Exception {
+        AnalyzeTestUtil.getStarRocksAssert().withTable("CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');");
+        String sql;
+
+        try {
+            sql = "ALTER TABLE vector_t1 ADD INDEX index_vector1 (v1) USING VECTOR (\n" +
+                    "        'index_type' = 'IVFPQ', \n" +
+                    "        'dim'='4', \n" +
+                    "        'metric_type' = 'l2_distance', \n" +
+                    "        'is_vector_normed' = 'false', \n" +
+                    "        'Nbits' = '8',\n" +
+                    "        'Nlist' = '16', \n" +
+                    "        'M_IVFPQ' = '2'\n" +
+                    "    )\n";
+            analyzeSuccess(sql);
+
+            sql = "ALTER TABLE vector_t1 ADD INDEX index_vector1 (v1) USING VECTOR (\n" +
+                    "        'index_type' = 'HNSW', \n" +
+                    "        'dim'='5', \n" +
+                    "        'metric_type' = 'l2_distance', \n" +
+                    "        'is_vector_normed' = 'false', \n" +
+                    "        'M' = '2', \n" +
+                    "        'efconstruction' = '1'\n" +
+                    "    )\n";
+            analyzeSuccess(sql);
+
+            sql = "ALTER TABLE vector_t1 ADD INDEX index_vector1 (v1) USING VECTOR (\n" +
+                    "        'index_type' = 'aIVFPQ', \n" +
+                    "        'dim'='5', \n" +
+                    "        'metric_type' = 'l2_distance', \n" +
+                    "        'is_vector_normed' = 'false', \n" +
+                    "        'Nbits' = '8',\n" +
+                    "        'Nlist' = '16' \n" +
+                    "    )\n";
+            analyzeSuccess(sql);
+
+            StatementBase statement = SqlParser.parseSingleStatement(sql, connectContext.getSessionVariable().getSqlMode());
+            StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement);
+            stmtExecutor.execute();
+            assertThat(connectContext.getState().getErrType()).isEqualTo(QueryState.ErrType.INTERNAL_ERR);
+            assertThat(connectContext.getState().getErrorMessage()).contains("Value of `index_type` must in (IVFPQ,HNSW)");
+
+        } finally {
+            AnalyzeTestUtil.getStarRocksAssert().dropTables(List.of("vector_t1"));
+        }
+    }
+
+    @Test
+    public void testOnlyOneVectorIndexForCreateTable() {
+        String sql;
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL,\n" +
+                "    v2 ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_v1 (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    ),\n" +
+                "    INDEX index_v2 (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "At most one vector index is allowed for a table, but 2 were found: [index_v1, index_v2]");
+    }
+
+    @Test
+    public void testOnlyOneVectorIndexForAlterTableSuccess() throws Exception {
+        AnalyzeTestUtil.getStarRocksAssert().withTable("CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL,\n" +
+                "    v2 ARRAY<FLOAT> NOT NULL\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');");
+        String sql;
+
+        try {
+            sql = "ALTER TABLE vector_t1 ADD INDEX index_v1 (v1) USING VECTOR (\n" +
+                    "        'index_type' = 'IVFPQ', \n" +
+                    "        'dim'='4', \n" +
+                    "        'metric_type' = 'l2_distance', \n" +
+                    "        'is_vector_normed' = 'false', \n" +
+                    "        'Nbits' = '8',\n" +
+                    "        'Nlist' = '16', \n" +
+                    "        'M_IVFPQ' = '2'\n" +
+                    "    )\n";
+            StatementBase statement = SqlParser.parseSingleStatement(sql, connectContext.getSessionVariable().getSqlMode());
+            StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement);
+            stmtExecutor.execute();
+            assertThat(connectContext.getState().isError()).isFalse();
+        } finally {
+            AnalyzeTestUtil.getStarRocksAssert().dropTables(List.of("vector_t1"));
+        }
+    }
+
+    @Test
+    public void testOnlyOneVectorIndexForAlterTableFail() throws Exception {
+        AnalyzeTestUtil.getStarRocksAssert().withTable("CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL,\n" +
+                "    v2 ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_v1 (v1) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16',\n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');");
+        String sql;
+
+        try {
+            {
+                sql = "ALTER TABLE vector_t1 ADD INDEX index_v2 (v2) USING VECTOR (\n" +
+                        "        'index_type' = 'IVFPQ', \n" +
+                        "        'dim'='4', \n" +
+                        "        'metric_type' = 'l2_distance', \n" +
+                        "        'is_vector_normed' = 'false', \n" +
+                        "        'Nbits' = '8',\n" +
+                        "        'Nlist' = '16', \n" +
+                        "        'M_IVFPQ' = '2'\n" +
+                        "    )\n";
+                StatementBase statement = SqlParser.parseSingleStatement(sql, connectContext.getSessionVariable().getSqlMode());
+                StmtExecutor stmtExecutor = new StmtExecutor(connectContext, statement);
+                stmtExecutor.execute();
+                assertThat(connectContext.getState().getErrorMessage()).contains(
+                        "At most one vector index is allowed for a table, but there is already a vector index [index_v1]");
+            }
+        } finally {
+            AnalyzeTestUtil.getStarRocksAssert().dropTables(List.of("vector_t1"));
+        }
+    }
+
+    @Test
+    public void testCreateOnNullableColumn() {
+        String sql;
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1";
+        analyzeFail(sql, "The vector index can only build on non-nullable column");
+    }
+
+    @Test
+    public void testIVFPQ() {
+        String sql;
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL,\n" +
+                "    v2 ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_v1 (v1) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='4', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16'" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');";
+        analyzeFail(sql, "`M_IVFPQ` is required for IVFPQ index");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL,\n" +
+                "    v2 ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_v1 (v1) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='10', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '3' \n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');";
+        analyzeFail(sql, "`DIM` should be a multiple of `M_IVFPQ` for IVFPQ index");
+
+        sql = "CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL,\n" +
+                "    v2 ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_v1 (v1) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='10', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16', \n" +
+                "        'M_IVFPQ' = '2' \n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');";
+        analyzeSuccess(sql);
+    }
+
+    @Test
+    public void testShowIndex() throws Exception {
+        String sql;
+        String show;
+
+        AnalyzeTestUtil.getStarRocksAssert().withTable("CREATE TABLE vector_t1 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    v1 ARRAY<FLOAT> NOT NULL,\n" +
+                "    v2 ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_v1 (v1) USING VECTOR (\n" +
+                "        'index_type' = 'IVFPQ', \n" +
+                "        'dim'='10', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'Nbits' = '8',\n" +
+                "        'Nlist' = '16',\n" +
+                "        'M_IVFPQ' = '2'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');");
+        show = AnalyzeTestUtil.getStarRocksAssert().showCreateTable("show create table vector_t1");
+        assertThat(show).contains(
+                "INDEX index_v1 (`v1`) USING VECTOR(\"dim\" = \"10\", \"index_type\" = \"ivfpq\", " +
+                        "\"is_vector_normed\" = \"false\", \"m_ivfpq\" = \"2\", \"metric_type\" = \"l2_distance\", " +
+                        "\"nbits\" = \"8\", \"nlist\" = \"16\")");
+
+        sql = "CREATE TABLE vector_t2 (\n" +
+                "    id bigint(20) NOT NULL,\n" +
+                "    vector ARRAY<FLOAT> NOT NULL,\n" +
+                "    INDEX index_vector (vector) USING VECTOR (\n" +
+                "        'index_type' = 'HNSW', \n" +
+                "        'dim'='5', \n" +
+                "        'metric_type' = 'l2_distance', \n" +
+                "        'is_vector_normed' = 'false', \n" +
+                "        'M' = '2', \n" +
+                "        'efconstruction' = '1'\n" +
+                "    )\n" +
+                ") ENGINE=OLAP\n" +
+                "DUPLICATE KEY(id)\n" +
+                "DISTRIBUTED BY HASH(id) BUCKETS 1\n" +
+                "PROPERTIES ('replication_num'='1');";
+        AnalyzeTestUtil.getStarRocksAssert().withTable(sql);
+        show = AnalyzeTestUtil.getStarRocksAssert().showCreateTable("show create table vector_t2");
+        assertThat(show).contains(
+                "INDEX index_vector (`vector`) USING VECTOR(\"dim\" = \"5\", \"efconstruction\" = \"1\", " +
+                        "\"index_type\" = \"hnsw\", \"is_vector_normed\" = \"false\", \"m\" = \"2\", " +
+                        "\"metric_type\" = \"l2_distance\")");
+
+        AnalyzeTestUtil.getStarRocksAssert().dropTables(List.of("vector_t1", "vector_t2"));
+    }
+}
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 2c6372d654e4a0..499d19d602d588 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -570,6 +570,7 @@ struct TVectorSearchOptions {
   8: optional bool use_ivfpq;
   9: optional double pq_refine_factor;
   10: optional double k_factor;
+  11: optional i32 vector_slot_id;
 }
 
 enum SampleMethod {
diff --git a/test/sql/test_vector_index/R/test_vector_index b/test/sql/test_vector_index/R/test_vector_index
index 09c176a587166b..1b8b6b8e7cc777 100644
--- a/test/sql/test_vector_index/R/test_vector_index
+++ b/test/sql/test_vector_index/R/test_vector_index
@@ -1,4 +1,4 @@
--- name: test_create_vector_index
+-- name: test_create_vector_index @sequential
 ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
 -- result:
 -- !result
@@ -18,7 +18,14 @@ PROPERTIES (
 );
 -- result:
 -- !result
-CREATE INDEX index_vector2 ON t_test_vector_table (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="5", "nlist" = "256", "nbits"="10");
+DROP INDEX index_vector1 ON t_test_vector_table;
+-- result:
+-- !result
+function: wait_alter_table_finish()
+-- result:
+None
+-- !result
+CREATE INDEX index_vector2 ON t_test_vector_table (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="4", "nlist" = "256", "nbits"="8", "M_IVFPQ"="2");
 -- result:
 -- !result
 function: wait_alter_table_finish()
@@ -32,7 +39,7 @@ function: wait_alter_table_finish()
 -- result:
 None
 -- !result
-ALTER TABLE t_test_vector_table add index index_vector2 (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="5", "nlist" = "256", "nbits"="10");
+ALTER TABLE t_test_vector_table add index index_vector2 (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="4", "nlist" = "256", "nbits"="8", "M_IVFPQ"="2");
 -- result:
 -- !result
 function: wait_alter_table_finish()
@@ -50,8 +57,14 @@ None
 DROP TABLE t_test_vector_table;
 -- result:
 -- !result
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+-- result:
+-- !result
 
--- name: test_vector_index
+-- name: test_vector_index @sequential
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+-- result:
+-- !result
 CREATE TABLE `t_test_vector_table` (
   `id` bigint(20) NOT NULL COMMENT "",
   `vector1` ARRAY<FLOAT> NOT NULL COMMENT "",
@@ -88,4 +101,8 @@ select * from (select id, approx_l2_distance([1,1,1,1,1], vector1) score from t_
 
 DROP TABLE t_test_vector_table;
 -- result:
--- !result
\ No newline at end of file
+-- !result
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+-- result:
+-- !result
diff --git a/test/sql/test_vector_index/R/test_vector_index_hnsw b/test/sql/test_vector_index/R/test_vector_index_hnsw
new file mode 100644
index 00000000000000..8ad6d634257b16
--- /dev/null
+++ b/test/sql/test_vector_index/R/test_vector_index_hnsw
@@ -0,0 +1,484 @@
+-- name: test_vector_index_hnsw @sequential
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+-- result:
+-- !result
+CREATE TABLE __row_util_base (
+  k1 bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`k1`)
+DISTRIBUTED BY HASH(`k1`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+insert into __row_util_base select generate_series from TABLE(generate_series(0, 10000 - 1));
+-- result:
+-- !result
+insert into __row_util_base select * from __row_util_base; -- 20000
+insert into __row_util_base select * from __row_util_base; -- 40000
+insert into __row_util_base select * from __row_util_base; -- 80000
+insert into __row_util_base select * from __row_util_base; -- 160000
+insert into __row_util_base select * from __row_util_base; -- 320000
+insert into __row_util_base select * from __row_util_base; -- 640000
+
+CREATE TABLE __row_util (
+  idx bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`idx`)
+DISTRIBUTED BY HASH(`idx`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+insert into __row_util 
+select 
+    row_number() over() as idx
+from __row_util_base;
+-- result:
+-- !result
+CREATE TABLE t2 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NOT NULL,
+    i1 bigint(20) NOT NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "hnsw", 
+        "dim"="5", 
+        "metric_type" = "l2_distance", 
+        "is_vector_normed" = "false", 
+        "M" = "160", 
+        "efconstruction" = "400")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+insert into t2 
+select 
+    idx,
+    array_generate(10000, 10004),
+    array_generate(10000, 10004),
+    idx
+from __row_util
+order by idx 
+limit 20;
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    order by dis limit 21
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	1	0.0
+2	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	2	0.0
+3	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	3	0.0
+4	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	4	0.0
+5	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	5	0.0
+6	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	6	0.0
+7	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	7	0.0
+8	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	8	0.0
+9	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	9	0.0
+10	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	10	0.0
+11	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	11	0.0
+12	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	12	0.0
+13	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	13	0.0
+14	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	14	0.0
+15	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	15	0.0
+16	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	16	0.0
+17	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	17	0.0
+18	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	18	0.0
+19	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	19	0.0
+20	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	20	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+order by dis;
+-- result:
+9	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	9	500100000.0
+16	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	16	500100000.0
+15	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	15	500100000.0
+13	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	13	500100000.0
+20	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	20	500100000.0
+12	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	12	500100000.0
+2	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	2	500100000.0
+7	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	7	500100000.0
+14	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	14	500100000.0
+10	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	10	500100000.0
+19	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	19	500100000.0
+11	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	11	500100000.0
+18	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	18	500100000.0
+8	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	8	500100000.0
+17	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	17	500100000.0
+1	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	1	500100000.0
+3	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	3	500100000.0
+5	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	5	500100000.0
+4	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	4	500100000.0
+6	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	6	500100000.0
+-- !result
+insert into t2 
+select 
+    idx + 20,
+    array_repeat(idx, 5),
+    array_repeat(idx, 5),
+    idx + 20
+from __row_util;
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 order by dis limit 10;
+-- result:
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	0.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	5.0
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	20.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	45.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	80.0
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	125.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	180.0
+28	[8,8,8,8,8]	[8,8,8,8,8]	28	245.0
+29	[9,9,9,9,9]	[9,9,9,9,9]	29	320.0
+30	[10,10,10,10,10]	[10,10,10,10,10]	30	405.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+-- result:
+640020	[640000,640000,640000,640000,640000]	[640000,640000,640000,640000,640000]	640020	20480.0
+640019	[639999,639999,639999,639999,639999]	[639999,639999,639999,639999,639999]	640019	21125.0
+640018	[639998,639998,639998,639998,639998]	[639998,639998,639998,639998,639998]	640018	21780.0
+640017	[639997,639997,639997,639997,639997]	[639997,639997,639997,639997,639997]	640017	22445.0
+640016	[639996,639996,639996,639996,639996]	[639996,639996,639996,639996,639996]	640016	23120.0
+640015	[639995,639995,639995,639995,639995]	[639995,639995,639995,639995,639995]	640015	23805.0
+640014	[639994,639994,639994,639994,639994]	[639994,639994,639994,639994,639994]	640014	24500.0
+640013	[639993,639993,639993,639993,639993]	[639993,639993,639993,639993,639993]	640013	25205.0
+640012	[639992,639992,639992,639992,639992]	[639992,639992,639992,639992,639992]	640012	25920.0
+640011	[639991,639991,639991,639991,639991]	[639991,639991,639991,639991,639991]	640011	26645.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+-- result:
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	10.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	15.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	15.0
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	30.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	30.0
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	55.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	90.0
+28	[8,8,8,8,8]	[8,8,8,8,8]	28	135.0
+29	[9,9,9,9,9]	[9,9,9,9,9]	29	190.0
+30	[10,10,10,10,10]	[10,10,10,10,10]	30	255.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	1	0.0
+2	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	2	0.0
+3	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	3	0.0
+4	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	4	0.0
+5	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	5	0.0
+6	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	6	0.0
+7	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	7	0.0
+8	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	8	0.0
+9	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	9	0.0
+10	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	10	0.0
+11	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	11	0.0
+12	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	12	0.0
+13	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	13	0.0
+14	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	14	0.0
+15	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	15	0.0
+16	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	16	0.0
+17	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	17	0.0
+18	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	18	0.0
+19	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	19	0.0
+20	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	20	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 0
+order by dis limit 10;
+-- result:
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	0.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	5.0
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	20.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	45.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	80.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	10.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	15.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	15.0
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	30.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	30.0
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	55.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	90.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	1	0.0
+2	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	2	0.0
+3	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	3	0.0
+4	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	4	0.0
+5	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	5	0.0
+6	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	6	0.0
+7	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	7	0.0
+8	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	8	0.0
+9	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	9	0.0
+10	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	10	0.0
+11	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	11	0.0
+12	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	12	0.0
+13	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	13	0.0
+14	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	14	0.0
+15	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	15	0.0
+16	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	16	0.0
+17	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	17	0.0
+18	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	18	0.0
+19	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	19	0.0
+20	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	20	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis >= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	125.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	180.0
+28	[8,8,8,8,8]	[8,8,8,8,8]	28	245.0
+29	[9,9,9,9,9]	[9,9,9,9,9]	29	320.0
+30	[10,10,10,10,10]	[10,10,10,10,10]	30	405.0
+31	[11,11,11,11,11]	[11,11,11,11,11]	31	500.0
+32	[12,12,12,12,12]	[12,12,12,12,12]	32	605.0
+33	[13,13,13,13,13]	[13,13,13,13,13]	33	720.0
+34	[14,14,14,14,14]	[14,14,14,14,14]	34	845.0
+35	[15,15,15,15,15]	[15,15,15,15,15]	35	980.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 100 and id >= 0
+order by dis limit 10;
+-- result:
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	0.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	5.0
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	20.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	45.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	80.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+order by dis, id limit 10;
+-- result:
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	0.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	5.0
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	20.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	45.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	80.0
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	125.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	180.0
+28	[8,8,8,8,8]	[8,8,8,8,8]	28	245.0
+29	[9,9,9,9,9]	[9,9,9,9,9]	29	320.0
+30	[10,10,10,10,10]	[10,10,10,10,10]	30	405.0
+-- !result
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 order by dis limit 10;
+-- result:
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	0.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	5.0
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	20.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	45.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	80.0
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	125.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	180.0
+28	[8,8,8,8,8]	[8,8,8,8,8]	28	245.0
+29	[9,9,9,9,9]	[9,9,9,9,9]	29	320.0
+30	[10,10,10,10,10]	[10,10,10,10,10]	30	405.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+-- result:
+640020	[640000,640000,640000,640000,640000]	[640000,640000,640000,640000,640000]	640020	20480.0
+640019	[639999,639999,639999,639999,639999]	[639999,639999,639999,639999,639999]	640019	21125.0
+640018	[639998,639998,639998,639998,639998]	[639998,639998,639998,639998,639998]	640018	21780.0
+640017	[639997,639997,639997,639997,639997]	[639997,639997,639997,639997,639997]	640017	22445.0
+640016	[639996,639996,639996,639996,639996]	[639996,639996,639996,639996,639996]	640016	23120.0
+640015	[639995,639995,639995,639995,639995]	[639995,639995,639995,639995,639995]	640015	23805.0
+640014	[639994,639994,639994,639994,639994]	[639994,639994,639994,639994,639994]	640014	24500.0
+640013	[639993,639993,639993,639993,639993]	[639993,639993,639993,639993,639993]	640013	25205.0
+640012	[639992,639992,639992,639992,639992]	[639992,639992,639992,639992,639992]	640012	25920.0
+640011	[639991,639991,639991,639991,639991]	[639991,639991,639991,639991,639991]	640011	26645.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+-- result:
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	10.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	15.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	15.0
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	30.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	30.0
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	55.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	90.0
+28	[8,8,8,8,8]	[8,8,8,8,8]	28	135.0
+29	[9,9,9,9,9]	[9,9,9,9,9]	29	190.0
+30	[10,10,10,10,10]	[10,10,10,10,10]	30	255.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	1	0.0
+2	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	2	0.0
+3	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	3	0.0
+4	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	4	0.0
+5	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	5	0.0
+6	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	6	0.0
+7	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	7	0.0
+8	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	8	0.0
+9	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	9	0.0
+10	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	10	0.0
+11	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	11	0.0
+12	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	12	0.0
+13	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	13	0.0
+14	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	14	0.0
+15	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	15	0.0
+16	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	16	0.0
+17	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	17	0.0
+18	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	18	0.0
+19	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	19	0.0
+20	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	20	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	0.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	5.0
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	20.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	45.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	80.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+23	[3,3,3,3,3]	[3,3,3,3,3]	23	10.0
+22	[2,2,2,2,2]	[2,2,2,2,2]	22	15.0
+24	[4,4,4,4,4]	[4,4,4,4,4]	24	15.0
+21	[1,1,1,1,1]	[1,1,1,1,1]	21	30.0
+25	[5,5,5,5,5]	[5,5,5,5,5]	25	30.0
+26	[6,6,6,6,6]	[6,6,6,6,6]	26	55.0
+27	[7,7,7,7,7]	[7,7,7,7,7]	27	90.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	1	0.0
+2	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	2	0.0
+3	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	3	0.0
+4	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	4	0.0
+5	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	5	0.0
+6	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	6	0.0
+7	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	7	0.0
+8	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	8	0.0
+9	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	9	0.0
+10	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	10	0.0
+11	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	11	0.0
+12	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	12	0.0
+13	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	13	0.0
+14	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	14	0.0
+15	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	15	0.0
+16	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	16	0.0
+17	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	17	0.0
+18	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	18	0.0
+19	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	19	0.0
+20	[10000,10001,10002,10003,10004]	[10000,10001,10002,10003,10004]	20	0.0
+-- !result
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+-- result:
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_vector_index/R/test_vector_index_insert b/test/sql/test_vector_index/R/test_vector_index_insert
new file mode 100644
index 00000000000000..31de99e0e4c6b3
--- /dev/null
+++ b/test/sql/test_vector_index/R/test_vector_index_insert
@@ -0,0 +1,170 @@
+-- name: test_vector_index_insert @sequential
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+-- result:
+-- !result
+CREATE TABLE t1 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "hnsw", 
+        "dim"="5", 
+        "metric_type" = "cosine_similarity", 
+        "is_vector_normed" = "true", 
+        "M" = "16", 
+        "efconstruction" = "40")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+INSERT into t1 values 
+    (1, null, null);
+-- result:
+[REGEX].*Insert has filtered data.*
+-- !result
+INSERT into t1 values 
+    (1, [null, null, null, null, null], [1,2,3,4,5]);
+-- result:
+[REGEX].*The input vector is not normalized but `metric_type` is cosine_similarity and `is_vector_normed` is true.*
+-- !result
+INSERT into t1 values 
+    (1, [1,2,3,4], [1,2,3,4]);
+-- result:
+[REGEX].*The dimensions of the vector written are inconsistent, index dim is 5 but data dim is 4.*
+-- !result
+INSERT into t1 values 
+    (1, [], []);
+-- result:
+[REGEX].*The dimensions of the vector written are inconsistent, index dim is 5 but data dim is 0.*
+-- !result
+INSERT INTO t1 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7,8], [4,5,6,7,8]);
+-- result:
+[REGEX].*The input vector is not normalized but `metric_type` is cosine_similarity and `is_vector_normed` is true.*
+-- !result
+INSERT INTO t1 values
+    (1, [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242], 
+        [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242]),
+    (2, [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093],
+        [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093]),
+    (3, [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152],
+        [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152]),
+    (4, [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152],
+        null);
+-- result:
+-- !result
+INSERT INTO t1 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7,8], [4,5,6,7,8]),
+    (3, null, null);
+-- result:
+[REGEX].*The input vector is not normalized but `metric_type` is cosine_similarity and `is_vector_normed` is true.*
+-- !result
+INSERT INTO t1 values
+    (1, [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242], 
+        [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242]),
+    (4, null, null),
+    (2, [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093],
+        [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093]),
+    (3, [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152],
+        [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152]),
+    (3, null, null),
+    (5, null, null),
+    (6, null, null),
+    (7, null, null),
+    (8, null, null),
+    (9, null, null),
+    (10, null, null);
+-- result:
+[REGEX].*Insert has filtered data.*
+-- !result
+select * from t1 order by id;
+-- result:
+1	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]
+2	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]
+3	[0.33686078,0.42107597,0.50529116,null,0.67372155]	[0.33686078,0.42107597,0.50529116,null,0.67372155]
+4	[0.33686078,0.42107597,0.50529116,null,0.67372155]	None
+-- !result
+CREATE TABLE t2 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "hnsw", 
+        "dim"="5", 
+        "metric_type" = "cosine_similarity", 
+        "is_vector_normed" = "false", 
+        "M" = "16", 
+        "efconstruction" = "40")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+INSERT INTO t2 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7,8], [4,5,6,7,8]),
+    (3, [4,5,6,null,8], [4,5,6,null,8]),
+    (4, [null, null, null, null], [null, null, null, null]),
+    (5, [4,5,6,7,8], null);
+-- result:
+[REGEX].*The dimensions of the vector written are inconsistent, index dim is 5 but data dim is 4.*
+-- !result
+INSERT INTO t2 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7], [4,5,6,7,8]),
+    (3, [4,5,6,null,8], [4,5,6,null,8]),
+    (4, [null, null, null, null], [null, null, null, null]),
+    (5, [4,5,6,7,8], null);
+-- result:
+[REGEX].*The dimensions of the vector written are inconsistent, index dim is 5 but data dim is 4.*
+-- !result
+select * from t2 order by id;
+-- result:
+-- !result
+insert into t1 select * from t2;
+-- result:
+-- !result
+insert into t1 select * from t1;
+-- result:
+-- !result
+select * from t1 order by id, v1, v2;
+-- result:
+1	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]
+1	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]
+2	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]
+2	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]
+3	[0.33686078,0.42107597,0.50529116,null,0.67372155]	[0.33686078,0.42107597,0.50529116,null,0.67372155]
+3	[0.33686078,0.42107597,0.50529116,null,0.67372155]	[0.33686078,0.42107597,0.50529116,null,0.67372155]
+4	[0.33686078,0.42107597,0.50529116,null,0.67372155]	None
+4	[0.33686078,0.42107597,0.50529116,null,0.67372155]	None
+-- !result
+insert into t2 select * from t1;
+-- result:
+-- !result
+insert into t2 select id, v2, v1 from t2;
+-- result:
+[REGEX].*Insert has filtered data.*
+-- !result
+select * from t2 order by id, v1, v2;
+-- result:
+1	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]
+1	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]	[0.13483997,0.26967993,0.40451992,0.53935987,0.6741999]
+2	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]
+2	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]	[0.2901905,0.36273813,0.43528575,0.50783336,0.580381]
+3	[0.33686078,0.42107597,0.50529116,null,0.67372155]	[0.33686078,0.42107597,0.50529116,null,0.67372155]
+3	[0.33686078,0.42107597,0.50529116,null,0.67372155]	[0.33686078,0.42107597,0.50529116,null,0.67372155]
+4	[0.33686078,0.42107597,0.50529116,null,0.67372155]	None
+4	[0.33686078,0.42107597,0.50529116,null,0.67372155]	None
+-- !result
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+-- result:
+-- !result
diff --git a/test/sql/test_vector_index/R/test_vector_index_ivfpq b/test/sql/test_vector_index/R/test_vector_index_ivfpq
new file mode 100644
index 00000000000000..999d35a0936c9c
--- /dev/null
+++ b/test/sql/test_vector_index/R/test_vector_index_ivfpq
@@ -0,0 +1,536 @@
+-- name: test_vector_index_ivfpq @sequential
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+-- result:
+-- !result
+CREATE TABLE __row_util_base (
+  k1 bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`k1`)
+DISTRIBUTED BY HASH(`k1`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+insert into __row_util_base select generate_series from TABLE(generate_series(0, 10000 - 1));
+-- result:
+-- !result
+insert into __row_util_base select * from __row_util_base; -- 20000
+insert into __row_util_base select * from __row_util_base; -- 40000
+insert into __row_util_base select * from __row_util_base; -- 80000
+insert into __row_util_base select * from __row_util_base; -- 160000
+insert into __row_util_base select * from __row_util_base; -- 320000
+insert into __row_util_base select * from __row_util_base; -- 640000
+
+CREATE TABLE __row_util (
+  idx bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`idx`)
+DISTRIBUTED BY HASH(`idx`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+insert into __row_util 
+select 
+    row_number() over() as idx
+from __row_util_base;
+-- result:
+-- !result
+CREATE TABLE t1 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NOT NULL,
+    i1 bigint(20) NOT NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "IVFPQ", 
+        "dim"="4", 
+        "metric_type" = "l2_distance", 
+        "is_vector_normed" = "false", 
+        "nbits" = "8", 
+        "nlist" = "40",
+        "M_IVFPQ" = "2")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+-- !result
+CREATE TABLE t1 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NOT NULL,
+    i1 bigint(20) NOT NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "ivfpq", 
+        "dim"="4", 
+        "metric_type" = "l2_distance", 
+        "is_vector_normed" = "false", 
+        "nbits" = "8", 
+        "nlist" = "16",
+        "M_IVFPQ" = "2")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+-- result:
+E: (1050, "Getting analyzing error. Detail message: Table 't1' already exists.")
+-- !result
+insert into t1 
+select 
+    idx,
+    array_generate(10000, 10003),
+    array_generate(10000, 10003),
+    idx
+from __row_util
+order by idx 
+limit 20;
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    order by dis limit 21
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003]	[10000,10001,10002,10003]	1	0.0
+2	[10000,10001,10002,10003]	[10000,10001,10002,10003]	2	0.0
+3	[10000,10001,10002,10003]	[10000,10001,10002,10003]	3	0.0
+4	[10000,10001,10002,10003]	[10000,10001,10002,10003]	4	0.0
+5	[10000,10001,10002,10003]	[10000,10001,10002,10003]	5	0.0
+6	[10000,10001,10002,10003]	[10000,10001,10002,10003]	6	0.0
+7	[10000,10001,10002,10003]	[10000,10001,10002,10003]	7	0.0
+8	[10000,10001,10002,10003]	[10000,10001,10002,10003]	8	0.0
+9	[10000,10001,10002,10003]	[10000,10001,10002,10003]	9	0.0
+10	[10000,10001,10002,10003]	[10000,10001,10002,10003]	10	0.0
+11	[10000,10001,10002,10003]	[10000,10001,10002,10003]	11	0.0
+12	[10000,10001,10002,10003]	[10000,10001,10002,10003]	12	0.0
+13	[10000,10001,10002,10003]	[10000,10001,10002,10003]	13	0.0
+14	[10000,10001,10002,10003]	[10000,10001,10002,10003]	14	0.0
+15	[10000,10001,10002,10003]	[10000,10001,10002,10003]	15	0.0
+16	[10000,10001,10002,10003]	[10000,10001,10002,10003]	16	0.0
+17	[10000,10001,10002,10003]	[10000,10001,10002,10003]	17	0.0
+18	[10000,10001,10002,10003]	[10000,10001,10002,10003]	18	0.0
+19	[10000,10001,10002,10003]	[10000,10001,10002,10003]	19	0.0
+20	[10000,10001,10002,10003]	[10000,10001,10002,10003]	20	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+order by dis;
+-- result:
+8	[10000,10001,10002,10003]	[10000,10001,10002,10003]	8	400040000.0
+17	[10000,10001,10002,10003]	[10000,10001,10002,10003]	17	400040000.0
+1	[10000,10001,10002,10003]	[10000,10001,10002,10003]	1	400040000.0
+3	[10000,10001,10002,10003]	[10000,10001,10002,10003]	3	400040000.0
+5	[10000,10001,10002,10003]	[10000,10001,10002,10003]	5	400040000.0
+4	[10000,10001,10002,10003]	[10000,10001,10002,10003]	4	400040000.0
+6	[10000,10001,10002,10003]	[10000,10001,10002,10003]	6	400040000.0
+9	[10000,10001,10002,10003]	[10000,10001,10002,10003]	9	400040000.0
+16	[10000,10001,10002,10003]	[10000,10001,10002,10003]	16	400040000.0
+15	[10000,10001,10002,10003]	[10000,10001,10002,10003]	15	400040000.0
+13	[10000,10001,10002,10003]	[10000,10001,10002,10003]	13	400040000.0
+20	[10000,10001,10002,10003]	[10000,10001,10002,10003]	20	400040000.0
+12	[10000,10001,10002,10003]	[10000,10001,10002,10003]	12	400040000.0
+2	[10000,10001,10002,10003]	[10000,10001,10002,10003]	2	400040000.0
+7	[10000,10001,10002,10003]	[10000,10001,10002,10003]	7	400040000.0
+14	[10000,10001,10002,10003]	[10000,10001,10002,10003]	14	400040000.0
+10	[10000,10001,10002,10003]	[10000,10001,10002,10003]	10	400040000.0
+19	[10000,10001,10002,10003]	[10000,10001,10002,10003]	19	400040000.0
+11	[10000,10001,10002,10003]	[10000,10001,10002,10003]	11	400040000.0
+18	[10000,10001,10002,10003]	[10000,10001,10002,10003]	18	400040000.0
+-- !result
+insert into t1 
+select 
+    idx + 20,
+    array_repeat(idx, 4),
+    array_repeat(idx, 4),
+    idx + 20
+from __row_util;
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select id, v1, dis from w1 order by dis limit 10;
+-- result:
+21	[1,1,1,1]	0.0
+22	[2,2,2,2]	4.0
+23	[3,3,3,3]	16.0
+24	[4,4,4,4]	36.0
+25	[5,5,5,5]	64.0
+26	[6,6,6,6]	100.0
+27	[7,7,7,7]	144.0
+28	[8,8,8,8]	196.0
+29	[9,9,9,9]	256.0
+30	[10,10,10,10]	324.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+-- result:
+640020	[640000,640000,640000,640000]	[640000,640000,640000,640000]	640020	16384.0
+640019	[639999,639999,639999,639999]	[639999,639999,639999,639999]	640019	16900.0
+640018	[639998,639998,639998,639998]	[639998,639998,639998,639998]	640018	17424.0
+640017	[639997,639997,639997,639997]	[639997,639997,639997,639997]	640017	17956.0
+640016	[639996,639996,639996,639996]	[639996,639996,639996,639996]	640016	18496.0
+640015	[639995,639995,639995,639995]	[639995,639995,639995,639995]	640015	19044.0
+640014	[639994,639994,639994,639994]	[639994,639994,639994,639994]	640014	19600.0
+640013	[639993,639993,639993,639993]	[639993,639993,639993,639993]	640013	20164.0
+640012	[639992,639992,639992,639992]	[639992,639992,639992,639992]	640012	20736.0
+640011	[639991,639991,639991,639991]	[639991,639991,639991,639991]	640011	21316.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+-- result:
+22	[2,2,2,2]	[2,2,2,2]	22	6.0
+23	[3,3,3,3]	[3,3,3,3]	23	6.0
+24	[4,4,4,4]	[4,4,4,4]	24	14.0
+21	[1,1,1,1]	[1,1,1,1]	21	14.0
+25	[5,5,5,5]	[5,5,5,5]	25	30.0
+26	[6,6,6,6]	[6,6,6,6]	26	54.0
+27	[7,7,7,7]	[7,7,7,7]	27	86.0
+28	[8,8,8,8]	[8,8,8,8]	28	126.0
+29	[9,9,9,9]	[9,9,9,9]	29	174.0
+30	[10,10,10,10]	[10,10,10,10]	30	230.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003]	[10000,10001,10002,10003]	1	0.0
+2	[10000,10001,10002,10003]	[10000,10001,10002,10003]	2	0.0
+3	[10000,10001,10002,10003]	[10000,10001,10002,10003]	3	0.0
+4	[10000,10001,10002,10003]	[10000,10001,10002,10003]	4	0.0
+5	[10000,10001,10002,10003]	[10000,10001,10002,10003]	5	0.0
+6	[10000,10001,10002,10003]	[10000,10001,10002,10003]	6	0.0
+7	[10000,10001,10002,10003]	[10000,10001,10002,10003]	7	0.0
+8	[10000,10001,10002,10003]	[10000,10001,10002,10003]	8	0.0
+9	[10000,10001,10002,10003]	[10000,10001,10002,10003]	9	0.0
+10	[10000,10001,10002,10003]	[10000,10001,10002,10003]	10	0.0
+11	[10000,10001,10002,10003]	[10000,10001,10002,10003]	11	0.0
+12	[10000,10001,10002,10003]	[10000,10001,10002,10003]	12	0.0
+13	[10000,10001,10002,10003]	[10000,10001,10002,10003]	13	0.0
+14	[10000,10001,10002,10003]	[10000,10001,10002,10003]	14	0.0
+15	[10000,10001,10002,10003]	[10000,10001,10002,10003]	15	0.0
+16	[10000,10001,10002,10003]	[10000,10001,10002,10003]	16	0.0
+17	[10000,10001,10002,10003]	[10000,10001,10002,10003]	17	0.0
+18	[10000,10001,10002,10003]	[10000,10001,10002,10003]	18	0.0
+19	[10000,10001,10002,10003]	[10000,10001,10002,10003]	19	0.0
+20	[10000,10001,10002,10003]	[10000,10001,10002,10003]	20	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select v1, dis from w1 
+where dis <= 0
+order by dis limit 10;
+-- result:
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select v1, dis from w1 
+where dis <= 100
+order by dis limit 10;
+-- result:
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+[10000,10001,10002,10003]	400040000.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select v1, dis from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+[10000,10001,10002,10003]	1587915000000.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select v1, dis from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+[10000,10001,10002,10003]	399920000.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select v1, dis from w2 order by dis, id;
+-- result:
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+[10000,10001,10002,10003]	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+where dis >= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+26	[6,6,6,6]	[6,6,6,6]	26	100.0
+27	[7,7,7,7]	[7,7,7,7]	27	144.0
+28	[8,8,8,8]	[8,8,8,8]	28	196.0
+29	[9,9,9,9]	[9,9,9,9]	29	256.0
+30	[10,10,10,10]	[10,10,10,10]	30	324.0
+31	[11,11,11,11]	[11,11,11,11]	31	400.0
+32	[12,12,12,12]	[12,12,12,12]	32	484.0
+33	[13,13,13,13]	[13,13,13,13]	33	576.0
+34	[14,14,14,14]	[14,14,14,14]	34	676.0
+35	[15,15,15,15]	[15,15,15,15]	35	784.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+where dis <= 100 and id >= 0
+order by dis limit 10;
+-- result:
+21	[1,1,1,1]	[1,1,1,1]	21	0.0
+22	[2,2,2,2]	[2,2,2,2]	22	4.0
+23	[3,3,3,3]	[3,3,3,3]	23	16.0
+24	[4,4,4,4]	[4,4,4,4]	24	36.0
+25	[5,5,5,5]	[5,5,5,5]	25	64.0
+26	[6,6,6,6]	[6,6,6,6]	26	100.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+order by dis, id limit 10;
+-- result:
+21	[1,1,1,1]	[1,1,1,1]	21	0.0
+22	[2,2,2,2]	[2,2,2,2]	22	4.0
+23	[3,3,3,3]	[3,3,3,3]	23	16.0
+24	[4,4,4,4]	[4,4,4,4]	24	36.0
+25	[5,5,5,5]	[5,5,5,5]	25	64.0
+26	[6,6,6,6]	[6,6,6,6]	26	100.0
+27	[7,7,7,7]	[7,7,7,7]	27	144.0
+28	[8,8,8,8]	[8,8,8,8]	28	196.0
+29	[9,9,9,9]	[9,9,9,9]	29	256.0
+30	[10,10,10,10]	[10,10,10,10]	30	324.0
+-- !result
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 order by dis limit 10;
+-- result:
+21	[1,1,1,1]	[1,1,1,1]	21	0.0
+22	[2,2,2,2]	[2,2,2,2]	22	4.0
+23	[3,3,3,3]	[3,3,3,3]	23	16.0
+24	[4,4,4,4]	[4,4,4,4]	24	36.0
+25	[5,5,5,5]	[5,5,5,5]	25	64.0
+26	[6,6,6,6]	[6,6,6,6]	26	100.0
+27	[7,7,7,7]	[7,7,7,7]	27	144.0
+28	[8,8,8,8]	[8,8,8,8]	28	196.0
+29	[9,9,9,9]	[9,9,9,9]	29	256.0
+30	[10,10,10,10]	[10,10,10,10]	30	324.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+-- result:
+640020	[640000,640000,640000,640000]	[640000,640000,640000,640000]	640020	16384.0
+640019	[639999,639999,639999,639999]	[639999,639999,639999,639999]	640019	16900.0
+640018	[639998,639998,639998,639998]	[639998,639998,639998,639998]	640018	17424.0
+640017	[639997,639997,639997,639997]	[639997,639997,639997,639997]	640017	17956.0
+640016	[639996,639996,639996,639996]	[639996,639996,639996,639996]	640016	18496.0
+640015	[639995,639995,639995,639995]	[639995,639995,639995,639995]	640015	19044.0
+640014	[639994,639994,639994,639994]	[639994,639994,639994,639994]	640014	19600.0
+640013	[639993,639993,639993,639993]	[639993,639993,639993,639993]	640013	20164.0
+640012	[639992,639992,639992,639992]	[639992,639992,639992,639992]	640012	20736.0
+640011	[639991,639991,639991,639991]	[639991,639991,639991,639991]	640011	21316.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+-- result:
+22	[2,2,2,2]	[2,2,2,2]	22	6.0
+23	[3,3,3,3]	[3,3,3,3]	23	6.0
+24	[4,4,4,4]	[4,4,4,4]	24	14.0
+21	[1,1,1,1]	[1,1,1,1]	21	14.0
+25	[5,5,5,5]	[5,5,5,5]	25	30.0
+26	[6,6,6,6]	[6,6,6,6]	26	54.0
+27	[7,7,7,7]	[7,7,7,7]	27	86.0
+28	[8,8,8,8]	[8,8,8,8]	28	126.0
+29	[9,9,9,9]	[9,9,9,9]	29	174.0
+30	[10,10,10,10]	[10,10,10,10]	30	230.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003]	[10000,10001,10002,10003]	1	0.0
+2	[10000,10001,10002,10003]	[10000,10001,10002,10003]	2	0.0
+3	[10000,10001,10002,10003]	[10000,10001,10002,10003]	3	0.0
+4	[10000,10001,10002,10003]	[10000,10001,10002,10003]	4	0.0
+5	[10000,10001,10002,10003]	[10000,10001,10002,10003]	5	0.0
+6	[10000,10001,10002,10003]	[10000,10001,10002,10003]	6	0.0
+7	[10000,10001,10002,10003]	[10000,10001,10002,10003]	7	0.0
+8	[10000,10001,10002,10003]	[10000,10001,10002,10003]	8	0.0
+9	[10000,10001,10002,10003]	[10000,10001,10002,10003]	9	0.0
+10	[10000,10001,10002,10003]	[10000,10001,10002,10003]	10	0.0
+11	[10000,10001,10002,10003]	[10000,10001,10002,10003]	11	0.0
+12	[10000,10001,10002,10003]	[10000,10001,10002,10003]	12	0.0
+13	[10000,10001,10002,10003]	[10000,10001,10002,10003]	13	0.0
+14	[10000,10001,10002,10003]	[10000,10001,10002,10003]	14	0.0
+15	[10000,10001,10002,10003]	[10000,10001,10002,10003]	15	0.0
+16	[10000,10001,10002,10003]	[10000,10001,10002,10003]	16	0.0
+17	[10000,10001,10002,10003]	[10000,10001,10002,10003]	17	0.0
+18	[10000,10001,10002,10003]	[10000,10001,10002,10003]	18	0.0
+19	[10000,10001,10002,10003]	[10000,10001,10002,10003]	19	0.0
+20	[10000,10001,10002,10003]	[10000,10001,10002,10003]	20	0.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+21	[1,1,1,1]	[1,1,1,1]	21	0.0
+22	[2,2,2,2]	[2,2,2,2]	22	4.0
+23	[3,3,3,3]	[3,3,3,3]	23	16.0
+24	[4,4,4,4]	[4,4,4,4]	24	36.0
+25	[5,5,5,5]	[5,5,5,5]	25	64.0
+26	[6,6,6,6]	[6,6,6,6]	26	100.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+-- result:
+22	[2,2,2,2]	[2,2,2,2]	22	6.0
+23	[3,3,3,3]	[3,3,3,3]	23	6.0
+24	[4,4,4,4]	[4,4,4,4]	24	14.0
+21	[1,1,1,1]	[1,1,1,1]	21	14.0
+25	[5,5,5,5]	[5,5,5,5]	25	30.0
+26	[6,6,6,6]	[6,6,6,6]	26	54.0
+27	[7,7,7,7]	[7,7,7,7]	27	86.0
+-- !result
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select * from w2 order by dis, id;
+-- result:
+1	[10000,10001,10002,10003]	[10000,10001,10002,10003]	1	0.0
+2	[10000,10001,10002,10003]	[10000,10001,10002,10003]	2	0.0
+3	[10000,10001,10002,10003]	[10000,10001,10002,10003]	3	0.0
+4	[10000,10001,10002,10003]	[10000,10001,10002,10003]	4	0.0
+5	[10000,10001,10002,10003]	[10000,10001,10002,10003]	5	0.0
+6	[10000,10001,10002,10003]	[10000,10001,10002,10003]	6	0.0
+7	[10000,10001,10002,10003]	[10000,10001,10002,10003]	7	0.0
+8	[10000,10001,10002,10003]	[10000,10001,10002,10003]	8	0.0
+9	[10000,10001,10002,10003]	[10000,10001,10002,10003]	9	0.0
+10	[10000,10001,10002,10003]	[10000,10001,10002,10003]	10	0.0
+11	[10000,10001,10002,10003]	[10000,10001,10002,10003]	11	0.0
+12	[10000,10001,10002,10003]	[10000,10001,10002,10003]	12	0.0
+13	[10000,10001,10002,10003]	[10000,10001,10002,10003]	13	0.0
+14	[10000,10001,10002,10003]	[10000,10001,10002,10003]	14	0.0
+15	[10000,10001,10002,10003]	[10000,10001,10002,10003]	15	0.0
+16	[10000,10001,10002,10003]	[10000,10001,10002,10003]	16	0.0
+17	[10000,10001,10002,10003]	[10000,10001,10002,10003]	17	0.0
+18	[10000,10001,10002,10003]	[10000,10001,10002,10003]	18	0.0
+19	[10000,10001,10002,10003]	[10000,10001,10002,10003]	19	0.0
+20	[10000,10001,10002,10003]	[10000,10001,10002,10003]	20	0.0
+-- !result
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+-- result:
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_vector_index/T/test_vector_index b/test/sql/test_vector_index/T/test_vector_index
index f50e60de399ad3..e4dae64b235232 100644
--- a/test/sql/test_vector_index/T/test_vector_index
+++ b/test/sql/test_vector_index/T/test_vector_index
@@ -1,5 +1,6 @@
--- name: test_create_vector_index
+-- name: test_create_vector_index @sequential
 ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+
 CREATE TABLE `t_test_vector_table` (
   `id` bigint(20) NOT NULL COMMENT "",
   `vector1` ARRAY<FLOAT> NOT NULL COMMENT "",
@@ -14,15 +15,23 @@ PROPERTIES (
 "replicated_storage" = "false",
 "compression" = "LZ4"
 );
-CREATE INDEX index_vector2 ON t_test_vector_table (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="5", "nlist" = "256", "nbits"="10");
+
+DROP INDEX index_vector1 ON t_test_vector_table;
+
+CREATE INDEX index_vector2 ON t_test_vector_table (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="4", "nlist" = "256", "nbits"="8", "M_IVFPQ"="2");
 DROP INDEX index_vector2 ON t_test_vector_table;
 
-ALTER TABLE t_test_vector_table add index index_vector2 (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="5", "nlist" = "256", "nbits"="10");
+ALTER TABLE t_test_vector_table add index index_vector2 (vector2) USING VECTOR ("metric_type" = "l2_distance", "is_vector_normed" = "false", "index_type" = "ivfpq", "dim"="4", "nlist" = "256", "nbits"="8", "M_IVFPQ"="2");
 ALTER TABLE t_test_vector_table drop index index_vector2;
 
 DROP TABLE t_test_vector_table;
 
--- name: test_vector_index
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+
+-- name: test_vector_index @sequential
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+
 CREATE TABLE `t_test_vector_table` (
   `id` bigint(20) NOT NULL COMMENT "",
   `vector1` ARRAY<FLOAT> NOT NULL COMMENT "",
@@ -42,4 +51,7 @@ insert into t_test_vector_table values(2, [4,5,6,7,8]);
 select id, approx_l2_distance([1,1,1,1,1], vector1) from t_test_vector_table order by approx_l2_distance([1,1,1,1,1], vector1) limit 1;
 select * from (select id, approx_l2_distance([1,1,1,1,1], vector1) score from t_test_vector_table) a where score < 40 order by score limit 1;
 
-DROP TABLE t_test_vector_table;
\ No newline at end of file
+DROP TABLE t_test_vector_table;
+
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
diff --git a/test/sql/test_vector_index/T/test_vector_index_hnsw b/test/sql/test_vector_index/T/test_vector_index_hnsw
new file mode 100644
index 00000000000000..65944332a7435b
--- /dev/null
+++ b/test/sql/test_vector_index/T/test_vector_index_hnsw
@@ -0,0 +1,249 @@
+-- name: test_vector_index_hnsw @sequential
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+
+
+
+CREATE TABLE __row_util_base (
+  k1 bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`k1`)
+DISTRIBUTED BY HASH(`k1`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+insert into __row_util_base select generate_series from TABLE(generate_series(0, 10000 - 1));
+insert into __row_util_base select * from __row_util_base; -- 20000
+insert into __row_util_base select * from __row_util_base; -- 40000
+insert into __row_util_base select * from __row_util_base; -- 80000
+insert into __row_util_base select * from __row_util_base; -- 160000
+insert into __row_util_base select * from __row_util_base; -- 320000
+insert into __row_util_base select * from __row_util_base; -- 640000
+
+CREATE TABLE __row_util (
+  idx bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`idx`)
+DISTRIBUTED BY HASH(`idx`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+
+insert into __row_util 
+select 
+    row_number() over() as idx
+from __row_util_base;
+
+
+CREATE TABLE t2 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NOT NULL,
+    i1 bigint(20) NOT NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "hnsw", 
+        "dim"="5", 
+        "metric_type" = "l2_distance", 
+        "is_vector_normed" = "false", 
+        "M" = "160", 
+        "efconstruction" = "400")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+
+
+insert into t2 
+select 
+    idx,
+    array_generate(10000, 10004),
+    array_generate(10000, 10004),
+    idx
+from __row_util
+order by idx 
+limit 20;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    order by dis limit 21
+) select * from w2 order by dis, id;
+
+-- cannot use vector index
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+order by dis;
+
+insert into t2 
+select 
+    idx + 20,
+    array_repeat(idx, 5),
+    array_repeat(idx, 5),
+    idx + 20
+from __row_util;
+
+-- basic queries.
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+
+-- queries predicates
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 0
+order by dis limit 10;
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select * from w2 order by dis, id;
+
+
+-- cannot use vector index
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis >= 100 and dis <= 1000
+order by dis limit 10;
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 100 and id >= 0
+order by dis limit 10;
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+order by dis, id limit 10;
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+
+-- basic queries.
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+
+-- queries predicates
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1, 1]) as dis from t2
+)
+select * from w1 
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064, 640064]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4, 5]) as dis from t2
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003, 10004]) as dis from t2
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select * from w2 order by dis, id;
+
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
diff --git a/test/sql/test_vector_index/T/test_vector_index_insert b/test/sql/test_vector_index/T/test_vector_index_insert
new file mode 100644
index 00000000000000..c7a5b6e18d06bc
--- /dev/null
+++ b/test/sql/test_vector_index/T/test_vector_index_insert
@@ -0,0 +1,120 @@
+-- name: test_vector_index_insert @sequential
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+
+CREATE TABLE t1 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "hnsw", 
+        "dim"="5", 
+        "metric_type" = "cosine_similarity", 
+        "is_vector_normed" = "true", 
+        "M" = "16", 
+        "efconstruction" = "40")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+
+INSERT into t1 values 
+    (1, null, null);
+
+INSERT into t1 values 
+    (1, [null, null, null, null, null], [1,2,3,4,5]);
+
+INSERT into t1 values 
+    (1, [1,2,3,4], [1,2,3,4]);
+
+INSERT into t1 values 
+    (1, [], []);
+
+INSERT INTO t1 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7,8], [4,5,6,7,8]);
+
+INSERT INTO t1 values
+    (1, [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242], 
+        [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242]),
+    (2, [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093],
+        [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093]),
+    (3, [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152],
+        [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152]),
+    (4, [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152],
+        null);
+
+
+INSERT INTO t1 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7,8], [4,5,6,7,8]),
+    (3, null, null);
+
+INSERT INTO t1 values
+    (1, [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242], 
+        [0.13483997249264842, 0.26967994498529685, 0.40451991747794525, 0.5393598899705937, 0.674199862463242]),
+    (4, null, null),
+    (2, [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093],
+        [0.29019050004400465, 0.36273812505500586, 0.435285750066007, 0.5078333750770082, 0.5803810000880093]),
+    (3, [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152],
+        [0.3368607684266076, 0.42107596053325946, 0.5052911526399114, null, 0.6737215368532152]),
+    (3, null, null),
+    (5, null, null),
+    (6, null, null),
+    (7, null, null),
+    (8, null, null),
+    (9, null, null),
+    (10, null, null);
+
+select * from t1 order by id;
+
+
+CREATE TABLE t2 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "hnsw", 
+        "dim"="5", 
+        "metric_type" = "cosine_similarity", 
+        "is_vector_normed" = "false", 
+        "M" = "16", 
+        "efconstruction" = "40")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+
+INSERT INTO t2 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7,8], [4,5,6,7,8]),
+    (3, [4,5,6,null,8], [4,5,6,null,8]),
+    (4, [null, null, null, null], [null, null, null, null]),
+    (5, [4,5,6,7,8], null);
+
+
+INSERT INTO t2 values
+    (1, [1,2,3,4,5], [1,2,3,4,5]),
+    (2, [4,5,6,7], [4,5,6,7,8]),
+    (3, [4,5,6,null,8], [4,5,6,null,8]),
+    (4, [null, null, null, null], [null, null, null, null]),
+    (5, [4,5,6,7,8], null);
+
+
+select * from t2 order by id, v1, v2;
+
+
+insert into t1 select * from t2;
+insert into t1 select * from t1;
+select * from t1 order by id, v1, v2;
+
+insert into t2 select * from t1;
+insert into t2 select id, v2, v1 from t2;
+select * from t2 order by id, v1, v2;
+
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
diff --git a/test/sql/test_vector_index/T/test_vector_index_ivfpq b/test/sql/test_vector_index/T/test_vector_index_ivfpq
new file mode 100644
index 00000000000000..f415e7c27fc233
--- /dev/null
+++ b/test/sql/test_vector_index/T/test_vector_index_ivfpq
@@ -0,0 +1,271 @@
+-- name: test_vector_index_ivfpq @sequential
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "true");
+
+
+
+CREATE TABLE __row_util_base (
+  k1 bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`k1`)
+DISTRIBUTED BY HASH(`k1`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+insert into __row_util_base select generate_series from TABLE(generate_series(0, 10000 - 1));
+insert into __row_util_base select * from __row_util_base; -- 20000
+insert into __row_util_base select * from __row_util_base; -- 40000
+insert into __row_util_base select * from __row_util_base; -- 80000
+insert into __row_util_base select * from __row_util_base; -- 160000
+insert into __row_util_base select * from __row_util_base; -- 320000
+insert into __row_util_base select * from __row_util_base; -- 640000
+
+CREATE TABLE __row_util (
+  idx bigint NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`idx`)
+DISTRIBUTED BY HASH(`idx`) BUCKETS 32
+PROPERTIES (
+    "replication_num" = "1"
+);
+
+insert into __row_util 
+select 
+    row_number() over() as idx
+from __row_util_base;
+
+
+CREATE TABLE t1 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NOT NULL,
+    i1 bigint(20) NOT NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "IVFPQ", 
+        "dim"="4", 
+        "metric_type" = "l2_distance", 
+        "is_vector_normed" = "false", 
+        "nbits" = "8", 
+        "nlist" = "40",
+        "M_IVFPQ" = "2")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+
+
+CREATE TABLE t1 (
+    id bigint(20) NOT NULL,
+    v1 ARRAY<FLOAT> NOT NULL,
+    v2 ARRAY<FLOAT> NOT NULL,
+    i1 bigint(20) NOT NULL,
+    INDEX index_vector (v1) USING VECTOR (
+        "index_type" = "ivfpq", 
+        "dim"="4", 
+        "metric_type" = "l2_distance", 
+        "is_vector_normed" = "false", 
+        "nbits" = "8", 
+        "nlist" = "16",
+        "M_IVFPQ" = "2")
+) ENGINE=OLAP
+DUPLICATE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 64
+PROPERTIES (
+    "replication_num" = "1"
+);
+
+
+insert into t1 
+select 
+    idx,
+    array_generate(10000, 10003),
+    array_generate(10000, 10003),
+    idx
+from __row_util
+order by idx 
+limit 20;
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    order by dis limit 21
+) select * from w2 order by dis, id;
+
+-- cannot use vector index
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+order by dis;
+
+insert into t1 
+select 
+    idx + 20,
+    array_repeat(idx, 4),
+    array_repeat(idx, 4),
+    idx + 20
+from __row_util;
+
+-- basic queries.
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select id, v1, dis from w1 order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+
+-- queries predicates
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select v1, dis from w1 
+where dis <= 0
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select v1, dis from w1 
+where dis <= 100
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select v1, dis from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select v1, dis from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select v1, dis from w2 order by dis, id;
+
+
+-- cannot use vector index
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+where dis >= 100 and dis <= 1000
+order by dis limit 10;
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+where dis <= 100 and id >= 0
+order by dis limit 10;
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+order by dis, id limit 10;
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");
+
+-- basic queries.
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select * from w1
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    order by dis limit 20
+) select * from w2 order by dis, id;
+
+-- queries predicates
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 1, 1, 1]) as dis from t1
+)
+select * from w1 
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [640064, 640064, 640064, 640064]) as dis from t1
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [1, 2, 3, 4]) as dis from t1
+)
+select * from w1
+where dis <= 100 and dis <= 1000
+order by dis limit 10;
+
+
+with w1 as (
+    select *, approx_l2_distance(v1, [10000, 10001, 10002, 10003]) as dis from t1
+), w2 as (
+    select * from w1
+    where dis <= 100 and dis <= 1000
+    order by dis limit 20
+) select * from w2 order by dis, id;
+
+
+ADMIN SET FRONTEND CONFIG("enable_experimental_vector" = "false");

From a859197705cb08d5e68df663fe1af5c1c271dca5 Mon Sep 17 00:00:00 2001
From: "shuming.li" <ming.moriarty@gmail.com>
Date: Mon, 20 Jan 2025 14:23:24 +0800
Subject: [PATCH 47/71] [BugFix] Fix mv partition compensation for iceberg
 table with transform partition in range materialized view (#55151)

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
---
 .../java/com/starrocks/common/Config.java     |   4 +
 .../analyzer/MaterializedViewAnalyzer.java    |  18 +-
 .../com/starrocks/sql/common/PRangeCell.java  |   8 +
 .../materialization/MvUtils.java              |  68 +++++++-
 .../ExternalTableCompensation.java            |  89 ++++++----
 .../materialization/MVTestBase.java           |   4 +
 .../MvRefreshAndRewriteIcebergTest.java       |  60 +++++++
 .../test_mv_with_iceberg_partition_transform  | 162 ++++++++++++++++++
 .../test_mv_with_iceberg_partition_transform  |  57 ++++++
 9 files changed, 428 insertions(+), 42 deletions(-)
 create mode 100644 test/sql/test_transparent_mv/R/test_mv_with_iceberg_partition_transform
 create mode 100644 test/sql/test_transparent_mv/T/test_mv_with_iceberg_partition_transform

diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
index ea78c0cf00f81a..569333a210e2d2 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/Config.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
@@ -3135,6 +3135,10 @@ public class Config extends ConfigBase {
             "occupying too much meta memory")
     public static int max_mv_task_run_meta_message_values_length = 16;
 
+    @ConfField(mutable = true, comment = "Whether enable to use list partition rather than range partition for " +
+            "all external table partition types")
+    public static boolean enable_mv_list_partition_for_external_table = false;
+
     /**
      * The refresh partition number when refreshing materialized view at once by default.
      */
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java
index e2c74c98740fe8..b3ccf1800ae66d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java
@@ -59,6 +59,7 @@
 import com.starrocks.catalog.ScalarType;
 import com.starrocks.catalog.Table;
 import com.starrocks.catalog.Type;
+import com.starrocks.common.Config;
 import com.starrocks.common.DdlException;
 import com.starrocks.common.ErrorCode;
 import com.starrocks.common.ErrorReport;
@@ -1028,14 +1029,19 @@ private void checkMVPartitionInfoType(CreateMaterializedViewStatement statement,
                 // - otherwise use range partition as before.
                 // To be compatible with old implementations, if the partition column is not a string type,
                 // still use range partition.
-                // TODO: remove this compatibility code in the future, use list partition directly later.
-                if (partitionExprType.isStringType() &&
-                        mvPartitionByExprs.stream().allMatch(t -> t instanceof SlotRef) &&
-                        !(partitionRefTableExpr instanceof FunctionCallExpr)) {
+                // NOTE: If enable_mv_list_partition_for_external_table is true, create list partition mv
+                // for all external tables. Otherwise, use original range partition instead.
+                if (Config.enable_mv_list_partition_for_external_table) {
                     statement.setPartitionType(PartitionType.LIST);
                 } else {
-                    statement.setPartitionType(PartitionType.RANGE);
-                    checkRangePartitionColumnLimit(mvPartitionByExprs);
+                    if (partitionExprType.isStringType() &&
+                            mvPartitionByExprs.stream().allMatch(t -> t instanceof SlotRef) &&
+                            !(partitionRefTableExpr instanceof FunctionCallExpr)) {
+                        statement.setPartitionType(PartitionType.LIST);
+                    } else {
+                        statement.setPartitionType(PartitionType.RANGE);
+                        checkRangePartitionColumnLimit(mvPartitionByExprs);
+                    }
                 }
             }
         }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/common/PRangeCell.java b/fe/fe-core/src/main/java/com/starrocks/sql/common/PRangeCell.java
index 4817043d071d0c..112eff2f86dd1e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/common/PRangeCell.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/common/PRangeCell.java
@@ -31,6 +31,14 @@ public PRangeCell(Range<PartitionKey> partitionKeyRange) {
         this.range = partitionKeyRange;
     }
 
+    public static PRangeCell of(PartitionKey partitionKey) {
+        return new PRangeCell(Range.singleton(partitionKey));
+    }
+
+    public static PRangeCell of(Range<PartitionKey> partitionKeyRange) {
+        return new PRangeCell(partitionKeyRange);
+    }
+
     public Range<PartitionKey> getRange() {
         return range;
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
index 83f9356198b5b1..f9c4b9bc37814b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
@@ -61,6 +61,7 @@
 import com.starrocks.sql.ast.RandomDistributionDesc;
 import com.starrocks.sql.ast.StatementBase;
 import com.starrocks.sql.common.MetaUtils;
+import com.starrocks.sql.common.PRangeCell;
 import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder;
 import com.starrocks.sql.optimizer.ExpressionContext;
 import com.starrocks.sql.optimizer.JoinHelper;
@@ -1472,21 +1473,32 @@ public static String formatBaseTableInfos(List<BaseTableInfo> baseTableInfos) {
         return baseTableInfos.stream().map(BaseTableInfo::getReadableString).collect(Collectors.joining(","));
     }
 
+    public static ScalarOperator convertPartitionKeyRangesToListPredicate(List<? extends ScalarOperator> partitionColRefs,
+                                                                          Collection<PRangeCell> pRangeCells,
+                                                                          boolean areAllRangePartitionsSingleton) {
+        final List<Range<PartitionKey>> partitionRanges = pRangeCells
+                .stream()
+                .map(PRangeCell::getRange)
+                .collect(Collectors.toList());
+
+        return convertPartitionKeysToListPredicate(partitionColRefs, partitionRanges, areAllRangePartitionsSingleton);
+    }
+
     public static ScalarOperator convertPartitionKeysToListPredicate(List<? extends ScalarOperator> partitionColRefs,
                                                                      Collection<PartitionKey> partitionRanges) {
-        List<ScalarOperator> values = Lists.newArrayList();
+        final List<ScalarOperator> values = Lists.newArrayList();
         if (partitionColRefs.size() == 1) {
-            for (PartitionKey partitionKey : partitionRanges) {
-                List<LiteralExpr> literalExprs = partitionKey.getKeys();
+            for (PartitionKey key : partitionRanges) {
+                final List<LiteralExpr> literalExprs = key.getKeys();
                 Preconditions.checkArgument(literalExprs.size() == partitionColRefs.size());
-                LiteralExpr literalExpr = literalExprs.get(0);
-                ConstantOperator upperBound = (ConstantOperator) SqlToScalarOperatorTranslator.translate(literalExpr);
+                final LiteralExpr literalExpr = literalExprs.get(0);
+                final ConstantOperator upperBound = (ConstantOperator) SqlToScalarOperatorTranslator.translate(literalExpr);
                 values.add(upperBound);
             }
             return MvUtils.convertToInPredicate(partitionColRefs.get(0), values);
         } else {
-            for (PartitionKey partitionKey : partitionRanges) {
-                List<LiteralExpr> literalExprs = partitionKey.getKeys();
+            for (PartitionKey key : partitionRanges) {
+                List<LiteralExpr> literalExprs = key.getKeys();
                 Preconditions.checkArgument(literalExprs.size() == partitionColRefs.size());
                 // TODO: use row operator instead
                 List<ScalarOperator> predicates = Lists.newArrayList();
@@ -1503,6 +1515,48 @@ public static ScalarOperator convertPartitionKeysToListPredicate(List<? extends
         }
     }
 
+    private static ScalarOperator convertPartitionKeysToListPredicate(List<? extends ScalarOperator> partitionColRefs,
+                                                                      Collection<Range<PartitionKey>> partitionRanges,
+                                                                      boolean areAllRangePartitionsSingleton) {
+
+        if (areAllRangePartitionsSingleton) {
+            List<PartitionKey> partitionKeys = partitionRanges
+                    .stream()
+                    .map(Range::lowerEndpoint)
+                    .collect(Collectors.toList());
+            return convertPartitionKeysToListPredicate(partitionColRefs, partitionKeys);
+        } else {
+            final List<ScalarOperator> values = Lists.newArrayList();
+            partitionRanges
+                    .stream()
+                    .map(range -> getPartitionKeyRangePredicate(partitionColRefs, range))
+                    .forEach(values::add);
+            return Utils.compoundOr(values);
+        }
+    }
+
+    private static ScalarOperator getPartitionKeyRangePredicate(List<? extends ScalarOperator> partitionColRefs,
+                                                                Range<PartitionKey> range) {
+        final List<LiteralExpr> lowerLiteralExprs = range.lowerEndpoint().getKeys();
+        final List<LiteralExpr> upperLiteralExprs = range.upperEndpoint().getKeys();
+        Preconditions.checkArgument(lowerLiteralExprs.size() == upperLiteralExprs.size());
+        Preconditions.checkArgument(lowerLiteralExprs.size() == partitionColRefs.size());
+        final List<ScalarOperator> predicates = Lists.newArrayList();
+        for (int i = 0; i < lowerLiteralExprs.size(); i++) {
+            final ScalarOperator partitionColRef = partitionColRefs.get(i);
+            final LiteralExpr lowerLiteralExpr = lowerLiteralExprs.get(i);
+            final LiteralExpr upperLiteralExpr = upperLiteralExprs.get(i);
+            final ConstantOperator lowerBound =
+                    (ConstantOperator) SqlToScalarOperatorTranslator.translate(lowerLiteralExpr);
+            final ConstantOperator upperBound =
+                    (ConstantOperator) SqlToScalarOperatorTranslator.translate(upperLiteralExpr);
+            final ScalarOperator gt = new BinaryPredicateOperator(BinaryType.GE, partitionColRef, lowerBound);
+            final ScalarOperator ls = new BinaryPredicateOperator(BinaryType.LT, partitionColRef, upperBound);
+            predicates.add(Utils.compoundAnd(gt, ls));
+        }
+        return Utils.compoundAnd(predicates);
+    }
+
     /**
      * Optimize the inlined view plan.
      * @param logicalTree logical opt expression tree which has not been optimized
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/ExternalTableCompensation.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/ExternalTableCompensation.java
index 0e09c1c7a8ef06..2b5ff4cbfd4f85 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/ExternalTableCompensation.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/compensation/ExternalTableCompensation.java
@@ -17,6 +17,7 @@
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
 import com.starrocks.analysis.BinaryType;
 import com.starrocks.analysis.Expr;
 import com.starrocks.analysis.LiteralExpr;
@@ -61,6 +62,7 @@
 import com.starrocks.sql.optimizer.transformer.ExpressionMapping;
 import com.starrocks.sql.optimizer.transformer.SqlToScalarOperatorTranslator;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.iceberg.PartitionField;
 import org.apache.iceberg.Snapshot;
 
 import java.util.Collection;
@@ -72,12 +74,12 @@
 
 import static com.starrocks.connector.iceberg.IcebergPartitionUtils.getIcebergTablePartitionPredicateExpr;
 import static com.starrocks.sql.optimizer.OptimizerTraceUtil.logMVRewrite;
-import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.convertPartitionKeysToListPredicate;
+import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.convertPartitionKeyRangesToListPredicate;
 
 public final class ExternalTableCompensation extends TableCompensation {
-    private List<PartitionKey> compensations;
+    private List<PRangeCell> compensations;
 
-    public ExternalTableCompensation(Table refBaseTable, List<PartitionKey> compensations) {
+    public ExternalTableCompensation(Table refBaseTable, List<PRangeCell> compensations) {
         super(refBaseTable, MVTransparentState.COMPENSATE);
         this.compensations = compensations;
     }
@@ -122,10 +124,10 @@ public LogicalScanOperator compensate(OptimizerContext optimizerContext,
                     Optional.ofNullable(((IcebergTable) currentTable).getNativeTable().currentSnapshot())
                             .map(Snapshot::snapshotId));
             builder.setTableVersionRange(versionRange);
-            externalExtraPredicate = getIcebergTableCompensation(optimizerContext, mv, refBaseTable, refTableName,
-                    refPartitionColRefs);
+            externalExtraPredicate = getIcebergTableCompensation(optimizerContext, mv, (IcebergTable) currentTable,
+                    refTableName, refPartitionColRefs);
         } else {
-            externalExtraPredicate = convertPartitionKeysToListPredicate(refPartitionColRefs, compensations);
+            externalExtraPredicate = convertPartitionKeyRangesToListPredicate(refPartitionColRefs, compensations, true);
         }
         Preconditions.checkState(externalExtraPredicate != null);
         externalExtraPredicate.setRedundant(true);
@@ -136,12 +138,28 @@ public LogicalScanOperator compensate(OptimizerContext optimizerContext,
 
     private ScalarOperator getIcebergTableCompensation(OptimizerContext optimizerContext,
                                                        MaterializedView mv,
-                                                       Table refBaseTable,
+                                                       IcebergTable icebergTable,
                                                        TableName refTableName,
                                                        List<ColumnRefOperator> refPartitionColRefs) {
         PartitionInfo mvPartitionInfo = mv.getPartitionInfo();
         if (!mvPartitionInfo.isListPartition()) {
-            return convertPartitionKeysToListPredicate(refPartitionColRefs, compensations);
+            // check whether the iceberg table contains partition transformations
+            final List<Column> refBaseTablePartitionCols = refPartitionColRefs.stream()
+                    .map(ref -> icebergTable.getColumn(ref.getName()))
+                    .collect(Collectors.toList());
+            final List<PartitionField> partitionFields = Lists.newArrayList();
+            for (Column column : refBaseTablePartitionCols) {
+                for (PartitionField field : icebergTable.getNativeTable().spec().fields()) {
+                    final String partitionFieldName = icebergTable.getNativeTable().schema().findColumnName(field.sourceId());
+                    if (partitionFieldName.equalsIgnoreCase(column.getName())) {
+                        partitionFields.add(field);
+                    }
+                }
+            }
+            final boolean isContainPartitionTransform = partitionFields
+                    .stream()
+                    .anyMatch(field -> field.transform().dedupName().equalsIgnoreCase("time"));
+            return convertPartitionKeyRangesToListPredicate(refPartitionColRefs, compensations, !isContainPartitionTransform);
         }
         List<Column> mvPartitionCols = mv.getPartitionColumns();
         // to iceberg, `partitionKeys` are using LocalTime as partition values which cannot be used to prune iceberg
@@ -149,8 +167,8 @@ private ScalarOperator getIcebergTableCompensation(OptimizerContext optimizerCon
         // convert `partitionKeys` to iceberg utc time here.
         // Please see MVPCTRefreshListPartitioner#genPartitionPredicate for more details.
         Map<Table, List<SlotRef>> refBaseTablePartitionSlotRefs = mv.getRefBaseTablePartitionSlots();
-        Preconditions.checkArgument(refBaseTablePartitionSlotRefs.containsKey(refBaseTable));
-        List<SlotRef> refBaseTableSlotRefs = refBaseTablePartitionSlotRefs.get(refBaseTable);
+        Preconditions.checkArgument(refBaseTablePartitionSlotRefs.containsKey(icebergTable));
+        final List<SlotRef> refBaseTableSlotRefs = refBaseTablePartitionSlotRefs.get(icebergTable);
         ExpressionMapping expressionMapping =
                 new ExpressionMapping(new Scope(RelationId.anonymous(), new RelationFields()),
                         Lists.newArrayList());
@@ -160,17 +178,17 @@ private ScalarOperator getIcebergTableCompensation(OptimizerContext optimizerCon
             expressionMapping.put(refBaseTablePartitionExpr, refPartitionColRef);
         }
         AnalyzeState analyzeState = new AnalyzeState();
-        Scope scope = new Scope(RelationId.anonymous(), new RelationFields(
-                refBaseTable.getBaseSchema().stream()
+        final Scope scope = new Scope(RelationId.anonymous(), new RelationFields(
+                icebergTable.getBaseSchema().stream()
                         .map(col -> new Field(col.getName(),
                                 col.getType(), refTableName, null))
                         .collect(Collectors.toList())));
         List<ScalarOperator> externalPredicates = Lists.newArrayList();
-        List<Column> refBaseTablePartitionCols = refPartitionColRefs.stream()
-                .map(ref -> refBaseTable.getColumn(ref.getName()))
+        final List<Column> refBaseTablePartitionCols = refPartitionColRefs.stream()
+                .map(ref -> icebergTable.getColumn(ref.getName()))
                 .collect(Collectors.toList());
-        for (PartitionKey partitionKey : compensations) {
-            List<LiteralExpr> literalExprs = partitionKey.getKeys();
+        for (PRangeCell pRangeCell : compensations) {
+            List<LiteralExpr> literalExprs = pRangeCell.getRange().lowerEndpoint().getKeys();
             Preconditions.checkState(literalExprs.size() == refPartitionColRefs.size());
             List<ScalarOperator> predicates = Lists.newArrayList();
             for (int i = 0; i < literalExprs.size(); i++) {
@@ -186,7 +204,7 @@ private ScalarOperator getIcebergTableCompensation(OptimizerContext optimizerCon
                 } else {
                     SlotRef refBaseTablePartitionExpr = refBaseTableSlotRefs.get(i);
                     Column refColumn = refBaseTablePartitionCols.get(i);
-                    Expr predicateExpr = getIcebergTablePartitionPredicateExpr((IcebergTable) refBaseTable,
+                    Expr predicateExpr = getIcebergTablePartitionPredicateExpr(icebergTable,
                             refColumn.getName(), refBaseTablePartitionExpr, literalExpr);
                     ExpressionAnalyzer.analyzeExpression(predicateExpr, analyzeState, scope, ConnectContext.get());
                     ScalarOperator predicate = SqlToScalarOperatorTranslator.translate(predicateExpr, expressionMapping,
@@ -210,17 +228,27 @@ public String toString() {
         sb.append(" [");
         List<String> partitions = Lists.newArrayList();
         for (int i = 0; i < size; i++) {
-            PartitionKey key = compensations.get(i);
-            List<String> keys = key.getKeys()
-                    .stream()
-                    .map(LiteralExpr::getStringValue)
-                    .collect(Collectors.toList());
-            partitions.add("(" + Joiner.on(",").join(keys) + ")");
+            PRangeCell key = compensations.get(i);
+            Range<PartitionKey> range = key.getRange();
+            if (range.lowerEndpoint().equals(range.upperEndpoint())) {
+                partitions.add(getPartitionKeyString(range.lowerEndpoint()));
+            } else {
+                sb.append(getPartitionKeyString(key.getRange().lowerEndpoint()))
+                        .append(" - ")
+                        .append(getPartitionKeyString(key.getRange().upperEndpoint()));
+            }
         }
         sb.append(Joiner.on(",").join(partitions));
         sb.append("]");
         return sb.toString();
     }
+    private String getPartitionKeyString(PartitionKey key) {
+        List<String> keys = key.getKeys()
+                .stream()
+                .map(LiteralExpr::getStringValue)
+                .collect(Collectors.toList());
+        return "(" + Joiner.on(",").join(keys) + ")";
+    }
 
     public static TableCompensation build(Table refBaseTable,
                                           MvUpdateInfo mvUpdateInfo,
@@ -238,7 +266,7 @@ public static TableCompensation build(Table refBaseTable,
             return TableCompensation.noCompensation();
         }
 
-        List<PartitionKey> toRefreshPartitionKeys = Lists.newArrayList();
+        final List<PRangeCell> toRefreshPartitionKeys = Lists.newArrayList();
         MVTransparentState state;
         if (MvPartitionCompensator.isSupportPartitionPruneCompensate(refBaseTable) && scanOperatorOpt.isPresent()) {
             state = getToRefreshPartitionKeysWithPruner(refBaseTable, mv, toRefreshPartitionNames, toRefreshPartitionKeys,
@@ -270,7 +298,7 @@ public static TableCompensation build(Table refBaseTable,
     private static MVTransparentState getToRefreshPartitionKeysWithoutPruner(Table refBaseTable,
                                                                              MvUpdateInfo mvUpdateInfo,
                                                                              Set<String> toRefreshPartitionNames,
-                                                                             List<PartitionKey> toRefreshPartitionKeys) {
+                                                                             final List<PRangeCell> toRefreshPartitionKeys) {
         MvBaseTableUpdateInfo baseTableUpdateInfo = mvUpdateInfo.getBaseTableUpdateInfos().get(refBaseTable);
         if (baseTableUpdateInfo == null) {
             return null;
@@ -290,10 +318,12 @@ private static MVTransparentState getToRefreshPartitionKeysWithoutPruner(Table r
                 }
                 PCell pCell = nameToPartitionKeys.get(partitionName);
                 if (pCell instanceof PRangeCell) {
-                    toRefreshPartitionKeys.add(((PRangeCell) pCell).getRange().lowerEndpoint());
+                    toRefreshPartitionKeys.add(((PRangeCell) pCell));
                 } else if (pCell instanceof PListCell) {
-                    List<PartitionKey> keys = ((PListCell) pCell).toPartitionKeys(partitionColumns);
-                    toRefreshPartitionKeys.addAll(keys);
+                    final List<PartitionKey> keys = ((PListCell) pCell).toPartitionKeys(partitionColumns);
+                    keys.stream()
+                            .map(key -> PRangeCell.of(key))
+                            .forEach(toRefreshPartitionKeys::add);
                 }
             }
         } catch (Exception e) {
@@ -312,7 +342,7 @@ private static MVTransparentState getToRefreshPartitionKeysWithoutPruner(Table r
     private static MVTransparentState getToRefreshPartitionKeysWithPruner(Table refBaseTable,
                                                                           MaterializedView mv,
                                                                           Set<String> toRefreshPartitionNames,
-                                                                          List<PartitionKey> toRefreshPartitionKeys,
+                                                                          List<PRangeCell> toRefreshPartitionKeys,
                                                                           LogicalScanOperator scanOperator) {
         // selected partition ids/keys are only set for scan operator that supports partition prune.
         List<PartitionKey> selectPartitionKeys = null;
@@ -378,6 +408,7 @@ private static MVTransparentState getToRefreshPartitionKeysWithPruner(Table refB
         toRefreshPartitionNames
                 .stream()
                 .map(selectPartitionNameToKeys::get)
+                .map(key -> PRangeCell.of(key))
                 .forEach(toRefreshPartitionKeys::add);
         return MVTransparentState.COMPENSATE;
     }
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java
index 348e6aab5be908..27b9fcf259cc2c 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java
@@ -146,6 +146,10 @@ public static Table getTable(String dbName, String mvName) {
         return table;
     }
 
+    protected MaterializedView getMv(String mvName) {
+        return getMv(DB_NAME, mvName);
+    }
+
     protected MaterializedView getMv(String dbName, String mvName) {
         Table table = getTable(dbName, mvName);
         Assert.assertTrue(table instanceof MaterializedView);
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java
index 766977228bd1c0..1932307e7aa885 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRefreshAndRewriteIcebergTest.java
@@ -17,6 +17,7 @@
 import com.starrocks.catalog.Database;
 import com.starrocks.catalog.MaterializedView;
 import com.starrocks.catalog.Partition;
+import com.starrocks.common.Config;
 import com.starrocks.connector.iceberg.MockIcebergMetadata;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.sql.plan.ConnectorPlanTestBase;
@@ -1954,4 +1955,63 @@ public void testViewBasedRewrite() throws Exception {
         starRocksAssert.dropMaterializedView(mvName);
         connectContext.getSessionVariable().setEnableViewBasedMvRewrite(false);
     }
+
+    @Test
+    public void testListMVWithIcebergTable1() {
+        final String mvName = "test_mv1";
+        Config.enable_mv_list_partition_for_external_table = true;
+        try {
+            starRocksAssert.withMaterializedView("create materialized view " + mvName + " " +
+                    "partition by str2date(d,'%Y-%m-%d') " +
+                    "distributed by hash(a) " +
+                    "REFRESH DEFERRED MANUAL " +
+                    "PROPERTIES (\n" +
+                    "'replication_num' = '1'" +
+                    ") " +
+                    "as select a, b, d, bitmap_union(to_bitmap(t1.c))" +
+                    " from iceberg0.partitioned_db.part_tbl1 as t1 " +
+                    " group by a, b, d;");
+            Assert.fail();
+        } catch (Exception e) {
+            Assert.assertTrue(e.getMessage().contains("List partition expression can only be ref-base-table's " +
+                    "partition expression but contain"));
+        }
+        Config.enable_mv_list_partition_for_external_table = false;
+    }
+
+    @Test
+    public void testListMVWithIcebergTable2() throws Exception {
+        Config.enable_mv_list_partition_for_external_table = true;
+        final String mvName = "test_mv1";
+        starRocksAssert.withMaterializedView("create materialized view " + mvName + " " +
+                "partition by d " +
+                "distributed by hash(a) " +
+                "REFRESH DEFERRED MANUAL " +
+                "PROPERTIES (\n" +
+                "'replication_num' = '1'" +
+                ") " +
+                "as select a, b, d, bitmap_union(to_bitmap(t1.c))" +
+                " from iceberg0.partitioned_db.part_tbl1 as t1 " +
+                " group by a, b, d;");
+        final MaterializedView mv = getMv(mvName);
+        Assert.assertTrue(mv.getPartitionInfo().isListPartition());
+        Config.enable_mv_list_partition_for_external_table = false;
+    }
+
+    @Test
+    public void testListMVWithIcebergTable3() throws Exception {
+        String mvName = "test_mv1";
+        Config.enable_mv_list_partition_for_external_table = true;
+        starRocksAssert.withMaterializedView("CREATE MATERIALIZED VIEW test_mv1\n" +
+                "PARTITION BY date_trunc('month', ts)\n" +
+                "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" +
+                "REFRESH DEFERRED MANUAL\n" +
+                "PROPERTIES (\n" +
+                "\"replication_num\" = \"1\"\n" +
+                ")\n" +
+                "AS SELECT id, data, ts  FROM `iceberg0`.`partitioned_transforms_db`.`t0_month` as a;");
+        final MaterializedView mv = getMv(mvName);
+        Assert.assertTrue(mv.getPartitionInfo().isListPartition());
+        Config.enable_mv_list_partition_for_external_table = false;
+    }
 }
diff --git a/test/sql/test_transparent_mv/R/test_mv_with_iceberg_partition_transform b/test/sql/test_transparent_mv/R/test_mv_with_iceberg_partition_transform
new file mode 100644
index 00000000000000..e846fe00f209ce
--- /dev/null
+++ b/test/sql/test_transparent_mv/R/test_mv_with_iceberg_partition_transform
@@ -0,0 +1,162 @@
+-- name: test_mv_with_iceberg_partition_transform
+create database db_${uuid0};
+-- result:
+-- !result
+use db_${uuid0};
+-- result:
+-- !result
+create external catalog mv_iceberg_${uuid0}
+properties
+(
+    "type" = "iceberg",
+    "iceberg.catalog.type" = "hive",
+    "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}"
+);
+-- result:
+-- !result
+CREATE MATERIALIZED VIEW test_mv1
+PARTITION BY (date_trunc('month', prcdate))
+REFRESH DEFERRED MANUAL
+PROPERTIES ("replication_num" = "1")
+AS
+  SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month;
+-- result:
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 PARTITION start('2025-01-01') end('2025-01-03') WITH SYNC MODE;
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+2025-02-03
+2025-03-03
+2025-04-01
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;
+-- result:
+2025-02-03
+2025-03-03
+2025-04-01
+-- !result
+select distinct prcdate from test_mv1 order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+-- !result
+select * from test_mv1 order by prcdate;
+-- result:
+2025-01-01	b	1.0
+2025-01-02	b	2.0
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 PARTITION start('2025-01-03') end('2025-01-04') WITH SYNC MODE;
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+2025-02-03
+2025-03-03
+2025-04-01
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;
+-- result:
+2025-02-03
+2025-03-03
+2025-04-01
+-- !result
+select distinct prcdate from test_mv1 order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+-- !result
+select * from test_mv1 order by prcdate;
+-- result:
+2025-01-01	b	1.0
+2025-01-02	b	2.0
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;", "test_mv1")
+-- result:
+True
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+2025-02-03
+2025-03-03
+2025-04-01
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+-- !result
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;
+-- result:
+2025-02-03
+2025-03-03
+2025-04-01
+-- !result
+select distinct prcdate from test_mv1 order by prcdate;
+-- result:
+2025-01-01
+2025-01-02
+2025-02-03
+2025-03-03
+2025-04-01
+-- !result
+select * from test_mv1 order by prcdate;
+-- result:
+2025-01-01	b	1.0
+2025-01-02	b	2.0
+2025-02-03	b	3.0
+2025-03-03	b	4.0
+2025-04-01	b	5.0
+-- !result
+drop database db_${uuid0} force;
+-- result:
+-- !result
+drop catalog mv_iceberg_${uuid0};
+-- result:
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_transparent_mv/T/test_mv_with_iceberg_partition_transform b/test/sql/test_transparent_mv/T/test_mv_with_iceberg_partition_transform
new file mode 100644
index 00000000000000..8d386f42f49f28
--- /dev/null
+++ b/test/sql/test_transparent_mv/T/test_mv_with_iceberg_partition_transform
@@ -0,0 +1,57 @@
+-- name: test_mv_with_iceberg_partition_transform
+
+-- create mv
+create database db_${uuid0};
+use db_${uuid0};
+
+-- admin set frontend config("enable_mv_list_partition_for_external_table"="false");
+create external catalog mv_iceberg_${uuid0}
+properties
+(
+    "type" = "iceberg",
+    "iceberg.catalog.type" = "hive",
+    "hive.metastore.uris" = "${iceberg_catalog_hive_metastore_uris}"
+);
+
+CREATE MATERIALIZED VIEW test_mv1
+PARTITION BY (date_trunc('month', prcdate))
+REFRESH DEFERRED MANUAL
+PROPERTIES ("replication_num" = "1")
+AS
+  SELECT * FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month;
+
+-- partial refresh
+REFRESH MATERIALIZED VIEW test_mv1 PARTITION start('2025-01-01') end('2025-01-03') WITH SYNC MODE;
+
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;", "test_mv1")
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;", "test_mv1")
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;", "test_mv1")
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;
+select distinct prcdate from test_mv1 order by prcdate;
+select * from test_mv1 order by prcdate;
+
+REFRESH MATERIALIZED VIEW test_mv1 PARTITION start('2025-01-03') end('2025-01-04') WITH SYNC MODE;
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;", "test_mv1")
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;", "test_mv1")
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;", "test_mv1")
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;
+select distinct prcdate from test_mv1 order by prcdate;
+select * from test_mv1 order by prcdate;
+
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;", "test_mv1")
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;", "test_mv1")
+function: print_hit_materialized_view("SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;", "test_mv1")
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month order by prcdate;
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate < '2025-01-03' order by prcdate;
+SELECT distinct prcdate FROM mv_iceberg_${uuid0}.sql_test_db.test_iceberg_with_month where prcdate > '2025-01-03' order by prcdate;
+select distinct prcdate from test_mv1 order by prcdate;
+select * from test_mv1 order by prcdate;
+
+drop database db_${uuid0} force;
+drop catalog mv_iceberg_${uuid0};
\ No newline at end of file

From f34f1a229df1fc78b191d6042a69f54223969bcd Mon Sep 17 00:00:00 2001
From: "shuming.li" <ming.moriarty@gmail.com>
Date: Mon, 20 Jan 2025 14:24:33 +0800
Subject: [PATCH 48/71] [BugFix] Support more patterns for mv union rewrite by
 default (#55199)

Signed-off-by: shuming.li <ming.moriarty@gmail.com>
---
 .../MaterializedViewRewriter.java             |  59 ++-
 .../R/test_transparent_mv_union_hive2         | 470 ++++++++++++++++++
 .../T/test_transparent_mv_union_hive2         | 175 +++++++
 3 files changed, 694 insertions(+), 10 deletions(-)
 create mode 100644 test/sql/test_transparent_mv/R/test_transparent_mv_union_hive2
 create mode 100644 test/sql/test_transparent_mv/T/test_transparent_mv_union_hive2

diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java
index 2fb5b5f1ef18ab..965b472f16ff81 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MaterializedViewRewriter.java
@@ -1660,6 +1660,30 @@ private boolean isPullUpQueryPredicate(ScalarOperator predicate,
                 });
     }
 
+    private boolean isPullPredicateRewrite(List<ScalarOperator> queryPredicates,
+                                           Set<ColumnRefOperator> mvPredicateUsedColRefs,
+                                           MVUnionRewriteMode unionRewriteMode) {
+        if (unionRewriteMode.isPullPredicateRewrite()) {
+            return true;
+        }
+        // if mv predicates are empty, try to union rewrite.
+        if (CollectionUtils.isEmpty(mvPredicateUsedColRefs)) {
+            return true;
+        }
+        // if query predicates and mv predicates have the same column, try to union rewrite.
+        if (queryPredicates.stream()
+                .anyMatch(pred -> pred.getColumnRefs().stream().anyMatch(col -> mvPredicateUsedColRefs.contains(col)))) {
+            return true;
+        }
+        // TODO: if query predicates are not contained in mv predicates, there maybe a chance to union rewrite
+        //  but the performance may not be good.
+        // eg:
+        //   query: select dt, k2 from tblA where date_trunc('day', dt) >= '2023-11-01'
+        //   mv : select dt, k2 from tblA where k2 > 10;
+        // whether to rewrite query by mv should be considered carefully later.
+        return false;
+    }
+
     private PredicateSplit getUnionRewriteQueryCompensation(RewriteContext rewriteContext,
                                                             ColumnRewriter columnRewriter) {
         final PredicateSplit mvCompensationToQuery = getCompensationPredicates(columnRewriter,
@@ -1674,23 +1698,38 @@ private PredicateSplit getUnionRewriteQueryCompensation(RewriteContext rewriteCo
         SessionVariable sessionVariable = optimizerContext.getSessionVariable();
         MVUnionRewriteMode unionRewriteMode =
                 MVUnionRewriteMode.getInstance(sessionVariable.getMaterializedViewUnionRewriteMode());
-        if (!unionRewriteMode.isPullPredicateRewrite()) {
-            return null;
-        }
 
-        logMVRewrite(mvRewriteContext, "Try to pull up query's predicates to make possible for union rewrite, " +
-                "unionRewriteMode:{}", unionRewriteMode);
         // try to pull up query's predicates to make possible for rewrite from mv to query.
         PredicateSplit mvPredicateSplit = rewriteContext.getMvPredicateSplit();
         PredicateSplit queryPredicateSplit = rewriteContext.getQueryPredicateSplit();
-        Set<ColumnRefOperator> mvPredicateUsedColRefs = Utils.compoundAnd(mvPredicateSplit.getPredicates())
-                .getColumnRefs().stream()
-                .map(colRef -> (ColumnRefOperator) columnRewriter.rewriteViewToQuery(colRef))
+
+        // filter redundant predicates since they are not used for rewrite.
+        Set<ColumnRefOperator> mvPredicateUsedColRefs = mvPredicateSplit.getPredicates()
+                .stream()
+                .filter(pred -> pred != null && !pred.isRedundant())
+                .map(pred -> pred.getColumnRefs()
+                        .stream()
+                        .map(colRef -> (ColumnRefOperator) columnRewriter.rewriteViewToQuery(colRef))
+                        .collect(Collectors.toSet()))
+                .flatMap(Set::stream)
                 .collect(Collectors.toSet());
 
         List<ScalarOperator> queryPredicates = Lists.newArrayList();
-        queryPredicates.addAll(Utils.extractConjuncts(queryPredicateSplit.getRangePredicates()));
-        queryPredicates.addAll(Utils.extractConjuncts(queryPredicateSplit.getResidualPredicates()));
+        Utils.extractConjuncts(queryPredicateSplit.getRangePredicates())
+                .stream()
+                .filter(pred -> pred != null && !pred.isRedundant())
+                .forEach(queryPredicates::add);
+        Utils.extractConjuncts(queryPredicateSplit.getResidualPredicates())
+                .stream()
+                .filter(pred -> pred != null && !pred.isRedundant())
+                .forEach(queryPredicates::add);
+        
+        // if query and mv contains the same column, we can rewrite query by mv.
+        if (!isPullPredicateRewrite(queryPredicates, mvPredicateUsedColRefs, unionRewriteMode)) {
+            return null;
+        }
+        logMVRewrite(mvRewriteContext, "Try to pull up query's predicates to make possible for union rewrite, " +
+                "unionRewriteMode:{}", unionRewriteMode);
 
         Set<ScalarOperator> queryOnPredicates = MvUtils.getJoinOnPredicates(rewriteContext.getQueryExpression());
         ColumnRefSet queryOnPredicateUsedColRefs = Optional.ofNullable(Utils.compoundAnd(queryOnPredicates))
diff --git a/test/sql/test_transparent_mv/R/test_transparent_mv_union_hive2 b/test/sql/test_transparent_mv/R/test_transparent_mv_union_hive2
new file mode 100644
index 00000000000000..231053d4343c5b
--- /dev/null
+++ b/test/sql/test_transparent_mv/R/test_transparent_mv_union_hive2
@@ -0,0 +1,470 @@
+-- name: test_transparent_mv_union_hive2
+create external catalog mv_hive_${uuid0}
+properties
+(
+    "type" = "hive",
+    "hive.catalog.type" = "hive",
+    "hive.metastore.uris" = "${hive_metastore_uris}"
+);
+-- result:
+-- !result
+set new_planner_optimize_timeout=10000;
+-- result:
+-- !result
+set catalog mv_hive_${uuid0};
+-- result:
+-- !result
+create database mv_hive_db_${uuid0};
+-- result:
+-- !result
+use mv_hive_db_${uuid0};
+-- result:
+-- !result
+CREATE TABLE mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 (
+  num int,
+  dt date
+)
+PARTITION BY (dt);
+-- result:
+-- !result
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES 
+  (1,"2024-11-15"),(2,"2024-11-18"),(3,"2024-11-21"),(4,"2024-11-24"),
+  (1,"2024-12-02"),(2,"2024-12-05"),(3,"2024-12-08"),(4,"2024-12-11"),
+  (1,"2024-12-16"),(2,"2024-12-19"),(3,"2024-12-22"),(4,"2024-12-25"),
+  (2,"2024-11-15"),(3,"2024-11-18"),(4,"2024-11-21"),(5,"2024-11-24"),
+  (2,"2024-12-02"),(3,"2024-12-05"),(4,"2024-12-08"),(5,"2024-12-11");
+-- result:
+-- !result
+CREATE TABLE mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 (
+  num int,
+  dt date
+)
+PARTITION BY (dt);
+-- result:
+-- !result
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 VALUES 
+  (1,"2024-11-15"),(2,"2024-11-18"),(3,"2024-11-21"),(4,"2024-11-24"),
+  (1,"2024-12-02"),(2,"2024-12-05"),(3,"2024-12-08"),(4,"2024-12-11"),
+  (1,"2024-12-16"),(2,"2024-12-19"),(3,"2024-12-22"),(4,"2024-12-25"),
+  (2,"2024-11-15"),(3,"2024-11-18"),(4,"2024-11-21"),(5,"2024-11-24"),
+  (2,"2024-12-02"),(3,"2024-12-05"),(4,"2024-12-08"),(5,"2024-12-11");
+-- result:
+-- !result
+set catalog default_catalog;
+-- result:
+-- !result
+create database db_${uuid0};
+-- result:
+-- !result
+use db_${uuid0};
+-- result:
+-- !result
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+PROPERTIES (
+  "replication_num" = "1",
+  "query_rewrite_consistency" = "checked"
+  -- "partition_ttl"="60 day"
+)
+AS SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1;
+-- result:
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (1, "2024-11-18"), (1, "2024-11-23"), (4, "2024-12-25");
+-- result:
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;
+-- result:
+1	2024-12-02
+1	2024-12-16
+2	2024-12-02
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;
+-- result:
+5	2024-12-11
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;
+-- result:
+1	2024-11-23
+1	2024-12-02
+1	2024-12-16
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;
+-- result:
+5	2024-11-24
+5	2024-12-11
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;
+-- result:
+1	2024-11-15
+1	2024-11-15
+1	2024-11-18
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;
+-- result:
+4	2024-11-21
+4	2024-11-24
+4	2024-12-08
+-- !result
+SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3)t order by 1, 2 limit 3;
+-- result:
+4	2024-11-21
+4	2024-11-21
+4	2024-11-24
+-- !result
+DROP MATERIALIZED VIEW test_mv1;
+-- result:
+-- !result
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+PROPERTIES (
+  "replication_num" = "1",
+  "query_rewrite_consistency" = "loose"
+)
+AS SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1;
+-- result:
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (1, "2024-11-18"), (1, "2024-11-23"), (4, "2024-12-25");
+-- result:
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;
+-- result:
+1	2024-12-02
+1	2024-12-16
+2	2024-12-02
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;
+-- result:
+5	2024-12-11
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;
+-- result:
+1	2024-11-23
+1	2024-12-02
+1	2024-12-16
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;
+-- result:
+5	2024-11-24
+5	2024-12-11
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;
+-- result:
+1	2024-11-15
+1	2024-11-15
+1	2024-11-18
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;
+-- result:
+4	2024-11-21
+4	2024-11-24
+4	2024-12-08
+-- !result
+SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3)t order by 1, 2 limit 3;
+-- result:
+4	2024-11-21
+4	2024-11-21
+4	2024-11-24
+-- !result
+DROP MATERIALIZED VIEW test_mv1;
+-- result:
+-- !result
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+PROPERTIES (
+  "replication_num" = "1",
+  "query_rewrite_consistency" = "loose"
+)
+AS SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01';
+-- result:
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (1, "2024-11-18"), (1, "2024-11-23"), (4, "2024-12-25");
+-- result:
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+False
+-- !result
+function: print_hit_materialized_view("SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;
+-- result:
+1	2024-12-02
+1	2024-12-16
+2	2024-12-02
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;
+-- result:
+5	2024-12-11
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;
+-- result:
+1	2024-11-23
+1	2024-11-23
+1	2024-11-23
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;
+-- result:
+5	2024-11-24
+5	2024-12-11
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;
+-- result:
+1	2024-11-15
+1	2024-11-15
+1	2024-11-15
+-- !result
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;
+-- result:
+4	2024-11-21
+4	2024-11-24
+4	2024-12-08
+-- !result
+SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3)t order by 1, 2 limit 3;
+-- result:
+4	2024-11-21
+4	2024-11-21
+4	2024-11-24
+-- !result
+DROP MATERIALIZED VIEW test_mv1;
+-- result:
+-- !result
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+AS SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' GROUP BY dt;
+-- result:
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (4, "2024-12-25");
+-- result:
+-- !result
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  GROUP BY dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-12-01' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  GROUP BY dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-11-20' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM (SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 UNION ALL SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  GROUP BY dt order by 1, 2 limit 3;
+-- result:
+2024-12-02	3
+2024-12-05	5
+2024-12-08	7
+-- !result
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-12-01' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;
+-- result:
+2024-12-25	20
+-- !result
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  GROUP BY dt order by 1, 2 limit 3;
+-- result:
+2024-11-21	7
+2024-11-23	3
+2024-11-24	9
+-- !result
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-11-20' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;
+-- result:
+2024-12-25	20
+-- !result
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt order by 1, 2 limit 3;
+-- result:
+2024-11-15	7
+2024-11-18	8
+2024-11-21	7
+-- !result
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;
+-- result:
+2024-12-25	20
+-- !result
+SELECT * FROM (SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 UNION ALL SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt) t order by 1, 2 limit 3;
+-- result:
+2024-11-15	7
+2024-11-18	8
+2024-11-21	7
+-- !result
+DROP MATERIALIZED VIEW test_mv1;
+-- result:
+-- !result
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+AS SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt=t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt;
+-- result:
+-- !result
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (4, "2024-12-25");
+-- result:
+-- !result
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+function: print_hit_materialized_view("SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3", "test_mv1", "UNION")
+-- result:
+True
+-- !result
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt order by 1, 2 limit 3;
+-- result:
+2024-12-02	6
+2024-12-05	10
+2024-12-08	14
+-- !result
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;
+-- result:
+2024-12-08	14
+2024-12-11	18
+2024-12-25	24
+-- !result
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt order by 1, 2 limit 3;
+-- result:
+2024-11-21	14
+2024-11-24	18
+2024-12-02	6
+-- !result
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;
+-- result:
+2024-11-21	14
+2024-11-24	18
+2024-12-08	14
+-- !result
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;
+-- result:
+2024-11-15	21
+2024-11-18	25
+2024-11-21	14
+-- !result
+SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3;
+-- result:
+2024-11-21	14
+2024-11-21	14
+2024-11-24	18
+-- !result
+DROP MATERIALIZED VIEW test_mv1;
+-- result:
+-- !result
+drop table mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 force;
+-- result:
+-- !result
+drop table mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 force;
+-- result:
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_transparent_mv/T/test_transparent_mv_union_hive2 b/test/sql/test_transparent_mv/T/test_transparent_mv_union_hive2
new file mode 100644
index 00000000000000..f91bc981d73952
--- /dev/null
+++ b/test/sql/test_transparent_mv/T/test_transparent_mv_union_hive2
@@ -0,0 +1,175 @@
+-- name: test_transparent_mv_union_hive2
+create external catalog mv_hive_${uuid0}
+properties
+(
+    "type" = "hive",
+    "hive.catalog.type" = "hive",
+    "hive.metastore.uris" = "${hive_metastore_uris}"
+);
+
+set new_planner_optimize_timeout=10000;
+-- create hive table
+set catalog mv_hive_${uuid0};
+create database mv_hive_db_${uuid0};
+use mv_hive_db_${uuid0};
+
+CREATE TABLE mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 (
+  num int,
+  dt date
+)
+PARTITION BY (dt);
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES 
+  (1,"2024-11-15"),(2,"2024-11-18"),(3,"2024-11-21"),(4,"2024-11-24"),
+  (1,"2024-12-02"),(2,"2024-12-05"),(3,"2024-12-08"),(4,"2024-12-11"),
+  (1,"2024-12-16"),(2,"2024-12-19"),(3,"2024-12-22"),(4,"2024-12-25"),
+  (2,"2024-11-15"),(3,"2024-11-18"),(4,"2024-11-21"),(5,"2024-11-24"),
+  (2,"2024-12-02"),(3,"2024-12-05"),(4,"2024-12-08"),(5,"2024-12-11");
+
+CREATE TABLE mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 (
+  num int,
+  dt date
+)
+PARTITION BY (dt);
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 VALUES 
+  (1,"2024-11-15"),(2,"2024-11-18"),(3,"2024-11-21"),(4,"2024-11-24"),
+  (1,"2024-12-02"),(2,"2024-12-05"),(3,"2024-12-08"),(4,"2024-12-11"),
+  (1,"2024-12-16"),(2,"2024-12-19"),(3,"2024-12-22"),(4,"2024-12-25"),
+  (2,"2024-11-15"),(3,"2024-11-18"),(4,"2024-11-21"),(5,"2024-11-24"),
+  (2,"2024-12-02"),(3,"2024-12-05"),(4,"2024-12-08"),(5,"2024-12-11");
+
+-- create mv
+set catalog default_catalog;
+create database db_${uuid0};
+use db_${uuid0};
+-- set materialized_view_union_rewrite_mode=2;
+
+-- NOTE: test mv with the single table
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+PROPERTIES (
+  "replication_num" = "1",
+  "query_rewrite_consistency" = "checked"
+  -- "partition_ttl"="60 day"
+)
+AS SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1;
+
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (1, "2024-11-18"), (1, "2024-11-23"), (4, "2024-12-25");
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;
+SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3)t order by 1, 2 limit 3;
+
+-- NOTE: test mv with the single table
+DROP MATERIALIZED VIEW test_mv1;
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+PROPERTIES (
+  "replication_num" = "1",
+  "query_rewrite_consistency" = "loose"
+)
+AS SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1;
+
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (1, "2024-11-18"), (1, "2024-11-23"), (4, "2024-12-25");
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;
+SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3)t order by 1, 2 limit 3;
+
+-- NOTE: test mv with the single table
+DROP MATERIALIZED VIEW test_mv1;
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+PROPERTIES (
+  "replication_num" = "1",
+  "query_rewrite_consistency" = "loose"
+)
+AS SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01';
+
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (1, "2024-11-18"), (1, "2024-11-23"), (4, "2024-12-25");
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' and num > 4 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20' and num > 4 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 order by 1, 2 limit 3;
+SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 order by 1, 2 limit 3;
+SELECT * FROM (SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3 UNION ALL SELECT * FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where num > 3)t order by 1, 2 limit 3;
+
+DROP MATERIALIZED VIEW test_mv1;
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+AS SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01' GROUP BY dt;
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (4, "2024-12-25");
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  GROUP BY dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-12-01' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  GROUP BY dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-11-20' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM (SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 UNION ALL SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt) t order by 1, 2 limit 3;", "test_mv1", "UNION")
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-12-01'  GROUP BY dt order by 1, 2 limit 3;
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-12-01' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt > '2024-11-20'  GROUP BY dt order by 1, 2 limit 3;
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 where dt >'2024-11-20' GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt order by 1, 2 limit 3;
+SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 order by 1, 2 limit 3;
+SELECT * FROM (SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt having sum(num) > 10 UNION ALL SELECT dt, sum(num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 GROUP BY dt) t order by 1, 2 limit 3;
+
+DROP MATERIALIZED VIEW test_mv1;
+CREATE MATERIALIZED VIEW test_mv1 
+PARTITION BY dt 
+REFRESH DEFERRED MANUAL 
+AS SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt=t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt;
+REFRESH MATERIALIZED VIEW test_mv1 WITH SYNC MODE;
+INSERT INTO mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 VALUES (1, "2024-11-15"), (4, "2024-12-25");
+
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1")
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1")
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;", "test_mv1", "UNION")
+function: print_hit_materialized_view("SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3", "test_mv1", "UNION")
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt order by 1, 2 limit 3;
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-12-01' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt order by 1, 2 limit 3;
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;
+SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt GROUP BY t2.dt having sum(t2.num) > 10 order by 1, 2 limit 3;
+SELECT * FROM (SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10 UNION ALL SELECT t2.dt, sum(t2.num) as num FROM mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 join mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 on t1.dt = t2.dt where t2.dt > '2024-11-20' GROUP BY t2.dt having sum(t2.num) > 10) t order by 1, 2 limit 3;
+
+DROP MATERIALIZED VIEW test_mv1;
+drop table mv_hive_${uuid0}.mv_hive_db_${uuid0}.t1 force;
+drop table mv_hive_${uuid0}.mv_hive_db_${uuid0}.t2 force;
\ No newline at end of file

From 790bc80f64dd88a9b456c2ed708304a2efbb09dd Mon Sep 17 00:00:00 2001
From: stephen <91597003+stephen-shelby@users.noreply.github.com>
Date: Mon, 20 Jan 2025 15:00:41 +0800
Subject: [PATCH 49/71] [Enhancement] Avoid sample collecting table partitions
 that have not been updated (#55026)

Signed-off-by: stephen <stephen5217@163.com>
---
 .../StatisticsCollectJobFactory.java          | 22 ++++++++++++++++++-
 .../statistic/StatisticsCollectJobTest.java   | 17 ++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java
index 0337b6d1d278a4..9df08fb686ba82 100644
--- a/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java
+++ b/fe/fe-core/src/main/java/com/starrocks/statistic/StatisticsCollectJobFactory.java
@@ -35,6 +35,7 @@
 
 import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -533,7 +534,26 @@ private static void createJob(List<StatisticsCollectJob> allTableJobMap, NativeA
     private static void createSampleStatsJob(List<StatisticsCollectJob> allTableJobMap, NativeAnalyzeJob job,
                                              Database db, Table table, List<String> columnNames,
                                              List<Type> columnTypes) {
-        StatisticsCollectJob sample = buildStatisticsCollectJob(db, table, null, columnNames, columnTypes,
+        Collection<Partition> partitions = table.getPartitions();
+        BasicStatsMeta basicStatsMeta = GlobalStateMgr.getCurrentState().getAnalyzeMgr()
+                .getTableBasicStatsMeta(table.getId());
+        List<Long> partitionIdList;
+        if (basicStatsMeta != null) {
+            partitionIdList = partitions.stream()
+                    .filter(partition -> {
+                        LocalDateTime partitionUpdateTime = StatisticUtils.getPartitionLastUpdateTime(partition);
+                        return basicStatsMeta.getUpdateTime().isBefore(partitionUpdateTime) && partition.hasData();
+                    })
+                    .map(Partition::getId)
+                    .collect(Collectors.toList());
+        } else {
+            partitionIdList = partitions.stream()
+                    .filter(Partition::hasData)
+                    .map(Partition::getId)
+                    .collect(Collectors.toList());
+        }
+
+        StatisticsCollectJob sample = buildStatisticsCollectJob(db, table, partitionIdList, columnNames, columnTypes,
                 StatsConstants.AnalyzeType.SAMPLE, job.getScheduleType(), job.getProperties());
         allTableJobMap.add(sample);
     }
diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java
index 79cda7520a2e7c..afa47baa067d1f 100644
--- a/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/statistic/StatisticsCollectJobTest.java
@@ -394,6 +394,23 @@ public void testAnalyzeColumnSample2() {
                         StatsConstants.ScheduleStatus.PENDING,
                         LocalDateTime.MIN));
         Assert.assertEquals(1, jobs2.size());
+
+        BasicStatsMeta basicStatsMeta3 = new BasicStatsMeta(db.getId(), olapTable.getId(), null,
+                StatsConstants.AnalyzeType.SAMPLE,
+                LocalDateTime.of(2021, 1, 1, 1, 1, 1), Maps.newHashMap());
+        basicStatsMeta3.increaseDeltaRows(10000000L);
+        GlobalStateMgr.getCurrentState().getAnalyzeMgr().addBasicStatsMeta(basicStatsMeta3);
+
+        List<StatisticsCollectJob> job3 = StatisticsCollectJobFactory.buildStatisticsCollectJob(
+                new NativeAnalyzeJob(db.getId(), olapTable.getId(), Lists.newArrayList("v2"),
+                        Lists.newArrayList(Type.BIGINT),
+                        StatsConstants.AnalyzeType.SAMPLE, StatsConstants.ScheduleType.SCHEDULE,
+                        Maps.newHashMap(),
+                        StatsConstants.ScheduleStatus.PENDING,
+                        LocalDateTime.MIN));
+        Assert.assertEquals(1, job3.size());
+        Assert.assertTrue(job3.get(0) instanceof HyperStatisticsCollectJob);
+        Assert.assertTrue(job3.get(0).toString().contains("partitionIdList=[10010]"));
     }
 
     @Test

From 050730db007a2d99dcca88885dbefee62a09523c Mon Sep 17 00:00:00 2001
From: stephen <91597003+stephen-shelby@users.noreply.github.com>
Date: Mon, 20 Jan 2025 15:20:16 +0800
Subject: [PATCH 50/71] [BugFix] fix using incorrect read ratio when sampling
 to collect table statistics (#55198)

Signed-off-by: stephen <stephen5217@163.com>
---
 .../statistic/base/PartitionSampler.java      | 28 +++----------------
 .../statistic/hyper/HyperStatisticSQLs.java   | 13 ++++++---
 .../statistic/hyper/HyperJobTest.java         | 12 ++++++--
 3 files changed, 23 insertions(+), 30 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java b/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java
index dd7b216f63f1ec..682daecc832261 100644
--- a/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/statistic/base/PartitionSampler.java
@@ -27,10 +27,10 @@
 import java.util.Map;
 
 public class PartitionSampler {
-    private static final double HIGH_WEIGHT_READ_RATIO = 0.001;
-    private static final double MEDIUM_HIGH_WEIGHT_READ_RATIO = 0.01;
-    private static final double MEDIUM_LOW_WEIGHT_READ_RATIO = 0.1;
-    private static final double LOW_WEIGHT_READ_RATIO = 0.8;
+    public static final double HIGH_WEIGHT_READ_RATIO = 0.001;
+    public static final double MEDIUM_HIGH_WEIGHT_READ_RATIO = 0.01;
+    public static final double MEDIUM_LOW_WEIGHT_READ_RATIO = 0.1;
+    public static final double LOW_WEIGHT_READ_RATIO = 0.8;
     private static final long HIGH_WEIGHT_ROWS_THRESHOLD = 10000000L;
     private static final long MEDIUM_HIGH_WEIGHT_ROWS_THRESHOLD = 1000000L;
     private static final long MEDIUM_LOW_WEIGHT_ROWS_THRESHOLD = 100000L;
@@ -56,26 +56,6 @@ public PartitionSampler(double highSampleRatio, double mediumHighRatio, double m
         this.sampleRowsLimit = sampleRowLimit;
     }
 
-    public double getHighRatio() {
-        return highRatio;
-    }
-
-    public double getMediumHighRatio() {
-        return mediumHighRatio;
-    }
-
-    public double getMediumLowRatio() {
-        return mediumLowRatio;
-    }
-
-    public double getLowRatio() {
-        return lowRatio;
-    }
-
-    public int getMaxSize() {
-        return maxSize;
-    }
-
     public long getSampleRowsLimit() {
         return sampleRowsLimit;
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/statistic/hyper/HyperStatisticSQLs.java b/fe/fe-core/src/main/java/com/starrocks/statistic/hyper/HyperStatisticSQLs.java
index e9eb65cd2fd43d..7e6922b7a38942 100644
--- a/fe/fe-core/src/main/java/com/starrocks/statistic/hyper/HyperStatisticSQLs.java
+++ b/fe/fe-core/src/main/java/com/starrocks/statistic/hyper/HyperStatisticSQLs.java
@@ -32,6 +32,11 @@
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import static com.starrocks.statistic.base.PartitionSampler.HIGH_WEIGHT_READ_RATIO;
+import static com.starrocks.statistic.base.PartitionSampler.LOW_WEIGHT_READ_RATIO;
+import static com.starrocks.statistic.base.PartitionSampler.MEDIUM_HIGH_WEIGHT_READ_RATIO;
+import static com.starrocks.statistic.base.PartitionSampler.MEDIUM_LOW_WEIGHT_READ_RATIO;
+
 public class HyperStatisticSQLs {
     private static final VelocityEngine DEFAULT_VELOCITY_ENGINE;
 
@@ -131,13 +136,13 @@ public static String buildSampleSQL(Database db, Table table, Partition p, List<
         List<String> groupSQLs = Lists.newArrayList();
         StringBuilder sqlBuilder = new StringBuilder();
         groupSQLs.add(generateRatioTable(tableName, sampler.getSampleRowsLimit(), info.getHighWeightTablets(),
-                sampler.getHighRatio(), "t_high"));
+                HIGH_WEIGHT_READ_RATIO, "t_high"));
         groupSQLs.add(generateRatioTable(tableName, sampler.getSampleRowsLimit(), info.getMediumHighWeightTablets(),
-                sampler.getMediumHighRatio(), "t_medium_high"));
+                MEDIUM_HIGH_WEIGHT_READ_RATIO, "t_medium_high"));
         groupSQLs.add(generateRatioTable(tableName, sampler.getSampleRowsLimit(), info.getMediumLowWeightTablets(),
-                sampler.getMediumLowRatio(), "t_medium_low"));
+                MEDIUM_LOW_WEIGHT_READ_RATIO, "t_medium_low"));
         groupSQLs.add(generateRatioTable(tableName, sampler.getSampleRowsLimit(), info.getLowWeightTablets(),
-                sampler.getLowRatio(), "t_low"));
+                LOW_WEIGHT_READ_RATIO, "t_low"));
         if (groupSQLs.stream().allMatch(Objects::isNull)) {
             groupSQLs.add("SELECT * FROM " + tableName + " LIMIT " + Config.statistic_sample_collect_rows);
         }
diff --git a/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java b/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java
index 67231af387816a..4dcfa24467f1b2 100644
--- a/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/statistic/hyper/HyperJobTest.java
@@ -44,6 +44,7 @@
 import com.starrocks.statistic.base.PrimitiveTypeColumnStats;
 import com.starrocks.statistic.base.SubFieldColumnStats;
 import com.starrocks.statistic.base.TabletSampler;
+import com.starrocks.statistic.sample.SampleInfo;
 import com.starrocks.statistic.sample.TabletStats;
 import com.starrocks.utframe.StarRocksAssert;
 import mockit.Mock;
@@ -193,6 +194,13 @@ public void testFullJobs() {
     public void testSampleJobs() {
         Pair<List<String>, List<Type>> pair = initColumn(List.of("c1", "c2", "c3"));
 
+        new MockUp<SampleInfo>() {
+            @Mock
+            public List<TabletStats> getMediumHighWeightTablets() {
+                return List.of(new TabletStats(1, pid, 5000000));
+            }
+        };
+
         List<HyperQueryJob> jobs = HyperQueryJob.createSampleQueryJobs(connectContext, db, table, pair.first,
                 pair.second, List.of(pid), 1, sampler);
 
@@ -203,8 +211,8 @@ public void testSampleJobs() {
         List<String> sql = jobs.get(1).buildQuerySQL();
         Assert.assertEquals(1, sql.size());
 
-        assertContains(sql.get(0), "with base_cte_table as " +
-                "(SELECT * FROM `test`.`t_struct` LIMIT 200000) ");
+        assertContains(sql.get(0), "with base_cte_table as ( SELECT * FROM (SELECT *  FROM `test`.`t_struct` TABLET(1)" +
+                " SAMPLE('percent'='1')  LIMIT 200000)");
         assertContains(sql.get(0), "cast(IFNULL(SUM(CHAR_LENGTH(`c2`)) * 0/ COUNT(*), 0) as BIGINT), " +
                 "hex(hll_serialize(IFNULL(hll_raw(`c2`), hll_empty())))," +
                 " cast((COUNT(*) - COUNT(`c2`)) * 0 / COUNT(*) as BIGINT), " +

From 9eb9756b09cbbcbf0f6102528fd49dd829fd305c Mon Sep 17 00:00:00 2001
From: stdpain <34912776+stdpain@users.noreply.github.com>
Date: Mon, 20 Jan 2025 15:34:10 +0800
Subject: [PATCH 51/71] [BugFix] Fix reserve error with a large limit (#55241)

Signed-off-by: stdpain <drfeng08@gmail.com>
---
 be/src/exec/chunks_sorter_topn.cpp            |  4 ++--
 .../test_sort/R/test_topn_with_large_limit    | 20 +++++++++++++++++++
 .../test_sort/T/test_topn_with_large_limit    | 12 +++++++++++
 3 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100644 test/sql/test_sort/R/test_topn_with_large_limit
 create mode 100644 test/sql/test_sort/T/test_topn_with_large_limit

diff --git a/be/src/exec/chunks_sorter_topn.cpp b/be/src/exec/chunks_sorter_topn.cpp
index a2496e31ac5361..68015cf1304460 100644
--- a/be/src/exec/chunks_sorter_topn.cpp
+++ b/be/src/exec/chunks_sorter_topn.cpp
@@ -472,11 +472,11 @@ Status ChunksSorterTopn::_hybrid_sort_common(RuntimeState* state, std::pair<Perm
     // case2: rows_to_keep > 0, which means `SMALLER_THAN_MIN_OF_SEGMENT` part itself not suffice, we need to get more elements
     // from both `INCLUDE_IN_SEGMENT` part and _merged_segment. And notice that `INCLUDE_IN_SEGMENT` part may be empty
     if (rows_to_keep > 0) {
+        const size_t sorted_size = _merged_segment.chunk->num_rows();
+        rows_to_keep = std::min(rows_to_keep, sorted_size + second_size);
         if (big_chunk == nullptr) {
             big_chunk.reset(segments[new_permutation.second[0].chunk_index].chunk->clone_empty(rows_to_keep).release());
         }
-        const size_t sorted_size = _merged_segment.chunk->num_rows();
-        rows_to_keep = std::min(rows_to_keep, sorted_size + second_size);
         if (_topn_type == TTopNType::RANK && sorted_size + second_size > rows_to_keep) {
             // For rank type, there may exist a wide equal range, so we need to keep all elements of part2 and part3
             rows_to_keep = sorted_size + second_size;
diff --git a/test/sql/test_sort/R/test_topn_with_large_limit b/test/sql/test_sort/R/test_topn_with_large_limit
new file mode 100644
index 00000000000000..cec0aead7a07a8
--- /dev/null
+++ b/test/sql/test_sort/R/test_topn_with_large_limit
@@ -0,0 +1,20 @@
+-- name: test_topn_with_large_limit
+set pipeline_dop=1;
+-- result:
+-- !result
+create table t0 (
+    c0 INT,
+    c1 BIGINT
+) DUPLICATE KEY(c0) DISTRIBUTED BY HASH(c0) BUCKETS 1 PROPERTIES('replication_num' = '1');
+-- result:
+-- !result
+insert into t0 SELECT null, null FROM TABLE(generate_series(1,  65536));
+-- result:
+-- !result
+insert into t0 SELECT generate_series, generate_series FROM TABLE(generate_series(1,  257* 4096));
+-- result:
+-- !result
+select count(c0) from ( select c0 from t0 order by 1 asc nulls first limit 9223372036854775807 )t;
+-- result:
+1052672
+-- !result
\ No newline at end of file
diff --git a/test/sql/test_sort/T/test_topn_with_large_limit b/test/sql/test_sort/T/test_topn_with_large_limit
new file mode 100644
index 00000000000000..281fab67a44a34
--- /dev/null
+++ b/test/sql/test_sort/T/test_topn_with_large_limit
@@ -0,0 +1,12 @@
+-- name: test_topn_with_large_limit
+
+set pipeline_dop=1;
+create table t0 (
+    c0 INT,
+    c1 BIGINT
+) DUPLICATE KEY(c0) DISTRIBUTED BY HASH(c0) BUCKETS 1 PROPERTIES('replication_num' = '1');
+insert into t0 SELECT null, null FROM TABLE(generate_series(1,  65536));
+-- see ChunksSorterTopn::tunning_buffered_chunks
+insert into t0 SELECT generate_series, generate_series FROM TABLE(generate_series(1,  257* 4096));
+
+select count(c0) from ( select c0 from t0 order by 1 asc nulls first limit 9223372036854775807 )t;
\ No newline at end of file

From 21d2fc774f6be373845c2add0308054d2990f1de Mon Sep 17 00:00:00 2001
From: sfwang218 <wangshifa@58.com>
Date: Mon, 20 Jan 2025 16:05:49 +0800
Subject: [PATCH 52/71] [Feature] Add array function of array_flatten (#50080)

Signed-off-by: wangshifa <wangshifa@58.com>
---
 be/src/exprs/array_functions.cpp              | 68 +++++++++++++++++++
 be/src/exprs/array_functions.h                |  2 +
 be/test/exprs/array_functions_test.cpp        | 18 +++++
 .../array-functions/array_flatten.md          | 47 +++++++++++++
 .../array-functions/array_flatten.md          | 47 +++++++++++++
 .../com/starrocks/catalog/FunctionSet.java    |  1 +
 .../sql/analyzer/ExpressionAnalyzer.java      | 15 ++++
 .../starrocks/catalog/FunctionSetTest.java    | 25 +++++++
 .../sql/analyzer/AnalyzeArrayTest.java        |  8 +++
 gensrc/script/functions.py                    |  2 +
 test/sql/test_array/R/test_array              | 55 +++++++++++++++
 test/sql/test_array/T/test_array              | 16 ++++-
 12 files changed, 303 insertions(+), 1 deletion(-)
 create mode 100644 docs/en/sql-reference/sql-functions/array-functions/array_flatten.md
 create mode 100644 docs/zh/sql-reference/sql-functions/array-functions/array_flatten.md

diff --git a/be/src/exprs/array_functions.cpp b/be/src/exprs/array_functions.cpp
index e62fa5c4cdb06a..c025448bdf12cc 100644
--- a/be/src/exprs/array_functions.cpp
+++ b/be/src/exprs/array_functions.cpp
@@ -1723,4 +1723,72 @@ StatusOr<ColumnPtr> ArrayFunctions::repeat(FunctionContext* ctx, const Columns&
         return dest_column;
     }
 }
+
+StatusOr<ColumnPtr> ArrayFunctions::array_flatten(FunctionContext* ctx, const Columns& columns) {
+    DCHECK_EQ(1, columns.size());
+    RETURN_IF_COLUMNS_ONLY_NULL(columns);
+
+    size_t chunk_size = columns[0]->size();
+
+    // Helper function to init result array elements and offsets
+    auto build_result_array_elements_and_offsets = [](const ColumnPtr& elements_column, size_t offsets_size) {
+        DCHECK(elements_column->is_array());
+        auto [array_null, elements, offsets] = unpack_array_column(elements_column);
+        ColumnPtr result_elements = elements->clone_empty();
+        auto result_offsets = UInt32Column::create();
+        result_offsets->reserve(offsets_size);
+        result_offsets->append(0);
+        return std::make_pair(result_elements, result_offsets);
+    };
+
+    // Helper function to flatten a single array item
+    auto flatten_array_item = [](const Datum& v, ColumnPtr& result_elements, auto& result_offsets) {
+        if (!v.is_null()) {
+            const auto& items = v.get<DatumArray>();
+            for (const auto& item : items) {
+                if (!item.is_null()) {
+                    const auto& sub_items = item.get<DatumArray>();
+                    for (const auto& sub_item : sub_items) {
+                        result_elements->append_datum(sub_item);
+                    }
+                }
+            }
+        }
+        result_offsets->append(result_elements->size());
+    };
+
+    // Special handle const column
+    if (columns[0]->is_constant()) {
+        auto* const_column = down_cast<ConstColumn*>(columns[0].get());
+        ArrayColumn* const_array = down_cast<ArrayColumn*>(const_column->mutable_data_column()->get());
+
+        auto [result_elements, result_offsets] =
+                build_result_array_elements_and_offsets(const_array->elements_column(), 1);
+        Datum v = const_array->get(0);
+        flatten_array_item(v, result_elements, result_offsets);
+        return ConstColumn::create(ArrayColumn::create(result_elements, result_offsets), chunk_size);
+    }
+
+    const NullableColumn* src_nullable_column = nullptr;
+    ArrayColumn* array_column = nullptr;
+    if (columns[0]->is_nullable()) {
+        src_nullable_column = down_cast<const NullableColumn*>(columns[0].get());
+        array_column = down_cast<ArrayColumn*>(src_nullable_column->data_column().get());
+    } else {
+        array_column = down_cast<ArrayColumn*>(columns[0].get());
+    }
+
+    auto [result_elements, result_offsets] =
+            build_result_array_elements_and_offsets(array_column->elements_column(), array_column->offsets().size());
+    for (size_t i = 0; i < chunk_size; i++) {
+        Datum v = array_column->get(i);
+        flatten_array_item(v, result_elements, result_offsets);
+    }
+
+    auto result = ArrayColumn::create(result_elements, result_offsets);
+    if (src_nullable_column != nullptr) {
+        return NullableColumn::create(result, src_nullable_column->null_column());
+    }
+    return result;
+}
 } // namespace starrocks
diff --git a/be/src/exprs/array_functions.h b/be/src/exprs/array_functions.h
index e49a9f52e106a3..9c8712a59d0c21 100644
--- a/be/src/exprs/array_functions.h
+++ b/be/src/exprs/array_functions.h
@@ -196,6 +196,8 @@ class ArrayFunctions {
     DEFINE_VECTORIZED_FN(array_intersect_any_type);
 
     DEFINE_VECTORIZED_FN(array_sortby_multi);
+
+    DEFINE_VECTORIZED_FN(array_flatten);
 };
 
 } // namespace starrocks
diff --git a/be/test/exprs/array_functions_test.cpp b/be/test/exprs/array_functions_test.cpp
index e05f8f76a48310..19cb12f4d79a00 100644
--- a/be/test/exprs/array_functions_test.cpp
+++ b/be/test/exprs/array_functions_test.cpp
@@ -5966,4 +5966,22 @@ TEST_F(ArrayFunctionsTest, array_repeat_map) {
         }
     }
 }
+
+TEST_F(ArrayFunctionsTest, array_flatten_int) {
+    // array_flatten(NULL): NULL
+    // array_flatten([[1, 2], [1, 4]]): [1,2,1,4]
+    // array_flatten([[1, 2], [3]]): [1,2,3]
+    {
+        auto array = ColumnHelper::create_column(TYPE_ARRAY_ARRAY_INT, true);
+        array->append_nulls(1);
+        array->append_datum(DatumArray{DatumArray{1, 2}, DatumArray{1, 4}});
+        array->append_datum(DatumArray{DatumArray{1, 2}, DatumArray{3}});
+
+        auto result = ArrayFunctions::array_flatten(nullptr, {array}).value();
+        EXPECT_EQ(3, result->size());
+        EXPECT_TRUE(result->get(0).is_null());
+        EXPECT_EQ("[1,2,1,4]", result->debug_item(1));
+        EXPECT_EQ("[1,2,3]", result->debug_item(2));
+    }
+}
 } // namespace starrocks
diff --git a/docs/en/sql-reference/sql-functions/array-functions/array_flatten.md b/docs/en/sql-reference/sql-functions/array-functions/array_flatten.md
new file mode 100644
index 00000000000000..c729710dc220e1
--- /dev/null
+++ b/docs/en/sql-reference/sql-functions/array-functions/array_flatten.md
@@ -0,0 +1,47 @@
+---
+displayed_sidebar: docs
+---
+
+# array_flatten
+
+## Description
+
+Flatten one layer of nested arrays.
+
+## Syntax
+
+```Haskell
+array_flatten(param)
+```
+
+## Parameters
+
+`param`: a nested array that needs to be flattened. Only nested arrays are supported, and it can be a multi-level nested array, the elements of the array can be any data type supported by StarRocks.
+
+## Return value
+
+The data type of the return value is an array type after flattening one layer.
+
+## Examples
+
+Example 1: Flatten a 2-level nested array.
+
+```plaintext
+mysql> SELECT array_flatten([[1, 2], [1, 4]]) as res;
++-----------+
+| res       |
++-----------+
+| [1,2,1,4] |
++-----------+
+```
+
+Example 2: Flatten a 3-level nested array.
+
+```plaintext
+mysql> SELECT array_flatten([[[1],[2]], [[3],[4]]]) as res;
++-------------------+
+| res               |
++-------------------+
+| [[1],[2],[3],[4]] |
++-------------------+
+```
diff --git a/docs/zh/sql-reference/sql-functions/array-functions/array_flatten.md b/docs/zh/sql-reference/sql-functions/array-functions/array_flatten.md
new file mode 100644
index 00000000000000..dad9f501dd4e8e
--- /dev/null
+++ b/docs/zh/sql-reference/sql-functions/array-functions/array_flatten.md
@@ -0,0 +1,47 @@
+---
+displayed_sidebar: docs
+---
+
+# array_flatten
+
+## 功能
+
+将嵌套数组展平一层
+
+## 语法
+
+```Haskell
+array_flatten(param)
+```
+
+## 参数说明
+
+`param`：需要展平的嵌套数组，只支持嵌套数组，可以是多层嵌套数组，数组元素可以是StarRocks支持的所有数据类型。
+
+## 返回值说明
+
+返回值的数据类型为展平一层后的数组类型。
+
+## 示例
+
+**示例一：2层嵌套数组展平。**
+
+```plain text
+mysql> SELECT array_flatten([[1, 2], [1, 4]]) as res;
++-----------+
+| res       |
++-----------+
+| [1,2,1,4] |
++-----------+
+```
+
+**示例二：3层嵌套数组展平。**
+
+```plain text
+mysql> SELECT array_flatten([[[1],[2]], [[3],[4]]]) as res;
++-------------------+
+| res               |
++-------------------+
+| [[1],[2],[3],[4]] |
++-------------------+
+```
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java b/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java
index 6fbf0c0d87e8ef..e0b3e3d8274064 100644
--- a/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/com/starrocks/catalog/FunctionSet.java
@@ -357,6 +357,7 @@ public class FunctionSet {
     public static final String ARRAY_GENERATE = "array_generate";
 
     public static final String ARRAY_TO_BITMAP = "array_to_bitmap";
+    public static final String ARRAY_FLATTEN = "array_flatten";
 
     // Bit functions:
     public static final String BITAND = "bitand";
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java
index d7c29ee5f36d00..ae32154b488240 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ExpressionAnalyzer.java
@@ -1287,6 +1287,21 @@ private void checkFunction(String fnName, FunctionCallExpr node, Type[] argument
                     }
                     break;
                 }
+                case FunctionSet.ARRAY_FLATTEN: {
+                    if (node.getChildren().size() != 1) {
+                        throw new SemanticException(fnName + " should have only one input", node.getPos());
+                    }
+                    Type inputType = node.getChild(0).getType();
+                    if (!inputType.isArrayType() && !inputType.isNull()) {
+                        throw new SemanticException("The only one input of " + fnName +
+                                " should be an array of arrays, rather than " + inputType.toSql(), node.getPos());
+                    }
+                    if (inputType.isArrayType() && !((ArrayType) inputType).getItemType().isArrayType()) {
+                        throw new SemanticException("The only one input of " + fnName +
+                                " should be an array of arrays, rather than " + inputType.toSql(), node.getPos());
+                    }
+                    break;
+                }
             }
         }
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/FunctionSetTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/FunctionSetTest.java
index 8a1c8e5af14d6a..b5a36c3164966e 100644
--- a/fe/fe-core/src/test/java/com/starrocks/catalog/FunctionSetTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/catalog/FunctionSetTest.java
@@ -30,6 +30,7 @@ public class FunctionSetTest {
     private static final Type DOUBLE_ARRAY = new ArrayType(Type.DOUBLE);
     private static final Type INT_ARRAY_ARRAY = new ArrayType(INT_ARRAY);
     private static final Type TINYINT_ARRAY_ARRAY = new ArrayType(TINYINT_ARRAY);
+    private static final Type VARCHAR_ARRAY_ARRAY = new ArrayType(VARCHAR_ARRAY);
 
     @Before
     public void setUp() {
@@ -267,6 +268,30 @@ public void testPolymorphicFunction() {
         fn = functionSet.getFunction(desc, Function.CompareMode.IS_SUPERTYPE_OF);
         Assert.assertNotNull(fn);
         Assert.assertEquals(fn.functionId, 150216L);
+
+        // array_flatten(ARRAY<ARRAY<TINYINT>>)
+        argTypes = new Type[] {TINYINT_ARRAY_ARRAY};
+        desc = new Function(new FunctionName("array_flatten"), argTypes, Type.INVALID, false);
+        fn = functionSet.getFunction(desc, Function.CompareMode.IS_SUPERTYPE_OF);
+        Assert.assertNotNull(fn);
+        Assert.assertEquals(TINYINT_ARRAY, fn.getReturnType());
+        Assert.assertEquals(TINYINT_ARRAY_ARRAY, fn.getArgs()[0]);
+
+        // array_flatten(ARRAY<ARRAY<INT>>)
+        argTypes = new Type[] {INT_ARRAY_ARRAY};
+        desc = new Function(new FunctionName("array_flatten"), argTypes, Type.INVALID, false);
+        fn = functionSet.getFunction(desc, Function.CompareMode.IS_SUPERTYPE_OF);
+        Assert.assertNotNull(fn);
+        Assert.assertEquals(INT_ARRAY, fn.getReturnType());
+        Assert.assertEquals(INT_ARRAY_ARRAY, fn.getArgs()[0]);
+
+        // array_flatten(ARRAY<ARRAY<INT>>)
+        argTypes = new Type[] {VARCHAR_ARRAY_ARRAY};
+        desc = new Function(new FunctionName("array_flatten"), argTypes, Type.INVALID, false);
+        fn = functionSet.getFunction(desc, Function.CompareMode.IS_SUPERTYPE_OF);
+        Assert.assertNotNull(fn);
+        Assert.assertEquals(VARCHAR_ARRAY, fn.getReturnType());
+        Assert.assertEquals(VARCHAR_ARRAY_ARRAY, fn.getArgs()[0]);
     }
 
     @Test
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeArrayTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeArrayTest.java
index b74a687b07c620..5f61b99386d125 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeArrayTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/analyzer/AnalyzeArrayTest.java
@@ -74,4 +74,12 @@ public void testArrayConcat() {
         analyzeSuccess("select array_concat([1.0, 2.0, 3.0], ['2.00', '2.0'])");
         analyzeFail("select array_concat([1, 2, 3], [[1, 1], [2, 2]])");
     }
+
+    @Test
+    public void testArrayFlatten() {
+        analyzeFail("select array_flatten()");
+        analyzeFail("select array_flatten(1)");
+        analyzeFail("select array_flatten([1, 2, 3])");
+        analyzeSuccess("select array_flatten([[1, 2], [1, 4]])");
+    }
 }
diff --git a/gensrc/script/functions.py b/gensrc/script/functions.py
index 1887c1ecaa58dd..623f5f512519da 100644
--- a/gensrc/script/functions.py
+++ b/gensrc/script/functions.py
@@ -1307,6 +1307,8 @@
 
     [150340, 'array_repeat', True, False, 'ANY_ARRAY', ['ANY_ELEMENT', 'INT'], 'ArrayFunctions::repeat'],
 
+    [150345, 'array_flatten', True, False, 'ANY_ELEMENT', ['ANY_ARRAY'], 'ArrayFunctions::array_flatten'],
+
     # high-order functions related to lambda functions.
     [160100, 'array_map', True, False, 'ANY_ARRAY', ['FUNCTION', 'ANY_ARRAY', "..."], 'ArrayFunctions::array_map'],
 
diff --git a/test/sql/test_array/R/test_array b/test/sql/test_array/R/test_array
index cd409444681b7e..863265ef42628f 100644
--- a/test/sql/test_array/R/test_array
+++ b/test/sql/test_array/R/test_array
@@ -345,4 +345,59 @@ SELECT array_repeat(COLA, COLB) FROM repeat_test ORDER BY COLA;
 [None,None,None]
 [1,1,1]
 None
+-- !result
+
+-- name: test_array_flatten
+select array_flatten([[1, 2], [1, 4]]);
+-- result:
+[1,2,1,4]
+-- !result
+select array_flatten([[[1],[2]],[[3],[4]]]);
+-- result:
+[[1],[2],[3],[4]]
+-- !result
+CREATE TABLE IF NOT EXISTS flatten_test (COLA INT, COLB ARRAY<ARRAY<INT>>) PROPERTIES ("replication_num"="1");
+-- result:
+-- !result
+INSERT INTO flatten_test (COLA, COLB) VALUES (1, [[1, 2], [1, 4]]), (2, NULL), (3, [[5], [6, 7, 8], [9]]), (4, [[2, 3], [4, 5, 6], NULL]);
+-- result:
+-- !result
+SELECT array_flatten(COLB) FROM flatten_test ORDER BY COLA;
+-- result:
+[1,2,1,4]
+None
+[5,6,7,8,9]
+[2,3,4,5,6]
+-- !result
+SELECT array_slice(array_flatten(COLB), 1, 2) FROM flatten_test ORDER BY COLA;
+-- result:
+[1,2]
+None
+[5,6]
+[2,3]
+-- !result
+SELECT array_flatten(array_slice(COLB, 1, 2)) FROM flatten_test ORDER BY COLA;
+-- result:
+[1,2,1,4]
+None
+[5,6,7,8]
+[2,3,4,5,6]
+-- !result
+CREATE TABLE IF NOT EXISTS flatten_one_layer_arr_test (COLA INT, COLB ARRAY<INT>) PROPERTIES ("replication_num"="1");
+-- result:
+-- !result
+INSERT INTO flatten_one_layer_arr_test (COLA, COLB) VALUES (1, [1, 2, 3]);
+-- result:
+-- !result
+SELECT array_flatten(COLB) FROM flatten_one_layer_arr_test ORDER BY COLA;
+-- result:
+E: (1064, 'Getting analyzing error from line 1, column 7 to line 1, column 25. Detail message: The only one input of array_flatten should be an array of arrays, rather than array<int(11)>.')
+-- !result
+SELECT array_slice(array_flatten(COLB), 1, 2) FROM flatten_one_layer_arr_test ORDER BY COLA;
+-- result:
+E: (1064, 'Getting analyzing error from line 1, column 19 to line 1, column 37. Detail message: The only one input of array_flatten should be an array of arrays, rather than array<int(11)>.')
+-- !result
+SELECT array_flatten(array_slice(COLB, 1, 2)) FROM flatten_one_layer_arr_test ORDER BY COLA;
+-- result:
+E: (1064, 'Getting analyzing error from line 1, column 7 to line 1, column 44. Detail message: The only one input of array_flatten should be an array of arrays, rather than array<int(11)>.')
 -- !result
\ No newline at end of file
diff --git a/test/sql/test_array/T/test_array b/test/sql/test_array/T/test_array
index 312f6f572020e3..e3f61198b44b47 100644
--- a/test/sql/test_array/T/test_array
+++ b/test/sql/test_array/T/test_array
@@ -170,4 +170,18 @@ select array_repeat([1,2],3);
 select array_repeat(1,-1);
 CREATE TABLE IF NOT EXISTS repeat_test (COLA INT, COLB INT) PROPERTIES ("replication_num"="1");
 INSERT INTO repeat_test (COLA, COLB) VALUES (1, 3), (NULL, 3), (2, NULL);
-SELECT array_repeat(COLA, COLB) FROM repeat_test ORDER BY COLA;
\ No newline at end of file
+SELECT array_repeat(COLA, COLB) FROM repeat_test ORDER BY COLA;
+
+-- name: test_array_flatten
+select array_flatten([[1, 2], [1, 4]]);
+select array_flatten([[[1],[2]],[[3],[4]]]);
+CREATE TABLE IF NOT EXISTS flatten_test (COLA INT, COLB ARRAY<ARRAY<INT>>) PROPERTIES ("replication_num"="1");
+INSERT INTO flatten_test (COLA, COLB) VALUES (1, [[1, 2], [1, 4]]), (2, NULL), (3, [[5], [6, 7, 8], [9]]), (4, [[2, 3], [4, 5, 6], NULL]);
+SELECT array_flatten(COLB) FROM flatten_test ORDER BY COLA;
+SELECT array_slice(array_flatten(COLB), 1, 2) FROM flatten_test ORDER BY COLA;
+SELECT array_flatten(array_slice(COLB, 1, 2)) FROM flatten_test ORDER BY COLA;
+CREATE TABLE IF NOT EXISTS flatten_one_layer_arr_test (COLA INT, COLB ARRAY<INT>) PROPERTIES ("replication_num"="1");
+INSERT INTO flatten_one_layer_arr_test (COLA, COLB) VALUES (1, [1, 2, 3]);
+SELECT array_flatten(COLB) FROM flatten_one_layer_arr_test ORDER BY COLA;
+SELECT array_slice(array_flatten(COLB), 1, 2) FROM flatten_one_layer_arr_test ORDER BY COLA;
+SELECT array_flatten(array_slice(COLB, 1, 2)) FROM flatten_one_layer_arr_test ORDER BY COLA;
\ No newline at end of file

From fef49b5764b57b781b47960c747091c2748c49a0 Mon Sep 17 00:00:00 2001
From: andyziye <108652123+andyziye@users.noreply.github.com>
Date: Mon, 20 Jan 2025 16:43:44 +0800
Subject: [PATCH 53/71] [Tool] Fixed the CreateArtifact error in re-run
 workflow run(#55256)

Signed-off-by: andyziye <108652123+andyziye@users.noreply.github.com>
---
 .github/workflows/ci-pipeline-branch.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-pipeline-branch.yml b/.github/workflows/ci-pipeline-branch.yml
index 1692401dc1d89d..3d245977e3b432 100644
--- a/.github/workflows/ci-pipeline-branch.yml
+++ b/.github/workflows/ci-pipeline-branch.yml
@@ -504,7 +504,8 @@ jobs:
           name: head_sha
           path: ./head_sha.txt
           retention-days: 3
+          overwrite: true
 
       - name: Clean
         run: |
-          rm -rf ${{ github.workspace }}/*
\ No newline at end of file
+          rm -rf ${{ github.workspace }}/*

From 9af1024b993386195711003cb67b3f5cbce8062c Mon Sep 17 00:00:00 2001
From: kaijianding <kaijian.ding@gmail.com>
Date: Mon, 20 Jan 2025 17:46:54 +0800
Subject: [PATCH 54/71] [BugFix] should acquire db lock out of createLoadTask
 to avoid dead lock (#55219)

Signed-off-by: kaijian.ding <kaijian.ding@gmail.com>
---
 .../load/routineload/RoutineLoadJob.java      |  2 +-
 .../load/streamload/StreamLoadMgr.java        | 53 ++++++-------------
 2 files changed, 18 insertions(+), 37 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java
index 3ac757133ecf83..5583c35e533792 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/routineload/RoutineLoadJob.java
@@ -894,7 +894,7 @@ public TExecPlanFragmentParams plan(TUniqueId loadId, long txnId, String label)
             planParams.query_options.setLoad_job_type(TLoadJobType.ROUTINE_LOAD);
             StreamLoadMgr streamLoadManager = GlobalStateMgr.getCurrentState().getStreamLoadMgr();
 
-            StreamLoadTask streamLoadTask = streamLoadManager.createLoadTaskWithoutLock(db, table.getName(), label, "", "",
+            StreamLoadTask streamLoadTask = streamLoadManager.createLoadTaskWithoutLock(db, table, label, "", "",
                     taskTimeoutSecond * 1000, true, warehouseId);
             streamLoadTask.setTxnId(txnId);
             streamLoadTask.setLabel(label);
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java
index 05e8843f60f066..10b18489e49378 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/streamload/StreamLoadMgr.java
@@ -133,6 +133,7 @@ public void beginLoadTaskFromFrontend(String dbName, String tableName, String la
         } finally {
             readUnlock();
         }
+        Table table = checkMeta(db, tableName);
 
         boolean createTask = true;
 
@@ -144,7 +145,7 @@ public void beginLoadTaskFromFrontend(String dbName, String tableName, String la
                 task.beginTxnFromFrontend(channelId, channelNum, resp);
                 return;
             }
-            task = createLoadTask(db, tableName, label, user, clientIp, timeoutMillis, channelNum, channelId, warehouseId);
+            task = createLoadTaskWithoutLock(db, table, label, user, clientIp, timeoutMillis, channelNum, channelId, warehouseId);
             LOG.info(new LogBuilder(LogKey.STREAM_LOAD_TASK, task.getId())
                     .add("msg", "create load task").build());
             addLoadTask(task);
@@ -166,10 +167,12 @@ public void beginLoadTaskFromBackend(String dbName, String tableName, String lab
         StreamLoadTask task = null;
         Database db = checkDbName(dbName);
         long dbId = db.getId();
+        Table table = checkMeta(db, tableName);
 
         writeLock();
         try {
-            task = createLoadTask(db, tableName, label, user, clientIp, timeoutMillis, isRoutineLoad, warehouseId);
+            task = createLoadTaskWithoutLock(db, table, label, user, clientIp, timeoutMillis, isRoutineLoad,
+                    warehouseId);
             LOG.info(new LogBuilder(LogKey.STREAM_LOAD_TASK, task.getId())
                     .add("msg", "create load task").build());
 
@@ -180,20 +183,8 @@ public void beginLoadTaskFromBackend(String dbName, String tableName, String lab
         }
     }
 
-    // for sync stream load
-    public StreamLoadTask createLoadTask(Database db, String tableName, String label, String user, String clientIp,
-                                         long timeoutMillis, boolean isRoutineLoad, long warehouseId)
-            throws StarRocksException {
-        Table table;
-        Locker locker = new Locker();
-        locker.lockDatabase(db.getId(), LockType.READ);
-        try {
-            unprotectedCheckMeta(db, tableName);
-            table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName);
-        } finally {
-            locker.unLockDatabase(db.getId(), LockType.READ);
-        }
-
+    public StreamLoadTask createLoadTaskWithoutLock(Database db, Table table, String label, String user, String clientIp,
+                                                    long timeoutMillis, boolean isRoutineLoad, long warehouseId) {
         // init stream load task
         long id = GlobalStateMgr.getCurrentState().getNextId();
         StreamLoadTask streamLoadTask = new StreamLoadTask(id, db, (OlapTable) table,
@@ -201,38 +192,27 @@ public StreamLoadTask createLoadTask(Database db, String tableName, String label
         return streamLoadTask;
     }
 
-    public StreamLoadTask createLoadTaskWithoutLock(Database db, String tableName, String label, String user, String clientIp,
-                                                    long timeoutMillis, boolean isRoutineLoad, long warehouseId)
-            throws StarRocksException {
+    private StreamLoadTask createLoadTaskWithoutLock(Database db, Table table, String label, String user,
+                                                     String clientIp, long timeoutMillis, int channelNum,
+                                                     int channelId, long warehouseId) {
         // init stream load task
         long id = GlobalStateMgr.getCurrentState().getNextId();
-        StreamLoadTask streamLoadTask = new StreamLoadTask(id, db,
-                (OlapTable) GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName),
-                label, user, clientIp, timeoutMillis, System.currentTimeMillis(), isRoutineLoad, warehouseId);
+        StreamLoadTask streamLoadTask = new StreamLoadTask(id, db, (OlapTable) table,
+                label, user, clientIp, timeoutMillis, channelNum, channelId, System.currentTimeMillis(), warehouseId);
         return streamLoadTask;
     }
 
-    public StreamLoadTask createLoadTask(Database db, String tableName, String label, String user, String clientIp,
-                                         long timeoutMillis, int channelNum,
-                                         int channelId, long warehouseId) throws StarRocksException {
-        Table table;
+    private Table checkMeta(Database db, String tableName) throws StarRocksException {
         Locker locker = new Locker();
         locker.lockDatabase(db.getId(), LockType.READ);
         try {
-            unprotectedCheckMeta(db, tableName);
-            table = GlobalStateMgr.getCurrentState().getLocalMetastore().getTable(db.getFullName(), tableName);
+            return unprotectedCheckMeta(db, tableName);
         } finally {
             locker.unLockDatabase(db.getId(), LockType.READ);
         }
-
-        // init stream load task
-        long id = GlobalStateMgr.getCurrentState().getNextId();
-        StreamLoadTask streamLoadTask = new StreamLoadTask(id, db, (OlapTable) table,
-                label, user, clientIp, timeoutMillis, channelNum, channelId, System.currentTimeMillis(), warehouseId);
-        return streamLoadTask;
     }
 
-    public void unprotectedCheckMeta(Database db, String tblName)
+    private Table unprotectedCheckMeta(Database db, String tblName)
             throws StarRocksException {
         if (tblName == null) {
             throw new AnalysisException("Table name must be specified when calling /begin/transaction/ first time");
@@ -253,6 +233,7 @@ public void unprotectedCheckMeta(Database db, String tblName)
         if (!table.isOlapOrCloudNativeTable()) {
             throw new AnalysisException("Only olap/lake table support stream load");
         }
+        return table;
     }
 
     public void replayCreateLoadTask(StreamLoadTask loadJob) {
@@ -262,7 +243,7 @@ public void replayCreateLoadTask(StreamLoadTask loadJob) {
                 .build());
     }
 
-    public Database checkDbName(String dbName) throws StarRocksException {
+    private Database checkDbName(String dbName) throws StarRocksException {
         Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbName);
         if (db == null) {
             LOG.warn("Database {} does not exist", dbName);

From e8ef907e53d39fab2170e200042a11b4530ad9f7 Mon Sep 17 00:00:00 2001
From: Youngwb <yangwenbo_mailbox@163.com>
Date: Mon, 20 Jan 2025 23:10:01 +0800
Subject: [PATCH 55/71] [BugFix] parquet writer do not perform time zone
 adjustments when writing timestamp types. (#55194)

Signed-off-by: Youngwb <yangwenbo_mailbox@163.com>
---
 be/src/formats/parquet/file_writer.cpp        |  3 ++-
 be/src/formats/parquet/level_builder.cpp      |  5 +++--
 .../formats/parquet/parquet_file_writer.cpp   |  2 +-
 .../R/test_iceberg_catalog_timestamp          | 20 +++++++++++++++++--
 .../T/test_iceberg_catalog_timestamp          | 11 ++++++++--
 5 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/be/src/formats/parquet/file_writer.cpp b/be/src/formats/parquet/file_writer.cpp
index 950e0606614c39..fa2a55f96afedf 100644
--- a/be/src/formats/parquet/file_writer.cpp
+++ b/be/src/formats/parquet/file_writer.cpp
@@ -298,7 +298,8 @@ arrow::Result<::parquet::schema::NodePtr> ParquetBuildHelper::_make_schema_node(
     }
     case TYPE_DATETIME: {
         return ::parquet::schema::PrimitiveNode::Make(
-                name, rep_type, ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::unit::MICROS),
+                name, rep_type,
+                ::parquet::LogicalType::Timestamp(false, ::parquet::LogicalType::TimeUnit::unit::MICROS),
                 ::parquet::Type::INT64, -1, file_column_id.field_id);
     }
     case TYPE_DECIMAL32: {
diff --git a/be/src/formats/parquet/level_builder.cpp b/be/src/formats/parquet/level_builder.cpp
index b7b82e992c5a7a..27de8577dc39ce 100644
--- a/be/src/formats/parquet/level_builder.cpp
+++ b/be/src/formats/parquet/level_builder.cpp
@@ -357,8 +357,9 @@ Status LevelBuilder::_write_datetime_column_chunk(const LevelBuilderContext& ctx
     DeferOp defer([&] { delete[] values; });
 
     for (size_t i = 0; i < col->size(); i++) {
-        // normalize to utc
-        auto timestamp = timestamp::sub<TimeUnit::SECOND>(data_col[i]._timestamp, _offset);
+        auto timestamp = use_int96_timestamp_encoding
+                                 ? timestamp::sub<TimeUnit::SECOND>(data_col[i]._timestamp, _offset)
+                                 : data_col[i]._timestamp;
         if constexpr (use_int96_timestamp_encoding) {
             auto date = reinterpret_cast<int32_t*>(values[i].value + 2);
             auto nanosecond = reinterpret_cast<int64_t*>(values[i].value);
diff --git a/be/src/formats/parquet/parquet_file_writer.cpp b/be/src/formats/parquet/parquet_file_writer.cpp
index ebebcf12d72c27..4c65405d61ce05 100644
--- a/be/src/formats/parquet/parquet_file_writer.cpp
+++ b/be/src/formats/parquet/parquet_file_writer.cpp
@@ -338,7 +338,7 @@ arrow::Result<::parquet::schema::NodePtr> ParquetFileWriter::_make_schema_node(c
         } else {
             return ::parquet::schema::PrimitiveNode::Make(
                     name, rep_type,
-                    ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::unit::MICROS),
+                    ::parquet::LogicalType::Timestamp(false, ::parquet::LogicalType::TimeUnit::unit::MICROS),
                     ::parquet::Type::INT64, -1, file_column_id.field_id);
         }
     }
diff --git a/test/sql/test_iceberg/R/test_iceberg_catalog_timestamp b/test/sql/test_iceberg/R/test_iceberg_catalog_timestamp
index 2941d6cc592832..bf3e4a0f8dd471 100644
--- a/test/sql/test_iceberg/R/test_iceberg_catalog_timestamp
+++ b/test/sql/test_iceberg/R/test_iceberg_catalog_timestamp
@@ -10,8 +10,8 @@ set catalog ice_cat_${uuid0};
 create database ice_db_${uuid0};
 use ice_db_${uuid0};
 create table ice_tbl_${uuid0} (
-  col_str int,
-  col_int datetime
+  col_int int,
+  col_datetime datetime
 );
 insert into ice_tbl_${uuid0} values (1, '2024-01-29 01:00:00'),(2, '2024-01-30 20:10:00'),(3,null);
 -- result:
@@ -22,6 +22,22 @@ select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0};
 2	2024-01-30 20:10:00
 3	None
 -- !result
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = '2024-01-29 01:00:00';
+-- result:
+1	2024-01-29 01:00:00
+-- !result
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = cast('2024-01-29 01:00:00' as datetime);
+-- result:
+1	2024-01-29 01:00:00
+-- !result
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = cast('2024-01-29 01:00:00.000' as datetime);
+-- result:
+1	2024-01-29 01:00:00
+-- !result
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = cast('2024-01-29 01:00:00:000' as datetime);
+-- result:
+1	2024-01-29 01:00:00
+-- !result
 drop table ice_tbl_${uuid0} force;
 drop database ice_db_${uuid0};
 drop catalog ice_cat_${uuid0};
diff --git a/test/sql/test_iceberg/T/test_iceberg_catalog_timestamp b/test/sql/test_iceberg/T/test_iceberg_catalog_timestamp
index 8ed87be2a97ac4..5d020e3ce58b88 100644
--- a/test/sql/test_iceberg/T/test_iceberg_catalog_timestamp
+++ b/test/sql/test_iceberg/T/test_iceberg_catalog_timestamp
@@ -10,13 +10,20 @@ set catalog ice_cat_${uuid0};
 create database ice_db_${uuid0};
 use ice_db_${uuid0};
 create table ice_tbl_${uuid0} (
-  col_str int,
-  col_int datetime
+  col_int int,
+  col_datetime datetime
 );
 insert into ice_tbl_${uuid0} values (1, '2024-01-29 01:00:00'),(2, '2024-01-30 20:10:00'),(3,null);
 
 select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0};
 
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = '2024-01-29 01:00:00';
+
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = cast('2024-01-29 01:00:00' as datetime);
+
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = cast('2024-01-29 01:00:00.000' as datetime);
+
+select * from ice_cat_${uuid0}.ice_db_${uuid0}.ice_tbl_${uuid0} where col_datetime = cast('2024-01-29 01:00:00:000' as datetime);
 drop table ice_tbl_${uuid0} force;
 drop database ice_db_${uuid0};
 drop catalog ice_cat_${uuid0};
\ No newline at end of file

From 1f7391729e4931420653acbb56088f01b5ac074a Mon Sep 17 00:00:00 2001
From: zombee0 <ewang2027@gmail.com>
Date: Mon, 20 Jan 2025 23:10:14 +0800
Subject: [PATCH 56/71] [Enhancement]parquet reader supports low cardinality
 optimization (#55167)

Signed-off-by: zombee0 <ewang2027@gmail.com>
---
 be/src/column/nullable_column.h               |   6 +
 be/src/connector/hive_connector.cpp           |  33 +++
 be/src/connector/hive_connector.h             |   1 +
 be/src/exec/hdfs_scanner.cpp                  |   5 +-
 be/src/exec/hdfs_scanner.h                    |   4 +
 be/src/exprs/runtime_filter_bank.cpp          |   1 +
 .../formats/parquet/column_reader_factory.cpp |  24 ++
 .../formats/parquet/column_reader_factory.h   |   3 +
 .../formats/parquet/complex_column_reader.cpp |  30 +-
 .../formats/parquet/complex_column_reader.h   |   4 +
 be/src/formats/parquet/file_reader.cpp        |   5 +-
 be/src/formats/parquet/group_reader.cpp       |   6 +
 be/src/formats/parquet/group_reader.h         |   2 +
 .../formats/parquet/scalar_column_reader.cpp  | 181 +++++++++++-
 be/src/formats/parquet/scalar_column_reader.h |  87 ++++++
 be/test/formats/parquet/file_reader_test.cpp  | 274 ++++++++++++++++++
 be/test/formats/parquet/parquet_ut_base.cpp   |  44 ++-
 be/test/formats/parquet/parquet_ut_base.h     |   3 +
 .../test_data/low_rows_non_dict.parquet       | Bin 0 -> 6525 bytes
 19 files changed, 701 insertions(+), 12 deletions(-)
 create mode 100644 be/test/formats/parquet/test_data/low_rows_non_dict.parquet

diff --git a/be/src/column/nullable_column.h b/be/src/column/nullable_column.h
index e37b99ee86c8f4..a542434f0bec1f 100644
--- a/be/src/column/nullable_column.h
+++ b/be/src/column/nullable_column.h
@@ -284,6 +284,12 @@ class NullableColumn : public ColumnFactory<Column, NullableColumn> {
         update_has_null();
     }
 
+    void swap_null_column(Column& rhs) {
+        auto& r = down_cast<NullableColumn&>(rhs);
+        _null_column->swap_column(*r._null_column);
+        std::swap(_has_null, r._has_null);
+    }
+
     void reset_column() override {
         Column::reset_column();
         _data_column->reset_column();
diff --git a/be/src/connector/hive_connector.cpp b/be/src/connector/hive_connector.cpp
index 330ac514006941..e2f682d25bd247 100644
--- a/be/src/connector/hive_connector.cpp
+++ b/be/src/connector/hive_connector.cpp
@@ -444,6 +444,8 @@ Status HiveDataSource::_decompose_conjunct_ctxs(RuntimeState* state) {
             _conjunct_ctxs_by_slot[slot_id].emplace_back(ctx);
         }
     }
+    // rewrite dict
+    RETURN_IF_ERROR(state->mutable_dict_optimize_parser()->rewrite_conjuncts(&_scanner_conjunct_ctxs));
     return Status::OK();
 }
 
@@ -580,6 +582,36 @@ void HiveDataSource::_init_rf_counters() {
     }
 }
 
+Status HiveDataSource::_init_global_dicts(HdfsScannerParams* params) {
+    const THdfsScanNode& hdfs_scan_node = _provider->_hdfs_scan_node;
+    const auto& global_dict_map = _runtime_state->get_query_global_dict_map();
+    auto global_dict = _pool.add(new ColumnIdToGlobalDictMap());
+    // mapping column id to storage column ids
+    TupleDescriptor* tuple_desc = _runtime_state->desc_tbl().get_tuple_descriptor(hdfs_scan_node.tuple_id);
+    DictOptimizeParser::rewrite_descriptor(_runtime_state, {}, {}, &tuple_desc->decoded_slots());
+    for (auto slot : tuple_desc->slots()) {
+        if (!slot->is_materialized()) {
+            continue;
+        }
+        auto iter = global_dict_map.find(slot->id());
+        if (iter != global_dict_map.end()) {
+            auto& dict_map = iter->second.first;
+            global_dict->emplace(slot->id(), const_cast<GlobalDictMap*>(&dict_map));
+#ifdef DEBUG
+            std::stringstream ss;
+            ss << "slot_id: " << slot->id() << " global dict: ";
+            for (const auto& kv : dict_map) {
+                ss << "<" << kv.first << " " << kv.second << ">"
+                   << ", ";
+            }
+            LOG(INFO) << ss.str();
+#endif
+        }
+    }
+    params->global_dictmaps = global_dict;
+    return Status::OK();
+}
+
 Status HiveDataSource::_init_scanner(RuntimeState* state) {
     SCOPED_TIMER(_profile.open_file_timer);
 
@@ -609,6 +641,7 @@ Status HiveDataSource::_init_scanner(RuntimeState* state) {
     ASSIGN_OR_RETURN(auto fs, FileSystem::CreateUniqueFromString(native_file_path, fsOptions));
 
     HdfsScannerParams scanner_params;
+    RETURN_IF_ERROR(_init_global_dicts(&scanner_params));
     scanner_params.runtime_filter_collector = _runtime_filters;
     scanner_params.scan_range = &scan_range;
     scanner_params.fs = _pool.add(fs.release());
diff --git a/be/src/connector/hive_connector.h b/be/src/connector/hive_connector.h
index 50f097434fd8c4..952dc5566a3b8f 100644
--- a/be/src/connector/hive_connector.h
+++ b/be/src/connector/hive_connector.h
@@ -99,6 +99,7 @@ class HiveDataSource final : public DataSource {
 
     Status _init_partition_values();
     Status _init_extended_values();
+    Status _init_global_dicts(HdfsScannerParams* params);
     Status _init_scanner(RuntimeState* state);
     HdfsScanner* _create_hudi_jni_scanner(const FSOptions& options);
     HdfsScanner* _create_paimon_jni_scanner(const FSOptions& options);
diff --git a/be/src/exec/hdfs_scanner.cpp b/be/src/exec/hdfs_scanner.cpp
index 31e630baae1b44..30db96050e70f9 100644
--- a/be/src/exec/hdfs_scanner.cpp
+++ b/be/src/exec/hdfs_scanner.cpp
@@ -23,6 +23,7 @@
 #include "io/compressed_input_stream.h"
 #include "io/shared_buffered_input_stream.h"
 #include "pipeline/fragment_context.h"
+#include "runtime/global_dict/parser.h"
 #include "storage/predicate_parser.h"
 #include "storage/runtime_range_pruner.hpp"
 #include "util/compression/compression_utils.h"
@@ -172,6 +173,7 @@ Status HdfsScanner::_build_scanner_context() {
     ctx.split_context = _scanner_params.split_context;
     ctx.enable_split_tasks = _scanner_params.enable_split_tasks;
     ctx.connector_max_split_size = _scanner_params.connector_max_split_size;
+    ctx.global_dictmaps = _scanner_params.global_dictmaps;
 
     if (config::parquet_advance_zonemap_filter) {
         ScanConjunctsManagerOptions opts;
@@ -185,7 +187,8 @@ Status HdfsScanner::_build_scanner_context() {
         opts.pred_tree_params = _runtime_state->fragment_ctx()->pred_tree_params();
         ctx.conjuncts_manager = std::make_unique<ScanConjunctsManager>(std::move(opts));
         RETURN_IF_ERROR(ctx.conjuncts_manager->parse_conjuncts());
-        auto* predicate_parser = opts.obj_pool->add(new ConnectorPredicateParser(&ctx.slot_descs));
+        auto* predicate_parser =
+                opts.obj_pool->add(new ConnectorPredicateParser(&_scanner_params.tuple_desc->decoded_slots()));
         ASSIGN_OR_RETURN(ctx.predicate_tree,
                          ctx.conjuncts_manager->get_predicate_tree(predicate_parser, ctx.predicate_free_pool));
         ctx.rf_scan_range_pruner = opts.obj_pool->add(
diff --git a/be/src/exec/hdfs_scanner.h b/be/src/exec/hdfs_scanner.h
index dc0bcca90ee2da..cd564e735748e1 100644
--- a/be/src/exec/hdfs_scanner.h
+++ b/be/src/exec/hdfs_scanner.h
@@ -244,6 +244,8 @@ struct HdfsScannerParams {
     bool orc_use_column_names = false;
 
     int64_t connector_max_split_size = 0;
+
+    ColumnIdToGlobalDictMap* global_dictmaps = &EMPTY_GLOBAL_DICTMAPS;
 };
 
 struct HdfsScannerContext {
@@ -365,6 +367,8 @@ struct HdfsScannerContext {
     std::unique_ptr<ScanConjunctsManager> conjuncts_manager = nullptr;
     std::vector<std::unique_ptr<ColumnPredicate>> predicate_free_pool;
     PredicateTree predicate_tree;
+
+    ColumnIdToGlobalDictMap* global_dictmaps = &EMPTY_GLOBAL_DICTMAPS;
 };
 
 struct OpenFileOptions {
diff --git a/be/src/exprs/runtime_filter_bank.cpp b/be/src/exprs/runtime_filter_bank.cpp
index 2c5c21301bcdf9..4f04bd0d387f37 100644
--- a/be/src/exprs/runtime_filter_bank.cpp
+++ b/be/src/exprs/runtime_filter_bank.cpp
@@ -778,6 +778,7 @@ void RuntimeFilterHelper::create_min_max_value_predicate(ObjectPool* pool, SlotI
                                                          const JoinRuntimeFilter* filter, Expr** min_max_predicate) {
     *min_max_predicate = nullptr;
     if (filter == nullptr) return;
+    // TODO, if you want to enable it for string, pls adapt for low-cardinality string
     if (slot_type == TYPE_CHAR || slot_type == TYPE_VARCHAR) return;
     auto res = type_dispatch_filter(slot_type, (Expr*)nullptr, MinMaxPredicateBuilder(pool, slot_id, filter));
     *min_max_predicate = res;
diff --git a/be/src/formats/parquet/column_reader_factory.cpp b/be/src/formats/parquet/column_reader_factory.cpp
index 1f5be92d6c1574..9a9c72be66593c 100644
--- a/be/src/formats/parquet/column_reader_factory.cpp
+++ b/be/src/formats/parquet/column_reader_factory.cpp
@@ -156,6 +156,30 @@ StatusOr<ColumnReaderPtr> ColumnReaderFactory::create(const ColumnReaderOptions&
     }
 }
 
+StatusOr<ColumnReaderPtr> ColumnReaderFactory::create(ColumnReaderPtr ori_reader, const GlobalDictMap* dict,
+                                                      SlotId slot_id, int64_t num_rows) {
+    if (ori_reader->get_column_parquet_field()->type == ColumnType::ARRAY) {
+        ASSIGN_OR_RETURN(ColumnReaderPtr child_reader,
+                         ColumnReaderFactory::create(
+                                 std::move((down_cast<ListColumnReader*>(ori_reader.get()))->get_element_reader()),
+                                 dict, slot_id, num_rows));
+        return std::make_unique<ListColumnReader>(ori_reader->get_column_parquet_field(), std::move(child_reader));
+    } else {
+        RawColumnReader* raw_reader = dynamic_cast<RawColumnReader*>(ori_reader.get());
+        if (raw_reader == nullptr) {
+            return Status::InternalError("Error on reader transform for low cardinality reader");
+        }
+        if (raw_reader->column_all_pages_dict_encoded()) {
+            return std::make_unique<LowCardColumnReader>(*raw_reader, dict, slot_id);
+        } else if (num_rows <= DICT_DECODE_MAX_SIZE) {
+            return std::make_unique<LowRowsColumnReader>(*raw_reader, dict, slot_id);
+        } else {
+            return Status::GlobalDictNotMatch(
+                    fmt::format("SlotId: {}, Not dict encoded and not low rows on global dict column. ", slot_id));
+        }
+    }
+}
+
 void ColumnReaderFactory::get_subfield_pos_with_pruned_type(const ParquetField& field, const TypeDescriptor& col_type,
                                                             bool case_sensitive, std::vector<int32_t>& pos) {
     DCHECK(field.type == ColumnType::STRUCT);
diff --git a/be/src/formats/parquet/column_reader_factory.h b/be/src/formats/parquet/column_reader_factory.h
index 376cad971f5787..9bc15001cfa85f 100644
--- a/be/src/formats/parquet/column_reader_factory.h
+++ b/be/src/formats/parquet/column_reader_factory.h
@@ -28,6 +28,9 @@ class ColumnReaderFactory {
                                             const TypeDescriptor& col_type,
                                             const TIcebergSchemaField* iceberg_schema_field);
 
+    static StatusOr<ColumnReaderPtr> create(ColumnReaderPtr raw_reader, const GlobalDictMap* dict, const SlotId slot_id,
+                                            int64_t num_rows);
+
 private:
     // for struct type without schema change
     static void get_subfield_pos_with_pruned_type(const ParquetField& field, const TypeDescriptor& col_type,
diff --git a/be/src/formats/parquet/complex_column_reader.cpp b/be/src/formats/parquet/complex_column_reader.cpp
index 60890816f250e7..35a80bbd213c65 100644
--- a/be/src/formats/parquet/complex_column_reader.cpp
+++ b/be/src/formats/parquet/complex_column_reader.cpp
@@ -105,6 +105,30 @@ Status ListColumnReader::read_range(const Range<uint64_t>& range, const Filter*
     return Status::OK();
 }
 
+Status ListColumnReader::fill_dst_column(ColumnPtr& dst, ColumnPtr& src) {
+    ArrayColumn* array_column_src = nullptr;
+    ArrayColumn* array_column_dst = nullptr;
+    if (src->is_nullable()) {
+        NullableColumn* nullable_column_src = down_cast<NullableColumn*>(src.get());
+        DCHECK(nullable_column_src->mutable_data_column()->is_array());
+        array_column_src = down_cast<ArrayColumn*>(nullable_column_src->mutable_data_column());
+        NullableColumn* nullable_column_dst = down_cast<NullableColumn*>(dst.get());
+        DCHECK(nullable_column_dst->mutable_data_column()->is_array());
+        array_column_dst = down_cast<ArrayColumn*>(nullable_column_dst->mutable_data_column());
+        nullable_column_dst->swap_null_column(*nullable_column_src);
+    } else {
+        DCHECK(src->is_array());
+        DCHECK(dst->is_array());
+        DCHECK(!get_column_parquet_field()->is_nullable);
+        array_column_src = down_cast<ArrayColumn*>(src.get());
+        array_column_dst = down_cast<ArrayColumn*>(dst.get());
+    }
+    array_column_dst->offsets_column()->swap_column(*(array_column_src->offsets_column()));
+    RETURN_IF_ERROR(
+            _element_reader->fill_dst_column(array_column_dst->elements_column(), array_column_src->elements_column()));
+    return Status::OK();
+}
+
 Status MapColumnReader::read_range(const Range<uint64_t>& range, const Filter* filter, ColumnPtr& dst) {
     NullableColumn* nullable_column = nullptr;
     MapColumn* map_column = nullptr;
@@ -273,14 +297,10 @@ Status StructColumnReader::fill_dst_column(ColumnPtr& dst, ColumnPtr& src) {
         NullableColumn* nullable_column_src = down_cast<NullableColumn*>(src.get());
         DCHECK(nullable_column_src->mutable_data_column()->is_struct());
         struct_column_src = down_cast<StructColumn*>(nullable_column_src->mutable_data_column());
-        NullColumn* null_column_src = nullable_column_src->mutable_null_column();
         NullableColumn* nullable_column_dst = down_cast<NullableColumn*>(dst.get());
         DCHECK(nullable_column_dst->mutable_data_column()->is_struct());
         struct_column_dst = down_cast<StructColumn*>(nullable_column_dst->mutable_data_column());
-        NullColumn* null_column_dst = nullable_column_dst->mutable_null_column();
-        null_column_dst->swap_column(*null_column_src);
-        nullable_column_src->update_has_null();
-        nullable_column_dst->update_has_null();
+        nullable_column_dst->swap_null_column(*nullable_column_src);
     } else {
         DCHECK(src->is_struct());
         DCHECK(dst->is_struct());
diff --git a/be/src/formats/parquet/complex_column_reader.h b/be/src/formats/parquet/complex_column_reader.h
index 0e65f70efd1f18..c3b9bdce4156a1 100644
--- a/be/src/formats/parquet/complex_column_reader.h
+++ b/be/src/formats/parquet/complex_column_reader.h
@@ -28,6 +28,8 @@ class ListColumnReader final : public ColumnReader {
 
     Status read_range(const Range<uint64_t>& range, const Filter* filter, ColumnPtr& dst) override;
 
+    Status fill_dst_column(ColumnPtr& dst, ColumnPtr& src) override;
+
     void get_levels(level_t** def_levels, level_t** rep_levels, size_t* num_levels) override {
         _element_reader->get_levels(def_levels, rep_levels, num_levels);
     }
@@ -45,6 +47,8 @@ class ListColumnReader final : public ColumnReader {
         _element_reader->select_offset_index(range, rg_first_row);
     }
 
+    ColumnReaderPtr& get_element_reader() { return _element_reader; }
+
 private:
     std::unique_ptr<ColumnReader> _element_reader;
 };
diff --git a/be/src/formats/parquet/file_reader.cpp b/be/src/formats/parquet/file_reader.cpp
index 38b7072ac5c1de..d19dffba9fc2bf 100644
--- a/be/src/formats/parquet/file_reader.cpp
+++ b/be/src/formats/parquet/file_reader.cpp
@@ -384,7 +384,7 @@ StatusOr<bool> FileReader::_update_rf_and_filter_group(const GroupReaderPtr& gro
     bool filter = false;
     if (config::parquet_advance_zonemap_filter && _rf_scan_range_pruner != nullptr) {
         RETURN_IF_ERROR(_rf_scan_range_pruner->update_range_if_arrived(
-                &EMPTY_GLOBAL_DICTMAPS,
+                _scanner_ctx->global_dictmaps,
                 [&filter, &group_reader](auto cid, const PredicateList& predicates) {
                     PredicateCompoundNode<CompoundNodeType::AND> pred_tree;
                     for (const auto& pred : predicates) {
@@ -550,6 +550,7 @@ Status FileReader::_init_group_readers() {
     // for pageIndex
     _group_reader_param.min_max_conjunct_ctxs = fd_scanner_ctx.min_max_conjunct_ctxs;
     _group_reader_param.predicate_tree = &fd_scanner_ctx.predicate_tree;
+    _group_reader_param.global_dictmaps = fd_scanner_ctx.global_dictmaps;
 
     int64_t row_group_first_row = 0;
     // select and create row group readers.
@@ -658,7 +659,7 @@ Status FileReader::get_next(ChunkPtr* chunk) {
             auto s = strings::Substitute("FileReader::get_next failed. reason = $0, file = $1", status.to_string(),
                                          _file->filename());
             LOG(WARNING) << s;
-            return Status::InternalError(s);
+            return status;
         }
 
         return status;
diff --git a/be/src/formats/parquet/group_reader.cpp b/be/src/formats/parquet/group_reader.cpp
index a7ea3cd949f6d9..d5ef1eaf0a4880 100644
--- a/be/src/formats/parquet/group_reader.cpp
+++ b/be/src/formats/parquet/group_reader.cpp
@@ -372,6 +372,12 @@ StatusOr<ColumnReaderPtr> GroupReader::_create_column_reader(const GroupReaderPa
                              ColumnReaderFactory::create(_column_reader_opts, schema_node, column.slot_type(),
                                                          column.t_iceberg_schema_field));
         }
+        if (_param.global_dictmaps->contains(column.slot_id())) {
+            ASSIGN_OR_RETURN(
+                    column_reader,
+                    ColumnReaderFactory::create(std::move(column_reader), _param.global_dictmaps->at(column.slot_id()),
+                                                column.slot_id(), _row_group_metadata->num_rows));
+        }
         if (column_reader == nullptr) {
             // this shouldn't happen but guard
             return Status::InternalError("No valid column reader.");
diff --git a/be/src/formats/parquet/group_reader.h b/be/src/formats/parquet/group_reader.h
index df0c6e1c14fdf8..b0457a551f041b 100644
--- a/be/src/formats/parquet/group_reader.h
+++ b/be/src/formats/parquet/group_reader.h
@@ -102,6 +102,8 @@ struct GroupReaderParam {
     const std::vector<ColumnPtr>* partition_values = nullptr;
     // not existed column
     const std::vector<SlotDescriptor*>* not_existed_slots = nullptr;
+    // used for global low cardinality optimization
+    ColumnIdToGlobalDictMap* global_dictmaps = &EMPTY_GLOBAL_DICTMAPS;
 };
 
 class PageIndexReader;
diff --git a/be/src/formats/parquet/scalar_column_reader.cpp b/be/src/formats/parquet/scalar_column_reader.cpp
index cd3fa5089a671d..773308a5d95dd5 100644
--- a/be/src/formats/parquet/scalar_column_reader.cpp
+++ b/be/src/formats/parquet/scalar_column_reader.cpp
@@ -19,6 +19,9 @@
 #include "formats/parquet/zone_map_filter_evaluator.h"
 #include "gutil/casts.h"
 #include "io/shared_buffered_input_stream.h"
+#include "runtime/global_dict/dict_column.h"
+#include "runtime/types.h"
+#include "simd/gather.h"
 #include "simd/simd.h"
 #include "statistics_helper.h"
 
@@ -412,10 +415,8 @@ Status ScalarColumnReader::fill_dst_column(ColumnPtr& dst, ColumnPtr& src) {
                     _reader->get_dict_values(codes_column->get_data(), *codes_nullable_column, dict_values.get()));
             DCHECK_EQ(dict_codes->size(), dict_values->size());
             if (dict_values->is_nullable()) {
-                auto* nullable_codes = down_cast<NullableColumn*>(dict_codes.get());
                 auto* nullable_values = down_cast<NullableColumn*>(dict_values.get());
-                nullable_values->null_column_data().swap(nullable_codes->null_column_data());
-                nullable_values->set_has_null(nullable_codes->has_null());
+                nullable_values->swap_null_column(*codes_nullable_column);
             }
         } else {
             dst->append_default(src->size());
@@ -426,4 +427,178 @@ Status ScalarColumnReader::fill_dst_column(ColumnPtr& dst, ColumnPtr& src) {
     }
     return Status::OK();
 }
+
+// LowCardColumnReader
+
+Status LowCardColumnReader::read_range(const Range<uint64_t>& range, const Filter* filter, ColumnPtr& dst) {
+    DCHECK(get_column_parquet_field()->is_nullable ? dst->is_nullable() : true);
+    ColumnContentType content_type = ColumnContentType::DICT_CODE;
+
+    if (_dict_code == nullptr) {
+        _dict_code = ColumnHelper::create_column(
+                TypeDescriptor::from_logical_type(ColumnDictFilterContext::kDictCodePrimitiveType), true);
+    }
+    _ori_column = dst;
+    dst = _dict_code;
+    dst->reserve(range.span_size());
+
+    {
+        SCOPED_RAW_TIMER(&_opts.stats->column_read_ns);
+        return _reader->read_range(range, filter, content_type, dst.get());
+    }
+}
+
+bool LowCardColumnReader::try_to_use_dict_filter(ExprContext* ctx, bool is_decode_needed, const SlotId slotId,
+                                                 const std::vector<std::string>& sub_field_path, const size_t& layer) {
+    if (sub_field_path.size() != layer) {
+        return false;
+    }
+
+    if (column_all_pages_dict_encoded()) {
+        if (_dict_filter_ctx == nullptr) {
+            _dict_filter_ctx = std::make_unique<ColumnDictFilterContext>();
+            _dict_filter_ctx->is_decode_needed = is_decode_needed;
+            _dict_filter_ctx->sub_field_path = sub_field_path;
+            _dict_filter_ctx->slot_id = slotId;
+        }
+        _dict_filter_ctx->conjunct_ctxs.push_back(ctx);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+Status LowCardColumnReader::fill_dst_column(ColumnPtr& dst, ColumnPtr& src) {
+    if (!_code_convert_map.has_value()) {
+        RETURN_IF_ERROR(_check_current_dict());
+    }
+
+    dst->resize(src->size());
+
+    const ColumnPtr& dict_codes = src;
+    auto* codes_nullable_column = ColumnHelper::as_raw_column<NullableColumn>(dict_codes);
+    auto* codes_column = ColumnHelper::as_raw_column<FixedLengthColumn<int32_t>>(codes_nullable_column->data_column());
+    const NullData& null_data_ptr = codes_nullable_column->immutable_null_column_data();
+
+    auto& codes = codes_column->get_data();
+    if (codes_nullable_column->has_null()) {
+        for (size_t i = 0; i < src->size(); i++) {
+            // if null, we assign dict code 0
+            // null = 0, mask = 0xffffffff
+            // null = 1, mask = 0x00000000
+            uint32_t mask = ~(static_cast<uint32_t>(-null_data_ptr[i]));
+            codes[i] = mask & codes[i];
+        }
+    }
+
+    auto* dst_data_column = down_cast<LowCardDictColumn*>(ColumnHelper::get_data_column(dst.get()));
+    SIMDGather::gather(dst_data_column->get_data().data(), _code_convert_map->data(), codes.data(),
+                       DICT_DECODE_MAX_SIZE, src->size());
+
+    if (dst->is_nullable()) {
+        auto* nullable_dst = down_cast<NullableColumn*>(dst.get());
+        nullable_dst->swap_null_column(*codes_nullable_column);
+    }
+
+    src->reset_column();
+    src = _ori_column;
+
+    return Status::OK();
+}
+
+Status LowCardColumnReader::_check_current_dict() {
+    std::vector<int16_t> code_convert_map;
+
+    // create dict value chunk for evaluation.
+    ColumnPtr dict_value_column = ColumnHelper::create_column(TypeDescriptor(TYPE_VARCHAR), true);
+    RETURN_IF_ERROR(_reader->get_dict_values(dict_value_column.get()));
+
+    size_t dict_size = dict_value_column->size();
+
+    code_convert_map.resize(dict_size + 2);
+    std::fill(code_convert_map.begin(), code_convert_map.end(), 0);
+    auto* local_to_global = code_convert_map.data();
+
+    auto viewer = ColumnViewer<TYPE_VARCHAR>(dict_value_column);
+
+    for (int i = 0; i < dict_size; ++i) {
+        auto slice = viewer.value(i);
+        auto res = _dict->find(slice);
+        if (res == _dict->end()) {
+            if (slice.size > 0) {
+                // error message format used to extract info, carefully
+                return Status::GlobalDictNotMatch(
+                        fmt::format("SlotId: {}, FileName: {} , file doesn't match global dict. ", _slot_id,
+                                    _opts.file->filename()));
+            }
+        } else {
+            local_to_global[i] = res->second;
+        }
+    }
+
+#ifdef DEBUG
+    std::stringstream ss;
+    ss << "dict mapping: ";
+    for (int i = 0; i < dict_size; ++i) {
+        ss << code_convert_map[i] << " ";
+    }
+    LOG(INFO) << ss.str();
+#endif
+
+    _code_convert_map = std::move(code_convert_map);
+
+    return Status::OK();
+}
+
+// LowRowsColumnReader
+
+Status LowRowsColumnReader::read_range(const Range<uint64_t>& range, const Filter* filter, ColumnPtr& dst) {
+    DCHECK(get_column_parquet_field()->is_nullable ? dst->is_nullable() : true);
+    ColumnContentType content_type = ColumnContentType::VALUE;
+
+    if (_tmp_column == nullptr) {
+        _tmp_column = ColumnHelper::create_column(TYPE_VARCHAR_DESC, true);
+    }
+    _ori_column = dst;
+    dst = _tmp_column;
+    dst->reserve(range.span_size());
+
+    {
+        SCOPED_RAW_TIMER(&_opts.stats->column_read_ns);
+        return _reader->read_range(range, filter, content_type, dst.get());
+    }
+}
+
+Status LowRowsColumnReader::fill_dst_column(ColumnPtr& dst, ColumnPtr& src) {
+    dst->resize(src->size());
+
+    const ColumnPtr& readed_column = src;
+    auto* nullable_string_column = ColumnHelper::as_raw_column<NullableColumn>(readed_column);
+    auto* binary_column = ColumnHelper::as_raw_column<BinaryColumn>(nullable_string_column->data_column());
+    auto* dst_data_column = down_cast<LowCardDictColumn*>(ColumnHelper::get_data_column(dst.get()));
+    for (size_t i = 0; i < src->size(); i++) {
+        const auto& slice = binary_column->get_slice(i);
+        auto res = _dict->find(slice);
+        if (res == _dict->end()) {
+            if (slice.size > 0) {
+                // error message format used to extract info, carefully
+                return Status::GlobalDictNotMatch(
+                        fmt::format("SlotId: {}, FileName: {} , file doesn't match global dict. ", _slot_id,
+                                    _opts.file->filename()));
+            }
+        } else {
+            dst_data_column->get_data()[i] = res->second;
+        }
+    }
+
+    if (dst->is_nullable()) {
+        auto* nullable_dst = down_cast<NullableColumn*>(dst.get());
+        nullable_dst->swap_null_column(*nullable_string_column);
+    }
+
+    src->reset_column();
+    src = _ori_column;
+
+    return Status::OK();
+}
 } // namespace starrocks::parquet
\ No newline at end of file
diff --git a/be/src/formats/parquet/scalar_column_reader.h b/be/src/formats/parquet/scalar_column_reader.h
index 773b341bb1de62..e1ec5c4a0b9dfb 100644
--- a/be/src/formats/parquet/scalar_column_reader.h
+++ b/be/src/formats/parquet/scalar_column_reader.h
@@ -60,6 +60,10 @@ class RawColumnReader : public ColumnReader {
     explicit RawColumnReader(const ParquetField* parquet_field, const tparquet::ColumnChunk* column_chunk_metadata,
                              const ColumnReaderOptions& opts)
             : ColumnReader(parquet_field), _opts(opts), _chunk_metadata(column_chunk_metadata) {}
+    explicit RawColumnReader(const RawColumnReader& reader)
+            : ColumnReader(reader.get_column_parquet_field()),
+              _opts(reader._opts),
+              _chunk_metadata(reader._chunk_metadata) {}
     ~RawColumnReader() override = default;
 
     Status prepare() override {
@@ -182,4 +186,87 @@ class ScalarColumnReader final : public RawColumnReader {
     ColumnPtr _ori_column = nullptr;
 };
 
+class LowCardColumnReader final : public RawColumnReader {
+public:
+    explicit LowCardColumnReader(const RawColumnReader& reader, const GlobalDictMap* dict, SlotId slot_id)
+            : RawColumnReader(reader), _dict(dict), _slot_id(slot_id) {}
+
+    Status read_range(const Range<uint64_t>& range, const Filter* filter, ColumnPtr& dst) override;
+
+    bool try_to_use_dict_filter(ExprContext* ctx, bool is_decode_needed, const SlotId slotId,
+                                const std::vector<std::string>& sub_field_path, const size_t& layer) override;
+
+    Status rewrite_conjunct_ctxs_to_predicate(bool* is_group_filtered, const std::vector<std::string>& sub_field_path,
+                                              const size_t& layer) override {
+        DCHECK_EQ(sub_field_path.size(), layer);
+        return _dict_filter_ctx->rewrite_conjunct_ctxs_to_predicate(_reader.get(), is_group_filtered);
+    }
+
+    Status filter_dict_column(const ColumnPtr& column, Filter* filter, const std::vector<std::string>& sub_field_path,
+                              const size_t& layer) override {
+        DCHECK_EQ(sub_field_path.size(), layer);
+        return _dict_filter_ctx->predicate->evaluate_and(column.get(), filter->data());
+    }
+
+    Status fill_dst_column(ColumnPtr& dst, ColumnPtr& src) override;
+
+    StatusOr<bool> row_group_zone_map_filter(const std::vector<const ColumnPredicate*>& predicates,
+                                             CompoundNodeType pred_relation, const uint64_t rg_first_row,
+                                             const uint64_t rg_num_rows) const override {
+        return _row_group_zone_map_filter(predicates, pred_relation, TypeDescriptor(LogicalType::TYPE_VARCHAR),
+                                          rg_first_row, rg_num_rows);
+    }
+
+    StatusOr<bool> page_index_zone_map_filter(const std::vector<const ColumnPredicate*>& predicates,
+                                              SparseRange<uint64_t>* row_ranges, CompoundNodeType pred_relation,
+                                              const uint64_t rg_first_row, const uint64_t rg_num_rows) override {
+        return _page_index_zone_map_filter(predicates, row_ranges, pred_relation,
+                                           TypeDescriptor(LogicalType::TYPE_VARCHAR), rg_first_row, rg_num_rows);
+    }
+
+private:
+    Status _check_current_dict();
+
+    std::unique_ptr<ColumnDictFilterContext> _dict_filter_ctx;
+
+    const GlobalDictMap* _dict = nullptr;
+    const SlotId _slot_id;
+
+    std::optional<std::vector<int16_t>> _code_convert_map;
+
+    ColumnPtr _dict_code = nullptr;
+    ColumnPtr _ori_column = nullptr;
+};
+
+class LowRowsColumnReader final : public RawColumnReader {
+public:
+    explicit LowRowsColumnReader(const RawColumnReader& reader, const GlobalDictMap* dict, SlotId slot_id)
+            : RawColumnReader(reader), _dict(dict), _slot_id(slot_id) {}
+
+    Status read_range(const Range<uint64_t>& range, const Filter* filter, ColumnPtr& dst) override;
+
+    Status fill_dst_column(ColumnPtr& dst, ColumnPtr& src) override;
+
+    StatusOr<bool> row_group_zone_map_filter(const std::vector<const ColumnPredicate*>& predicates,
+                                             CompoundNodeType pred_relation, const uint64_t rg_first_row,
+                                             const uint64_t rg_num_rows) const override {
+        return _row_group_zone_map_filter(predicates, pred_relation, TypeDescriptor(LogicalType::TYPE_VARCHAR),
+                                          rg_first_row, rg_num_rows);
+    }
+
+    StatusOr<bool> page_index_zone_map_filter(const std::vector<const ColumnPredicate*>& predicates,
+                                              SparseRange<uint64_t>* row_ranges, CompoundNodeType pred_relation,
+                                              const uint64_t rg_first_row, const uint64_t rg_num_rows) override {
+        return _page_index_zone_map_filter(predicates, row_ranges, pred_relation,
+                                           TypeDescriptor(LogicalType::TYPE_VARCHAR), rg_first_row, rg_num_rows);
+    }
+
+private:
+    const GlobalDictMap* _dict = nullptr;
+    const SlotId _slot_id;
+
+    ColumnPtr _tmp_column = nullptr;
+    ColumnPtr _ori_column = nullptr;
+};
+
 } // namespace starrocks::parquet
\ No newline at end of file
diff --git a/be/test/formats/parquet/file_reader_test.cpp b/be/test/formats/parquet/file_reader_test.cpp
index f9d9ba11f4a42c..3ff9a62b93c2c4 100644
--- a/be/test/formats/parquet/file_reader_test.cpp
+++ b/be/test/formats/parquet/file_reader_test.cpp
@@ -37,6 +37,7 @@
 #include "io/shared_buffered_input_stream.h"
 #include "runtime/descriptor_helper.h"
 #include "runtime/mem_tracker.h"
+#include "runtime/types.h"
 #include "testutil/assert.h"
 #include "testutil/column_test_helper.h"
 #include "testutil/exprs_test_helper.h"
@@ -3383,4 +3384,277 @@ TEST_F(FileReaderTest, min_max_filter_all_null_group) {
     ASSERT_EQ(file_reader->row_group_size(), 0);
 }
 
+TEST_F(FileReaderTest, low_card_reader) {
+    auto chunk = std::make_shared<Chunk>();
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+
+    const std::string small_page_file = "./be/test/formats/parquet/test_data/page_index_small_page.parquet";
+
+    Utils::SlotDesc slot_descs[] = {{"c0", TYPE_INT_DESC}, {"c2", TYPE_INT_DESC}, {""}};
+    auto ctx = _create_file_random_read_context(small_page_file, slot_descs);
+
+    std::vector<std::string> values;
+    for (int i = 0; i < 100; ++i) {
+        values.push_back(std::to_string(i));
+    }
+    std::sort(values.begin(), values.end());
+
+    ColumnIdToGlobalDictMap dict_map;
+    GlobalDictMap g_dict;
+    for (int i = 0; i < 100; ++i) {
+        g_dict[Slice(values[i])] = i;
+    }
+    dict_map[1] = &g_dict;
+
+    ctx->global_dictmaps = &dict_map;
+
+    std::vector<TExpr> t_conjuncts;
+    ParquetUTBase::create_dictmapping_string_conjunct(TExprOpcode::EQ, 1, "2", &t_conjuncts);
+    ParquetUTBase::create_conjunct_ctxs(&_pool, _runtime_state, &t_conjuncts, &ctx->conjunct_ctxs_by_slot[1]);
+    TupleDescriptor* tuple_desc = Utils::create_tuple_descriptor(_runtime_state, &_pool, slot_descs);
+    ParquetUTBase::setup_conjuncts_manager(ctx->conjunct_ctxs_by_slot[1], tuple_desc, _runtime_state, ctx);
+
+    auto file_reader = _create_file_reader(small_page_file);
+    Status status = file_reader->init(ctx);
+    ASSERT_TRUE(status.ok());
+    size_t total_row_nums = 0;
+    while (!status.is_end_of_file()) {
+        chunk->reset();
+        status = file_reader->get_next(&chunk);
+        chunk->check_or_die();
+        total_row_nums += chunk->num_rows();
+        ColumnPtr c0 = chunk->get_column_by_index(0);
+        ColumnPtr c1 = chunk->get_column_by_index(1);
+        for (size_t row_index = 0; row_index < chunk->num_rows(); row_index++) {
+            int32_t c0_value = c0->get(row_index).get_int32();
+            if (c0_value % 10 == 0) {
+                EXPECT_TRUE(c1->is_null(row_index));
+            } else {
+                EXPECT_FALSE(c1->is_null(row_index));
+                std::string expected_string = std::to_string(c0_value % 100);
+                int32_t global_code = g_dict.at(Slice(expected_string));
+                EXPECT_EQ(global_code, c1->get(row_index).get_int32());
+            }
+        }
+    }
+
+    EXPECT_EQ(200, total_row_nums);
+    ctx->predicate_free_pool.clear();
+    ctx->conjuncts_manager = nullptr;
+}
+
+TEST_F(FileReaderTest, low_card_reader_filter_group) {
+    const std::string small_page_file = "./be/test/formats/parquet/test_data/page_index_small_page.parquet";
+
+    Utils::SlotDesc slot_descs[] = {{"c0", TYPE_INT_DESC}, {"c2", TYPE_INT_DESC}, {""}};
+    auto ctx = _create_file_random_read_context(small_page_file, slot_descs);
+
+    std::vector<std::string> values;
+    for (int i = 0; i < 100; ++i) {
+        values.push_back(std::to_string(i));
+    }
+    std::sort(values.begin(), values.end());
+
+    ColumnIdToGlobalDictMap dict_map;
+    GlobalDictMap g_dict;
+    for (int i = 0; i < 100; ++i) {
+        g_dict[Slice(values[i])] = i;
+    }
+    dict_map[1] = &g_dict;
+
+    ctx->global_dictmaps = &dict_map;
+
+    std::vector<TExpr> t_conjuncts;
+    ParquetUTBase::create_dictmapping_string_conjunct(TExprOpcode::GT, 1, "a", &t_conjuncts);
+    ParquetUTBase::create_conjunct_ctxs(&_pool, _runtime_state, &t_conjuncts, &ctx->conjunct_ctxs_by_slot[1]);
+    TupleDescriptor* tuple_desc = Utils::create_tuple_descriptor(_runtime_state, &_pool, slot_descs);
+    tuple_desc->decoded_slots()[1]->type().type = TYPE_VARCHAR;
+    ParquetUTBase::setup_conjuncts_manager(ctx->conjunct_ctxs_by_slot[1], tuple_desc, _runtime_state, ctx);
+
+    auto file_reader = _create_file_reader(small_page_file);
+    Status status = file_reader->init(ctx);
+    ASSERT_TRUE(status.ok());
+    EXPECT_EQ(file_reader->row_group_size(), 0);
+    ctx->predicate_free_pool.clear();
+    ctx->conjuncts_manager = nullptr;
+}
+
+TEST_F(FileReaderTest, low_card_reader_dict_not_match) {
+    auto chunk = std::make_shared<Chunk>();
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+
+    const std::string small_page_file = "./be/test/formats/parquet/test_data/page_index_small_page.parquet";
+
+    Utils::SlotDesc slot_descs[] = {{"c0", TYPE_INT_DESC}, {"c2", TYPE_INT_DESC}, {""}};
+    auto ctx = _create_file_random_read_context(small_page_file, slot_descs);
+
+    std::vector<std::string> values;
+    for (int i = 0; i < 90; ++i) {
+        values.push_back(std::to_string(i));
+    }
+    std::sort(values.begin(), values.end());
+
+    ColumnIdToGlobalDictMap dict_map;
+    GlobalDictMap g_dict;
+    for (int i = 0; i < 90; ++i) {
+        g_dict[Slice(values[i])] = i;
+    }
+    dict_map[1] = &g_dict;
+
+    ctx->global_dictmaps = &dict_map;
+
+    auto file_reader = _create_file_reader(small_page_file);
+    Status status = file_reader->init(ctx);
+    ASSERT_TRUE(status.ok());
+    while (!status.is_end_of_file()) {
+        chunk->reset();
+        status = file_reader->get_next(&chunk);
+        if (!status.ok()) {
+            ASSERT_EQ("Global dictionary not match", status.code_as_string());
+            return;
+        }
+    }
+
+    ASSERT_TRUE(false);
+}
+
+TEST_F(FileReaderTest, no_matched_reader) {
+    auto chunk = std::make_shared<Chunk>();
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+
+    const std::string file = "./be/test/formats/parquet/test_data/page_index_repeated_nodict.parquet";
+
+    Utils::SlotDesc slot_descs[] = {{"c0", TYPE_INT_DESC}, {"c2", TYPE_INT_DESC}, {""}};
+    auto ctx = _create_file_random_read_context(file, slot_descs);
+
+    std::vector<std::string> values;
+    for (int i = 0; i < 100; ++i) {
+        values.push_back(std::to_string(i));
+    }
+    std::sort(values.begin(), values.end());
+
+    ColumnIdToGlobalDictMap dict_map;
+    GlobalDictMap g_dict;
+    for (int i = 0; i < 100; ++i) {
+        g_dict[Slice(values[i])] = i;
+    }
+    dict_map[1] = &g_dict;
+
+    ctx->global_dictmaps = &dict_map;
+
+    auto file_reader = _create_file_reader(file);
+    Status status = file_reader->init(ctx);
+    ASSERT_EQ("Global dictionary not match", status.code_as_string());
+}
+
+TEST_F(FileReaderTest, low_rows_reader) {
+    auto chunk = std::make_shared<Chunk>();
+
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_DESC, true), chunk->num_columns());
+    chunk->append_column(ColumnHelper::create_column(TYPE_INT_ARRAY_DESC, true), chunk->num_columns());
+
+    const std::string low_rows_file = "./be/test/formats/parquet/test_data/low_rows_non_dict.parquet";
+
+    Utils::SlotDesc slot_descs[] = {
+            {"c0", TYPE_INT_DESC}, {"c1", TYPE_INT_DESC}, {"c2", TYPE_INT_DESC}, {"c3", TYPE_INT_ARRAY_DESC}, {""}};
+    auto ctx = _create_file_random_read_context(low_rows_file, slot_descs);
+
+    std::vector<std::string> values;
+    for (int i = 0; i < 100; ++i) {
+        values.push_back(std::to_string(i));
+    }
+    std::sort(values.begin(), values.end());
+
+    ColumnIdToGlobalDictMap dict_map;
+    GlobalDictMap g_dict;
+    for (int i = 0; i < 100; ++i) {
+        g_dict[Slice(values[i])] = i;
+    }
+    dict_map[2] = &g_dict;
+    dict_map[3] = &g_dict;
+
+    ctx->global_dictmaps = &dict_map;
+
+    auto file_reader = _create_file_reader(low_rows_file);
+    Status status = file_reader->init(ctx);
+
+    ASSERT_TRUE(status.ok());
+    size_t total_row_nums = 0;
+    while (!status.is_end_of_file()) {
+        chunk->reset();
+        status = file_reader->get_next(&chunk);
+        chunk->check_or_die();
+        total_row_nums += chunk->num_rows();
+        ColumnPtr c0 = chunk->get_column_by_index(0);
+        ColumnPtr c1 = chunk->get_column_by_index(1);
+        ColumnPtr c2 = chunk->get_column_by_index(2);
+        ColumnPtr c3 = chunk->get_column_by_index(3);
+        for (size_t row_index = 0; row_index < chunk->num_rows(); row_index++) {
+            int32_t c0_value = c0->get(row_index).get_int32();
+            if (c0_value % 10 == 0) {
+                EXPECT_TRUE(c2->is_null(row_index));
+                EXPECT_TRUE(c3->is_null(row_index));
+            } else {
+                EXPECT_FALSE(c2->is_null(row_index));
+                EXPECT_FALSE(c3->is_null(row_index));
+                int32_t c1_value = c1->get(row_index).get_int32();
+                std::string expected_c0_string = std::to_string(c0_value % 100);
+                std::string expected_c1_string = std::to_string(c1_value % 100);
+                int32_t c0_global_code = g_dict.at(Slice(expected_c0_string));
+                int32_t c1_global_code = g_dict.at(Slice(expected_c1_string));
+                EXPECT_EQ(c0_global_code, c2->get(row_index).get_int32());
+                DatumArray c3_value = c3->get(row_index).get_array();
+                EXPECT_EQ(3, c3_value.size());
+                EXPECT_EQ(c0_global_code, c3_value[0].get_int32());
+                EXPECT_TRUE(c3_value[1].is_null());
+                EXPECT_EQ(c1_global_code, c3_value[2].get_int32());
+            }
+        }
+    }
+
+    EXPECT_EQ(100, total_row_nums);
+}
+
+TEST_F(FileReaderTest, low_rows_reader_filter_group) {
+    const std::string small_page_file = "./be/test/formats/parquet/test_data/low_rows_non_dict.parquet";
+
+    Utils::SlotDesc slot_descs[] = {{"c0", TYPE_INT_DESC}, {"c2", TYPE_INT_DESC}, {""}};
+    auto ctx = _create_file_random_read_context(small_page_file, slot_descs);
+
+    std::vector<std::string> values;
+    for (int i = 0; i < 100; ++i) {
+        values.push_back(std::to_string(i));
+    }
+    std::sort(values.begin(), values.end());
+
+    ColumnIdToGlobalDictMap dict_map;
+    GlobalDictMap g_dict;
+    for (int i = 0; i < 100; ++i) {
+        g_dict[Slice(values[i])] = i;
+    }
+    dict_map[1] = &g_dict;
+
+    ctx->global_dictmaps = &dict_map;
+
+    std::vector<TExpr> t_conjuncts;
+    ParquetUTBase::create_dictmapping_string_conjunct(TExprOpcode::EQ, 1, "a", &t_conjuncts);
+    ParquetUTBase::create_conjunct_ctxs(&_pool, _runtime_state, &t_conjuncts, &ctx->conjunct_ctxs_by_slot[1]);
+    TupleDescriptor* tuple_desc = Utils::create_tuple_descriptor(_runtime_state, &_pool, slot_descs);
+    tuple_desc->decoded_slots()[1]->type().type = TYPE_VARCHAR;
+    ParquetUTBase::setup_conjuncts_manager(ctx->conjunct_ctxs_by_slot[1], tuple_desc, _runtime_state, ctx);
+
+    auto file_reader = _create_file_reader(small_page_file);
+    Status status = file_reader->init(ctx);
+    ASSERT_TRUE(status.ok());
+    EXPECT_EQ(file_reader->row_group_size(), 0);
+
+    ctx->predicate_free_pool.clear();
+    ctx->conjuncts_manager = nullptr;
+}
+
 } // namespace starrocks::parquet
diff --git a/be/test/formats/parquet/parquet_ut_base.cpp b/be/test/formats/parquet/parquet_ut_base.cpp
index 85f7128a532af4..8bbdd51fb152ed 100644
--- a/be/test/formats/parquet/parquet_ut_base.cpp
+++ b/be/test/formats/parquet/parquet_ut_base.cpp
@@ -29,6 +29,7 @@ void ParquetUTBase::create_conjunct_ctxs(ObjectPool* pool, RuntimeState* runtime
                                          std::vector<ExprContext*>* conjunct_ctxs) {
     ASSERT_OK(Expr::create_expr_trees(pool, *tExprs, conjunct_ctxs, nullptr));
     ASSERT_OK(Expr::prepare(*conjunct_ctxs, runtime_state));
+    DictOptimizeParser::disable_open_rewrite(conjunct_ctxs);
     ASSERT_OK(Expr::open(*conjunct_ctxs, runtime_state));
 }
 
@@ -264,10 +265,51 @@ void ParquetUTBase::setup_conjuncts_manager(std::vector<ExprContext*>& conjuncts
     opts.pred_tree_params = {true, true};
     params->conjuncts_manager = std::make_unique<ScanConjunctsManager>(std::move(opts));
     ASSERT_TRUE(params->conjuncts_manager->parse_conjuncts().ok());
-    ConnectorPredicateParser predicate_parser{&params->slot_descs};
+    ConnectorPredicateParser predicate_parser{&tuple_desc->decoded_slots()};
     auto st = params->conjuncts_manager->get_predicate_tree(&predicate_parser, params->predicate_free_pool);
     ASSERT_TRUE(st.ok());
     params->predicate_tree = st.value();
 }
 
+void ParquetUTBase::create_dictmapping_string_conjunct(TExprOpcode::type opcode, starrocks::SlotId slot_id,
+                                                       const std::string& value, std::vector<TExpr>* tExprs) {
+    std::vector<TExprNode> nodes;
+
+    TExprNode node0;
+    node0.node_type = TExprNodeType::DICT_EXPR;
+    node0.num_children = 2;
+    node0.type = gen_type_desc(TPrimitiveType::VARCHAR);
+    node0.__set_has_nullable_child(true);
+    node0.__set_is_nullable(true);
+    nodes.emplace_back(node0);
+
+    TExprNode node1 = ExprsTestHelper::create_slot_expr_node_t<TYPE_INT>(0, slot_id, true);
+    nodes.emplace_back(node1);
+
+    TExprNode pre_node = ExprsTestHelper::create_binary_pred_node(TPrimitiveType::VARCHAR, opcode);
+    if (opcode == TExprOpcode::LT || opcode == TExprOpcode::LE || opcode == TExprOpcode::GE ||
+        opcode == TExprOpcode::GT) {
+        pre_node.__set_is_monotonic(true);
+    }
+    nodes.emplace_back(pre_node);
+
+    TExprNode place_holder;
+    place_holder.node_type = TExprNodeType::PLACEHOLDER_EXPR;
+    place_holder.type = gen_type_desc(TPrimitiveType::VARCHAR);
+    place_holder.num_children = 0;
+    place_holder.__set_is_nullable(true);
+    TPlaceHolder holder;
+    holder.__set_slot_id(slot_id);
+    holder.__set_nullable(true);
+    place_holder.__set_vslot_ref(holder);
+    nodes.emplace_back(place_holder);
+
+    TExprNode varchar_literal = ExprsTestHelper::create_literal<TYPE_VARCHAR, std::string>(value, false);
+    nodes.emplace_back(varchar_literal);
+
+    TExpr t_expr;
+    t_expr.nodes = nodes;
+    tExprs->emplace_back(t_expr);
+}
+
 } // namespace starrocks::parquet
diff --git a/be/test/formats/parquet/parquet_ut_base.h b/be/test/formats/parquet/parquet_ut_base.h
index 35e848a4753d40..3e09a0738e9c13 100644
--- a/be/test/formats/parquet/parquet_ut_base.h
+++ b/be/test/formats/parquet/parquet_ut_base.h
@@ -58,6 +58,9 @@ class ParquetUTBase {
 
     static void setup_conjuncts_manager(std::vector<ExprContext*>& conjuncts, TupleDescriptor* tuple_desc,
                                         RuntimeState* runtime_state, HdfsScannerContext* params);
+
+    static void create_dictmapping_string_conjunct(TExprOpcode::type opcode, SlotId slot_id, const std::string& value,
+                                                   std::vector<TExpr>* tExprs);
 };
 
 } // namespace starrocks::parquet
\ No newline at end of file
diff --git a/be/test/formats/parquet/test_data/low_rows_non_dict.parquet b/be/test/formats/parquet/test_data/low_rows_non_dict.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..926b3d5d1073eac71122cb558081a3d3365bb428
GIT binary patch
literal 6525
zcmcIp&vP3^7M2~yaflOwiHr;m7(txu;sni%M$&}rE+g5NC2S{FV*ScygJvY#az<mJ
zAC}~7Zd?2T`~~|bP#ie!?lH9oP81io?qx4jab=6z?(e;JWG7Gsgo)DMOm~0%y5D=>
zo2lfMozsGW;1`qPw-9_hBJRh7<FlcOYJjfL#0cL_2d+G$R}7vIe;<#GsClD2Uv|rD
zj3985ah&lK;{@Yr#upgRFitX_Wjx3DBI9|+ml&rQUuL|(7-XDge1$Q@IK%iV<7<qw
zj29VSXMBTkj`2;#FymW{Z!=zEoM*huc!lwIjK61mhjD@ND&sZAcNrs$8l%o=Fh&_o
z#u#IqF~MjtUT1ud@do1`82`xlC&u>~ZN?;HigA%~iE)`R&A7stVazgqz_`k|#<<RS
zlkpbg24jx#Hsc+}O~yQ9fw9O~Vk|S>W&AVaJ;uK<-e>%fvBKyux{OLm^Ix5v4UWxR
z8V}5bM%_p-F#ULB`iryTH$QRXX9#X)R){nZlrnox%2hj5J60!9C;E_2qE4(lpgf{H
zq&%iPs647XtURtFpdz9oq#~vws3NK&tRk*5KxKr=5S1}1gH%ST3{x4WGEilt%21WD
zDuY!<zt87a8Lt$e6rmKN6r&WR6r~iV6sHuZ6sZ)d6sr`h6s;7l6t60PstBq=sEVN~
zh^i>6!l;U)Dv+v3Z}a)yVhl6B$vDUO2IK3D7a3<6Ut@fgafUI(_zL4RW03Iz<I9Xw
zj4v^sXMB<I9OGHWNyambFEE~FoM1e~IL>&Iag6Z<qm-7sMhF7)|DGLEu6SQQQSSfF
z{c`DkIT8GFV(6tCyKrUHvO*&=?9vreBd34({@Z_^S8(!&6LLqIY;@TevWd#Zlub-F
zaoHqfW65J^8d@E#fi{ZPL>ogJN1H$kmJXH<mJXH<mJXH<mJXH<mJXHymI0OlmI0Ol
zmI0OlmI0OlRurr#SW&Q|U`4@-f)xcT3RV;>6D$)f6D$)f6D$)f6D$)f6Ra3mF|cA_
z#lVVz6$2{<Rt&5dSaGo8V8y|TgB1rW4ptnjI9PG85@033N`RFBD*;vltOQsIuo7Td
zU|C>UU|C>UU|C>UU|C>U*8JZt&ju&eYelEz{N(iXC#UDXyLjVCP91%1d^Q#QY-%C+
z=@@_T#U77P^2u5w>p}YS#@Ab4f8O{hko7eE{Xf1HzxRG6$b>~=N8}wH-;%fF10*y#
z0twFEkQ2xy5CHiFf*|KWAmmppNouen$aIPfMm35LNE$^6YiUEV(kNo+(I{@1sZsQ>
zf<}?Y%A}zLY>|>EFGx?6DWodO9nuzM6Df@Hi*!aAM{1)SB+XG)lJerPMf#)6B^6RG
zlNKqvNs*NAq)So&sgpE88YPvGQb{+Y*BES(YDrt9T~Zq<nDj?FCPk8(Nu#7`QY|T)
zbj<pO4_W1~MOx=kc$-;>0<VVXuoc9Mts`E%E5wU;i+Ir~s5V4li%!a?hAs96Y_XSM
zi@gV1>{ZxeZ^ITx0k$YQR8b7rq6l&{VT+>-TO57Z;;euz&KB6>EP^f0F4&^1qdKI+
z7F8q8Qp#A$Udm+3YRYiRcC@SljIuhwV%5NwhImo+(qPNND++@abO??Gbz&)tURqL%
z7G7O2Y(d}Hj|J6Zn-*$;KL5>iRqGO?t}*7-+nrN2uhyvf9Z~o`9h?4cO8n;kd-=vt
zK^v}oV>5p>1EKM@yHjgaf)jz@*rieVHLh;7KyVU0`lZ0C{H`(Oun`;$j<3qkJaa4r
ztHDv4sI2pB@nJACJG1&5eCq!e-ypx>)$yw%F#Y$qaMJvQK=Ii(vk)3@R{UzE9s1A7
zhvB+kt@ZD_dtSHUx5L-x?mi4Z5Pjk<+^YE7weZ4R*snBdqQmcco_K0^Dy@!qjNpE+
zDQ>k0J=E~<=$;s%74KvD8ZUfwovnJ!tKL7_M;p7}88oHAaM!Q9dsW&)x6vFN4(vI*
zHMb-8)~IzV)k>!#rauhVd@;T5Z%gjxo#m^E@S~y49Nk6OjY$5HU2xyAUDW+fEDApy
z*~4M?&=#bN9xaHS@Ug<K_6#HFd!9!0XBa`>F(W_EpHCmgc!nMH9kU~g#9cOa?`h9H
zGkUOw%xTpsoxK(U>eZcAr8SW8Yz``|)?QDpwqI+t>wCWJ)neDQYmu<<d6@SwG^k$_
zf5MLfp-DTJyIHtSB`EZ-Gw;ePvb|*6_J;T?uMccHYA3htW!v7iQ`_iECXbBwWT)Wp
zc(S)h&!WXD7|+;t&lWs^a*@S|1o3R!H#bw_E}m(4-Euk~E$Zw0&Pu-H_-pl>^`ui9
zWLuSj`5?2qx1I5`J5HnOW$L}{OvBu93Y&6Hw_518S5tP~HF7(yzajeU9zXB9sZ1=h
zxYS!q368%WDHm-y-rvec+L=^#*U|f$Q!uxrjdF3va|dGna!2gv>GjJE&vVkbEvK>E
zDc=_J(x&F5dvcv#N$^X>^{wiP*TYyUt2qr%te;c+)tq9|EE+k}P0M}Tbw|&BR4%MX
z=scQNO)rZzay#;@;@qCoSl=(Fg<s^JQrUcdsnofd%38<vejw);jCTESU*cSAsf^rL
zqf*GX%ga``RM0&5u(8&IJwGXYDg32bky<gyV_vtqSmOPNvmM*7@KI!accU(S>lE^l
zQX#ifO)nkBVC!~I?t9o)pUzfpmJ9s{MP2#Su9ps$u#THv7I9xb81~B%JAS_7u6V<K
zZ94vjzE0=sl!d<pKT<l3N8ega$$T2Nm4U6N;wSx7ho7E%{2V{uc2-1=tsINX;78+f
z@Z)jmH$`qc)q>e9>DGgCK0+~g+Bf^9M$;=9KM^C&d$*Anv9ku8Ywb!QJ17;iCUV42
zn&N)o7%uEo8~K5&d;3mZ=3%D4UXS)yckQ~2cjieR>E&Yfpi;29SYtQSU8_e289yRb
zo|cV@wYZfv#_r1|Ba1|`ByUj<mJdV`N{FYWHBod#CyEZ{qn==rEoz2smk!o!d-aZ(
z|IU&uU2=9ZCr9ij>*0~?Rf;*!K}}k8<^0wy(a7?&v58)6^04%YerY<rHW00_zd`$3
zm-eu~6gHIvmG&nlqQu$B-8IqOSL@62DK0T-Q!Xlwv?A-FxOh>F@_MSC%079Ck}1~{
ztvIv*!SA6{4q`>RvF$);u32e))U9={y3OWQycp)V@XygJVMcyR9{vI!i+`ZHCH~{(
E|KkQTiU0rr

literal 0
HcmV?d00001


From 92b45a6a25b915bcd4e906cf7e9330ef63810a57 Mon Sep 17 00:00:00 2001
From: Harbor Liu <460660596@qq.com>
Date: Tue, 21 Jan 2025 10:04:45 +0800
Subject: [PATCH 57/71] [Feature] Add Transaction Commit/Rollback logical and
 state (#54218)

---
 .../java/com/starrocks/common/ErrorCode.java  |  12 +-
 .../com/starrocks/common/LoadException.java   |   4 +
 .../common/NoAliveBackendException.java       |   4 +
 .../starrocks/common/TimeoutException.java    |   4 +
 .../com/starrocks/load/loadv2/LoadMgr.java    |  25 +-
 .../java/com/starrocks/qe/ConnectContext.java |  14 +
 .../java/com/starrocks/qe/StmtExecutor.java   |  35 +-
 .../com/starrocks/sql/StatementPlanner.java   |   5 +
 .../sql/analyzer/DMLStmtAnalyzer.java         |   4 +
 .../transaction/DatabaseTransactionMgr.java   |  89 +++-
 .../transaction/ExplicitTxnState.java         | 107 ++++
 .../transaction/TransactionState.java         | 212 +++++---
 .../transaction/TransactionStateBatch.java    |  14 +-
 .../transaction/TransactionStmtExecutor.java  | 499 ++++++++++++++++++
 .../DatabaseTransactionMgrTest.java           |   2 +-
 15 files changed, 914 insertions(+), 116 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/transaction/ExplicitTxnState.java
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStmtExecutor.java

diff --git a/fe/fe-core/src/main/java/com/starrocks/common/ErrorCode.java b/fe/fe-core/src/main/java/com/starrocks/common/ErrorCode.java
index 4338e922de985b..5910c1eb7976b9 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/ErrorCode.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/ErrorCode.java
@@ -151,8 +151,7 @@ public enum ErrorCode {
     ERR_INVALID_VALUE(5018, new byte[] {'H', 'Y', '0', '0', '0'}, "Invalid %s: '%s'. Expected values should be %s"),
     ERR_NO_ALTER_OPERATION(5023, new byte[] {'H', 'Y', '0', '0', '0'},
             "No operation in alter statement"),
-    ERR_TIMEOUT(5024, new byte[] {'H', 'Y', '0', '0', '0'}, "%s reached its timeout of %d seconds, %s"),
-    ERR_FAILED_WHEN_INSERT(5025, new byte[] {'H', 'Y', '0', '0', '0'}, "Failed when INSERT execute"),
+    ERR_TIMEOUT(5024, new byte[] {'5', '3', '4', '0', '0'}, "%s reached its timeout of %d seconds, %s"),
     ERR_UNSUPPORTED_TYPE_IN_CTAS(5026, new byte[] {'H', 'Y', '0', '0', '0'},
             "Unsupported type '%s' in create table as select statement"),
     ERR_MISSING_PARAM(5027, new byte[] {'H', 'Y', '0', '0', '0'}, "Missing param: %s "),
@@ -208,7 +207,7 @@ public enum ErrorCode {
             "Create table like does not support create view."),
     ERROR_SET_CONFIG_FAILED(5076, new byte[] {'4', '2', '0', '0', '0'},
             "set config failed: %s"),
-    ERR_QUERY_EXCEPTION(5077, new byte[] {'4', '2', '0', '0', '0'},
+    ERR_QUERY_CANCELLED_BY_CRASH(5077, new byte[] {'X', 'X', '0', '0', '0'},
             "Query cancelled by crash of backends."),
     ERR_BAD_CATALOG_ERROR(5078, new byte[] {'4', '2', '0', '0', '0'},
             "Unknown catalog '%s'"),
@@ -274,6 +273,11 @@ public enum ErrorCode {
      */
     ERR_LOCK_ERROR(5300, new byte[] {'5', '5', 'P', '0', '3'}, "Failed to acquire lock: %s"),
     ERR_BEGIN_TXN_FAILED(5301, new byte[] {'5', '5', 'P', '0', '3'}, "Failed to begin transaction: %s"),
+    ERR_TXN_NOT_EXIST(5302, new byte[] {'2', '5', 'P', '0', '1'}, "Transaction %s does not exist"),
+    ERR_TXN_IMPORT_SAME_TABLE(5303, new byte[] {'2', '5', 'P', '0', '1'},
+            "NOT allowed to read or write tables that have been subjected to DML operations before"),
+    ERR_TXN_FORBID_CROSS_DB(5304, new byte[] {'2', '5', 'P', '0', '1'},
+            "Cannot execute cross-database transactions. All DML target tables must belong to the same db"),
 
     /**
      * 5400 - 5499: Internal error
@@ -339,6 +343,8 @@ public enum ErrorCode {
                     "You can modify 'kafka_offsets' property through ALTER ROUTINE LOAD and RESUME the job"),
     ERR_INSERT_COLUMN_NAME_MISMATCH(5608, new byte[] {'4', '2', '6', '0', '1'},
             "%s column: %s has no matching %s column"),
+    ERR_FAILED_WHEN_INSERT(5609, new byte[] {'2', '2', '0', '0', '0'}, "Failed when executing INSERT : '%s'"),
+    ERR_LOAD_HAS_FILTERED_DATA(5610, new byte[] {'2', '2', '0', '0', '0'}, "Insert has filtered data : %s"),
 
     /**
      * 5700 - 5799: Partition
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/LoadException.java b/fe/fe-core/src/main/java/com/starrocks/common/LoadException.java
index 1dc8b22373efea..2df789087600f1 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/LoadException.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/LoadException.java
@@ -31,4 +31,8 @@ public LoadException(String msg) {
     public LoadException(String msg, Throwable e) {
         super(msg, e);
     }
+
+    public LoadException(ErrorCode errorCode, Object... objs) {
+        super(errorCode, objs);
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/NoAliveBackendException.java b/fe/fe-core/src/main/java/com/starrocks/common/NoAliveBackendException.java
index db643a0291bf66..10baace37bb899 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/NoAliveBackendException.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/NoAliveBackendException.java
@@ -23,4 +23,8 @@ public NoAliveBackendException(String msg, Throwable cause) {
     public NoAliveBackendException(String msg) {
         super(msg);
     }
+
+    public NoAliveBackendException() {
+        super(ErrorCode.ERR_QUERY_CANCELLED_BY_CRASH);
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/TimeoutException.java b/fe/fe-core/src/main/java/com/starrocks/common/TimeoutException.java
index 3c75f0a88bf62b..02f5a99ca9e88c 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/TimeoutException.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/TimeoutException.java
@@ -22,4 +22,8 @@ public class TimeoutException extends StarRocksException {
     public TimeoutException(String msg) {
         super(msg);
     }
+
+    public TimeoutException(String type, long timeout, String errMsg) {
+        super(ErrorCode.ERR_TIMEOUT, type, timeout, errMsg);
+    }
 }
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java
index d0b71126731263..73fd8a624aa409 100644
--- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/LoadMgr.java
@@ -234,9 +234,24 @@ public void recordFinishedOrCancelledLoadJob(long jobId, EtlJobType jobType, Str
         }
     }
 
-    public InsertLoadJob registerInsertLoadJob(String label, String dbName, long tableId, long txnId, String loadId, String user,
-                                               EtlJobType jobType, long createTimestamp, long estimateScanRows,
-                                               int estimateFileNum, long estimateFileSize, long timeout,
+    public static class EstimateStats {
+        long estimateScanRows;
+        int estimateFileNum;
+        long estimateFileSize;
+
+        public EstimateStats(long estimateScanRows, int estimateFileNum, long estimateFileSize) {
+            this.estimateScanRows = estimateScanRows;
+            this.estimateFileNum = estimateFileNum;
+            this.estimateFileSize = estimateFileSize;
+        }
+    }
+
+    public InsertLoadJob registerInsertLoadJob(String label, String dbName, long tableId, long txnId, String loadId,
+                                               String user,
+                                               EtlJobType jobType,
+                                               long createTimestamp,
+                                               EstimateStats estimateStats,
+                                               long timeout,
                                                long warehouseId,
                                                Coordinator coordinator) throws StarRocksException {
         // get db id
@@ -249,8 +264,8 @@ public InsertLoadJob registerInsertLoadJob(String label, String dbName, long tab
         if (Objects.requireNonNull(jobType) == EtlJobType.INSERT) {
             loadJob = new InsertLoadJob(label, db.getId(), tableId, txnId, loadId, user,
                     createTimestamp, timeout, warehouseId, coordinator);
-            loadJob.setLoadFileInfo(estimateFileNum, estimateFileSize);
-            loadJob.setEstimateScanRow(estimateScanRows);
+            loadJob.setLoadFileInfo(estimateStats.estimateFileNum, estimateStats.estimateFileSize);
+            loadJob.setEstimateScanRow(estimateStats.estimateScanRows);
             loadJob.setTransactionId(txnId);
         } else {
             throw new LoadException("Unknown job type [" + jobType.name() + "]");
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ConnectContext.java b/fe/fe-core/src/main/java/com/starrocks/qe/ConnectContext.java
index 0ee70cee6c61f4..31a2a534b5f9df 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/ConnectContext.java
@@ -85,6 +85,7 @@
 import com.starrocks.thrift.TPipelineProfileLevel;
 import com.starrocks.thrift.TUniqueId;
 import com.starrocks.thrift.TWorkGroup;
+import com.starrocks.transaction.ExplicitTxnState;
 import com.starrocks.warehouse.Warehouse;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -251,6 +252,19 @@ public class ConnectContext {
     // `insert into table select external table`. Currently, this feature only supports hive table.
     private Optional<Boolean> useConnectorMetadataCache = Optional.empty();
 
+
+    // Explicit transaction in a session. The temporary state generated by multiple statements in a transaction is recorded in
+    // ExplicitTxnStateItem, and the transaction state is recorded in TransactionState.
+    private ExplicitTxnState explicitTxnState;
+
+    public void setExplicitTxnState(ExplicitTxnState explicitTxnState) {
+        this.explicitTxnState = explicitTxnState;
+    }
+
+    public ExplicitTxnState getExplicitTxnState() {
+        return explicitTxnState;
+    }
+
     public StmtExecutor getExecutor() {
         return executor;
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
index b20c5fabf0ac93..d5fe522d56fe8b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
@@ -99,6 +99,7 @@
 import com.starrocks.load.InsertOverwriteJobMgr;
 import com.starrocks.load.loadv2.InsertLoadJob;
 import com.starrocks.load.loadv2.LoadJob;
+import com.starrocks.load.loadv2.LoadMgr;
 import com.starrocks.metric.MetricRepo;
 import com.starrocks.metric.TableMetricsEntity;
 import com.starrocks.metric.TableMetricsRegistry;
@@ -233,6 +234,7 @@
 import com.starrocks.transaction.TransactionCommitFailedException;
 import com.starrocks.transaction.TransactionState;
 import com.starrocks.transaction.TransactionStatus;
+import com.starrocks.transaction.TransactionStmtExecutor;
 import com.starrocks.transaction.VisibleStateWaiter;
 import com.starrocks.warehouse.WarehouseIdleChecker;
 import org.apache.commons.collections4.CollectionUtils;
@@ -523,7 +525,7 @@ public void execute() throws Exception {
         context.setExecutionId(UUIDUtil.toTUniqueId(uuid));
         SessionVariable sessionVariableBackup = context.getSessionVariable();
 
-        // if use http protocal, use httpResultSender to send result to netty channel
+        // if use http protocol, use httpResultSender to send result to netty channel
         if (context instanceof HttpConnectContext) {
             httpResultSender = new HttpResultSender((HttpConnectContext) context);
         }
@@ -793,12 +795,14 @@ public void execute() throws Exception {
                 handleDelBackendBlackListStmt();
             } else if (parsedStmt instanceof PlanAdvisorStmt) {
                 handlePlanAdvisorStmt();
-            } else if (parsedStmt instanceof BeginStmt
-                    || parsedStmt instanceof CommitStmt
-                    || parsedStmt instanceof RollbackStmt) {
-                handleUnsupportedStmt();
             } else if (parsedStmt instanceof TranslateStmt) {
                 handleTranslateStmt();
+            } else if (parsedStmt instanceof BeginStmt) {
+                TransactionStmtExecutor.beginStmt(context, (BeginStmt) parsedStmt);
+            } else if (parsedStmt instanceof CommitStmt) {
+                TransactionStmtExecutor.commitStmt(context, (CommitStmt) parsedStmt);
+            } else if (parsedStmt instanceof RollbackStmt) {
+                TransactionStmtExecutor.rollbackStmt(context, (RollbackStmt) parsedStmt);
             } else {
                 context.getState().setError("Do not support this query.");
             }
@@ -820,7 +824,7 @@ public void execute() throws Exception {
             } else if (e instanceof NoAliveBackendException) {
                 context.getState().setErrType(QueryState.ErrType.INTERNAL_ERR);
             } else {
-                // TODO: some UserException doesn't belong to analysis error
+                // TODO: some StarRocksException doesn't belong to analysis error
                 // we should set such error type to internal error
                 context.getState().setErrType(QueryState.ErrType.ANALYSIS_ERR);
             }
@@ -2314,6 +2318,12 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception {
         }
 
         MetricRepo.COUNTER_LOAD_ADD.increase(1L);
+
+        if (context.getExplicitTxnState() != null) {
+            TransactionStmtExecutor.loadData(database, targetTable, execPlan, stmt, originStmt, context);
+            return;
+        }
+
         long transactionId = stmt.getTxnId();
         TransactionState txnState = null;
         String label = DebugUtil.printId(context.getExecutionId());
@@ -2346,6 +2356,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception {
         TransactionStatus txnStatus = TransactionStatus.ABORTED;
         boolean insertError = false;
         String trackingSql = "";
+
         try {
             coord = getCoordinatorFactory().createInsertScheduler(
                     context, execPlan.getFragments(), execPlan.getScanNodes(), execPlan.getDescTbl().toThrift());
@@ -2386,16 +2397,14 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception {
                         context.getQualifiedUser(),
                         EtlJobType.INSERT,
                         createTime,
-                        estimateScanRows,
-                        estimateFileNum,
-                        estimateScanFileSize,
+                        new LoadMgr.EstimateStats(estimateScanRows, estimateFileNum, estimateScanFileSize),
                         getExecTimeout(),
                         context.getCurrentWarehouseId(),
                         coord);
                 loadJob.setJobProperties(stmt.getProperties());
                 jobId = loadJob.getId();
                 if (txnState != null) {
-                    txnState.setCallbackId(jobId);
+                    txnState.addCallbackId(jobId);
                 }
             }
 
@@ -2435,8 +2444,8 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception {
                                 execPlan.getExplainString(TExplainLevel.COSTS));
                     }
 
-                    coord.cancel(ErrorCode.ERR_QUERY_EXCEPTION.formatErrorMsg());
-                    ErrorReport.reportNoAliveBackendException(ErrorCode.ERR_QUERY_EXCEPTION);
+                    coord.cancel(ErrorCode.ERR_QUERY_CANCELLED_BY_CRASH.formatErrorMsg());
+                    ErrorReport.reportNoAliveBackendException(ErrorCode.ERR_QUERY_CANCELLED_BY_CRASH);
                 } else {
                     coord.cancel(ErrorCode.ERR_TIMEOUT.formatErrorMsg(getExecType(), timeout, ""));
                     if (coord.isThriftServerHighLoad()) {
@@ -2725,7 +2734,7 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception {
                         "",
                         coord.getTrackingUrl());
             }
-        } catch (MetaNotFoundException e) {
+        } catch (StarRocksException e) {
             LOG.warn("Record info of insert load with error {}", e.getMessage(), e);
             errMsg = "Record info of insert load with error " + e.getMessage();
         }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
index ababf5b585c2c5..4bab98be1a6b2f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
@@ -423,6 +423,11 @@ private static void setOutfileSink(QueryStatement queryStmt, ExecPlan plan) {
     private static void beginTransaction(DmlStmt stmt, ConnectContext session)
             throws BeginTransactionException, RunningTxnExceedException, AnalysisException, LabelAlreadyUsedException,
             DuplicatedRequestException {
+        if (session.getExplicitTxnState() != null) {
+            stmt.setTxnId(session.getExplicitTxnState().getTransactionState().getTransactionId());
+            return;
+        }
+
         // not need begin transaction here
         // 1. explain (exclude explain analyze)
         // 2. insert into files
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DMLStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DMLStmtAnalyzer.java
index 5b9925a60b48ad..8ae5c05b2069a3 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DMLStmtAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/DMLStmtAnalyzer.java
@@ -20,8 +20,12 @@
 import com.starrocks.sql.ast.DmlStmt;
 import com.starrocks.sql.ast.InsertStmt;
 import com.starrocks.sql.ast.UpdateStmt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class DMLStmtAnalyzer {
+    private static final Logger LOGGER = LoggerFactory.getLogger(DMLStmtAnalyzer.DMLStmtAnalyzerVisitor.class);
+
     public static void analyze(DmlStmt stmt, ConnectContext context) {
         new DMLStmtAnalyzer.DMLStmtAnalyzerVisitor().analyze(stmt, context);
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java
index fed05d1bb36ea0..4522e444ec8137 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java
@@ -173,7 +173,7 @@ public DatabaseTransactionMgr(long dbId, GlobalStateMgr globalStateMgr) {
     public long beginTransaction(List<Long> tableIdList, String label, TUniqueId requestId,
                                  TransactionState.TxnCoordinator coordinator,
                                  TransactionState.LoadJobSourceType sourceType,
-                                 long listenerId,
+                                 long callbackId,
                                  long timeoutSecond,
                                  long warehouseId)
             throws DuplicatedRequestException, LabelAlreadyUsedException, RunningTxnExceedException, AnalysisException {
@@ -185,9 +185,10 @@ public long beginTransaction(List<Long> tableIdList, String label, TUniqueId req
         long tid = globalStateMgr.getGlobalTransactionMgr().getTransactionIDGenerator().getNextTransactionId();
         boolean combinedTxnLog = LakeTableHelper.supportCombinedTxnLog(sourceType);
         LOG.info("begin transaction: txn_id: {} with label {} from coordinator {}, listner id: {}",
-                tid, label, coordinator, listenerId);
+                tid, label, coordinator, callbackId);
         TransactionState transactionState = new TransactionState(dbId, tableIdList, tid, label, requestId, sourceType,
-                coordinator, listenerId, timeoutSecond * 1000);
+                coordinator, callbackId, timeoutSecond * 1000);
+
         transactionState.setPrepareTime(System.currentTimeMillis());
         transactionState.setWarehouseId(warehouseId);
         transactionState.setUseCombinedTxnLog(combinedTxnLog);
@@ -251,6 +252,66 @@ public long beginTransaction(List<Long> tableIdList, String label, TUniqueId req
         }
     }
 
+    public void upsertTransactionState(TransactionState transactionState)
+            throws DuplicatedRequestException, LabelAlreadyUsedException, RunningTxnExceedException, AnalysisException {
+        checkDatabaseDataQuota();
+
+        writeLock();
+        try {
+            checkLabel(transactionState.getLabel(), transactionState.getRequestId());
+            checkRunningTxnExceedLimit(transactionState.getSourceType());
+            unprotectUpsertTransactionState(transactionState);
+
+            if (MetricRepo.hasInit) {
+                MetricRepo.COUNTER_TXN_BEGIN.increase(1L);
+            }
+        } catch (DuplicatedRequestException e) {
+            throw e;
+        } catch (Exception e) {
+            if (MetricRepo.hasInit) {
+                MetricRepo.COUNTER_TXN_REJECT.increase(1L);
+            }
+            throw e;
+        } finally {
+            writeUnlock();
+        }
+    }
+
+    private void checkLabel(String label, TUniqueId requestId)
+            throws LabelAlreadyUsedException, DuplicatedRequestException {
+        /*
+         * Check if label already used, by following steps
+         * 1. get all existing transactions
+         * 2. if there is a PREPARE transaction, check if this is a retry request. If yes, return the
+         *    existing txn id.
+         * 3. if there is a non-aborted transaction, throw label already used exception.
+         */
+        Set<Long> existingTxnIds = unprotectedGetTxnIdsByLabel(label);
+        if (existingTxnIds != null && !existingTxnIds.isEmpty()) {
+            List<TransactionState> notAbortedTxns = Lists.newArrayList();
+            for (long txnId : existingTxnIds) {
+                TransactionState txn = unprotectedGetTransactionState(txnId);
+                Preconditions.checkNotNull(txn);
+                if (txn.getTransactionStatus() != TransactionStatus.ABORTED) {
+                    notAbortedTxns.add(txn);
+                }
+            }
+            // there should be at most 1 txn in PREPARE/COMMITTED/VISIBLE status
+            Preconditions.checkState(notAbortedTxns.size() <= 1, notAbortedTxns);
+            if (!notAbortedTxns.isEmpty()) {
+                TransactionState notAbortedTxn = notAbortedTxns.get(0);
+                if (requestId != null && notAbortedTxn.getTransactionStatus() == TransactionStatus.PREPARE
+                        && notAbortedTxn.getRequestId() != null &&
+                        notAbortedTxn.getRequestId().equals(requestId)) {
+                    // this may be a retry request for same job, just return existing txn id.
+                    throw new DuplicatedRequestException(DebugUtil.printId(requestId),
+                            notAbortedTxn.getTransactionId(), "");
+                }
+                throw new LabelAlreadyUsedException(label, notAbortedTxn.getTransactionStatus());
+            }
+        }
+    }
+
     /**
      * Change the transaction status to Prepared, indicating that the data has been prepared and is waiting for commit
      * prepared transaction process as follows:
@@ -262,7 +323,8 @@ public long beginTransaction(List<Long> tableIdList, String label, TUniqueId req
      * @param transactionId     transactionId
      * @param tabletCommitInfos tabletCommitInfos
      */
-    public void prepareTransaction(long transactionId, List<TabletCommitInfo> tabletCommitInfos,
+    public void prepareTransaction(long transactionId,
+                                   List<TabletCommitInfo> tabletCommitInfos,
                                    List<TabletFailInfo> tabletFailInfos,
                                    TxnCommitAttachment txnCommitAttachment,
                                    boolean writeEditLog)
@@ -327,7 +389,7 @@ public void prepareTransaction(long transactionId, List<TabletCommitInfo> tablet
                 listener.preCommit(transactionState, tabletCommitInfos, tabletFailInfos);
             }
 
-            TxnStateChangeCallback callback = transactionState.beforeStateTransform(TransactionStatus.PREPARED);
+            transactionState.beforeStateTransform(TransactionStatus.PREPARED);
             boolean txnOperated = false;
 
             Span unprotectedCommitSpan = TraceManager.startSpan("unprotectedPreparedTransaction", txnSpan);
@@ -362,7 +424,7 @@ public void prepareTransaction(long transactionId, List<TabletCommitInfo> tablet
                 txnSpan.setAttribute("num_partition", numPartitions);
                 unprotectedCommitSpan.end();
                 // after state transform
-                transactionState.afterStateTransform(TransactionStatus.PREPARED, txnOperated, callback, null);
+                transactionState.afterStateTransform(TransactionStatus.PREPARED, txnOperated, null);
             }
             if (writeEditLog) {
                 persistTxnStateInTxnLevelLock(transactionState);
@@ -437,7 +499,7 @@ public VisibleStateWaiter commitPreparedTransaction(long transactionId) throws S
             txnSpan.setAttribute("tables", tableListString.toString());
 
             // before state transform
-            TxnStateChangeCallback callback = transactionState.beforeStateTransform(TransactionStatus.COMMITTED);
+            transactionState.beforeStateTransform(TransactionStatus.COMMITTED);
             // transaction state transform
             boolean txnOperated = false;
 
@@ -456,7 +518,7 @@ public VisibleStateWaiter commitPreparedTransaction(long transactionId) throws S
                 txnSpan.setAttribute("num_partition", numPartitions);
                 unprotectedCommitSpan.end();
                 // after state transform
-                transactionState.afterStateTransform(TransactionStatus.COMMITTED, txnOperated, callback, null);
+                transactionState.afterStateTransform(TransactionStatus.COMMITTED, txnOperated, null);
             }
 
             persistTxnStateInTxnLevelLock(transactionState);
@@ -530,7 +592,7 @@ public void abortTransaction(long transactionId, boolean abortPrepared, String r
         }
 
         // before state transform
-        TxnStateChangeCallback callback = transactionState.beforeStateTransform(TransactionStatus.ABORTED);
+        transactionState.beforeStateTransform(TransactionStatus.ABORTED);
         boolean txnOperated = false;
 
         transactionState.writeLock();
@@ -540,7 +602,7 @@ public void abortTransaction(long transactionId, boolean abortPrepared, String r
                 txnOperated = unprotectAbortTransaction(transactionId, abortPrepared, reason);
             } finally {
                 writeUnlock();
-                transactionState.afterStateTransform(TransactionStatus.ABORTED, txnOperated, callback, reason);
+                transactionState.afterStateTransform(TransactionStatus.ABORTED, txnOperated, reason);
             }
 
             persistTxnStateInTxnLevelLock(transactionState);
@@ -1192,7 +1254,7 @@ public void finishTransaction(long transactionId, Set<Long> errorReplicaIds) thr
                     LOG.debug("after set transaction {} to visible", transactionState);
                 } finally {
                     writeUnlock();
-                    transactionState.afterStateTransform(TransactionStatus.VISIBLE, txnOperated);
+                    transactionState.afterStateTransform(TransactionStatus.VISIBLE, txnOperated, "");
                 }
 
                 persistTxnStateInTxnLevelLock(transactionState);
@@ -1845,7 +1907,8 @@ GlobalStateMgr getGlobalStateMgr() {
         return globalStateMgr;
     }
 
-    public void finishTransactionNew(TransactionState transactionState, Set<Long> publishErrorReplicas) {
+    public void finishTransactionNew(TransactionState transactionState, Set<Long> publishErrorReplicas)
+            throws StarRocksException {
         Database db = globalStateMgr.getLocalMetastore().getDb(transactionState.getDbId());
         if (db == null) {
             transactionState.writeLock();
@@ -1890,7 +1953,7 @@ public void finishTransactionNew(TransactionState transactionState, Set<Long> pu
                     txnOperated = true;
                 } finally {
                     writeUnlock();
-                    transactionState.afterStateTransform(TransactionStatus.VISIBLE, txnOperated);
+                    transactionState.afterStateTransform(TransactionStatus.VISIBLE, txnOperated, "");
                 }
                 persistTxnStateInTxnLevelLock(transactionState);
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/ExplicitTxnState.java b/fe/fe-core/src/main/java/com/starrocks/transaction/ExplicitTxnState.java
new file mode 100644
index 00000000000000..7c41573f04541d
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/ExplicitTxnState.java
@@ -0,0 +1,107 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.transaction;
+
+import com.starrocks.sql.ast.DmlStmt;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Explicit transaction in a session. The temporary state generated by multiple statements in a transaction is recorded in
+ * ExplicitTxnStateItem, and the transaction state is recorded in TransactionState.
+ */
+public class ExplicitTxnState {
+    private TransactionState transactionState;
+    private final List<ExplicitTxnStateItem> explicitTxnStateItems = new ArrayList<>();
+
+    public ExplicitTxnState() {
+    }
+
+    public TransactionState getTransactionState() {
+        return transactionState;
+    }
+
+    public void setTransactionState(TransactionState transactionState) {
+        this.transactionState = transactionState;
+    }
+
+    public List<ExplicitTxnStateItem> getTransactionStateItems() {
+        return explicitTxnStateItems;
+    }
+
+    public void addTransactionItem(ExplicitTxnStateItem explicitTxnStateItem) {
+        explicitTxnStateItems.add(explicitTxnStateItem);
+    }
+
+    public static class ExplicitTxnStateItem {
+        private long loadedRows;
+        private long loadedBytes;
+        private long filteredRows;
+
+        private List<TabletCommitInfo> tabletCommitInfos = null;
+        private List<TabletFailInfo> tabletFailInfos = null;
+
+        private DmlStmt dmlStmt;
+
+        public void addLoadedRows(long loadedRows) {
+            this.loadedRows += loadedRows;
+        }
+
+        public long getLoadedRows() {
+            return loadedRows;
+        }
+
+        public void addLoadedBytes(long loadedBytes) {
+            this.loadedBytes += loadedBytes;
+        }
+
+        public long getLoadedBytes() {
+            return loadedBytes;
+        }
+
+        public void addFilteredRows(long filteredRows) {
+            this.filteredRows += filteredRows;
+        }
+
+        public long getFilteredRows() {
+            return filteredRows;
+        }
+
+        public List<TabletCommitInfo> getTabletCommitInfos() {
+            return tabletCommitInfos;
+        }
+
+        public void setTabletCommitInfos(List<TabletCommitInfo> tabletCommitInfos) {
+            this.tabletCommitInfos = tabletCommitInfos;
+        }
+
+        public List<TabletFailInfo> getTabletFailInfos() {
+            return tabletFailInfos;
+        }
+
+        public void setTabletFailInfos(List<TabletFailInfo> tabletFailInfos) {
+            this.tabletFailInfos = tabletFailInfos;
+        }
+
+        public DmlStmt getDmlStmt() {
+            return dmlStmt;
+        }
+
+        public void setDmlStmt(DmlStmt dmlStmt) {
+            this.dmlStmt = dmlStmt;
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java
index f831e788ece224..ef9d8a896519c0 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionState.java
@@ -52,6 +52,7 @@
 import com.starrocks.common.TraceManager;
 import com.starrocks.common.io.Writable;
 import com.starrocks.metric.MetricRepo;
+import com.starrocks.persist.gson.GsonPreProcessable;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.server.WarehouseManager;
 import com.starrocks.service.FrontendOptions;
@@ -83,7 +84,7 @@
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
 
-public class TransactionState implements Writable {
+public class TransactionState implements Writable, GsonPreProcessable {
     private static final Logger LOG = LogManager.getLogger(TransactionState.class);
 
     // compare the TransactionState by txn id, desc
@@ -304,8 +305,14 @@ public String toString() {
     // NOTE: This field is only used in shared data mode.
     private long allowCommitTimeMs = -1;
 
+    //This is for compatibility and is not deleted. callbackIdList will be used later. can be deleted at 3.6
+    @Deprecated
     @SerializedName("cb")
     private long callbackId = -1;
+
+    @SerializedName("cbl")
+    private List<Long> callbackIdList;
+
     @SerializedName("to")
     private long timeoutMs = Config.stream_load_default_timeout_second * 1000L;
 
@@ -334,6 +341,7 @@ public String toString() {
 
     private Span txnSpan = null;
     private String traceParent = null;
+
     private Set<TabletCommitInfo> tabletCommitInfos = null;
 
     // For a transaction, we need to ensure that different clients obtain consistent partition information,
@@ -373,6 +381,8 @@ public TransactionState() {
         this.latch = new CountDownLatch(1);
         this.txnSpan = TraceManager.startNoopSpan();
         this.traceParent = TraceManager.toTraceParent(txnSpan.getSpanContext());
+
+        this.callbackIdList = Lists.newArrayList();
     }
 
     public TransactionState(long dbId, List<Long> tableIdList, long transactionId, String label, TUniqueId requestId,
@@ -395,7 +405,39 @@ public TransactionState(long dbId, List<Long> tableIdList, long transactionId, S
         this.publishVersionTasks = Maps.newHashMap();
         this.hasSendTask = false;
         this.latch = new CountDownLatch(1);
-        this.callbackId = callbackId;
+        this.callbackIdList = Lists.newArrayList(callbackId);
+
+        this.timeoutMs = timeoutMs;
+        this.txnSpan = TraceManager.startSpan("txn");
+        txnSpan.setAttribute("txn_id", transactionId);
+        txnSpan.setAttribute("label", label);
+        this.traceParent = TraceManager.toTraceParent(txnSpan.getSpanContext());
+    }
+
+    public TransactionState(long transactionId,
+                            String label,
+                            TUniqueId requestId,
+                            LoadJobSourceType sourceType,
+                            TxnCoordinator txnCoordinator,
+                            long timeoutMs) {
+        this.tableIdList = Lists.newArrayList();
+        this.transactionId = transactionId;
+        this.label = label;
+        this.requestId = requestId;
+        this.idToTableCommitInfos = Maps.newHashMap();
+        this.txnCoordinator = txnCoordinator;
+        this.transactionStatus = TransactionStatus.PREPARE;
+        this.sourceType = sourceType;
+        this.prepareTime = -1;
+        this.commitTime = -1;
+        this.finishTime = -1;
+        this.reason = "";
+        this.errorReplicas = Sets.newHashSet();
+        this.publishVersionTasks = Maps.newHashMap();
+        this.hasSendTask = false;
+        this.latch = new CountDownLatch(1);
+        this.callbackIdList = Lists.newArrayList();
+
         this.timeoutMs = timeoutMs;
         this.txnSpan = TraceManager.startSpan("txn");
         txnSpan.setAttribute("txn_id", transactionId);
@@ -403,8 +445,8 @@ public TransactionState(long dbId, List<Long> tableIdList, long transactionId, S
         this.traceParent = TraceManager.toTraceParent(txnSpan.getSpanContext());
     }
 
-    public void setCallbackId(long callbackId) {
-        this.callbackId = callbackId;
+    public void addCallbackId(long callbackId) {
+        this.callbackIdList.add(callbackId);
     }
 
     public void setErrorReplicas(Set<Long> newErrorReplicas) {
@@ -421,11 +463,13 @@ public Set<TabletCommitInfo> getTabletCommitInfos() {
     }
 
     public void setTabletCommitInfos(List<TabletCommitInfo> infos) {
-        this.tabletCommitInfos = Sets.newHashSet();
+        if (this.tabletCommitInfos == null) {
+            this.tabletCommitInfos = Sets.newHashSet();
+        }
+
         this.tabletCommitInfos.addAll(infos);
     }
 
-
     public boolean checkReplicaNeedSkip(Tablet tablet, Replica replica, PartitionCommitInfo partitionCommitInfo) {
         boolean isContain = tabletCommitInfosContainsReplica(tablet.getId(), replica.getBackendId(), replica.getState());
         if (isContain) {
@@ -447,7 +491,7 @@ public boolean checkReplicaNeedSkip(Tablet tablet, Replica replica, PartitionCom
 
         return true;
     }
-  
+
     public void resetTabletCommitInfos() {
         // With a high streamload frequency and too many tablets involved,
         // TabletCommitInfos will take up too much memory.
@@ -548,8 +592,12 @@ public TxnCommitAttachment getTxnCommitAttachment() {
         return txnCommitAttachment;
     }
 
-    public long getCallbackId() {
-        return callbackId;
+    public List<Long> getCallbackId() {
+        if (callbackId != -1) {
+            return Lists.newArrayList(callbackId);
+        } else {
+            return new ArrayList<>(callbackIdList);
+        }
     }
 
     public long getTimeoutMs() {
@@ -597,84 +645,82 @@ public void notifyVisible() {
         }
     }
 
-    public TxnStateChangeCallback beforeStateTransform(TransactionStatus transactionStatus)
+    public void beforeStateTransform(TransactionStatus transactionStatus)
             throws TransactionException {
-        // callback will pass to afterStateTransform since it may be deleted from
-        // GlobalTransactionMgr between beforeStateTransform and afterStateTransform
-        TxnStateChangeCallback callback = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
-                .getCallbackFactory().getCallback(callbackId);
-        // before status changed
-        if (callback != null) {
-            switch (transactionStatus) {
-                case ABORTED:
-                    callback.beforeAborted(this);
-                    break;
-                case COMMITTED:
-                    callback.beforeCommitted(this);
-                    break;
-                case PREPARED:
-                    callback.beforePrepared(this);
-                    break;
-                default:
-                    break;
-            }
-        } else if (callbackId > 0) {
-            if (Objects.requireNonNull(transactionStatus) == TransactionStatus.COMMITTED) {
-                // Maybe listener has been deleted. The txn need to be aborted later.
-                throw new TransactionException(
-                        "Failed to commit txn when callback " + callbackId + "could not be found");
-            }
-        }
-
-        return callback;
-    }
-
-    public void afterStateTransform(TransactionStatus transactionStatus, boolean txnOperated) {
-        // after status changed
-        TxnStateChangeCallback callback = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
-                .getCallbackFactory().getCallback(callbackId);
-        if (callback != null) {
-            if (Objects.requireNonNull(transactionStatus) == TransactionStatus.VISIBLE) {
-                callback.afterVisible(this, txnOperated);
+        for (Long callbackId : getCallbackId()) {
+            // callback will pass to afterStateTransform since it may be deleted from
+            // GlobalTransactionMgr between beforeStateTransform and afterStateTransform
+            TxnStateChangeCallback callback = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
+                    .getCallbackFactory().getCallback(callbackId);
+            // before status changed
+            if (callback != null) {
+                switch (transactionStatus) {
+                    case ABORTED:
+                        callback.beforeAborted(this);
+                        break;
+                    case COMMITTED:
+                        callback.beforeCommitted(this);
+                        break;
+                    case PREPARED:
+                        callback.beforePrepared(this);
+                        break;
+                    default:
+                        break;
+                }
+            } else if (callbackId > 0) {
+                if (Objects.requireNonNull(transactionStatus) == TransactionStatus.COMMITTED) {
+                    // Maybe listener has been deleted. The txn need to be aborted later.
+                    throw new TransactionException(
+                            "Failed to commit txn when callback " + callbackId + "could not be found");
+                }
             }
         }
     }
 
     public void afterStateTransform(TransactionStatus transactionStatus, boolean txnOperated,
-                                    TxnStateChangeCallback callback,
                                     String txnStatusChangeReason)
             throws StarRocksException {
-        // after status changed
-        if (callback != null) {
-            switch (transactionStatus) {
-                case ABORTED:
-                    callback.afterAborted(this, txnOperated, txnStatusChangeReason);
-                    break;
-                case COMMITTED:
-                    callback.afterCommitted(this, txnOperated);
-                    break;
-                case PREPARED:
-                    callback.afterPrepared(this, txnOperated);
-                    break;
-                default:
-                    break;
+        for (Long callbackId : getCallbackId()) {
+
+            TxnStateChangeCallback callback = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
+                    .getCallbackFactory().getCallback(callbackId);
+
+            // after status changed
+            if (callback != null) {
+                switch (transactionStatus) {
+                    case ABORTED:
+                        callback.afterAborted(this, txnOperated, txnStatusChangeReason);
+                        break;
+                    case COMMITTED:
+                        callback.afterCommitted(this, txnOperated);
+                        break;
+                    case PREPARED:
+                        callback.afterPrepared(this, txnOperated);
+                        break;
+                    case VISIBLE:
+                        callback.afterVisible(this, txnOperated);
+                        break;
+                    default:
+                        break;
+                }
             }
         }
     }
 
     public void replaySetTransactionStatus() {
-        TxnStateChangeCallback callback =
-                GlobalStateMgr.getCurrentState().getGlobalTransactionMgr().getCallbackFactory().getCallback(
-                        callbackId);
-        if (callback != null) {
-            if (transactionStatus == TransactionStatus.ABORTED) {
-                callback.replayOnAborted(this);
-            } else if (transactionStatus == TransactionStatus.COMMITTED) {
-                callback.replayOnCommitted(this);
-            } else if (transactionStatus == TransactionStatus.VISIBLE) {
-                callback.replayOnVisible(this);
-            } else if (transactionStatus == TransactionStatus.PREPARED) {
-                callback.replayOnPrepared(this);
+        for (Long callbackId : getCallbackId()) {
+            TxnStateChangeCallback callback =
+                    GlobalStateMgr.getCurrentState().getGlobalTransactionMgr().getCallbackFactory().getCallback(callbackId);
+            if (callback != null) {
+                if (transactionStatus == TransactionStatus.ABORTED) {
+                    callback.replayOnAborted(this);
+                } else if (transactionStatus == TransactionStatus.COMMITTED) {
+                    callback.replayOnCommitted(this);
+                } else if (transactionStatus == TransactionStatus.VISIBLE) {
+                    callback.replayOnVisible(this);
+                } else if (transactionStatus == TransactionStatus.PREPARED) {
+                    callback.replayOnPrepared(this);
+                }
             }
         }
     }
@@ -719,10 +765,18 @@ public long getDbId() {
         return dbId;
     }
 
+    public void setDbId(long dbId) {
+        this.dbId = dbId;
+    }
+
     public List<Long> getTableIdList() {
         return tableIdList;
     }
 
+    public void addTableIdList(Long tableId) {
+        this.tableIdList.add(tableId);
+    }
+
     public Map<Long, TableCommitInfo> getIdToTableCommitInfos() {
         return idToTableCommitInfos;
     }
@@ -796,7 +850,7 @@ public String toString() {
         sb.append(", label: ").append(label);
         sb.append(", db id: ").append(dbId);
         sb.append(", table id list: ").append(StringUtils.join(tableIdList, ","));
-        sb.append(", callback id: ").append(callbackId);
+        sb.append(", callback id: ").append(getCallbackId());
         sb.append(", coordinator: ").append(txnCoordinator.toString());
         sb.append(", transaction status: ").append(transactionStatus);
         sb.append(", error replicas num: ").append(errorReplicas.size());
@@ -1090,4 +1144,12 @@ public void clearAutomaticPartitionSnapshot() {
     public void write(DataOutput out) throws IOException {
 
     }
+
+    @Override
+    public void gsonPreProcess() throws IOException {
+        //For compatibility, if the implicit transaction can be rolled back, duplicates will be removed in getCallbackId.
+        if (callbackId == -1 && !callbackIdList.isEmpty()) {
+            callbackId = callbackIdList.get(0);
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateBatch.java b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateBatch.java
index f82425a13e8ee7..dd04fe91f56708 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateBatch.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateBatch.java
@@ -98,12 +98,14 @@ public void setTransactionStatus(TransactionStatus transactionStatus) {
     // a proxy method
     public void afterVisible(TransactionStatus transactionStatus, boolean txnOperated) {
         for (TransactionState transactionState : transactionStates) {
-            // after status changed
-            TxnStateChangeCallback callback = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
-                    .getCallbackFactory().getCallback(transactionState.getCallbackId());
-            if (callback != null) {
-                if (Objects.requireNonNull(transactionStatus) == TransactionStatus.VISIBLE) {
-                    callback.afterVisible(transactionState, txnOperated);
+            for (Long callbackId : transactionState.getCallbackId()) {
+                // after status changed
+                TxnStateChangeCallback callback = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
+                        .getCallbackFactory().getCallback(callbackId);
+                if (callback != null) {
+                    if (Objects.requireNonNull(transactionStatus) == TransactionStatus.VISIBLE) {
+                        callback.afterVisible(transactionState, txnOperated);
+                    }
                 }
             }
         }
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStmtExecutor.java
new file mode 100644
index 00000000000000..cb611d1c051284
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStmtExecutor.java
@@ -0,0 +1,499 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.transaction;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Ints;
+import com.starrocks.catalog.Database;
+import com.starrocks.catalog.OlapTable;
+import com.starrocks.catalog.Table;
+import com.starrocks.common.Config;
+import com.starrocks.common.ErrorCode;
+import com.starrocks.common.ErrorReportException;
+import com.starrocks.common.LoadException;
+import com.starrocks.common.NoAliveBackendException;
+import com.starrocks.common.StarRocksException;
+import com.starrocks.common.TimeoutException;
+import com.starrocks.common.util.DebugUtil;
+import com.starrocks.common.util.concurrent.lock.LockTimeoutException;
+import com.starrocks.lake.LakeTableHelper;
+import com.starrocks.load.EtlJobType;
+import com.starrocks.load.loadv2.InsertLoadJob;
+import com.starrocks.load.loadv2.LoadJob;
+import com.starrocks.load.loadv2.LoadMgr;
+import com.starrocks.metric.MetricRepo;
+import com.starrocks.metric.TableMetricsEntity;
+import com.starrocks.metric.TableMetricsRegistry;
+import com.starrocks.planner.FileScanNode;
+import com.starrocks.planner.OlapScanNode;
+import com.starrocks.planner.ScanNode;
+import com.starrocks.qe.ConnectContext;
+import com.starrocks.qe.DefaultCoordinator;
+import com.starrocks.qe.DmlType;
+import com.starrocks.qe.OriginStatement;
+import com.starrocks.qe.QeProcessorImpl;
+import com.starrocks.qe.SessionVariable;
+import com.starrocks.qe.scheduler.Coordinator;
+import com.starrocks.rpc.RpcException;
+import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.service.FrontendOptions;
+import com.starrocks.sql.ast.CreateTableAsSelectStmt;
+import com.starrocks.sql.ast.DeleteStmt;
+import com.starrocks.sql.ast.DmlStmt;
+import com.starrocks.sql.ast.InsertStmt;
+import com.starrocks.sql.ast.StatementBase;
+import com.starrocks.sql.ast.UpdateStmt;
+import com.starrocks.sql.ast.txn.BeginStmt;
+import com.starrocks.sql.ast.txn.CommitStmt;
+import com.starrocks.sql.ast.txn.RollbackStmt;
+import com.starrocks.sql.plan.ExecPlan;
+import com.starrocks.task.LoadEtlTask;
+import com.starrocks.thrift.TExplainLevel;
+import com.starrocks.thrift.TLoadJobType;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static com.starrocks.common.ErrorCode.ERR_TXN_NOT_EXIST;
+
+public class TransactionStmtExecutor {
+    private static final Logger LOG = LogManager.getLogger(TransactionStmtExecutor.class);
+
+    public static void beginStmt(ConnectContext context, BeginStmt stmt) {
+        if (context.getExplicitTxnState() != null) {
+            //Repeated begin does not create a new transaction
+            ExplicitTxnState explicitTxnState = context.getExplicitTxnState();
+            String label = explicitTxnState.getTransactionState().getLabel();
+            long transactionId = explicitTxnState.getTransactionState().getTransactionId();
+            context.getState().setOk(0, 0, buildMessage(label, TransactionStatus.PREPARE, transactionId, -1));
+            return;
+        }
+
+        long transactionId = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
+                .getTransactionIDGenerator().getNextTransactionId();
+        String label = DebugUtil.printId(context.getExecutionId());
+        TransactionState transactionState = new TransactionState(transactionId, label, null,
+                TransactionState.LoadJobSourceType.INSERT_STREAMING,
+                new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, FrontendOptions.getLocalHostAddress()),
+                context.getExecTimeout() * 1000L);
+
+        transactionState.setPrepareTime(System.currentTimeMillis());
+        transactionState.setWarehouseId(context.getCurrentWarehouseId());
+        boolean combinedTxnLog = LakeTableHelper.supportCombinedTxnLog(TransactionState.LoadJobSourceType.INSERT_STREAMING);
+        transactionState.setUseCombinedTxnLog(combinedTxnLog);
+
+        ExplicitTxnState explicitTxnState = new ExplicitTxnState();
+        explicitTxnState.setTransactionState(transactionState);
+        context.setExplicitTxnState(explicitTxnState);
+
+        context.getState().setOk(0, 0, buildMessage(label, TransactionStatus.PREPARE, transactionId, -1));
+    }
+
+    public static void loadData(Database database,
+                                Table targetTable,
+                                ExecPlan execPlan,
+                                DmlStmt dmlStmt,
+                                OriginStatement originStmt,
+                                ConnectContext context) {
+        Coordinator coordinator = new DefaultCoordinator.Factory().createInsertScheduler(
+                context, execPlan.getFragments(), execPlan.getScanNodes(), execPlan.getDescTbl().toThrift());
+        ExplicitTxnState explicitTxnState = context.getExplicitTxnState();
+        TransactionState transactionState = explicitTxnState.getTransactionState();
+
+        try {
+            if (transactionState.getDbId() == 0) {
+                transactionState.setDbId(database.getId());
+                DatabaseTransactionMgr databaseTransactionMgr =
+                        GlobalStateMgr.getCurrentState().getGlobalTransactionMgr()
+                                .getDatabaseTransactionMgr(database.getId());
+                databaseTransactionMgr.upsertTransactionState(transactionState);
+            }
+
+            if (database.getId() != transactionState.getDbId()) {
+                throw ErrorReportException.report(ErrorCode.ERR_TXN_FORBID_CROSS_DB);
+            }
+
+            if (transactionState.getTableIdList().contains(targetTable.getId())) {
+                throw ErrorReportException.report(ErrorCode.ERR_TXN_IMPORT_SAME_TABLE);
+            }
+            transactionState.addTableIdList(targetTable.getId());
+
+            ExplicitTxnState.ExplicitTxnStateItem item =
+                    load(database, targetTable, execPlan, dmlStmt, originStmt, context, coordinator);
+            explicitTxnState.addTransactionItem(item);
+
+            context.getState().setOk(item.getLoadedRows(), Ints.saturatedCast(item.getFilteredRows()),
+                    buildMessage(transactionState.getLabel(), TransactionStatus.PREPARE,
+                            transactionState.getTransactionId(), database.getId()));
+        } catch (StarRocksException | RpcException | InterruptedException e) {
+            context.getState().setError(e.getMessage());
+        }
+    }
+
+    public static void commitStmt(ConnectContext context, CommitStmt stmt) {
+        ExplicitTxnState explicitTxnState = context.getExplicitTxnState();
+        if (explicitTxnState == null) {
+            //commit statement not in after begin, do nothing
+            return;
+        }
+
+        if (explicitTxnState.getTransactionStateItems().isEmpty()) {
+            TransactionState transactionState = explicitTxnState.getTransactionState();
+            context.setExplicitTxnState(null);
+            context.getState().setOk(0, 0, buildMessage(transactionState.getLabel(),
+                    TransactionStatus.VISIBLE, transactionState.getTransactionId(), -1));
+            return;
+        }
+
+        GlobalTransactionMgr transactionMgr = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr();
+
+        long transactionId = context.getExplicitTxnState().getTransactionState().getTransactionId();
+        TransactionState transactionState = context.getExplicitTxnState().getTransactionState();
+        long databaseId = transactionState.getDbId();
+        Database database = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(databaseId);
+
+        try {
+            int timeout = context.getSessionVariable().getQueryTimeoutS();
+            long jobDeadLineMs = System.currentTimeMillis() + timeout * 1000L;
+
+            List<TabletCommitInfo> commitInfos = Lists.newArrayList();
+            List<TabletFailInfo> failInfos = Lists.newArrayList();
+            long loadedRows = 0;
+            for (ExplicitTxnState.ExplicitTxnStateItem item : explicitTxnState.getTransactionStateItems()) {
+                commitInfos.addAll(item.getTabletCommitInfos());
+                failInfos.addAll(item.getTabletFailInfos());
+                loadedRows += item.getLoadedRows();
+            }
+
+            TxnCommitAttachment txnCommitAttachment = new InsertTxnCommitAttachment(loadedRows);
+            VisibleStateWaiter visibleWaiter = transactionMgr.retryCommitOnRateLimitExceeded(
+                    database,
+                    transactionId,
+                    commitInfos,
+                    failInfos,
+                    txnCommitAttachment,
+                    timeout);
+
+            long publishWaitMs = Config.enable_sync_publish ? jobDeadLineMs - System.currentTimeMillis() :
+                    context.getSessionVariable().getTransactionVisibleWaitTimeout() * 1000;
+
+            TransactionStatus txnStatus;
+            if (visibleWaiter.await(publishWaitMs, TimeUnit.MILLISECONDS)) {
+                txnStatus = TransactionStatus.VISIBLE;
+            } else {
+                txnStatus = TransactionStatus.COMMITTED;
+            }
+
+            List<ExplicitTxnState.ExplicitTxnStateItem> explicitTxnStateItems
+                    = explicitTxnState.getTransactionStateItems();
+            List<Long> callbackIds = transactionState.getCallbackId();
+            Preconditions.checkArgument(explicitTxnStateItems.size() == callbackIds.size());
+
+            for (int i = 0; i < explicitTxnStateItems.size(); i++) {
+                ExplicitTxnState.ExplicitTxnStateItem item = explicitTxnStateItems.get(i);
+
+                DmlStmt dmlStmt = item.getDmlStmt();
+                Table targetTable = GlobalStateMgr.getCurrentState().getLocalMetastore()
+                        .getTable(database.getFullName(), dmlStmt.getTableName().getTbl());
+
+                MetricRepo.COUNTER_LOAD_FINISHED.increase(1L);
+                // collect table-level metrics
+                TableMetricsEntity entity = TableMetricsRegistry.getInstance().getMetricsEntity(targetTable.getId());
+                entity.counterInsertLoadFinishedTotal.increase(1L);
+                entity.counterInsertLoadRowsTotal.increase(item.getLoadedRows());
+                entity.counterInsertLoadBytesTotal.increase(item.getLoadedBytes());
+
+                GlobalStateMgr.getCurrentState().getOperationListenerBus()
+                        .onDMLStmtJobTransactionFinish(transactionState, database, targetTable, DmlType.fromStmt(dmlStmt));
+
+                context.getGlobalStateMgr().getLoadMgr().recordFinishedOrCancelledLoadJob(
+                        callbackIds.get(i),
+                        EtlJobType.INSERT,
+                        "",
+                        "");
+            }
+
+            context.getState().setOk(0, 0,
+                    buildMessage(transactionState.getLabel(), txnStatus, transactionId, database.getId()));
+        } catch (StarRocksException | LockTimeoutException e) {
+            LOG.warn("errors when abort txn", e);
+            context.getState().setError(e.getMessage());
+        } finally {
+            //clean global explicit transaction state
+            context.setExplicitTxnState(null);
+        }
+    }
+
+    public static void rollbackStmt(ConnectContext context, RollbackStmt stmt) {
+        ExplicitTxnState explicitTxnState = context.getExplicitTxnState();
+        if (explicitTxnState == null) {
+            //rollback statement not in after begin, do nothing
+            return;
+        }
+
+        if (explicitTxnState.getTransactionStateItems().isEmpty()) {
+            TransactionState transactionState = explicitTxnState.getTransactionState();
+            context.setExplicitTxnState(null);
+            context.getState().setOk(0, 0, buildMessage(transactionState.getLabel(),
+                    TransactionStatus.ABORTED, transactionState.getTransactionId(), -1));
+            return;
+        }
+
+        LoadMgr loadMgr = GlobalStateMgr.getCurrentState().getLoadMgr();
+        long transactionId = context.getExplicitTxnState().getTransactionState().getTransactionId();
+        try {
+            TransactionState transactionState = context.getExplicitTxnState().getTransactionState();
+            long databaseId = transactionState.getDbId();
+
+            GlobalTransactionMgr transactionMgr = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr();
+            List<TabletCommitInfo> commitInfos = Lists.newArrayList();
+            List<TabletFailInfo> failInfos = Lists.newArrayList();
+
+            List<ExplicitTxnState.ExplicitTxnStateItem> explicitTxnStateItems
+                    = explicitTxnState.getTransactionStateItems();
+            List<Long> callbackIds = transactionState.getCallbackId();
+            Preconditions.checkArgument(explicitTxnStateItems.size() == callbackIds.size());
+
+            for (int i = 0; i < explicitTxnStateItems.size(); i++) {
+                ExplicitTxnState.ExplicitTxnStateItem item = explicitTxnStateItems.get(i);
+                commitInfos.addAll(item.getTabletCommitInfos());
+                failInfos.addAll(item.getTabletFailInfos());
+                loadMgr.recordFinishedOrCancelledLoadJob(callbackIds.get(i), EtlJobType.INSERT, "", "");
+            }
+
+            transactionMgr.abortTransaction(
+                    databaseId,
+                    transactionId,
+                    "rollback transaction by user",
+                    commitInfos,
+                    failInfos,
+                    null);
+
+            context.getState().setOk(0, 0,
+                    buildMessage(transactionState.getLabel(), TransactionStatus.ABORTED, transactionId, -1));
+        } catch (StarRocksException e) {
+            // just print a log if abort txn failed. This failure do not need to pass to user.
+            // user only concern abort how txn failed.
+            LOG.warn("errors when abort txn", e);
+            context.getState().setError(e.getMessage());
+        } finally {
+            //clean global explicit transaction state
+            context.setExplicitTxnState(null);
+        }
+    }
+
+    public static String buildMessage(String label, TransactionStatus txnStatus, long transactionId, long databaseId) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("{");
+        sb.append("'label':'").append(label).append("', ");
+        sb.append("'status':'").append(txnStatus.name()).append("', ");
+        sb.append("'txnId':'").append(transactionId).append("'");
+
+        if (txnStatus == TransactionStatus.COMMITTED) {
+            GlobalTransactionMgr transactionMgr = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr();
+            String timeoutInfo = transactionMgr.getTxnPublishTimeoutDebugInfo(databaseId, transactionId);
+            LOG.warn("txn {} publish timeout {}", transactionId, timeoutInfo);
+            if (timeoutInfo.length() > 240) {
+                timeoutInfo = timeoutInfo.substring(0, 240) + "...";
+            }
+            String errMsg = "Publish timeout " + timeoutInfo;
+
+            sb.append(", 'err':'").append(errMsg).append("'");
+        }
+
+        sb.append("}");
+
+        return sb.toString();
+    }
+
+    public static ExplicitTxnState.ExplicitTxnStateItem load(
+            Database database,
+            Table targetTable,
+            ExecPlan execPlan,
+            DmlStmt dmlStmt,
+            OriginStatement originStmt,
+            ConnectContext context,
+            Coordinator coord) throws StarRocksException, InterruptedException, RpcException {
+        try {
+            GlobalTransactionMgr transactionMgr = GlobalStateMgr.getCurrentState().getGlobalTransactionMgr();
+
+            MetricRepo.COUNTER_LOAD_ADD.increase(1L);
+
+            // Every time set no send flag and clean all data in buffer
+            if (context.getMysqlChannel() != null) {
+                context.getMysqlChannel().reset();
+            }
+
+            long transactionId = dmlStmt.getTxnId();
+            TransactionState txnState = transactionMgr.getTransactionState(database.getId(), transactionId);
+            if (txnState == null) {
+                throw ErrorReportException.report(ERR_TXN_NOT_EXIST, transactionId);
+            }
+            if (!txnState.getTableIdList().contains(targetTable.getId())) {
+                txnState.getTableIdList().add(targetTable.getId());
+                txnState.addTableIndexes((OlapTable) targetTable);
+            }
+
+            String label = txnState.getLabel();
+            if (execPlan.getScanNodes().stream()
+                    .anyMatch(scanNode -> scanNode instanceof OlapScanNode || scanNode instanceof FileScanNode)) {
+                coord.setLoadJobType(TLoadJobType.INSERT_QUERY);
+            } else {
+                coord.setLoadJobType(TLoadJobType.INSERT_VALUES);
+            }
+
+            InsertLoadJob loadJob = context.getGlobalStateMgr().getLoadMgr().registerInsertLoadJob(
+                    label,
+                    database.getFullName(),
+                    targetTable.getId(),
+                    transactionId,
+                    DebugUtil.printId(context.getExecutionId()),
+                    context.getQualifiedUser(),
+                    EtlJobType.INSERT,
+                    System.currentTimeMillis(),
+                    estimate(execPlan),
+                    context.getSessionVariable().getQueryTimeoutS(),
+                    context.getCurrentWarehouseId(),
+                    coord);
+            loadJob.setJobProperties(dmlStmt.getProperties());
+            long jobId = loadJob.getId();
+            txnState.addCallbackId(jobId);
+            coord.setLoadJobId(jobId);
+
+            QeProcessorImpl.QueryInfo queryInfo =
+                    new QeProcessorImpl.QueryInfo(context, originStmt.originStmt, coord);
+            QeProcessorImpl.INSTANCE.registerQuery(context.getExecutionId(), queryInfo);
+
+            coord.exec();
+            coord.setExecPlan(execPlan);
+
+            int timeout = context.getSessionVariable().getQueryTimeoutS();
+            coord.join(timeout);
+            if (!coord.isDone()) {
+                /*
+                 * In this case, There are two factors that lead query cancelled:
+                 * 1: TIMEOUT
+                 * 2: BE EXCEPTION
+                 * So we should distinguish these two factors.
+                 */
+                if (!coord.checkBackendState()) {
+                    // When enable_collect_query_detail_info is set to true, the plan will be recorded in the query detail,
+                    // and hence there is no need to log it here.
+                    if (Config.log_plan_cancelled_by_crash_be && context.getQueryDetail() == null) {
+                        LOG.warn("Query cancelled by crash of backends [QueryId={}] [SQL={}] [Plan={}]",
+                                DebugUtil.printId(context.getExecutionId()),
+                                originStmt == null ? "" : originStmt.originStmt,
+                                execPlan.getExplainString(TExplainLevel.COSTS));
+                    }
+
+                    coord.cancel(ErrorCode.ERR_QUERY_CANCELLED_BY_CRASH.formatErrorMsg());
+                    throw new NoAliveBackendException();
+                } else {
+                    coord.cancel(ErrorCode.ERR_TIMEOUT.formatErrorMsg(getExecType(dmlStmt), timeout, ""));
+                    if (coord.isThriftServerHighLoad()) {
+                        throw new TimeoutException(getExecType(dmlStmt),
+                                timeout,
+                                "Please check the thrift-server-pool metrics, " +
+                                        "if the pool size reaches thrift_server_max_worker_threads(default is 4096), " +
+                                        "you can set the config to a higher value in fe.conf, " +
+                                        "or set parallel_fragment_exec_instance_num to a lower value in session variable");
+                    } else {
+                        throw new TimeoutException(getExecType(dmlStmt), timeout,
+                                String.format("please increase the '%s' session variable and retry",
+                                        SessionVariable.INSERT_TIMEOUT));
+                    }
+                }
+            }
+
+            if (!coord.getExecStatus().ok()) {
+                throw new LoadException(ErrorCode.ERR_FAILED_WHEN_INSERT, coord.getExecStatus().getErrorMsg().isEmpty() ?
+                        coord.getExecStatus().getErrorCodeString() : coord.getExecStatus().getErrorMsg());
+            }
+
+            LOG.debug("delta files is {}", coord.getDeltaUrls());
+
+            loadJob.updateLoadingStatus(coord.getLoadCounters());
+
+            long loadedRows = coord.getLoadCounters().get(LoadEtlTask.DPP_NORMAL_ALL) != null ?
+                    Long.parseLong(coord.getLoadCounters().get(LoadEtlTask.DPP_NORMAL_ALL)) : 0;
+
+            // filteredRows is stored in int64_t in the backend, so use long here.
+            long filteredRows = coord.getLoadCounters().get(LoadEtlTask.DPP_ABNORMAL_ALL) != null ?
+                    Long.parseLong(coord.getLoadCounters().get(LoadEtlTask.DPP_ABNORMAL_ALL)) : 0;
+
+            // insert will fail if 'filtered rows / total rows' exceeds max_filter_ratio
+            // for native table and external catalog table(without insert load job)
+            if (filteredRows > (filteredRows + loadedRows) * dmlStmt.getMaxFilterRatio()) {
+                String trackingSql = "select tracking_log from information_schema.load_tracking_logs where job_id=" + jobId;
+                throw new LoadException(ErrorCode.ERR_LOAD_HAS_FILTERED_DATA,
+                        "txn_id = " + transactionId + ", tracking sql = " + trackingSql);
+            }
+
+            long loadedBytes = coord.getLoadCounters().get(LoadJob.LOADED_BYTES) != null ?
+                    Long.parseLong(coord.getLoadCounters().get(LoadJob.LOADED_BYTES)) : 0;
+
+            ExplicitTxnState.ExplicitTxnStateItem item = new ExplicitTxnState.ExplicitTxnStateItem();
+            item.setDmlStmt(dmlStmt);
+            item.setTabletCommitInfos(TabletCommitInfo.fromThrift(coord.getCommitInfos()));
+            item.setTabletFailInfos(TabletFailInfo.fromThrift(coord.getFailInfos()));
+            item.addLoadedRows(loadedRows);
+            item.addFilteredRows(filteredRows);
+            item.addLoadedBytes(loadedBytes);
+            return item;
+        } finally {
+            QeProcessorImpl.INSTANCE.unregisterQuery(context.getExecutionId());
+        }
+    }
+
+    public static LoadMgr.EstimateStats estimate(ExecPlan execPlan) {
+        long estimateScanRows = -1;
+        int estimateFileNum = 0;
+        long estimateScanFileSize = 0;
+
+        boolean needQuery = false;
+        for (ScanNode scanNode : execPlan.getScanNodes()) {
+            if (scanNode instanceof OlapScanNode) {
+                estimateScanRows += ((OlapScanNode) scanNode).getActualRows();
+                needQuery = true;
+            }
+            if (scanNode instanceof FileScanNode) {
+                estimateFileNum += ((FileScanNode) scanNode).getFileNum();
+                estimateScanFileSize += ((FileScanNode) scanNode).getFileTotalSize();
+                needQuery = true;
+            }
+        }
+
+        if (needQuery) {
+            estimateScanRows = execPlan.getFragments().get(0).getPlanRoot().getCardinality();
+        }
+
+        return new LoadMgr.EstimateStats(estimateScanRows, estimateFileNum, estimateScanFileSize);
+    }
+
+    public static String getExecType(StatementBase stmt) {
+        if (stmt instanceof InsertStmt || stmt instanceof CreateTableAsSelectStmt) {
+            return "Insert";
+        } else if (stmt instanceof UpdateStmt) {
+            return "Update";
+        } else if (stmt instanceof DeleteStmt) {
+            return "Delete";
+        } else {
+            return "Query";
+        }
+    }
+}
diff --git a/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java
index 07a193bc38a709..e177ea12017a28 100644
--- a/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java
@@ -414,7 +414,7 @@ public void testGetSingleTranInfo() throws AnalysisException {
         assertTrue(currentTime > TimeUtils.timeStringToLong(txnInfo.get(8)));
         assertEquals("", txnInfo.get(9));
         assertEquals("0", txnInfo.get(10));
-        assertEquals("-1", txnInfo.get(11));
+        assertEquals("[-1]", txnInfo.get(11));
         assertEquals(String.valueOf(Config.stream_load_default_timeout_second * 1000L), txnInfo.get(12));
     }
 

From ea3ff40f01037f6deae068277a6217d8ba0f6abe Mon Sep 17 00:00:00 2001
From: Seaven <seaven_7@qq.com>
Date: Tue, 21 Jan 2025 10:25:12 +0800
Subject: [PATCH 58/71] [Refactor] Refactor Optimizer&OptimizerContext
 interface (#55117)

Signed-off-by: Seaven <seaven_7@qq.com>
---
 .../authorization/ColumnPrivilege.java        |   18 +-
 .../java/com/starrocks/sql/DeletePlanner.java |    7 +-
 .../java/com/starrocks/sql/InsertPlanner.java |    7 +-
 .../starrocks/sql/ShortCircuitPlanner.java    |   14 +-
 .../com/starrocks/sql/StatementPlanner.java   |   55 +-
 .../java/com/starrocks/sql/UpdatePlanner.java |   11 +-
 .../sql/analyzer/AnalyzeStmtAnalyzer.java     |    7 +-
 .../analyzer/MaterializedViewAnalyzer.java    |    9 +-
 .../optimizer/MaterializedViewOptimizer.java  |   25 +-
 .../sql/optimizer/MvRewritePreprocessor.java  |    3 +-
 .../sql/optimizer/OptExpression.java          |   17 -
 .../starrocks/sql/optimizer/Optimizer.java    | 1018 +----------------
 .../sql/optimizer/OptimizerContext.java       |  291 ++---
 .../sql/optimizer/OptimizerFactory.java       |   62 +
 ...mizerConfig.java => OptimizerOptions.java} |   46 +-
 .../sql/optimizer/QueryOptimizer.java         |  957 ++++++++++++++++
 .../sql/optimizer/ShortCircuitOptimizer.java  |   68 ++
 .../sql/optimizer/dump/QueryDumpInfo.java     |    2 +-
 .../rewrite/JoinPredicatePushdown.java        |    4 +-
 .../PruneProjectColumnsRule.java              |    2 +-
 .../materialization/MvRewriteStrategy.java    |   12 +-
 .../materialization/MvUtils.java              |   26 +-
 .../rule/TextMatchBasedRewriteRule.java       |    2 +-
 .../sql/optimizer/task/ApplyRuleTask.java     |    2 -
 .../sql/optimizer/task/OptimizerTask.java     |    6 +-
 .../sql/optimizer/task/RewriteTreeTask.java   |    2 +-
 .../sql/optimizer/task/TaskContext.java       |   14 -
 .../sql/optimizer/task/TaskScheduler.java     |   27 +-
 .../validate/MVRewriteValidator.java          |   18 +-
 .../java/com/starrocks/sql/plan/ExecPlan.java |   11 +-
 .../sql/plan/PlanFragmentBuilder.java         |   25 +-
 .../connector/hive/HiveMetadataTest.java      |    4 +-
 .../hive/HiveStatisticsProviderTest.java      |    4 +-
 .../iceberg/IcebergMetadataTest.java          |    8 +-
 .../connector/paimon/PaimonMetadataTest.java  |    5 +-
 .../sql/optimizer/OptimizerTaskTest.java      |  184 +--
 .../sql/optimizer/rule/BinderTest.java        |    7 +-
 .../OlapScanImplementationRuleTest.java       |    7 +-
 .../DistributionPrunerRuleTest.java           |    5 +-
 .../MergeLimitWithSortRuleTest.java           |    5 +-
 .../PartitionPruneRuleTest.java               |   11 +-
 .../transformation/PushDownAggRuleTest.java   |    5 +-
 .../transformation/PushDownScanRuleTest.java  |    5 +-
 .../materialization/MVTestBase.java           |    7 +-
 .../MvRewritePreprocessorTest.java            |   77 +-
 .../MvRewriteStrategyTest.java                |   13 +-
 .../materialization/MvRewriteTest.java        |   12 +-
 .../statistics/StatisticsCalculatorTest.java  |    4 +-
 .../sql/plan/PartitionPruneTest.java          |    4 +-
 .../com/starrocks/utframe/UtFrameUtils.java   |   22 +-
 50 files changed, 1595 insertions(+), 1562 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java
 rename fe/fe-core/src/main/java/com/starrocks/sql/optimizer/{OptimizerConfig.java => OptimizerOptions.java} (54%)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java

diff --git a/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java b/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java
index 843a3e30a3d8eb..be4884095484e4 100644
--- a/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java
+++ b/fe/fe-core/src/main/java/com/starrocks/authorization/ColumnPrivilege.java
@@ -37,7 +37,8 @@
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptExpressionVisitor;
 import com.starrocks.sql.optimizer.Optimizer;
-import com.starrocks.sql.optimizer.OptimizerConfig;
+import com.starrocks.sql.optimizer.OptimizerFactory;
+import com.starrocks.sql.optimizer.OptimizerOptions;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
@@ -110,13 +111,14 @@ public static void check(ConnectContext context, QueryStatement stmt, List<Table
             TransformerContext transformerContext = new TransformerContext(columnRefFactory, context, mvTransformerContext);
             logicalPlan = new RelationTransformer(transformerContext).transformWithSelectLimit(stmt.getQueryRelation());
 
-            OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED);
-            optimizerConfig.disableRule(RuleType.GP_SINGLE_TABLE_MV_REWRITE);
-            optimizerConfig.disableRule(RuleType.GP_MULTI_TABLE_MV_REWRITE);
-            optimizerConfig.disableRule(RuleType.GP_PRUNE_EMPTY_OPERATOR);
-            Optimizer optimizer = new Optimizer(optimizerConfig);
-            optimizedPlan = optimizer.optimize(context, logicalPlan.getRoot(),
-                    new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+            OptimizerOptions optimizerOptions = new OptimizerOptions(OptimizerOptions.OptimizerStrategy.RULE_BASED);
+            optimizerOptions.disableRule(RuleType.GP_SINGLE_TABLE_MV_REWRITE);
+            optimizerOptions.disableRule(RuleType.GP_MULTI_TABLE_MV_REWRITE);
+            optimizerOptions.disableRule(RuleType.GP_PRUNE_EMPTY_OPERATOR);
+            Optimizer optimizer =
+                    OptimizerFactory.create(OptimizerFactory.initContext(context, columnRefFactory, optimizerOptions));
+            optimizedPlan = optimizer.optimize(logicalPlan.getRoot(),
+                    new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn()));
 
             optimizedPlan.getOp().accept(new ScanColumnCollector(tableObjectToTableName, scanColumns), optimizedPlan, null);
         }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java
index db013ed7c2fb45..1eee21104c90c7 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/DeletePlanner.java
@@ -37,6 +37,7 @@
 import com.starrocks.sql.ast.QueryRelation;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
@@ -72,13 +73,11 @@ public ExecPlan plan(DeleteStmt deleteStatement, ConnectContext session) {
             // Non-query must use the strategy assign scan ranges per driver sequence, which local shuffle agg cannot use.
             session.getSessionVariable().setEnableLocalShuffleAgg(false);
 
-            Optimizer optimizer = new Optimizer();
+            Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.initContext(session, columnRefFactory));
             OptExpression optimizedPlan = optimizer.optimize(
-                    session,
                     logicalPlan.getRoot(),
                     new PhysicalPropertySet(),
-                    new ColumnRefSet(logicalPlan.getOutputColumn()),
-                    columnRefFactory);
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
             ExecPlan execPlan = PlanFragmentBuilder.createPhysicalPlan(optimizedPlan, session,
                     logicalPlan.getOutputColumn(), columnRefFactory,
                     colNames, TResultSinkType.MYSQL_PROTOCAL, false);
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/InsertPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/InsertPlanner.java
index ad72ac2b80f9ee..c9dc8b1f64cc7f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/InsertPlanner.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/InsertPlanner.java
@@ -78,6 +78,7 @@
 import com.starrocks.sql.common.TypeManager;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.DistributionProperty;
@@ -511,18 +512,16 @@ private ExecPlan buildExecPlanWithRetry(InsertStmt insertStmt, ConnectContext se
     private ExecPlan buildExecPlan(InsertStmt insertStmt, ConnectContext session, List<ColumnRefOperator> outputColumns,
                                    LogicalPlan logicalPlan, ColumnRefFactory columnRefFactory,
                                    QueryRelation queryRelation, Table targetTable) {
-        Optimizer optimizer = new Optimizer();
         PhysicalPropertySet requiredPropertySet = createPhysicalPropertySet(insertStmt, outputColumns,
                 session.getSessionVariable());
         OptExpression optimizedPlan;
 
         try (Timer ignore2 = Tracers.watchScope("Optimizer")) {
+            Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.initContext(session, columnRefFactory));
             optimizedPlan = optimizer.optimize(
-                    session,
                     logicalPlan.getRoot(),
                     requiredPropertySet,
-                    new ColumnRefSet(logicalPlan.getOutputColumn()),
-                    columnRefFactory);
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
         }
 
         //8. Build fragment exec plan
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ShortCircuitPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/ShortCircuitPlanner.java
index b279767c527468..e70ed97376159e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/ShortCircuitPlanner.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/ShortCircuitPlanner.java
@@ -24,7 +24,6 @@
 import com.starrocks.sql.optimizer.OptExpressionVisitor;
 import com.starrocks.sql.optimizer.operator.ColumnFilterConverter;
 import com.starrocks.sql.optimizer.operator.Operator;
-import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalFilterOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalLimitOperator;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
@@ -73,19 +72,12 @@ public static BaseLogicalPlanChecker createLogicalPlanChecker(OptExpression root
         }
     }
 
-    public static OptExpression checkSupportShortCircuitRead(OptExpression root, ConnectContext connectContext) {
+    public static boolean checkSupportShortCircuitRead(OptExpression root, ConnectContext connectContext) {
         if (!connectContext.getSessionVariable().isEnableShortCircuit()) {
-            root.setShortCircuit(false);
-            return root;
-        }
-        boolean supportShortCircuit = root.getOp().accept(new LogicalPlanChecker(), root, null);
-        if (supportShortCircuit && OperatorType.LOGICAL_LIMIT.equals(root.getOp().getOpType())) {
-            root = root.getInputs().get(0);
+            return false;
         }
-        root.setShortCircuit(supportShortCircuit);
-        return root;
+        return root.getOp().accept(new LogicalPlanChecker(), root, null);
     }
-
     protected static boolean isRedundant(Map<ColumnRefOperator, ScalarOperator> projections) {
         for (Map.Entry<ColumnRefOperator, ScalarOperator> entry : projections.entrySet()) {
             if (!entry.getKey().equals(entry.getValue())) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
index 4bab98be1a6b2f..680497ee907529 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/StatementPlanner.java
@@ -60,6 +60,9 @@
 import com.starrocks.sql.common.UnsupportedException;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
+import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
+import com.starrocks.sql.optimizer.OptimizerOptions;
 import com.starrocks.sql.optimizer.OptimizerTraceUtil;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
@@ -234,7 +237,7 @@ private static ExecPlan createQueryPlan(StatementBase stmt,
                                             ConnectContext session,
                                             TResultSinkType resultSinkType) {
         QueryStatement queryStmt = (QueryStatement) stmt;
-        QueryRelation query = (QueryRelation) queryStmt.getQueryRelation();
+        QueryRelation query = queryStmt.getQueryRelation();
         List<String> colNames = query.getColumnOutputNames();
         // 1. Build Logical plan
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
@@ -247,20 +250,21 @@ private static ExecPlan createQueryPlan(StatementBase stmt,
             logicalPlan = new RelationTransformer(transformerContext).transformWithSelectLimit(query);
         }
 
-        OptExpression root = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session);
-
+        boolean isShortCircuit = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session);
         OptExpression optimizedPlan;
         try (Timer ignored = Tracers.watchScope("Optimizer")) {
             // 2. Optimize logical plan and build physical plan
-            Optimizer optimizer = new Optimizer();
-            optimizedPlan = optimizer.optimize(
-                    session,
-                    root,
-                    mvTransformerContext,
-                    stmt,
+            OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory);
+            optimizerContext.setMvTransformerContext(mvTransformerContext);
+            optimizerContext.setStatement(stmt);
+            if (isShortCircuit) {
+                optimizerContext.setOptimizerOptions(OptimizerOptions.newShortCircuitOpt());
+            }
+
+            Optimizer optimizer = OptimizerFactory.create(optimizerContext);
+            optimizedPlan = optimizer.optimize(logicalPlan.getRoot(),
                     new PhysicalPropertySet(),
-                    new ColumnRefSet(logicalPlan.getOutputColumn()),
-                    columnRefFactory);
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
         }
 
         try (Timer ignored = Tracers.watchScope("ExecPlanBuild")) {
@@ -273,7 +277,7 @@ private static ExecPlan createQueryPlan(StatementBase stmt,
             ExecPlan execPlan = PlanFragmentBuilder.createPhysicalPlan(
                     optimizedPlan, session, logicalPlan.getOutputColumn(), columnRefFactory, colNames,
                     resultSinkType,
-                    !session.getSessionVariable().isSingleNodeExecPlan());
+                    !session.getSessionVariable().isSingleNodeExecPlan(), isShortCircuit);
             execPlan.setLogicalPlan(logicalPlan);
             execPlan.setColumnRefFactory(columnRefFactory);
             return execPlan;
@@ -312,25 +316,26 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt,
                 logicalPlan = new RelationTransformer(transformerContext).transformWithSelectLimit(query);
             }
 
-            OptExpression root = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session);
-
+            boolean isShortCircuit = ShortCircuitPlanner.checkSupportShortCircuitRead(logicalPlan.getRoot(), session);
             OptExpression optimizedPlan;
             try (Timer ignored = Tracers.watchScope("Optimizer")) {
+                OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory);
                 // 2. Optimize logical plan and build physical plan
-                Optimizer optimizer = new Optimizer();
                 // FIXME: refactor this into Optimizer.optimize() method.
                 // set query tables into OptimizeContext so can be added for mv rewrite
                 if (Config.skip_whole_phase_lock_mv_limit >= 0) {
-                    optimizer.setQueryTables(olapTables);
+                    optimizerContext.setQueryTables(olapTables);
+                }
+
+                if (isShortCircuit) {
+                    optimizerContext.setOptimizerOptions(OptimizerOptions.newShortCircuitOpt());
                 }
-                optimizedPlan = optimizer.optimize(
-                        session,
-                        root,
-                        mvTransformerContext,
-                        queryStmt,
-                        new PhysicalPropertySet(),
-                        new ColumnRefSet(logicalPlan.getOutputColumn()),
-                        columnRefFactory);
+                optimizerContext.setMvTransformerContext(mvTransformerContext);
+                optimizerContext.setStatement(queryStmt);
+
+                Optimizer optimizer = OptimizerFactory.create(optimizerContext);
+                optimizedPlan = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(),
+                        new ColumnRefSet(logicalPlan.getOutputColumn()));
             }
 
             try (Timer ignored = Tracers.watchScope("ExecPlanBuild")) {
@@ -341,7 +346,7 @@ public static ExecPlan createQueryPlanWithReTry(QueryStatement queryStmt,
                 ExecPlan plan = PlanFragmentBuilder.createPhysicalPlan(
                         optimizedPlan, session, logicalPlan.getOutputColumn(), columnRefFactory, colNames,
                         resultSinkType,
-                        !session.getSessionVariable().isSingleNodeExecPlan());
+                        !session.getSessionVariable().isSingleNodeExecPlan(), isShortCircuit);
                 final long finalPlanStartTime = planStartTime;
                 isSchemaValid = olapTables.stream().allMatch(t -> OptimisticVersion.validateTableUpdate(t,
                         finalPlanStartTime));
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java b/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java
index 6cbfdcde33ceb4..82c42b0ea08b7f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/UpdatePlanner.java
@@ -40,6 +40,8 @@
 import com.starrocks.sql.ast.UpdateStmt;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
+import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
@@ -95,15 +97,14 @@ public ExecPlan plan(UpdateStmt updateStmt, ConnectContext session) {
             session.getSessionVariable().setEnableLocalShuffleAgg(false);
 
             long tableId = targetTable.getId();
-            Optimizer optimizer = new Optimizer();
-            optimizer.setUpdateTableId(tableId);
+            OptimizerContext optimizerContext = OptimizerFactory.initContext(session, columnRefFactory);
+            optimizerContext.setUpdateTableId(tableId);
 
+            Optimizer optimizer = OptimizerFactory.create(optimizerContext);
             OptExpression optimizedPlan = optimizer.optimize(
-                    session,
                     optExprBuilder.getRoot(),
                     new PhysicalPropertySet(),
-                    new ColumnRefSet(outputColumns),
-                    columnRefFactory);
+                    new ColumnRefSet(outputColumns));
             ExecPlan execPlan = PlanFragmentBuilder.createPhysicalPlan(optimizedPlan, session,
                     outputColumns, columnRefFactory, colNames, TResultSinkType.MYSQL_PROTOCAL, false);
             DescriptorTable descriptorTable = execPlan.getDescTbl();
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java
index e205e666553256..90ce27d2708430 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzeStmtAnalyzer.java
@@ -42,9 +42,7 @@
 import com.starrocks.sql.ast.DropStatsStmt;
 import com.starrocks.sql.ast.StatementBase;
 import com.starrocks.sql.common.MetaUtils;
-import com.starrocks.sql.optimizer.Memo;
-import com.starrocks.sql.optimizer.OptimizerConfig;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.statistics.Statistics;
 import com.starrocks.statistic.StatisticUtils;
@@ -381,8 +379,7 @@ private void analyzeAnalyzeTypeDesc(ConnectContext session, StatementBase statem
                     }
 
                     Statistics tableStats = session.getGlobalStateMgr().getMetadataMgr().
-                            getTableStatistics(new OptimizerContext(new Memo(), new ColumnRefFactory(), session,
-                                            OptimizerConfig.defaultConfig()),
+                            getTableStatistics(OptimizerFactory.initContext(session, new ColumnRefFactory()),
                                     tableName.getCatalog(), analyzeTable, Maps.newHashMap(), keys, null);
                     totalRows = tableStats.getOutputRowCount();
                 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java
index b3ccf1800ae66d..231d5c7702e37e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/MaterializedViewAnalyzer.java
@@ -96,6 +96,8 @@
 import com.starrocks.sql.common.PListCell;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
+import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
@@ -457,14 +459,13 @@ private void planMVQuery(CreateMaterializedViewStatement createStmt, QueryStatem
                 // Build logical plan for view query
                 OptExprBuilder optExprBuilder = logicalPlan.getRootBuilder();
                 logicalPlan = new LogicalPlan(optExprBuilder, outputColumns, logicalPlan.getCorrelation());
-                Optimizer optimizer = new Optimizer();
+                OptimizerContext optimizerContext = OptimizerFactory.initContext(ctx, columnRefFactory);
+                Optimizer optimizer = OptimizerFactory.create(optimizerContext);
                 PhysicalPropertySet requiredPropertySet = PhysicalPropertySet.EMPTY;
                 OptExpression optimizedPlan = optimizer.optimize(
-                        ctx,
                         logicalPlan.getRoot(),
                         requiredPropertySet,
-                        new ColumnRefSet(logicalPlan.getOutputColumn()),
-                        columnRefFactory);
+                        new ColumnRefSet(logicalPlan.getOutputColumn()));
                 optimizedPlan.deriveMVProperty();
 
                 // TODO: refine rules for mv plan
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java
index 4c855eaa92d7a7..98b2bc4b863657 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MaterializedViewOptimizer.java
@@ -36,24 +36,24 @@ public MvPlanContext optimize(MaterializedView mv,
                                   ConnectContext connectContext,
                                   boolean inlineView) {
         // optimize the sql by rule and disable rule based materialized view rewrite
-        OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED);
+        OptimizerOptions optimizerOptions = OptimizerOptions.newRuleBaseOpt();
         // Disable partition prune for mv's plan so no needs  to compensate pruned predicates anymore.
         // Only needs to compensate mv's ref-base-table's partition predicates when mv's freshness cannot be satisfied.
-        optimizerConfig.disableRule(RuleType.GP_PARTITION_PRUNE);
-        optimizerConfig.disableRule(RuleType.GP_ALL_MV_REWRITE);
+        optimizerOptions.disableRule(RuleType.GP_PARTITION_PRUNE);
+        optimizerOptions.disableRule(RuleType.GP_ALL_MV_REWRITE);
         // INTERSECT_REWRITE is used for INTERSECT related plan optimize, which can not be SPJG;
         // And INTERSECT_REWRITE should be based on PARTITION_PRUNE rule set.
         // So exclude it
-        optimizerConfig.disableRule(RuleType.GP_INTERSECT_REWRITE);
-        optimizerConfig.disableRule(RuleType.TF_REWRITE_GROUP_BY_COUNT_DISTINCT);
-        optimizerConfig.disableRule(RuleType.TF_PRUNE_EMPTY_SCAN);
-        optimizerConfig.disableRule(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE);
-        optimizerConfig.disableRule(RuleType.TF_MV_TRANSPARENT_REWRITE_RULE);
-        optimizerConfig.disableRule(RuleType.TF_ELIMINATE_AGG);
-        optimizerConfig.disableRule(RuleType.TF_PULL_UP_PREDICATE_SCAN);
+        optimizerOptions.disableRule(RuleType.GP_INTERSECT_REWRITE);
+        optimizerOptions.disableRule(RuleType.TF_REWRITE_GROUP_BY_COUNT_DISTINCT);
+        optimizerOptions.disableRule(RuleType.TF_PRUNE_EMPTY_SCAN);
+        optimizerOptions.disableRule(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE);
+        optimizerOptions.disableRule(RuleType.TF_MV_TRANSPARENT_REWRITE_RULE);
+        optimizerOptions.disableRule(RuleType.TF_ELIMINATE_AGG);
+        optimizerOptions.disableRule(RuleType.TF_PULL_UP_PREDICATE_SCAN);
         // For sync mv, no rewrite query by original sync mv rule to avoid useless rewrite.
         if (mv.getRefreshScheme().isSync()) {
-            optimizerConfig.disableRule(RuleType.TF_MATERIALIZED_VIEW);
+            optimizerOptions.disableRule(RuleType.TF_MATERIALIZED_VIEW);
         }
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
         String mvSql = mv.getViewDefineSql();
@@ -77,7 +77,8 @@ public MvPlanContext optimize(MaterializedView mv,
         try {
             // get optimized plan of mv's defined query
             Pair<OptExpression, LogicalPlan> plans =
-                    MvUtils.getRuleOptimizedLogicalPlan(stmt, columnRefFactory, connectContext, optimizerConfig, inlineView);
+                    MvUtils.getRuleOptimizedLogicalPlan(stmt, columnRefFactory, connectContext, optimizerOptions,
+                            inlineView);
             if (plans == null) {
                 return new MvPlanContext(false, "No query plan for it");
             }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java
index 56197134799f43..029c0036160765 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/MvRewritePreprocessor.java
@@ -229,7 +229,8 @@ public void prepare(OptExpression queryOptExpression) {
             // use a new context rather than reuse the existed context to avoid cache conflict.
             try {
                 // 1. get related mvs for all input tables
-                Set<MaterializedView> relatedMVs = getRelatedMVs(queryTables, context.getOptimizerConfig().isRuleBased());
+                Set<MaterializedView> relatedMVs =
+                        getRelatedMVs(queryTables, context.getOptimizerOptions().isRuleBased());
                 if (relatedMVs.isEmpty()) {
                     return;
                 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java
index c876390dc2fb48..ee3d99bf6de4e0 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptExpression.java
@@ -63,8 +63,6 @@ public class OptExpression {
 
     private UKFKConstraints constraints;
 
-    private Boolean isShortCircuit = false;
-
     // the flag if its parent has required data distribution property for this expression
     private boolean existRequiredDistribution = true;
 
@@ -82,13 +80,6 @@ public static OptExpression create(Operator op, OptExpression... inputs) {
         return expr;
     }
 
-    public static OptExpression createForShortCircuit(Operator op, OptExpression input, boolean isShortCircuit) {
-        OptExpression expr = new OptExpression(op);
-        expr.inputs = Lists.newArrayList(input);
-        expr.setShortCircuit(isShortCircuit);
-        return expr;
-    }
-
     public static OptExpression create(Operator op, List<OptExpression> inputs) {
         OptExpression expr = new OptExpression(op);
         expr.inputs = inputs;
@@ -231,14 +222,6 @@ public void setCost(double cost) {
         this.cost = cost;
     }
 
-    public Boolean getShortCircuit() {
-        return isShortCircuit;
-    }
-
-    public void setShortCircuit(Boolean shortCircuit) {
-        isShortCircuit = shortCircuit;
-    }
-
     @Override
     public String toString() {
         return op + " child size " + inputs.size();
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
index 003e5fd4a4be30..6403470fe490ce 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java
@@ -11,857 +11,30 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-
 package com.starrocks.sql.optimizer;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.starrocks.analysis.JoinOperator;
-import com.starrocks.catalog.MaterializedView;
-import com.starrocks.catalog.OlapTable;
-import com.starrocks.common.profile.Timer;
-import com.starrocks.common.profile.Tracers;
-import com.starrocks.qe.ConnectContext;
-import com.starrocks.qe.SessionVariable;
-import com.starrocks.qe.feedback.OperatorTuningGuides;
-import com.starrocks.qe.feedback.PlanTuningAdvisor;
-import com.starrocks.sql.Explain;
-import com.starrocks.sql.ast.StatementBase;
-import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
-import com.starrocks.sql.optimizer.cost.CostEstimate;
-import com.starrocks.sql.optimizer.operator.Operator;
-import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
-import com.starrocks.sql.optimizer.operator.logical.LogicalTreeAnchorOperator;
-import com.starrocks.sql.optimizer.operator.logical.LogicalViewScanOperator;
-import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator;
-import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown;
-import com.starrocks.sql.optimizer.rule.RuleSet;
-import com.starrocks.sql.optimizer.rule.implementation.OlapScanImplementationRule;
-import com.starrocks.sql.optimizer.rule.join.JoinReorderFactory;
-import com.starrocks.sql.optimizer.rule.join.ReorderJoinRule;
-import com.starrocks.sql.optimizer.rule.mv.MaterializedViewRule;
-import com.starrocks.sql.optimizer.rule.transformation.ApplyExceptionRule;
-import com.starrocks.sql.optimizer.rule.transformation.ArrayDistinctAfterAggRule;
-import com.starrocks.sql.optimizer.rule.transformation.CTEProduceAddProjectionRule;
-import com.starrocks.sql.optimizer.rule.transformation.ConvertToEqualForNullRule;
-import com.starrocks.sql.optimizer.rule.transformation.DeriveRangeJoinPredicateRule;
-import com.starrocks.sql.optimizer.rule.transformation.EliminateAggRule;
-import com.starrocks.sql.optimizer.rule.transformation.EliminateConstantCTERule;
-import com.starrocks.sql.optimizer.rule.transformation.EliminateSortColumnWithEqualityPredicateRule;
-import com.starrocks.sql.optimizer.rule.transformation.ForceCTEReuseRule;
-import com.starrocks.sql.optimizer.rule.transformation.GroupByCountDistinctRewriteRule;
-import com.starrocks.sql.optimizer.rule.transformation.IcebergEqualityDeleteRewriteRule;
-import com.starrocks.sql.optimizer.rule.transformation.IcebergPartitionsTableRewriteRule;
-import com.starrocks.sql.optimizer.rule.transformation.JoinLeftAsscomRule;
-import com.starrocks.sql.optimizer.rule.transformation.MaterializedViewTransparentRewriteRule;
-import com.starrocks.sql.optimizer.rule.transformation.MergeProjectWithChildRule;
-import com.starrocks.sql.optimizer.rule.transformation.MergeTwoAggRule;
-import com.starrocks.sql.optimizer.rule.transformation.MergeTwoProjectRule;
-import com.starrocks.sql.optimizer.rule.transformation.OnPredicateMoveAroundRule;
-import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnMinMaxRewriteRule;
-import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnValueOnlyOnScanRule;
-import com.starrocks.sql.optimizer.rule.transformation.PruneEmptyWindowRule;
-import com.starrocks.sql.optimizer.rule.transformation.PullUpScanPredicateRule;
-import com.starrocks.sql.optimizer.rule.transformation.PushDownAggregateGroupingSetsRule;
-import com.starrocks.sql.optimizer.rule.transformation.PushDownJoinOnExpressionToChildProject;
-import com.starrocks.sql.optimizer.rule.transformation.PushDownLimitRankingWindowRule;
-import com.starrocks.sql.optimizer.rule.transformation.PushDownPredicateRankingWindowRule;
-import com.starrocks.sql.optimizer.rule.transformation.PushDownProjectLimitRule;
-import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowOuterJoinRule;
-import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowUnionRule;
-import com.starrocks.sql.optimizer.rule.transformation.PushLimitAndFilterToCTEProduceRule;
-import com.starrocks.sql.optimizer.rule.transformation.RemoveAggregationFromAggTable;
-import com.starrocks.sql.optimizer.rule.transformation.RewriteGroupingSetsByCTERule;
-import com.starrocks.sql.optimizer.rule.transformation.RewriteMultiDistinctRule;
-import com.starrocks.sql.optimizer.rule.transformation.RewriteSimpleAggToHDFSScanRule;
-import com.starrocks.sql.optimizer.rule.transformation.RewriteUnnestBitmapRule;
-import com.starrocks.sql.optimizer.rule.transformation.SchemaTableEvaluateRule;
-import com.starrocks.sql.optimizer.rule.transformation.SeparateProjectRule;
-import com.starrocks.sql.optimizer.rule.transformation.SkewJoinOptimizeRule;
-import com.starrocks.sql.optimizer.rule.transformation.SplitScanORToUnionRule;
-import com.starrocks.sql.optimizer.rule.transformation.UnionToValuesRule;
-import com.starrocks.sql.optimizer.rule.transformation.materialization.MVCompensationPruneUnionRule;
-import com.starrocks.sql.optimizer.rule.transformation.materialization.MvRewriteStrategy;
-import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils;
-import com.starrocks.sql.optimizer.rule.transformation.materialization.rule.TextMatchBasedRewriteRule;
-import com.starrocks.sql.optimizer.rule.transformation.pruner.CboTablePruneRule;
-import com.starrocks.sql.optimizer.rule.transformation.pruner.PrimaryKeyUpdateTableRule;
-import com.starrocks.sql.optimizer.rule.transformation.pruner.RboTablePruneRule;
-import com.starrocks.sql.optimizer.rule.transformation.pruner.UniquenessBasedTablePruneRule;
-import com.starrocks.sql.optimizer.rule.tree.AddDecodeNodeForDictStringRule;
-import com.starrocks.sql.optimizer.rule.tree.AddIndexOnlyPredicateRule;
-import com.starrocks.sql.optimizer.rule.tree.ApplyTuningGuideRule;
-import com.starrocks.sql.optimizer.rule.tree.CloneDuplicateColRefRule;
-import com.starrocks.sql.optimizer.rule.tree.DataCachePopulateRewriteRule;
-import com.starrocks.sql.optimizer.rule.tree.EliminateOveruseColumnAccessPathRule;
-import com.starrocks.sql.optimizer.rule.tree.ExchangeSortToMergeRule;
-import com.starrocks.sql.optimizer.rule.tree.ExtractAggregateColumn;
-import com.starrocks.sql.optimizer.rule.tree.InlineCteProjectPruneRule;
-import com.starrocks.sql.optimizer.rule.tree.JoinLocalShuffleRule;
-import com.starrocks.sql.optimizer.rule.tree.MarkParentRequiredDistributionRule;
-import com.starrocks.sql.optimizer.rule.tree.PhysicalDistributionAggOptRule;
-import com.starrocks.sql.optimizer.rule.tree.PreAggregateTurnOnRule;
-import com.starrocks.sql.optimizer.rule.tree.PredicateReorderRule;
-import com.starrocks.sql.optimizer.rule.tree.PruneAggregateNodeRule;
-import com.starrocks.sql.optimizer.rule.tree.PruneShuffleColumnRule;
-import com.starrocks.sql.optimizer.rule.tree.PruneSubfieldsForComplexType;
-import com.starrocks.sql.optimizer.rule.tree.PushDownAggregateRule;
-import com.starrocks.sql.optimizer.rule.tree.PushDownDistinctAggregateRule;
-import com.starrocks.sql.optimizer.rule.tree.ScalarOperatorsReuseRule;
-import com.starrocks.sql.optimizer.rule.tree.SimplifyCaseWhenPredicateRule;
-import com.starrocks.sql.optimizer.rule.tree.SubfieldExprNoCopyRule;
-import com.starrocks.sql.optimizer.rule.tree.lowcardinality.LowCardinalityRewriteRule;
-import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PruneSubfieldRule;
-import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PushDownSubfieldRule;
-import com.starrocks.sql.optimizer.task.OptimizeGroupTask;
-import com.starrocks.sql.optimizer.task.PrepareCollectMetaTask;
-import com.starrocks.sql.optimizer.task.TaskContext;
-import com.starrocks.sql.optimizer.task.TaskScheduler;
-import com.starrocks.sql.optimizer.transformer.MVTransformerContext;
-import com.starrocks.sql.optimizer.validate.MVRewriteValidator;
-import com.starrocks.sql.optimizer.validate.OptExpressionValidator;
-import com.starrocks.sql.optimizer.validate.PlanValidator;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_TRANSPARENT_REWRITE;
-import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE;
-import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED;
-import static com.starrocks.sql.optimizer.rule.RuleType.TF_MATERIALIZED_VIEW;
-
-/**
- * Optimizer's entrance class
- */
-public class Optimizer {
-    private static final Logger LOG = LogManager.getLogger(Optimizer.class);
-    private OptimizerContext context;
-    private final OptimizerConfig optimizerConfig;
-    private MvRewriteStrategy mvRewriteStrategy = new MvRewriteStrategy();
-    private TaskScheduler scheduler;
 
-    private long updateTableId = -1;
-
-    private Set<OlapTable> queryTables;
-
-    public Optimizer() {
-        this(OptimizerConfig.defaultConfig());
-    }
+public abstract class Optimizer {
+    protected final OptimizerContext context;
 
-    public Optimizer(OptimizerConfig config) {
-        this.optimizerConfig = config;
-    }
-
-    @VisibleForTesting
-    public OptimizerConfig getOptimizerConfig() {
-        return optimizerConfig;
+    Optimizer(OptimizerContext context) {
+        this.context = context;
     }
 
     public OptimizerContext getContext() {
         return context;
     }
 
-    @VisibleForTesting
-    public MvRewriteStrategy getMvRewriteStrategy() {
-        return mvRewriteStrategy;
-    }
-
-    public OptExpression optimize(ConnectContext connectContext,
-                                  OptExpression logicOperatorTree,
-                                  PhysicalPropertySet requiredProperty,
-                                  ColumnRefSet requiredColumns,
-                                  ColumnRefFactory columnRefFactory) {
-        return optimize(connectContext, logicOperatorTree, null, null, requiredProperty,
-                requiredColumns, columnRefFactory);
-    }
-
-    public OptExpression optimize(ConnectContext connectContext,
-                                  OptExpression logicOperatorTree,
-                                  MVTransformerContext mvTransformerContext,
-                                  StatementBase stmt,
-                                  PhysicalPropertySet requiredProperty,
-                                  ColumnRefSet requiredColumns,
-                                  ColumnRefFactory columnRefFactory) {
-        try {
-            // prepare for optimizer
-            prepare(connectContext, columnRefFactory, logicOperatorTree);
-
-            // prepare for mv rewrite
-            prepareMvRewrite(connectContext, logicOperatorTree, columnRefFactory, requiredColumns);
-            try (Timer ignored = Tracers.watchScope("MVTextRewrite")) {
-                logicOperatorTree = new TextMatchBasedRewriteRule(connectContext, stmt, mvTransformerContext)
-                        .transform(logicOperatorTree, context).get(0);
-            }
-
-            OptExpression result = optimizerConfig.isRuleBased() ?
-                    optimizeByRule(logicOperatorTree, requiredProperty, requiredColumns) :
-                    optimizeByCost(connectContext, logicOperatorTree, requiredProperty, requiredColumns);
-            return result;
-        } finally {
-            // make sure clear caches in OptimizerContext
-            context.clear();
-            connectContext.setQueryMVContext(null);
-        }
-    }
-
-    public void setQueryTables(Set<OlapTable> queryTables) {
-        this.queryTables = queryTables;
-    }
-
-    public void setUpdateTableId(long updateTableId) {
-        this.updateTableId = updateTableId;
-    }
-
-    // Optimize by rule will return logical plan.
-    // Used by materialized view query rewrite optimization.
-    private OptExpression optimizeByRule(OptExpression logicOperatorTree,
-                                         PhysicalPropertySet requiredProperty,
-                                         ColumnRefSet requiredColumns) {
-        OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree);
-        TaskContext rootTaskContext =
-                new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE);
-        logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext);
-        OptimizerTraceUtil.log("after logical rewrite, new logicOperatorTree:\n%s", logicOperatorTree);
-        return logicOperatorTree;
-    }
-
-    /**
-     * Optimizer will transform and implement the logical operator based on
-     * the {@see Rule}, then cost the physical operator, and finally find the
-     * lowest cost physical operator tree
-     *
-     * @param logicOperatorTree the input for query Optimizer
-     * @param requiredProperty  the required physical property from sql or groupExpression
-     * @param requiredColumns   the required output columns from sql or groupExpression
-     * @return the lowest cost physical operator for this query
-     */
-    private OptExpression optimizeByCost(ConnectContext connectContext,
-                                         OptExpression logicOperatorTree,
-                                         PhysicalPropertySet requiredProperty,
-                                         ColumnRefSet requiredColumns) {
-        // Phase 1: none
-        OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree);
-        // Phase 2: rewrite based on memo and group
-        Memo memo = context.getMemo();
-        TaskContext rootTaskContext =
-                new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE);
-
-        try (Timer ignored = Tracers.watchScope("RuleBaseOptimize")) {
-            logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext);
-        }
-
-        if (logicOperatorTree.getShortCircuit()) {
-            return logicOperatorTree;
-        }
-
-        memo.init(logicOperatorTree);
-        if (context.getQueryMaterializationContext() != null) {
-            // LogicalTreeWithView is logically equivalent to logicOperatorTree
-            addViewBasedPlanIntoMemo(context.getQueryMaterializationContext().getQueryOptPlanWithView());
-        }
-        OptimizerTraceUtil.log("after logical rewrite, root group:\n%s", memo.getRootGroup());
-
-        // Currently, we cache output columns in logic property.
-        // We derive logic property Bottom Up firstly when new group added to memo,
-        // but we do column prune rewrite top down later.
-        // So after column prune rewrite, the output columns for each operator maybe change,
-        // but the logic property is cached and never change.
-        // So we need to explicitly derive all group logic property again
-        memo.deriveAllGroupLogicalProperty();
-
-        // Phase 3: optimize based on memo and group
-        try (Timer ignored = Tracers.watchScope("CostBaseOptimize")) {
-            memoOptimize(connectContext, memo, rootTaskContext);
-        }
-
-        OptExpression result;
-        if (connectContext.getSessionVariable().isSetUseNthExecPlan()) {
-            // extract the nth execution plan
-            int nthExecPlan = connectContext.getSessionVariable().getUseNthExecPlan();
-            result = EnumeratePlan.extractNthPlan(requiredProperty, memo.getRootGroup(), nthExecPlan);
-        } else {
-            result = extractBestPlan(requiredProperty, memo.getRootGroup());
-        }
-        OptimizerTraceUtil.logOptExpression("after extract best plan:\n%s", result);
-
-        // set costs audio log before physicalRuleRewrite
-        // statistics won't set correctly after physicalRuleRewrite.
-        // we need set plan costs before physical rewrite stage.
-        final CostEstimate costs = Explain.buildCost(result);
-        connectContext.getAuditEventBuilder().setPlanCpuCosts(costs.getCpuCost())
-                .setPlanMemCosts(costs.getMemoryCost());
-        OptExpression finalPlan;
-        try (Timer ignored = Tracers.watchScope("PhysicalRewrite")) {
-            finalPlan = physicalRuleRewrite(connectContext, rootTaskContext, result);
-            OptimizerTraceUtil.logOptExpression("final plan after physical rewrite:\n%s", finalPlan);
-        }
-
-        try (Timer ignored = Tracers.watchScope("DynamicRewrite")) {
-            finalPlan = dynamicRewrite(connectContext, rootTaskContext, finalPlan);
-            OptimizerTraceUtil.logOptExpression("final plan after dynamic rewrite:\n%s", finalPlan);
-        }
-
-        // collect all mv scan operator
-        collectAllPhysicalOlapScanOperators(result, rootTaskContext);
-        List<PhysicalOlapScanOperator> mvScan = rootTaskContext.getAllPhysicalOlapScanOperators().stream().
-                filter(scan -> scan.getTable().isMaterializedView()).collect(Collectors.toList());
-        // add mv db id to currentSqlDbIds, the resource group could use this to distinguish sql patterns
-        Set<Long> currentSqlDbIds = rootTaskContext.getOptimizerContext().getCurrentSqlDbIds();
-        mvScan.stream().map(scan -> ((MaterializedView) scan.getTable()).getDbId()).forEach(currentSqlDbIds::add);
-
-        try (Timer ignored = Tracers.watchScope("PlanValidate")) {
-            // valid the final plan
-            PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext);
-            // validate mv and log tracer if needed
-            MVRewriteValidator.getInstance().validateMV(connectContext, finalPlan, rootTaskContext);
-            // audit mv
-            MVRewriteValidator.getInstance().auditMv(connectContext, finalPlan, rootTaskContext);
-            return finalPlan;
-        }
-    }
-
-    private void addViewBasedPlanIntoMemo(OptExpression logicalTreeWithView) {
-        if (logicalTreeWithView == null) {
-            return;
-        }
-        Memo memo = context.getMemo();
-        memo.copyIn(memo.getRootGroup(), logicalTreeWithView);
-    }
-
-    private void prepare(ConnectContext connectContext,
-                         ColumnRefFactory columnRefFactory,
-                         OptExpression logicOperatorTree) {
-        Memo memo = null;
-        if (!optimizerConfig.isRuleBased()) {
-            memo = new Memo();
-        }
-
-        context = new OptimizerContext(memo, columnRefFactory, connectContext, optimizerConfig);
-        context.setQueryTables(queryTables);
-        context.setUpdateTableId(updateTableId);
-        this.scheduler = context.getTaskScheduler();
-
-        // collect all olap scan operator
-        collectAllLogicalOlapScanOperators(logicOperatorTree, context);
-
-    }
-
-    private void prepareMvRewrite(ConnectContext connectContext, OptExpression logicOperatorTree,
-                                  ColumnRefFactory columnRefFactory, ColumnRefSet requiredColumns) {
-        SessionVariable sessionVariable = connectContext.getSessionVariable();
-        // MV Rewrite will be used when cbo is enabled.
-        if (context.getOptimizerConfig().isRuleBased() || sessionVariable.isDisableMaterializedViewRewrite() ||
-                !sessionVariable.isEnableMaterializedViewRewrite()) {
-            return;
-        }
-        // prepare related mvs if needed and initialize mv rewrite strategy
-        new MvRewritePreprocessor(connectContext, columnRefFactory, context, requiredColumns)
-                .prepare(logicOperatorTree);
-
-        // initialize mv rewrite strategy finally
-        mvRewriteStrategy = MvRewriteStrategy.prepareRewriteStrategy(context, connectContext, logicOperatorTree);
-        OptimizerTraceUtil.logMVPrepare("MV rewrite strategy: {}", mvRewriteStrategy);
-
-        // TODO(stephen): enable agg push down when query exists related mvs.
-        if (context.getQueryMaterializationContext() != null &&
-                !context.getQueryMaterializationContext().getValidCandidateMVs().isEmpty()) {
-            context.getSessionVariable().setCboPushDownAggregateMode(-1);
-        }
-    }
-
-    private void pruneTables(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) {
-        if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableRboTablePrune()) {
-            if (!Utils.hasPrunableJoin(tree)) {
-                return;
-            }
-            // PARTITION_PRUNE is required to run before ReorderJoinRule because ReorderJoinRule's
-            // Statistics calculation on Operators depends on row count yielded by the PARTITION_PRUNE.
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
-            // ReorderJoinRule is a in-memo rule, when it is used outside memo, we must apply
-            // MergeProjectWithChildRule to merge LogicalProjectionOperator into its child's
-            // projection before ReorderJoinRule's application, after that, we must separate operator's
-            // projection as LogicalProjectionOperator from the operator by applying SeparateProjectRule.
-            scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-            scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
-            CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context);
-            tree = new UniquenessBasedTablePruneRule().rewrite(tree, rootTaskContext);
-            deriveLogicalProperty(tree);
-            tree = new ReorderJoinRule().rewrite(tree, context);
-            tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
-            deriveLogicalProperty(tree);
-            // TODO(by satanson): bucket shuffle join interpolation in PK table's update query can adjust layout
-            //  of the data ingested by OlapTableSink and eliminate race introduced by multiple concurrent write
-            //  operations on the same tablets, pruning this bucket shuffle join make update statement performance
-            //  regression, so we can turn on this rule after we put an bucket-shuffle exchange in front of
-            //  OlapTableSink in future, at present we turn off this rule.
-            if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableTablePruneOnUpdate()) {
-                tree = new PrimaryKeyUpdateTableRule().rewrite(tree, rootTaskContext);
-                deriveLogicalProperty(tree);
-            }
-            tree = new RboTablePruneRule().rewrite(tree, rootTaskContext);
-            scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-            rootTaskContext.setRequiredColumns(requiredColumns.clone());
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
-            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
-        }
-    }
-
-    /**
-     * Rewrite transparent materialized view.
-     */
-    private OptExpression transparentMVRewrite(OptExpression tree, TaskContext rootTaskContext) {
-        scheduler.rewriteOnce(tree, rootTaskContext, new MaterializedViewTransparentRewriteRule());
-        if (Utils.isOptHasAppliedRule(tree, OP_MV_TRANSPARENT_REWRITE)) {
-            tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
-        }
-        return tree;
-    }
-
-    private void ruleBasedMaterializedViewRewrite(OptExpression tree,
-                                                  TaskContext rootTaskContext,
-                                                  ColumnRefSet requiredColumns) {
-        // skip if mv rewrite is disabled
-        if (!mvRewriteStrategy.enableMaterializedViewRewrite || context.getQueryMaterializationContext() == null) {
-            return;
-        }
-
-        // do rule based mv rewrite if needed
-        if (!context.getQueryMaterializationContext().hasRewrittenSuccess()) {
-            doRuleBasedMaterializedViewRewrite(tree, rootTaskContext);
-        }
-
-        // NOTE: Since union rewrite will generate Filter -> Union -> OlapScan -> OlapScan, need to push filter below Union
-        // and do partition predicate again.
-        // TODO: move this into doRuleBasedMaterializedViewRewrite
-        // TODO: Do it in CBO if needed later.
-        boolean isNeedFurtherPartitionPrune =
-                Utils.isOptHasAppliedRule(tree, op -> op.isOpRuleBitSet(OP_MV_UNION_REWRITE));
-        OptimizerTraceUtil.logMVPrepare("is further partition prune: {}", isNeedFurtherPartitionPrune);
-        if (isNeedFurtherPartitionPrune && context.getQueryMaterializationContext().hasRewrittenSuccess()) {
-            // reset partition prune bit to do partition prune again.
-            MvUtils.getScanOperator(tree).forEach(scan -> {
-                scan.resetOpRuleBit(OP_PARTITION_PRUNED);
-            });
-            // Do predicate push down if union rewrite successes.
-            tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
-            deriveLogicalProperty(tree);
-            // Do partition prune again to avoid unnecessary scan.
-            rootTaskContext.setRequiredColumns(requiredColumns.clone());
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
-            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
-            // It's necessary for external table since its predicate is not used directly after push down.
-            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
-            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES);
-            scheduler.rewriteIterative(tree, rootTaskContext, new MVCompensationPruneUnionRule());
-            scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-        }
-    }
-
-    private void doRuleBasedMaterializedViewRewrite(OptExpression tree,
-                                                    TaskContext rootTaskContext) {
-        if (mvRewriteStrategy.enableViewBasedRewrite) {
-            // try view based mv rewrite first, then try normal mv rewrite rules
-            viewBasedMvRuleRewrite(tree, rootTaskContext);
-        }
-        if (mvRewriteStrategy.enableForceRBORewrite) {
-            // use rule based mv rewrite strategy to do mv rewrite for multi tables query
-            if (mvRewriteStrategy.enableMultiTableRewrite) {
-                scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES);
-            }
-            if (mvRewriteStrategy.enableSingleTableRewrite) {
-                scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES);
-            }
-        } else if (mvRewriteStrategy.enableSingleTableRewrite) {
-            // now add single table materialized view rewrite rules in rule based rewrite phase to boost optimization
-            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES);
-        }
-    }
-
-    private void doMVRewriteWithMultiStages(OptExpression tree,
-                                            TaskContext rootTaskContext) {
-        if (!mvRewriteStrategy.enableMaterializedViewRewrite || !mvRewriteStrategy.mvStrategy.isMultiStages()) {
-            return;
-        }
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-        scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
-        // do rule based mv rewrite
-        doRuleBasedMaterializedViewRewrite(tree, rootTaskContext);
-        new SeparateProjectRule().rewrite(tree, rootTaskContext);
-        deriveLogicalProperty(tree);
-    }
-
-    private OptExpression logicalRuleRewrite(
-            OptExpression tree,
-            TaskContext rootTaskContext) {
-        rootTaskContext.getOptimizerContext().setShortCircuit(tree.getShortCircuit());
-        tree = OptExpression.createForShortCircuit(new LogicalTreeAnchorOperator(), tree, tree.getShortCircuit());
-        // for short circuit
-        Optional<OptExpression> result = ruleRewriteForShortCircuit(tree, rootTaskContext);
-        if (result.isPresent()) {
-            return result.get();
-        }
-
-        ColumnRefSet requiredColumns = rootTaskContext.getRequiredColumns().clone();
-        deriveLogicalProperty(tree);
-
-        SessionVariable sessionVariable = rootTaskContext.getOptimizerContext().getSessionVariable();
-        CTEContext cteContext = context.getCteContext();
-        CTEUtils.collectCteOperators(tree, context);
-
-        // see JoinPredicatePushdown
-        JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext =
-                context.getJoinPushDownParams();
-        joinPredicatePushDownContext.prepare(context, sessionVariable, mvRewriteStrategy);
-
-        // inline CTE if consume use once
-        while (cteContext.hasInlineCTE()) {
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES);
-            CTEUtils.collectCteOperators(tree, context);
-        }
-
-        scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
-        CTEUtils.collectCteOperators(tree, context);
-
-        scheduler.rewriteOnce(tree, rootTaskContext, new IcebergPartitionsTableRewriteRule());
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.AGGREGATE_REWRITE_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_SUBQUERY_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_COMMON_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_WINDOW_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_JOIN_RULES);
-        scheduler.rewriteOnce(tree, rootTaskContext, new ApplyExceptionRule());
-        CTEUtils.collectCteOperators(tree, context);
-
-        if (sessionVariable.isEnableFineGrainedRangePredicate()) {
-            scheduler.rewriteAtMostOnce(tree, rootTaskContext, RuleSet.FINE_GRAINED_RANGE_PREDICATE_RULES);
-        }
-
-        // rewrite transparent materialized view
-        tree = transparentMVRewrite(tree, rootTaskContext);
-
-        // Note: PUSH_DOWN_PREDICATE tasks should be executed before MERGE_LIMIT tasks
-        // because of the Filter node needs to be merged first to avoid the Limit node
-        // cannot merge
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
-        scheduler.rewriteOnce(tree, rootTaskContext, SchemaTableEvaluateRule.getInstance());
-
-        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.ELIMINATE_OP_WITH_CONSTANT_RULES);
-        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownPredicateRankingWindowRule());
-
-        scheduler.rewriteOnce(tree, rootTaskContext, new ConvertToEqualForNullRule());
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
-        // Put EliminateAggRule after PRUNE_COLUMNS to give a chance to prune group bys before eliminate aggregations.
-        scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance());
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_UKFK_JOIN_RULES);
-        deriveLogicalProperty(tree);
-
-        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownJoinOnExpressionToChildProject());
-
-        scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule());
-        // @todo: resolve recursive optimization question:
-        //  MergeAgg -> PruneColumn -> PruneEmptyWindow -> MergeAgg/Project -> PruneColumn...
-        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoAggRule());
-
-        rootTaskContext.setRequiredColumns(requiredColumns.clone());
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
-
-        pruneTables(tree, rootTaskContext, requiredColumns);
-
-        scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule());
-        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-
-        // rule-based materialized view rewrite: early stage
-        doMVRewriteWithMultiStages(tree, rootTaskContext);
-        joinPredicatePushDownContext.reset();
-
-        // Limit push must be after the column prune,
-        // otherwise the Node containing limit may be prune
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, new PushDownProjectLimitRule());
-
-        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownLimitRankingWindowRule());
-        rewriteGroupingSets(tree, rootTaskContext, sessionVariable);
-
-        // No heavy metadata operation before external table partition prune
-        prepareMetaOnlyOnce(tree, rootTaskContext);
-
-        // apply skew join optimize after push down join on expression to child project,
-        // we need to compute the stats of child project(like subfield).
-        skewJoinOptimize(tree, rootTaskContext);
-        scheduler.rewriteOnce(tree, rootTaskContext, new IcebergEqualityDeleteRewriteRule());
-
-        tree = pruneSubfield(tree, rootTaskContext, requiredColumns);
-
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);
-
-        CTEUtils.collectCteOperators(tree, context);
-        if (cteContext.needOptimizeCTE()) {
-            cteContext.reset();
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.COLLECT_CTE_RULES);
-            rootTaskContext.setRequiredColumns(requiredColumns.clone());
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
-            if (cteContext.needPushLimit() || cteContext.needPushPredicate()) {
-                scheduler.rewriteOnce(tree, rootTaskContext, new PushLimitAndFilterToCTEProduceRule());
-            }
-
-            if (cteContext.needPushPredicate()) {
-                scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
-            }
-
-            if (cteContext.needPushLimit()) {
-                scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
-            }
-
-            scheduler.rewriteOnce(tree, rootTaskContext, new ForceCTEReuseRule());
-        }
-
-        // Add a config to decide whether to rewrite sync mv.
-        if (!optimizerConfig.isRuleDisable(TF_MATERIALIZED_VIEW)
-                && sessionVariable.isEnableSyncMaterializedViewRewrite()) {
-            // Split or predicates to union all so can be used by mv rewrite to choose the best sort key indexes.
-            // TODO: support adaptive for or-predicates to union all.
-            if (SplitScanORToUnionRule.isForceRewrite()) {
-                scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance());
-            }
-
-            OptimizerTraceUtil.logOptExpression("before MaterializedViewRule:\n%s", tree);
-            tree = new MaterializedViewRule().transform(tree, context).get(0);
-            OptimizerTraceUtil.logOptExpression("after MaterializedViewRule:\n%s", tree);
-
-            deriveLogicalProperty(tree);
-        }
-
-        scheduler.rewriteDownTop(tree, rootTaskContext, OnPredicateMoveAroundRule.INSTANCE);
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
-
-        scheduler.rewriteIterative(tree, rootTaskContext, new PartitionColumnMinMaxRewriteRule());
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMultiDistinctRule());
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, new CTEProduceAddProjectionRule());
-        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);
-
-        // ArrayDistinctAfterAggRule must run before pushDownAggregation,
-        // because push down agg won't have array_distinct project
-        if (sessionVariable.getEnableArrayDistinctAfterAggOpt()) {
-            scheduler.rewriteOnce(tree, rootTaskContext, new ArrayDistinctAfterAggRule());
-        }
-
-        tree = pushDownAggregation(tree, rootTaskContext, requiredColumns);
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
-
-        CTEUtils.collectCteOperators(tree, context);
-        // inline CTE if consume use once
-        while (cteContext.hasInlineCTE()) {
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES);
-            CTEUtils.collectCteOperators(tree, context);
-        }
-
-        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.META_SCAN_REWRITE_RULES);
-        scheduler.rewriteOnce(tree, rootTaskContext, new PartitionColumnValueOnlyOnScanRule());
-        // before MergeProjectWithChildRule, after INLINE_CTE and MergeApplyWithTableFunction
-        scheduler.rewriteIterative(tree, rootTaskContext, RewriteUnnestBitmapRule.getInstance());
-
-        // After this rule, we shouldn't generate logical project operator
-        scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
-
-        scheduler.rewriteOnce(tree, rootTaskContext, new EliminateSortColumnWithEqualityPredicateRule());
-        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowOuterJoinRule());
-        // intersect rewrite depend on statistics
-        Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext());
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INTERSECT_REWRITE_RULES);
-        scheduler.rewriteIterative(tree, rootTaskContext, new RemoveAggregationFromAggTable());
-
-        scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance());
-        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowUnionRule());
-
-        // rule based materialized view rewrite
-        ruleBasedMaterializedViewRewrite(tree, rootTaskContext, requiredColumns);
-
-        // this rewrite rule should be after mv.
-        scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT);
-
-        // NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle
-        // select count(distinct c) from t group by a, b
-        // if this rule has applied before MV.
-        scheduler.rewriteOnce(tree, rootTaskContext, new GroupByCountDistinctRewriteRule());
-
-        scheduler.rewriteOnce(tree, rootTaskContext, new DeriveRangeJoinPredicateRule());
-
-        scheduler.rewriteOnce(tree, rootTaskContext, UnionToValuesRule.getInstance());
-
-        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.VECTOR_REWRITE_RULES);
-        // this rule should be after mv
-        // @TODO: it can also be applied to other table scan operator
-        if (context.getSessionVariable().isEnableScanPredicateExprReuse()) {
-            scheduler.rewriteOnce(tree, rootTaskContext, PullUpScanPredicateRule.OLAP_SCAN);
-        }
-
-        tree = SimplifyCaseWhenPredicateRule.INSTANCE.rewrite(tree, rootTaskContext);
-        deriveLogicalProperty(tree);
-        return tree.getInputs().get(0);
-    }
-
-    private void rewriteGroupingSets(OptExpression tree, TaskContext rootTaskContext, SessionVariable sessionVariable) {
-        if (sessionVariable.isEnableRewriteGroupingsetsToUnionAll()) {
-            scheduler.rewriteIterative(tree, rootTaskContext, new RewriteGroupingSetsByCTERule());
-        }
-        if (sessionVariable.isCboPushDownGroupingSet()) {
-            scheduler.rewriteOnce(tree, rootTaskContext, new PushDownAggregateGroupingSetsRule());
-        }
-    }
-
-    private Optional<OptExpression> ruleRewriteForShortCircuit(OptExpression tree, TaskContext rootTaskContext) {
-        Boolean isShortCircuit = tree.getShortCircuit();
-
-        if (isShortCircuit) {
-            deriveLogicalProperty(tree);
-            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SHORT_CIRCUIT_SET_RULES);
-            scheduler.rewriteOnce(tree, rootTaskContext, new MergeProjectWithChildRule());
-            OptExpression result = tree.getInputs().get(0);
-            result.setShortCircuit(true);
-            return Optional.of(result);
-        }
-        return Optional.empty();
-    }
-
-    // for single scan node, to make sure we can rewrite
-    private void viewBasedMvRuleRewrite(OptExpression tree, TaskContext rootTaskContext) {
-        QueryMaterializationContext queryMaterializationContext = context.getQueryMaterializationContext();
-        Preconditions.checkArgument(queryMaterializationContext != null);
-
-        try (Timer ignored = Tracers.watchScope("MVViewRewrite")) {
-            OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "try VIEW_BASED_MV_REWRITE");
-            OptExpression treeWithView = queryMaterializationContext.getQueryOptPlanWithView();
-            // should add a LogicalTreeAnchorOperator for rewrite
-            treeWithView = OptExpression.create(new LogicalTreeAnchorOperator(), treeWithView);
-            if (mvRewriteStrategy.enableMultiTableRewrite) {
-                scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES);
-            }
-            if (mvRewriteStrategy.enableSingleTableRewrite) {
-                scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES);
-            }
-
-            List<Operator> leftViewScanOperators = Lists.newArrayList();
-            MvUtils.collectViewScanOperator(treeWithView, leftViewScanOperators);
-            List<LogicalViewScanOperator> origQueryViewScanOperators =
-                    queryMaterializationContext.getQueryViewScanOps();
-            if (leftViewScanOperators.size() < origQueryViewScanOperators.size()) {
-                // replace original tree plan
-                tree.setChild(0, treeWithView.inputAt(0));
-                deriveLogicalProperty(tree);
-
-                // if there are view scan operator left, we should replace it back to original plans
-                if (!leftViewScanOperators.isEmpty()) {
-                    MvUtils.replaceLogicalViewScanOperator(tree);
-                }
-            }
-            OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "original view scans size: {}, " +
-                    "left view scans size: {}", origQueryViewScanOperators.size(), leftViewScanOperators.size());
-        } catch (Exception e) {
-            OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE",
-                    "single table view based mv rule rewrite failed.", e);
-        }
-    }
-
-    private OptExpression rewriteAndValidatePlan(
-            OptExpression tree,
-            TaskContext rootTaskContext) {
-        OptExpression result = logicalRuleRewrite(tree, rootTaskContext);
-        OptExpressionValidator validator = new OptExpressionValidator();
-        validator.validate(result);
-        // skip memo
-        if (result.getShortCircuit()) {
-            result = new OlapScanImplementationRule().transform(result, null).get(0);
-            result.setShortCircuit(true);
-        }
-        return result;
-    }
-
-    private OptExpression pushDownAggregation(OptExpression tree, TaskContext rootTaskContext,
-                                              ColumnRefSet requiredColumns) {
-        boolean pushDistinctFlag = false;
-        boolean pushAggFlag = false;
-        if (context.getSessionVariable().isCboPushDownDistinctBelowWindow()) {
-            // TODO(by satanson): in future, PushDownDistinctAggregateRule and PushDownAggregateRule should be
-            //  fused one rule to tackle with all scenarios of agg push-down.
-            PushDownDistinctAggregateRule rule = new PushDownDistinctAggregateRule(rootTaskContext);
-            tree = rule.rewrite(tree, rootTaskContext);
-            pushDistinctFlag = rule.getRewriter().hasRewrite();
-        }
-
-        if (context.getSessionVariable().getCboPushDownAggregateMode() != -1) {
-            if (context.getSessionVariable().isCboPushDownAggregateOnBroadcastJoin()) {
-                // Reorder joins before applying PushDownAggregateRule to better decide where to push down aggregator.
-                // For example, do not push down a not very efficient aggregator below a very small broadcast join.
-                scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
-                scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
-                scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
-                CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context);
-                deriveLogicalProperty(tree);
-                tree = new ReorderJoinRule().rewrite(tree, JoinReorderFactory.createJoinReorderAdaptive(), context);
-                tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
-                deriveLogicalProperty(tree);
-                Utils.calculateStatistics(tree, context);
-            }
-
-            PushDownAggregateRule rule = new PushDownAggregateRule(rootTaskContext);
-            rule.getRewriter().collectRewriteContext(tree);
-            if (rule.getRewriter().isNeedRewrite()) {
-                pushAggFlag = true;
-                tree = rule.rewrite(tree, rootTaskContext);
-            }
-        }
-
-        if (pushDistinctFlag || pushAggFlag) {
-            deriveLogicalProperty(tree);
-            rootTaskContext.setRequiredColumns(requiredColumns.clone());
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
-            scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance());
-        }
-
-        return tree;
-    }
-
-    private void skewJoinOptimize(OptExpression tree, TaskContext rootTaskContext) {
-        if (context.getSessionVariable().isEnableStatsToOptimizeSkewJoin()) {
-            // merge projects before calculate statistics
-            scheduler.rewriteOnce(tree, rootTaskContext, new MergeTwoProjectRule());
-            Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext());
-        }
-        scheduler.rewriteOnce(tree, rootTaskContext, new SkewJoinOptimizeRule());
+    public OptExpression optimize(OptExpression tree, ColumnRefSet requiredColumns) {
+        return optimize(tree, new PhysicalPropertySet(), requiredColumns);
     }
 
-    private OptExpression pruneSubfield(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) {
-        if (!context.getSessionVariable().isCboPruneSubfield()) {
-            return tree;
-        }
+    public abstract OptExpression optimize(OptExpression tree, PhysicalPropertySet requiredProperty,
+                                           ColumnRefSet requiredColumns);
 
-        PushDownSubfieldRule pushDownRule = new PushDownSubfieldRule();
-        tree = pushDownRule.rewrite(tree, rootTaskContext);
-
-        if (pushDownRule.hasRewrite()) {
-            rootTaskContext.setRequiredColumns(requiredColumns.clone());
-            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
-        }
-        scheduler.rewriteOnce(tree, rootTaskContext, new PruneSubfieldRule());
-
-        return tree;
-    }
-
-    private void deriveLogicalProperty(OptExpression root) {
+    protected void deriveLogicalProperty(OptExpression root) {
         for (OptExpression child : root.getInputs()) {
             deriveLogicalProperty(child);
         }
@@ -870,177 +43,4 @@ private void deriveLogicalProperty(OptExpression root) {
         context.deriveLogicalProperty();
         root.setLogicalProperty(context.getRootProperty());
     }
-
-    void memoOptimize(ConnectContext connectContext, Memo memo, TaskContext rootTaskContext) {
-        context.setInMemoPhase(true);
-        OptExpression tree = memo.getRootGroup().extractLogicalTree();
-        SessionVariable sessionVariable = connectContext.getSessionVariable();
-        // add CboTablePruneRule
-        if (Utils.countJoinNodeSize(tree, CboTablePruneRule.JOIN_TYPES) < 10 &&
-                sessionVariable.isEnableCboTablePrune()) {
-            context.getRuleSet().addCboTablePruneRule();
-        }
-        // Join reorder
-        int innerCrossJoinNode = Utils.countJoinNodeSize(tree, JoinOperator.innerCrossJoinSet());
-        if (!sessionVariable.isDisableJoinReorder() && innerCrossJoinNode < sessionVariable.getCboMaxReorderNode()) {
-            if (innerCrossJoinNode > sessionVariable.getCboMaxReorderNodeUseExhaustive()) {
-                CTEUtils.collectForceCteStatistics(memo, context);
-
-                OptimizerTraceUtil.logOptExpression("before ReorderJoinRule:\n%s", tree);
-                new ReorderJoinRule().transform(tree, context);
-                OptimizerTraceUtil.logOptExpression("after ReorderJoinRule:\n%s", tree);
-
-                context.getRuleSet().addJoinCommutativityWithoutInnerRule();
-            } else {
-                if (Utils.countJoinNodeSize(tree, JoinOperator.semiAntiJoinSet()) <
-                        sessionVariable.getCboMaxReorderNodeUseExhaustive()) {
-                    context.getRuleSet().getTransformRules().add(JoinLeftAsscomRule.INNER_JOIN_LEFT_ASSCOM_RULE);
-                }
-                context.getRuleSet().addJoinTransformationRules();
-            }
-        }
-
-        if (!sessionVariable.isDisableJoinReorder() && sessionVariable.isEnableOuterJoinReorder()
-                && Utils.capableOuterReorder(tree, sessionVariable.getCboReorderThresholdUseExhaustive())) {
-            context.getRuleSet().addOuterJoinTransformationRules();
-        }
-
-        if (!sessionVariable.isMVPlanner()) {
-            // add join implementRule
-            String joinImplementationMode = connectContext.getSessionVariable().getJoinImplementationMode();
-            if ("merge".equalsIgnoreCase(joinImplementationMode)) {
-                context.getRuleSet().addMergeJoinImplementationRule();
-            } else if ("hash".equalsIgnoreCase(joinImplementationMode)) {
-                context.getRuleSet().addHashJoinImplementationRule();
-            } else if ("nestloop".equalsIgnoreCase(joinImplementationMode)) {
-                context.getRuleSet().addNestLoopJoinImplementationRule();
-            } else {
-                context.getRuleSet().addAutoJoinImplementationRule();
-            }
-        } else {
-            context.getRuleSet().addRealtimeMVRules();
-        }
-
-        if (mvRewriteStrategy.enableMultiTableRewrite) {
-            context.getRuleSet().addSingleTableMvRewriteRule();
-            context.getRuleSet().addMultiTableMvRewriteRule();
-        }
-
-        scheduler.pushTask(new OptimizeGroupTask(rootTaskContext, memo.getRootGroup()));
-        scheduler.executeTasks(rootTaskContext);
-    }
-
-    private OptExpression physicalRuleRewrite(ConnectContext connectContext, TaskContext rootTaskContext,
-                                              OptExpression result) {
-        Preconditions.checkState(result.getOp().isPhysical());
-
-        int planCount = result.getPlanCount();
-
-        // Since there may be many different plans in the logic phase, it's possible
-        // that this switch can't turned on after logical optimization, so we only determine
-        // whether the PreAggregate can be turned on in the final
-        result = new PreAggregateTurnOnRule().rewrite(result, rootTaskContext);
-
-        // Rewrite Exchange on top of Sort to Final Sort
-        result = new ExchangeSortToMergeRule().rewrite(result, rootTaskContext);
-        result = new PruneAggregateNodeRule().rewrite(result, rootTaskContext);
-        result = new PruneShuffleColumnRule().rewrite(result, rootTaskContext);
-        result = new PhysicalDistributionAggOptRule().rewrite(result, rootTaskContext);
-        result = new AddDecodeNodeForDictStringRule().rewrite(result, rootTaskContext);
-        result = new LowCardinalityRewriteRule().rewrite(result, rootTaskContext);
-        // Put before ScalarOperatorsReuseRule
-        result = new PruneSubfieldsForComplexType().rewrite(result, rootTaskContext);
-        result = new InlineCteProjectPruneRule().rewrite(result, rootTaskContext);
-        // This rule should be last
-        result = new ScalarOperatorsReuseRule().rewrite(result, rootTaskContext);
-        // Reorder predicates
-        result = new PredicateReorderRule(rootTaskContext.getOptimizerContext().getSessionVariable()).rewrite(result,
-                rootTaskContext);
-        result = new ExtractAggregateColumn().rewrite(result, rootTaskContext);
-        result = new JoinLocalShuffleRule().rewrite(result, rootTaskContext);
-
-        // This must be put at last of the optimization. Because wrapping reused ColumnRefOperator with CloneOperator
-        // too early will prevent it from certain optimizations that depend on the equivalence of the ColumnRefOperator.
-        result = new CloneDuplicateColRefRule().rewrite(result, rootTaskContext);
-
-        // set subfield expr copy flag
-        if (rootTaskContext.getOptimizerContext().getSessionVariable().getEnableSubfieldNoCopy()) {
-            result = new SubfieldExprNoCopyRule().rewrite(result, rootTaskContext);
-        }
-
-        result = new AddIndexOnlyPredicateRule().rewrite(result, rootTaskContext);
-        result = new DataCachePopulateRewriteRule(connectContext).rewrite(result, rootTaskContext);
-        result = new EliminateOveruseColumnAccessPathRule().rewrite(result, rootTaskContext);
-        result.setPlanCount(planCount);
-        return result;
-    }
-
-    private OptExpression dynamicRewrite(ConnectContext connectContext, TaskContext rootTaskContext,
-                                         OptExpression result) {
-        // update the existRequiredDistribution value in optExpression. The next rules need it to determine
-        // if we can change the distribution to adjust the plan because of skew data, bad statistics or something else.
-        result = new MarkParentRequiredDistributionRule().rewrite(result, rootTaskContext);
-        result = new ApplyTuningGuideRule(connectContext).rewrite(result, rootTaskContext);
-
-        OperatorTuningGuides.OptimizedRecord optimizedRecord = PlanTuningAdvisor.getInstance()
-                .getOptimizedRecord(context.getQueryId());
-        if (optimizedRecord != null) {
-            Tracers.record(Tracers.Module.BASE, "DynamicApplyTuningGuides", optimizedRecord.getExplainString());
-        }
-        return result;
-    }
-
-    /**
-     * Extract the lowest cost physical operator tree from memo
-     *
-     * @param requiredProperty the required physical property from sql or groupExpression
-     * @param rootGroup        the current group to find the lowest cost physical operator
-     * @return the lowest cost physical operator for this query
-     */
-    private OptExpression extractBestPlan(PhysicalPropertySet requiredProperty,
-                                          Group rootGroup) {
-        GroupExpression groupExpression = rootGroup.getBestExpression(requiredProperty);
-        if (groupExpression == null) {
-            String msg = "no executable plan for this sql. group: %s. required property: %s";
-            throw new IllegalArgumentException(String.format(msg, rootGroup, requiredProperty));
-        }
-        List<PhysicalPropertySet> inputProperties = groupExpression.getInputProperties(requiredProperty);
-
-        List<OptExpression> childPlans = Lists.newArrayList();
-        for (int i = 0; i < groupExpression.arity(); ++i) {
-            OptExpression childPlan = extractBestPlan(inputProperties.get(i), groupExpression.inputAt(i));
-            childPlans.add(childPlan);
-        }
-
-        OptExpression expression = OptExpression.create(groupExpression.getOp(),
-                childPlans);
-        // record inputProperties at optExpression, used for planFragment builder to determine join type
-        expression.setRequiredProperties(inputProperties);
-        expression.setStatistics(groupExpression.getGroup().getStatistics());
-        expression.setCost(groupExpression.getCost(requiredProperty));
-        expression.setOutputProperty(requiredProperty);
-
-        // When build plan fragment, we need the output column of logical property
-        expression.setLogicalProperty(rootGroup.getLogicalProperty());
-        return expression;
-    }
-
-    private void collectAllLogicalOlapScanOperators(OptExpression tree, OptimizerContext optimizerContext) {
-        List<LogicalOlapScanOperator> list = Lists.newArrayList();
-        Utils.extractOperator(tree, list, op -> op instanceof LogicalOlapScanOperator);
-        optimizerContext.setAllLogicalOlapScanOperators(Collections.unmodifiableList(list));
-    }
-
-    private void collectAllPhysicalOlapScanOperators(OptExpression tree, TaskContext rootTaskContext) {
-        List<PhysicalOlapScanOperator> list = Lists.newArrayList();
-        Utils.extractOperator(tree, list, op -> op instanceof PhysicalOlapScanOperator);
-        rootTaskContext.setAllPhysicalOlapScanOperators(Collections.unmodifiableList(list));
-    }
-
-    private void prepareMetaOnlyOnce(OptExpression tree, TaskContext rootTaskContext) {
-        if (rootTaskContext.getOptimizerContext().getSessionVariable().enableParallelPrepareMetadata()) {
-            scheduler.pushTask(new PrepareCollectMetaTask(rootTaskContext, tree));
-            scheduler.executeTasks(rootTaskContext);
-        }
-    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java
index 7b7c5d77e7723e..22699912710b60 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java
@@ -14,27 +14,25 @@
 
 package com.starrocks.sql.optimizer;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.starrocks.catalog.OlapTable;
 import com.starrocks.qe.ConnectContext;
 import com.starrocks.qe.SessionVariable;
-import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.sql.ast.StatementBase;
 import com.starrocks.sql.common.ErrorType;
 import com.starrocks.sql.common.StarRocksPlannerException;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.dump.DumpInfo;
-import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.operator.scalar.IsNullPredicateOperator;
 import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown;
 import com.starrocks.sql.optimizer.rule.RuleSet;
 import com.starrocks.sql.optimizer.rule.RuleType;
 import com.starrocks.sql.optimizer.task.TaskContext;
 import com.starrocks.sql.optimizer.task.TaskScheduler;
+import com.starrocks.sql.optimizer.transformer.MVTransformerContext;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -42,151 +40,100 @@
 import java.util.concurrent.TimeUnit;
 
 public class OptimizerContext {
-    private final UUID queryId;
-    private final Memo memo;
+    // ============================ Query ============================
+    private StatementBase statement;
+    private ConnectContext connectContext;
+    private ColumnRefFactory columnRefFactory;
+    private Set<OlapTable> queryTables;
+    private long updateTableId = -1;
+
+    private OptimizerOptions optimizerOptions;
+
+    // ============================ Optimizer ============================
+    private Memo memo;
     private final RuleSet ruleSet;
-    private final GlobalStateMgr globalStateMgr;
-    private final TaskScheduler taskScheduler;
-    private final ColumnRefFactory columnRefFactory;
-    private SessionVariable sessionVariable;
-    private DumpInfo dumpInfo;
-    private Set<Long> currentSqlDbIds;
-    private CTEContext cteContext;
-    private TaskContext currentTaskContext;
-    private final OptimizerConfig optimizerConfig;
+    private TaskScheduler taskScheduler;
 
-    private Set<OlapTable> queryTables;
+    private final CTEContext cteContext;
+    private TaskContext currentTaskContext;
+    private final QueryMaterializationContext queryMaterializationContext = new QueryMaterializationContext();
 
-    private long updateTableId = -1;
+    private MVTransformerContext mvTransformerContext;
 
-    private boolean isObtainedFromInternalStatistics = false;
+    // uniquePartitionIdGenerator for external catalog
+    private long uniquePartitionIdGenerator = 0L;
     private final Stopwatch optimizerTimer = Stopwatch.createStarted();
     private final Map<RuleType, Stopwatch> ruleWatchMap = Maps.newHashMap();
 
+    // ============================ Task Variables ============================
     // The context for join predicate pushdown rule
-    private JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext =
+    private final JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext =
             new JoinPredicatePushdown.JoinPredicatePushDownContext();
     // QueryMaterializationContext is different from MaterializationContext that it keeps the context during the query
     // lifecycle instead of per materialized view.
-    private QueryMaterializationContext queryMaterializationContext = new QueryMaterializationContext();
 
-    private boolean isShortCircuit = false;
+    private boolean isObtainedFromInternalStatistics = false;
     private boolean inMemoPhase = false;
 
     // Is not null predicate can be derived from inner join or semi join,
     // which should be kept to be used to convert outer join into inner join.
-    private List<IsNullPredicateOperator> pushdownNotNullPredicates = Lists.newArrayList();
-
-    // uniquePartitionIdGenerator for external catalog
-    private long uniquePartitionIdGenerator = 0L;
-
-    // collect all LogicalOlapScanOperators in the query before any optimization
-    private List<LogicalOlapScanOperator> allLogicalOlapScanOperators;
-
-    @VisibleForTesting
-    public OptimizerContext(Memo memo, ColumnRefFactory columnRefFactory) {
-        this.memo = memo;
-        this.ruleSet = new RuleSet();
-        this.globalStateMgr = GlobalStateMgr.getCurrentState();
-        this.taskScheduler = TaskScheduler.create();
-        this.columnRefFactory = columnRefFactory;
-        this.sessionVariable = GlobalStateMgr.getCurrentState().getVariableMgr().newSessionVariable();
-        this.optimizerConfig = new OptimizerConfig();
-        this.queryId = UUID.randomUUID();
-        this.allLogicalOlapScanOperators = Collections.emptyList();
-    }
-
-    @VisibleForTesting
-    public OptimizerContext(Memo memo, ColumnRefFactory columnRefFactory, ConnectContext connectContext) {
-        this(memo, columnRefFactory, connectContext, OptimizerConfig.defaultConfig());
-    }
+    private final List<IsNullPredicateOperator> pushdownNotNullPredicates = Lists.newArrayList();
 
-    public OptimizerContext(Memo memo, ColumnRefFactory columnRefFactory, ConnectContext connectContext,
-                            OptimizerConfig optimizerConfig) {
-        this.memo = memo;
+    OptimizerContext(ConnectContext context) {
+        this.connectContext = context;
         this.ruleSet = new RuleSet();
-        this.globalStateMgr = GlobalStateMgr.getCurrentState();
-        this.taskScheduler = TaskScheduler.create();
-        this.columnRefFactory = columnRefFactory;
-        this.queryId = connectContext.getQueryId();
-        this.sessionVariable = connectContext.getSessionVariable();
-        this.dumpInfo = connectContext.getDumpInfo();
-        this.currentSqlDbIds = connectContext.getCurrentSqlDbIds();
         this.cteContext = new CTEContext();
         cteContext.reset();
-        this.cteContext.setEnableCTE(sessionVariable.isCboCteReuse());
-        this.cteContext.setInlineCTERatio(sessionVariable.getCboCTERuseRatio());
-        this.cteContext.setMaxCTELimit(sessionVariable.getCboCTEMaxLimit());
-        this.optimizerConfig = optimizerConfig;
-    }
+        this.cteContext.setEnableCTE(getSessionVariable().isCboCteReuse());
+        this.cteContext.setInlineCTERatio(getSessionVariable().getCboCTERuseRatio());
+        this.cteContext.setMaxCTELimit(getSessionVariable().getCboCTEMaxLimit());
 
-    public Memo getMemo() {
-        return memo;
+        this.optimizerOptions = OptimizerOptions.defaultOpt();
     }
 
-    public RuleSet getRuleSet() {
-        return ruleSet;
+    // ============================ Query ============================
+    public StatementBase getStatement() {
+        return statement;
     }
 
-    public GlobalStateMgr getCatalog() {
-        return globalStateMgr;
+    public void setStatement(StatementBase statement) {
+        this.statement = statement;
     }
 
-    public TaskScheduler getTaskScheduler() {
-        return taskScheduler;
+    public ConnectContext getConnectContext() {
+        return connectContext;
     }
 
-    public ColumnRefFactory getColumnRefFactory() {
-        return columnRefFactory;
+    public void setConnectContext(ConnectContext connectContext) {
+        this.connectContext = connectContext;
     }
 
-    public final SessionVariable getSessionVariable() {
-        return sessionVariable;
+    public UUID getQueryId() {
+        return connectContext.getQueryId();
     }
 
-    public void setSessionVariable(SessionVariable sessionVariable) {
-        this.sessionVariable = sessionVariable;
+    public final SessionVariable getSessionVariable() {
+        return connectContext.getSessionVariable();
     }
 
     public DumpInfo getDumpInfo() {
-        return dumpInfo;
-    }
-
-    public Set<Long> getCurrentSqlDbIds() {
-        return currentSqlDbIds;
-    }
-
-    public CTEContext getCteContext() {
-        return cteContext;
-    }
-
-    public void setTaskContext(TaskContext context) {
-        this.currentTaskContext = context;
+        return connectContext.getDumpInfo();
     }
 
-    public TaskContext getTaskContext() {
-        return currentTaskContext;
-    }
-
-    public UUID getQueryId() {
-        return queryId;
+    public void setColumnRefFactory(ColumnRefFactory columnRefFactory) {
+        this.columnRefFactory = columnRefFactory;
     }
 
-    public OptimizerConfig getOptimizerConfig() {
-        return optimizerConfig;
+    public ColumnRefFactory getColumnRefFactory() {
+        return columnRefFactory;
     }
 
-    /**
-     * Get all valid candidate materialized views for the query:
-     * - The materialized view is valid to rewrite by rule(SPJG)
-     * - The materialized view's refresh-ness is valid to rewrite.
-     */
-    public List<MaterializationContext> getCandidateMvs() {
-        return queryMaterializationContext.getValidCandidateMVs();
+    public Set<OlapTable> getQueryTables() {
+        return queryTables;
     }
 
-    public JoinPredicatePushdown.JoinPredicatePushDownContext getJoinPushDownParams() {
-        return joinPredicatePushDownContext;
+    public void setQueryTables(Set<OlapTable> queryTables) {
+        this.queryTables = queryTables;
     }
 
     public void setUpdateTableId(long updateTableId) {
@@ -197,87 +144,70 @@ public long getUpdateTableId() {
         return updateTableId;
     }
 
-    public long optimizerElapsedMs() {
-        return optimizerTimer.elapsed(TimeUnit.MILLISECONDS);
+    public OptimizerOptions getOptimizerOptions() {
+        return optimizerOptions;
     }
 
-    public Stopwatch getStopwatch(RuleType ruleType) {
-        return ruleWatchMap.computeIfAbsent(ruleType, (k) -> Stopwatch.createStarted());
+    public void setOptimizerOptions(OptimizerOptions optimizerOptions) {
+        this.optimizerOptions = optimizerOptions;
     }
 
-    public boolean ruleExhausted(RuleType ruleType) {
-        Stopwatch watch = getStopwatch(ruleType);
-        long elapsed = watch.elapsed(TimeUnit.MILLISECONDS);
-        long timeLimit = Math.min(sessionVariable.getOptimizerMaterializedViewTimeLimitMillis(),
-                sessionVariable.getOptimizerExecuteTimeout());
-        return elapsed > timeLimit;
+    // ============================ Optimizer ============================
+    public Memo getMemo() {
+        return memo;
     }
 
-    public boolean isObtainedFromInternalStatistics() {
-        return isObtainedFromInternalStatistics;
+    public void setMemo(Memo memo) {
+        this.memo = memo;
     }
 
-    public void setObtainedFromInternalStatistics(boolean obtainedFromInternalStatistics) {
-        isObtainedFromInternalStatistics = obtainedFromInternalStatistics;
+    public RuleSet getRuleSet() {
+        return ruleSet;
     }
 
-    /**
-     * Whether reach optimizer timeout
-     */
-    public boolean reachTimeout() {
-        long timeout = getSessionVariable().getOptimizerExecuteTimeout();
-        return optimizerElapsedMs() > timeout;
+    public TaskScheduler getTaskScheduler() {
+        return taskScheduler;
     }
 
-    public Set<OlapTable> getQueryTables() {
-        return queryTables;
+    public void setTaskScheduler(TaskScheduler taskScheduler) {
+        this.taskScheduler = taskScheduler;
     }
 
-    public void setQueryTables(Set<OlapTable> queryTables) {
-        this.queryTables = queryTables;
+    public CTEContext getCteContext() {
+        return cteContext;
     }
 
-    /**
-     * Throw exception if reach optimizer timeout
-     */
-    public void checkTimeout() {
-        if (!reachTimeout()) {
-            return;
-        }
-        Memo memo = getMemo();
-        Group group = memo == null ? null : memo.getRootGroup();
-        throw new StarRocksPlannerException("StarRocks planner use long time " + optimizerElapsedMs() +
-                " ms in " + (group == null ? "logical" : "memo") + " phase, This probably because " +
-                "1. FE Full GC, " +
-                "2. Hive external table fetch metadata took a long time, " +
-                "3. The SQL is very complex. " +
-                "You could " +
-                "1. adjust FE JVM config, " +
-                "2. try query again, " +
-                "3. enlarge new_planner_optimize_timeout session variable",
-                ErrorType.INTERNAL_ERROR);
+    public void setTaskContext(TaskContext context) {
+        this.currentTaskContext = context;
     }
 
-    public void setQueryMaterializationContext(QueryMaterializationContext queryMaterializationContext) {
-        this.queryMaterializationContext = queryMaterializationContext;
+    public TaskContext getTaskContext() {
+        return currentTaskContext;
     }
 
     public QueryMaterializationContext getQueryMaterializationContext() {
         return queryMaterializationContext;
     }
 
-    public boolean isShortCircuit() {
-        return isShortCircuit;
+    public MVTransformerContext getMvTransformerContext() {
+        return mvTransformerContext;
     }
 
-    public void setShortCircuit(boolean shortCircuit) {
-        isShortCircuit = shortCircuit;
+    public void setMvTransformerContext(MVTransformerContext mvTransformerContext) {
+        this.mvTransformerContext = mvTransformerContext;
     }
 
-    public void clear() {
-        if (this.queryMaterializationContext != null) {
-            this.queryMaterializationContext.clear();
-        }
+    // ============================ Task Variables ============================
+    public JoinPredicatePushdown.JoinPredicatePushDownContext getJoinPushDownParams() {
+        return joinPredicatePushDownContext;
+    }
+
+    public boolean isObtainedFromInternalStatistics() {
+        return isObtainedFromInternalStatistics;
+    }
+
+    public void setObtainedFromInternalStatistics(boolean obtainedFromInternalStatistics) {
+        isObtainedFromInternalStatistics = obtainedFromInternalStatistics;
     }
 
     public void setInMemoPhase(boolean inMemoPhase) {
@@ -288,6 +218,27 @@ public boolean isInMemoPhase() {
         return this.inMemoPhase;
     }
 
+    /**
+     * Get all valid candidate materialized views for the query:
+     * - The materialized view is valid to rewrite by rule(SPJG)
+     * - The materialized view's refresh-ness is valid to rewrite.
+     */
+    public List<MaterializationContext> getCandidateMvs() {
+        return queryMaterializationContext.getValidCandidateMVs();
+    }
+
+    public Stopwatch getStopwatch(RuleType ruleType) {
+        return ruleWatchMap.computeIfAbsent(ruleType, (k) -> Stopwatch.createStarted());
+    }
+
+    public boolean ruleExhausted(RuleType ruleType) {
+        Stopwatch watch = getStopwatch(ruleType);
+        long elapsed = watch.elapsed(TimeUnit.MILLISECONDS);
+        long timeLimit = Math.min(getSessionVariable().getOptimizerMaterializedViewTimeLimitMillis(),
+                getSessionVariable().getOptimizerExecuteTimeout());
+        return elapsed > timeLimit;
+    }
+
     public List<IsNullPredicateOperator> getPushdownNotNullPredicates() {
         return pushdownNotNullPredicates;
     }
@@ -305,11 +256,23 @@ public long getNextUniquePartitionId() {
         return uniquePartitionIdGenerator++;
     }
 
-    public void setAllLogicalOlapScanOperators(List<LogicalOlapScanOperator> allScanOperators) {
-        this.allLogicalOlapScanOperators = allScanOperators;
-    }
-
-    public List<LogicalOlapScanOperator> getAllLogicalOlapScanOperators() {
-        return allLogicalOlapScanOperators;
+    /**
+     * Throw exception if reach optimizer timeout
+     */
+    public void checkTimeout() {
+        long timeout = getSessionVariable().getOptimizerExecuteTimeout();
+        long now = optimizerTimer.elapsed(TimeUnit.MILLISECONDS);
+        if (timeout > 0 && now > timeout) {
+            throw new StarRocksPlannerException("StarRocks planner use long time " + now +
+                    " ms in " + (inMemoPhase ? "memo" : "logical") + " phase, This probably because " +
+                    "1. FE Full GC, " +
+                    "2. Hive external table fetch metadata took a long time, " +
+                    "3. The SQL is very complex. " +
+                    "You could " +
+                    "1. adjust FE JVM config, " +
+                    "2. try query again, " +
+                    "3. enlarge new_planner_optimize_timeout session variable",
+                    ErrorType.INTERNAL_ERROR);
+        }
     }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java
new file mode 100644
index 00000000000000..e1e55ae1116914
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerFactory.java
@@ -0,0 +1,62 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.starrocks.qe.ConnectContext;
+import com.starrocks.sql.optimizer.base.ColumnRefFactory;
+
+public class OptimizerFactory {
+    @VisibleForTesting
+    public static OptimizerContext mockContext(ConnectContext context, ColumnRefFactory columnRefFactory,
+                                               OptimizerOptions config) {
+        OptimizerContext oc = new OptimizerContext(context);
+        oc.setColumnRefFactory(columnRefFactory);
+        oc.setOptimizerOptions(config);
+        return oc;
+    }
+
+    @VisibleForTesting
+    public static OptimizerContext mockContext(ConnectContext context, ColumnRefFactory columnRefFactory) {
+        return mockContext(context, columnRefFactory, OptimizerOptions.defaultOpt());
+    }
+
+    @VisibleForTesting
+    public static OptimizerContext mockContext(ColumnRefFactory columnRefFactory) {
+        OptimizerContext oc = new OptimizerContext(new ConnectContext());
+        oc.setColumnRefFactory(columnRefFactory);
+        oc.setOptimizerOptions(OptimizerOptions.defaultOpt());
+        return oc;
+    }
+
+    public static OptimizerContext initContext(ConnectContext context, ColumnRefFactory columnRefFactory,
+                                               OptimizerOptions config) {
+        OptimizerContext oc = new OptimizerContext(context);
+        oc.setColumnRefFactory(columnRefFactory);
+        oc.setOptimizerOptions(config);
+        return oc;
+    }
+
+    public static OptimizerContext initContext(ConnectContext context, ColumnRefFactory columnRefFactory) {
+        return initContext(context, columnRefFactory, OptimizerOptions.defaultOpt());
+    }
+
+    public static Optimizer create(OptimizerContext context) {
+        if (context.getOptimizerOptions().isShortCircuit()) {
+            return new ShortCircuitOptimizer(context);
+        }
+        return new QueryOptimizer(context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerOptions.java
similarity index 54%
rename from fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java
rename to fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerOptions.java
index 2839d980c6b087..055de4af800660 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerConfig.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerOptions.java
@@ -12,41 +12,39 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-
 package com.starrocks.sql.optimizer;
 
 import com.starrocks.sql.optimizer.rule.RuleType;
 
 import java.util.BitSet;
 
-public class OptimizerConfig {
-    public enum OptimizerAlgorithm {
+public class OptimizerOptions {
+    public enum OptimizerStrategy {
         RULE_BASED,
-        COST_BASED
+        COST_BASED,
+        SHORT_CIRCUIT,
     }
 
-    private final OptimizerAlgorithm optimizerAlgorithm;
+    private final OptimizerStrategy optimizerStrategy;
 
     private final BitSet ruleSwitches;
 
-    private static final OptimizerConfig DEFAULT_CONFIG = new OptimizerConfig();
-
-    public static OptimizerConfig defaultConfig() {
-        return DEFAULT_CONFIG;
-    }
-
-    public OptimizerConfig() {
-        this(OptimizerAlgorithm.COST_BASED);
+    public OptimizerOptions() {
+        this(OptimizerStrategy.COST_BASED);
     }
-
-    public OptimizerConfig(OptimizerAlgorithm optimizerAlgorithm) {
-        this.optimizerAlgorithm = optimizerAlgorithm;
+    
+    public OptimizerOptions(OptimizerStrategy optimizerStrategy) {
+        this.optimizerStrategy = optimizerStrategy;
         this.ruleSwitches = new BitSet(RuleType.NUM_RULES.ordinal());
         this.ruleSwitches.flip(0, ruleSwitches.size());
     }
 
     public boolean isRuleBased() {
-        return optimizerAlgorithm.equals(OptimizerAlgorithm.RULE_BASED);
+        return optimizerStrategy.equals(OptimizerStrategy.RULE_BASED);
+    }
+
+    public boolean isShortCircuit() {
+        return optimizerStrategy.equals(OptimizerStrategy.SHORT_CIRCUIT);
     }
 
     public void disableRule(RuleType ruleType) {
@@ -56,4 +54,18 @@ public void disableRule(RuleType ruleType) {
     public boolean isRuleDisable(RuleType ruleType) {
         return !ruleSwitches.get(ruleType.ordinal());
     }
+
+    private static final OptimizerOptions DEFAULT_OPTIONS = new OptimizerOptions(OptimizerStrategy.COST_BASED);
+
+    public static OptimizerOptions defaultOpt() {
+        return DEFAULT_OPTIONS;
+    }
+
+    public static OptimizerOptions newRuleBaseOpt() {
+        return new OptimizerOptions(OptimizerStrategy.RULE_BASED);
+    }
+
+    public static OptimizerOptions newShortCircuitOpt() {
+        return new OptimizerOptions(OptimizerStrategy.SHORT_CIRCUIT);
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java
new file mode 100644
index 00000000000000..da92a1a160ee26
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/QueryOptimizer.java
@@ -0,0 +1,957 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.starrocks.analysis.JoinOperator;
+import com.starrocks.catalog.MaterializedView;
+import com.starrocks.common.profile.Timer;
+import com.starrocks.common.profile.Tracers;
+import com.starrocks.qe.ConnectContext;
+import com.starrocks.qe.SessionVariable;
+import com.starrocks.qe.feedback.OperatorTuningGuides;
+import com.starrocks.qe.feedback.PlanTuningAdvisor;
+import com.starrocks.sql.Explain;
+import com.starrocks.sql.optimizer.base.ColumnRefFactory;
+import com.starrocks.sql.optimizer.base.ColumnRefSet;
+import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
+import com.starrocks.sql.optimizer.cost.CostEstimate;
+import com.starrocks.sql.optimizer.operator.Operator;
+import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
+import com.starrocks.sql.optimizer.operator.logical.LogicalTreeAnchorOperator;
+import com.starrocks.sql.optimizer.operator.logical.LogicalViewScanOperator;
+import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator;
+import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown;
+import com.starrocks.sql.optimizer.rule.RuleSet;
+import com.starrocks.sql.optimizer.rule.join.JoinReorderFactory;
+import com.starrocks.sql.optimizer.rule.join.ReorderJoinRule;
+import com.starrocks.sql.optimizer.rule.mv.MaterializedViewRule;
+import com.starrocks.sql.optimizer.rule.transformation.ApplyExceptionRule;
+import com.starrocks.sql.optimizer.rule.transformation.ArrayDistinctAfterAggRule;
+import com.starrocks.sql.optimizer.rule.transformation.CTEProduceAddProjectionRule;
+import com.starrocks.sql.optimizer.rule.transformation.ConvertToEqualForNullRule;
+import com.starrocks.sql.optimizer.rule.transformation.DeriveRangeJoinPredicateRule;
+import com.starrocks.sql.optimizer.rule.transformation.EliminateAggRule;
+import com.starrocks.sql.optimizer.rule.transformation.EliminateConstantCTERule;
+import com.starrocks.sql.optimizer.rule.transformation.EliminateSortColumnWithEqualityPredicateRule;
+import com.starrocks.sql.optimizer.rule.transformation.ForceCTEReuseRule;
+import com.starrocks.sql.optimizer.rule.transformation.GroupByCountDistinctRewriteRule;
+import com.starrocks.sql.optimizer.rule.transformation.IcebergEqualityDeleteRewriteRule;
+import com.starrocks.sql.optimizer.rule.transformation.IcebergPartitionsTableRewriteRule;
+import com.starrocks.sql.optimizer.rule.transformation.JoinLeftAsscomRule;
+import com.starrocks.sql.optimizer.rule.transformation.MaterializedViewTransparentRewriteRule;
+import com.starrocks.sql.optimizer.rule.transformation.MergeProjectWithChildRule;
+import com.starrocks.sql.optimizer.rule.transformation.MergeTwoAggRule;
+import com.starrocks.sql.optimizer.rule.transformation.MergeTwoProjectRule;
+import com.starrocks.sql.optimizer.rule.transformation.OnPredicateMoveAroundRule;
+import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnMinMaxRewriteRule;
+import com.starrocks.sql.optimizer.rule.transformation.PartitionColumnValueOnlyOnScanRule;
+import com.starrocks.sql.optimizer.rule.transformation.PruneEmptyWindowRule;
+import com.starrocks.sql.optimizer.rule.transformation.PullUpScanPredicateRule;
+import com.starrocks.sql.optimizer.rule.transformation.PushDownAggregateGroupingSetsRule;
+import com.starrocks.sql.optimizer.rule.transformation.PushDownJoinOnExpressionToChildProject;
+import com.starrocks.sql.optimizer.rule.transformation.PushDownLimitRankingWindowRule;
+import com.starrocks.sql.optimizer.rule.transformation.PushDownPredicateRankingWindowRule;
+import com.starrocks.sql.optimizer.rule.transformation.PushDownProjectLimitRule;
+import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowOuterJoinRule;
+import com.starrocks.sql.optimizer.rule.transformation.PushDownTopNBelowUnionRule;
+import com.starrocks.sql.optimizer.rule.transformation.PushLimitAndFilterToCTEProduceRule;
+import com.starrocks.sql.optimizer.rule.transformation.RemoveAggregationFromAggTable;
+import com.starrocks.sql.optimizer.rule.transformation.RewriteGroupingSetsByCTERule;
+import com.starrocks.sql.optimizer.rule.transformation.RewriteMultiDistinctRule;
+import com.starrocks.sql.optimizer.rule.transformation.RewriteSimpleAggToHDFSScanRule;
+import com.starrocks.sql.optimizer.rule.transformation.RewriteUnnestBitmapRule;
+import com.starrocks.sql.optimizer.rule.transformation.SchemaTableEvaluateRule;
+import com.starrocks.sql.optimizer.rule.transformation.SeparateProjectRule;
+import com.starrocks.sql.optimizer.rule.transformation.SkewJoinOptimizeRule;
+import com.starrocks.sql.optimizer.rule.transformation.SplitScanORToUnionRule;
+import com.starrocks.sql.optimizer.rule.transformation.UnionToValuesRule;
+import com.starrocks.sql.optimizer.rule.transformation.materialization.MVCompensationPruneUnionRule;
+import com.starrocks.sql.optimizer.rule.transformation.materialization.MvRewriteStrategy;
+import com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils;
+import com.starrocks.sql.optimizer.rule.transformation.materialization.rule.TextMatchBasedRewriteRule;
+import com.starrocks.sql.optimizer.rule.transformation.pruner.CboTablePruneRule;
+import com.starrocks.sql.optimizer.rule.transformation.pruner.PrimaryKeyUpdateTableRule;
+import com.starrocks.sql.optimizer.rule.transformation.pruner.RboTablePruneRule;
+import com.starrocks.sql.optimizer.rule.transformation.pruner.UniquenessBasedTablePruneRule;
+import com.starrocks.sql.optimizer.rule.tree.AddDecodeNodeForDictStringRule;
+import com.starrocks.sql.optimizer.rule.tree.AddIndexOnlyPredicateRule;
+import com.starrocks.sql.optimizer.rule.tree.ApplyTuningGuideRule;
+import com.starrocks.sql.optimizer.rule.tree.CloneDuplicateColRefRule;
+import com.starrocks.sql.optimizer.rule.tree.DataCachePopulateRewriteRule;
+import com.starrocks.sql.optimizer.rule.tree.EliminateOveruseColumnAccessPathRule;
+import com.starrocks.sql.optimizer.rule.tree.ExchangeSortToMergeRule;
+import com.starrocks.sql.optimizer.rule.tree.ExtractAggregateColumn;
+import com.starrocks.sql.optimizer.rule.tree.InlineCteProjectPruneRule;
+import com.starrocks.sql.optimizer.rule.tree.JoinLocalShuffleRule;
+import com.starrocks.sql.optimizer.rule.tree.MarkParentRequiredDistributionRule;
+import com.starrocks.sql.optimizer.rule.tree.PhysicalDistributionAggOptRule;
+import com.starrocks.sql.optimizer.rule.tree.PreAggregateTurnOnRule;
+import com.starrocks.sql.optimizer.rule.tree.PredicateReorderRule;
+import com.starrocks.sql.optimizer.rule.tree.PruneAggregateNodeRule;
+import com.starrocks.sql.optimizer.rule.tree.PruneShuffleColumnRule;
+import com.starrocks.sql.optimizer.rule.tree.PruneSubfieldsForComplexType;
+import com.starrocks.sql.optimizer.rule.tree.PushDownAggregateRule;
+import com.starrocks.sql.optimizer.rule.tree.PushDownDistinctAggregateRule;
+import com.starrocks.sql.optimizer.rule.tree.ScalarOperatorsReuseRule;
+import com.starrocks.sql.optimizer.rule.tree.SimplifyCaseWhenPredicateRule;
+import com.starrocks.sql.optimizer.rule.tree.SubfieldExprNoCopyRule;
+import com.starrocks.sql.optimizer.rule.tree.lowcardinality.LowCardinalityRewriteRule;
+import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PruneSubfieldRule;
+import com.starrocks.sql.optimizer.rule.tree.prunesubfield.PushDownSubfieldRule;
+import com.starrocks.sql.optimizer.task.OptimizeGroupTask;
+import com.starrocks.sql.optimizer.task.PrepareCollectMetaTask;
+import com.starrocks.sql.optimizer.task.TaskContext;
+import com.starrocks.sql.optimizer.task.TaskScheduler;
+import com.starrocks.sql.optimizer.validate.MVRewriteValidator;
+import com.starrocks.sql.optimizer.validate.OptExpressionValidator;
+import com.starrocks.sql.optimizer.validate.PlanValidator;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_TRANSPARENT_REWRITE;
+import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_MV_UNION_REWRITE;
+import static com.starrocks.sql.optimizer.operator.OpRuleBit.OP_PARTITION_PRUNED;
+import static com.starrocks.sql.optimizer.rule.RuleType.TF_MATERIALIZED_VIEW;
+
+/**
+ * QueryOptimizer's entrance class
+ */
+public class QueryOptimizer extends Optimizer {
+    private static final Logger LOG = LogManager.getLogger(QueryOptimizer.class);
+    private OptimizerOptions optimizerOptions;
+    private MvRewriteStrategy mvRewriteStrategy = new MvRewriteStrategy();
+    private final TaskScheduler scheduler = new TaskScheduler();
+    private Memo memo;
+
+    // collect all LogicalOlapScanOperators in the query before any optimization
+    private final List<LogicalOlapScanOperator> allLogicalOlapScanOperators = Lists.newArrayList();
+
+    QueryOptimizer(OptimizerContext context) {
+        super(context);
+    }
+
+    @VisibleForTesting
+    public MvRewriteStrategy getMvRewriteStrategy() {
+        return mvRewriteStrategy;
+    }
+
+    private void prepare(OptExpression logicOperatorTree) {
+        optimizerOptions = context.getOptimizerOptions();
+
+        if (!optimizerOptions.isRuleBased()) {
+            memo = new Memo();
+            context.setMemo(memo);
+        }
+        context.setTaskScheduler(scheduler);
+
+        // collect all olap scan operator
+        Utils.extractOperator(logicOperatorTree, allLogicalOlapScanOperators,
+                op -> op instanceof LogicalOlapScanOperator);
+    }
+
+    public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty,
+                                  ColumnRefSet requiredColumns) {
+        try {
+            // prepare for optimizer
+            prepare(logicOperatorTree);
+
+            // prepare for mv rewrite
+            prepareMvRewrite(context.getConnectContext(), logicOperatorTree, context.getColumnRefFactory(),
+                    requiredColumns);
+            try (Timer ignored = Tracers.watchScope("MVTextRewrite")) {
+                logicOperatorTree = new TextMatchBasedRewriteRule(context.getConnectContext(), context.getStatement(),
+                        context.getMvTransformerContext()).transform(logicOperatorTree, context).get(0);
+            }
+
+            OptExpression result = optimizerOptions.isRuleBased() ?
+                    optimizeByRule(logicOperatorTree, requiredProperty, requiredColumns) :
+                    optimizeByCost(context.getConnectContext(), logicOperatorTree, requiredProperty,
+                            requiredColumns);
+            return result;
+        } finally {
+            // make sure clear caches in OptimizerContext
+            context.getQueryMaterializationContext().clear();
+            context.getConnectContext().setQueryMVContext(null);
+        }
+    }
+
+    // Optimize by rule will return logical plan.
+    // Used by materialized view query rewrite optimization.
+    private OptExpression optimizeByRule(OptExpression logicOperatorTree,
+                                         PhysicalPropertySet requiredProperty,
+                                         ColumnRefSet requiredColumns) {
+        OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree);
+        TaskContext rootTaskContext =
+                new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE);
+        logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext);
+        OptimizerTraceUtil.log("after logical rewrite, new logicOperatorTree:\n%s", logicOperatorTree);
+        return logicOperatorTree;
+    }
+
+    /**
+     * Optimizer will transform and implement the logical operator based on
+     * the {@see Rule}, then cost the physical operator, and finally find the
+     * lowest cost physical operator tree
+     *
+     * @param logicOperatorTree the input for query Optimizer
+     * @param requiredProperty  the required physical property from sql or groupExpression
+     * @param requiredColumns   the required output columns from sql or groupExpression
+     * @return the lowest cost physical operator for this query
+     */
+    private OptExpression optimizeByCost(ConnectContext connectContext,
+                                         OptExpression logicOperatorTree,
+                                         PhysicalPropertySet requiredProperty,
+                                         ColumnRefSet requiredColumns) {
+        // Phase 1: none
+        OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", logicOperatorTree);
+        // Phase 2: rewrite based on memo and group
+        TaskContext rootTaskContext =
+                new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE);
+
+        try (Timer ignored = Tracers.watchScope("RuleBaseOptimize")) {
+            logicOperatorTree = rewriteAndValidatePlan(logicOperatorTree, rootTaskContext);
+        }
+
+        Preconditions.checkNotNull(memo);
+        memo.init(logicOperatorTree);
+        if (context.getQueryMaterializationContext() != null) {
+            // LogicalTreeWithView is logically equivalent to logicOperatorTree
+            addViewBasedPlanIntoMemo(context.getQueryMaterializationContext().getQueryOptPlanWithView());
+        }
+        OptimizerTraceUtil.log("after logical rewrite, root group:\n%s", memo.getRootGroup());
+
+        // Currently, we cache output columns in logic property.
+        // We derive logic property Bottom Up firstly when new group added to memo,
+        // but we do column prune rewrite top down later.
+        // So after column prune rewrite, the output columns for each operator maybe change,
+        // but the logic property is cached and never change.
+        // So we need to explicitly derive all group logic property again
+        memo.deriveAllGroupLogicalProperty();
+
+        // Phase 3: optimize based on memo and group
+        try (Timer ignored = Tracers.watchScope("CostBaseOptimize")) {
+            memoOptimize(connectContext, memo, rootTaskContext);
+        }
+
+        OptExpression result;
+        if (connectContext.getSessionVariable().isSetUseNthExecPlan()) {
+            // extract the nth execution plan
+            int nthExecPlan = connectContext.getSessionVariable().getUseNthExecPlan();
+            result = EnumeratePlan.extractNthPlan(requiredProperty, memo.getRootGroup(), nthExecPlan);
+        } else {
+            result = extractBestPlan(requiredProperty, memo.getRootGroup());
+        }
+        OptimizerTraceUtil.logOptExpression("after extract best plan:\n%s", result);
+
+        // set costs audio log before physicalRuleRewrite
+        // statistics won't set correctly after physicalRuleRewrite.
+        // we need set plan costs before physical rewrite stage.
+        final CostEstimate costs = Explain.buildCost(result);
+        connectContext.getAuditEventBuilder().setPlanCpuCosts(costs.getCpuCost())
+                .setPlanMemCosts(costs.getMemoryCost());
+        OptExpression finalPlan;
+        try (Timer ignored = Tracers.watchScope("PhysicalRewrite")) {
+            finalPlan = physicalRuleRewrite(connectContext, rootTaskContext, result);
+            OptimizerTraceUtil.logOptExpression("final plan after physical rewrite:\n%s", finalPlan);
+        }
+
+        try (Timer ignored = Tracers.watchScope("DynamicRewrite")) {
+            finalPlan = dynamicRewrite(connectContext, rootTaskContext, finalPlan);
+            OptimizerTraceUtil.logOptExpression("final plan after dynamic rewrite:\n%s", finalPlan);
+        }
+
+        // collect all mv scan operator
+        List<PhysicalOlapScanOperator> mvScan = collectAllPhysicalOlapScanOperators(result).stream().
+                filter(scan -> scan.getTable().isMaterializedView()).collect(Collectors.toList());
+        // add mv db id to currentSqlDbIds, the resource group could use this to distinguish sql patterns
+        Set<Long> currentSqlDbIds = context.getConnectContext().getCurrentSqlDbIds();
+        mvScan.stream().map(scan -> ((MaterializedView) scan.getTable()).getDbId()).forEach(currentSqlDbIds::add);
+
+        try (Timer ignored = Tracers.watchScope("PlanValidate")) {
+            // valid the final plan
+            PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext);
+            // validate mv and log tracer if needed
+            MVRewriteValidator mvRewriteValidator = new MVRewriteValidator(allLogicalOlapScanOperators);
+            mvRewriteValidator.validateMV(connectContext, finalPlan, rootTaskContext);
+            // audit mv
+            mvRewriteValidator.auditMv(connectContext, finalPlan, rootTaskContext);
+            return finalPlan;
+        }
+    }
+
+    private void addViewBasedPlanIntoMemo(OptExpression logicalTreeWithView) {
+        if (logicalTreeWithView == null) {
+            return;
+        }
+        Memo memo = context.getMemo();
+        memo.copyIn(memo.getRootGroup(), logicalTreeWithView);
+    }
+
+    private void prepareMvRewrite(ConnectContext connectContext, OptExpression logicOperatorTree,
+                                  ColumnRefFactory columnRefFactory, ColumnRefSet requiredColumns) {
+        SessionVariable sessionVariable = connectContext.getSessionVariable();
+        // MV Rewrite will be used when cbo is enabled.
+        if (context.getOptimizerOptions().isRuleBased() || sessionVariable.isDisableMaterializedViewRewrite() ||
+                !sessionVariable.isEnableMaterializedViewRewrite()) {
+            return;
+        }
+        // prepare related mvs if needed and initialize mv rewrite strategy
+        new MvRewritePreprocessor(connectContext, columnRefFactory, context, requiredColumns)
+                .prepare(logicOperatorTree);
+
+        // initialize mv rewrite strategy finally
+        mvRewriteStrategy = MvRewriteStrategy.prepareRewriteStrategy(context, connectContext, logicOperatorTree);
+        OptimizerTraceUtil.logMVPrepare("MV rewrite strategy: {}", mvRewriteStrategy);
+
+        // TODO(stephen): enable agg push down when query exists related mvs.
+        if (context.getQueryMaterializationContext() != null &&
+                !context.getQueryMaterializationContext().getValidCandidateMVs().isEmpty()) {
+            context.getSessionVariable().setCboPushDownAggregateMode(-1);
+        }
+    }
+
+    private void pruneTables(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) {
+        if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableRboTablePrune()) {
+            if (!Utils.hasPrunableJoin(tree)) {
+                return;
+            }
+            // PARTITION_PRUNE is required to run before ReorderJoinRule because ReorderJoinRule's
+            // Statistics calculation on Operators depends on row count yielded by the PARTITION_PRUNE.
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
+            // ReorderJoinRule is a in-memo rule, when it is used outside memo, we must apply
+            // MergeProjectWithChildRule to merge LogicalProjectionOperator into its child's
+            // projection before ReorderJoinRule's application, after that, we must separate operator's
+            // projection as LogicalProjectionOperator from the operator by applying SeparateProjectRule.
+            scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+            scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
+            CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context);
+            tree = new UniquenessBasedTablePruneRule().rewrite(tree, rootTaskContext);
+            deriveLogicalProperty(tree);
+            tree = new ReorderJoinRule().rewrite(tree, context);
+            tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
+            deriveLogicalProperty(tree);
+            // TODO(by satanson): bucket shuffle join interpolation in PK table's update query can adjust layout
+            //  of the data ingested by OlapTableSink and eliminate race introduced by multiple concurrent write
+            //  operations on the same tablets, pruning this bucket shuffle join make update statement performance
+            //  regression, so we can turn on this rule after we put an bucket-shuffle exchange in front of
+            //  OlapTableSink in future, at present we turn off this rule.
+            if (rootTaskContext.getOptimizerContext().getSessionVariable().isEnableTablePruneOnUpdate()) {
+                tree = new PrimaryKeyUpdateTableRule().rewrite(tree, rootTaskContext);
+                deriveLogicalProperty(tree);
+            }
+            tree = new RboTablePruneRule().rewrite(tree, rootTaskContext);
+            scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+            rootTaskContext.setRequiredColumns(requiredColumns.clone());
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
+            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
+        }
+    }
+
+    /**
+     * Rewrite transparent materialized view.
+     */
+    private OptExpression transparentMVRewrite(OptExpression tree, TaskContext rootTaskContext) {
+        scheduler.rewriteOnce(tree, rootTaskContext, new MaterializedViewTransparentRewriteRule());
+        if (Utils.isOptHasAppliedRule(tree, OP_MV_TRANSPARENT_REWRITE)) {
+            tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
+        }
+        return tree;
+    }
+
+    private void ruleBasedMaterializedViewRewrite(OptExpression tree,
+                                                  TaskContext rootTaskContext,
+                                                  ColumnRefSet requiredColumns) {
+        // skip if mv rewrite is disabled
+        if (!mvRewriteStrategy.enableMaterializedViewRewrite || context.getQueryMaterializationContext() == null) {
+            return;
+        }
+
+        // do rule based mv rewrite if needed
+        if (!context.getQueryMaterializationContext().hasRewrittenSuccess()) {
+            doRuleBasedMaterializedViewRewrite(tree, rootTaskContext);
+        }
+
+        // NOTE: Since union rewrite will generate Filter -> Union -> OlapScan -> OlapScan, need to push filter below Union
+        // and do partition predicate again.
+        // TODO: move this into doRuleBasedMaterializedViewRewrite
+        // TODO: Do it in CBO if needed later.
+        boolean isNeedFurtherPartitionPrune =
+                Utils.isOptHasAppliedRule(tree, op -> op.isOpRuleBitSet(OP_MV_UNION_REWRITE));
+        OptimizerTraceUtil.logMVPrepare("is further partition prune: {}", isNeedFurtherPartitionPrune);
+        if (isNeedFurtherPartitionPrune && context.getQueryMaterializationContext().hasRewrittenSuccess()) {
+            // reset partition prune bit to do partition prune again.
+            MvUtils.getScanOperator(tree).forEach(scan -> {
+                scan.resetOpRuleBit(OP_PARTITION_PRUNED);
+            });
+            // Do predicate push down if union rewrite successes.
+            tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
+            deriveLogicalProperty(tree);
+            // Do partition prune again to avoid unnecessary scan.
+            rootTaskContext.setRequiredColumns(requiredColumns.clone());
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
+            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
+            // It's necessary for external table since its predicate is not used directly after push down.
+            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
+            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES);
+            scheduler.rewriteIterative(tree, rootTaskContext, new MVCompensationPruneUnionRule());
+            scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+        }
+    }
+
+    private void doRuleBasedMaterializedViewRewrite(OptExpression tree,
+                                                    TaskContext rootTaskContext) {
+        if (mvRewriteStrategy.enableViewBasedRewrite) {
+            // try view based mv rewrite first, then try normal mv rewrite rules
+            viewBasedMvRuleRewrite(tree, rootTaskContext);
+        }
+        if (mvRewriteStrategy.enableForceRBORewrite) {
+            // use rule based mv rewrite strategy to do mv rewrite for multi tables query
+            if (mvRewriteStrategy.enableMultiTableRewrite) {
+                scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES);
+            }
+            if (mvRewriteStrategy.enableSingleTableRewrite) {
+                scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES);
+            }
+        } else if (mvRewriteStrategy.enableSingleTableRewrite) {
+            // now add single table materialized view rewrite rules in rule based rewrite phase to boost optimization
+            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES);
+        }
+    }
+
+    private void doMVRewriteWithMultiStages(OptExpression tree,
+                                            TaskContext rootTaskContext) {
+        if (!mvRewriteStrategy.enableMaterializedViewRewrite || !mvRewriteStrategy.mvStrategy.isMultiStages()) {
+            return;
+        }
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+        scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
+        // do rule based mv rewrite
+        doRuleBasedMaterializedViewRewrite(tree, rootTaskContext);
+        new SeparateProjectRule().rewrite(tree, rootTaskContext);
+        deriveLogicalProperty(tree);
+    }
+
+    private OptExpression logicalRuleRewrite(
+            OptExpression tree,
+            TaskContext rootTaskContext) {
+        tree = OptExpression.create(new LogicalTreeAnchorOperator(), tree);
+
+        ColumnRefSet requiredColumns = rootTaskContext.getRequiredColumns().clone();
+        deriveLogicalProperty(tree);
+
+        SessionVariable sessionVariable = rootTaskContext.getOptimizerContext().getSessionVariable();
+        CTEContext cteContext = context.getCteContext();
+        CTEUtils.collectCteOperators(tree, context);
+
+        // see JoinPredicatePushdown
+        JoinPredicatePushdown.JoinPredicatePushDownContext joinPredicatePushDownContext =
+                context.getJoinPushDownParams();
+        joinPredicatePushDownContext.prepare(context, sessionVariable, mvRewriteStrategy);
+
+        // inline CTE if consume use once
+        while (cteContext.hasInlineCTE()) {
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES);
+            CTEUtils.collectCteOperators(tree, context);
+        }
+
+        scheduler.rewriteIterative(tree, rootTaskContext, new EliminateConstantCTERule());
+        CTEUtils.collectCteOperators(tree, context);
+
+        scheduler.rewriteOnce(tree, rootTaskContext, new IcebergPartitionsTableRewriteRule());
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.AGGREGATE_REWRITE_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_SUBQUERY_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_COMMON_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_WINDOW_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SUBQUERY_REWRITE_TO_JOIN_RULES);
+        scheduler.rewriteOnce(tree, rootTaskContext, new ApplyExceptionRule());
+        CTEUtils.collectCteOperators(tree, context);
+
+        if (sessionVariable.isEnableFineGrainedRangePredicate()) {
+            scheduler.rewriteAtMostOnce(tree, rootTaskContext, RuleSet.FINE_GRAINED_RANGE_PREDICATE_RULES);
+        }
+
+        // rewrite transparent materialized view
+        tree = transparentMVRewrite(tree, rootTaskContext);
+
+        // Note: PUSH_DOWN_PREDICATE tasks should be executed before MERGE_LIMIT tasks
+        // because of the Filter node needs to be merged first to avoid the Limit node
+        // cannot merge
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
+        scheduler.rewriteOnce(tree, rootTaskContext, SchemaTableEvaluateRule.getInstance());
+
+        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.ELIMINATE_OP_WITH_CONSTANT_RULES);
+        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownPredicateRankingWindowRule());
+
+        scheduler.rewriteOnce(tree, rootTaskContext, new ConvertToEqualForNullRule());
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
+        // Put EliminateAggRule after PRUNE_COLUMNS to give a chance to prune group bys before eliminate aggregations.
+        scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance());
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_UKFK_JOIN_RULES);
+        deriveLogicalProperty(tree);
+
+        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownJoinOnExpressionToChildProject());
+
+        scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule());
+        // @todo: resolve recursive optimization question:
+        //  MergeAgg -> PruneColumn -> PruneEmptyWindow -> MergeAgg/Project -> PruneColumn...
+        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoAggRule());
+
+        rootTaskContext.setRequiredColumns(requiredColumns.clone());
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
+
+        pruneTables(tree, rootTaskContext, requiredColumns);
+
+        scheduler.rewriteIterative(tree, rootTaskContext, new PruneEmptyWindowRule());
+        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+
+        // rule-based materialized view rewrite: early stage
+        doMVRewriteWithMultiStages(tree, rootTaskContext);
+        joinPredicatePushDownContext.reset();
+
+        // Limit push must be after the column prune,
+        // otherwise the Node containing limit may be prune
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, new PushDownProjectLimitRule());
+
+        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownLimitRankingWindowRule());
+        rewriteGroupingSets(tree, rootTaskContext, sessionVariable);
+
+        // No heavy metadata operation before external table partition prune
+        prepareMetaOnlyOnce(tree, rootTaskContext);
+
+        // apply skew join optimize after push down join on expression to child project,
+        // we need to compute the stats of child project(like subfield).
+        skewJoinOptimize(tree, rootTaskContext);
+        scheduler.rewriteOnce(tree, rootTaskContext, new IcebergEqualityDeleteRewriteRule());
+
+        tree = pruneSubfield(tree, rootTaskContext, requiredColumns);
+
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_ASSERT_ROW_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);
+
+        CTEUtils.collectCteOperators(tree, context);
+        if (cteContext.needOptimizeCTE()) {
+            cteContext.reset();
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.COLLECT_CTE_RULES);
+            rootTaskContext.setRequiredColumns(requiredColumns.clone());
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
+            if (cteContext.needPushLimit() || cteContext.needPushPredicate()) {
+                scheduler.rewriteOnce(tree, rootTaskContext, new PushLimitAndFilterToCTEProduceRule());
+            }
+
+            if (cteContext.needPushPredicate()) {
+                scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
+            }
+
+            if (cteContext.needPushLimit()) {
+                scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
+            }
+
+            scheduler.rewriteOnce(tree, rootTaskContext, new ForceCTEReuseRule());
+        }
+
+        // Add a config to decide whether to rewrite sync mv.
+        if (!optimizerOptions.isRuleDisable(TF_MATERIALIZED_VIEW)
+                && sessionVariable.isEnableSyncMaterializedViewRewrite()) {
+            // Split or predicates to union all so can be used by mv rewrite to choose the best sort key indexes.
+            // TODO: support adaptive for or-predicates to union all.
+            if (SplitScanORToUnionRule.isForceRewrite()) {
+                scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance());
+            }
+
+            OptimizerTraceUtil.logOptExpression("before MaterializedViewRule:\n%s", tree);
+            tree = new MaterializedViewRule().transform(tree, context).get(0);
+            OptimizerTraceUtil.logOptExpression("after MaterializedViewRule:\n%s", tree);
+
+            deriveLogicalProperty(tree);
+        }
+
+        scheduler.rewriteDownTop(tree, rootTaskContext, OnPredicateMoveAroundRule.INSTANCE);
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
+
+        scheduler.rewriteIterative(tree, rootTaskContext, new PartitionColumnMinMaxRewriteRule());
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, new RewriteMultiDistinctRule());
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PUSH_DOWN_PREDICATE_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_EMPTY_OPERATOR_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, new CTEProduceAddProjectionRule());
+        scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.PRUNE_PROJECT_RULES);
+
+        // ArrayDistinctAfterAggRule must run before pushDownAggregation,
+        // because push down agg won't have array_distinct project
+        if (sessionVariable.getEnableArrayDistinctAfterAggOpt()) {
+            scheduler.rewriteOnce(tree, rootTaskContext, new ArrayDistinctAfterAggRule());
+        }
+
+        tree = pushDownAggregation(tree, rootTaskContext, requiredColumns);
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.MERGE_LIMIT_RULES);
+
+        CTEUtils.collectCteOperators(tree, context);
+        // inline CTE if consume use once
+        while (cteContext.hasInlineCTE()) {
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INLINE_CTE_RULES);
+            CTEUtils.collectCteOperators(tree, context);
+        }
+
+        scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.META_SCAN_REWRITE_RULES);
+        scheduler.rewriteOnce(tree, rootTaskContext, new PartitionColumnValueOnlyOnScanRule());
+        // before MergeProjectWithChildRule, after INLINE_CTE and MergeApplyWithTableFunction
+        scheduler.rewriteIterative(tree, rootTaskContext, RewriteUnnestBitmapRule.getInstance());
+
+        // After this rule, we shouldn't generate logical project operator
+        scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
+
+        scheduler.rewriteOnce(tree, rootTaskContext, new EliminateSortColumnWithEqualityPredicateRule());
+        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowOuterJoinRule());
+        // intersect rewrite depend on statistics
+        Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext());
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.INTERSECT_REWRITE_RULES);
+        scheduler.rewriteIterative(tree, rootTaskContext, new RemoveAggregationFromAggTable());
+
+        scheduler.rewriteOnce(tree, rootTaskContext, SplitScanORToUnionRule.getInstance());
+        scheduler.rewriteOnce(tree, rootTaskContext, new PushDownTopNBelowUnionRule());
+
+        // rule based materialized view rewrite
+        ruleBasedMaterializedViewRewrite(tree, rootTaskContext, requiredColumns);
+
+        // this rewrite rule should be after mv.
+        scheduler.rewriteOnce(tree, rootTaskContext, RewriteSimpleAggToHDFSScanRule.SCAN_NO_PROJECT);
+
+        // NOTE: This rule should be after MV Rewrite because MV Rewrite cannot handle
+        // select count(distinct c) from t group by a, b
+        // if this rule has applied before MV.
+        scheduler.rewriteOnce(tree, rootTaskContext, new GroupByCountDistinctRewriteRule());
+
+        scheduler.rewriteOnce(tree, rootTaskContext, new DeriveRangeJoinPredicateRule());
+
+        scheduler.rewriteOnce(tree, rootTaskContext, UnionToValuesRule.getInstance());
+
+        scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.VECTOR_REWRITE_RULES);
+        // this rule should be after mv
+        // @TODO: it can also be applied to other table scan operator
+        if (context.getSessionVariable().isEnableScanPredicateExprReuse()) {
+            scheduler.rewriteOnce(tree, rootTaskContext, PullUpScanPredicateRule.OLAP_SCAN);
+        }
+
+        tree = SimplifyCaseWhenPredicateRule.INSTANCE.rewrite(tree, rootTaskContext);
+        deriveLogicalProperty(tree);
+        return tree.getInputs().get(0);
+    }
+
+    private void rewriteGroupingSets(OptExpression tree, TaskContext rootTaskContext, SessionVariable sessionVariable) {
+        if (sessionVariable.isEnableRewriteGroupingsetsToUnionAll()) {
+            scheduler.rewriteIterative(tree, rootTaskContext, new RewriteGroupingSetsByCTERule());
+        }
+        if (sessionVariable.isCboPushDownGroupingSet()) {
+            scheduler.rewriteOnce(tree, rootTaskContext, new PushDownAggregateGroupingSetsRule());
+        }
+    }
+
+    // for single scan node, to make sure we can rewrite
+    private void viewBasedMvRuleRewrite(OptExpression tree, TaskContext rootTaskContext) {
+        QueryMaterializationContext queryMaterializationContext = context.getQueryMaterializationContext();
+        Preconditions.checkArgument(queryMaterializationContext != null);
+
+        try (Timer ignored = Tracers.watchScope("MVViewRewrite")) {
+            OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "try VIEW_BASED_MV_REWRITE");
+            OptExpression treeWithView = queryMaterializationContext.getQueryOptPlanWithView();
+            // should add a LogicalTreeAnchorOperator for rewrite
+            treeWithView = OptExpression.create(new LogicalTreeAnchorOperator(), treeWithView);
+            if (mvRewriteStrategy.enableMultiTableRewrite) {
+                scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.MULTI_TABLE_MV_REWRITE_RULES);
+            }
+            if (mvRewriteStrategy.enableSingleTableRewrite) {
+                scheduler.rewriteIterative(treeWithView, rootTaskContext, RuleSet.SINGLE_TABLE_MV_REWRITE_RULES);
+            }
+
+            List<Operator> leftViewScanOperators = Lists.newArrayList();
+            MvUtils.collectViewScanOperator(treeWithView, leftViewScanOperators);
+            List<LogicalViewScanOperator> origQueryViewScanOperators =
+                    queryMaterializationContext.getQueryViewScanOps();
+            if (leftViewScanOperators.size() < origQueryViewScanOperators.size()) {
+                // replace original tree plan
+                tree.setChild(0, treeWithView.inputAt(0));
+                deriveLogicalProperty(tree);
+
+                // if there are view scan operator left, we should replace it back to original plans
+                if (!leftViewScanOperators.isEmpty()) {
+                    MvUtils.replaceLogicalViewScanOperator(tree);
+                }
+            }
+            OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE", "original view scans size: {}, " +
+                    "left view scans size: {}", origQueryViewScanOperators.size(), leftViewScanOperators.size());
+        } catch (Exception e) {
+            OptimizerTraceUtil.logMVRewriteRule("VIEW_BASED_MV_REWRITE",
+                    "single table view based mv rule rewrite failed.", e);
+        }
+    }
+
+    private OptExpression rewriteAndValidatePlan(
+            OptExpression tree,
+            TaskContext rootTaskContext) {
+        OptExpression result = logicalRuleRewrite(tree, rootTaskContext);
+        OptExpressionValidator validator = new OptExpressionValidator();
+        validator.validate(result);
+        return result;
+    }
+
+    private OptExpression pushDownAggregation(OptExpression tree, TaskContext rootTaskContext,
+                                              ColumnRefSet requiredColumns) {
+        boolean pushDistinctFlag = false;
+        boolean pushAggFlag = false;
+        if (context.getSessionVariable().isCboPushDownDistinctBelowWindow()) {
+            // TODO(by satanson): in future, PushDownDistinctAggregateRule and PushDownAggregateRule should be
+            //  fused one rule to tackle with all scenarios of agg push-down.
+            PushDownDistinctAggregateRule rule = new PushDownDistinctAggregateRule(rootTaskContext);
+            tree = rule.rewrite(tree, rootTaskContext);
+            pushDistinctFlag = rule.getRewriter().hasRewrite();
+        }
+
+        if (context.getSessionVariable().getCboPushDownAggregateMode() != -1) {
+            if (context.getSessionVariable().isCboPushDownAggregateOnBroadcastJoin()) {
+                // Reorder joins before applying PushDownAggregateRule to better decide where to push down aggregator.
+                // For example, do not push down a not very efficient aggregator below a very small broadcast join.
+                scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PARTITION_PRUNE_RULES);
+                scheduler.rewriteIterative(tree, rootTaskContext, new MergeTwoProjectRule());
+                scheduler.rewriteIterative(tree, rootTaskContext, new MergeProjectWithChildRule());
+                CTEUtils.collectForceCteStatisticsOutsideMemo(tree, context);
+                deriveLogicalProperty(tree);
+                tree = new ReorderJoinRule().rewrite(tree, JoinReorderFactory.createJoinReorderAdaptive(), context);
+                tree = new SeparateProjectRule().rewrite(tree, rootTaskContext);
+                deriveLogicalProperty(tree);
+                Utils.calculateStatistics(tree, context);
+            }
+
+            PushDownAggregateRule rule = new PushDownAggregateRule(rootTaskContext);
+            rule.getRewriter().collectRewriteContext(tree);
+            if (rule.getRewriter().isNeedRewrite()) {
+                pushAggFlag = true;
+                tree = rule.rewrite(tree, rootTaskContext);
+            }
+        }
+
+        if (pushDistinctFlag || pushAggFlag) {
+            deriveLogicalProperty(tree);
+            rootTaskContext.setRequiredColumns(requiredColumns.clone());
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
+            scheduler.rewriteOnce(tree, rootTaskContext, EliminateAggRule.getInstance());
+        }
+
+        return tree;
+    }
+
+    private void skewJoinOptimize(OptExpression tree, TaskContext rootTaskContext) {
+        if (context.getSessionVariable().isEnableStatsToOptimizeSkewJoin()) {
+            // merge projects before calculate statistics
+            scheduler.rewriteOnce(tree, rootTaskContext, new MergeTwoProjectRule());
+            Utils.calculateStatistics(tree, rootTaskContext.getOptimizerContext());
+        }
+        scheduler.rewriteOnce(tree, rootTaskContext, new SkewJoinOptimizeRule());
+    }
+
+    private OptExpression pruneSubfield(OptExpression tree, TaskContext rootTaskContext, ColumnRefSet requiredColumns) {
+        if (!context.getSessionVariable().isCboPruneSubfield()) {
+            return tree;
+        }
+
+        PushDownSubfieldRule pushDownRule = new PushDownSubfieldRule();
+        tree = pushDownRule.rewrite(tree, rootTaskContext);
+
+        if (pushDownRule.hasRewrite()) {
+            rootTaskContext.setRequiredColumns(requiredColumns.clone());
+            scheduler.rewriteOnce(tree, rootTaskContext, RuleSet.PRUNE_COLUMNS_RULES);
+        }
+        scheduler.rewriteOnce(tree, rootTaskContext, new PruneSubfieldRule());
+
+        return tree;
+    }
+
+    void memoOptimize(ConnectContext connectContext, Memo memo, TaskContext rootTaskContext) {
+        context.setInMemoPhase(true);
+        OptExpression tree = memo.getRootGroup().extractLogicalTree();
+        SessionVariable sessionVariable = connectContext.getSessionVariable();
+        // add CboTablePruneRule
+        if (Utils.countJoinNodeSize(tree, CboTablePruneRule.JOIN_TYPES) < 10 &&
+                sessionVariable.isEnableCboTablePrune()) {
+            context.getRuleSet().addCboTablePruneRule();
+        }
+        // Join reorder
+        int innerCrossJoinNode = Utils.countJoinNodeSize(tree, JoinOperator.innerCrossJoinSet());
+        if (!sessionVariable.isDisableJoinReorder() && innerCrossJoinNode < sessionVariable.getCboMaxReorderNode()) {
+            if (innerCrossJoinNode > sessionVariable.getCboMaxReorderNodeUseExhaustive()) {
+                CTEUtils.collectForceCteStatistics(memo, context);
+
+                OptimizerTraceUtil.logOptExpression("before ReorderJoinRule:\n%s", tree);
+                new ReorderJoinRule().transform(tree, context);
+                OptimizerTraceUtil.logOptExpression("after ReorderJoinRule:\n%s", tree);
+
+                context.getRuleSet().addJoinCommutativityWithoutInnerRule();
+            } else {
+                if (Utils.countJoinNodeSize(tree, JoinOperator.semiAntiJoinSet()) <
+                        sessionVariable.getCboMaxReorderNodeUseExhaustive()) {
+                    context.getRuleSet().getTransformRules().add(JoinLeftAsscomRule.INNER_JOIN_LEFT_ASSCOM_RULE);
+                }
+                context.getRuleSet().addJoinTransformationRules();
+            }
+        }
+
+        if (!sessionVariable.isDisableJoinReorder() && sessionVariable.isEnableOuterJoinReorder()
+                && Utils.capableOuterReorder(tree, sessionVariable.getCboReorderThresholdUseExhaustive())) {
+            context.getRuleSet().addOuterJoinTransformationRules();
+        }
+
+        if (!sessionVariable.isMVPlanner()) {
+            // add join implementRule
+            String joinImplementationMode = connectContext.getSessionVariable().getJoinImplementationMode();
+            if ("merge".equalsIgnoreCase(joinImplementationMode)) {
+                context.getRuleSet().addMergeJoinImplementationRule();
+            } else if ("hash".equalsIgnoreCase(joinImplementationMode)) {
+                context.getRuleSet().addHashJoinImplementationRule();
+            } else if ("nestloop".equalsIgnoreCase(joinImplementationMode)) {
+                context.getRuleSet().addNestLoopJoinImplementationRule();
+            } else {
+                context.getRuleSet().addAutoJoinImplementationRule();
+            }
+        } else {
+            context.getRuleSet().addRealtimeMVRules();
+        }
+
+        if (mvRewriteStrategy.enableMultiTableRewrite) {
+            context.getRuleSet().addSingleTableMvRewriteRule();
+            context.getRuleSet().addMultiTableMvRewriteRule();
+        }
+
+        scheduler.pushTask(new OptimizeGroupTask(rootTaskContext, memo.getRootGroup()));
+        scheduler.executeTasks(rootTaskContext);
+    }
+
+    private OptExpression physicalRuleRewrite(ConnectContext connectContext, TaskContext rootTaskContext,
+                                              OptExpression result) {
+        Preconditions.checkState(result.getOp().isPhysical());
+
+        int planCount = result.getPlanCount();
+
+        // Since there may be many different plans in the logic phase, it's possible
+        // that this switch can't turned on after logical optimization, so we only determine
+        // whether the PreAggregate can be turned on in the final
+        result = new PreAggregateTurnOnRule().rewrite(result, rootTaskContext);
+
+        // Rewrite Exchange on top of Sort to Final Sort
+        result = new ExchangeSortToMergeRule().rewrite(result, rootTaskContext);
+        result = new PruneAggregateNodeRule().rewrite(result, rootTaskContext);
+        result = new PruneShuffleColumnRule().rewrite(result, rootTaskContext);
+        result = new PhysicalDistributionAggOptRule().rewrite(result, rootTaskContext);
+        result = new AddDecodeNodeForDictStringRule().rewrite(result, rootTaskContext);
+        result = new LowCardinalityRewriteRule().rewrite(result, rootTaskContext);
+        // Put before ScalarOperatorsReuseRule
+        result = new PruneSubfieldsForComplexType().rewrite(result, rootTaskContext);
+        result = new InlineCteProjectPruneRule().rewrite(result, rootTaskContext);
+        // This rule should be last
+        result = new ScalarOperatorsReuseRule().rewrite(result, rootTaskContext);
+        // Reorder predicates
+        result = new PredicateReorderRule(rootTaskContext.getOptimizerContext().getSessionVariable()).rewrite(result,
+                rootTaskContext);
+        result = new ExtractAggregateColumn().rewrite(result, rootTaskContext);
+        result = new JoinLocalShuffleRule().rewrite(result, rootTaskContext);
+
+        // This must be put at last of the optimization. Because wrapping reused ColumnRefOperator with CloneOperator
+        // too early will prevent it from certain optimizations that depend on the equivalence of the ColumnRefOperator.
+        result = new CloneDuplicateColRefRule().rewrite(result, rootTaskContext);
+
+        // set subfield expr copy flag
+        if (rootTaskContext.getOptimizerContext().getSessionVariable().getEnableSubfieldNoCopy()) {
+            result = new SubfieldExprNoCopyRule().rewrite(result, rootTaskContext);
+        }
+
+        result = new AddIndexOnlyPredicateRule().rewrite(result, rootTaskContext);
+        result = new DataCachePopulateRewriteRule(connectContext).rewrite(result, rootTaskContext);
+        result = new EliminateOveruseColumnAccessPathRule().rewrite(result, rootTaskContext);
+        result.setPlanCount(planCount);
+        return result;
+    }
+
+    private OptExpression dynamicRewrite(ConnectContext connectContext, TaskContext rootTaskContext,
+                                         OptExpression result) {
+        // update the existRequiredDistribution value in optExpression. The next rules need it to determine
+        // if we can change the distribution to adjust the plan because of skew data, bad statistics or something else.
+        result = new MarkParentRequiredDistributionRule().rewrite(result, rootTaskContext);
+        result = new ApplyTuningGuideRule(connectContext).rewrite(result, rootTaskContext);
+
+        OperatorTuningGuides.OptimizedRecord optimizedRecord = PlanTuningAdvisor.getInstance()
+                .getOptimizedRecord(context.getQueryId());
+        if (optimizedRecord != null) {
+            Tracers.record(Tracers.Module.BASE, "DynamicApplyTuningGuides", optimizedRecord.getExplainString());
+        }
+        return result;
+    }
+
+    /**
+     * Extract the lowest cost physical operator tree from memo
+     *
+     * @param requiredProperty the required physical property from sql or groupExpression
+     * @param rootGroup        the current group to find the lowest cost physical operator
+     * @return the lowest cost physical operator for this query
+     */
+    private OptExpression extractBestPlan(PhysicalPropertySet requiredProperty,
+                                          Group rootGroup) {
+        GroupExpression groupExpression = rootGroup.getBestExpression(requiredProperty);
+        if (groupExpression == null) {
+            String msg = "no executable plan for this sql. group: %s. required property: %s";
+            throw new IllegalArgumentException(String.format(msg, rootGroup, requiredProperty));
+        }
+        List<PhysicalPropertySet> inputProperties = groupExpression.getInputProperties(requiredProperty);
+
+        List<OptExpression> childPlans = Lists.newArrayList();
+        for (int i = 0; i < groupExpression.arity(); ++i) {
+            OptExpression childPlan = extractBestPlan(inputProperties.get(i), groupExpression.inputAt(i));
+            childPlans.add(childPlan);
+        }
+
+        OptExpression expression = OptExpression.create(groupExpression.getOp(),
+                childPlans);
+        // record inputProperties at optExpression, used for planFragment builder to determine join type
+        expression.setRequiredProperties(inputProperties);
+        expression.setStatistics(groupExpression.getGroup().getStatistics());
+        expression.setCost(groupExpression.getCost(requiredProperty));
+        expression.setOutputProperty(requiredProperty);
+
+        // When build plan fragment, we need the output column of logical property
+        expression.setLogicalProperty(rootGroup.getLogicalProperty());
+        return expression;
+    }
+
+    private List<PhysicalOlapScanOperator> collectAllPhysicalOlapScanOperators(OptExpression tree) {
+        List<PhysicalOlapScanOperator> list = Lists.newArrayList();
+        Utils.extractOperator(tree, list, op -> op instanceof PhysicalOlapScanOperator);
+        return list;
+    }
+
+    private void prepareMetaOnlyOnce(OptExpression tree, TaskContext rootTaskContext) {
+        if (rootTaskContext.getOptimizerContext().getSessionVariable().enableParallelPrepareMetadata()) {
+            scheduler.pushTask(new PrepareCollectMetaTask(rootTaskContext, tree));
+            scheduler.executeTasks(rootTaskContext);
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java
new file mode 100644
index 00000000000000..a880ce9f9793a9
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/ShortCircuitOptimizer.java
@@ -0,0 +1,68 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.sql.optimizer;
+
+import com.starrocks.common.profile.Timer;
+import com.starrocks.common.profile.Tracers;
+import com.starrocks.sql.optimizer.base.ColumnRefSet;
+import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
+import com.starrocks.sql.optimizer.operator.OperatorType;
+import com.starrocks.sql.optimizer.operator.logical.LogicalTreeAnchorOperator;
+import com.starrocks.sql.optimizer.rule.RuleSet;
+import com.starrocks.sql.optimizer.rule.implementation.OlapScanImplementationRule;
+import com.starrocks.sql.optimizer.rule.transformation.MergeProjectWithChildRule;
+import com.starrocks.sql.optimizer.task.TaskContext;
+import com.starrocks.sql.optimizer.task.TaskScheduler;
+import com.starrocks.sql.optimizer.validate.OptExpressionValidator;
+
+public class ShortCircuitOptimizer extends Optimizer {
+
+    ShortCircuitOptimizer(OptimizerContext context) {
+        super(context);
+    }
+
+    @Override
+    public OptExpression optimize(OptExpression tree, PhysicalPropertySet requiredProperty,
+                                  ColumnRefSet requiredColumns) {
+        TaskScheduler scheduler = new TaskScheduler();
+        context.setTaskScheduler(scheduler);
+
+        // Phase 1: none
+        OptimizerTraceUtil.logOptExpression("origin logicOperatorTree:\n%s", tree);
+        // Phase 2: rewrite based on memo and group
+        TaskContext rootTaskContext =
+                new TaskContext(context, requiredProperty, requiredColumns.clone(), Double.MAX_VALUE);
+
+        try (Timer ignored = Tracers.watchScope("ShortCircuitOptimize")) {
+            tree = OptExpression.create(new LogicalTreeAnchorOperator(), tree);
+            // for short circuit
+            deriveLogicalProperty(tree);
+            scheduler.rewriteIterative(tree, rootTaskContext, RuleSet.SHORT_CIRCUIT_SET_RULES);
+            scheduler.rewriteOnce(tree, rootTaskContext, new MergeProjectWithChildRule());
+            tree = tree.getInputs().get(0);
+
+            if (OperatorType.LOGICAL_LIMIT.equals(tree.getOp().getOpType())) {
+                tree = tree.getInputs().get(0);
+            }
+
+            OptExpressionValidator validator = new OptExpressionValidator();
+            validator.validate(tree);
+
+            // skip memo
+            tree = new OlapScanImplementationRule().transform(tree, null).get(0);
+            return tree;
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java
index 55036cf51eb603..f3aea9d56b177b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumpInfo.java
@@ -62,7 +62,7 @@ public class QueryDumpInfo implements DumpInfo {
     private final List<String> exceptionList = new ArrayList<>();
     private int beNum;
     private int cachedAvgNumOfHardwareCores = -1;
-    private Map<Long, Integer> numOfHardwareCoresPerBe = Maps.newHashMap();
+    private final Map<Long, Integer> numOfHardwareCoresPerBe = Maps.newHashMap();
 
     private SessionVariable sessionVariable;
     private final ConnectContext connectContext;
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java
index d405d85f7f0c44..d37aa41bbcdc74 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/JoinPredicatePushdown.java
@@ -59,8 +59,8 @@ public class JoinPredicatePushdown {
 
     private OptExpression joinOptExpression;
 
-    private List<ScalarOperator> leftPushDown;
-    private List<ScalarOperator> rightPushDown;
+    private final List<ScalarOperator> leftPushDown;
+    private final List<ScalarOperator> rightPushDown;
 
     /**
      * Whether to do complete equivalence derive in
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java
index 4bff43fc5ea2d3..737dff24a94bc7 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/PruneProjectColumnsRule.java
@@ -67,7 +67,7 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
             ColumnRefOperator constCol = context.getColumnRefFactory()
                     .create("auto_fill_col", Type.TINYINT, false);
             newMap.put(constCol, ConstantOperator.createTinyInt((byte) 1));
-        } else if (newMap.equals(projectOperator.getColumnRefMap()) && context.isShortCircuit()) {
+        } else if (newMap.equals(projectOperator.getColumnRefMap()) && context.getOptimizerOptions().isShortCircuit()) {
             // Change the requiredOutputColumns in context
             requiredOutputColumns.union(requiredInputColumns);
             // make sure this rule only executed once
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java
index d047b36e25f55b..bea986321e321f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategy.java
@@ -19,8 +19,8 @@
 import com.starrocks.qe.SessionVariable;
 import com.starrocks.sql.optimizer.MaterializationContext;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerConfig;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerOptions;
 import com.starrocks.sql.optimizer.rule.RuleType;
 
 public class MvRewriteStrategy {
@@ -59,14 +59,14 @@ public boolean isMultiStages() {
     public boolean enableMultiTableRewrite = false;
 
     static class MvStrategyArbitrator {
-        private final OptimizerConfig optimizerConfig;
+        private final OptimizerOptions optimizerOptions;
         private final OptimizerContext optimizerContext;
         private final SessionVariable sessionVariable;
 
         public MvStrategyArbitrator(OptimizerContext optimizerContext,
                                     ConnectContext connectContext) {
             this.optimizerContext = optimizerContext;
-            this.optimizerConfig = optimizerContext.getOptimizerConfig();
+            this.optimizerOptions = optimizerContext.getOptimizerOptions();
             // from connectContext rather than optimizerContext
             this.sessionVariable = connectContext.getSessionVariable();
         }
@@ -84,8 +84,8 @@ private boolean isEnableMaterializedViewRewrite() {
                     optimizerContext.getCandidateMvs().isEmpty()) {
                 return false;
             }
-            if (optimizerConfig.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE) &&
-                    optimizerConfig.isRuleDisable(RuleType.GP_MULTI_TABLE_MV_REWRITE)) {
+            if (optimizerOptions.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE) &&
+                    optimizerOptions.isRuleDisable(RuleType.GP_MULTI_TABLE_MV_REWRITE)) {
                 return false;
             }
             return true;
@@ -100,7 +100,7 @@ private boolean isEnableRBOViewBasedRewrite() {
 
         private boolean isEnableRBOSingleTableRewrite(OptExpression queryPlan) {
             // if disable single mv rewrite, return false.
-            if (optimizerConfig.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE)) {
+            if (optimizerOptions.isRuleDisable(RuleType.GP_SINGLE_TABLE_MV_REWRITE)) {
                 return false;
             }
             // If query only has one table use single table rewrite, view delta only rewrites multi-tables query.
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
index f9c4b9bc37814b..671baf41fcfdfe 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvUtils.java
@@ -71,8 +71,9 @@
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptExpressionVisitor;
 import com.starrocks.sql.optimizer.Optimizer;
-import com.starrocks.sql.optimizer.OptimizerConfig;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
+import com.starrocks.sql.optimizer.OptimizerOptions;
 import com.starrocks.sql.optimizer.QueryMaterializationContext;
 import com.starrocks.sql.optimizer.Utils;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
@@ -478,7 +479,7 @@ public static StatementBase parse(MaterializedView mv,
     public static Pair<OptExpression, LogicalPlan> getRuleOptimizedLogicalPlan(StatementBase mvStmt,
                                                                                ColumnRefFactory columnRefFactory,
                                                                                ConnectContext connectContext,
-                                                                               OptimizerConfig optimizerConfig,
+                                                                               OptimizerOptions optimizerOptions,
                                                                                boolean inlineView) {
         Preconditions.checkState(mvStmt instanceof QueryStatement);
         Analyzer.analyze(mvStmt, connectContext);
@@ -486,13 +487,13 @@ public static Pair<OptExpression, LogicalPlan> getRuleOptimizedLogicalPlan(State
         TransformerContext transformerContext =
                 new TransformerContext(columnRefFactory, connectContext, inlineView, null);
         LogicalPlan logicalPlan = new RelationTransformer(transformerContext).transform(query);
-        Optimizer optimizer = new Optimizer(optimizerConfig);
+        Optimizer optimizer =
+                OptimizerFactory.create(
+                        OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerOptions));
         OptExpression optimizedPlan = optimizer.optimize(
-                connectContext,
                 logicalPlan.getRoot(),
                 new PhysicalPropertySet(),
-                new ColumnRefSet(logicalPlan.getOutputColumn()),
-                columnRefFactory);
+                new ColumnRefSet(logicalPlan.getOutputColumn()));
         return Pair.create(optimizedPlan, logicalPlan);
     }
 
@@ -1569,12 +1570,13 @@ public static OptExpression optimizeViewPlan(OptExpression logicalTree,
                                                  ConnectContext connectContext,
                                                  ColumnRefSet requiredColumns,
                                                  ColumnRefFactory columnRefFactory) {
-        OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED);
-        optimizerConfig.disableRule(RuleType.GP_SINGLE_TABLE_MV_REWRITE);
-        optimizerConfig.disableRule(RuleType.GP_MULTI_TABLE_MV_REWRITE);
-        Optimizer optimizer = new Optimizer(optimizerConfig);
-        OptExpression optimizedViewPlan = optimizer.optimize(connectContext, logicalTree,
-                new PhysicalPropertySet(), requiredColumns, columnRefFactory);
+        OptimizerOptions optimizerOptions = OptimizerOptions.newRuleBaseOpt();
+        optimizerOptions.disableRule(RuleType.GP_SINGLE_TABLE_MV_REWRITE);
+        optimizerOptions.disableRule(RuleType.GP_MULTI_TABLE_MV_REWRITE);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.initContext(connectContext, columnRefFactory, optimizerOptions));
+        OptExpression optimizedViewPlan = optimizer.optimize(logicalTree,
+                new PhysicalPropertySet(), requiredColumns);
         return optimizedViewPlan;
     }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java
index 7d5bef5af7dd2a..7437f2e3feab94 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rule/transformation/materialization/rule/TextMatchBasedRewriteRule.java
@@ -115,7 +115,7 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
     private OptExpression doTransform(OptimizerContext context,
                                       OptExpression input,
                                       ParseNode parseNode) {
-        if (context.getOptimizerConfig().isRuleDisable(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE)) {
+        if (context.getOptimizerOptions().isRuleDisable(RuleType.TF_MV_TEXT_MATCH_REWRITE_RULE)) {
             return null;
         }
         SessionVariable sessionVariable = connectContext.getSessionVariable();
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java
index 9865ba2522b1b9..7baebceb1fb86a 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/ApplyRuleTask.java
@@ -73,7 +73,6 @@ public void execute() {
         final Stopwatch ruleStopWatch = optimizerContext.getStopwatch(rule.type());
         final Binder binder = new Binder(optimizerContext, pattern, groupExpression, ruleStopWatch);
         final List<OptExpression> newExpressions = Lists.newArrayList();
-        final List<OptExpression> extractExpressions = Lists.newArrayList();
         OptExpression extractExpr = binder.next();
         while (extractExpr != null) {
             // Check if the rule has exhausted or not to avoid optimization time exceeding the limit.:
@@ -88,7 +87,6 @@ public void execute() {
                 extractExpr = binder.next();
                 continue;
             }
-            extractExpressions.add(extractExpr);
             List<OptExpression> targetExpressions;
             OptimizerTraceUtil.logApplyRuleBefore(context.getOptimizerContext(), rule, extractExpr);
             try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, rule.getClass().getSimpleName())) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java
index c3f65210918b42..1d554b6ce780f7 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/OptimizerTask.java
@@ -16,7 +16,7 @@
 package com.starrocks.sql.optimizer.task;
 
 import com.starrocks.sql.optimizer.GroupExpression;
-import com.starrocks.sql.optimizer.OptimizerConfig;
+import com.starrocks.sql.optimizer.OptimizerOptions;
 import com.starrocks.sql.optimizer.rule.Rule;
 
 import java.util.List;
@@ -47,7 +47,7 @@ public void pushTask(OptimizerTask task) {
     void filterInValidRules(GroupExpression groupExpression,
                             List<Rule> candidateRules,
                             List<Rule> validRules) {
-        OptimizerConfig optimizerConfig = context.getOptimizerContext().getOptimizerConfig();
+        OptimizerOptions optimizerOptions = context.getOptimizerContext().getOptimizerOptions();
         for (Rule rule : candidateRules) {
             if (groupExpression.hasRuleExplored(rule)) {
                 continue;
@@ -55,7 +55,7 @@ void filterInValidRules(GroupExpression groupExpression,
             if (!rule.getPattern().matchWithoutChild(groupExpression)) {
                 continue;
             }
-            if (optimizerConfig.isRuleDisable(rule.type())) {
+            if (optimizerOptions.isRuleDisable(rule.type())) {
                 continue;
             }
             if (rule.exhausted(context.getOptimizerContext())) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java
index 17380d201a7ff9..4cdf198750f15f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/RewriteTreeTask.java
@@ -79,7 +79,7 @@ protected void rewrite(OptExpression parent, int childIndex, OptExpression root)
 
     protected OptExpression applyRules(OptExpression parent, int childIndex, OptExpression root, List<Rule> rules) {
         for (Rule rule : rules) {
-            if (context.getOptimizerContext().getOptimizerConfig().isRuleDisable(rule.type())) {
+            if (context.getOptimizerContext().getOptimizerOptions().isRuleDisable(rule.type())) {
                 continue;
             }
             if (rule.exhausted(context.getOptimizerContext())) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java
index 9eff255ab4e1c0..3f98c8f796da63 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskContext.java
@@ -18,10 +18,6 @@
 import com.starrocks.sql.optimizer.OptimizerContext;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
-import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator;
-
-import java.util.Collections;
-import java.util.List;
 
 // The context for optimizer task
 public class TaskContext {
@@ -29,7 +25,6 @@ public class TaskContext {
     private final PhysicalPropertySet requiredProperty;
     private ColumnRefSet requiredColumns;
     private double upperBoundCost;
-    private List<PhysicalOlapScanOperator> allPhysicalOlapScanOperators;
 
     public TaskContext(OptimizerContext context,
                        PhysicalPropertySet physicalPropertySet,
@@ -39,7 +34,6 @@ public TaskContext(OptimizerContext context,
         this.requiredProperty = physicalPropertySet;
         this.requiredColumns = requiredColumns;
         this.upperBoundCost = cost;
-        this.allPhysicalOlapScanOperators = Collections.emptyList();
     }
 
     public OptimizerContext getOptimizerContext() {
@@ -65,12 +59,4 @@ public void setRequiredColumns(ColumnRefSet requiredColumns) {
     public void setUpperBoundCost(double upperBoundCost) {
         this.upperBoundCost = upperBoundCost;
     }
-
-    public void setAllPhysicalOlapScanOperators(List<PhysicalOlapScanOperator> allScanOperators) {
-        this.allPhysicalOlapScanOperators = allScanOperators;
-    }
-
-    public List<PhysicalOlapScanOperator> getAllPhysicalOlapScanOperators() {
-        return allPhysicalOlapScanOperators;
-    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java
index a5eb700b9648ab..1bbf01d98f5c83 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/task/TaskScheduler.java
@@ -16,10 +16,6 @@
 
 import com.starrocks.common.profile.Timer;
 import com.starrocks.common.profile.Tracers;
-import com.starrocks.sql.common.ErrorType;
-import com.starrocks.sql.common.StarRocksPlannerException;
-import com.starrocks.sql.optimizer.Group;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.rule.Rule;
 
@@ -39,29 +35,8 @@ public static TaskScheduler create() {
     }
 
     public void executeTasks(TaskContext context) {
-        long timeout = context.getOptimizerContext().getSessionVariable().getOptimizerExecuteTimeout();
         while (!tasks.empty()) {
-            long watch = context.getOptimizerContext().optimizerElapsedMs();
-            if (timeout > 0 && watch > timeout) {
-                // Should have at least one valid plan
-                // group will be null when in rewrite phase
-                // memo may be null for rule-based optimizer
-                Memo memo = context.getOptimizerContext().getMemo();
-                Group group = memo == null ? null : memo.getRootGroup();
-                if (group == null || !group.hasBestExpression(context.getRequiredProperty())) {
-                    throw new StarRocksPlannerException("StarRocks planner use long time " + timeout +
-                            " ms in " + (group == null ? "logical" : "memo") + " phase, This probably because " +
-                            "1. FE Full GC, " +
-                            "2. Hive external table fetch metadata took a long time, " +
-                            "3. The SQL is very complex. " +
-                            "You could " +
-                            "1. adjust FE JVM config, " +
-                            "2. try query again, " +
-                            "3. enlarge new_planner_optimize_timeout session variable",
-                            ErrorType.INTERNAL_ERROR);
-                }
-                break;
-            }
+            context.getOptimizerContext().checkTimeout();
             OptimizerTask task = tasks.pop();
             context.getOptimizerContext().setTaskContext(context);
             try (Timer ignore = Tracers.watchScope(Tracers.Module.OPTIMIZER, task.getClass().getSimpleName())) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java
index 8d141a99e9bb18..ba06194d0f2fd4 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java
@@ -24,6 +24,7 @@
 import com.starrocks.sql.optimizer.MaterializationContext;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.rule.transformation.materialization.MaterializedViewRewriter;
 import com.starrocks.sql.optimizer.task.TaskContext;
 import org.apache.commons.collections.CollectionUtils;
@@ -36,10 +37,15 @@
 import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.collectMaterializedViews;
 
 public class MVRewriteValidator {
-    private static final MVRewriteValidator INSTANCE = new MVRewriteValidator();
-
-    public static MVRewriteValidator getInstance() {
-        return INSTANCE;
+    //    private static final MVRewriteValidator INSTANCE = new MVRewriteValidator();
+    //
+    //    public static MVRewriteValidator getInstance() {
+    //        return INSTANCE;
+    //    }
+    private List<LogicalOlapScanOperator> allLogicalOlapScanOperators;
+
+    public MVRewriteValidator(List<LogicalOlapScanOperator> allLogicalOlapScanOperators) {
+        this.allLogicalOlapScanOperators = allLogicalOlapScanOperators;
     }
 
     private static boolean isUpdateMaterializedViewMetrics(ConnectContext connectContext) {
@@ -79,7 +85,7 @@ public void auditMv(ConnectContext connectContext, OptExpression physicalPlan, T
         List<MaterializedView> mvs = collectMaterializedViews(physicalPlan);
         // To avoid queries that query the materialized view directly, only consider materialized views
         // that are not used in rewriting before.
-        Set<Long> beforeTableIds = optimizerContext.getAllLogicalOlapScanOperators().stream()
+        Set<Long> beforeTableIds = allLogicalOlapScanOperators.stream()
                 .map(op -> op.getTable().getId())
                 .collect(Collectors.toSet());
         if (CollectionUtils.isNotEmpty(mvs)) {
@@ -130,7 +136,7 @@ public void validateMV(ConnectContext connectContext, OptExpression physicalPlan
         }
 
         List<MaterializedView> mvs = collectMaterializedViews(physicalPlan);
-        Set<Long> beforeTableIds = taskContext.getOptimizerContext().getAllLogicalOlapScanOperators().stream()
+        Set<Long> beforeTableIds = allLogicalOlapScanOperators.stream()
                 .map(op -> op.getTable().getId())
                 .collect(Collectors.toSet());
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java b/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java
index cbdc81af3710dc..e32f1b3a12703d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java
@@ -67,6 +67,8 @@ public class ExecPlan {
 
     private List<Integer> collectExecStatsIds;
 
+    private final boolean isShortCircuit;
+
     @VisibleForTesting
     public ExecPlan() {
         connectContext = new ConnectContext();
@@ -74,14 +76,16 @@ public ExecPlan() {
         colNames = new ArrayList<>();
         physicalPlan = null;
         outputColumns = new ArrayList<>();
+        isShortCircuit = false;
     }
 
     public ExecPlan(ConnectContext connectContext, List<String> colNames,
-                    OptExpression physicalPlan, List<ColumnRefOperator> outputColumns) {
+                    OptExpression physicalPlan, List<ColumnRefOperator> outputColumns, boolean isShortCircuit) {
         this.connectContext = connectContext;
         this.colNames = colNames;
         this.physicalPlan = physicalPlan;
         this.outputColumns = outputColumns;
+        this.isShortCircuit = isShortCircuit;
     }
 
     // for broker load plan
@@ -91,6 +95,7 @@ public ExecPlan(ConnectContext connectContext, List<PlanFragment> fragments) {
         this.physicalPlan = null;
         this.outputColumns = new ArrayList<>();
         this.fragments.addAll(fragments);
+        this.isShortCircuit = false;
     }
 
     public ConnectContext getConnectContext() {
@@ -271,4 +276,8 @@ public List<Integer> getCollectExecStatsIds() {
     public void setCollectExecStatsIds(List<Integer> collectExecStatsIds) {
         this.collectExecStatsIds = collectExecStatsIds;
     }
+
+    public boolean isShortCircuit() {
+        return isShortCircuit;
+    }
 }
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java
index 01396c8b4b0773..1ac67ab3dc730b 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/plan/PlanFragmentBuilder.java
@@ -240,15 +240,24 @@ public static ExecPlan createPhysicalPlan(OptExpression plan, ConnectContext con
                                               List<ColumnRefOperator> outputColumns, ColumnRefFactory columnRefFactory,
                                               List<String> colNames,
                                               TResultSinkType resultSinkType,
-                                              boolean hasOutputFragment) {
+                                              boolean hasOutputFragment, boolean isShortCircuit) {
         UKFKConstraintsCollector.collectColumnConstraints(plan);
-        ExecPlan execPlan = new ExecPlan(connectContext, colNames, plan, outputColumns);
+        ExecPlan execPlan = new ExecPlan(connectContext, colNames, plan, outputColumns, isShortCircuit);
         createOutputFragment(new PhysicalPlanTranslator(columnRefFactory).translate(plan, execPlan), execPlan,
                 outputColumns, hasOutputFragment);
         execPlan.setPlanCount(plan.getPlanCount());
         return finalizeFragments(execPlan, resultSinkType);
     }
 
+    public static ExecPlan createPhysicalPlan(OptExpression plan, ConnectContext connectContext,
+                                              List<ColumnRefOperator> outputColumns, ColumnRefFactory columnRefFactory,
+                                              List<String> colNames,
+                                              TResultSinkType resultSinkType,
+                                              boolean hasOutputFragment) {
+        return createPhysicalPlan(plan, connectContext, outputColumns, columnRefFactory, colNames, resultSinkType,
+                hasOutputFragment, false);
+    }
+
     public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext,
                                                    CreateMaterializedViewStatement createStmt,
                                                    OptExpression optExpr,
@@ -257,7 +266,7 @@ public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext,
                                                    ColumnRefFactory columnRefFactory) throws DdlException {
         List<String> colNames = queryRelation.getColumnOutputNames();
         List<ColumnRefOperator> outputColumns = logicalPlan.getOutputColumn();
-        ExecPlan execPlan = new ExecPlan(connectContext, colNames, optExpr, outputColumns);
+        ExecPlan execPlan = new ExecPlan(connectContext, colNames, optExpr, outputColumns, false);
         PlanFragment planFragment = new PhysicalPlanTranslator(columnRefFactory).translate(optExpr, execPlan);
         // createOutputFragment(planFragment, execPlan, outputColumns, false);
         execPlan.setPlanCount(optExpr.getPlanCount());
@@ -288,7 +297,7 @@ public static ExecPlan createPhysicalPlanForMV(ConnectContext connectContext,
         return execPlan;
     }
 
-    public static TupleDescriptor buildTupleDesc(ExecPlan execPlan, Table table) {
+    private static TupleDescriptor buildTupleDesc(ExecPlan execPlan, Table table) {
         DescriptorTable descriptorTable = execPlan.getDescTbl();
         TupleDescriptor olapTuple = descriptorTable.createTupleDescriptor();
         for (Column column : table.getFullSchema()) {
@@ -332,7 +341,7 @@ private static void createOutputFragment(PlanFragment inputFragment, ExecPlan ex
                 && !inputFragment.hashLocalBucketShuffleRightOrFullJoin(inputFragment.getPlanRoot())
                 && execPlan.getScanNodes().stream().allMatch(d -> d instanceof OlapScanNode)
                 && (execPlan.getScanNodes().stream().map(d -> ((OlapScanNode) d).getScanTabletIds().size())
-                .reduce(Integer::sum).orElse(2) <= 1)) || execPlan.getPhysicalPlan().getShortCircuit()) {
+                .reduce(Integer::sum).orElse(2) <= 1)) || execPlan.isShortCircuit()) {
             inputFragment.setOutputExprs(outputExprs);
             return;
         }
@@ -656,7 +665,7 @@ public PlanFragment visitPhysicalProject(OptExpression optExpr, ExecPlan context
 
             projectNode.setLimit(inputFragment.getPlanRoot().getLimit());
             inputFragment.setPlanRoot(projectNode);
-            inputFragment.setShortCircuit(optExpr.getShortCircuit());
+            inputFragment.setShortCircuit(context.isShortCircuit());
             return inputFragment;
         }
 
@@ -959,11 +968,11 @@ public PlanFragment visitPhysicalOlapScan(OptExpression optExpr, ExecPlan contex
                     new PlanFragment(context.getNextFragmentId(), scanNode, DataPartition.RANDOM);
             fragment.setQueryGlobalDicts(node.getGlobalDicts());
             fragment.setQueryGlobalDictExprs(getGlobalDictsExprs(node.getGlobalDictsExpr(), context));
-            fragment.setShortCircuit(optExpr.getShortCircuit());
+            fragment.setShortCircuit(context.isShortCircuit());
             context.getFragments().add(fragment);
 
             // set row store literal
-            if (optExpr.getShortCircuit()) {
+            if (context.isShortCircuit()) {
                 scanNode.computePointScanRangeLocations();
             }
             return fragment;
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java
index 27f698e464c6df..203d3713965f26 100644
--- a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveMetadataTest.java
@@ -51,8 +51,8 @@
 import com.starrocks.sql.analyzer.AstToStringBuilder;
 import com.starrocks.sql.ast.CreateTableStmt;
 import com.starrocks.sql.ast.DropTableStmt;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.statistics.ColumnStatistic;
@@ -133,7 +133,7 @@ public void setUp() throws Exception {
         // create connect context
         connectContext = UtFrameUtils.createDefaultCtx();
         columnRefFactory = new ColumnRefFactory();
-        optimizerContext = new OptimizerContext(new Memo(), columnRefFactory, connectContext);
+        optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory);
         hiveMetadata = new HiveMetadata("hive_catalog", new HdfsEnvironment(), hmsOps, fileOps, statisticsProvider,
                 Optional.empty(), executorForHmsRefresh, executorForHmsRefresh,
                 new ConnectorProperties(ConnectorType.HIVE));
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java
index b761a36cef2544..cf3dd8d90ea846 100644
--- a/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/hive/HiveStatisticsProviderTest.java
@@ -26,8 +26,8 @@
 import com.starrocks.connector.PartitionUtil;
 import com.starrocks.connector.RemoteFileOperations;
 import com.starrocks.qe.ConnectContext;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
 import com.starrocks.sql.optimizer.statistics.ColumnStatistic;
@@ -98,7 +98,7 @@ public void setUp() throws Exception {
         // create connect context
         connectContext = UtFrameUtils.createDefaultCtx();
         columnRefFactory = new ColumnRefFactory();
-        optimizerContext = new OptimizerContext(new Memo(), columnRefFactory, connectContext);
+        optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory);
     }
 
     @After
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java
index 49358b31f711d9..d2e5c692523495 100644
--- a/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/iceberg/IcebergMetadataTest.java
@@ -79,8 +79,8 @@
 import com.starrocks.sql.ast.ModifyColumnClause;
 import com.starrocks.sql.ast.ModifyTablePropertiesClause;
 import com.starrocks.sql.ast.TableRenameClause;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator;
 import com.starrocks.sql.optimizer.operator.scalar.CallOperator;
@@ -794,7 +794,7 @@ public void testGetTableStatistics() {
         ColumnRefOperator columnRefOperator2 = new ColumnRefOperator(4, Type.STRING, "data", true);
         colRefToColumnMetaMap.put(columnRefOperator1, new Column("id", Type.INT));
         colRefToColumnMetaMap.put(columnRefOperator2, new Column("data", Type.STRING));
-        OptimizerContext context = new OptimizerContext(new Memo(), new ColumnRefFactory());
+        OptimizerContext context = OptimizerFactory.mockContext(new ColumnRefFactory());
         Assert.assertFalse(context.getSessionVariable().enableIcebergColumnStatistics());
         Assert.assertTrue(context.getSessionVariable().enableReadIcebergPuffinNdv());
         TableVersionRange versionRange = TableVersionRange.withEnd(Optional.of(
@@ -830,7 +830,7 @@ public void testGetTableStatisticsWithColumnStats() {
         TableVersionRange versionRange = TableVersionRange.withEnd(Optional.of(
                 mockedNativeTableB.currentSnapshot().snapshotId()));
         Statistics statistics = metadata.getTableStatistics(
-                new OptimizerContext(null, null, ConnectContext.get()),
+                OptimizerFactory.mockContext(ConnectContext.get(), null),
                 icebergTable, colRefToColumnMetaMap, null, null, -1, versionRange);
         Assert.assertEquals(4.0, statistics.getOutputRowCount(), 0.001);
         Assert.assertEquals(2, statistics.getColumnStatistics().size());
@@ -940,7 +940,7 @@ public void testGetRepeatedTableStats() {
         mockedNativeTableA.refresh();
 
         new ConnectContext().setThreadLocalInfo();
-        OptimizerContext context = new OptimizerContext(new Memo(), new ColumnRefFactory(), ConnectContext.get());
+        OptimizerContext context = OptimizerFactory.mockContext(ConnectContext.get(), new ColumnRefFactory());
         context.getSessionVariable().setEnableIcebergColumnStatistics(true);
 
         TableVersionRange version = TableVersionRange.withEnd(Optional.of(
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java
index 57372c09a3b3cf..a8863686b76a93 100644
--- a/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonMetadataTest.java
@@ -29,9 +29,8 @@
 import com.starrocks.credential.CloudConfiguration;
 import com.starrocks.credential.CloudType;
 import com.starrocks.server.MetadataMgr;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.logical.LogicalPaimonScanOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator;
@@ -443,7 +442,7 @@ public long getTableCreateTime(String dbName, String tblName) {
         OptExpression scan =
                 new OptExpression(new LogicalPaimonScanOperator(paimonTable, colRefToColumnMetaMap, columnMetaToColRefMap,
                         -1, null));
-        rule0.transform(scan, new OptimizerContext(new Memo(), new ColumnRefFactory()));
+        rule0.transform(scan, OptimizerFactory.mockContext(new ColumnRefFactory()));
         assertEquals(1, ((LogicalPaimonScanOperator) scan.getOp()).getScanOperatorPredicates()
                 .getSelectedPartitionIds().size());
     }
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java
index 3917eadd17aa10..8df67e9e1fcff0 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/OptimizerTaskTest.java
@@ -39,7 +39,6 @@
 import com.starrocks.sql.optimizer.base.DistributionSpec;
 import com.starrocks.sql.optimizer.base.HashDistributionDesc;
 import com.starrocks.sql.optimizer.base.Ordering;
-import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
 import com.starrocks.sql.optimizer.dump.MockDumpInfo;
 import com.starrocks.sql.optimizer.operator.AggType;
 import com.starrocks.sql.optimizer.operator.Operator;
@@ -169,9 +168,8 @@ public void testTaskScheduler(@Mocked OlapTable olapTable1,
             }
         };
 
-        Optimizer optimizer = new Optimizer();
-        optimizer.optimize(ctx, logicOperatorTree, new PhysicalPropertySet(), new ColumnRefSet(),
-                columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        optimizer.optimize(logicOperatorTree, new ColumnRefSet());
         Memo memo = optimizer.getContext().getMemo();
         assertEquals(3, memo.getGroups().size());
         assertEquals(8, memo.getGroupExpressions().size());
@@ -257,9 +255,8 @@ public void testTwoJoin(@Mocked OlapTable olapTable1,
             }
         };
 
-        Optimizer optimizer = new Optimizer();
-        optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(),
-                columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        optimizer.optimize(topJoin, new ColumnRefSet());
 
         Memo memo = optimizer.getContext().getMemo();
 
@@ -379,9 +376,8 @@ public void testThreeJoin(@Mocked OlapTable olapTable1,
                                 scan4ColumnMap, Maps.newHashMap(), null,
                                 -1, null)));
 
-        Optimizer optimizer = new Optimizer();
-        optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(),
-                columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        optimizer.optimize(topJoin, new ColumnRefSet());
     }
 
     @Test
@@ -475,9 +471,8 @@ public void testFourJoin(@Mocked OlapTable olapTable1,
             }
         };
 
-        Optimizer optimizer = new Optimizer();
-        optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(),
-                columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        optimizer.optimize(topJoin, new ColumnRefSet());
     }
 
     @Test
@@ -622,9 +617,8 @@ public void testSevenJoin(@Mocked OlapTable olapTable1,
             }
         };
 
-        Optimizer optimizer = new Optimizer();
-        optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(), new ColumnRefSet(),
-                columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        optimizer.optimize(topJoin, new ColumnRefSet());
     }
 
     @Test
@@ -670,8 +664,9 @@ public void testDeriveOutputColumns(@Mocked OlapTable olapTable1,
         outputColumns.addAll(outputColumns2);
         ColumnRefSet outputColumnsSet = new ColumnRefSet(outputColumns);
 
-        Optimizer optimizer = new Optimizer();
-        optimizer.optimize(ctx, logicOperatorTree, new PhysicalPropertySet(), outputColumnsSet, columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        optimizer.optimize(logicOperatorTree, outputColumnsSet);
+
         Memo memo = optimizer.getContext().getMemo();
 
         MemoStatusChecker checker = new MemoStatusChecker(memo, 2, outputColumnsSet);
@@ -715,10 +710,10 @@ public void testExtractBestPlanForThreeTable(@Mocked OlapTable olapTable1,
                         new LogicalOlapScanOperator(olapTable3, scanColumnMap,
                                 Maps.newHashMap(), null, -1, null)));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(),
-                new ColumnRefSet(Lists.newArrayList(column1)),
-                columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(topJoin,
+                new ColumnRefSet(List.of(column1)));
+
         assertEquals(physicalTree.getOp().getOpType(), OperatorType.PHYSICAL_NESTLOOP_JOIN);
         assertEquals(physicalTree.inputAt(0).getOp().getOpType(), OperatorType.PHYSICAL_NESTLOOP_JOIN);
         assertEquals(physicalTree.inputAt(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION);
@@ -750,9 +745,9 @@ public void testTopDownRewrite(@Mocked OlapTable olapTable1) {
             }
         };
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(), new ColumnRefSet(),
-                columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet());
+
         Operator root = physicalTree.getOp();
         assertEquals(root.getOpType(), OperatorType.PHYSICAL_LIMIT);
     }
@@ -783,9 +778,9 @@ public void testPruneOlapScanColumnsRule(@Mocked OlapTable olapTable1) {
                         new LogicalOlapScanOperator(olapTable1, scanColumnMap, Maps.newHashMap(), null,
                                 -1, null)));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns1), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression,
+                new ColumnRefSet(outputColumns1));
 
         assertEquals(OperatorType.PHYSICAL_OLAP_SCAN, physicalTree.getOp().getOpType());
         PhysicalOlapScanOperator physicalOlapScan = (PhysicalOlapScanOperator) physicalTree.getOp();
@@ -829,9 +824,10 @@ public void testPruneOlapScanColumnsRuleWithConstant(@Mocked OlapTable olapTable
             }
         };
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns1), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression,
+                new ColumnRefSet(outputColumns1));
 
         assertNotNull(physicalTree.getOp().getProjection());
         assertEquals(physicalTree.getOp().getOpType(), OperatorType.PHYSICAL_OLAP_SCAN);
@@ -901,9 +897,9 @@ public void testPruneAggregateColumnsRule(@Mocked OlapTable olapTable1) {
 
         ColumnRefSet outputColumns = new ColumnRefSet(column2.getId());
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(expression, outputColumns);
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 2: t2 in " +
@@ -917,8 +913,10 @@ public void testPruneAggregateColumnsRule(@Mocked OlapTable olapTable1) {
             tmp.getSessionVariable().setJoinImplementationMode("auto");
             tmp.getSessionVariable().setEnablePlanValidation(false);
             tmp.setDumpInfo(new MockDumpInfo());
-            OptExpression expression1 = optimizer.optimize(tmp, expression, new PhysicalPropertySet(), outputColumns,
-                    columnRefFactory);
+
+            Optimizer optimizer = OptimizerFactory.create(
+                    OptimizerFactory.mockContext(ctx, columnRefFactory));
+            OptExpression expression1 = optimizer.optimize(expression, outputColumns);
             Map<ColumnRefOperator, CallOperator> aggs =
                     ((PhysicalHashAggregateOperator) expression1.getOp()).getAggregations();
             assertEquals(Type.INT, column2.getType());
@@ -994,9 +992,10 @@ public void testPruneCountStarRule(@Mocked OlapTable olapTable1) {
 
         ColumnRefSet outputColumns = new ColumnRefSet(column5.getId());
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(expression, outputColumns);
+
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 5: t5 in " +
@@ -1047,9 +1046,11 @@ public void testPruneAggregateConstantRule(@Mocked OlapTable olapTable1) {
             }
         };
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(
+                    OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(expression, outputColumns);
+
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " +
@@ -1104,9 +1105,11 @@ public void testMergeAggregateWithLimitRule(@Mocked OlapTable olapTable1) {
             }
         };
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(
+                    OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(expression, outputColumns);
+
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " +
@@ -1154,9 +1157,9 @@ public void testPruneSortColumnsRule(@Mocked OlapTable olapTable1,
 
         ColumnRefSet outputColumns = new ColumnRefSet(column2.getId());
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                outputColumns, columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, outputColumns);
 
         assertEquals(physicalTree.getLogicalProperty().getOutputColumns(), new ColumnRefSet(column2.getId()));
 
@@ -1209,9 +1212,9 @@ public void testSplitAggregateRule(@Mocked OlapTable olapTable1) {
 
         ColumnRefSet outputColumns = new ColumnRefSet(column1.getId());
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                outputColumns, columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, outputColumns);
 
         Operator operator = physicalTree.getOp();
         assertEquals(operator.getOpType(), OperatorType.PHYSICAL_HASH_AGG);
@@ -1281,9 +1284,11 @@ public void testSplitAggregateRuleNoGroupBy(@Mocked OlapTable olapTable1) {
             }
         };
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(
+                    OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(expression, outputColumns);
+
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " +
@@ -1342,9 +1347,10 @@ public void testSplitAggregateRuleWithDistinctAndGroupBy(@Mocked OlapTable olapT
 
         ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column3, column2));
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(
+                    OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(expression, outputColumns);
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " +
@@ -1403,9 +1409,10 @@ public void testSplitAggregateRuleWithOnlyOneDistinct(@Mocked OlapTable olapTabl
 
         ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column3));
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, expression, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(expression, outputColumns);
+
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 3: t3 in " +
@@ -1458,9 +1465,8 @@ public void testSplitAggregateRuleWithProject(@Mocked OlapTable olapTable1) {
 
         ColumnRefSet outputColumns = new ColumnRefSet(column2.getId());
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, topProject, new PhysicalPropertySet(),
-                outputColumns, columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(topProject, outputColumns);
 
         Operator operator = physicalTree.getOp();
         assertEquals(operator.getOpType(), OperatorType.PHYSICAL_HASH_AGG);
@@ -1524,9 +1530,10 @@ public void testFilterPushDownWithHaving(@Mocked OlapTable olapTable1) {
 
         ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column6));
 
-        Optimizer optimizer = new Optimizer();
         try {
-            optimizer.optimize(ctx, root, new PhysicalPropertySet(), outputColumns, columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+            optimizer.optimize(root, outputColumns);
+
             fail("invalid plan. can not optimize success");
         } catch (Exception e) {
             Assert.assertTrue(e.getMessage(), e.getMessage().contains("Type check failed. the type of arg 5: t5 in " +
@@ -1583,9 +1590,9 @@ public void testFilterPushDownWithHaving2(@Mocked OlapTable olapTable1) {
 
         ColumnRefSet outputColumns = new ColumnRefSet(Lists.newArrayList(column4));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, root, new PhysicalPropertySet(),
-                outputColumns, columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(root, outputColumns);
 
         Operator operator = physicalTree.getOp();
         assertEquals(operator.getOpType(), OperatorType.PHYSICAL_HASH_AGG);
@@ -1636,9 +1643,8 @@ public void testFilterPushDownRule(@Mocked OlapTable olapTable1) {
                                         null, -1,
                                         null))));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns));
 
         assertNotNull(physicalTree.getOp().getProjection());
         assertEquals(physicalTree.getOp().getOpType(), OperatorType.PHYSICAL_OLAP_SCAN);
@@ -1685,9 +1691,8 @@ public void testFilterPushDownRuleWithMultiProjects(@Mocked OlapTable olapTable1
 
         OptExpression filter = OptExpression.create(new LogicalFilterOperator(predicate), project2);
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, filter, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(filter, new ColumnRefSet(outputColumns));
 
         assertNotNull(physicalTree.getOp().getProjection());
         Projection pp = physicalTree.getOp().getProjection();
@@ -1748,9 +1753,9 @@ public void testCommonOperatorReuseRule(@Mocked OlapTable olapTable1) {
                         new LogicalOlapScanOperator(olapTable1, scanColumnMap, Maps.newHashMap(), null, -1,
                                 null)));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns));
 
         PhysicalOlapScanOperator olapScanOperator = (PhysicalOlapScanOperator) physicalTree.getOp();
         Projection projection = olapScanOperator.getProjection();
@@ -1834,9 +1839,8 @@ public void testShuffleTwoJoin(@Mocked OlapTable olapTable1,
                 OptExpression.create(scan1),
                 OptExpression.create(scan2));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns));
     }
 
     @Test
@@ -1932,9 +1936,8 @@ public void testShuffleThreeJoin(@Mocked OlapTable olapTable1,
                 OptExpression.create(scan2));
         OptExpression topJoin = OptExpression.create(join2,
                 join1, OptExpression.create(scan3));
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, topJoin, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(topJoin, new ColumnRefSet(outputColumns));
     }
 
     @Test
@@ -2031,9 +2034,10 @@ public void testBroadcastExceedRowLimitWithHugeGapInRowCount(@Mocked OlapTable o
                 OptExpression.create(scan1),
                 OptExpression.create(scan2));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns));
+
         assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION);
         PhysicalDistributionOperator rightOperator =
                 (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp();
@@ -2152,9 +2156,10 @@ public void testBroadcastExceedRowLimitWithoutHugeGapInRowCount(@Mocked OlapTabl
                 OptExpression.create(scan1),
                 OptExpression.create(scan2));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns));
+
         assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION);
         PhysicalDistributionOperator rightOperator =
                 (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp();
@@ -2277,9 +2282,9 @@ public void testOlapTablePartitionRowCount(@Mocked OlapTable olapTable1,
                 OptExpression.create(scan1),
                 OptExpression.create(scan2));
 
-        Optimizer optimizer = new Optimizer();
-        OptExpression physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        OptExpression physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns));
         assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION);
         PhysicalDistributionOperator rightOperator =
                 (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp();
@@ -2334,7 +2339,6 @@ public void testOlapTablePartitionRowCount(@Mocked OlapTable olapTable1,
             }
         };
 
-        optimizer = new Optimizer();
         expression = OptExpression.create(new LogicalJoinOperator(JoinOperator.INNER_JOIN, predicate),
                 OptExpression.create(new LogicalOlapScanOperator(olapTable1, scan1ColumnMap, Maps.newHashMap(),
                         DistributionSpec.createHashDistributionSpec(
@@ -2344,8 +2348,10 @@ public void testOlapTablePartitionRowCount(@Mocked OlapTable olapTable1,
                         DistributionSpec.createHashDistributionSpec(
                                 new HashDistributionDesc(Lists.newArrayList(this.column4.getId()),
                                         HashDistributionDesc.SourceType.LOCAL)), -1, null)));
-        physicalTree = optimizer.optimize(ctx, expression, new PhysicalPropertySet(),
-                new ColumnRefSet(outputColumns), columnRefFactory);
+        optimizer = OptimizerFactory.create(
+                OptimizerFactory.mockContext(ctx, columnRefFactory));
+        physicalTree = optimizer.optimize(expression, new ColumnRefSet(outputColumns));
+
         assertEquals(physicalTree.getInputs().get(1).getOp().getOpType(), OperatorType.PHYSICAL_DISTRIBUTION);
         rightOperator = (PhysicalDistributionOperator) physicalTree.getInputs().get(1).getOp();
         assertEquals(rightOperator.getDistributionSpec().getType(), DistributionSpec.DistributionType.BROADCAST);
@@ -2395,7 +2401,7 @@ public void testSplitAggregateRuleConstantColumns(@Mocked OlapTable olapTable1)
 
         SplitTwoPhaseAggRule splitTwoPhaseAggRule = SplitTwoPhaseAggRule.getInstance();
         List<OptExpression> list = splitTwoPhaseAggRule.transform(
-                expression, new OptimizerContext(new Memo(), new ColumnRefFactory()));
+                expression, OptimizerFactory.mockContext(new ColumnRefFactory()));
 
         assertEquals(OperatorType.LOGICAL_AGGR, list.get(0).getOp().getOpType());
         LogicalAggregationOperator result = (LogicalAggregationOperator) list.get(0).getOp();
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java
index 2eff6fa0ee1c8c..f31946f16dee4d 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/BinderTest.java
@@ -22,6 +22,7 @@
 import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.OperatorType;
 import com.starrocks.sql.optimizer.operator.logical.LogicalJoinOperator;
@@ -37,21 +38,21 @@ public class BinderTest {
 
     private Binder buildBinder(Pattern pattern, OptExpression expr) {
         Memo memo = new Memo();
-        OptimizerContext optimizerContext = new OptimizerContext(memo, new ColumnRefFactory());
+        OptimizerContext optimizerContext = OptimizerFactory.mockContext(new ColumnRefFactory());
         Stopwatch stopwatch = Stopwatch.createStarted();
         return new Binder(optimizerContext, pattern, memo.init(expr), stopwatch);
     }
 
     private Binder buildBinder(Pattern pattern, GroupExpression qe) {
         Memo memo = new Memo();
-        OptimizerContext optimizerContext = new OptimizerContext(memo, new ColumnRefFactory());
+        OptimizerContext optimizerContext = OptimizerFactory.mockContext(new ColumnRefFactory());
         Stopwatch stopwatch = Stopwatch.createStarted();
         return new Binder(optimizerContext, pattern, qe, stopwatch);
     }
 
     private OptExpression bindNext(Pattern pattern, OptExpression expr) {
         Memo memo = new Memo();
-        OptimizerContext optimizerContext = new OptimizerContext(memo, new ColumnRefFactory());
+        OptimizerContext optimizerContext = OptimizerFactory.mockContext(new ColumnRefFactory());
         Stopwatch stopwatch = Stopwatch.createStarted();
         Binder binder = new Binder(optimizerContext, pattern, memo.init(expr), stopwatch);
         return binder.next();
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java
index ea7e8126092b56..0878c80271f013 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/implementation/OlapScanImplementationRuleTest.java
@@ -17,9 +17,8 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.starrocks.catalog.OlapTable;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
 import com.starrocks.sql.optimizer.operator.physical.PhysicalOlapScanOperator;
@@ -41,8 +40,8 @@ public void transform(@Mocked OlapTable table) {
                 false, Lists.newArrayList(4L), null, null, false);
 
         List<OptExpression> output =
-                new OlapScanImplementationRule().transform(new OptExpression(logical), new OptimizerContext(
-                        new Memo(), new ColumnRefFactory()));
+                new OlapScanImplementationRule().transform(new OptExpression(logical),
+                        OptimizerFactory.mockContext(new ColumnRefFactory()));
 
         assertEquals(1, output.size());
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java
index c7f286f4c2a183..57944bd89d6252 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/DistributionPrunerRuleTest.java
@@ -32,9 +32,8 @@
 import com.starrocks.catalog.ScalarType;
 import com.starrocks.catalog.Type;
 import com.starrocks.planner.PartitionColumnFilter;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.Utils;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
@@ -197,7 +196,7 @@ public void transform(@Mocked OlapTable olapTable, @Mocked Partition partition,
 
         assertEquals(0, operator.getSelectedTabletId().size());
         OptExpression optExpression =
-                rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), new ColumnRefFactory()))
+                rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(new ColumnRefFactory()))
                         .get(0);
 
         assertEquals(20, ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedTabletId().size());
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java
index 2f19bf394d95aa..93070f493c0c70 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/MergeLimitWithSortRuleTest.java
@@ -17,9 +17,8 @@
 
 import com.google.common.collect.Lists;
 import com.starrocks.catalog.Type;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.Ordering;
 import com.starrocks.sql.optimizer.operator.OperatorType;
@@ -43,7 +42,7 @@ public void transform() {
         limit.getInputs().add(sort);
 
         MergeLimitWithSortRule rule = new MergeLimitWithSortRule();
-        List<OptExpression> list = rule.transform(limit, new OptimizerContext(new Memo(), new ColumnRefFactory()));
+        List<OptExpression> list = rule.transform(limit, OptimizerFactory.mockContext(new ColumnRefFactory()));
 
         assertEquals(OperatorType.LOGICAL_TOPN, list.get(0).getOp().getOpType());
         assertEquals(2, ((LogicalTopNOperator) list.get(0).getOp()).getOffset());
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java
index fe6cf641e18ec9..d28b3228c702fe 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PartitionPruneRuleTest.java
@@ -37,9 +37,8 @@
 import com.starrocks.pseudocluster.PseudoCluster;
 import com.starrocks.sql.ast.PartitionNames;
 import com.starrocks.sql.ast.PartitionValue;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.Utils;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
@@ -161,7 +160,7 @@ public void transform1(@Mocked OlapTable olapTable, @Mocked RangePartitionInfo p
 
         assertNull(operator.getSelectedPartitionId());
         OptExpression optExpression =
-                rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0);
+                rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0);
 
         assertEquals(3, ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId().size());
     }
@@ -282,7 +281,7 @@ public void transform2(@Mocked OlapTable olapTable, @Mocked RangePartitionInfo p
 
         assertNull(operator.getSelectedPartitionId());
         OptExpression optExpression =
-                rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0);
+                rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0);
 
         assertEquals(3, ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId().size());
     }
@@ -363,7 +362,7 @@ public void transformForSingleItemListPartition(@Mocked OlapTable olapTable,
         PartitionPruneRule rule = new PartitionPruneRule();
         assertNull(operator.getSelectedPartitionId());
         OptExpression optExpression =
-                rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0);
+                rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0);
 
         List<Long> selectPartitionIds = ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId();
         assertEquals(1, selectPartitionIds.size());
@@ -454,7 +453,7 @@ public void transformForSingleItemListPartitionWithTemp(@Mocked OlapTable olapTa
         PartitionPruneRule rule = new PartitionPruneRule();
         assertNull(operator.getSelectedPartitionId());
         OptExpression optExpression =
-                rule.transform(new OptExpression(operator), new OptimizerContext(new Memo(), columnRefFactory)).get(0);
+                rule.transform(new OptExpression(operator), OptimizerFactory.mockContext(columnRefFactory)).get(0);
 
         List<Long> selectPartitionIds = ((LogicalOlapScanOperator) optExpression.getOp()).getSelectedPartitionId();
         assertEquals(1, selectPartitionIds.size());
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java
index 753a41d8cedd45..3371d38e731c39 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownAggRuleTest.java
@@ -20,9 +20,8 @@
 import com.starrocks.analysis.BinaryType;
 import com.starrocks.catalog.Column;
 import com.starrocks.catalog.Type;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.AggType;
 import com.starrocks.sql.optimizer.operator.OperatorType;
@@ -77,7 +76,7 @@ public void transform(@Mocked LogicalOlapScanOperator scanOp) {
 
         PushDownPredicateAggRule rule = new PushDownPredicateAggRule();
 
-        List<OptExpression> list = rule.transform(filter, new OptimizerContext(new Memo(), new ColumnRefFactory()));
+        List<OptExpression> list = rule.transform(filter, OptimizerFactory.mockContext(new ColumnRefFactory()));
 
         assertEquals(OperatorType.LOGICAL_AGGR, list.get(0).getOp().getOpType());
         assertEquals(OperatorType.COMPOUND,
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java
index c56e0ca90bddcc..a2b34453f9e9a9 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/PushDownScanRuleTest.java
@@ -19,9 +19,8 @@
 import com.starrocks.analysis.BinaryType;
 import com.starrocks.catalog.OlapTable;
 import com.starrocks.catalog.Type;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.Operator;
 import com.starrocks.sql.optimizer.operator.OperatorType;
@@ -57,7 +56,7 @@ public void transform(@Mocked OlapTable table) {
 
         assertNull(((LogicalOlapScanOperator) scan.getOp()).getPredicate());
         List<OptExpression> result =
-                rule.transform(optExpression, new OptimizerContext(new Memo(), new ColumnRefFactory()));
+                rule.transform(optExpression, OptimizerFactory.mockContext(new ColumnRefFactory()));
 
         Operator scanOperator = result.get(0).inputAt(0).getOp();
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java
index 27b9fcf259cc2c..0e0f8f1ade69d0 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MVTestBase.java
@@ -51,6 +51,7 @@
 import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.QueryMaterializationContext;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
@@ -228,13 +229,11 @@ public static OptExpression getOptimizedPlan(String sql, ConnectContext connectC
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
         LogicalPlan logicalPlan =
                 new RelationTransformer(columnRefFactory, connectContext).transformWithSelectLimit(query);
-        Optimizer optimizer = new Optimizer();
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory));
         return optimizer.optimize(
-                connectContext,
                 logicalPlan.getRoot(),
                 new PhysicalPropertySet(),
-                new ColumnRefSet(logicalPlan.getOutputColumn()),
-                columnRefFactory);
+                new ColumnRefSet(logicalPlan.getOutputColumn()));
     }
 
     public List<PhysicalScanOperator> getScanOperators(OptExpression root, String name) {
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java
index 9c87dd4f7fbc0e..d03edbc4e45b40 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java
@@ -26,12 +26,12 @@
 import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder;
 import com.starrocks.sql.optimizer.MaterializationContext;
 import com.starrocks.sql.optimizer.MaterializedViewOptimizer;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.MvRewritePreprocessor;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
-import com.starrocks.sql.optimizer.OptimizerConfig;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
+import com.starrocks.sql.optimizer.OptimizerOptions;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
@@ -92,21 +92,6 @@ public List<OptExpression> transform(OptExpression input, OptimizerContext conte
 
     @Test
     public void testOptimizer() throws Exception {
-        Optimizer optimizer = new Optimizer();
-        Assert.assertFalse(optimizer.getOptimizerConfig().isRuleBased());
-        Assert.assertFalse(optimizer.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT));
-        Assert.assertFalse(optimizer.getOptimizerConfig().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE));
-
-        OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED);
-        optimizerConfig.disableRule(RuleType.TF_MERGE_TWO_PROJECT);
-        optimizerConfig.disableRule(RuleType.GP_PUSH_DOWN_PREDICATE);
-        Optimizer optimizer1 = new Optimizer(optimizerConfig);
-        Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleBased());
-        Assert.assertFalse(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE));
-        Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT));
-        Assert.assertFalse(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.GP_COLLECT_CTE));
-        Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE));
-
         String sql = "select v1, sum(v3) from t0 where v1 < 10 group by v1";
         Pair<String, ExecPlan> result = UtFrameUtils.getPlanAndFragment(connectContext, sql);
         Assert.assertNotNull(result);
@@ -118,13 +103,32 @@ public void testOptimizer() throws Exception {
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
         LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext)
                 .transformWithSelectLimit(query.getQueryRelation());
-        OptExpression expr = optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(),
-                new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+
+        OptimizerContext optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(optimizerContext);
+        Assert.assertFalse(optimizerContext.getOptimizerOptions().isRuleBased());
+        Assert.assertFalse(optimizerContext.getOptimizerOptions().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT));
+        Assert.assertFalse(optimizerContext.getOptimizerOptions().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE));
+
+        OptExpression expr = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(),
+                new ColumnRefSet(logicalPlan.getOutputColumn()));
         Assert.assertTrue(expr.getInputs().get(0).getOp() instanceof PhysicalOlapScanOperator);
         Assert.assertNotNull(expr.getInputs().get(0).getOp().getPredicate());
 
-        OptExpression expr1 = optimizer1.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(),
-                new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+        OptimizerOptions optimizerOptions = new OptimizerOptions(OptimizerOptions.OptimizerStrategy.RULE_BASED);
+        optimizerOptions.disableRule(RuleType.TF_MERGE_TWO_PROJECT);
+        optimizerOptions.disableRule(RuleType.GP_PUSH_DOWN_PREDICATE);
+        OptimizerContext optimizerContext1 = OptimizerFactory.mockContext(connectContext, columnRefFactory,
+                optimizerOptions);
+        Optimizer optimizer1 = OptimizerFactory.create(optimizerContext1);
+        Assert.assertTrue(optimizerContext1.getOptimizerOptions().isRuleBased());
+        Assert.assertFalse(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE));
+        Assert.assertTrue(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT));
+        Assert.assertFalse(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.GP_COLLECT_CTE));
+        Assert.assertTrue(optimizerContext1.getOptimizerOptions().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE));
+
+        OptExpression expr1 = optimizer1.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(),
+                new ColumnRefSet(logicalPlan.getOutputColumn()));
         Assert.assertTrue(expr1.getInputs().get(0).getOp() instanceof LogicalFilterOperator);
 
         // test timeout
@@ -163,9 +167,9 @@ public void testPreprocessMvNonPartitionMv() throws Exception {
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
         LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext)
                 .transformWithSelectLimit(query.getQueryRelation());
-        Optimizer optimizer = new Optimizer();
-        OptExpression expr = optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(),
-                new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+        Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory));
+        OptExpression expr = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(),
+                new ColumnRefSet(logicalPlan.getOutputColumn()));
         Assert.assertNotNull(expr);
         Assert.assertEquals(2, optimizer.getContext().getCandidateMvs().size());
 
@@ -208,9 +212,10 @@ public void testPreprocessMvPartitionMv() throws Exception {
             ColumnRefFactory columnRefFactory = new ColumnRefFactory();
             LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext)
                     .transformWithSelectLimit(query.getQueryRelation());
-            Optimizer optimizer = new Optimizer();
-            OptExpression expr = optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(),
-                    new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+            Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext,
+                    columnRefFactory));
+            OptExpression expr = optimizer.optimize(logicalPlan.getRoot(),
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
             Assert.assertNotNull(expr);
             Assert.assertEquals(1, optimizer.getContext().getCandidateMvs().size());
             MaterializationContext materializationContext = optimizer.getContext().getCandidateMvs().iterator().next();
@@ -222,9 +227,10 @@ public void testPreprocessMvPartitionMv() throws Exception {
 
             refreshMaterializedView("test", "mv_4");
             executeInsertSql(connectContext, "insert into tbl_with_mv partition(p2) values(\"2020-02-20\", 20, 30)");
-            Optimizer optimizer2 = new Optimizer();
-            OptExpression expr2 = optimizer2.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(),
-                    new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+            Optimizer optimizer2 = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext,
+                    columnRefFactory));
+            OptExpression expr2 = optimizer2.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(),
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
             Assert.assertNotNull(expr2);
             MaterializationContext materializationContext2 =
                     optimizer2.getContext().getCandidateMvs().iterator().next();
@@ -245,12 +251,13 @@ public void testPreprocessMvPartitionMv() throws Exception {
             StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
             QueryStatement query = (QueryStatement) stmt;
 
-            Optimizer optimizer3 = new Optimizer();
             ColumnRefFactory columnRefFactory = new ColumnRefFactory();
             LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext)
                     .transformWithSelectLimit(query.getQueryRelation());
-            OptExpression expr3 = optimizer3.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(),
-                    new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+            Optimizer optimizer3 =
+                    OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory));
+            OptExpression expr3 = optimizer3.optimize(logicalPlan.getRoot(),
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
             Assert.assertNotNull(expr3);
             MaterializationContext materializationContext3 =
                     optimizer3.getContext().getCandidateMvs().iterator().next();
@@ -264,8 +271,8 @@ public void testPreprocessMvPartitionMv() throws Exception {
 
     private Pair<MvRewritePreprocessor, OptExpression> buildMvProcessor(String query) {
         ColumnRefFactory columnRefFactory = new ColumnRefFactory();
-        OptimizerConfig optimizerConfig = new OptimizerConfig();
-        OptimizerContext context = new OptimizerContext(new Memo(), columnRefFactory, connectContext, optimizerConfig);
+        OptimizerOptions optimizerOptions = new OptimizerOptions();
+        OptimizerContext context = OptimizerFactory.mockContext(connectContext, columnRefFactory, optimizerOptions);
 
         try {
             StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(query, connectContext);
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java
index f688c8ed147ac5..4980a17247fa3a 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteStrategyTest.java
@@ -18,6 +18,8 @@
 import com.starrocks.sql.ast.StatementBase;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
+import com.starrocks.sql.optimizer.OptimizerFactory;
+import com.starrocks.sql.optimizer.QueryOptimizer;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
@@ -40,11 +42,11 @@ private OptExpression optimize(Optimizer optimizer, String sql) {
         try {
             StatementBase stmt = UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
             QueryStatement queryStatement = (QueryStatement) stmt;
-            ColumnRefFactory columnRefFactory = new ColumnRefFactory();
-            LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext)
+            LogicalPlan logicalPlan =
+                    new RelationTransformer(optimizer.getContext().getColumnRefFactory(), connectContext)
                     .transformWithSelectLimit(queryStatement.getQueryRelation());
-            return optimizer.optimize(connectContext, logicalPlan.getRoot(), new PhysicalPropertySet(),
-                    new ColumnRefSet(logicalPlan.getOutputColumn()), columnRefFactory);
+            return optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(),
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
         } catch (Exception e) {
             Assert.fail(e.getMessage());
             return null;
@@ -59,7 +61,8 @@ public void testSingleTableRewriteStrategy() throws Exception {
                 " as" +
                 " select t1a, id_date, t1b from table_with_partition");
         String sql =  "select t1a, id_date, t1b from table_with_partition";
-        Optimizer optimizer = new Optimizer();
+        QueryOptimizer optimizer = (QueryOptimizer) OptimizerFactory.create(
+                OptimizerFactory.mockContext(connectContext, new ColumnRefFactory()));
         OptExpression optExpression = optimize(optimizer, sql);
         Assert.assertTrue(optExpression != null);
         MvRewriteStrategy mvRewriteStrategy = optimizer.getMvRewriteStrategy();
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java
index d865e757fb6abc..30b90cd2f398c8 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewriteTest.java
@@ -29,15 +29,13 @@
 import com.starrocks.common.AnalysisException;
 import com.starrocks.common.Config;
 import com.starrocks.common.FeConstants;
-import com.starrocks.qe.ConnectContext;
 import com.starrocks.qe.SessionVariable;
 import com.starrocks.qe.ShowResultSet;
 import com.starrocks.schema.MSchema;
 import com.starrocks.schema.MTable;
 import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder;
 import com.starrocks.sql.optimizer.OptExpression;
-import com.starrocks.sql.optimizer.Optimizer;
-import com.starrocks.sql.optimizer.base.ColumnRefFactory;
+import com.starrocks.sql.optimizer.QueryOptimizer;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
 import com.starrocks.sql.optimizer.base.PhysicalPropertySet;
 import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator;
@@ -1853,12 +1851,10 @@ public void testPlanCache() throws Exception {
 
             MaterializedView mv = getMv("test", "mv_with_window");
 
-            new MockUp<Optimizer>() {
-
+            new MockUp<QueryOptimizer>() {
                 @Mock
-                public OptExpression optimize(ConnectContext connectContext, OptExpression logicOperatorTree,
-                                              PhysicalPropertySet requiredProperty, ColumnRefSet requiredColumns,
-                                              ColumnRefFactory columnRefFactory) {
+                public OptExpression optimize(OptExpression logicOperatorTree, PhysicalPropertySet requiredProperty,
+                                              ColumnRefSet requiredColumns) {
                     throw new RuntimeException("optimize failed");
                 }
             };
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java
index 987cc90772f507..1107c44a379f8f 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/statistics/StatisticsCalculatorTest.java
@@ -33,8 +33,8 @@
 import com.starrocks.sql.optimizer.ExpressionContext;
 import com.starrocks.sql.optimizer.Group;
 import com.starrocks.sql.optimizer.GroupExpression;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.Utils;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
@@ -87,7 +87,7 @@ public static void beforeClass() throws Exception {
         // create connect context
         connectContext = UtFrameUtils.createDefaultCtx();
         columnRefFactory = new ColumnRefFactory();
-        optimizerContext = new OptimizerContext(new Memo(), columnRefFactory, connectContext);
+        optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory);
 
         starRocksAssert = new StarRocksAssert(connectContext);
         ConnectorPlanTestBase.mockAllCatalogs(connectContext, temp.newFolder().toURI().toString());
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java
index efb143cb6dbb0e..a30e3a277afc7e 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/PartitionPruneTest.java
@@ -19,8 +19,8 @@
 import com.starrocks.catalog.PartitionInfo;
 import com.starrocks.common.FeConstants;
 import com.starrocks.common.Pair;
-import com.starrocks.sql.optimizer.Memo;
 import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.operator.logical.LogicalScanOperator;
 import com.starrocks.sql.optimizer.operator.scalar.ScalarOperator;
@@ -251,7 +251,7 @@ private static Pair<ScalarOperator, LogicalScanOperator> buildConjunctAndScan(St
     private void testRemovePredicate(String sql, String expected) throws Exception {
         Pair<ScalarOperator, LogicalScanOperator> pair = buildConjunctAndScan(sql);
         StatisticsCalculator calculator = new StatisticsCalculator();
-        OptimizerContext context = new OptimizerContext(new Memo(), new ColumnRefFactory());
+        OptimizerContext context = OptimizerFactory.mockContext(new ColumnRefFactory());
         ScalarOperator newPredicate = calculator.removePartitionPredicate(pair.first, pair.second, context);
         Assert.assertEquals(expected, newPredicate.toString());
     }
diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java
index 3edae4e55ee879..e52726250fe6b5 100644
--- a/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java
+++ b/fe/fe-core/src/test/java/com/starrocks/utframe/UtFrameUtils.java
@@ -119,7 +119,9 @@
 import com.starrocks.sql.optimizer.LogicalPlanPrinter;
 import com.starrocks.sql.optimizer.OptExpression;
 import com.starrocks.sql.optimizer.Optimizer;
-import com.starrocks.sql.optimizer.OptimizerConfig;
+import com.starrocks.sql.optimizer.OptimizerContext;
+import com.starrocks.sql.optimizer.OptimizerFactory;
+import com.starrocks.sql.optimizer.OptimizerOptions;
 import com.starrocks.sql.optimizer.QueryMaterializationContext;
 import com.starrocks.sql.optimizer.base.ColumnRefFactory;
 import com.starrocks.sql.optimizer.base.ColumnRefSet;
@@ -847,21 +849,19 @@ public static LogicalPlan getQueryLogicalPlan(ConnectContext connectContext,
     public static OptExpression getQueryOptExpression(ConnectContext connectContext,
                                                       ColumnRefFactory columnRefFactory,
                                                       LogicalPlan logicalPlan,
-                                                      OptimizerConfig optimizerConfig) {
+                                                      OptimizerOptions optimizerOptions) {
         OptExpression optimizedPlan;
         try (Timer t = Tracers.watchScope("Optimizer")) {
             Optimizer optimizer = null;
-            if (optimizerConfig != null) {
-                optimizer = new Optimizer(optimizerConfig);
-            } else {
-                optimizer = new Optimizer();
+            OptimizerContext context = OptimizerFactory.mockContext(connectContext, columnRefFactory);
+            if (optimizerOptions != null) {
+                context.setOptimizerOptions(optimizerOptions);
             }
+            optimizer = OptimizerFactory.create(context);
             optimizedPlan = optimizer.optimize(
-                        connectContext,
                         logicalPlan.getRoot(),
                         new PhysicalPropertySet(),
-                        new ColumnRefSet(logicalPlan.getOutputColumn()),
-                        columnRefFactory);
+                    new ColumnRefSet(logicalPlan.getOutputColumn()));
         }
         return optimizedPlan;
     }
@@ -1442,9 +1442,9 @@ public static OptExpression getQueryOptExpression(ConnectContext connectContext,
             Assert.fail("Parse query failed:" + DebugUtil.getStackTrace(e));
         }
         LogicalPlan logicalPlan = UtFrameUtils.getQueryLogicalPlan(connectContext, columnRefFactory, statement);
-        OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED);
+        OptimizerOptions optimizerOptions = new OptimizerOptions(OptimizerOptions.OptimizerStrategy.RULE_BASED);
         OptExpression optExpression = UtFrameUtils.getQueryOptExpression(connectContext, columnRefFactory,
-                logicalPlan, optimizerConfig);
+                logicalPlan, optimizerOptions);
         return optExpression;
     }
 

From 0a580fd32d743cdea95374e346fc671024415439 Mon Sep 17 00:00:00 2001
From: Seaven <seaven_7@qq.com>
Date: Tue, 21 Jan 2025 13:48:01 +0800
Subject: [PATCH 59/71] [Enhancement] limit clause support user variables
 (#55257)

Signed-off-by: Seaven <seaven_7@qq.com>
---
 .../com/starrocks/analysis/LimitElement.java  |  49 +++++---
 .../sql/analyzer/SelectAnalyzer.java          |  39 ++++++-
 .../com/starrocks/sql/parser/AstBuilder.java  |  24 +++-
 .../com/starrocks/sql/parser/StarRocks.g4     |  10 +-
 .../common/proc/OptimizeProcDirTest.java      |   2 +-
 .../com/starrocks/sql/plan/LimitTest.java     | 105 ++++++++++++++++++
 6 files changed, 201 insertions(+), 28 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java b/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java
index 7b371c7fad6902..cd2c2b2d15c6fd 100644
--- a/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java
+++ b/fe/fe-core/src/main/java/com/starrocks/analysis/LimitElement.java
@@ -34,6 +34,7 @@
 
 package com.starrocks.analysis;
 
+import com.google.common.base.Preconditions;
 import com.starrocks.sql.ast.AstVisitor;
 import com.starrocks.sql.parser.NodePosition;
 
@@ -46,8 +47,8 @@ public class LimitElement implements ParseNode {
     /////////////////////////////////////////
     // BEGIN: Members that need to be reset()
 
-    private long limit;
-    private long offset;
+    private final Expr limit;
+    private final Expr offset;
 
     // END: Members that need to be reset()
     /////////////////////////////////////////
@@ -56,12 +57,8 @@ public class LimitElement implements ParseNode {
 
     public LimitElement() {
         pos = NodePosition.ZERO;
-        limit = -1;
-        offset = 0;
-    }
-
-    public LimitElement(long limit) {
-        this(0, limit, NodePosition.ZERO);
+        limit = new IntLiteral(-1);
+        offset = new IntLiteral(0);
     }
 
     public LimitElement(long offset, long limit) {
@@ -69,6 +66,12 @@ public LimitElement(long offset, long limit) {
     }
 
     public LimitElement(long offset, long limit, NodePosition pos) {
+        this.pos = pos;
+        this.offset = new IntLiteral(offset);
+        this.limit = new IntLiteral(limit);
+    }
+
+    public LimitElement(Expr offset, Expr limit, NodePosition pos) {
         this.pos = pos;
         this.offset = offset;
         this.limit = limit;
@@ -90,11 +93,12 @@ public LimitElement clone() {
      * first. If no limit was set, then -1 is returned.
      */
     public long getLimit() {
-        return limit;
+        Preconditions.checkState(limit instanceof LiteralExpr);
+        return ((LiteralExpr) limit).getLongValue();
     }
 
     public boolean hasLimit() {
-        return limit != -1;
+        return getLimit() != -1;
     }
 
     /**
@@ -102,36 +106,45 @@ public boolean hasLimit() {
      * analyze() first. If no offsetExpr exists, then 0 (the default offset) is returned.
      */
     public long getOffset() {
-        return offset;
+        Preconditions.checkState(offset instanceof LiteralExpr);
+        return ((LiteralExpr) offset).getLongValue();
     }
 
     public boolean hasOffset() {
-        return offset != 0;
+        return getOffset() != 0;
     }
 
     public String toSql() {
-        if (limit == -1) {
+        if (getLimit() == -1) {
             return "";
         }
         StringBuilder sb = new StringBuilder(" LIMIT ");
-        if (offset != 0) {
-            sb.append(offset).append(", ");
+        if (getOffset() != 0) {
+            sb.append(getOffset()).append(", ");
         }
-        sb.append("").append(limit);
+        sb.append("").append(getLimit());
         return sb.toString();
     }
 
+    public Expr getLimitExpr() {
+        return limit;
+    }
+
+    public Expr getOffsetExpr() {
+        return offset;
+    }
+
     @Override
     public NodePosition getPos() {
         return pos;
     }
 
     public String toDigest() {
-        if (limit == -1) {
+        if (getLimit() == -1) {
             return "";
         }
         StringBuilder sb = new StringBuilder(" limit ");
-        if (offset != 0) {
+        if (getOffset() != 0) {
             sb.append(" ?, ");
         }
         sb.append("").append(" ? ");
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SelectAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SelectAnalyzer.java
index 81dad06a4e1b0b..f784cc011583f0 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SelectAnalyzer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/SelectAnalyzer.java
@@ -29,6 +29,7 @@
 import com.starrocks.analysis.OrderByElement;
 import com.starrocks.analysis.ParseNode;
 import com.starrocks.analysis.SlotRef;
+import com.starrocks.analysis.UserVariableExpr;
 import com.starrocks.catalog.FunctionSet;
 import com.starrocks.catalog.PrimitiveType;
 import com.starrocks.catalog.Type;
@@ -189,9 +190,7 @@ public void analyze(AnalyzeState analyzeState,
             analyzeState.setOrderSourceExpressions(orderSourceExpressions);
         }
 
-        if (limitElement != null && limitElement.hasLimit()) {
-            analyzeState.setLimit(new LimitElement(limitElement.getOffset(), limitElement.getLimit()));
-        }
+        analyzeState.setLimit(analyzeLimit(limitElement, analyzeState, sourceScope));
     }
 
     private List<Expr> analyzeSelect(SelectList selectList, Relation fromRelation, boolean hasGroupByClause,
@@ -574,6 +573,40 @@ private void analyzeHaving(Expr havingClause, AnalyzeState analyzeState,
         }
     }
 
+    private LimitElement analyzeLimit(LimitElement limitElement, AnalyzeState analyzeState, Scope scope) {
+        if (limitElement == null) {
+            return null;
+        }
+
+        Expr limitExpr = limitElement.getLimitExpr();
+        Expr offsetExpr = limitElement.getOffsetExpr();
+        long limit;
+        long offset;
+        analyzeExpression(limitExpr, analyzeState, scope);
+        analyzeExpression(offsetExpr, analyzeState, scope);
+        if (limitExpr.isLiteral()) {
+            limit = limitElement.getLimit();
+        } else if (limitExpr instanceof UserVariableExpr &&
+                ((UserVariableExpr) limitExpr).getValue() instanceof IntLiteral) {
+            limit = ((IntLiteral) ((UserVariableExpr) limitExpr).getValue()).getLongValue();
+        } else {
+            throw new SemanticException("LIMIT clause %s must be number", limitExpr.toMySql());
+        }
+        if (limit == -1) {
+            return null;
+        }
+
+        if (offsetExpr.isLiteral()) {
+            offset = limitElement.getOffset();
+        } else if (offsetExpr instanceof UserVariableExpr &&
+                ((UserVariableExpr) offsetExpr).getValue() instanceof IntLiteral) {
+            offset = ((IntLiteral) ((UserVariableExpr) offsetExpr).getValue()).getLongValue();
+        } else {
+            throw new SemanticException("OFFSET clause %s must be number", offsetExpr.toMySql());
+        }
+        return new LimitElement(offset, limit, limitElement.getPos());
+    }
+
     // If alias is same with table column name, we directly use table name.
     // otherwise, we use output expression according to the alias
     public static class RewriteAliasVisitor implements AstVisitor<Expr, Void> {
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java
index 4bb009d6fa4cdc..a2184ba77e7d89 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java
@@ -5572,13 +5572,29 @@ private static boolean getOrderingType(Token token) {
 
     @Override
     public ParseNode visitLimitElement(StarRocksParser.LimitElementContext context) {
-        if (context.limit.getText().equals("?") || (context.offset != null && context.offset.getText().equals("?"))) {
+        if (context.limit.PARAMETER() != null || (context.offset != null && context.offset.PARAMETER() != null)) {
             throw new ParsingException("using parameter(?) as limit or offset not supported");
         }
-        long limit = Long.parseLong(context.limit.getText());
-        long offset = 0;
+
+        Expr limit;
+        Expr offset = new IntLiteral(0);
+
+        if (context.limit.INTEGER_VALUE() != null) {
+            limit = new IntLiteral(Long.parseLong(context.limit.INTEGER_VALUE().getText()));
+        } else if (context.limit.userVariable() != null) {
+            limit = (UserVariableExpr) visit(context.limit.userVariable());
+        } else {
+            throw new ParsingException("unsupported invalid limit value", createPos(context.limit));
+        }
+
         if (context.offset != null) {
-            offset = Long.parseLong(context.offset.getText());
+            if (context.offset.INTEGER_VALUE() != null) {
+                offset = new IntLiteral(Long.parseLong(context.offset.INTEGER_VALUE().getText()));
+            } else if (context.offset.userVariable() != null) {
+                offset = (UserVariableExpr) visit(context.offset.userVariable());
+            } else {
+                throw new ParsingException("unsupported invalid offset value", createPos(context.offset));
+            }
         }
         return new LimitElement(offset, limit, createPos(context));
     }
diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4
index 5670d248bba1d0..ae4c7fb30159c2 100644
--- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4
+++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4
@@ -2160,9 +2160,15 @@ sortItem
     : expression ordering = (ASC | DESC)? (NULLS nullOrdering=(FIRST | LAST))?
     ;
 
+limitConstExpr
+    : INTEGER_VALUE
+    | PARAMETER
+    | userVariable
+    ;
+
 limitElement
-    : LIMIT limit =(INTEGER_VALUE|PARAMETER) (OFFSET offset=(INTEGER_VALUE|PARAMETER))?
-    | LIMIT offset =(INTEGER_VALUE|PARAMETER) ',' limit=(INTEGER_VALUE|PARAMETER)
+    : LIMIT limit=limitConstExpr (OFFSET offset=limitConstExpr)?
+    | LIMIT offset=limitConstExpr ',' limit=limitConstExpr
     ;
 
 querySpecification
diff --git a/fe/fe-core/src/test/java/com/starrocks/common/proc/OptimizeProcDirTest.java b/fe/fe-core/src/test/java/com/starrocks/common/proc/OptimizeProcDirTest.java
index a8b777d65d0728..dd1fe33c67f47f 100644
--- a/fe/fe-core/src/test/java/com/starrocks/common/proc/OptimizeProcDirTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/common/proc/OptimizeProcDirTest.java
@@ -212,7 +212,7 @@ public void testFetchResultByFilter() throws AnalysisException {
         ArrayList<OrderByPair> orderByPairs = Lists.newArrayList();
         orderByPairs.add(new OrderByPair(0));
 
-        LimitElement limitElement = new LimitElement(1);
+        LimitElement limitElement = new LimitElement(0, 1);
 
         BaseProcResult result = (BaseProcResult) optimizeProcDir.fetchResultByFilter(
                 filter, orderByPairs, limitElement);
diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java
index 2e135ccd1f371d..d7fe5bf13a1012 100644
--- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/LimitTest.java
@@ -20,7 +20,11 @@
 import com.starrocks.catalog.Replica;
 import com.starrocks.common.FeConstants;
 import com.starrocks.qe.SessionVariable;
+import com.starrocks.qe.SetExecutor;
 import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.sql.analyzer.SemanticException;
+import com.starrocks.sql.ast.SetStmt;
+import com.starrocks.utframe.UtFrameUtils;
 import mockit.Expectations;
 import org.junit.Assert;
 import org.junit.Test;
@@ -954,4 +958,105 @@ public void testTransformGroupByToLimit() throws Exception {
                         "  2:EXCHANGE\n" +
                         "     limit: 1");
     }
+
+    @Test
+    public void testLimitUserVariable() throws Exception {
+        {
+            String sql = "set @var = 123";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            sql = "select * from t0 limit @var";
+            String plan = getFragmentPlan(sql);
+            assertContains(plan, "EXCHANGE\n" +
+                    "     limit: 123");
+        }
+        {
+            String sql = "set @var = 123";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            sql = "select * from t0 limit @var, @var";
+            String plan = getFragmentPlan(sql);
+            assertContains(plan, "offset: 123\n" +
+                    "     limit: 123");
+        }
+        {
+            String sql = "set @var = 123";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            sql = "select * from t0 limit @var OFFSET @var";
+            String plan = getFragmentPlan(sql);
+            assertContains(plan, "offset: 123\n" +
+                    "     limit: 123");
+        }
+        {
+            String sql = "set @var = 123";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            sql = "select * from t0 limit @var, 11";
+            String plan = getFragmentPlan(sql);
+            assertContains(plan, "offset: 123\n" +
+                    "     limit: 11");
+        }
+        {
+            String sql = "set @var = 123";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            sql = "select * from t0 limit 12, @var";
+            String plan = getFragmentPlan(sql);
+            assertContains(plan, "offset: 12\n" +
+                    "     limit: 123");
+        }
+        {
+            String sql = "set @var = 31 + 16";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            sql = "select * from t0 limit 12, @var";
+            String plan = getFragmentPlan(sql);
+            assertContains(plan, "offset: 12\n" +
+                    "     limit: 47");
+        }
+    }
+
+    @Test
+    public void testLimitUserVariableError() throws Exception {
+        {
+            String sql = "set @var = '123'";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            String tql = "select * from t0 limit @var";
+            Assert.assertThrows(SemanticException.class, () -> getFragmentPlan(tql));
+        }
+        {
+            String sql = "set @var = 'abc'";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            String tql = "select * from t0 limit @var";
+            Assert.assertThrows(SemanticException.class, () -> getFragmentPlan(tql));
+        }
+        {
+            String sql = "set @var = 'abc'";
+            SetStmt stmt = (SetStmt) UtFrameUtils.parseStmtWithNewParser(sql, connectContext);
+            SetExecutor executor = new SetExecutor(connectContext, stmt);
+            executor.execute();
+
+            String tql = "select * from t0 limit @var, 2";
+            Assert.assertThrows(SemanticException.class, () -> getFragmentPlan(tql));
+        }
+    }
 }

From 82165777c7ec755f9947695e0dbb644e18c77ccf Mon Sep 17 00:00:00 2001
From: duanyyyyyyy <139062392+duanyyyyyyy@users.noreply.github.com>
Date: Tue, 21 Jan 2025 14:29:08 +0800
Subject: [PATCH 60/71] [BugFix] Fix a bug that when catalog is unified catalog
 the background refresh for hive connector do not work (#55215)

Signed-off-by: duanyyyyyyy <yan.duan9759@gmail.com>
---
 .../connector/delta/DeltaLakeConnector.java   |  7 +++-
 .../connector/hive/CatalogNameType.java       | 41 +++++++++++++++++++
 .../hive/ConnectorTableMetadataProcessor.java | 25 ++++++-----
 .../connector/hive/HiveConnector.java         |  6 ++-
 .../connector/hudi/HudiConnector.java         |  5 ++-
 5 files changed, 69 insertions(+), 15 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/connector/hive/CatalogNameType.java

diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java
index 07f9dac068043a..226d3ca54b0821 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java
@@ -19,6 +19,7 @@
 import com.starrocks.connector.ConnectorContext;
 import com.starrocks.connector.ConnectorMetadata;
 import com.starrocks.connector.HdfsEnvironment;
+import com.starrocks.connector.hive.CatalogNameType;
 import com.starrocks.credential.CloudConfiguration;
 import com.starrocks.credential.CloudConfigurationFactory;
 import com.starrocks.server.GlobalStateMgr;
@@ -36,12 +37,14 @@ public class DeltaLakeConnector implements Connector {
     private final Map<String, String> properties;
     private final CloudConfiguration cloudConfiguration;
     private final String catalogName;
+    private final CatalogNameType catalogNameType;
     private final DeltaLakeInternalMgr internalMgr;
     private final DeltaLakeMetadataFactory metadataFactory;
     private IDeltaLakeMetastore metastore;
 
     public DeltaLakeConnector(ConnectorContext context) {
         this.catalogName = context.getCatalogName();
+        this.catalogNameType = new CatalogNameType(catalogName, "delta_lake");
         this.properties = context.getProperties();
         this.cloudConfiguration = CloudConfigurationFactory.buildCloudConfigurationForStorage(properties);
         HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(cloudConfiguration);
@@ -75,13 +78,13 @@ public CloudConfiguration getCloudConfiguration() {
     public void shutdown() {
         internalMgr.shutdown();
         metadataFactory.metastoreCacheInvalidateCache();
-        GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor().unRegisterCacheUpdateProcessor(catalogName);
+        GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor().unRegisterCacheUpdateProcessor(catalogNameType);
     }
 
     public void onCreate() {
         Optional<DeltaLakeCacheUpdateProcessor> updateProcessor = metadataFactory.getCacheUpdateProcessor();
         updateProcessor.ifPresent(processor -> GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor()
-                        .registerCacheUpdateProcessor(catalogName, updateProcessor.get()));
+                        .registerCacheUpdateProcessor(catalogNameType, updateProcessor.get()));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/CatalogNameType.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/CatalogNameType.java
new file mode 100644
index 00000000000000..6d4742475e0382
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/hive/CatalogNameType.java
@@ -0,0 +1,41 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.connector.hive;
+
+/**
+ * This class will be used for register background refresh in `ConnectorTableMetadataProcessor`.
+ * As the unified catalog feature is implemented, we can not use the catalog name as the key
+ * in the `cacheUpdateProcessors` map of the `ConnectorTableMetadataProcessor`.
+ * So here we introduce this class and use it as the key for that map.
+ */
+public class CatalogNameType {
+
+    private final String catalogName;
+    private final String catalogType;
+
+    public CatalogNameType(String catalogName, String catalogType) {
+        this.catalogName = catalogName;
+        this.catalogType = catalogType;
+    }
+
+    public String getCatalogName() {
+        return this.catalogName;
+    }
+
+    public String getCatalogType() {
+        return this.catalogType;
+    }
+
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java
index 868c26674aae5b..647a7c945ce780 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/hive/ConnectorTableMetadataProcessor.java
@@ -48,7 +48,8 @@ public class ConnectorTableMetadataProcessor extends FrontendDaemon {
 
     private final Set<BaseTableInfo> registeredTableInfos = Sets.newConcurrentHashSet();
 
-    private final Map<String, CacheUpdateProcessor> cacheUpdateProcessors = new ConcurrentHashMap<>();
+    private final Map<CatalogNameType, CacheUpdateProcessor> cacheUpdateProcessors =
+            new ConcurrentHashMap<>();
 
     private final ExecutorService refreshRemoteFileExecutor;
     private final Map<String, IcebergCatalog> cachingIcebergCatalogs = new ConcurrentHashMap<>();
@@ -57,14 +58,16 @@ public void registerTableInfo(BaseTableInfo tableInfo) {
         registeredTableInfos.add(tableInfo);
     }
 
-    public void registerCacheUpdateProcessor(String catalogName, CacheUpdateProcessor cache) {
-        LOG.info("register to update {} metadata cache in the ConnectorTableMetadataProcessor", catalogName);
-        cacheUpdateProcessors.put(catalogName, cache);
+    public void registerCacheUpdateProcessor(CatalogNameType catalogNameType, CacheUpdateProcessor cache) {
+        LOG.info("register to update {}:{} metadata cache in the ConnectorTableMetadataProcessor",
+                catalogNameType.getCatalogName(), catalogNameType.getCatalogType());
+        cacheUpdateProcessors.put(catalogNameType, cache);
     }
 
-    public void unRegisterCacheUpdateProcessor(String catalogName) {
-        LOG.info("unregister to update {} metadata cache in the ConnectorTableMetadataProcessor", catalogName);
-        cacheUpdateProcessors.remove(catalogName);
+    public void unRegisterCacheUpdateProcessor(CatalogNameType catalogNameType) {
+        LOG.info("unregister to update {}:{} metadata cache in the ConnectorTableMetadataProcessor",
+                catalogNameType.getCatalogName(), catalogNameType.getCatalogType());
+        cacheUpdateProcessors.remove(catalogNameType);
     }
 
     public void registerCachingIcebergCatalog(String catalogName, IcebergCatalog icebergCatalog) {
@@ -99,9 +102,11 @@ protected void runAfterCatalogReady() {
 
     private void refreshCatalogTable() {
         MetadataMgr metadataMgr = GlobalStateMgr.getCurrentState().getMetadataMgr();
-        List<String> catalogNames = Lists.newArrayList(cacheUpdateProcessors.keySet());
-        for (String catalogName : catalogNames) {
-            CacheUpdateProcessor updateProcessor = cacheUpdateProcessors.get(catalogName);
+        List<CatalogNameType> catalogNameTypes = Lists.newArrayList(cacheUpdateProcessors.keySet());
+        for (CatalogNameType catalogNameType : catalogNameTypes) {
+            String catalogName = catalogNameType.getCatalogName();
+            LOG.info("Starting to refresh tables from {}:{} metadata cache", catalogName, catalogNameType.getCatalogType());
+            CacheUpdateProcessor updateProcessor = cacheUpdateProcessors.get(catalogNameType);
             if (updateProcessor == null) {
                 LOG.error("Failed to get cacheUpdateProcessor by catalog {}.", catalogName);
                 continue;
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveConnector.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveConnector.java
index 01b6fdf14e6d8f..9d067f3cae3851 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveConnector.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveConnector.java
@@ -34,12 +34,14 @@ public class HiveConnector implements Connector {
     public static final String HIVE_METASTORE_CONNECTION_POOL_SIZE = "hive.metastore.connection.pool.size";
     private final Map<String, String> properties;
     private final String catalogName;
+    private final CatalogNameType catalogNameType;
     private final HiveConnectorInternalMgr internalMgr;
     private final HiveMetadataFactory metadataFactory;
 
     public HiveConnector(ConnectorContext context) {
         this.properties = context.getProperties();
         this.catalogName = context.getCatalogName();
+        this.catalogNameType = new CatalogNameType(catalogName, "hive");
         CloudConfiguration cloudConfiguration = CloudConfigurationFactory.buildCloudConfigurationForStorage(properties);
         HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(cloudConfiguration);
         this.internalMgr = new HiveConnectorInternalMgr(catalogName, properties, hdfsEnvironment);
@@ -83,7 +85,7 @@ public void onCreate() {
                     internalMgr.isEnableBackgroundRefreshHiveMetadata()) {
                 updateProcessor
                         .ifPresent(processor -> GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor()
-                                .registerCacheUpdateProcessor(catalogName, updateProcessor.get()));
+                                .registerCacheUpdateProcessor(catalogNameType, updateProcessor.get()));
             }
         }
     }
@@ -93,6 +95,6 @@ public void shutdown() {
         internalMgr.shutdown();
         metadataFactory.getCacheUpdateProcessor().ifPresent(HiveCacheUpdateProcessor::invalidateAll);
         GlobalStateMgr.getCurrentState().getMetastoreEventsProcessor().unRegisterCacheUpdateProcessor(catalogName);
-        GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor().unRegisterCacheUpdateProcessor(catalogName);
+        GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor().unRegisterCacheUpdateProcessor(catalogNameType);
     }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hudi/HudiConnector.java b/fe/fe-core/src/main/java/com/starrocks/connector/hudi/HudiConnector.java
index da45e9baac6462..135744e9c61f16 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/hudi/HudiConnector.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/hudi/HudiConnector.java
@@ -20,6 +20,7 @@
 import com.starrocks.connector.ConnectorMetadata;
 import com.starrocks.connector.HdfsEnvironment;
 import com.starrocks.connector.RemoteFileIO;
+import com.starrocks.connector.hive.CatalogNameType;
 import com.starrocks.connector.hive.IHiveMetastore;
 import com.starrocks.credential.CloudConfiguration;
 import com.starrocks.credential.CloudConfigurationFactory;
@@ -33,6 +34,7 @@ public class HudiConnector implements Connector {
     public static final List<String> SUPPORTED_METASTORE_TYPE = Lists.newArrayList("hive", "glue", "dlf");
     private final Map<String, String> properties;
     private final String catalogName;
+    private final CatalogNameType catalogNameType;
     private final HudiConnectorInternalMgr internalMgr;
     private final HudiMetadataFactory metadataFactory;
 
@@ -41,6 +43,7 @@ public HudiConnector(ConnectorContext context) {
         CloudConfiguration cloudConfiguration = CloudConfigurationFactory.buildCloudConfigurationForStorage(properties);
         HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(cloudConfiguration);
         this.catalogName = context.getCatalogName();
+        this.catalogNameType = new CatalogNameType(catalogName, "hudi");
         this.internalMgr = new HudiConnectorInternalMgr(catalogName, properties, hdfsEnvironment);
         this.metadataFactory = createMetadataFactory(hdfsEnvironment);
         onCreate();
@@ -74,6 +77,6 @@ public void onCreate() {
     @Override
     public void shutdown() {
         internalMgr.shutdown();
-        GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor().unRegisterCacheUpdateProcessor(catalogName);
+        GlobalStateMgr.getCurrentState().getConnectorTableMetadataProcessor().unRegisterCacheUpdateProcessor(catalogNameType);
     }
 }
\ No newline at end of file

From b924bf631f62a72179390eb0c366f4f4deaf7f1b Mon Sep 17 00:00:00 2001
From: Youngwb <yangwenbo_mailbox@163.com>
Date: Tue, 21 Jan 2025 15:54:42 +0800
Subject: [PATCH 61/71] [Enhancement] upgrade iceberg version to 1.7.1 (#55271)

Signed-off-by: Youngwb <yangwenbo_mailbox@163.com>
---
 fe/fe-core/pom.xml      | 2 +-
 java-extensions/pom.xml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index bf702654516fbc..0dfd2b87167712 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -41,7 +41,7 @@ under the License.
         <starrocks.home>${basedir}/../../</starrocks.home>
         <fe_ut_parallel>${env.FE_UT_PARALLEL}</fe_ut_parallel>
         <jacoco.version>0.8.8</jacoco.version>
-        <iceberg.version>1.6.0</iceberg.version>
+        <iceberg.version>1.7.1</iceberg.version>
         <paimon.version>0.8.2</paimon.version>
         <delta-kernel.version>4.0.0rc1</delta-kernel.version>
         <staros.version>3.4-rc3</staros.version>
diff --git a/java-extensions/pom.xml b/java-extensions/pom.xml
index a67b9cf580351b..e0f4e059d266b9 100644
--- a/java-extensions/pom.xml
+++ b/java-extensions/pom.xml
@@ -35,7 +35,7 @@
         <java-extensions.home>${basedir}</java-extensions.home>
         <aws-v2-sdk.version>2.23.19</aws-v2-sdk.version>
         <hadoop.version>3.4.0</hadoop.version>
-        <iceberg.version>1.6.0</iceberg.version>
+        <iceberg.version>1.7.1</iceberg.version>
         <log4j2.version>2.23.1</log4j2.version>
         <junit.version>5.10.3</junit.version>
         <hive-apache.version>3.1.2-22</hive-apache.version>

From 1773164e7665bde31f7981e75f3bc35d8e2f0890 Mon Sep 17 00:00:00 2001
From: Murphy <96611012+murphyatwork@users.noreply.github.com>
Date: Tue, 21 Jan 2025 19:56:30 +0800
Subject: [PATCH 62/71] [UT] fix GlobalStateMgrTest.testReloadTables (#55284)

Signed-off-by: Murphy <mofei@starrocks.com>
---
 .../com/starrocks/server/GlobalStateMgrTest.java   | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java
index a616b1ba437cf9..9287c128295b69 100644
--- a/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/server/GlobalStateMgrTest.java
@@ -69,6 +69,7 @@
 import org.mockito.Mockito;
 
 import java.lang.reflect.Field;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
@@ -343,10 +344,15 @@ public void testReloadTables() throws Exception {
         Path targetPath = Path.of(Config.meta_dir, GlobalStateMgr.IMAGE_DIR, "/v2",
                 Path.of(imagePath).getFileName().toString());
         Files.move(Path.of(imagePath), targetPath);
-        // move checksum file
-        Path checksumPath = Path.of(Config.meta_dir, "checksum.0");
-        Path checksumTarget = Path.of(Config.meta_dir, GlobalStateMgr.IMAGE_DIR, "/v2", "checksum.0");
-        Files.move(checksumPath, checksumTarget);
+        // Move all checksum files instead of a single file
+        Path checksumDir = Path.of(Config.meta_dir);
+        Path checksumTargetDir = Path.of(Config.meta_dir, GlobalStateMgr.IMAGE_DIR, "/v2");
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(checksumDir, "checksum.*")) {
+            for (Path file : stream) {
+                Path target = checksumTargetDir.resolve(file.getFileName());
+                Files.move(file, target);
+            }
+        }
 
         GlobalStateMgr newState = new MyGlobalStateMgr(false);
         newState.loadImage(Config.meta_dir + GlobalStateMgr.IMAGE_DIR);

From 95f95158a9544776d97074f2fce0023f4a93af8e Mon Sep 17 00:00:00 2001
From: RyanZ <dirtysalt1987@gmail.com>
Date: Tue, 21 Jan 2025 04:10:48 -0800
Subject: [PATCH 63/71] [BugFix] fix resource leak when doing checkpoint
 (#55270)

---
 .../com/starrocks/connector/ConnectorMetadata.java   |  3 +++
 .../java/com/starrocks/connector/ConnectorMgr.java   |  6 ++++++
 .../com/starrocks/connector/jdbc/JDBCConnector.java  |  7 +++++++
 .../com/starrocks/connector/jdbc/JDBCMetadata.java   |  7 +++++++
 .../java/com/starrocks/server/GlobalStateMgr.java    | 12 +++++++++++-
 5 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java b/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java
index eb23fcaba3fd4a..3216407d19f0aa 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMetadata.java
@@ -344,5 +344,8 @@ default Set<DeleteFile> getDeleteFiles(IcebergTable icebergTable, Long snapshotI
                                            ScalarOperator predicate, FileContent fileContent) {
         throw new StarRocksConnectorException("This connector doesn't support getting delete files");
     }
+
+    default void shutdown() {
+    }
 }
 
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMgr.java b/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMgr.java
index 793bdc8421a6e8..f7003f4dfb5e79 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/ConnectorMgr.java
@@ -133,4 +133,10 @@ public Map<String, MemoryTrackable> getMemTrackers() {
         }
         return memoryTrackers;
     }
+
+    public void shutdown() {
+        for (CatalogConnector cc : connectors.values()) {
+            cc.shutdown();
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCConnector.java b/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCConnector.java
index 9914fd1dfd6a09..a38871591ab661 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCConnector.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCConnector.java
@@ -103,4 +103,11 @@ public ConnectorMetadata getMetadata() {
         }
         return metadata;
     }
+
+    @Override
+    public void shutdown() {
+        if (metadata != null) {
+            metadata.shutdown();
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCMetadata.java b/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCMetadata.java
index 5ab4db92fd19ca..cd83327de5b6ca 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCMetadata.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/jdbc/JDBCMetadata.java
@@ -302,4 +302,11 @@ public void refreshTable(String srDbName, Table table, List<String> partitionNam
     public void refreshCache(Map<String, String> properties) {
         createMetaAsyncCacheInstances(properties);
     }
+
+    @Override
+    public void shutdown() {
+        if (dataSource != null) {
+            dataSource.close();
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
index bdfa794dd53774..d83906d159e229 100644
--- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
@@ -834,6 +834,11 @@ public void transferToNonLeader(FrontendNodeType newType) {
 
     public static void destroyCheckpoint() {
         if (CHECKPOINT != null) {
+            try {
+                CHECKPOINT.shutdown();
+            } catch (Exception e) {
+                LOG.warn("exception when destroy checkpoint", e);
+            }
             CHECKPOINT = null;
         }
     }
@@ -1357,7 +1362,7 @@ private void startLeaderOnlyDaemonThreads() {
         checkpointController.start();
 
         clusterSnapshotCheckpointScheduler = new ClusterSnapshotCheckpointScheduler(checkpointController,
-                                                  StarMgrServer.getCurrentState().getCheckpointController());
+                StarMgrServer.getCurrentState().getCheckpointController());
         clusterSnapshotCheckpointScheduler.start();
 
         keyRotationDaemon.start();
@@ -2708,4 +2713,9 @@ public VariableMgr getVariableMgr() {
     public WarehouseIdleChecker getWarehouseIdleChecker() {
         return warehouseIdleChecker;
     }
+
+    public void shutdown() {
+        // in a single thread.
+        connectorMgr.shutdown();
+    }
 }

From cd512cc3b71f62293506f2cb850695d7b9bcf068 Mon Sep 17 00:00:00 2001
From: kaijianding <kaijian.ding@gmail.com>
Date: Tue, 21 Jan 2025 20:49:14 +0800
Subject: [PATCH 64/71] [Enhancement] move time-consuming
 listMaterializedViewStatus out of db lock scope (#55281)

Signed-off-by: kaijian.ding <kaijian.ding@gmail.com>
---
 .../src/main/java/com/starrocks/qe/ShowExecutor.java | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
index a121f085a93667..3d43e379efe389 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
@@ -374,18 +374,18 @@ public ShowResultSet visitShowMaterializedViewStatement(ShowMaterializedViewsStm
                         }
                     }
                 }
-
-                List<ShowMaterializedViewStatus> mvStatusList =
-                        listMaterializedViewStatus(dbName, materializedViews, singleTableMVs);
-                List<List<String>> rowSets = mvStatusList.stream().map(ShowMaterializedViewStatus::toResultSet)
-                        .collect(Collectors.toList());
-                return new ShowResultSet(statement.getMetaData(), rowSets);
             } catch (Exception e) {
                 LOG.warn("listMaterializedViews failed:", e);
                 throw e;
             } finally {
                 locker.unLockDatabase(db.getId(), LockType.READ);
             }
+
+            List<ShowMaterializedViewStatus> mvStatusList =
+                    listMaterializedViewStatus(dbName, materializedViews, singleTableMVs);
+            List<List<String>> rowSets = mvStatusList.stream().map(ShowMaterializedViewStatus::toResultSet)
+                    .collect(Collectors.toList());
+            return new ShowResultSet(statement.getMetaData(), rowSets);
         }
 
         @Override

From 84b1b4a05b3c819aac3732eabb5ca51064201695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=B5=B5=E7=A9=BA=E4=BA=8B=E3=82=B9=E3=83=94=E3=83=AA?=
 =?UTF-8?q?=E3=83=83=E3=83=88?= <wanglichen@starrocks.com>
Date: Wed, 22 Jan 2025 09:51:21 +0800
Subject: [PATCH 65/71] [Doc] Fix links (#55298)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 絵空事スピリット <wanglichen@starrocks.com>
---
 docs/zh/release_notes/release-3.4.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/zh/release_notes/release-3.4.md b/docs/zh/release_notes/release-3.4.md
index df4f74f6d7e906..78b405d2d642a2 100644
--- a/docs/zh/release_notes/release-3.4.md
+++ b/docs/zh/release_notes/release-3.4.md
@@ -50,11 +50,11 @@ displayed_sidebar: docs
 
 ### 数据导入
 
-- INSERT OVERWRITE 新增 Dynamic Overwrite 语义，启用后，系统将根据导入的数据自动创建分区或覆盖对应的现有分区，导入不涉及的分区不会被清空或删除，适用于恢复特定分区数据的场景。更多内容，参考 []()。
+- INSERT OVERWRITE 新增 Dynamic Overwrite 语义，启用后，系统将根据导入的数据自动创建分区或覆盖对应的现有分区，导入不涉及的分区不会被清空或删除，适用于恢复特定分区数据的场景。更多内容，参考 [INSERT](https://docs.starrocks.io/zh/docs/sql-reference/sql-statements/loading_unloading/INSERT/)。
 - 优化了 INSERT from FILES 导入，使其可以基本取代 Broker Load 成为首选导入方式： 
   - FILES 支持 LIST 远程存储中的文件，并提供文件的基本统计信息。更多内容，参考 [FILES - list_files_only](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#list_files_only)。
   - INSERT 支持按名称匹配列，特别适用于导入列很多且列名相同的数据。（默认按位置匹配列。）更多内容，参考 [INSERT 按名称匹配列](https://docs.starrocks.io/zh/docs/loading/InsertInto/#insert-%E6%8C%89%E5%90%8D%E7%A7%B0%E5%8C%B9%E9%85%8D%E5%88%97)。
-  - INSERT 支持指定 PROPERTIES，与其他导入方式保持一致。用户可通过指定 `strict_mode`、`max_filter_ratio` 和 `timeout` 来控制数据导入的行为和质量。更多内容，参考 [](https://docs.starrocks.io/zh/docs/sql-reference/sql-statements/loading_unloading/INSERT/#properties)。
+  - INSERT 支持指定 PROPERTIES，与其他导入方式保持一致。用户可通过指定 `strict_mode`、`max_filter_ratio` 和 `timeout` 来控制数据导入的行为和质量。更多内容，参考 [INSERT](https://docs.starrocks.io/zh/docs/sql-reference/sql-statements/loading_unloading/INSERT/)。
   - INSERT from FILES 支持将目标表的 Schema 检查下推到 FILES 的扫描阶段，从而更准确地推断源数据 Schema。更多内容，参考 [Target Table Schema 检查下推](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#target-table-schema-%E6%A3%80%E6%9F%A5%E4%B8%8B%E6%8E%A8)。
   - FILES 支持合并不同 Schema 的文件。Parquet 和 ORC 文件基于列名合并，CSV 文件基于列位置（顺序）合并。对于不匹配的列，用户可通过指定 `fill_mismatch_column_with` 属性选择填充 NULL 值或报错。更多内容，参考 [合并具有不同 Schema 的文件](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#%E5%90%88%E5%B9%B6%E5%85%B7%E6%9C%89%E4%B8%8D%E5%90%8C-schema-%E7%9A%84%E6%96%87%E4%BB%B6)。
   - FILES 支持从 Parquet 文件推断 STRUCT 类型数据。（在早期版本中，STRUCT 数据被推断为 STRING 类型。）更多内容，参考 [推断 Parquet 文件中的 STRUCT 类型](https://docs.starrocks.io/zh/docs/sql-reference/sql-functions/table-functions/files/#%E6%8E%A8%E6%96%AD-parquet-%E6%96%87%E4%BB%B6%E4%B8%AD%E7%9A%84-struct-%E7%B1%BB%E5%9E%8B)。

From 428db44f47004d0dd1bd43d086bc01902d937cca Mon Sep 17 00:00:00 2001
From: zombee0 <ewang2027@gmail.com>
Date: Wed, 22 Jan 2025 10:10:43 +0800
Subject: [PATCH 66/71] [Tool]bench filter performance (#48341)

Signed-off-by: zombee0 <ewang2027@gmail.com>
---
 be/src/bench/CMakeLists.txt        |   1 +
 be/src/bench/filter_data_bench.cpp | 291 +++++++++++++++++++++++++++++
 2 files changed, 292 insertions(+)
 create mode 100644 be/src/bench/filter_data_bench.cpp

diff --git a/be/src/bench/CMakeLists.txt b/be/src/bench/CMakeLists.txt
index 0a69af8c7bc287..45e08dba5710db 100644
--- a/be/src/bench/CMakeLists.txt
+++ b/be/src/bench/CMakeLists.txt
@@ -27,3 +27,4 @@ ADD_BE_BENCH(${SRC_DIR}/bench/binary_column_copy_bench)
 ADD_BE_BENCH(${SRC_DIR}/bench/hyperscan_vec_bench)
 ADD_BE_BENCH(${SRC_DIR}/bench/mem_equal_bench)
 ADD_BE_BENCH(${SRC_DIR}/bench/bit_unpack_bench)
+ADD_BE_BENCH(${SRC_DIR}/bench/filter_data_bench)
diff --git a/be/src/bench/filter_data_bench.cpp b/be/src/bench/filter_data_bench.cpp
new file mode 100644
index 00000000000000..75cffa228bd092
--- /dev/null
+++ b/be/src/bench/filter_data_bench.cpp
@@ -0,0 +1,291 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <benchmark/benchmark.h>
+#include <gtest/gtest.h>
+#include <testutil/assert.h>
+
+#include <cstdint>
+#include <random>
+#include <vector>
+
+#include "column/vectorized_fwd.h"
+#include "gutil/endian.h"
+#ifdef __AVX2__
+#include <emmintrin.h>
+#endif
+
+#include "column/column_helper.h"
+#include "runtime/current_thread.h"
+#include "simd/simd.h"
+
+namespace starrocks {
+
+enum class FilterType {
+    ORDINARY,
+    COLLECT_ASSIGN,
+    COMPRESS,
+    CUSTOM,
+};
+
+template <typename T>
+static inline size_t ordinary_filter(const Filter& filter, T* data) {
+    auto result_offset = 0;
+    for (auto i = 0; i < filter.size(); i++) {
+        if (filter[i]) {
+            *(data + result_offset) = *(data + i);
+            result_offset++;
+        }
+    }
+    return result_offset;
+}
+
+template <typename T>
+static inline size_t collect_assign_filter(const Filter& filter, T* data, std::vector<T>& res_data) {
+    auto res = 0;
+    for (auto i = 0; i < filter.size(); i++) {
+        res += filter[i] ? 1 : 0;
+    }
+
+    std::vector<size_t> pos(res);
+    auto index = 0;
+    for (auto i = 0; index < res; i++) {
+        pos[index] = i;
+        index += filter[i] ? 1 : 0;
+    }
+    res_data.resize(pos.size());
+    for (auto i = 0; i < pos.size(); i++) {
+        res_data[i] = *(data + pos[i]);
+    }
+    return res;
+}
+
+template <typename T>
+static inline size_t filter_simd_compress(const Filter& filter, T* data) {
+    std::vector<uint8> bit_mask(filter.size() / 8);
+
+    constexpr size_t filter_batch_size = 64;
+    auto mask_offset = 0;
+#ifdef __AVX512BW__
+    const __m512i zero64 = _mm512_setzero_si512();
+    for (size_t i = 0; i < filter.size(); i += filter_batch_size) {
+        int64 m = _mm512_cmpneq_epi8_mask(_mm512_loadu_si512(reinterpret_cast<const __m512i*>(filter.data() + i)),
+                                          zero64);
+        // int64 m_order = ghtonll(m);
+        memcpy(bit_mask.data() + mask_offset, &m, 8);
+        mask_offset += 8;
+    }
+#endif
+    constexpr size_t batch_size = 512 / (sizeof(T) * 8);
+    constexpr size_t mask_batch_size = batch_size / 8;
+    size_t res = 0;
+    size_t batches = bit_mask.size() * 8 / batch_size;
+    if constexpr (sizeof(T) * 8 == 32) {
+        for (size_t i = 0; i < batches; i++) {
+#ifdef __AVX512F__
+            __mmask16 m = *(reinterpret_cast<const uint16_t*>(bit_mask.data() + i * mask_batch_size));
+            __m512i src = _mm512_loadu_epi32(data + i * batch_size);
+            _mm512_mask_compressstoreu_epi32(data + res, m, src);
+            res += __builtin_popcount(m);
+#endif
+        }
+    } else if constexpr (sizeof(T) * 8 == 64) {
+        for (size_t i = 0; i < batches; i++) {
+#ifdef __AVX512F__
+            __mmask8 m = bit_mask[i * mask_batch_size];
+            __m512i src = _mm512_loadu_epi64(data + i * batch_size);
+            _mm512_mask_compressstoreu_epi64(data + res, m, src);
+            res += __builtin_popcount(m);
+#endif
+        }
+    } else {
+        // pass
+    }
+
+    return res;
+}
+
+template <typename T>
+class FilterDataBench {
+public:
+    void SetUp();
+    void TearDown() {}
+
+    FilterDataBench(size_t ratio) : _ratio(ratio) {}
+
+    template <FilterType type>
+    size_t do_bench();
+
+    // called before filter, used to check the filter result.
+    T expect_result() {
+        T sum = 0;
+        for (size_t i = 0; i < _data.size(); i++) {
+            if (_filter[i]) {
+                sum += _data[i];
+            }
+        }
+        return sum;
+    }
+
+    // called after filter, used to check the filter result.
+    template <FilterType type>
+    T final_result(size_t size) {
+        T sum = 0;
+        if constexpr (type != FilterType::COLLECT_ASSIGN) {
+            for (size_t i = 0; i < size; i++) {
+                sum += _data[i];
+            }
+        } else {
+            for (size_t i = 0; i < size; i++) {
+                sum += _res_data[i];
+            }
+        }
+        return sum;
+    }
+
+private:
+    size_t _ratio = 0;
+    static constexpr size_t _num_rows = 4096;
+    Filter _filter;
+    std::vector<T> _data;
+    std::vector<T> _res_data;
+};
+
+template <typename T>
+void FilterDataBench<T>::SetUp() {
+    _filter.resize(_num_rows);
+    _data.resize(_num_rows);
+
+    std::random_device rd;
+    std::mt19937 rng(rd());
+    std::uniform_int_distribution<T> dist(0, 999);
+
+    for (size_t i = 0; i < _num_rows; i++) {
+        int random_number = dist(rng);
+        _filter[i] = random_number < _ratio ? 0 : 1;
+        _data[i] = random_number;
+    }
+}
+
+template <typename T>
+template <FilterType type>
+size_t FilterDataBench<T>::do_bench() {
+    if constexpr (type == FilterType::ORDINARY) {
+        return ordinary_filter<T>(_filter, _data.data());
+    } else if constexpr (type == FilterType::COLLECT_ASSIGN) {
+        return collect_assign_filter<T>(_filter, _data.data(), _res_data);
+    } else if constexpr (type == FilterType::COMPRESS) {
+        return filter_simd_compress<T>(_filter, _data.data());
+    } else {
+        return ColumnHelper::filter<T>(_filter, _data.data());
+    }
+}
+
+template <FilterType type>
+static void BM_FilterData_T(benchmark::State& state) {
+    for (auto _ : state) {
+        state.PauseTiming();
+        FilterDataBench<uint64_t> bench(state.range(0));
+        bench.SetUp();
+        size_t sum = bench.expect_result();
+        size_t res = 0;
+        state.ResumeTiming();
+        res = bench.do_bench<type>();
+        state.PauseTiming();
+        ASSERT_EQ(sum, bench.final_result<type>(res));
+    }
+}
+
+static void BM_FilterData_Custom(benchmark::State& state) {
+    BM_FilterData_T<FilterType::CUSTOM>(state);
+}
+
+static void BM_FilterData_Ordinary(benchmark::State& state) {
+    BM_FilterData_T<FilterType::ORDINARY>(state);
+}
+
+static void BM_FilterData_CAndA(benchmark::State& state) {
+    BM_FilterData_T<FilterType::COLLECT_ASSIGN>(state);
+}
+
+static void BM_FilterData_Compress(benchmark::State& state) {
+    BM_FilterData_T<FilterType::COMPRESS>(state);
+}
+
+BENCHMARK(BM_FilterData_Custom)->ArgsProduct({{0, 20, 40, 60, 80, 100, 300, 500, 700, 900, 920, 940, 960, 980, 1000}});
+
+#if defined(__AVX512F__) && defined(__AVX512BW__)
+BENCHMARK(BM_FilterData_Compress)
+        ->ArgsProduct({{0, 20, 40, 60, 80, 100, 300, 500, 700, 900, 920, 940, 960, 980, 1000}});
+#endif
+
+BENCHMARK(BM_FilterData_Ordinary)
+        ->ArgsProduct({{0, 20, 40, 60, 80, 100, 300, 500, 700, 900, 920, 940, 960, 980, 1000}});
+
+BENCHMARK(BM_FilterData_CAndA)->ArgsProduct({{0, 20, 40, 60, 80, 100, 300, 500, 700, 900, 920, 940, 960, 980, 1000}});
+
+} //namespace starrocks
+
+BENCHMARK_MAIN();
+
+/*
+----------------------------------------------------------------------
+Benchmark                            Time             CPU   Iterations
+----------------------------------------------------------------------
+BM_FilterData_Custom/0            2402 ns         2357 ns       297359
+BM_FilterData_Custom/20           3490 ns         3461 ns       202131
+BM_FilterData_Custom/40           2930 ns         2885 ns       242048
+BM_FilterData_Custom/60           2431 ns         2394 ns       292127
+BM_FilterData_Custom/80           2197 ns         2158 ns       324461
+BM_FilterData_Custom/100          2086 ns         2045 ns       342555
+BM_FilterData_Custom/300          1963 ns         1915 ns       365499
+BM_FilterData_Custom/500          1946 ns         1898 ns       369684
+BM_FilterData_Custom/700          1922 ns         1881 ns       372513
+BM_FilterData_Custom/900          2686 ns         2634 ns       265878
+BM_FilterData_Custom/920          2999 ns         2940 ns       238365
+BM_FilterData_Custom/940          3303 ns         3250 ns       214932
+BM_FilterData_Custom/960          3563 ns         3506 ns       199595
+BM_FilterData_Custom/980          3248 ns         3203 ns       218570
+BM_FilterData_Custom/1000         1170 ns         1124 ns       623176
+BM_FilterData_Ordinary/0          7758 ns         7709 ns        90928
+BM_FilterData_Ordinary/20         9250 ns         9194 ns        76172
+BM_FilterData_Ordinary/40        10758 ns        10706 ns        65369
+BM_FilterData_Ordinary/60        12280 ns        12233 ns        57226
+BM_FilterData_Ordinary/80        13818 ns        13771 ns        50804
+BM_FilterData_Ordinary/100       15376 ns        15330 ns        45678
+BM_FilterData_Ordinary/300       31681 ns        31643 ns        22294
+BM_FilterData_Ordinary/500       40954 ns        40914 ns        17095
+BM_FilterData_Ordinary/700       32627 ns        32590 ns        21308
+BM_FilterData_Ordinary/900       16672 ns        16626 ns        42062
+BM_FilterData_Ordinary/920       14805 ns        14750 ns        47332
+BM_FilterData_Ordinary/940       12853 ns        12803 ns        54537
+BM_FilterData_Ordinary/960       10846 ns        10795 ns        64791
+BM_FilterData_Ordinary/980        8736 ns         8686 ns        80564
+BM_FilterData_Ordinary/1000       6678 ns         6629 ns       105723
+BM_FilterData_CAndA/0            14344 ns        14313 ns        48456
+BM_FilterData_CAndA/20           14187 ns        14153 ns        49442
+BM_FilterData_CAndA/40           14032 ns        13996 ns        50019
+BM_FilterData_CAndA/60           13906 ns        13866 ns        50418
+BM_FilterData_CAndA/80           13832 ns        13795 ns        50918
+BM_FilterData_CAndA/100          13692 ns        13660 ns        51185
+BM_FilterData_CAndA/300          12234 ns        12198 ns        57389
+BM_FilterData_CAndA/500          10867 ns        10835 ns        64551
+BM_FilterData_CAndA/700           9461 ns         9422 ns        74335
+BM_FilterData_CAndA/900           8028 ns         7978 ns        87790
+BM_FilterData_CAndA/920           7903 ns         7846 ns        89225
+BM_FilterData_CAndA/940           7753 ns         7703 ns        90843
+BM_FilterData_CAndA/960           7604 ns         7554 ns        92631
+BM_FilterData_CAndA/980           7473 ns         7421 ns        94427
+BM_FilterData_CAndA/1000          1405 ns         1356 ns       517270
+ */

From d33824055411515a9a3a2b32653807a8882e8a1d Mon Sep 17 00:00:00 2001
From: xiaochen <598457447@qq.com>
Date: Wed, 22 Jan 2025 10:27:49 +0800
Subject: [PATCH 67/71] [Enhancement] Implement trino function year_of_week 
 (#54850)

Signed-off-by: ClownXC <598457447@qq.com>
---
 .../Trino2SRFunctionCallTransformer.java      | 24 +++++++++++++++++++
 .../trino/TrinoFunctionTransformTest.java     |  6 +++++
 2 files changed, 30 insertions(+)

diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/parser/trino/Trino2SRFunctionCallTransformer.java b/fe/fe-core/src/main/java/com/starrocks/connector/parser/trino/Trino2SRFunctionCallTransformer.java
index b0449347a2a5e3..1f8eccc06a7c13 100644
--- a/fe/fe-core/src/main/java/com/starrocks/connector/parser/trino/Trino2SRFunctionCallTransformer.java
+++ b/fe/fe-core/src/main/java/com/starrocks/connector/parser/trino/Trino2SRFunctionCallTransformer.java
@@ -233,6 +233,30 @@ private static void registerDateFunctionTransformer() {
         // last_day_of_month(x)  -> last_day(x,'month')
         registerFunctionTransformer("last_day_of_month", 1, new FunctionCallExpr("last_day",
                 List.of(new PlaceholderExpr(1, Expr.class), new StringLiteral("month"))));
+
+        // year_of_week(x) -> floor(divide(yearweek('x', 1),100))
+        registerFunctionTransformer("year_of_week", 1,
+                new FunctionCallExpr("floor", List.of(
+                        new FunctionCallExpr("divide", List.of(
+                                new FunctionCallExpr("yearweek", List.of(
+                                        new PlaceholderExpr(1, Expr.class),
+                                        new IntLiteral(1))
+                                ),
+                                new IntLiteral(100))
+                        )
+                )));
+
+        // yow(x) -> floor(divide(yearweek('x', 1),100))
+        registerFunctionTransformer("yow", 1,
+                new FunctionCallExpr("floor", List.of(
+                        new FunctionCallExpr("divide", List.of(
+                                new FunctionCallExpr("yearweek", List.of(
+                                        new PlaceholderExpr(1, Expr.class),
+                                        new IntLiteral(1))
+                                ),
+                                new IntLiteral(100))
+                        )
+                )));
     }
 
     private static void registerStringFunctionTransformer() {
diff --git a/fe/fe-core/src/test/java/com/starrocks/connector/parser/trino/TrinoFunctionTransformTest.java b/fe/fe-core/src/test/java/com/starrocks/connector/parser/trino/TrinoFunctionTransformTest.java
index 34f5beae7836ce..0fd9150ca345c5 100644
--- a/fe/fe-core/src/test/java/com/starrocks/connector/parser/trino/TrinoFunctionTransformTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/connector/parser/trino/TrinoFunctionTransformTest.java
@@ -211,6 +211,12 @@ public void testDateFnTransform() throws Exception {
 
         sql = "select to_timestamp('2022-02-02', 'yyyy-mm-dd')";
         assertPlanContains(sql, " to_tera_timestamp('2022-02-02', 'yyyy-mm-dd')");
+
+        sql = "select year_of_week('2022-02-02')";
+        assertPlanContains(sql, "<slot 2> : 2022");
+
+        sql = "select yow('2022-02-02')";
+        assertPlanContains(sql, "<slot 2> : 2022");
     }
 
     @Test

From 11eaaf164abe45ff64ce95dd1d47b30208ce2a46 Mon Sep 17 00:00:00 2001
From: "zihe.liu" <ziheliu1024@gmail.com>
Date: Wed, 22 Jan 2025 10:34:22 +0800
Subject: [PATCH 68/71] [Enhancement] Update limitation of M_IVFPQ and
 is_input_normalized for vector index (#55283)

Signed-off-by: zihe.liu <ziheliu1024@gmail.com>
---
 be/src/storage/index/vector/tenann/tenann_index_builder.cpp     | 2 +-
 be/test/storage/index/vector_index_test.cpp                     | 2 +-
 be/test/storage/index/vector_search_test.cpp                    | 2 +-
 .../src/main/java/com/starrocks/common/VectorIndexParams.java   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/be/src/storage/index/vector/tenann/tenann_index_builder.cpp b/be/src/storage/index/vector/tenann/tenann_index_builder.cpp
index 84b07655ac4c29..c5fd7855790553 100644
--- a/be/src/storage/index/vector/tenann/tenann_index_builder.cpp
+++ b/be/src/storage/index/vector/tenann/tenann_index_builder.cpp
@@ -101,7 +101,7 @@ static Status valid_input_vector(const ArrayColumn& input_column, const size_t i
             for (int j = 0; j < input_dim; j++) {
                 sum += nums[offsets[i] + j] * nums[offsets[i] + j];
             }
-            if (std::abs(sum - 1) > 1e-6) {
+            if (std::abs(sum - 1) > 1e-3) {
                 return Status::InvalidArgument(
                         "The input vector is not normalized but `metric_type` is cosine_similarity and "
                         "`is_vector_normed` is true");
diff --git a/be/test/storage/index/vector_index_test.cpp b/be/test/storage/index/vector_index_test.cpp
index c45dbb6b405407..a518db46c047f2 100644
--- a/be/test/storage/index/vector_index_test.cpp
+++ b/be/test/storage/index/vector_index_test.cpp
@@ -64,7 +64,7 @@ class VectorIndexWriterTest : public testing::Test {
         DeferOp op([&] { ASSERT_TRUE(fs::path_exist(path)); });
 
         std::unique_ptr<VectorIndexWriter> vector_index_writer;
-        VectorIndexWriter::create(tablet_index, path, false, &vector_index_writer);
+        VectorIndexWriter::create(tablet_index, path, true, &vector_index_writer);
         CHECK_OK(vector_index_writer->init());
 
         // construct columns
diff --git a/be/test/storage/index/vector_search_test.cpp b/be/test/storage/index/vector_search_test.cpp
index 5627ee5c9fa215..3fc62c7fa6b0d5 100644
--- a/be/test/storage/index/vector_search_test.cpp
+++ b/be/test/storage/index/vector_search_test.cpp
@@ -65,7 +65,7 @@ class VectorIndexSearchTest : public testing::Test {
         DeferOp op([&] { ASSERT_TRUE(fs::path_exist(path)); });
 
         std::unique_ptr<VectorIndexWriter> vector_index_writer;
-        VectorIndexWriter::create(tablet_index, path, false, &vector_index_writer);
+        VectorIndexWriter::create(tablet_index, path, true, &vector_index_writer);
         CHECK_OK(vector_index_writer->init());
 
         // construct columns
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java b/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java
index 41f73c2279ccaa..5df7f3221a95e7 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/VectorIndexParams.java
@@ -140,7 +140,7 @@ public void check(String value) {
         M_IVFPQ(VectorIndexType.IVFPQ) {
             @Override
             public void check(String value) {
-                validateInteger(value, "M_IVFPQ", 2);
+                validateInteger(value, "M_IVFPQ", 1);
             }
         };
 

From e886985477012c174f65c6b54cc4d6c7719a7f8d Mon Sep 17 00:00:00 2001
From: Drake Wang <wxl24life@gmail.com>
Date: Wed, 22 Jan 2025 11:19:39 +0800
Subject: [PATCH 69/71] [Enhancement] Lake compaction scheduler optimize in fe
 restart scenarios (#54881)

Signed-off-by: drake_wang <wxl250059@alibaba-inc.com>
---
 .../lake/compaction/CompactionMgr.java        |  58 +++++++++-
 .../lake/compaction/CompactionScheduler.java  |  23 +---
 .../transaction/DatabaseTransactionMgr.java   |  14 +++
 .../transaction/GlobalTransactionMgr.java     |  14 +++
 .../transaction/LakeTableTxnLogApplier.java   |   3 +-
 .../transaction/TransactionStateSnapshot.java |   2 +-
 .../catalog/GlobalStateMgrTestUtil.java       |   2 +
 .../lake/compaction/CompactionMgrTest.java    | 108 +++++++++++++++++-
 .../compaction/CompactionSchedulerTest.java   |   2 +-
 .../DatabaseTransactionMgrTest.java           |  67 +++++++++--
 10 files changed, 252 insertions(+), 41 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionMgr.java b/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionMgr.java
index 686cf01ba3d5d4..1b35294675b23d 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionMgr.java
@@ -56,6 +56,21 @@ public class CompactionMgr implements MemoryTrackable {
     private Sorter sorter;
     private CompactionScheduler compactionScheduler;
 
+    /**
+     * We use `activeCompactionTransactionMap` to track all lake compaction txns that are not published on FE restart.
+     * The key of the map is the transaction id related to the compaction task, and the value is table id of the
+     * compaction task. It's possible that multiple keys have the same value, because there might be multiple compaction
+     * jobs on different partitions with the same table id.
+     *
+     * Note that, this will prevent all partitions whose tableId is maintained in the map from being compacted
+     */
+    private final ConcurrentHashMap<Long, Long> remainedActiveCompactionTxnWhenStart = new ConcurrentHashMap<>();
+
+    @VisibleForTesting
+    protected ConcurrentHashMap<Long, Long> getRemainedActiveCompactionTxnWhenStart() {
+        return remainedActiveCompactionTxnWhenStart;
+    }
+
     public CompactionMgr() {
         try {
             init();
@@ -90,6 +105,30 @@ public void start() {
         }
     }
 
+    /**
+     * iterate all transactions and find those with LAKE_COMPACTION labels and are not finished before FE restart.
+     **/
+    public void rebuildActiveCompactionTransactionMapOnRestart() {
+        Map<Long, Long> activeTxnStates =
+                GlobalStateMgr.getCurrentState().getGlobalTransactionMgr().getLakeCompactionActiveTxnStats();
+        for (Map.Entry<Long, Long> txnState : activeTxnStates.entrySet()) {
+            // for lake compaction txn, there can only be one table id for each txn state
+            remainedActiveCompactionTxnWhenStart.put(txnState.getKey(), txnState.getValue());
+            LOG.info("Found lake compaction transaction not finished on table {}, txn_id: {}", txnState.getValue(),
+                    txnState.getKey());
+        }
+    }
+
+    protected void removeFromStartupActiveCompactionTransactionMap(long txnId) {
+        if (remainedActiveCompactionTxnWhenStart.isEmpty()) {
+            return;
+        }
+        boolean ret = remainedActiveCompactionTxnWhenStart.keySet().removeIf(key -> key == txnId);
+        if (ret) {
+            LOG.info("Removed transaction {} from startup active compaction transaction map", txnId);
+        }
+    }
+
     public void handleLoadingFinished(PartitionIdentifier partition, long version, long versionTime,
                                       Quantiles compactionScore) {
         PartitionVersion currentVersion = new PartitionVersion(version, versionTime);
@@ -107,7 +146,8 @@ public void handleLoadingFinished(PartitionIdentifier partition, long version, l
     }
 
     public void handleCompactionFinished(PartitionIdentifier partition, long version, long versionTime,
-                                         Quantiles compactionScore) {
+                                         Quantiles compactionScore, long txnId) {
+        removeFromStartupActiveCompactionTransactionMap(txnId);
         PartitionVersion compactionVersion = new PartitionVersion(version, versionTime);
         PartitionStatistics statistics = partitionStatisticsHashMap.compute(partition, (k, v) -> {
             if (v == null) {
@@ -125,8 +165,10 @@ public void handleCompactionFinished(PartitionIdentifier partition, long version
 
     @NotNull
     List<PartitionStatisticsSnapshot> choosePartitionsToCompact(@NotNull Set<PartitionIdentifier> excludes,
-            @NotNull Set<Long> excludeTables) {
-        return choosePartitionsToCompact(excludeTables)
+                                                                @NotNull Set<Long> excludeTables) {
+        Set<Long> copiedExcludeTables = new HashSet<>(excludeTables);
+        copiedExcludeTables.addAll(remainedActiveCompactionTxnWhenStart.values());
+        return choosePartitionsToCompact(copiedExcludeTables)
                 .stream()
                 .filter(p -> !excludes.contains(p.getPartition()))
                 .collect(Collectors.toList());
@@ -215,6 +257,16 @@ public void save(ImageWriter imageWriter) throws IOException, SRMetaBlockExcepti
     public void load(SRMetaBlockReader reader) throws IOException, SRMetaBlockException, SRMetaBlockEOFException {
         CompactionMgr compactionManager = reader.readJson(CompactionMgr.class);
         partitionStatisticsHashMap = compactionManager.partitionStatisticsHashMap;
+
+        // In order to ensure that the input rowsets of compaction still exists when doing publishing version, it is
+        // necessary to ensure that the compaction task of the same partition is executed serially, that is, the next
+        // compaction task can be executed only after the status of the previous compaction task changes to visible or
+        // canceled.
+        // So when FE restarted, we should make sure all the active compaction transactions before restarting were tracked,
+        // and exclude them from choosing as candidates for compaction.
+        // Note here, the map is maintained on leader and follower fe, its keys were removed from the map after compaction
+        // transaction has finished, and for follower FE, this is done by replay process.
+        rebuildActiveCompactionTransactionMapOnRestart();
     }
 
     public long getPartitionStatsCount() {
diff --git a/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java b/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java
index 18a9452c138ab2..6db3717a9e6e22 100644
--- a/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/lake/compaction/CompactionScheduler.java
@@ -79,8 +79,6 @@ public class CompactionScheduler extends Daemon {
     private final GlobalStateMgr stateMgr;
     private final ConcurrentHashMap<PartitionIdentifier, CompactionJob> runningCompactions;
     private final SynchronizedCircularQueue<CompactionRecord> history;
-    private boolean finishedWaiting = false;
-    private long waitTxnId = -1;
     private long lastPartitionCleanTime;
     private Set<Long> disabledTables; // copy-on-write
 
@@ -105,31 +103,12 @@ protected void runOneCycle() {
         cleanPhysicalPartition();
 
         // Schedule compaction tasks only when this is a leader FE and all edit logs have finished replay.
-        // In order to ensure that the input rowsets of compaction still exists when doing publishing version, it is
-        // necessary to ensure that the compaction task of the same partition is executed serially, that is, the next
-        // compaction task can be executed only after the status of the previous compaction task changes to visible or
-        // canceled.
-        if (stateMgr.isLeader() && stateMgr.isReady() && allCommittedCompactionsBeforeRestartHaveFinished()) {
+        if (stateMgr.isLeader() && stateMgr.isReady()) {
             schedule();
             history.changeMaxSize(Config.lake_compaction_history_size);
         }
     }
 
-    // Returns true if all compaction transactions committed before this restart have finished(i.e., of VISIBLE state).
-    private boolean allCommittedCompactionsBeforeRestartHaveFinished() {
-        if (finishedWaiting) {
-            return true;
-        }
-        // Note: must call getMinActiveCompactionTxnId() before getNextTransactionId(), otherwise if there are
-        // no running transactions waitTxnId <= minActiveTxnId will always be false.
-        long minActiveTxnId = transactionMgr.getMinActiveCompactionTxnId();
-        if (waitTxnId < 0) {
-            waitTxnId = transactionMgr.getTransactionIDGenerator().getNextTransactionId();
-        }
-        finishedWaiting = waitTxnId <= minActiveTxnId;
-        return finishedWaiting;
-    }
-
     private void schedule() {
         // Check whether there are completed compaction jobs.
         for (Iterator<Map.Entry<PartitionIdentifier, CompactionJob>> iterator = runningCompactions.entrySet().iterator();
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java
index 4522e444ec8137..16c1faf01d6cd3 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/DatabaseTransactionMgr.java
@@ -838,6 +838,20 @@ public List<TransactionState> getCommittedTxnList() {
         }
     }
 
+    public Map<Long, Long> getLakeCompactionActiveTxnMap() {
+        readLock();
+        try {
+            // for lake compaction txn, there can only be one table id for each txn state
+            Map<Long, Long> txnIdToTableIdMap = new HashMap<>();
+            idToRunningTransactionState.values().stream()
+                    .filter(state -> state.getSourceType() == TransactionState.LoadJobSourceType.LAKE_COMPACTION)
+                    .forEach(state -> txnIdToTableIdMap.put(state.getTransactionId(), state.getTableIdList().get(0)));
+            return txnIdToTableIdMap;
+        } finally {
+            readUnlock();
+        }
+    }
+
     // Check whether there is committed txns on partitionId.
     public boolean hasCommittedTxnOnPartition(long tableId, long partitionId) {
         readLock();
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java b/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java
index 9274436366d7f4..9f2a216115d731 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/GlobalTransactionMgr.java
@@ -72,6 +72,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -647,6 +648,19 @@ public long getMinActiveCompactionTxnId() {
         return minId;
     }
 
+    /**
+     * Get the map of active txn [txnId, tableId] of compaction transactions.
+     * @return the list of active txn stats of compaction transactions.
+     */
+    public Map<Long, Long> getLakeCompactionActiveTxnStats() {
+        Map<Long, Long> txnIdToTableIdMap = new HashMap<>();
+        for (Map.Entry<Long, DatabaseTransactionMgr> entry : dbIdToDatabaseTransactionMgrs.entrySet()) {
+            DatabaseTransactionMgr dbTransactionMgr = entry.getValue();
+            txnIdToTableIdMap.putAll(dbTransactionMgr.getLakeCompactionActiveTxnMap());
+        }
+        return txnIdToTableIdMap;
+    }
+
     /**
      * Get the smallest transaction ID of active transactions in a database.
      * If there are no active transactions in the database, return the transaction ID that will be assigned to the
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnLogApplier.java b/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnLogApplier.java
index 03085ae28da831..180621d29bcba8 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnLogApplier.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/LakeTableTxnLogApplier.java
@@ -99,7 +99,8 @@ public void applyVisibleLog(TransactionState txnState, TableCommitInfo commitInf
             PartitionIdentifier partitionIdentifier =
                     new PartitionIdentifier(txnState.getDbId(), table.getId(), partition.getId());
             if (txnState.getSourceType() == TransactionState.LoadJobSourceType.LAKE_COMPACTION) {
-                compactionManager.handleCompactionFinished(partitionIdentifier, version, versionTime, compactionScore);
+                compactionManager.handleCompactionFinished(partitionIdentifier, version, versionTime, compactionScore,
+                        txnState.getTransactionId());
             } else {
                 compactionManager.handleLoadingFinished(partitionIdentifier, version, versionTime, compactionScore);
             }
diff --git a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateSnapshot.java b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateSnapshot.java
index 0e0ab4e42a219c..9514994801d52f 100644
--- a/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateSnapshot.java
+++ b/fe/fe-core/src/main/java/com/starrocks/transaction/TransactionStateSnapshot.java
@@ -40,4 +40,4 @@ public String toString() {
                 ", reason='" + reason + '\'' +
                 '}';
     }
-}
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java b/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java
index 83e317fa637d15..84a3e3bca158e9 100644
--- a/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java
+++ b/fe/fe-core/src/test/java/com/starrocks/catalog/GlobalStateMgrTestUtil.java
@@ -89,6 +89,8 @@ public class GlobalStateMgrTestUtil {
     public static String testTxnLable8 = "testTxnLable8";
     public static String testTxnLable9 = "testTxnLable9";
     public static String testTxnLable10 = "testTxnLable10";
+    public static String testTxnLableCompaction1 = "testTxnLableCompaction1";
+    public static String testTxnLableCompaction2 = "testTxnLableCompaction2";
     public static String testEsTable1 = "partitionedEsTable1";
     public static long testEsTableId1 = 14;
 
diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java
index 04125deb5fa0f0..82d3c50d1588a8 100644
--- a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionMgrTest.java
@@ -26,20 +26,31 @@
 import com.starrocks.persist.metablock.SRMetaBlockReaderV2;
 import com.starrocks.server.GlobalStateMgr;
 import com.starrocks.sql.common.MetaUtils;
+import com.starrocks.transaction.GlobalTransactionMgr;
 import com.starrocks.utframe.UtFrameUtils;
+import mockit.Expectations;
 import mockit.Mock;
 import mockit.MockUp;
+import mockit.Mocked;
 import org.junit.Assert;
 import org.junit.Test;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class CompactionMgrTest {
+    @Mocked
+    private GlobalStateMgr globalStateMgr;
+    @Mocked
+    private GlobalTransactionMgr globalTransactionMgr;
+    @Mocked
+    private Database db;
 
     @Test
     public void testChoosePartitionsToCompact() {
@@ -87,6 +98,65 @@ public void testChoosePartitionsToCompact() {
         Assert.assertSame(partition2, compactionList.get(0).getPartition());
     }
 
+    @Test
+    public void testChoosePartitionsToCompactWithActiveTxnFilter() {
+        long dbId = 10001L;
+        long tableId1 = 10002L;
+        long tableId2 = 10003L;
+        long partitionId10 = 20001L;
+        long partitionId11 = 20003L;
+        long partitionId20 = 20002L;
+
+        PartitionIdentifier partition10 = new PartitionIdentifier(dbId, tableId1, partitionId10);
+        PartitionIdentifier partition11 = new PartitionIdentifier(dbId, tableId1, partitionId11);
+        PartitionIdentifier partition20 = new PartitionIdentifier(dbId, tableId2, partitionId20);
+
+        CompactionMgr compactionManager = new CompactionMgr();
+        compactionManager.handleLoadingFinished(partition10, 1, System.currentTimeMillis(),
+                Quantiles.compute(Lists.newArrayList(100d)));
+        compactionManager.handleLoadingFinished(partition11, 2, System.currentTimeMillis(),
+                Quantiles.compute(Lists.newArrayList(100d)));
+        compactionManager.handleLoadingFinished(partition20, 3, System.currentTimeMillis(),
+                Quantiles.compute(Lists.newArrayList(100d)));
+
+        // build active txn on table1
+        long txnId = 10001L;
+        Map<Long, Long> txnIdToTableIdMap = new HashMap<>();
+        txnIdToTableIdMap.put(txnId, tableId1);
+        new Expectations() {
+            {
+                GlobalStateMgr.getCurrentState();
+                result = globalStateMgr;
+
+                globalStateMgr.getGlobalTransactionMgr();
+                result = globalTransactionMgr;
+
+                globalTransactionMgr.getLakeCompactionActiveTxnStats();
+                result = txnIdToTableIdMap;
+
+            }
+        };
+        compactionManager.rebuildActiveCompactionTransactionMapOnRestart();
+
+        Set<PartitionIdentifier> allPartitions = compactionManager.getAllPartitions();
+        Assert.assertEquals(3, allPartitions.size());
+        Assert.assertTrue(allPartitions.contains(partition10));
+        Assert.assertTrue(allPartitions.contains(partition11));
+        Assert.assertTrue(allPartitions.contains(partition20));
+
+        List<PartitionStatisticsSnapshot> compactionList =
+                compactionManager.choosePartitionsToCompact(new HashSet<>(), new HashSet<>());
+        // both partition10 and partition11 are filtered because table1 has active txn
+        Assert.assertEquals(1, compactionList.size());
+        Assert.assertSame(partition20, compactionList.get(0).getPartition());
+
+        Set<Long> excludeTables = new HashSet<>();
+        excludeTables.add(tableId2);
+        compactionList = compactionManager.choosePartitionsToCompact(new HashSet<>(), excludeTables);
+        // tableId2 is filtered by excludeTables
+        Assert.assertEquals(0, compactionList.size());
+    }
+
     @Test
     public void testGetMaxCompactionScore() {
         double delta = 0.001;
@@ -100,7 +170,7 @@ public void testGetMaxCompactionScore() {
                 Quantiles.compute(Lists.newArrayList(1d)));
         Assert.assertEquals(1, compactionMgr.getMaxCompactionScore(), delta);
         compactionMgr.handleCompactionFinished(partition1, 3, System.currentTimeMillis(),
-                Quantiles.compute(Lists.newArrayList(2d)));
+                Quantiles.compute(Lists.newArrayList(2d)), 1234);
         Assert.assertEquals(2, compactionMgr.getMaxCompactionScore(), delta);
 
         compactionMgr.handleLoadingFinished(partition2, 2, System.currentTimeMillis(),
@@ -205,4 +275,40 @@ public boolean isPhysicalPartitionExist(GlobalStateMgr stateMgr, long dbId, long
         compactionMgr2.load(reader);
         Assert.assertEquals(1, compactionMgr2.getPartitionStatsCount());
     }
+
+    @Test
+    public void testActiveCompactionTransactionMapOnRestart() {
+        long txnId = 10001L;
+        long tableId = 10002L;
+        Map<Long, Long> txnIdToTableIdMap = new HashMap<>();
+        txnIdToTableIdMap.put(txnId, tableId);
+        new Expectations() {
+            {
+                GlobalStateMgr.getCurrentState();
+                result = globalStateMgr;
+
+                globalStateMgr.getGlobalTransactionMgr();
+                result = globalTransactionMgr;
+
+                globalTransactionMgr.getLakeCompactionActiveTxnStats();
+                result = txnIdToTableIdMap;
+
+            }
+        };
+
+        CompactionMgr compactionMgr = new CompactionMgr();
+        compactionMgr.rebuildActiveCompactionTransactionMapOnRestart();
+        ConcurrentHashMap<Long, Long> activeCompactionTransactionMap =
+                compactionMgr.getRemainedActiveCompactionTxnWhenStart();
+        Assert.assertEquals(1, activeCompactionTransactionMap.size());
+        Assert.assertTrue(activeCompactionTransactionMap.containsValue(tableId));
+
+        // test for removeFromStartupActiveCompactionTransactionMap
+        long nonExistedTxnId = 10003L;
+        compactionMgr.removeFromStartupActiveCompactionTransactionMap(nonExistedTxnId);
+        Assert.assertEquals(1, activeCompactionTransactionMap.size());
+
+        compactionMgr.removeFromStartupActiveCompactionTransactionMap(txnId);
+        Assert.assertEquals(0, activeCompactionTransactionMap.size());
+    }
 }
diff --git a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java
index ae2ddd9fc8bd62..906926d49cbd98 100644
--- a/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/lake/compaction/CompactionSchedulerTest.java
@@ -189,7 +189,7 @@ public ConcurrentHashMap<PartitionIdentifier, CompactionJob> getRunningCompactio
 
     @Test
     public void testCompactionTaskLimit() {
-        CompactionScheduler compactionScheduler = new CompactionScheduler(null, null, null, null, "");
+        CompactionScheduler compactionScheduler = new CompactionScheduler(new CompactionMgr(), null, null, null, "");
 
         int defaultValue = Config.lake_compaction_max_tasks;
         // explicitly set config to a value bigger than default -1
diff --git a/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java b/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java
index e177ea12017a28..4667fa303a6156 100644
--- a/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/transaction/DatabaseTransactionMgrTest.java
@@ -151,16 +151,7 @@ public Map<String, Long> addTransactionToTransactionMgr() throws StarRocksExcept
         Assert.assertEquals(idGenerator.peekNextTransactionId(), masterTransMgr.getMinActiveCompactionTxnId());
 
         // commit a transaction
-        TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1,
-                GlobalStateMgrTestUtil.testBackendId1);
-        TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1,
-                GlobalStateMgrTestUtil.testBackendId2);
-        TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1,
-                GlobalStateMgrTestUtil.testBackendId3);
-        List<TabletCommitInfo> transTablets = Lists.newArrayList();
-        transTablets.add(tabletCommitInfo1);
-        transTablets.add(tabletCommitInfo2);
-        transTablets.add(tabletCommitInfo3);
+        List<TabletCommitInfo> transTablets = buildTabletCommitInfoList();
         masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, transactionId1, transTablets,
                 Lists.newArrayList(), null);
         DatabaseTransactionMgr masterDbTransMgr =
@@ -206,7 +197,7 @@ public Map<String, Long> addTransactionToTransactionMgr() throws StarRocksExcept
                         Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1),
                         GlobalStateMgrTestUtil.testTxnLable5,
                         feTransactionSource,
-                        TransactionState.LoadJobSourceType.LAKE_COMPACTION,
+                        TransactionState.LoadJobSourceType.BACKEND_STREAMING,
                         Config.max_load_timeout_second);
         // for test batch
         long transactionId6 = masterTransMgr
@@ -249,7 +240,6 @@ public Map<String, Long> addTransactionToTransactionMgr() throws StarRocksExcept
         lableToTxnId.put(GlobalStateMgrTestUtil.testTxnLable8, transactionId8);
 
         Assert.assertEquals(transactionId2, masterTransMgr.getMinActiveTxnId());
-        Assert.assertEquals(transactionId5, masterTransMgr.getMinActiveCompactionTxnId());
 
         transactionGraph.add(transactionId6, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1));
         transactionGraph.add(transactionId7, Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1));
@@ -264,6 +254,59 @@ public Map<String, Long> addTransactionToTransactionMgr() throws StarRocksExcept
         return lableToTxnId;
     }
 
+    private List<TabletCommitInfo> buildTabletCommitInfoList() {
+        TabletCommitInfo tabletCommitInfo1 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1,
+                GlobalStateMgrTestUtil.testBackendId1);
+        TabletCommitInfo tabletCommitInfo2 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1,
+                GlobalStateMgrTestUtil.testBackendId2);
+        TabletCommitInfo tabletCommitInfo3 = new TabletCommitInfo(GlobalStateMgrTestUtil.testTabletId1,
+                GlobalStateMgrTestUtil.testBackendId3);
+        List<TabletCommitInfo> transTablets = Lists.newArrayList();
+        transTablets.add(tabletCommitInfo1);
+        transTablets.add(tabletCommitInfo2);
+        transTablets.add(tabletCommitInfo3);
+        return transTablets;
+    }
+
+    @Test
+    public void getLakeCompactionActiveTxnListTest() throws StarRocksException {
+        TransactionState.TxnCoordinator feTransactionSource =
+                new TransactionState.TxnCoordinator(TransactionState.TxnSourceType.FE, "fe1");
+        long committedCompactionTransactionId = masterTransMgr
+                .beginTransaction(GlobalStateMgrTestUtil.testDbId1,
+                        Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1),
+                        GlobalStateMgrTestUtil.testTxnLableCompaction1,
+                        feTransactionSource,
+                        TransactionState.LoadJobSourceType.LAKE_COMPACTION,
+                        Config.lake_compaction_default_timeout_second);
+
+        DatabaseTransactionMgr masterDbTransMgr =
+                masterTransMgr.getDatabaseTransactionMgr(GlobalStateMgrTestUtil.testDbId1);
+        List<TabletCommitInfo> transTablets = buildTabletCommitInfoList();
+        masterTransMgr.commitTransaction(GlobalStateMgrTestUtil.testDbId1, committedCompactionTransactionId, transTablets,
+                Lists.newArrayList(), null);
+        assertEquals(TransactionStatus.COMMITTED, masterDbTransMgr.getTxnState(committedCompactionTransactionId).getStatus());
+
+        long preparedCompactionTransactionId = masterTransMgr
+                .beginTransaction(GlobalStateMgrTestUtil.testDbId1,
+                        Lists.newArrayList(GlobalStateMgrTestUtil.testTableId1),
+                        GlobalStateMgrTestUtil.testTxnLableCompaction2,
+                        feTransactionSource,
+                        TransactionState.LoadJobSourceType.LAKE_COMPACTION,
+                        Config.lake_compaction_default_timeout_second);
+
+        Map<Long, Long> compactionActiveTxnMap = masterDbTransMgr.getLakeCompactionActiveTxnMap();
+        Assert.assertEquals(2, compactionActiveTxnMap.size());
+        Assert.assertTrue(compactionActiveTxnMap.containsKey(committedCompactionTransactionId));
+        Assert.assertTrue(compactionActiveTxnMap.containsKey(preparedCompactionTransactionId));
+
+        // global transaction stats check
+        Map<Long, Long> globalCompactionActiveTxnMap = masterTransMgr.getLakeCompactionActiveTxnStats();
+        Assert.assertEquals(2, globalCompactionActiveTxnMap.size());
+        Assert.assertTrue(globalCompactionActiveTxnMap.containsKey(committedCompactionTransactionId));
+        Assert.assertTrue(globalCompactionActiveTxnMap.containsKey(preparedCompactionTransactionId));
+    }
+
     @Test
     public void testNormal() throws StarRocksException {
         DatabaseTransactionMgr masterDbTransMgr =

From 1066a4ca3630b5d33296344711e90311cdc9371e Mon Sep 17 00:00:00 2001
From: gengjun-git <gengjun@starrocks.com>
Date: Mon, 6 Jan 2025 20:28:46 +0800
Subject: [PATCH 70/71] add

Signed-off-by: gengjun-git <gengjun@starrocks.com>
---
 be/src/agent/heartbeat_server.cpp             |  19 +--
 be/src/agent/task_worker_pool.cpp             |  31 ++--
 be/src/agent/task_worker_pool.h               |   6 +
 be/src/service/service_be/backend_service.cpp |  13 ++
 be/src/service/service_be/backend_service.h   |   2 +
 be/test/agent/heartbeat_server_test.cpp       |   4 +-
 .../java/com/starrocks/common/Config.java     |   6 +
 .../java/com/starrocks/leader/LeaderImpl.java |  10 +-
 .../com/starrocks/leader/ReportHandler.java   |  46 ++++--
 .../com/starrocks/leader/TabletCollector.java | 146 ++++++++++++++++++
 .../starrocks/memory/MemoryUsageTracker.java  |   1 +
 .../java/com/starrocks/metric/MetricRepo.java |   9 ++
 .../com/starrocks/server/GlobalStateMgr.java  |  14 ++
 .../com/starrocks/system/HeartbeatMgr.java    |   1 +
 .../com/starrocks/common/GenericPoolTest.java |   7 +
 .../starrocks/leader/ReportHandlerTest.java   |  12 ++
 .../starrocks/leader/TabletCollectorTest.java |  35 +++++
 gensrc/thrift/BackendService.thrift           |  11 ++
 gensrc/thrift/HeartbeatService.thrift         |   1 +
 19 files changed, 326 insertions(+), 48 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/com/starrocks/leader/TabletCollector.java
 create mode 100644 fe/fe-core/src/test/java/com/starrocks/leader/TabletCollectorTest.java

diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp
index 7f67a6a82b8426..070124cf07735b 100644
--- a/be/src/agent/heartbeat_server.cpp
+++ b/be/src/agent/heartbeat_server.cpp
@@ -43,6 +43,7 @@
 #include <sstream>
 
 #include "agent/master_info.h"
+#include "agent/task_worker_pool.h"
 #include "common/process_exit.h"
 #include "common/status.h"
 #include "gen_cpp/HeartbeatService.h"
@@ -105,15 +106,6 @@ void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result, const TMaste
         bool r = update_master_info(master_info);
         LOG_IF(WARNING, !r) << "Fail to update master info, maybe the master info has been updated by another thread "
                                "with a larger epoch";
-    } else if (*res == kNeedUpdateAndReport) {
-        LOG(INFO) << "Updating master info: " << print_master_info(master_info);
-        bool r = update_master_info(master_info);
-        LOG_IF(WARNING, !r) << "Fail to update master info, maybe the master info has been updated by another thread "
-                               "with a larger epoch";
-        if (r) {
-            LOG(INFO) << "Master FE is changed or restarted. report tablet and disk info immediately";
-            _olap_engine->trigger_report();
-        }
     } else {
         DCHECK_EQ(kUnchanged, *res);
         // nothing to do
@@ -127,6 +119,12 @@ void HeartbeatServer::heartbeat(THeartbeatResult& heartbeat_result, const TMaste
         _olap_engine->decommission_disks(master_info.decommissioned_disks);
     }
 
+    if (master_info.__isset.stop_regular_tablet_report) {
+        ReportOlapTableTaskWorkerPool::set_regular_report_stopped(master_info.stop_regular_tablet_report);
+    } else {
+        ReportOlapTableTaskWorkerPool::set_regular_report_stopped(false);
+    }
+
     static auto num_hardware_cores = static_cast<int32_t>(CpuInfo::num_cores());
     if (res.ok()) {
         heartbeat_result.backend_info.__set_be_port(config::be_port);
@@ -270,9 +268,6 @@ StatusOr<HeartbeatServer::CmpResult> HeartbeatServer::compare_master_info(const
         heartbeat_flags->update(master_info.heartbeat_flags);
     }
 
-    if (curr_master_info->network_address != master_info.network_address) {
-        return kNeedUpdateAndReport;
-    }
     if (*curr_master_info != master_info) {
         return kNeedUpdate;
     }
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 78204b892a9792..2ce3e1245157e9 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -76,15 +76,27 @@
 namespace starrocks {
 
 namespace {
-static void wait_for_notify_small_steps(int32_t timeout_sec, bool from_report_tablet_thread,
-                                        const std::function<bool()>& stop_waiting) {
-    auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_sec);
+static void wait_for_disk_report_notify(const std::function<bool()>& stop_waiting) {
+    auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config::report_disk_state_interval_seconds);
     bool notified = false;
     do {
         // take 1 second per step
-        notified = StorageEngine::instance()->wait_for_report_notify(1, from_report_tablet_thread);
+        notified = StorageEngine::instance()->wait_for_report_notify(1, false);
     } while (!notified && std::chrono::steady_clock::now() < deadline && !stop_waiting());
 }
+
+static void wait_for_tablet_report_notify(const std::function<bool()>& stop_waiting) {
+    auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(config::report_tablet_interval_seconds);
+    bool notified = false;
+    do {
+        // take 1 second per step
+        notified = StorageEngine::instance()->wait_for_report_notify(1, true);
+    } while (!notified
+             // if the regular report is stopped, there will be no deadline
+             && (ReportOlapTableTaskWorkerPool::is_regular_report_stopped() ||
+                 std::chrono::steady_clock::now() < deadline) &&
+             !stop_waiting());
+}
 } // namespace
 
 const size_t PUBLISH_VERSION_BATCH_SIZE = 10;
@@ -665,8 +677,7 @@ void* ReportDiskStateTaskWorkerPool::_worker_thread_callback(void* arg_this) {
         }
 
         // wait for notifying until timeout
-        wait_for_notify_small_steps(config::report_disk_state_interval_seconds, false,
-                                    [&] { return worker_pool_this->_stopped.load(); });
+        wait_for_disk_report_notify([&] { return worker_pool_this->_stopped.load(); });
     }
 
     return nullptr;
@@ -695,8 +706,7 @@ void* ReportOlapTableTaskWorkerPool::_worker_thread_callback(void* arg_this) {
         if (!st_report.ok()) {
             LOG(WARNING) << "Fail to report all tablets info, err=" << st_report.to_string();
             // wait for notifying until timeout
-            wait_for_notify_small_steps(config::report_tablet_interval_seconds, true,
-                                        [&] { return worker_pool_this->_stopped.load(); });
+            wait_for_tablet_report_notify([&] { return worker_pool_this->_stopped.load(); });
             continue;
         }
         int64_t max_compaction_score =
@@ -717,13 +727,14 @@ void* ReportOlapTableTaskWorkerPool::_worker_thread_callback(void* arg_this) {
         }
 
         // wait for notifying until timeout
-        wait_for_notify_small_steps(config::report_tablet_interval_seconds, true,
-                                    [&] { return worker_pool_this->_stopped.load(); });
+        wait_for_tablet_report_notify([&] { return worker_pool_this->_stopped.load(); });
     }
 
     return nullptr;
 }
 
+std::atomic<bool> ReportOlapTableTaskWorkerPool::_regular_report_stopped(false);
+
 void* ReportWorkgroupTaskWorkerPool::_worker_thread_callback(void* arg_this) {
     auto* worker_pool_this = (ReportWorkgroupTaskWorkerPool*)arg_this;
 
diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h
index 1fa6cd2499e3c9..5c08f9027d0b63 100644
--- a/be/src/agent/task_worker_pool.h
+++ b/be/src/agent/task_worker_pool.h
@@ -194,7 +194,13 @@ class ReportOlapTableTaskWorkerPool : public TaskWorkerPool<AgentTaskRequestWith
         _callback_function = _worker_thread_callback;
     }
 
+    static void set_regular_report_stopped(bool stop) { _regular_report_stopped.store(stop); }
+
+    static bool is_regular_report_stopped() { return _regular_report_stopped.load(); }
+
 private:
+    static std::atomic<bool> _regular_report_stopped;
+
     static void* _worker_thread_callback(void* arg_this);
 
     AgentTaskRequestPtr _convert_task(const TAgentTaskRequest& task, time_t recv_time) override {
diff --git a/be/src/service/service_be/backend_service.cpp b/be/src/service/service_be/backend_service.cpp
index 015ee591506978..0c37fb48e604b0 100644
--- a/be/src/service/service_be/backend_service.cpp
+++ b/be/src/service/service_be/backend_service.cpp
@@ -36,6 +36,7 @@
 #include "backend_service.h"
 
 #include "agent/agent_server.h"
+#include "agent/task_worker_pool.h"
 #include "storage/storage_engine.h"
 #include "storage/tablet_manager.h"
 
@@ -66,4 +67,16 @@ void BackendService::publish_cluster_state(TAgentResult& result, const TAgentPub
     _agent_server->publish_cluster_state(result, request);
 }
 
+void BackendService::get_tablets_info(TGetTabletsInfoResult& result_, const TGetTabletsInfoRequest& request) {
+    result_.__set_report_version(curr_report_version());
+    result_.__isset.tablets = true;
+    TStatus t_status;
+    Status st_report = StorageEngine::instance()->tablet_manager()->report_all_tablets_info(&result_.tablets);
+    if (!st_report.ok()) {
+        LOG(WARNING) << "Fail to get all tablets info, err=" << st_report.to_string();
+    }
+    st_report.to_thrift(&t_status);
+    result_.status = t_status;
+}
+
 } // namespace starrocks
diff --git a/be/src/service/service_be/backend_service.h b/be/src/service/service_be/backend_service.h
index 4ca18525e6d8e6..ef6d00331a6dfe 100644
--- a/be/src/service/service_be/backend_service.h
+++ b/be/src/service/service_be/backend_service.h
@@ -58,6 +58,8 @@ class BackendService : public BackendServiceBase {
 
     void get_tablet_stat(TTabletStatResult& result) override;
 
+    void get_tablets_info(TGetTabletsInfoResult& result_, const TGetTabletsInfoRequest& request) override;
+
 private:
     AgentServer* _agent_server;
 };
diff --git a/be/test/agent/heartbeat_server_test.cpp b/be/test/agent/heartbeat_server_test.cpp
index a789019daaa894..c34e053b354803 100644
--- a/be/test/agent/heartbeat_server_test.cpp
+++ b/be/test/agent/heartbeat_server_test.cpp
@@ -50,7 +50,7 @@ TEST(HeartbeatServerTest, test_print_master_info_with_token_null) {
             "cluster_id=12345, epoch=100, token=<null>, backend_ip=192.168.1.1, "
             "http_port=<null>, heartbeat_flags=<null>, backend_id=<null>, "
             "min_active_txn_id=0, run_mode=<null>, disabled_disks=<null>, "
-            "decommissioned_disks=<null>, encrypted=<null>)";
+            "decommissioned_disks=<null>, encrypted=<null>, stop_regular_tablet_report=<null>)";
 
     EXPECT_EQ(server.print_master_info(master_info), expected_output);
 }
@@ -71,7 +71,7 @@ TEST(HeartbeatServerTest, test_print_master_info_with_token_hidden) {
             "cluster_id=12345, epoch=100, token=<hidden>, backend_ip=192.168.1.1, "
             "http_port=<null>, heartbeat_flags=<null>, backend_id=<null>, "
             "min_active_txn_id=0, run_mode=<null>, disabled_disks=<null>, "
-            "decommissioned_disks=<null>, encrypted=<null>)";
+            "decommissioned_disks=<null>, encrypted=<null>, stop_regular_tablet_report=<null>)";
 
     EXPECT_EQ(server.print_master_info(master_info), expected_output);
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
index 569333a210e2d2..e2f8bdf932ffa6 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/Config.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
@@ -1493,6 +1493,12 @@ public class Config extends ConfigBase {
     @ConfField
     public static int tablet_stat_update_interval_second = 300;  // 5 min
 
+    @ConfField(mutable = true, comment = "time interval to collect tablet info from backend")
+    public static long tablet_collect_interval_seconds = 60;
+
+    @ConfField(mutable = true, comment = "Timeout for calling BE get_tablets_info rpc")
+    public static int tablet_collect_timeout_seconds = 60;
+
     /**
      * The tryLock timeout configuration of globalStateMgr lock.
      * Normally it does not need to change, unless you need to test something.
diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java
index 94b45f511dcb68..578569e640d679 100644
--- a/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java
+++ b/fe/fe-core/src/main/java/com/starrocks/leader/LeaderImpl.java
@@ -83,7 +83,6 @@
 import com.starrocks.load.DeleteJob;
 import com.starrocks.load.OlapDeleteJob;
 import com.starrocks.load.loadv2.SparkLoadJob;
-import com.starrocks.memory.MemoryUsageTracker;
 import com.starrocks.rpc.ThriftConnectionPool;
 import com.starrocks.rpc.ThriftRPCRequestExecutor;
 import com.starrocks.server.GlobalStateMgr;
@@ -175,13 +174,6 @@
 public class LeaderImpl {
     private static final Logger LOG = LogManager.getLogger(LeaderImpl.class);
 
-    private final ReportHandler reportHandler = new ReportHandler();
-
-    public LeaderImpl() {
-        reportHandler.start();
-        MemoryUsageTracker.registerMemoryTracker("Report", reportHandler);
-    }
-
     public TMasterResult finishTask(TFinishTaskRequest request) {
         // if current node is not master, reject the request
         TMasterResult result = new TMasterResult();
@@ -873,7 +865,7 @@ public TMasterResult report(TReportRequest request) throws TException {
             result.setStatus(status);
             return result;
         }
-        return reportHandler.handleReport(request);
+        return GlobalStateMgr.getCurrentState().getReportHandler().handleReport(request);
     }
 
     private void finishAlterTask(AgentTask task) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java
index 9e97381bfe5377..38b4d94e517200 100644
--- a/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java
+++ b/fe/fe-core/src/main/java/com/starrocks/leader/ReportHandler.java
@@ -67,6 +67,7 @@
 import com.starrocks.catalog.TabletMeta;
 import com.starrocks.clone.TabletChecker;
 import com.starrocks.clone.TabletSchedCtx;
+import com.starrocks.common.CloseableLock;
 import com.starrocks.common.Config;
 import com.starrocks.common.FeConstants;
 import com.starrocks.common.InternalErrorCode;
@@ -79,9 +80,6 @@
 import com.starrocks.common.util.concurrent.lock.Locker;
 import com.starrocks.datacache.DataCacheMetrics;
 import com.starrocks.memory.MemoryTrackable;
-import com.starrocks.metric.GaugeMetric;
-import com.starrocks.metric.Metric.MetricUnit;
-import com.starrocks.metric.MetricRepo;
 import com.starrocks.persist.BackendTabletsInfo;
 import com.starrocks.persist.BatchDeleteReplicaInfo;
 import com.starrocks.persist.ReplicaPersistInfo;
@@ -142,12 +140,14 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 
 public class ReportHandler extends Daemon implements MemoryTrackable {
     @Override
     public List<Pair<List<Object>, Long>> getSamples() {
-        synchronized (pendingTaskMap) {
+        try (CloseableLock ignored = CloseableLock.lock(lock.readLock())) {
             List<Pair<List<Object>, Long>> result = new ArrayList<>();
             for (Map<Long, ReportTask> taskMap : pendingTaskMap.values()) {
                 result.add(Pair.create(taskMap.values()
@@ -162,7 +162,7 @@ public List<Pair<List<Object>, Long>> getSamples() {
 
     @Override
     public Map<String, Long> estimateCount() {
-        synchronized (pendingTaskMap) {
+        try (CloseableLock ignored = CloseableLock.lock(lock.readLock())) {
             long count = 0;
             for (Map<Long, ReportTask> taskMap : pendingTaskMap.values()) {
                 count += taskMap.size();
@@ -193,6 +193,8 @@ public enum ReportType {
 
     private final Map<ReportType, Map<Long, ReportTask>> pendingTaskMap = Maps.newHashMap();
 
+    private final ReadWriteLock lock = new ReentrantReadWriteLock();
+
     /**
      * Record the mapping of <tablet id, backend id> to the to be dropped time of tablet.
      * We will delay the drop of tablet based on configuration `tablet_report_drop_tablet_delay_sec`
@@ -208,14 +210,6 @@ public enum ReportType {
 
     public ReportHandler() {
         super("ReportHandler");
-        GaugeMetric<Long> gaugeQueueSize = new GaugeMetric<Long>(
-                "report_queue_size", MetricUnit.NOUNIT, "report queue size") {
-            @Override
-            public Long getValue() {
-                return (long) reportQueue.size();
-            }
-        };
-        MetricRepo.addMetric(gaugeQueueSize);
         pendingTaskMap.put(ReportType.TABLET_REPORT, Maps.newHashMap());
         pendingTaskMap.put(ReportType.DISK_REPORT, Maps.newHashMap());
         pendingTaskMap.put(ReportType.TASK_REPORT, Maps.newHashMap());
@@ -373,7 +367,7 @@ private void buildErrorResult(TStatus tStatus, String msg) {
     }
 
     private void putToQueue(ReportTask reportTask) throws Exception {
-        synchronized (pendingTaskMap) {
+        try (CloseableLock ignored = CloseableLock.lock(lock.writeLock())) {
             if (!pendingTaskMap.containsKey(reportTask.type)) {
                 throw new Exception("Unknown report task type" + reportTask.toString());
             }
@@ -427,6 +421,13 @@ public ReportTask(long beId, ReportType type, Map<TTaskType, Set<Long>> tasks,
             this.dataCacheMetrics = dataCacheMetrics;
         }
 
+        public ReportTask(long beId, ReportType type, Map<Long, TTablet> tablets, long reportVersion) {
+            this.beId = beId;
+            this.type = type;
+            this.tablets = tablets;
+            this.reportVersion = reportVersion;
+        }
+
         @Override
         protected void exec() {
             if (tasks != null) {
@@ -2184,13 +2185,28 @@ private static void addReplica(long tabletId, TTabletInfo backendTabletInfo, lon
         }
     }
 
+    public int getPendingTabletReportTaskCnt() {
+        try (CloseableLock ignored = CloseableLock.lock(lock.readLock())) {
+            Map<Long, ReportTask> tasks = pendingTaskMap.get(ReportType.TABLET_REPORT);
+            return tasks == null ? 0 : tasks.size();
+        }
+    }
+
+    public void putTabletReportTask(long beId, long reportVersion, Map<Long, TTablet> tablets) throws Exception {
+        putToQueue(new ReportTask(beId, ReportType.TABLET_REPORT, tablets, reportVersion));
+    }
+
+    public int getReportQueueSize() {
+        return reportQueue.size();
+    }
+
     @Override
     protected void runOneCycle() {
         while (true) {
             try {
                 Pair<Long, ReportType> pair = reportQueue.take();
                 ReportTask task = null;
-                synchronized (pendingTaskMap) {
+                try (CloseableLock ignored = CloseableLock.lock(lock.writeLock())) {
                     // using the lastest task
                     task = pendingTaskMap.get(pair.second).get(pair.first);
                     if (task == null) {
diff --git a/fe/fe-core/src/main/java/com/starrocks/leader/TabletCollector.java b/fe/fe-core/src/main/java/com/starrocks/leader/TabletCollector.java
new file mode 100644
index 00000000000000..458a99b1c1204c
--- /dev/null
+++ b/fe/fe-core/src/main/java/com/starrocks/leader/TabletCollector.java
@@ -0,0 +1,146 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.leader;
+
+import com.starrocks.common.Config;
+import com.starrocks.common.util.FrontendDaemon;
+import com.starrocks.rpc.ThriftConnectionPool;
+import com.starrocks.rpc.ThriftRPCRequestExecutor;
+import com.starrocks.server.GlobalStateMgr;
+import com.starrocks.server.RunMode;
+import com.starrocks.system.Backend;
+import com.starrocks.thrift.TGetTabletsInfoRequest;
+import com.starrocks.thrift.TGetTabletsInfoResult;
+import com.starrocks.thrift.TNetworkAddress;
+import com.starrocks.thrift.TStatusCode;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+public class TabletCollector extends FrontendDaemon {
+    private static final Logger LOG = LogManager.getLogger(TabletCollector.class);
+    private static final long CHECK_INTERVAL_MS = 100;
+
+    private final PriorityQueue<CollectStat> collectQueue;
+    private final Set<Long> queuedBeIds;
+
+    public TabletCollector() {
+        super("TabletCollector", CHECK_INTERVAL_MS);
+        collectQueue = new PriorityQueue<>();
+        queuedBeIds = new HashSet<>();
+    }
+
+    @Override
+    protected void runAfterCatalogReady() {
+        if (RunMode.isSharedDataMode()) {
+            return;
+        }
+
+        updateQueue();
+
+        collect(collectQueue.poll());
+    }
+
+    private void updateQueue() {
+        List<Backend> backends = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackends();
+        for (Backend backend : backends) {
+            if (backend.isAlive() && !queuedBeIds.contains(backend.getId())) {
+                queuedBeIds.add(backend.getId());
+                collectQueue.add(new CollectStat(backend.getId(), -1L));
+            }
+        }
+    }
+
+    private void collect(CollectStat collectStat) {
+        if (collectStat == null) {
+            return;
+        }
+
+        // 1. If there are more than 1 pending report task in ReportHandler
+        // or
+        // 2. The time since the last collection is less than Config.tablet_collect_interval_seconds.
+        // return back the stat to collectQueue and do nothing.
+        if (GlobalStateMgr.getCurrentState().getReportHandler().getPendingTabletReportTaskCnt() > 1
+                || System.currentTimeMillis() - collectStat.lastCollectTime < Config.tablet_collect_interval_seconds * 1000) {
+            collectQueue.add(collectStat);
+            return;
+        }
+
+        // If backend is invalid, remove this backend from queue.
+        // If resumed, the backend will be added back to the queue via updateQueue().
+        Backend backend = GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo().getBackend(collectStat.beId);
+        if (backend == null || !backend.isAlive()) {
+            queuedBeIds.remove(collectStat.beId);
+            return;
+        }
+
+        try {
+            long startMs = System.currentTimeMillis();
+            TGetTabletsInfoResult result = ThriftRPCRequestExecutor.call(
+                    ThriftConnectionPool.backendPool,
+                    new TNetworkAddress(backend.getHost(), backend.getBePort()),
+                    Config.tablet_collect_timeout_seconds * 1000,
+                    2,
+                    client -> client.get_tablets_info(new TGetTabletsInfoRequest()));
+
+            if (result.getStatus().getStatus_code() == TStatusCode.OK) {
+                GlobalStateMgr.getCurrentState().getReportHandler()
+                        .putTabletReportTask(backend.getId(), result.getReport_version(), result.getTablets());
+                LOG.debug("collect tablet from backend {} successfully, time used: {}ms", backend.getId(),
+                        System.currentTimeMillis() - startMs);
+            } else {
+                String errMsg = "";
+                if (result.getStatus().getError_msgs() != null) {
+                    errMsg = String.join(",", result.getStatus().getError_msgs());
+                }
+                LOG.warn("collect tablet from backend {} failed, error: {}", backend.getId(), errMsg);
+            }
+        } catch (Exception e) {
+            LOG.warn("collect tablets from backend {} failed", backend.getId(), e);
+        }
+
+        // Regardless of whether the collection succeeds or fails, lastCollectTime must be updated,
+        // otherwise it will block the collection of other backends.
+        collectStat.lastCollectTime = System.currentTimeMillis();
+        collectQueue.add(collectStat);
+    }
+
+    public static class CollectStat implements Comparable<CollectStat> {
+        long beId;
+        long lastCollectTime;
+
+        CollectStat(long beId, long lastCollectTime) {
+            this.beId = beId;
+            this.lastCollectTime = lastCollectTime;
+        }
+
+        public long getBeId() {
+            return beId;
+        }
+
+        public long getLastCollectTime() {
+            return lastCollectTime;
+        }
+
+        @Override
+        public int compareTo(CollectStat other) {
+            return Long.compare(lastCollectTime, other.lastCollectTime);
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/com/starrocks/memory/MemoryUsageTracker.java b/fe/fe-core/src/main/java/com/starrocks/memory/MemoryUsageTracker.java
index ebe78256230346..58a67b0b140189 100644
--- a/fe/fe-core/src/main/java/com/starrocks/memory/MemoryUsageTracker.java
+++ b/fe/fe-core/src/main/java/com/starrocks/memory/MemoryUsageTracker.java
@@ -70,6 +70,7 @@ private void initMemoryTracker() {
         registerMemoryTracker("Task", currentState.getTaskManager().getTaskRunManager());
         registerMemoryTracker("TabletInvertedIndex", currentState.getTabletInvertedIndex());
         registerMemoryTracker("LocalMetastore", currentState.getLocalMetastore());
+        registerMemoryTracker("Report", currentState.getReportHandler());
 
         // MV
         registerMemoryTracker("MV", currentState.getMaterializedViewMgr().getMvTimelinessMgr());
diff --git a/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java b/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java
index 3894a95f16c3c0..a9bd01f675c9ab 100644
--- a/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/com/starrocks/metric/MetricRepo.java
@@ -414,6 +414,15 @@ public Long getValue() {
         GAUGE_SAFE_MODE.setValue(0);
         STARROCKS_METRIC_REGISTER.addMetric(GAUGE_SAFE_MODE);
 
+        GaugeMetric<Long> gaugeReportQueueSize = new GaugeMetric<Long>(
+                "report_queue_size", MetricUnit.NOUNIT, "report queue size") {
+            @Override
+            public Long getValue() {
+                return (long) GlobalStateMgr.getCurrentState().getReportHandler().getReportQueueSize();
+            }
+        };
+        STARROCKS_METRIC_REGISTER.addMetric(gaugeReportQueueSize);
+
         // 2. counter
         COUNTER_REQUEST_ALL = new LongCounterMetric("request_total", MetricUnit.REQUESTS, "total request");
         STARROCKS_METRIC_REGISTER.addMetric(COUNTER_REQUEST_ALL);
diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
index d83906d159e229..7bc83ce54764b2 100644
--- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java
@@ -144,6 +144,8 @@
 import com.starrocks.lake.snapshot.ClusterSnapshotMgr;
 import com.starrocks.lake.vacuum.AutovacuumDaemon;
 import com.starrocks.leader.CheckpointController;
+import com.starrocks.leader.ReportHandler;
+import com.starrocks.leader.TabletCollector;
 import com.starrocks.leader.TaskRunStateSynchronizer;
 import com.starrocks.listener.GlobalLoadJobListenerBus;
 import com.starrocks.load.DeleteMgr;
@@ -524,6 +526,8 @@ public class GlobalStateMgr {
     private final ClusterSnapshotMgr clusterSnapshotMgr;
 
     private final SqlBlackList sqlBlackList;
+    private final ReportHandler reportHandler;
+    private final TabletCollector tabletCollector;
 
     public NodeMgr getNodeMgr() {
         return nodeMgr;
@@ -830,6 +834,9 @@ public void transferToNonLeader(FrontendNodeType newType) {
                         "query-deploy", true);
 
         this.warehouseIdleChecker = new WarehouseIdleChecker();
+
+        this.reportHandler = new ReportHandler();
+        this.tabletCollector = new TabletCollector();
     }
 
     public static void destroyCheckpoint() {
@@ -1442,6 +1449,9 @@ private void startLeaderOnlyDaemonThreads() {
         temporaryTableCleaner.start();
 
         connectorTableTriggerAnalyzeMgr.start();
+
+        reportHandler.start();
+        tabletCollector.start();
     }
 
     // start threads that should run on all FE
@@ -2718,4 +2728,8 @@ public void shutdown() {
         // in a single thread.
         connectorMgr.shutdown();
     }
+
+    public ReportHandler getReportHandler() {
+        return reportHandler;
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/com/starrocks/system/HeartbeatMgr.java
index e155d440e850ae..4f0b44071764e5 100644
--- a/fe/fe-core/src/main/java/com/starrocks/system/HeartbeatMgr.java
+++ b/fe/fe-core/src/main/java/com/starrocks/system/HeartbeatMgr.java
@@ -273,6 +273,7 @@ public HeartbeatResponse call() {
                 copiedMasterInfo.setMin_active_txn_id(
                         GlobalStateMgr.getCurrentState().getGlobalTransactionMgr().getMinActiveTxnId());
                 copiedMasterInfo.setRun_mode(RunMode.toTRunMode(RunMode.getCurrentRunMode()));
+                copiedMasterInfo.setStop_regular_tablet_report(true);
                 if (computeNode instanceof Backend) {
                     copiedMasterInfo.setDisabled_disks(((Backend) computeNode).getDisabledDisks());
                     copiedMasterInfo.setDecommissioned_disks(((Backend) computeNode).getDecommissionedDisks());
diff --git a/fe/fe-core/src/test/java/com/starrocks/common/GenericPoolTest.java b/fe/fe-core/src/test/java/com/starrocks/common/GenericPoolTest.java
index 93211eb270afad..f53c2e46a5f064 100644
--- a/fe/fe-core/src/test/java/com/starrocks/common/GenericPoolTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/common/GenericPoolTest.java
@@ -32,6 +32,8 @@
 import com.starrocks.thrift.TExportTaskRequest;
 import com.starrocks.thrift.TFetchDataParams;
 import com.starrocks.thrift.TFetchDataResult;
+import com.starrocks.thrift.TGetTabletsInfoRequest;
+import com.starrocks.thrift.TGetTabletsInfoResult;
 import com.starrocks.thrift.TMiniLoadEtlStatusRequest;
 import com.starrocks.thrift.TMiniLoadEtlStatusResult;
 import com.starrocks.thrift.TMiniLoadEtlTaskRequest;
@@ -199,6 +201,11 @@ public TTabletStatResult get_tablet_stat() throws TException {
             return null;
         }
 
+        @Override
+        public TGetTabletsInfoResult get_tablets_info(TGetTabletsInfoRequest request) throws TException {
+            return null;
+        }
+
         @Override
         public TStatus submit_routine_load_task(List<TRoutineLoadTask> tasks) throws TException {
             // TODO Auto-generated method stub
diff --git a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java
index 69a64f39e7359a..3a893913850db4 100644
--- a/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java
+++ b/fe/fe-core/src/test/java/com/starrocks/leader/ReportHandlerTest.java
@@ -469,4 +469,16 @@ public void testTabletDropDelay() throws InterruptedException {
         ready = ReportHandler.checkReadyToBeDropped(tabletId, backendId);
         Assert.assertTrue(ready);
     }
+
+    @Test
+    public void testGetPendingTabletReportTaskCnt() throws Exception {
+        ReportHandler reportHandler = new ReportHandler();
+        Assert.assertEquals(0, reportHandler.getPendingTabletReportTaskCnt());
+        reportHandler.putTabletReportTask(1L, 1L, new HashMap<>());
+        Assert.assertEquals(1, reportHandler.getPendingTabletReportTaskCnt());
+        reportHandler.putTabletReportTask(1L, 1L, new HashMap<>());
+        Assert.assertEquals(1, reportHandler.getPendingTabletReportTaskCnt());
+        reportHandler.putTabletReportTask(2L, 1L, new HashMap<>());
+        Assert.assertEquals(2, reportHandler.getPendingTabletReportTaskCnt());
+    }
 }
\ No newline at end of file
diff --git a/fe/fe-core/src/test/java/com/starrocks/leader/TabletCollectorTest.java b/fe/fe-core/src/test/java/com/starrocks/leader/TabletCollectorTest.java
new file mode 100644
index 00000000000000..b2171116a6fe07
--- /dev/null
+++ b/fe/fe-core/src/test/java/com/starrocks/leader/TabletCollectorTest.java
@@ -0,0 +1,35 @@
+// Copyright 2021-present StarRocks, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.starrocks.leader;
+
+import com.starrocks.leader.TabletCollector.CollectStat;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.PriorityQueue;
+
+public class TabletCollectorTest {
+
+    @Test
+    public void testCollectStat() {
+        PriorityQueue<CollectStat> queue = new PriorityQueue();
+        queue.add(new CollectStat(1L, 1L));
+        queue.add(new CollectStat(2L, 2L));
+        queue.add(new CollectStat(3L, 3L));
+        Assert.assertEquals(1L, queue.poll().lastCollectTime);
+        Assert.assertEquals(2L, queue.poll().lastCollectTime);
+        Assert.assertEquals(3L, queue.poll().lastCollectTime);
+    }
+}
diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift
index 37c60fb1fdb0ea..2274c2beba76bd 100644
--- a/gensrc/thrift/BackendService.thrift
+++ b/gensrc/thrift/BackendService.thrift
@@ -42,6 +42,7 @@ include "AgentService.thrift"
 include "InternalService.thrift"
 include "StarrocksExternalService.thrift"
 include "MVMaintenance.thrift"
+include "MasterService.thrift"
 
 struct TExportTaskRequest {
     1: required InternalService.TExecPlanFragmentParams params
@@ -116,6 +117,15 @@ struct TStreamLoadChannel {
     2: optional i32 channel_id
 }
 
+struct TGetTabletsInfoRequest {
+}
+
+struct TGetTabletsInfoResult {
+    1: required Status.TStatus status;
+    2: optional i64 report_version;
+    3: optional map<Types.TTabletId, MasterService.TTablet> tablets;
+}
+
 service BackendService {
     // Called by coord to start asynchronous execution of plan fragment in backend.
     // Returns as soon as all incoming data streams have been set up.
@@ -171,4 +181,5 @@ service BackendService {
     // release the context resource associated with the context_id
     StarrocksExternalService.TScanCloseResult close_scanner(1: StarrocksExternalService.TScanCloseParams params);
 
+    TGetTabletsInfoResult get_tablets_info(1: TGetTabletsInfoRequest request);
 }
diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift
index 2d7b671a544bdd..2b9e066a54c0fb 100644
--- a/gensrc/thrift/HeartbeatService.thrift
+++ b/gensrc/thrift/HeartbeatService.thrift
@@ -34,6 +34,7 @@ struct TMasterInfo {
     11: optional list<string> disabled_disks
     12: optional list<string> decommissioned_disks
     13: optional bool encrypted;
+    14: optional bool stop_regular_tablet_report;
 }
 
 struct TBackendInfo {

From cd578d07a3967c9d0a9a079ed8dcb8b23a862dbe Mon Sep 17 00:00:00 2001
From: gengjun-git <gengjun@starrocks.com>
Date: Wed, 22 Jan 2025 14:21:17 +0800
Subject: [PATCH 71/71] add comment

Signed-off-by: gengjun-git <gengjun@starrocks.com>
---
 gensrc/thrift/HeartbeatService.thrift | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift
index 2b9e066a54c0fb..bc4df3f9f1b393 100644
--- a/gensrc/thrift/HeartbeatService.thrift
+++ b/gensrc/thrift/HeartbeatService.thrift
@@ -34,7 +34,7 @@ struct TMasterInfo {
     11: optional list<string> disabled_disks
     12: optional list<string> decommissioned_disks
     13: optional bool encrypted;
-    14: optional bool stop_regular_tablet_report;
+    14: optional bool stop_regular_tablet_report; // used for upgrade/downgrade compatibility, can be removed after 3.5
 }
 
 struct TBackendInfo {