[Bug](materialized-view) adjust limit for create materialized view on…

… uniq/agg table (#21580)

adjust limit for create materialized view on uniq/agg table

fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java

@@ -21,18 +21,21 @@
 import org.apache.doris.analysis.AlterClause;
 import org.apache.doris.analysis.CancelAlterTableStmt;
 import org.apache.doris.analysis.CancelStmt;
+import org.apache.doris.analysis.CastExpr;
 import org.apache.doris.analysis.CreateMaterializedViewStmt;
 import org.apache.doris.analysis.CreateMultiTableMaterializedViewStmt;
 import org.apache.doris.analysis.DropMaterializedViewStmt;
 import org.apache.doris.analysis.DropRollupClause;
 import org.apache.doris.analysis.MVColumnItem;
+import org.apache.doris.analysis.SlotRef;
 import org.apache.doris.catalog.AggregateType;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.KeysType;
 import org.apache.doris.catalog.MaterializedIndex;
 import org.apache.doris.catalog.MaterializedIndex.IndexState;
+import org.apache.doris.catalog.MaterializedIndexMeta;
 import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.OlapTable.OlapTableState;
@@ -463,109 +466,117 @@ private List<Column> checkAndPrepareMaterializedView(CreateMaterializedViewStmt
             throw new DdlException("MergeOnWrite table can't create materialized view.");
         }
         // check if mv columns are valid
-        // a. Aggregate or Unique table:
-        //     1. For aggregate table, mv columns with aggregate function should be same as base schema
-        //     2. For aggregate table, the column which is the key of base table should be the key of mv as well.
-        // b. Duplicate table:
-        //     1. Columns resolved by semantics are legal
+        // a. Aggregate table:
+        // 1. all slot's aggregationType must same with value mv column
+        // 2. all slot's isKey must same with mv column
+        // 3. value column'define expr must be slot (except all slot belong replace family)
+        // b. Unique table:
+        // 1. mv must not contain group expr
+        // 2. all slot's isKey same with mv column
+        // c. Duplicate table:
+        // 1. Columns resolved by semantics are legal
+
         // update mv columns
         List<MVColumnItem> mvColumnItemList = addMVClause.getMVColumnItemList();
         List<Column> newMVColumns = Lists.newArrayList();
-        int numOfKeys = 0;
+
         if (olapTable.getKeysType().isAggregationFamily()) {
-            if (addMVClause.getMVKeysType() != KeysType.AGG_KEYS) {
-                throw new DdlException("The materialized view of aggregation"
-                        + " or unique table must has grouping columns");
+            if (!addMVClause.isReplay() && olapTable.getKeysType() == KeysType.AGG_KEYS
+                    && addMVClause.getMVKeysType() != KeysType.AGG_KEYS) {
+                throw new DdlException("The materialized view of aggregation table must has grouping columns");
+            }
+            if (!addMVClause.isReplay() && olapTable.getKeysType() == KeysType.UNIQUE_KEYS
+                    && addMVClause.getMVKeysType() == KeysType.AGG_KEYS) {
+                // check b.1
+                throw new DdlException("The materialized view of unique table must not has grouping columns");
             }
+            addMVClause.setMVKeysType(olapTable.getKeysType());
+
             for (MVColumnItem mvColumnItem : mvColumnItemList) {
-                if (mvColumnItem.getBaseColumnNames().size() != 1) {
-                    throw new DdlException(
-                            "mvColumnItem.getBaseColumnNames().size() != 1, mvColumnItem.getBaseColumnNames().size() = "
-                                    + mvColumnItem.getBaseColumnNames().size());
+                if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS && !mvColumnItem.isKey()) {
+                    mvColumnItem.setAggregationType(AggregateType.REPLACE, true);
                 }
 
-                String mvColumnName = mvColumnItem.getBaseColumnNames().iterator().next();
-                Column mvColumn = mvColumnItem.toMVColumn(olapTable);
-                if (mvColumnItem.isKey()) {
-                    ++numOfKeys;
+                if (olapTable.getKeysType() == KeysType.UNIQUE_KEYS) {
+                    for (String slotName : mvColumnItem.getBaseColumnNames()) {
+                        if (!addMVClause.isReplay()
+                                && olapTable
+                                        .getColumn(MaterializedIndexMeta
+                                                .normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName)))
+                                        .isKey()) {
+                            mvColumnItem.setIsKey(true);
+                        }
+                    }
                 }
 
-                AggregateType baseAggregationType = mvColumn.getAggregationType();
-                AggregateType mvAggregationType = mvColumnItem.getAggregationType();
-                if (mvColumn.isKey() && !mvColumnItem.isKey()) {
-                    throw new DdlException("The column[" + mvColumnName + "] must be the key of materialized view");
-                }
-                if (baseAggregationType != mvAggregationType) {
-                    throw new DdlException(
-                            "The aggregation type of column[" + mvColumnName + "] must be same as the aggregate "
-                                    + "type of base column in aggregate table");
-                }
-                if (baseAggregationType != null && baseAggregationType.isReplaceFamily() && olapTable
-                        .getKeysNum() != numOfKeys) {
-                    throw new DdlException(
-                            "The materialized view should contain all keys of base table if there is a" + " REPLACE "
-                                    + "value");
-                }
-                newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
-            }
-            // check useless rollup of same key columns and same order with base table
-            if (numOfKeys == olapTable.getBaseSchemaKeyColumns().size() && !addMVClause.isReplay()) {
-                boolean allKeysMatch = true;
-                for (int i = 0; i < numOfKeys; i++) {
-                    if (!CreateMaterializedViewStmt.mvColumnBreaker(newMVColumns.get(i).getName())
-                            .equalsIgnoreCase(olapTable.getBaseSchemaKeyColumns().get(i).getName())) {
-                        allKeysMatch = false;
-                        break;
+                // check a.2 and b.2
+                for (String slotName : mvColumnItem.getBaseColumnNames()) {
+                    if (!addMVClause.isReplay() && olapTable
+                            .getColumn(MaterializedIndexMeta
+                                    .normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName)))
+                            .isKey() != mvColumnItem.isKey()) {
+                        throw new DdlException("The mvItem[" + mvColumnItem.getName()
+                                + "]'s isKey must same with all slot, mvItem.isKey="
+                                + (mvColumnItem.isKey() ? "true" : "false"));
                     }
                 }
-                if (allKeysMatch) {
-                    throw new DdlException("MV contains all keys in base table with same order for "
-                            + "aggregation or unique table is useless.");
+
+                if (!addMVClause.isReplay() && !mvColumnItem.isKey() && olapTable.getKeysType() == KeysType.AGG_KEYS) {
+                    // check a.1
+                    for (String slotName : mvColumnItem.getBaseColumnNames()) {
+                        if (olapTable
+                                .getColumn(MaterializedIndexMeta
+                                        .normalizeName(CreateMaterializedViewStmt.mvColumnBreaker(slotName)))
+                                .getAggregationType() != mvColumnItem.getAggregationType()) {
+                            throw new DdlException("The mvItem[" + mvColumnItem.getName()
+                                    + "]'s AggregationType must same with all slot");
+                        }
+                    }
+
+                    // check a.3
+                    if (!mvColumnItem.getAggregationType().isReplaceFamily()
+                            && !(mvColumnItem.getDefineExpr() instanceof SlotRef)
+                            && !((mvColumnItem.getDefineExpr() instanceof CastExpr)
+                                    && mvColumnItem.getDefineExpr().getChild(0) instanceof SlotRef)) {
+                        throw new DdlException(
+                                "The mvItem[" + mvColumnItem.getName() + "] require slot because it is value column");
+                    }
                 }
+                newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
             }
         } else {
             Set<String> partitionOrDistributedColumnName = olapTable.getPartitionColumnNames();
             partitionOrDistributedColumnName.addAll(olapTable.getDistributionColumnNames());
-            boolean hasNewColumn = false;
             for (MVColumnItem mvColumnItem : mvColumnItemList) {
                 Set<String> names = mvColumnItem.getBaseColumnNames();
                 if (names == null) {
                     throw new DdlException("Base columns is null");
                 }
                 for (String str : names) {
-                    if (partitionOrDistributedColumnName.contains(str)
-                            && mvColumnItem.getAggregationType() != null) {
-                        throw new DdlException("The partition and distributed columns " + str
-                                + " must be key column in mv");
+                    if (partitionOrDistributedColumnName.contains(str) && mvColumnItem.getAggregationType() != null) {
+                        throw new DdlException(
+                                "The partition and distributed columns " + str + " must be key column in mv");
                     }
                 }
 
                 newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
-                if (mvColumnItem.isKey()) {
-                    ++numOfKeys;
-                }
-                if (olapTable
-                        .getBaseColumn(CreateMaterializedViewStmt.mvColumnBreaker(mvColumnItem.getName())) == null) {
-                    hasNewColumn = true;
-                }
             }
-            // check useless rollup of same key columns and same order with base table
-            if (!addMVClause.isReplay() && addMVClause.getMVKeysType() == KeysType.DUP_KEYS && !hasNewColumn) {
-                boolean allKeysMatch = true;
-                for (int i = 0; i < numOfKeys; i++) {
-                    if (!CreateMaterializedViewStmt.mvColumnBreaker(newMVColumns.get(i).getName())
-                            .equalsIgnoreCase(olapTable.getBaseSchema().get(i).getName())
-                            && olapTable.getBaseSchema().get(i).isKey()) {
-                        allKeysMatch = false;
-                        break;
-                    }
-                }
-                if (allKeysMatch && !olapTable.isDuplicateWithoutKey()) {
-                    throw new DdlException("MV contain the columns of the base table in prefix order for "
-                            + "duplicate table is useless.");
+        }
+
+        if (newMVColumns.size() == olapTable.getBaseSchema().size() && !addMVClause.isReplay()) {
+            boolean allKeysMatch = true;
+            for (int i = 0; i < newMVColumns.size(); i++) {
+                if (!CreateMaterializedViewStmt.mvColumnBreaker(newMVColumns.get(i).getName())
+                        .equalsIgnoreCase(olapTable.getBaseSchema().get(i).getName())) {
+                    allKeysMatch = false;
+                    break;
                 }
             }
+            if (allKeysMatch) {
+                throw new DdlException("MV same with base table is useless.");
+            }
         }
+
         if (KeysType.UNIQUE_KEYS == olapTable.getKeysType() && olapTable.hasDeleteSign()) {
             newMVColumns.add(new Column(olapTable.getDeleteSignColumn()));
         }

fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java

@@ -139,6 +139,10 @@ public String getDBName() {
         return dbName;
     }
 
+    public void setMVKeysType(KeysType type) {
+        mvKeysType = type;
+    }
+
     public KeysType getMVKeysType() {
         return mvKeysType;
     }

fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java

@@ -179,10 +179,6 @@ public Column toMVColumn(OlapTable olapTable) throws DdlException {
                 throw new DdlException("base column's type is null, column=" + result.getName());
             }
             result.setIsKey(isKey);
-            // If the mv column type is inconsistent with the base column type, the daily
-            // test will core.
-            // So, I comment this line firstly.
-            // result.setType(type);
         } else {
             if (type == null) {
                 throw new DdlException("MVColumnItem type is null");

fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java

@@ -753,8 +753,7 @@ public void testUniqueTableInQuery() throws Exception {
         String uniqueTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, k2 int, v1 int) UNIQUE KEY (k1, k2) "
                 + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1','enable_unique_key_merge_on_write' = 'false');";
         createTable(uniqueTable);
-        String createK1MV = "create materialized view only_k1 as select k2 from " + TEST_TABLE_NAME + " group by "
-                + "k2;";
+        String createK1MV = "create materialized view only_k1 as select k2 from " + TEST_TABLE_NAME;
         createMv(createK1MV);
         String query = "select * from " + TEST_TABLE_NAME + ";";
         singleTableTest(query, TEST_TABLE_NAME, false);

fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java

@@ -86,7 +86,6 @@ public void runBeforeAll() throws Exception {
                 + "properties(\"replication_num\" = \"1\")");
 
         createMv("create materialized view k12s3m as select k1,sum(k2),max(k2) from agg_have_dup_base group by k1");
-        createMv("create materialized view mv2 as select k1, k2, k3 from agg_have_dup_base group by k1, k2, k3");
         createMv("create materialized view mv3 as select k1, k2 + k3 from agg_have_dup_base group by k1, k2 + k3");
     }
 
@@ -119,9 +118,9 @@ public void testAnalysisException() {
     @Test
     public void testWithMV() throws Exception {
         String sql = "explain insert into agg_have_dup_base select -4, -4, -4, 'd'";
-        Assertions.assertEquals(10, getOutputFragment(sql).getOutputExprs().size());
+        Assertions.assertEquals(8, getOutputFragment(sql).getOutputExprs().size());
         String sql1 = "explain insert into agg_have_dup_base select -4, k2, -4, 'd' from agg_have_dup_base";
-        Assertions.assertEquals(10, getOutputFragment(sql1).getOutputExprs().size());
+        Assertions.assertEquals(8, getOutputFragment(sql1).getOutputExprs().size());
     }
 
     private PlanFragment getOutputFragment(String sql) throws Exception {

fe/fe-core/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java

@@ -666,8 +666,7 @@ public void testUniqueTableInQuery() throws Exception {
         String uniqueTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, k2 int, v1 int) UNIQUE KEY (k1, k2) "
                 + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1', 'enable_unique_key_merge_on_write' = 'false');";
         dorisAssert.withTable(uniqueTable);
-        String createK1K2MV = "create materialized view only_k1 as select k2, k1 from " + TEST_TABLE_NAME + " group by "
-                + "k2, k1;";
+        String createK1K2MV = "create materialized view only_k1 as select k2, k1 from " + TEST_TABLE_NAME;
         String query = "select * from " + TEST_TABLE_NAME + ";";
         dorisAssert.withMaterializedView(createK1K2MV).query(query).explainContains(TEST_TABLE_NAME);
         dorisAssert.dropTable(TEST_TABLE_NAME, true);

regression-test/data/mv_p0/unique/unique.out

@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_star --
+1	1	1	a
+2	2	2	b
+3	-3	\N	c
+

regression-test/suites/mv_p0/test_mv_useless/test_agg_mv_useless.groovy

@@ -52,11 +52,6 @@ suite ("test_agg_mv_useless") {
         exception "errCode = 2,"
     }
 
-    test {
-        sql "create materialized view k1_k2_u12 as select k1,k2 from ${testTable} group by k1,k2;"
-        exception "errCode = 2,"
-    }
-
     createMV("create materialized view k1_u1 as select k1 from ${testTable} group by k1;")
     createMV("create materialized view k1_k2_u21 as select k2,k1 from ${testTable} group by k2,k1 order by k2,k1;")
     createMV("create materialized view k1_sumk3 as select k1,sum(k3) from ${testTable} group by k1;")

regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy

@@ -38,14 +38,8 @@ suite ("test_dup_mv_useless") {
     sql "insert into ${testTable} select 2,2,2;"
     sql "insert into ${testTable} select 3,3,3;"
 
-    def result = "null"
     test {
-        sql "create materialized view k1 as select k1 from ${testTable};"
-        exception "errCode = 2,"
-    }
-
-    test {
-        sql "create materialized view k1_k2 as select k1,k2 from ${testTable};"
+        sql "create materialized view k1_k2_k3 as select k1,k2,k3 from ${testTable};"
         exception "errCode = 2,"
     }
 

regression-test/suites/mv_p0/test_mv_useless/test_uniq_mv_useless.groovy

@@ -38,16 +38,6 @@ suite ("test_uniq_mv_useless") {
     sql "insert into ${testTable} select 2,2,2;"
     sql "insert into ${testTable} select 3,3,3;"
 
-    test {
-        sql "create materialized view k1 as select k1 from ${testTable};"
-        exception "errCode = 2,"
-    }
-
-    test {
-        sql "create materialized view k1_k2 as select k1,k2 from ${testTable};"
-        exception "errCode = 2,"
-    }
-
     test {
         sql "create materialized view k1_k2_u12 as select k1,k2 from ${testTable} group by k1,k2;"
         exception "errCode = 2,"

regression-test/suites/mv_p0/unique/unique.groovy

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("unique") {
+
+    sql """ DROP TABLE IF EXISTS u_table; """
+
+    sql """
+            create table u_table(
+                k1 int null,
+                k2 int not null,
+                k3 bigint null,
+                k4 varchar(100) null
+            )
+            unique key (k1,k2,k3)
+            distributed BY hash(k1) buckets 3
+            properties("replication_num" = "1");
+        """
+
+    sql "insert into u_table select 1,1,1,'a';"
+    sql "insert into u_table select 2,2,2,'b';"
+    sql "insert into u_table select 3,-3,null,'c';"
+
+    test {
+        sql """create materialized view k12s3m as select k1,sum(k2),max(k2) from u_table group by k1;"""
+        exception "must not has grouping columns"
+    }
+    test {
+        sql """create materialized view kadj as select k4 from u_table"""
+        exception "must same with all slot"
+    }
+
+    createMV("create materialized view kadj as select k3,k2,k1,k4 from u_table;")
+    createMV("create materialized view k1l4 as select k1,length(k4) from u_table;")
+
+    test {
+        sql """create materialized view kadp as select k4 from u_table group by k4;"""
+        exception "must not has grouping columns"
+    }
+
+    qt_select_star "select * from u_table order by k1;"
+
+    // todo: support match query
+}