From 6ef2e62600de9a8e9ab336940af8b2ba7bc4d818 Mon Sep 17 00:00:00 2001 From: amory Date: Fri, 3 Nov 2023 22:55:39 +0800 Subject: [PATCH] [FIX](struct) fix struct be nested when read will make core (#26270) --- .../olap/rowset/segment_v2/column_reader.cpp | 1 + .../org/apache/doris/catalog/ArrayType.java | 10 ++++ .../org/apache/doris/catalog/MapType.java | 15 +++++ .../org/apache/doris/catalog/StructType.java | 24 ++++++++ .../java/org/apache/doris/catalog/Type.java | 27 ++++----- .../test_nestedtypes_insert_into_select.out | 7 +++ ...test_nestedtypes_insert_into_select.groovy | 58 +++++++++++++++++++ 7 files changed, 125 insertions(+), 17 deletions(-) create mode 100644 regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.out create mode 100644 regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index dd488e6e0acb26..3c02181573aa5c 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -88,6 +88,7 @@ Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB& case FieldType::OLAP_FIELD_TYPE_STRUCT: { // not support empty struct DCHECK(meta.children_columns_size() >= 1); + num_rows = meta.children_columns(0).num_rows(); // create struct column reader std::unique_ptr struct_reader( new ColumnReader(opts, meta, num_rows, file_reader)); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java index b88fd7d1321e27..a2915fb4b495e4 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -163,6 +163,16 @@ public static boolean canCastTo(ArrayType type, ArrayType targetType) { return Type.canCastTo(type.getItemType(), targetType.getItemType()); } + public static Type getAssignmentCompatibleType(ArrayType t1, ArrayType t2, boolean strict) { + Type itemCompatibleType = Type.getAssignmentCompatibleType(t1.getItemType(), t2.getItemType(), strict); + + if (itemCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + + return new ArrayType(itemCompatibleType, t1.getContainsNull() || t2.getContainsNull()); + } + @Override public void toThrift(TTypeDesc container) { TTypeNode node = new TTypeNode(); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 691c4d7e04db68..06dba8c2fb0219 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -198,6 +198,21 @@ public static boolean canCastTo(MapType type, MapType targetType) { || targetType.getValueType().isStringType() && type.getValueType().isStringType()); } + public static Type getAssignmentCompatibleType(MapType t1, MapType t2, boolean strict) { + Type keyCompatibleType = Type.getAssignmentCompatibleType(t1.getKeyType(), t2.getKeyType(), strict); + if (keyCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + Type valCompatibleType = Type.getAssignmentCompatibleType(t1.getValueType(), t2.getValueType(), strict); + if (valCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + + return new MapType(keyCompatibleType, valCompatibleType, + t1.getIsKeyContainsNull() || t2.getIsKeyContainsNull(), + t1.getIsValueContainsNull() || t2.getIsValueContainsNull()); + } + @Override public boolean supportSubType(Type subType) { for (Type supportedType : Type.getMapSubTypes()) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java index 0a058ae5406f85..1d6be19d28efbc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java @@ -29,6 +29,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Arrays; @@ -110,6 +111,29 @@ public static boolean canCastTo(StructType type, StructType targetType) { return true; } + public static Type getAssignmentCompatibleType(StructType t1, StructType t2, boolean strict) { + ArrayList fieldsLeft = t1.getFields(); + ArrayList fieldsRight = t2.getFields(); + ArrayList fieldsRes = new ArrayList<>(); + + for (int i = 0; i < t1.getFields().size(); ++i) { + StructField leftField = fieldsLeft.get(i); + StructField rightField = fieldsRight.get(i); + Type itemCompatibleType = Type.getAssignmentCompatibleType(leftField.getType(), rightField.getType(), + strict); + if (itemCompatibleType.isInvalid()) { + return ScalarType.INVALID; + } + fieldsRes.add(new StructField(StringUtils.isEmpty(leftField.getName()) ? rightField.getName() + : leftField.getName(), + itemCompatibleType, StringUtils.isEmpty(leftField.getComment()) ? rightField.getComment() + : leftField.getComment(), leftField.getContainsNull() || rightField.getContainsNull())); + + } + + return new StructType(fieldsRes); + } + @Override public boolean isSupported() { for (StructField f : fields) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 0121bed39ce7f9..8091858e249264 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -777,19 +777,14 @@ public static Type getAssignmentCompatibleType(Type t1, Type t2, boolean strict) } if (t1.isArrayType() && t2.isArrayType()) { - ArrayType arrayType1 = (ArrayType) t1; - ArrayType arrayType2 = (ArrayType) t2; - Type itemCompatibleType = Type.getAssignmentCompatibleType(arrayType1.getItemType(), - arrayType2.getItemType(), strict); - - if (itemCompatibleType.isInvalid()) { - return itemCompatibleType; - } - - return new ArrayType(itemCompatibleType, arrayType1.getContainsNull() || arrayType2.getContainsNull()); - } else if (t1.isArrayType() && t2.isNull()) { + return ArrayType.getAssignmentCompatibleType((ArrayType) t1, (ArrayType) t2, strict); + } else if (t1.isMapType() && t2.isMapType()) { + return MapType.getAssignmentCompatibleType((MapType) t1, (MapType) t2, strict); + } else if (t1.isStructType() && t2.isStructType()) { + return StructType.getAssignmentCompatibleType((StructType) t1, (StructType) t2, strict); + } else if (t1.isComplexType() && t2.isNull()) { return t1; - } else if (t1.isNull() && t2.isArrayType()) { + } else if (t1.isNull() && t2.isComplexType()) { return t2; } @@ -2197,17 +2192,15 @@ public static boolean matchExactType(Type type1, Type type2, boolean ignorePreci } else if (type2.isArrayType()) { // For types array, we also need to check contains null for case like // cast(array as array) - if (!((ArrayType) type2).getContainsNull() == ((ArrayType) type1).getContainsNull()) { + if (((ArrayType) type2).getContainsNull() != ((ArrayType) type1).getContainsNull()) { return false; } return matchExactType(((ArrayType) type2).getItemType(), ((ArrayType) type1).getItemType()); } else if (type2.isMapType()) { - // For types array, we also need to check contains null for case like - // cast(array as array) - if (!((MapType) type2).getIsKeyContainsNull() == ((MapType) type1).getIsKeyContainsNull()) { + if (((MapType) type2).getIsKeyContainsNull() != ((MapType) type1).getIsKeyContainsNull()) { return false; } - if (!((MapType) type2).getIsValueContainsNull() == ((MapType) type1).getIsValueContainsNull()) { + if (((MapType) type2).getIsValueContainsNull() != ((MapType) type1).getIsValueContainsNull()) { return false; } return matchExactType(((MapType) type2).getKeyType(), ((MapType) type1).getKeyType()) diff --git a/regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.out b/regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.out new file mode 100644 index 00000000000000..1b24ac06a846ea --- /dev/null +++ b/regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_as -- +text [{"a": 3, "b": "home"}, {"a": 4, "b": "work"}] + +-- !sql_as -- +text [{"a": 3, "b": "home"}, {"a": 4, "b": "work"}] + diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy new file mode 100644 index 00000000000000..3efb9d5b06f9c5 --- /dev/null +++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +import com.google.common.collect.Lists +import org.apache.commons.lang3.StringUtils +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_nestedtypes_insert_into_select", "p0") { + sql "set enable_nereids_planner=false" + sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')""" + + // create array struct + sql "DROP TABLE IF EXISTS ast;" + sql """ CREATE TABLE IF NOT EXISTS ast (col1 varchar(64) NULL, col2 array>) DUPLICATE KEY(`col1`) DISTRIBUTED BY HASH(`col1`) PROPERTIES ("replication_num" = "1"); """ + + // test insert into with literal + sql "INSERT INTO ast values ('text',[{3,'home'},{4,'work'}]);" + + order_qt_sql_as """ select * from ast; """ + + test { + sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" + exception "errCode = 2, detailMessage = Sql parser can't convert the result to array, please check your sql." + } + + + sql "set enable_nereids_planner=true" + sql " set enable_fallback_to_original_planner=false" + + // create array struct + sql "DROP TABLE IF EXISTS ast;" + sql """ CREATE TABLE IF NOT EXISTS ast (col1 varchar(64) NULL, col2 array>) DUPLICATE KEY(`col1`) DISTRIBUTED BY HASH(`col1`) PROPERTIES ("replication_num" = "1"); """ + + // test insert into with literal + sql "INSERT INTO ast values ('text',[{3,'home'},{4,'work'}]);" + + order_qt_sql_as """ select * from ast; """ + + test { + sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);" + exception "errCode = 2, detailMessage = Sql parser can't convert the result to array, please check your sql." + } +}