Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FIX](complextype) fxi array nested struct literal #26778

Merged
merged 3 commits into from
Nov 10, 2023
Merged

[FIX](complextype) fxi array nested struct literal #26778

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
04f369b
fix struct be nested when read will make core
amorynan Nov 2, 2023
7237800
update cases out and code
amorynan Nov 2, 2023
52fda10
fix groovy and array nested struct literal
amorynan Nov 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
update cases out and code
  • Loading branch information
@amorynan
amorynan committed Nov 10, 2023
commit 72378003dc4a8bcca1c95205862c7abb9243963a
10 changes: 10 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,16 @@ public static boolean canCastTo(ArrayType type, ArrayType targetType) {
return Type.canCastTo(type.getItemType(), targetType.getItemType());
}

public static Type getAssignmentCompatibleType(ArrayType t1, ArrayType t2, boolean strict) {
Type itemCompatibleType = Type.getAssignmentCompatibleType(t1.getItemType(), t2.getItemType(), strict);

if (itemCompatibleType.isInvalid()) {
return ScalarType.INVALID;
}

return new ArrayType(itemCompatibleType, t1.getContainsNull() || t2.getContainsNull());
}

@Override
public void toThrift(TTypeDesc container) {
TTypeNode node = new TTypeNode();
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,21 @@ public static boolean canCastTo(MapType type, MapType targetType) {
|| targetType.getValueType().isStringType() && type.getValueType().isStringType());
}

public static Type getAssignmentCompatibleType(MapType t1, MapType t2, boolean strict) {
Type keyCompatibleType = Type.getAssignmentCompatibleType(t1.getKeyType(), t2.getKeyType(), strict);
if (keyCompatibleType.isInvalid()) {
return ScalarType.INVALID;
}
Type valCompatibleType = Type.getAssignmentCompatibleType(t1.getValueType(), t2.getValueType(), strict);
if (valCompatibleType.isInvalid()) {
return ScalarType.INVALID;
}

return new MapType(keyCompatibleType, valCompatibleType,
t1.getIsKeyContainsNull() || t2.getIsKeyContainsNull(),
t1.getIsValueContainsNull() || t2.getIsValueContainsNull());
}

@Override
public boolean supportSubType(Type subType) {
for (Type supportedType : Type.getMapSubTypes()) {
Expand Down
24 changes: 24 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.StringUtils;

import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -110,6 +111,29 @@ public static boolean canCastTo(StructType type, StructType targetType) {
return true;
}

public static Type getAssignmentCompatibleType(StructType t1, StructType t2, boolean strict) {
ArrayList<StructField> fieldsLeft = t1.getFields();
ArrayList<StructField> fieldsRight = t2.getFields();
ArrayList<StructField> fieldsRes = new ArrayList<>();

for (int i = 0; i < t1.getFields().size(); ++i) {
StructField leftField = fieldsLeft.get(i);
StructField rightField = fieldsRight.get(i);
Type itemCompatibleType = Type.getAssignmentCompatibleType(leftField.getType(), rightField.getType(),
strict);
if (itemCompatibleType.isInvalid()) {
return ScalarType.INVALID;
}
fieldsRes.add(new StructField(StringUtils.isEmpty(leftField.getName()) ? rightField.getName()
: leftField.getName(),
itemCompatibleType, StringUtils.isEmpty(leftField.getComment()) ? rightField.getComment()
: leftField.getComment(), leftField.getContainsNull() || rightField.getContainsNull()));

}

return new StructType(fieldsRes);
}

@Override
public boolean isSupported() {
for (StructField f : fields) {
Expand Down
60 changes: 6 additions & 54 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand Down Expand Up @@ -747,56 +746,11 @@ public static Type getAssignmentCompatibleType(Type t1, Type t2, boolean strict)
}

if (t1.isArrayType() && t2.isArrayType()) {
ArrayType arrayType1 = (ArrayType) t1;
ArrayType arrayType2 = (ArrayType) t2;
Type itemCompatibleType = Type.getAssignmentCompatibleType(arrayType1.getItemType(),
arrayType2.getItemType(), strict);

if (itemCompatibleType.isInvalid()) {
return itemCompatibleType;
}

return new ArrayType(itemCompatibleType, arrayType1.getContainsNull() || arrayType2.getContainsNull());
return ArrayType.getAssignmentCompatibleType((ArrayType) t1, (ArrayType) t2, strict);
} else if (t1.isMapType() && t2.isMapType()) {
MapType mapType1 = (MapType) t1;
MapType mapType2 = (MapType) t2;
Type keyCompatibleType = Type.getAssignmentCompatibleType(mapType1.getKeyType(),
mapType2.getKeyType(), strict);
if (keyCompatibleType.isInvalid()) {
return keyCompatibleType;
}
Type valCompatibleType = Type.getAssignmentCompatibleType(mapType1.getValueType(),
mapType2.getValueType(), strict);
if (valCompatibleType.isInvalid()) {
return valCompatibleType;
}

return new MapType(keyCompatibleType, valCompatibleType,
mapType1.getIsKeyContainsNull() || mapType2.getIsKeyContainsNull(),
mapType1.getIsValueContainsNull() || mapType2.getIsValueContainsNull());
return MapType.getAssignmentCompatibleType((MapType) t1, (MapType) t2, strict);
} else if (t1.isStructType() && t2.isStructType()) {
StructType structType1 = (StructType) t1;
StructType structType2 = (StructType) t2;
ArrayList<StructField> fieldsLeft = structType1.getFields();
ArrayList<StructField> fieldsRight = structType2.getFields();
ArrayList<StructField> fieldsRes = new ArrayList<>();

for (int i = 0; i < structType1.getFields().size(); ++i) {
StructField leftField = fieldsLeft.get(i);
StructField rightField = fieldsRight.get(i);
Type itemCompatibleType = Type.getAssignmentCompatibleType(leftField.getType(), rightField.getType(),
strict);
if (itemCompatibleType.isInvalid()) {
return itemCompatibleType;
}
fieldsRes.add(new StructField(StringUtils.isEmpty(leftField.getName()) ? rightField.getName()
: leftField.getName(),
itemCompatibleType, StringUtils.isEmpty(leftField.getComment()) ? rightField.getComment()
: leftField.getComment(), leftField.getContainsNull() || rightField.getContainsNull()));

}

return new StructType(fieldsRes);
return StructType.getAssignmentCompatibleType((StructType) t1, (StructType) t2, strict);
} else if (t1.isComplexType() && t2.isNull()) {
return t1;
} else if (t1.isNull() && t2.isComplexType()) {
Expand Down Expand Up @@ -2004,17 +1958,15 @@ public static boolean matchExactType(Type type1, Type type2, boolean ignorePreci
} else if (type2.isArrayType()) {
// For types array, we also need to check contains null for case like
// cast(array<not_null(int)> as array<int>)
if (!((ArrayType) type2).getContainsNull() == ((ArrayType) type1).getContainsNull()) {
if (((ArrayType) type2).getContainsNull() != ((ArrayType) type1).getContainsNull()) {
return false;
}
return matchExactType(((ArrayType) type2).getItemType(), ((ArrayType) type1).getItemType());
} else if (type2.isMapType()) {
// For types array, we also need to check contains null for case like
// cast(array<not_null(int)> as array<int>)
if (!((MapType) type2).getIsKeyContainsNull() == ((MapType) type1).getIsKeyContainsNull()) {
if (((MapType) type2).getIsKeyContainsNull() != ((MapType) type1).getIsKeyContainsNull()) {
return false;
}
if (!((MapType) type2).getIsValueContainsNull() == ((MapType) type1).getIsValueContainsNull()) {
if (((MapType) type2).getIsValueContainsNull() != ((MapType) type1).getIsValueContainsNull()) {
return false;
}
return matchExactType(((MapType) type2).getKeyType(), ((MapType) type1).getKeyType())
Expand Down
10 changes: 2 additions & 8 deletions regression-test/data/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,7 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql_as --
text [{3, "home"}, {4, "work"}]

-- !sql_am --
txt [{6:"amory", 7:"now", 10:"commit"}]
text [{"a": 3, "b": "home"}, {"a": 4, "b": "work"}]

-- !sql_as --
text [{3, "home"}, {4, "work"}]

-- !sql_am --
txt [{6:"amory", 7:"now", 10:"commit"}]
text [{"a": 3, "b": "home"}, {"a": 4, "b": "work"}]

16 changes: 0 additions & 16 deletions ...ion-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,14 @@ suite("test_nestedtypes_insert_into_select", "p0") {
sql "set enable_nereids_planner=false"
sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')"""

// create array - map
sql "DROP TABLE IF EXISTS amt;"
sql """
CREATE TABLE IF NOT EXISTS amt (col1 varchar(64) NULL, col2 array<map<int, string>>) DUPLICATE KEY(`col1`) DISTRIBUTED BY HASH(`col1`) PROPERTIES ("replication_num" = "1"); """

// create array struct
sql "DROP TABLE IF EXISTS ast;"
sql """ CREATE TABLE IF NOT EXISTS ast (col1 varchar(64) NULL, col2 array<struct<a:int,b:string>>) DUPLICATE KEY(`col1`) DISTRIBUTED BY HASH(`col1`) PROPERTIES ("replication_num" = "1"); """

// test insert into with literal
sql "INSERT INTO ast values ('text',[{3,'home'},{4,'work'}]);"

sql "INSERT INTO amt values ('txt',[{6:'amory', 7:'now', 10: 'commit'}]);"

order_qt_sql_as """ select * from ast; """
order_qt_sql_am """ select * from amt; """

test {
sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);"
Expand All @@ -50,22 +42,14 @@ suite("test_nestedtypes_insert_into_select", "p0") {
sql "set enable_nereids_planner=true"
sql " set enable_fallback_to_original_planner=false"

// create array - map
sql "DROP TABLE IF EXISTS amt;"
sql """
CREATE TABLE IF NOT EXISTS amt (col1 varchar(64) NULL, col2 array<map<int, string>>) DUPLICATE KEY(`col1`) DISTRIBUTED BY HASH(`col1`) PROPERTIES ("replication_num" = "1"); """

// create array struct
sql "DROP TABLE IF EXISTS ast;"
sql """ CREATE TABLE IF NOT EXISTS ast (col1 varchar(64) NULL, col2 array<struct<a:int,b:string>>) DUPLICATE KEY(`col1`) DISTRIBUTED BY HASH(`col1`) PROPERTIES ("replication_num" = "1"); """

// test insert into with literal
sql "INSERT INTO ast values ('text',[{3,'home'},{4,'work'}]);"

sql "INSERT INTO amt values ('txt',[{6:'amory', 7:'now', 10: 'commit'}]);"

order_qt_sql_as """ select * from ast; """
order_qt_sql_am """ select * from amt; """

test {
sql "insert into ast values ('text' , [named_struct('a',1,'b','home'),named_struct('a',2,'b','work')]);"
Expand Down