Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions core/src/main/java/org/opensearch/sql/data/type/ExprType.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.expression.Expression;

Expand Down Expand Up @@ -54,4 +55,17 @@ default List<ExprType> getParent() {
default String legacyTypeName() {
return typeName();
}

/** Get the original path. Types like alias type will set the actual path in field property. */
default Optional<String> getOriginalPath() {
return Optional.empty();
}

/**
* Get the original path. Types like alias type should be derived from the type of the original
* field.
*/
default ExprType getOriginalExprType() {
return this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
public class ReferenceExpression implements Expression {
@Getter private final String attr;

@Getter private final String rawPath;

@Getter private final List<String> paths;

private final ExprType type;
Expand All @@ -36,8 +38,11 @@ public class ReferenceExpression implements Expression {
public ReferenceExpression(String ref, ExprType type) {
this.attr = ref;
// Todo. the define of paths need to be redefined after adding multiple index/variable support.
this.paths = Arrays.asList(ref.split("\\."));
this.type = type;
// For AliasType, the actual path is set in the property of `path` and the type is derived
// from the type of field on that path; Otherwise, use ref itself as the path
this.rawPath = type.getOriginalPath().orElse(ref);
this.paths = Arrays.asList(rawPath.split("\\."));
this.type = type.getOriginalExprType();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED;
import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN;

import java.util.Optional;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -85,4 +86,16 @@ void defaultLegacyTypeName() {
final ExprType exprType = () -> "dummy";
assertEquals("dummy", exprType.legacyTypeName());
}

@Test
void getOriginalPath() {
final ExprType exprType = () -> "dummy";
assertEquals(Optional.empty(), exprType.getOriginalPath());
}

@Test
void getOriginalExprType() {
final ExprType exprType = () -> "dummy";
assertEquals(exprType, exprType.getOriginalExprType());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import static com.google.common.base.Strings.isNullOrEmpty;
import static org.opensearch.sql.legacy.TestUtils.createIndexByRestClient;
import static org.opensearch.sql.legacy.TestUtils.getAccountIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getAliasIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping;
import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping;
Expand Down Expand Up @@ -751,7 +752,12 @@ public enum Index {
TestsConstants.TEST_INDEX_JSON_TEST,
"json",
getJsonTestIndexMapping(),
"src/test/resources/json_test.json");
"src/test/resources/json_test.json"),
DATA_TYPE_ALIAS(
TestsConstants.TEST_INDEX_ALIAS,
"alias",
getAliasIndexMapping(),
"src/test/resources/alias.json");

private final String name;
private final String type;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,11 @@ public static String getJsonTestIndexMapping() {
return getMappingFile(mappingFile);
}

public static String getAliasIndexMapping() {
String mappingFile = "alias_index_mapping.json";
return getMappingFile(mappingFile);
}

public static void loadBulk(Client client, String jsonPath, String defaultIndex)
throws Exception {
System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public class TestsConstants {
public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";
public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint";
public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test";
public static final String TEST_INDEX_ALIAS = TEST_INDEX + "_alias";
public static final String DATASOURCES = ".ql-datasources";

public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
Expand Down
13 changes: 13 additions & 0 deletions integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@

package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_ALIAS;
import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NONNUMERIC;
import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ALIAS;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC;
import static org.opensearch.sql.util.MatcherUtils.schema;
Expand All @@ -22,6 +24,7 @@ public class DataTypeIT extends PPLIntegTestCase {
public void init() throws IOException {
loadIndex(DATA_TYPE_NUMERIC);
loadIndex(DATA_TYPE_NONNUMERIC);
loadIndex(DATA_TYPE_ALIAS);
}

@Test
Expand Down Expand Up @@ -75,4 +78,14 @@ public void test_long_integer_data_type() throws IOException {
schema("long1", "long"),
schema("long2", "long"));
}

@Test
public void test_alias_data_type() throws IOException {
JSONObject result =
executeQuery(
String.format(
"source=%s | where alias_col > 1 " + "| fields original_col, alias_col ",
TEST_INDEX_ALIAS));
verifySchema(result, schema("original_col", "integer"), schema("alias_col", "integer"));
}
}
6 changes: 6 additions & 0 deletions integ-test/src/test/resources/alias.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{"index":{"_id":"1"}}
{"original_col" : 1}
{"index":{"_id":"2"}}
{"original_col" : 2}
{"index":{"_id":"3"}}
{"original_col" : 3}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"mappings": {
"properties": {
"original_col": {
"type": "integer"
},
"alias_col": {
"type": "alias",
"path": "original_col"
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.opensearch.data.type;

import java.util.List;
import java.util.Optional;
import java.util.Set;
import org.opensearch.sql.data.type.ExprType;

/**
* The type of alias. See <a
* href="https://opensearch.org/docs/latest/opensearch/supported-field-types/alias/">doc</a>
*/
public class OpenSearchAliasType extends OpenSearchDataType {

public static final String typeName = "alias";
public static final String pathPropertyName = "path";
public static final Set<MappingType> objectFieldTypes =
Set.of(MappingType.Object, MappingType.Nested);
private final String path;
private final OpenSearchDataType originalType;

public OpenSearchAliasType(String path, OpenSearchDataType type) {
super(type.getExprCoreType());
if (type instanceof OpenSearchAliasType) {
throw new IllegalStateException(
String.format("Alias field cannot refer to the path [%s] of alias type", path));
} else if (objectFieldTypes.contains(type.getMappingType())) {
throw new IllegalStateException(
String.format("Alias field cannot refer to the path [%s] of object type", path));
}
this.path = path;
this.originalType = type;
}

@Override
public Optional<String> getOriginalPath() {
return Optional.of(this.path);
}

@Override
public ExprType getOriginalExprType() {
return originalType.getExprType();
}

@Override
public ExprType getExprType() {
return this;
}

@Override
public OpenSearchDataType cloneEmpty() {
return new OpenSearchAliasType(this.path, originalType.cloneEmpty());
}

@Override
public boolean isCompatible(ExprType other) {
return originalType.isCompatible(other);
}

@Override
public List<ExprType> getParent() {
return originalType.getParent();
}

@Override
public String typeName() {
return originalType.typeName();
}

@Override
public String legacyTypeName() {
return originalType.legacyTypeName();
}

@Override
public boolean shouldCast(ExprType other) {
return originalType.shouldCast(other);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -109,14 +109,19 @@ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> i
return result;
}

Map<String, String> aliasMapping = new LinkedHashMap<>();
indexMapping.forEach(
(k, v) -> {
var innerMap = (Map<String, Object>) v;
// by default, the type is treated as an Object if "type" is not provided
var type = ((String) innerMap.getOrDefault("type", "object")).replace("_", "");
if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) {
// unknown type, e.g. `alias`
// TODO resolve alias reference
// Record fields of the alias type and resolve them later in case their references have
// not been resolved.
if (OpenSearchAliasType.typeName.equals(type)) {
aliasMapping.put(k, (String) innerMap.get(OpenSearchAliasType.pathPropertyName));
}
return;
}
// create OpenSearchDataType
Expand All @@ -126,6 +131,18 @@ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> i
EnumUtils.getEnumIgnoreCase(OpenSearchDataType.MappingType.class, type),
innerMap));
});

// Begin to parse alias type fields
aliasMapping.forEach(
(k, v) -> {
if (result.containsKey(v)) {
result.put(k, new OpenSearchAliasType(v, result.get(v)));
} else {
throw new IllegalStateException(
String.format("Cannot find the path [%s] for alias type field [%s]", v, k));
}
});

return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ public void pushDownHighlight(String field, Map<String, Literal> arguments) {
/** Push down project list to DSL requests. */
public void pushDownProjects(Set<ReferenceExpression> projects) {
sourceBuilder.fetchSource(
projects.stream().map(ReferenceExpression::getAttr).distinct().toArray(String[]::new),
projects.stream().map(ReferenceExpression::getRawPath).distinct().toArray(String[]::new),
new String[0]);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprTupleValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.opensearch.data.type.OpenSearchAliasType;
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
import org.opensearch.sql.opensearch.data.type.OpenSearchTextType;
import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory;
Expand Down Expand Up @@ -159,9 +160,9 @@ void get_index_mappings() throws IOException {
var parsedTypes = OpenSearchDataType.traverseAndFlatten(mapping);
assertAll(
() -> assertEquals(1, indexMappings.size()),
// 10 types extended to 17 after flattening
() -> assertEquals(10, mapping.size()),
() -> assertEquals(17, parsedTypes.size()),
// 11 types extended to 18 after flattening
() -> assertEquals(11, mapping.size()),
() -> assertEquals(18, parsedTypes.size()),
() -> assertEquals("TEXT", mapping.get("address").legacyTypeName()),
() -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("address")),
() -> assertEquals("INTEGER", mapping.get("age").legacyTypeName()),
Expand All @@ -186,6 +187,11 @@ void get_index_mappings() throws IOException {
() -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")),
// `employer` is a `text` with `fields`
() -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0),
() -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()),
() ->
assertEquals(
new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)),
parsedTypes.get("employer_alias")),
() -> assertEquals("NESTED", mapping.get("projects").legacyTypeName()),
() -> assertEquals(OpenSearchTextType.of(MappingType.Nested), parsedTypes.get("projects")),
() ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprTupleValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.opensearch.data.type.OpenSearchAliasType;
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
import org.opensearch.sql.opensearch.data.type.OpenSearchTextType;
import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory;
Expand Down Expand Up @@ -162,9 +163,9 @@ void get_index_mappings() throws IOException {
var parsedTypes = OpenSearchDataType.traverseAndFlatten(mapping);
assertAll(
() -> assertEquals(1, indexMappings.size()),
// 10 types extended to 17 after flattening
() -> assertEquals(10, mapping.size()),
() -> assertEquals(17, parsedTypes.size()),
// 11 types extended to 18 after flattening
() -> assertEquals(11, mapping.size()),
() -> assertEquals(18, parsedTypes.size()),
() -> assertEquals("TEXT", mapping.get("address").legacyTypeName()),
() -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("address")),
() -> assertEquals("INTEGER", mapping.get("age").legacyTypeName()),
Expand All @@ -189,6 +190,11 @@ void get_index_mappings() throws IOException {
() -> assertEquals(OpenSearchTextType.of(MappingType.Text), parsedTypes.get("employer")),
// `employer` is a `text` with `fields`
() -> assertTrue(((OpenSearchTextType) parsedTypes.get("employer")).getFields().size() > 0),
() -> assertEquals("TEXT", mapping.get("employer_alias").legacyTypeName()),
() ->
assertEquals(
new OpenSearchAliasType("employer", OpenSearchTextType.of(MappingType.Text)),
parsedTypes.get("employer_alias")),
() -> assertEquals("NESTED", mapping.get("projects").legacyTypeName()),
() -> assertEquals(OpenSearchTextType.of(MappingType.Nested), parsedTypes.get("projects")),
() ->
Expand Down
Loading
Loading