Merge 2.x to Maximus feature branch (#1062)

* Fix `FLOAT` -> `DOUBLE` cast. (#1025) Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> * Fix error messaging from prometheus. (#1029) (#1037) Signed-off-by: vamsi-amazon <reddyvam@amazon.com> (cherry picked from commit 4a9cef3) Co-authored-by: vamsi-amazon <reddyvam@amazon.com> * Add `query` function as alternate syntax to `query_string` function (#1010) This maintains backwards compatibility with the v1 engine. * Update DATE and TIME functions to parse string input as datetime (#991) Add option to accept datetime like string in both TIME and DATE (eg. accept "1999-01-02 12:12:12" for both TIME and DATE. Strict check on date for testing for valid dates (eg. Don't accept Feb 30th as a valid date) and throws a SemanticCheckException. Co-authored-by: Yury-Fridlyand <yuryf@bitquilltech.com> Signed-off-by: MitchellGale-BitQuill <mitchellg@bitquilltech.com> * back quote fix (#1041) (#1050) Signed-off-by: vamsi-amazon <reddyvam@amazon.com> (cherry picked from commit d3bb902) Co-authored-by: vamsi-amazon <reddyvam@amazon.com> * Catalog to Datasource changes (#1027) (#1049) Signed-off-by: vamsi-amazon <reddyvam@amazon.com> (cherry picked from commit 3e30379) * Bump jackson to 2.14.0 (#1058) Signed-off-by: Joshua Li <joshuali925@gmail.com> (cherry picked from commit 5a1adb2) * Add valueOf() to Expression (#1055) Signed-off-by: Joshua Li <joshuali925@gmail.com> Signed-off-by: Yury-Fridlyand <yuryf@bitquilltech.com> Signed-off-by: MitchellGale-BitQuill <mitchellg@bitquilltech.com> Signed-off-by: Joshua Li <joshuali925@gmail.com> Signed-off-by: Chen Dai <daichen@amazon.com> Co-authored-by: Yury-Fridlyand <yuryf@bitquilltech.com> Co-authored-by: opensearch-trigger-bot[bot] <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Co-authored-by: vamsi-amazon <reddyvam@amazon.com> Co-authored-by: GabeFernandez310 <64448015+GabeFernandez310@users.noreply.github.com> Co-authored-by: MitchellGale-BitQuill <104795536+MitchellGale-BitQuill@users.noreply.github.com> Co-authored-by: Joshua Li <joshuali925@gmail.com>
opensearch-project · Nov 9, 2022 · 3c6b37a · 3c6b37a
1 parent 5105022
commit 3c6b37a
Show file tree

Hide file tree

Showing 81 changed files with 1,347 additions and 414 deletions.
diff --git a/build.gradle b/build.gradle
@@ -8,8 +8,8 @@ buildscript {
     ext {
         opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT")
         spring_version = "5.3.22"
-        jackson_version = "2.13.4"
-        jackson_databind_version = "2.13.4.2"
+        jackson_version = "2.14.0"
+        jackson_databind_version = "2.14.0"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')

diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java
@@ -184,7 +184,7 @@ public Expression visitConstantFunction(ConstantFunction node, AnalysisContext c
     }
 
     var value = visitFunction(node, context);
-    value = DSL.literal(value.valueOf(null));
+    value = DSL.literal(value.valueOf());
     context.getConstantFunctionValues().put(valueName, value);
     return value;
   }

diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java
@@ -6,6 +6,8 @@
 
 package org.opensearch.sql.data.model;
 
+import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL;
+
 import com.google.common.base.Objects;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -33,7 +35,7 @@ public class ExprDateValue extends AbstractExprValue {
    */
   public ExprDateValue(String date) {
     try {
-      this.date = LocalDate.parse(date);
+      this.date = LocalDate.parse(date, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL);
     } catch (DateTimeParseException e) {
       throw new SemanticCheckException(String.format("date:%s in unsupported format, please use "
           + "yyyy-MM-dd", date));

diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java
@@ -6,10 +6,15 @@
 
 package org.opensearch.sql.data.model;
 
-import static org.opensearch.sql.utils.DateTimeFormatters.TIME_FORMATTER_VARIABLE_NANOS;
+import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME;
+import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL;
 
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
 import java.time.LocalTime;
-import java.time.format.DateTimeFormatter;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.time.format.DateTimeParseException;
 import java.util.Objects;
 import lombok.RequiredArgsConstructor;
@@ -22,14 +27,15 @@
  */
 @RequiredArgsConstructor
 public class ExprTimeValue extends AbstractExprValue {
+
   private final LocalTime time;
 
   /**
-   * Constructor.
+   * Constructor of ExprTimeValue.
    */
   public ExprTimeValue(String time) {
     try {
-      this.time = LocalTime.parse(time, TIME_FORMATTER_VARIABLE_NANOS);
+      this.time = LocalTime.parse(time, DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL);
     } catch (DateTimeParseException e) {
       throw new SemanticCheckException(String.format("time:%s in unsupported format, please use "
           + "HH:mm:ss[.SSSSSSSSS]", time));
@@ -38,7 +44,7 @@ public ExprTimeValue(String time) {
 
   @Override
   public String value() {
-    return DateTimeFormatter.ISO_LOCAL_TIME.format(time);
+    return ISO_LOCAL_TIME.format(time);
   }
 
   @Override

diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java
@@ -102,7 +102,7 @@ public static NamedExpression named(Expression expression) {
       return (NamedExpression) expression;
     }
     if (expression instanceof ParseExpression) {
-      return named(((ParseExpression) expression).getIdentifier().valueOf(null).stringValue(),
+      return named(((ParseExpression) expression).getIdentifier().valueOf().stringValue(),
           expression);
     }
     return named(expression.toString(), expression);
@@ -707,6 +707,10 @@ public FunctionExpression simple_query_string(Expression... args) {
     return compile(BuiltinFunctionName.SIMPLE_QUERY_STRING, args);
   }
 
+  public FunctionExpression query(Expression... args) {
+    return compile(BuiltinFunctionName.QUERY, args);
+  }
+
   public FunctionExpression query_string(Expression... args) {
     return compile(BuiltinFunctionName.QUERY_STRING, args);
   }

diff --git a/core/src/main/java/org/opensearch/sql/expression/Expression.java b/core/src/main/java/org/opensearch/sql/expression/Expression.java
@@ -16,6 +16,13 @@
  */
 public interface Expression extends Serializable {
 
+  /**
+   * Evaluate the value of expression that does not depend on value environment.
+   */
+  default ExprValue valueOf() {
+    return valueOf(null);
+  }
+
   /**
    * Evaluate the value of expression in the value environment.
    */

diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/TakeAggregator.java
@@ -29,7 +29,7 @@ public TakeAggregator(List<Expression> arguments, ExprCoreType returnType) {
 
   @Override
   public TakeState create() {
-    return new TakeState(getArguments().get(1).valueOf(null).integerValue());
+    return new TakeState(getArguments().get(1).valueOf().integerValue());
   }
 
   @Override

diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java
@@ -56,6 +56,7 @@
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.data.type.ExprCoreType;
 import org.opensearch.sql.exception.ExpressionEvaluationException;
+import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
 import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
 import org.opensearch.sql.expression.function.DefaultFunctionResolver;

diff --git a/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java b/core/src/main/java/org/opensearch/sql/expression/function/OpenSearchFunctions.java
@@ -30,6 +30,7 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(match());
     repository.register(multi_match());
     repository.register(simple_query_string());
+    repository.register(query());
     repository.register(query_string());
     // Register MATCHPHRASE as MATCH_PHRASE as well for backwards
     // compatibility.
@@ -68,6 +69,11 @@ private static FunctionResolver simple_query_string() {
     return new RelevanceFunctionResolver(funcName, STRUCT);
   }
 
+  private static FunctionResolver query() {
+    FunctionName funcName = BuiltinFunctionName.QUERY.getName();
+    return new RelevanceFunctionResolver(funcName, STRING);
+  }
+
   private static FunctionResolver query_string() {
     FunctionName funcName = BuiltinFunctionName.QUERY_STRING.getName();
     return new RelevanceFunctionResolver(funcName, STRUCT);

diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperator.java
@@ -125,7 +125,7 @@ private static DefaultFunctionResolver castToFloat() {
         impl(nullMissingHandling(
             (v) -> new ExprFloatValue(Float.valueOf(v.stringValue()))), FLOAT, STRING),
         impl(nullMissingHandling(
-            (v) -> new ExprFloatValue(v.longValue())), FLOAT, DOUBLE),
+            (v) -> new ExprFloatValue(v.floatValue())), FLOAT, DOUBLE),
         impl(nullMissingHandling(
             (v) -> new ExprFloatValue(v.booleanValue() ? 1f : 0f)), FLOAT, BOOLEAN)
     );

diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/GrokExpression.java
@@ -45,7 +45,7 @@ public class GrokExpression extends ParseExpression {
    */
   public GrokExpression(Expression sourceField, Expression pattern, Expression identifier) {
     super("grok", sourceField, pattern, identifier);
-    this.grok = grokCompiler.compile(pattern.valueOf(null).stringValue());
+    this.grok = grokCompiler.compile(pattern.valueOf().stringValue());
   }
 
   @Override

diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/ParseExpression.java
@@ -48,7 +48,7 @@ public ParseExpression(String functionName, Expression sourceField, Expression p
     this.sourceField = sourceField;
     this.pattern = pattern;
     this.identifier = identifier;
-    this.identifierStr = identifier.valueOf(null).stringValue();
+    this.identifierStr = identifier.valueOf().stringValue();
   }
 
   @Override

diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/PatternsExpression.java
@@ -44,7 +44,7 @@ public class PatternsExpression extends ParseExpression {
    */
   public PatternsExpression(Expression sourceField, Expression pattern, Expression identifier) {
     super("patterns", sourceField, pattern, identifier);
-    String patternStr = pattern.valueOf(null).stringValue();
+    String patternStr = pattern.valueOf().stringValue();
     useCustomPattern = !patternStr.isEmpty();
     if (useCustomPattern) {
       this.pattern = Pattern.compile(patternStr);

diff --git a/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java b/core/src/main/java/org/opensearch/sql/expression/parse/RegexExpression.java
@@ -40,7 +40,7 @@ public class RegexExpression extends ParseExpression {
    */
   public RegexExpression(Expression sourceField, Expression pattern, Expression identifier) {
     super("regex", sourceField, pattern, identifier);
-    this.regexPattern = Pattern.compile(pattern.valueOf(null).stringValue());
+    this.regexPattern = Pattern.compile(pattern.valueOf().stringValue());
   }
 
   @Override

diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/ValuesOperator.java
@@ -58,7 +58,7 @@ public boolean hasNext() {
   @Override
   public ExprValue next() {
     List<ExprValue> values = valuesIterator.next().stream()
-                                           .map(expr -> expr.valueOf(null))
+                                           .map(expr -> expr.valueOf())
                                            .collect(Collectors.toList());
     return new ExprCollectionValue(values);
   }

diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTable.java
@@ -21,7 +21,7 @@
 
 
 /**
- * Table implementation to handle show catalogs command.
+ * Table implementation to handle show datasources command.
  * Since catalog information is not tied to any storage engine, this info
  * is handled via Catalog Table.
  *

diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableScan.java
@@ -51,7 +51,7 @@ public void open() {
     for (Catalog catalog : catalogs) {
       exprValues.add(
           new ExprTupleValue(new LinkedHashMap<>(ImmutableMap.of(
-              "CATALOG_NAME", ExprValueUtils.stringValue(catalog.getName()),
+              "DATASOURCE_NAME", ExprValueUtils.stringValue(catalog.getName()),
               "CONNECTOR_TYPE", ExprValueUtils.stringValue(catalog.getConnectorType().name())))));
     }
     iterator = exprValues.iterator();

diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java b/core/src/main/java/org/opensearch/sql/planner/physical/catalog/CatalogTableSchema.java
@@ -22,7 +22,7 @@ public enum CatalogTableSchema {
 
   CATALOG_TABLE_SCHEMA(new LinkedHashMap<>() {
     {
-      put("CATALOG_NAME", STRING);
+      put("DATASOURCE_NAME", STRING);
       put("CONNECTOR_TYPE", STRING);
     }
   }

diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java b/core/src/main/java/org/opensearch/sql/planner/physical/collector/Rounding.java
@@ -44,7 +44,7 @@ public abstract class Rounding<T> {
    * Create Rounding instance.
    */
   public static Rounding<?> createRounding(SpanExpression span) {
-    ExprValue interval = span.getValue().valueOf(null);
+    ExprValue interval = span.getValue().valueOf();
     ExprType type = span.type();
 
     if (LONG.isCompatible(type)) {

diff --git a/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java b/core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java
@@ -113,23 +113,25 @@ public class DateTimeFormatters {
 
   public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS =
       new DateTimeFormatterBuilder()
-          .appendPattern("yyyy-MM-dd HH:mm:ss")
+          .appendPattern("uuuu-MM-dd HH:mm:ss")
           .appendFraction(
               ChronoField.NANO_OF_SECOND,
               MIN_FRACTION_SECONDS,
               MAX_FRACTION_SECONDS,
               true)
-          .toFormatter(Locale.ROOT);
+          .toFormatter(Locale.ROOT)
+          .withResolverStyle(ResolverStyle.STRICT);
 
-  public static final DateTimeFormatter TIME_FORMATTER_VARIABLE_NANOS =
+  public static final DateTimeFormatter DATE_TIME_FORMATTER_VARIABLE_NANOS_OPTIONAL =
       new DateTimeFormatterBuilder()
-          .appendPattern("HH:mm:ss")
+          .appendPattern("[uuuu-MM-dd HH:mm:ss][uuuu-MM-dd HH:mm][HH:mm:ss][HH:mm][uuuu-MM-dd]")
           .appendFraction(
               ChronoField.NANO_OF_SECOND,
               MIN_FRACTION_SECONDS,
               MAX_FRACTION_SECONDS,
               true)
-          .toFormatter();
+          .toFormatter(Locale.ROOT)
+          .withResolverStyle(ResolverStyle.STRICT);
 
   // YYMMDD
   public static final DateTimeFormatter DATE_FORMATTER_SHORT_YEAR =

diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java
@@ -487,6 +487,15 @@ void simple_query_string_expression_two_fields() {
             AstDSL.unresolvedArg("query", stringLiteral("sample query"))));
   }
 
+  @Test
+  void query_expression() {
+    assertAnalyzeEqual(
+            dsl.query(
+                    dsl.namedArgument("query", DSL.literal("field:query"))),
+            AstDSL.function("query",
+                    AstDSL.unresolvedArg("query", stringLiteral("field:query"))));
+  }
+
   @Test
   void query_string_expression() {
     assertAnalyzeEqual(
@@ -574,7 +583,7 @@ public void constant_function_returns_constant_cached_value() {
     var values = List.of(analyze(AstDSL.constantFunction("now")),
         analyze(AstDSL.constantFunction("now")), analyze(AstDSL.constantFunction("now")));
     assertTrue(values.stream().allMatch(v ->
-        v.valueOf(null) == analyze(AstDSL.constantFunction("now")).valueOf(null)));
+        v.valueOf() == analyze(AstDSL.constantFunction("now")).valueOf()));
   }
 
   @Test
@@ -584,17 +593,17 @@ public void function_returns_non_constant_value() {
     // different values
     var values = List.of(analyze(function("sysdate")), analyze(function("sysdate")),
         analyze(function("sysdate")), analyze(function("sysdate")));
-    var referenceValue = analyze(function("sysdate")).valueOf(null);
-    assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue));
+    var referenceValue = analyze(function("sysdate")).valueOf();
+    assertTrue(values.stream().noneMatch(v -> v.valueOf() == referenceValue));
   }
 
   @Test
   public void now_as_a_function_not_cached() {
     // // We can call `now()` as a function, in that case nothing should be cached
     var values = List.of(analyze(function("now")), analyze(function("now")),
         analyze(function("now")), analyze(function("now")));
-    var referenceValue = analyze(function("now")).valueOf(null);
-    assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue));
+    var referenceValue = analyze(function("now")).valueOf();
+    assertTrue(values.stream().noneMatch(v -> v.valueOf() == referenceValue));
   }
 
   protected Expression analyze(UnresolvedExpression unresolvedExpression) {

diff --git a/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java
@@ -59,7 +59,7 @@ void name_a_parse_expression() {
             DSL.literal("group"));
     NamedExpression named = DSL.named(parse);
     assertEquals(parse, named.getDelegated());
-    assertEquals(parse.getIdentifier().valueOf(null).stringValue(), named.getName());
+    assertEquals(parse.getIdentifier().valueOf().stringValue(), named.getName());
   }
 
 }
diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java
@@ -24,6 +24,7 @@
 import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;
 
 import com.google.common.collect.ImmutableList;
+import java.time.LocalDate;
 import java.util.List;
 import lombok.AllArgsConstructor;
 import org.junit.jupiter.api.BeforeEach;
@@ -240,6 +241,18 @@ public void date() {
     assertEquals(DATE, expr.type());
     assertEquals(new ExprDateValue("2020-08-17"), eval(expr));
     assertEquals("date(DATE '2020-08-17')", expr.toString());
+
+    expr = dsl.date(DSL.literal(new ExprDateValue("2020-08-17 12:12:00")));
+    assertEquals(DATE, expr.type());
+    assertEquals(new ExprDateValue("2020-08-17 12:12:00"), eval(expr));
+    assertEquals("date(DATE '2020-08-17')", expr.toString());
+
+    expr = dsl.date(DSL.literal(new ExprDateValue("2020-08-17 12:12")));
+    assertEquals(DATE, expr.type());
+    assertEquals(new ExprDateValue("2020-08-17 12:12"), eval(expr));
+    assertEquals("date(DATE '2020-08-17')", expr.toString());
+
+
   }
 
   @Test
@@ -795,6 +808,30 @@ public void time() {
     assertEquals(TIME, expr.type());
     assertEquals(new ExprTimeValue("01:01:01"), eval(expr));
     assertEquals("time(TIME '01:01:01')", expr.toString());
+
+    expr = dsl.time(DSL.literal(new ExprTimeValue("01:01")));
+    assertEquals(TIME, expr.type());
+    assertEquals(new ExprTimeValue("01:01"), eval(expr));
+    assertEquals("time(TIME '01:01:00')", expr.toString());
+
+    expr = dsl.time(DSL.literal(new ExprTimeValue("2019-04-19 01:01:01")));
+    assertEquals(TIME, expr.type());
+    assertEquals(new ExprTimeValue("2019-04-19 01:01:01"), eval(expr));
+    assertEquals("time(TIME '01:01:01')", expr.toString());
+
+    expr = dsl.time(DSL.literal(new ExprTimeValue("2019-04-19 01:01")));
+    assertEquals(TIME, expr.type());
+    assertEquals(new ExprTimeValue("2019-04-19 01:01"), eval(expr));
+    assertEquals("time(TIME '01:01:00')", expr.toString());
+
+    expr = dsl.time(DSL.literal(new ExprTimeValue("01:01:01.0123")));
+    assertEquals(TIME, expr.type());
+    assertEquals(new ExprTimeValue("01:01:01.0123"), eval(expr));
+    assertEquals("time(TIME '01:01:01.0123')", expr.toString());
+
+    expr = dsl.time(dsl.date(DSL.literal("2020-01-02")));
+    assertEquals(TIME, expr.type());
+    assertEquals(new ExprTimeValue("00:00:00"), expr.valueOf());
   }
 
   @Test