[backport to 2.19-dev] Backport calcite prs

.github/workflows/link-checker.yml

@@ -6,6 +6,7 @@ on:
     branches:
       - 'main'
       - '[1-9]+.[0-9x]+'
+      - '2.19-dev'
 
 jobs:
   linkchecker:

DEVELOPER_GUIDE.rst

@@ -235,6 +235,10 @@ Most of the time you just need to run ./gradlew build which will make sure you p
      - Run all unit tests.
    * - ./gradlew :integ-test:integTest
      - Run all integration test (this takes time).
+   * - ./gradlew :integ-test:yamlRestTest
+     - Run rest integration test.
+   * - ./gradlew :doctest:doctest
+     - Run doctests
    * - ./gradlew build
      - Build plugin by run all tasks above (this takes time).
    * - ./gradlew pitest
@@ -405,7 +409,7 @@ Sample test class:
 Doctest
 >>>>>>>
 
-Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_.
+Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_.
 
 
 Backports

async-query-core/src/main/antlr/OpenSearchPPLParser.g4

@@ -868,7 +868,6 @@ jsonFunctionName
    | JSON_OBJECT
    | JSON_ARRAY
    | JSON_ARRAY_LENGTH
-   | TO_JSON_STRING
    | JSON_EXTRACT
    | JSON_KEYS
    | JSON_VALID

async-query-core/src/main/java/org/opensearch/sql/spark/utils/SQLQueryUtils.java

@@ -315,6 +315,31 @@ public Void visitAlterMaterializedViewStatement(
       return super.visitAlterMaterializedViewStatement(ctx);
     }
 
+    @Override
+    public Void visitVacuumCoveringIndexStatement(
+        FlintSparkSqlExtensionsParser.VacuumCoveringIndexStatementContext ctx) {
+      indexQueryDetailsBuilder.indexQueryActionType(IndexQueryActionType.VACUUM);
+      indexQueryDetailsBuilder.indexType(FlintIndexType.COVERING);
+      return super.visitVacuumCoveringIndexStatement(ctx);
+    }
+
+    @Override
+    public Void visitVacuumSkippingIndexStatement(
+        FlintSparkSqlExtensionsParser.VacuumSkippingIndexStatementContext ctx) {
+      indexQueryDetailsBuilder.indexQueryActionType(IndexQueryActionType.VACUUM);
+      indexQueryDetailsBuilder.indexType(FlintIndexType.SKIPPING);
+      return super.visitVacuumSkippingIndexStatement(ctx);
+    }
+
+    @Override
+    public Void visitVacuumMaterializedViewStatement(
+        FlintSparkSqlExtensionsParser.VacuumMaterializedViewStatementContext ctx) {
+      indexQueryDetailsBuilder.indexQueryActionType(IndexQueryActionType.VACUUM);
+      indexQueryDetailsBuilder.indexType(FlintIndexType.MATERIALIZED_VIEW);
+      indexQueryDetailsBuilder.mvName(ctx.mvName.getText());
+      return super.visitVacuumMaterializedViewStatement(ctx);
+    }
+
     @Override
     public Void visitMaterializedViewQuery(MaterializedViewQueryContext ctx) {
       int a = ctx.start.getStartIndex();

async-query-core/src/main/java/org/opensearch/sql/spark/validator/FunctionType.java

@@ -32,6 +32,7 @@ public enum FunctionType {
   CSV("CSV"),
   MISC("Misc"),
   GENERATOR("Generator"),
+  OTHER("Other"),
   UDF("User Defined Function");
 
   private final String name;
@@ -422,6 +423,51 @@ public enum FunctionType {
                   "posexplode",
                   "posexplode_outer",
                   "stack"))
+          .put(
+              OTHER,
+              Set.of(
+                  "aggregate",
+                  "array_size",
+                  "array_sort",
+                  "cardinality",
+                  "crc32",
+                  "exists",
+                  "filter",
+                  "forall",
+                  "hash",
+                  "ilike",
+                  "in",
+                  "like",
+                  "map_filter",
+                  "map_zip_with",
+                  "md5",
+                  "mod",
+                  "named_struct",
+                  "parse_url",
+                  "raise_error",
+                  "reduce",
+                  "reverse",
+                  "sha",
+                  "sha1",
+                  "sha2",
+                  "size",
+                  "struct",
+                  "transform",
+                  "transform_keys",
+                  "transform_values",
+                  "url_decode",
+                  "url_encode",
+                  "xpath",
+                  "xpath_boolean",
+                  "xpath_double",
+                  "xpath_float",
+                  "xpath_int",
+                  "xpath_long",
+                  "xpath_number",
+                  "xpath_short",
+                  "xpath_string",
+                  "xxhash64",
+                  "zip_with"))
           .build();
 
   private static final Map<String, FunctionType> FUNCTION_NAME_TO_FUNCTION_TYPE_MAP =

async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLGrammarElement.java

@@ -78,6 +78,7 @@ public enum SQLGrammarElement implements GrammarElement {
   CSV_FUNCTIONS("CSV functions"),
   GENERATOR_FUNCTIONS("Generator functions"),
   MISC_FUNCTIONS("Misc functions"),
+  OTHER_FUNCTIONS("Other functions"),
 
   // UDF
   UDF("User Defined functions");

async-query-core/src/main/java/org/opensearch/sql/spark/validator/SQLQueryValidationVisitor.java

@@ -560,26 +560,30 @@ public Void visitFunctionName(FunctionNameContext ctx) {
     return super.visitFunctionName(ctx);
   }
 
-  private void validateFunctionAllowed(String function) {
-    FunctionType type = FunctionType.fromFunctionName(function.toLowerCase());
+  private void validateFunctionAllowed(String functionName) {
+    String lowerCaseFunctionName = functionName.toLowerCase();
+    FunctionType type = FunctionType.fromFunctionName(lowerCaseFunctionName);
     switch (type) {
       case MAP:
-        validateAllowed(SQLGrammarElement.MAP_FUNCTIONS);
+        validateAllowed(SQLGrammarElement.MAP_FUNCTIONS, lowerCaseFunctionName);
         break;
       case BITWISE:
-        validateAllowed(SQLGrammarElement.BITWISE_FUNCTIONS);
+        validateAllowed(SQLGrammarElement.BITWISE_FUNCTIONS, lowerCaseFunctionName);
         break;
       case CSV:
-        validateAllowed(SQLGrammarElement.CSV_FUNCTIONS);
+        validateAllowed(SQLGrammarElement.CSV_FUNCTIONS, lowerCaseFunctionName);
         break;
       case MISC:
-        validateAllowed(SQLGrammarElement.MISC_FUNCTIONS);
+        validateAllowed(SQLGrammarElement.MISC_FUNCTIONS, lowerCaseFunctionName);
         break;
       case GENERATOR:
-        validateAllowed(SQLGrammarElement.GENERATOR_FUNCTIONS);
+        validateAllowed(SQLGrammarElement.GENERATOR_FUNCTIONS, lowerCaseFunctionName);
+        break;
+      case OTHER:
+        validateAllowed(SQLGrammarElement.OTHER_FUNCTIONS, lowerCaseFunctionName);
         break;
       case UDF:
-        validateAllowed(SQLGrammarElement.UDF);
+        validateAllowed(SQLGrammarElement.UDF, lowerCaseFunctionName);
         break;
     }
   }
@@ -590,6 +594,12 @@ private void validateAllowed(SQLGrammarElement element) {
     }
   }
 
+  private void validateAllowed(SQLGrammarElement element, String detail) {
+    if (!grammarElementValidator.isValid(element)) {
+      throw new IllegalArgumentException(String.format("%s (%s) is not allowed.", element, detail));
+    }
+  }
+
   @Override
   public Void visitErrorCapturingIdentifier(ErrorCapturingIdentifierContext ctx) {
     ErrorCapturingIdentifierExtraContext extra = ctx.errorCapturingIdentifierExtra();

async-query-core/src/test/java/org/opensearch/sql/spark/utils/SQLQueryUtilsTest.java

@@ -141,6 +141,7 @@ void testExtractionFromFlintSkippingIndexQueries() {
           + " WITH (auto_refresh = true)",
       "DROP SKIPPING INDEX ON myS3.default.alb_logs",
       "ALTER SKIPPING INDEX ON myS3.default.alb_logs WITH (auto_refresh = false)",
+      "VACUUM SKIPPING INDEX ON myS3.default.alb_logs",
     };
 
     for (String query : createSkippingIndexQueries) {
@@ -170,6 +171,7 @@ void testExtractionFromFlintCoveringIndexQueries() {
           + " WHERE elb_status_code = 500 "
           + " WITH (auto_refresh = true)",
       "DROP INDEX elb_and_requestUri ON myS3.default.alb_logs",
+      "VACUUM INDEX elb_and_requestUri ON myS3.default.alb_logs",
       "ALTER INDEX elb_and_requestUri ON myS3.default.alb_logs WITH (auto_refresh = false)"
     };
 

async-query-core/src/test/java/org/opensearch/sql/spark/validator/FunctionTypeTest.java

@@ -42,6 +42,8 @@ public void test() {
     assertEquals(FunctionType.MISC, FunctionType.fromFunctionName("version"));
     assertEquals(FunctionType.GENERATOR, FunctionType.fromFunctionName("explode"));
     assertEquals(FunctionType.GENERATOR, FunctionType.fromFunctionName("stack"));
+    assertEquals(FunctionType.OTHER, FunctionType.fromFunctionName("aggregate"));
+    assertEquals(FunctionType.OTHER, FunctionType.fromFunctionName("forall"));
     assertEquals(FunctionType.UDF, FunctionType.fromFunctionName("unknown"));
   }
 }

async-query-core/src/test/java/org/opensearch/sql/spark/validator/SQLQueryValidatorTest.java

@@ -6,6 +6,7 @@
 package org.opensearch.sql.spark.validator;
 
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.when;
@@ -192,6 +193,10 @@ private enum TestElement {
     // Generator Functions
     GENERATOR_FUNCTIONS("SELECT explode(array(1, 2, 3));"),
 
+    // Other functions
+    NAMED_STRUCT("SELECT named_struct('a', 1);"),
+    PARSE_URL("SELECT parse_url(url) FROM my_table;"),
+
     // UDFs (User-Defined Functions)
     SCALAR_USER_DEFINED_FUNCTIONS("SELECT my_udf(name) FROM my_table;"),
     USER_DEFINED_AGGREGATE_FUNCTIONS("SELECT my_udaf(age) FROM my_table GROUP BY name;"),
@@ -323,6 +328,10 @@ void testDenyAllValidator() {
     // Generator Functions
     v.ng(TestElement.GENERATOR_FUNCTIONS);
 
+    // Other Functions
+    v.ng(TestElement.NAMED_STRUCT);
+    v.ng(TestElement.PARSE_URL);
+
     // UDFs
     v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS);
     v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS);
@@ -440,6 +449,10 @@ void testS3glueQueries() {
     // Generator Functions
     v.ok(TestElement.GENERATOR_FUNCTIONS);
 
+    // Other Functions
+    v.ok(TestElement.NAMED_STRUCT);
+    v.ok(TestElement.PARSE_URL);
+
     // UDFs
     v.ng(TestElement.SCALAR_USER_DEFINED_FUNCTIONS);
     v.ng(TestElement.USER_DEFINED_AGGREGATE_FUNCTIONS);
@@ -621,6 +634,14 @@ void testUnsupportedHiveNativeCommand() {
     v.ng("DFS");
   }
 
+  @Test
+  void testException() {
+    when(mockedProvider.getValidatorForDatasource(any())).thenReturn(element -> false);
+    VerifyValidator v = new VerifyValidator(sqlQueryValidator, DataSourceType.S3GLUE);
+
+    v.ng("SELECT named_struct('a', 1);", "Other functions (named_struct) is not allowed.");
+  }
+
   @AllArgsConstructor
   private static class VerifyValidator {
     private final SQLQueryValidator validator;
@@ -645,6 +666,15 @@ public void ng(String query) {
           "The query should throw: query=`" + query.toString() + "`");
     }
 
+    public void ng(String query, String expectedMessage) {
+      Exception e =
+          assertThrows(
+              IllegalArgumentException.class,
+              () -> runValidate(query),
+              "The query should throw: query=`" + query.toString() + "`");
+      assertEquals(expectedMessage, e.getMessage());
+    }
+
     void runValidate(String[] queries) {
       Arrays.stream(queries).forEach(query -> validator.validate(query, dataSourceType));
     }

benchmarks/build.gradle

@@ -14,10 +14,20 @@ repositories {
 
 dependencies {
     implementation project(':core')
+    implementation project(':opensearch')
 
     // Dependencies required by JMH micro benchmark
     api group: 'org.openjdk.jmh', name: 'jmh-core', version: '1.36'
     annotationProcessor group: 'org.openjdk.jmh', name: 'jmh-generator-annprocess', version: '1.36'
 }
 
+spotless {
+    java {
+        target fileTree('.') {
+            include '**/*.java'
+            exclude '**/jmh_generated/**'
+        }
+    }
+}
+
 compileJava.options.compilerArgs.addAll(["-processor", "org.openjdk.jmh.generators.BenchmarkProcessor"])