Added some integration tests for fillnull

Signed-off-by: Norman Jordan <norman.jordan@improving.com>
Bit-Quill · Oct 21, 2024 · a1f584f · a1f584f
1 parent bbe955a
commit a1f584f
Show file tree

Hide file tree

Showing 4 changed files with 190 additions and 0 deletions.
diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java
@@ -89,6 +89,18 @@ public void testLimitPushDownExplain() throws Exception {
                 + "| fields ageMinus"));
   }
 
+  @Test
+  public void testFillNullPushDownExplain() throws Exception {
+    String expected = loadFromFile("expectedOutput/ppl/explain_fillnull_push.json");
+
+    assertJsonEquals(
+        expected,
+        explainQueryToString(
+            "source=opensearch-sql_test_index_account"
+                + "| fields age, balance "
+                + "| fillnull with -1 in age,balance"));
+  }
+
   String loadFromFile(String filename) throws Exception {
     URI uri = Resources.getResource(filename).toURI();
     return new String(Files.readAllBytes(Paths.get(uri)));

diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.ppl;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.jupiter.api.Test;
+
+public class FillNullCommandIT extends PPLIntegTestCase {
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.CALCS);
+  }
+
+  @Test
+  public void testFillNullSameValueOneField() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | fields str2, num0 | fillnull with -1 in num0", TEST_INDEX_CALCS));
+    verifyDataRows(
+        result,
+        rows("one", 12.3),
+        rows("two", -12.3),
+        rows("three", 15.7),
+        rows(null, -15.7),
+        rows("five", 3.5),
+        rows("six", -3.5),
+        rows(null, 0),
+        rows("eight", -1),
+        rows("nine", 10),
+        rows("ten", -1),
+        rows("eleven", -1),
+        rows("twelve", -1),
+        rows(null, -1),
+        rows("fourteen", -1),
+        rows("fifteen", -1),
+        rows("sixteen", -1),
+        rows(null, -1));
+  }
+
+  @Test
+  public void testFillNullSameValueTwoFields() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | fields num0, num2 | fillnull with -1 in num0,num2", TEST_INDEX_CALCS));
+    verifyDataRows(
+        result,
+        rows(12.3, 17.86),
+        rows(-12.3, 16.73),
+        rows(15.7, -1),
+        rows(-15.7, 8.51),
+        rows(3.5, 6.46),
+        rows(-3.5, 8.98),
+        rows(0, 11.69),
+        rows(-1, 17.25),
+        rows(10, -1),
+        rows(-1, 11.5),
+        rows(-1, 6.8),
+        rows(-1, 3.79),
+        rows(-1, -1),
+        rows(-1, 13.04),
+        rows(-1, -1),
+        rows(-1, 10.98),
+        rows(-1, 7.87));
+  }
+
+  @Test
+  public void testFillNullVariousValuesOneField() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | fields str2, num0 | fillnull using num0 = -1", TEST_INDEX_CALCS));
+    verifyDataRows(
+        result,
+        rows("one", 12.3),
+        rows("two", -12.3),
+        rows("three", 15.7),
+        rows(null, -15.7),
+        rows("five", 3.5),
+        rows("six", -3.5),
+        rows(null, 0),
+        rows("eight", -1),
+        rows("nine", 10),
+        rows("ten", -1),
+        rows("eleven", -1),
+        rows("twelve", -1),
+        rows(null, -1),
+        rows("fourteen", -1),
+        rows("fifteen", -1),
+        rows("sixteen", -1),
+        rows(null, -1));
+  }
+
+  @Test
+  public void testFillNullVariousValuesTwoFields() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format(
+                "source=%s | fields num0, num2 | fillnull using num0 = -1, num2 = -2",
+                TEST_INDEX_CALCS));
+    verifyDataRows(
+        result,
+        rows(12.3, 17.86),
+        rows(-12.3, 16.73),
+        rows(15.7, -2),
+        rows(-15.7, 8.51),
+        rows(3.5, 6.46),
+        rows(-3.5, 8.98),
+        rows(0, 11.69),
+        rows(-1, 17.25),
+        rows(10, -2),
+        rows(-1, 11.5),
+        rows(-1, 6.8),
+        rows(-1, 3.79),
+        rows(-1, -2),
+        rows(-1, 13.04),
+        rows(-1, -2),
+        rows(-1, 10.98),
+        rows(-1, 7.87));
+  }
+}
diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json
@@ -0,0 +1,36 @@
+{
+  "root": {
+    "name": "ProjectOperator",
+    "description": {
+      "fields": "[age, balance]"
+    },
+    "children": [
+      {
+        "name":"EvalOperator",
+        "description": {
+          "expressions": {
+            "balance": "ifnull(balance, -1)",
+            "age":"ifnull(age, -1)"
+          }
+        },
+        "children": [
+          {
+            "name": "ProjectOperator",
+            "description": {
+              "fields": "[age, balance]"
+            },
+            "children": [
+              {
+                "name": "OpenSearchIndexScan",
+                "description": {
+                  "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\",\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)"
+                },
+                "children": []
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}
diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java
@@ -417,4 +417,16 @@ public void testCanParseTimestampdiffFunction() {
         new PPLSyntaxParser()
             .parse("SOURCE=test | eval k = TIMESTAMPDIFF(WEEK,'2003-01-02','2003-01-02')"));
   }
+
+  @Test
+  public void testCanParseFillNullSameValue() {
+    assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull with 0 in a"));
+    assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull with 0 in a,b"));
+  }
+
+  @Test
+  public void testCanParseFillNullVariousValues() {
+    assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull using a = 0"));
+    assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull using a = 0, b = 1"));
+  }
 }