Skip to content

Commit b6846ce

Browse files
normanj-bitquillacarbonetto
normanj-bitquill
and
acarbonetto
authored
Add FILLNULL command in PPL (#3032) (#3075)
Signed-off-by: Norman Jordan <norman.jordan@improving.com> Signed-off-by: normanj-bitquill <78755797+normanj-bitquill@users.noreply.github.com> Co-authored-by: Andrew Carbonetto <andrew.carbonetto@improving.com>
1 parent 6712526 commit b6846ce

File tree

17 files changed

+635
-0
lines changed

17 files changed

+635
-0
lines changed

‎core/src/main/java/org/opensearch/sql/analysis/Analyzer.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.opensearch.sql.ast.tree.Dedupe;
4747
import org.opensearch.sql.ast.tree.Eval;
4848
import org.opensearch.sql.ast.tree.FetchCursor;
49+
import org.opensearch.sql.ast.tree.FillNull;
4950
import org.opensearch.sql.ast.tree.Filter;
5051
import org.opensearch.sql.ast.tree.Head;
5152
import org.opensearch.sql.ast.tree.Kmeans;
@@ -558,6 +559,29 @@ public LogicalPlan visitAD(AD node, AnalysisContext context) {
558559
return new LogicalAD(child, options);
559560
}
560561

562+
/** Build {@link LogicalEval} for fillnull command. */
563+
@Override
564+
public LogicalPlan visitFillNull(final FillNull node, final AnalysisContext context) {
565+
LogicalPlan child = node.getChild().get(0).accept(this, context);
566+
567+
ImmutableList.Builder<Pair<ReferenceExpression, Expression>> expressionsBuilder =
568+
new Builder<>();
569+
for (FillNull.NullableFieldFill fieldFill : node.getNullableFieldFills()) {
570+
Expression fieldExpr =
571+
expressionAnalyzer.analyze(fieldFill.getNullableFieldReference(), context);
572+
ReferenceExpression ref =
573+
DSL.ref(fieldFill.getNullableFieldReference().getField().toString(), fieldExpr.type());
574+
FunctionExpression ifNullFunction =
575+
DSL.ifnull(ref, expressionAnalyzer.analyze(fieldFill.getReplaceNullWithMe(), context));
576+
expressionsBuilder.add(new ImmutablePair<>(ref, ifNullFunction));
577+
TypeEnvironment typeEnvironment = context.peek();
578+
// define the new reference in type env.
579+
typeEnvironment.define(ref);
580+
}
581+
582+
return new LogicalEval(child, expressionsBuilder.build());
583+
}
584+
561585
/** Build {@link LogicalML} for ml command. */
562586
@Override
563587
public LogicalPlan visitML(ML node, AnalysisContext context) {

‎core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.opensearch.sql.ast.tree.Dedupe;
4646
import org.opensearch.sql.ast.tree.Eval;
4747
import org.opensearch.sql.ast.tree.FetchCursor;
48+
import org.opensearch.sql.ast.tree.FillNull;
4849
import org.opensearch.sql.ast.tree.Filter;
4950
import org.opensearch.sql.ast.tree.Head;
5051
import org.opensearch.sql.ast.tree.Kmeans;
@@ -312,4 +313,8 @@ public T visitFetchCursor(FetchCursor cursor, C context) {
312313
public T visitCloseCursor(CloseCursor closeCursor, C context) {
313314
return visitChildren(closeCursor, context);
314315
}
316+
317+
public T visitFillNull(FillNull fillNull, C context) {
318+
return visitChildren(fillNull, context);
319+
}
315320
}

‎core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55

66
package org.opensearch.sql.ast.dsl;
77

8+
import com.google.common.collect.ImmutableList;
89
import java.util.Arrays;
910
import java.util.List;
1011
import java.util.stream.Collectors;
1112
import lombok.experimental.UtilityClass;
13+
import org.apache.commons.lang3.tuple.ImmutablePair;
1214
import org.apache.commons.lang3.tuple.Pair;
1315
import org.opensearch.sql.ast.expression.AggregateFunction;
1416
import org.opensearch.sql.ast.expression.Alias;
@@ -46,6 +48,7 @@
4648
import org.opensearch.sql.ast.tree.Aggregation;
4749
import org.opensearch.sql.ast.tree.Dedupe;
4850
import org.opensearch.sql.ast.tree.Eval;
51+
import org.opensearch.sql.ast.tree.FillNull;
4952
import org.opensearch.sql.ast.tree.Filter;
5053
import org.opensearch.sql.ast.tree.Head;
5154
import org.opensearch.sql.ast.tree.Limit;
@@ -471,4 +474,22 @@ public static Parse parse(
471474
java.util.Map<String, Literal> arguments) {
472475
return new Parse(parseMethod, sourceField, pattern, arguments, input);
473476
}
477+
478+
public static FillNull fillNull(UnresolvedExpression replaceNullWithMe, Field... fields) {
479+
return new FillNull(
480+
FillNull.ContainNullableFieldFill.ofSameValue(
481+
replaceNullWithMe, ImmutableList.copyOf(fields)));
482+
}
483+
484+
public static FillNull fillNull(
485+
List<ImmutablePair<Field, UnresolvedExpression>> fieldAndReplacements) {
486+
ImmutableList.Builder<FillNull.NullableFieldFill> replacementsBuilder = ImmutableList.builder();
487+
for (ImmutablePair<Field, UnresolvedExpression> fieldAndReplacement : fieldAndReplacements) {
488+
replacementsBuilder.add(
489+
new FillNull.NullableFieldFill(
490+
fieldAndReplacement.getLeft(), fieldAndReplacement.getRight()));
491+
}
492+
return new FillNull(
493+
FillNull.ContainNullableFieldFill.ofVariousValue(replacementsBuilder.build()));
494+
}
474495
}

‎core/src/main/java/org/opensearch/sql/ast/tree/FillNull.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.ast.tree;
7+
8+
import java.util.List;
9+
import java.util.Objects;
10+
import lombok.AllArgsConstructor;
11+
import lombok.Getter;
12+
import lombok.NonNull;
13+
import lombok.RequiredArgsConstructor;
14+
import org.opensearch.sql.ast.AbstractNodeVisitor;
15+
import org.opensearch.sql.ast.Node;
16+
import org.opensearch.sql.ast.expression.Field;
17+
import org.opensearch.sql.ast.expression.UnresolvedExpression;
18+
19+
/** AST node represent FillNull operation. */
20+
@RequiredArgsConstructor
21+
@AllArgsConstructor
22+
public class FillNull extends UnresolvedPlan {
23+
24+
@Getter
25+
@RequiredArgsConstructor
26+
public static class NullableFieldFill {
27+
@NonNull private final Field nullableFieldReference;
28+
@NonNull private final UnresolvedExpression replaceNullWithMe;
29+
}
30+
31+
public interface ContainNullableFieldFill {
32+
List<NullableFieldFill> getNullFieldFill();
33+
34+
static ContainNullableFieldFill ofVariousValue(List<NullableFieldFill> replacements) {
35+
return new VariousValueNullFill(replacements);
36+
}
37+
38+
static ContainNullableFieldFill ofSameValue(
39+
UnresolvedExpression replaceNullWithMe, List<Field> nullableFieldReferences) {
40+
return new SameValueNullFill(replaceNullWithMe, nullableFieldReferences);
41+
}
42+
}
43+
44+
private static class SameValueNullFill implements ContainNullableFieldFill {
45+
@Getter(onMethod_ = @Override)
46+
private final List<NullableFieldFill> nullFieldFill;
47+
48+
public SameValueNullFill(
49+
UnresolvedExpression replaceNullWithMe, List<Field> nullableFieldReferences) {
50+
Objects.requireNonNull(replaceNullWithMe, "Null replacement is required");
51+
this.nullFieldFill =
52+
Objects.requireNonNull(nullableFieldReferences, "Nullable field reference is required")
53+
.stream()
54+
.map(nullableReference -> new NullableFieldFill(nullableReference, replaceNullWithMe))
55+
.toList();
56+
}
57+
}
58+
59+
@RequiredArgsConstructor
60+
private static class VariousValueNullFill implements ContainNullableFieldFill {
61+
@NonNull
62+
@Getter(onMethod_ = @Override)
63+
private final List<NullableFieldFill> nullFieldFill;
64+
}
65+
66+
private UnresolvedPlan child;
67+
68+
@NonNull private final ContainNullableFieldFill containNullableFieldFill;
69+
70+
public List<NullableFieldFill> getNullableFieldFills() {
71+
return containNullableFieldFill.getNullFieldFill();
72+
}
73+
74+
@Override
75+
public UnresolvedPlan attach(UnresolvedPlan child) {
76+
this.child = child;
77+
return this;
78+
}
79+
80+
@Override
81+
public List<? extends Node> getChild() {
82+
return child == null ? List.of() : List.of(child);
83+
}
84+
85+
@Override
86+
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
87+
return nodeVisitor.visitFillNull(this, context);
88+
}
89+
}

‎core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import org.opensearch.sql.ast.dsl.AstDSL;
7474
import org.opensearch.sql.ast.expression.Argument;
7575
import org.opensearch.sql.ast.expression.DataType;
76+
import org.opensearch.sql.ast.expression.Field;
7677
import org.opensearch.sql.ast.expression.HighlightFunction;
7778
import org.opensearch.sql.ast.expression.Literal;
7879
import org.opensearch.sql.ast.expression.ParseMethod;
@@ -81,6 +82,7 @@
8182
import org.opensearch.sql.ast.tree.AD;
8283
import org.opensearch.sql.ast.tree.CloseCursor;
8384
import org.opensearch.sql.ast.tree.FetchCursor;
85+
import org.opensearch.sql.ast.tree.FillNull;
8486
import org.opensearch.sql.ast.tree.Kmeans;
8587
import org.opensearch.sql.ast.tree.ML;
8688
import org.opensearch.sql.ast.tree.Paginate;
@@ -1437,6 +1439,48 @@ public void kmeanns_relation() {
14371439
new Kmeans(AstDSL.relation("schema"), argumentMap));
14381440
}
14391441

1442+
@Test
1443+
public void fillnull_same_value() {
1444+
assertAnalyzeEqual(
1445+
LogicalPlanDSL.eval(
1446+
LogicalPlanDSL.relation("schema", table),
1447+
ImmutablePair.of(
1448+
DSL.ref("integer_value", INTEGER),
1449+
DSL.ifnull(DSL.ref("integer_value", INTEGER), DSL.literal(0))),
1450+
ImmutablePair.of(
1451+
DSL.ref("int_null_value", INTEGER),
1452+
DSL.ifnull(DSL.ref("int_null_value", INTEGER), DSL.literal(0)))),
1453+
new FillNull(
1454+
AstDSL.relation("schema"),
1455+
FillNull.ContainNullableFieldFill.ofSameValue(
1456+
AstDSL.intLiteral(0),
1457+
ImmutableList.<Field>builder()
1458+
.add(AstDSL.field("integer_value"))
1459+
.add(AstDSL.field("int_null_value"))
1460+
.build())));
1461+
}
1462+
1463+
@Test
1464+
public void fillnull_various_values() {
1465+
assertAnalyzeEqual(
1466+
LogicalPlanDSL.eval(
1467+
LogicalPlanDSL.relation("schema", table),
1468+
ImmutablePair.of(
1469+
DSL.ref("integer_value", INTEGER),
1470+
DSL.ifnull(DSL.ref("integer_value", INTEGER), DSL.literal(0))),
1471+
ImmutablePair.of(
1472+
DSL.ref("int_null_value", INTEGER),
1473+
DSL.ifnull(DSL.ref("int_null_value", INTEGER), DSL.literal(1)))),
1474+
new FillNull(
1475+
AstDSL.relation("schema"),
1476+
FillNull.ContainNullableFieldFill.ofVariousValue(
1477+
ImmutableList.of(
1478+
new FillNull.NullableFieldFill(
1479+
AstDSL.field("integer_value"), AstDSL.intLiteral(0)),
1480+
new FillNull.NullableFieldFill(
1481+
AstDSL.field("int_null_value"), AstDSL.intLiteral(1))))));
1482+
}
1483+
14401484
@Test
14411485
public void ad_batchRCF_relation() {
14421486
Map<String, Literal> argumentMap =

‎docs/category.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"user/ppl/cmd/information_schema.rst",
1515
"user/ppl/cmd/eval.rst",
1616
"user/ppl/cmd/fields.rst",
17+
"user/ppl/cmd/fillnull.rst",
1718
"user/ppl/cmd/grok.rst",
1819
"user/ppl/cmd/head.rst",
1920
"user/ppl/cmd/parse.rst",

‎docs/user/ppl/cmd/fillnull.rst‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
=============
2+
fillnull
3+
=============
4+
5+
.. rubric:: Table of contents
6+
7+
.. contents::
8+
:local:
9+
:depth: 2
10+
11+
12+
Description
13+
============
14+
Using ``fillnull`` command to fill null with provided value in one or more fields in the search result.
15+
16+
17+
Syntax
18+
============
19+
`fillnull [with <null-replacement> in <nullable-field>["," <nullable-field>]] | [using <source-field> = <null-replacement> [","<source-field> = <null-replacement>]]`
20+
21+
* null-replacement: mandatory. The value used to replace `null`s.
22+
* nullable-field: mandatory. Field reference. The `null` values in the field referred to by the property will be replaced with the values from the null-replacement.
23+
24+
Example 1: fillnull one field
25+
======================================================================
26+
27+
The example show fillnull one field.
28+
29+
PPL query::
30+
31+
os> source=accounts | fields email, employer | fillnull with '<not found>' in email ;
32+
fetched rows / total rows = 4/4
33+
+-----------------------+----------+
34+
| email | employer |
35+
|-----------------------+----------|
36+
| amberduke@pyrami.com | Pyrami |
37+
| hattiebond@netagy.com | Netagy |
38+
| <not found> | Quility |
39+
| daleadams@boink.com | null |
40+
+-----------------------+----------+
41+
42+
Example 2: fillnull applied to multiple fields
43+
========================================================================
44+
45+
The example show fillnull applied to multiple fields.
46+
47+
PPL query::
48+
49+
os> source=accounts | fields email, employer | fillnull using email = '<not found>', employer = '<no employer>' ;
50+
fetched rows / total rows = 4/4
51+
+-----------------------+---------------+
52+
| email | employer |
53+
|-----------------------+---------------|
54+
| amberduke@pyrami.com | Pyrami |
55+
| hattiebond@netagy.com | Netagy |
56+
| <not found> | Quility |
57+
| daleadams@boink.com | <no employer> |
58+
+-----------------------+---------------+
59+
60+
Limitation
61+
==========
62+
The ``fillnull`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node.

‎integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,17 @@ public void testLimitPushDownExplain() throws Exception {
8989
+ "| fields ageMinus"));
9090
}
9191

92+
@Test
93+
public void testFillNullPushDownExplain() throws Exception {
94+
String expected = loadFromFile("expectedOutput/ppl/explain_fillnull_push.json");
95+
96+
assertJsonEquals(
97+
expected,
98+
explainQueryToString(
99+
"source=opensearch-sql_test_index_account"
100+
+ " | fillnull with -1 in age,balance | fields age, balance"));
101+
}
102+
92103
String loadFromFile(String filename) throws Exception {
93104
URI uri = Resources.getResource(filename).toURI();
94105
return new String(Files.readAllBytes(Paths.get(uri)));

0 commit comments

Comments
 (0)