Skip to content

Commit 27299f2

Browse files
acarbonettoacarbonetto
committed
PPL: Add JSON_OBJECT function
Signed-off-by: Andrew Carbonetto <andrew.carbonetto@improving.com>
1 parent d44fc5a commit 27299f2

File tree

9 files changed

+238
-1
lines changed

9 files changed

+238
-1
lines changed

‎core/src/main/java/org/opensearch/sql/expression/DSL.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -687,6 +687,10 @@ public static FunctionExpression jsonValid(Expression... expressions) {
687687
return compile(FunctionProperties.None, BuiltinFunctionName.JSON_VALID, expressions);
688688
}
689689

690+
public static FunctionExpression jsonObject(Expression... expressions) {
691+
return compile(FunctionProperties.None, BuiltinFunctionName.JSON_OBJECT, expressions);
692+
}
693+
690694
public static Aggregator avg(Expression... expressions) {
691695
return aggregate(BuiltinFunctionName.AVG, expressions);
692696
}

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ public enum BuiltinFunctionName {
206206

207207
/** Json Functions. */
208208
JSON_VALID(FunctionName.of("json_valid")),
209+
JSON_OBJECT(FunctionName.of("json_object")),
209210

210211
/** NULL Test. */
211212
IS_NULL(FunctionName.of("is null")),

‎core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,25 +7,100 @@
77

88
import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
99
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
10+
import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
1011
import static org.opensearch.sql.expression.function.FunctionDSL.define;
1112
import static org.opensearch.sql.expression.function.FunctionDSL.impl;
1213
import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;
1314

15+
import java.util.Iterator;
16+
import java.util.LinkedHashMap;
17+
import java.util.List;
1418
import lombok.experimental.UtilityClass;
19+
import org.apache.commons.lang3.tuple.Pair;
20+
import org.opensearch.sql.data.model.ExprTupleValue;
21+
import org.opensearch.sql.data.model.ExprValue;
22+
import org.opensearch.sql.data.type.ExprCoreType;
23+
import org.opensearch.sql.data.type.ExprType;
24+
import org.opensearch.sql.exception.SemanticCheckException;
25+
import org.opensearch.sql.expression.Expression;
26+
import org.opensearch.sql.expression.FunctionExpression;
27+
import org.opensearch.sql.expression.env.Environment;
1528
import org.opensearch.sql.expression.function.BuiltinFunctionName;
1629
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
1730
import org.opensearch.sql.expression.function.DefaultFunctionResolver;
31+
import org.opensearch.sql.expression.function.FunctionBuilder;
32+
import org.opensearch.sql.expression.function.FunctionName;
33+
import org.opensearch.sql.expression.function.FunctionResolver;
34+
import org.opensearch.sql.expression.function.FunctionSignature;
1835
import org.opensearch.sql.utils.JsonUtils;
1936

2037
@UtilityClass
2138
public class JsonFunctions {
2239
public void register(BuiltinFunctionRepository repository) {
2340
repository.register(jsonValid());
41+
repository.register(jsonObject());
2442
}
2543

2644
private DefaultFunctionResolver jsonValid() {
2745
return define(
2846
BuiltinFunctionName.JSON_VALID.getName(),
2947
impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING));
3048
}
49+
50+
/** Creates a JSON Object/tuple expr from a given list of kv pairs. */
51+
private static FunctionResolver jsonObject() {
52+
return new FunctionResolver() {
53+
@Override
54+
public FunctionName getFunctionName() {
55+
return BuiltinFunctionName.JSON_OBJECT.getName();
56+
}
57+
58+
@Override
59+
public Pair<FunctionSignature, FunctionBuilder> resolve(
60+
FunctionSignature unresolvedSignature) {
61+
List<ExprType> paramList = unresolvedSignature.getParamTypeList();
62+
// check that we got an even number of arguments
63+
if (paramList.size() % 2 != 0) {
64+
throw new SemanticCheckException(
65+
String.format(
66+
"Expected an even number of arguments but instead got #%d arguments",
67+
paramList.size()));
68+
}
69+
70+
// check that each "key" argument (of key-value pair) is a string
71+
for (int i = 0; i < paramList.size(); i = i + 2) {
72+
ExprType paramType = paramList.get(i);
73+
if (!ExprCoreType.STRING.equals(paramType)) {
74+
throw new SemanticCheckException(
75+
String.format(
76+
"Expected type %s instead of %s for parameter #%d",
77+
ExprCoreType.STRING, paramType.typeName(), i + 1));
78+
}
79+
}
80+
81+
// return the unresolved signature and function builder
82+
return Pair.of(
83+
unresolvedSignature,
84+
(functionProperties, arguments) ->
85+
new FunctionExpression(getFunctionName(), arguments) {
86+
@Override
87+
public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
88+
LinkedHashMap<String, ExprValue> tupleValues = new LinkedHashMap<>();
89+
Iterator<Expression> iter = getArguments().iterator();
90+
while (iter.hasNext()) {
91+
tupleValues.put(
92+
iter.next().valueOf(valueEnv).stringValue(),
93+
iter.next().valueOf(valueEnv));
94+
}
95+
return ExprTupleValue.fromExprValueMap(tupleValues);
96+
}
97+
98+
@Override
99+
public ExprType type() {
100+
return STRUCT;
101+
}
102+
});
103+
}
104+
};
105+
}
31106
}

‎core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
import java.util.List;
2828
import java.util.stream.Stream;
2929
import lombok.AllArgsConstructor;
30-
import org.junit.jupiter.api.Disabled;
3130
import org.junit.jupiter.api.Test;
3231
import org.junit.jupiter.params.ParameterizedTest;
3332
import org.junit.jupiter.params.provider.Arguments;

‎core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,31 @@
66
package org.opensearch.sql.expression.json;
77

88
import static org.junit.jupiter.api.Assertions.assertEquals;
9+
import static org.junit.jupiter.api.Assertions.assertThrows;
10+
import static org.junit.jupiter.api.Assertions.assertTrue;
911
import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE;
12+
import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL;
1013
import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE;
1114

15+
import java.util.LinkedHashMap;
16+
import java.util.List;
17+
import java.util.Map;
1218
import org.junit.jupiter.api.Test;
1319
import org.junit.jupiter.api.extension.ExtendWith;
1420
import org.mockito.junit.jupiter.MockitoExtension;
21+
import org.opensearch.sql.data.model.ExprBooleanValue;
22+
import org.opensearch.sql.data.model.ExprCollectionValue;
23+
import org.opensearch.sql.data.model.ExprDoubleValue;
24+
import org.opensearch.sql.data.model.ExprLongValue;
25+
import org.opensearch.sql.data.model.ExprNullValue;
26+
import org.opensearch.sql.data.model.ExprStringValue;
27+
import org.opensearch.sql.data.model.ExprTupleValue;
1528
import org.opensearch.sql.data.model.ExprValue;
1629
import org.opensearch.sql.data.model.ExprValueUtils;
30+
import org.opensearch.sql.exception.SemanticCheckException;
1731
import org.opensearch.sql.expression.DSL;
1832
import org.opensearch.sql.expression.FunctionExpression;
33+
import org.opensearch.sql.expression.LiteralExpression;
1934

2035
@ExtendWith(MockitoExtension.class)
2136
public class JsonFunctionsTest {
@@ -46,4 +61,74 @@ private ExprValue execute(ExprValue jsonString) {
4661
FunctionExpression exp = DSL.jsonValid(DSL.literal(jsonString));
4762
return exp.valueOf();
4863
}
64+
65+
@Test
66+
public void json_object_returns_tuple() {
67+
FunctionExpression exp;
68+
69+
// Setup
70+
LinkedHashMap<String, ExprValue> objectMap = new LinkedHashMap<>();
71+
objectMap.put("foo", new ExprStringValue("foo"));
72+
objectMap.put("fuzz", ExprBooleanValue.of(true));
73+
objectMap.put("bar", new ExprLongValue(1234));
74+
objectMap.put("bar2", new ExprDoubleValue(12.34));
75+
objectMap.put("baz", ExprNullValue.of());
76+
objectMap.put(
77+
"obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value"))));
78+
// TODO: requires json_array()
79+
// objectMap.put(
80+
// "arr",
81+
// new ExprCollectionValue(
82+
// List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of())));
83+
ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap);
84+
85+
// exercise
86+
exp = DSL.jsonObject(
87+
DSL.literal("foo"), DSL.literal("foo"),
88+
DSL.literal("fuzz"), DSL.literal(true),
89+
DSL.literal("bar"), DSL.literal(1234),
90+
DSL.literal("bar2"), DSL.literal(12.34),
91+
DSL.literal("baz"), new LiteralExpression(ExprValueUtils.nullValue()),
92+
DSL.literal("obj"), DSL.jsonObject(
93+
DSL.literal("internal"), DSL.literal("value")
94+
)
95+
);
96+
97+
// Verify
98+
var value = exp.valueOf();
99+
assertTrue(value instanceof ExprTupleValue);
100+
assertEquals(expectedTupleExpr, value);
101+
}
102+
103+
@Test
104+
public void json_object_returns_empty_tuple() {
105+
FunctionExpression exp;
106+
107+
// Setup
108+
LinkedHashMap<String, ExprValue> objectMap = new LinkedHashMap<>();
109+
ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap);
110+
111+
// exercise
112+
exp = DSL.jsonObject();
113+
114+
// Verify
115+
var value = exp.valueOf();
116+
assertTrue(value instanceof ExprTupleValue);
117+
assertEquals(expectedTupleExpr, value);
118+
}
119+
120+
@Test
121+
public void json_object_throws_SemanticCheckException() {
122+
// wrong number of arguments
123+
assertThrows(
124+
SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal("only one")).valueOf());
125+
assertThrows(
126+
SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal("one"), DSL.literal("two"), DSL.literal("three")).valueOf());
127+
128+
// key argument is not a string
129+
assertThrows(
130+
SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal(1234), DSL.literal("two")).valueOf());
131+
assertThrows(
132+
SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal("one"), DSL.literal(true), DSL.literal(true), DSL.literal("four")).valueOf());
133+
}
49134
}

‎docs/user/ppl/functions/json.rst

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,43 @@ Example::
3333
| json empty string | | True |
3434
| json invalid object | {"invalid":"json", "string"} | True |
3535
+---------------------+------------------------------+----------+
36+
37+
JSON_OBJECT
38+
-----------
39+
40+
Description
41+
>>>>>>>>>>>
42+
43+
Usage: `json_object(<key>, <value>[, <key>, <value>]...)` returns a JSON object from key-value pairs.
44+
45+
Argument type:
46+
- A \<key\> must be STRING.
47+
- A \<value\> can be a scalar, another json object, or json array type. Note: scalar fields will be treated as single-value. Use `json_array` to construct an array value from a multi-value.
48+
49+
Return type: STRUCT
50+
51+
Example:
52+
53+
os> source=people | eval result = json_object('key', 123.45) | fields result
54+
fetched rows / total rows = 1/1
55+
+------------------+
56+
| result |
57+
+------------------+
58+
| {"key":123.45} |
59+
+------------------+
60+
61+
os> source=people | eval result = json_object('outer', json_object('inner', 123.45)) | fields result
62+
fetched rows / total rows = 1/1
63+
+------------------------------+
64+
| result |
65+
+------------------------------+
66+
| {"outer":{"inner":123.45}} |
67+
+------------------------------+
68+
69+
os> source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result
70+
fetched rows / total rows = 1/1
71+
+------------------------------+
72+
| result |
73+
+------------------------------+
74+
| {"array_doc":[123.45, "string", true, null]} |
75+
+------------------------------+

‎integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
1313

1414
import java.io.IOException;
15+
import java.util.List;
16+
import java.util.Map;
1517
import org.json.JSONObject;
1618
import org.junit.jupiter.api.Test;
1719

@@ -51,4 +53,24 @@ public void test_not_json_valid() throws IOException {
5153
verifySchema(result, schema("test_name", null, "string"));
5254
verifyDataRows(result, rows("json invalid object"));
5355
}
56+
57+
@Test
58+
public void test_json_object() throws IOException {
59+
JSONObject result;
60+
61+
result =
62+
executeQuery(
63+
String.format(
64+
"source=%s | eval obj=json_object(\"key\", json(json_string)) | fields test_name, obj"
65+
+ " test_name, casted",
66+
TEST_INDEX_JSON_TEST));
67+
verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined"));
68+
verifyDataRows(
69+
result,
70+
rows("json object", Map.of("key", Map.of("a", "1", "b", "2"))),
71+
rows("json array", Map.of("key", List.of(1, 2, 3, 4))),
72+
rows("json scalar string", Map.of("key", "abc")),
73+
rows("json empty string", Map.of("key", null))
74+
);
75+
}
5476
}

‎ppl/src/main/antlr/OpenSearchPPLLexer.g4

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH';
334334

335335
// JSON FUNCTIONS
336336
JSON_VALID: 'JSON_VALID';
337+
JSON_OBJECT: 'JSON_OBJECT';
337338

338339
// FLOWCONTROL FUNCTIONS
339340
IFNULL: 'IFNULL';

‎ppl/src/main/antlr/OpenSearchPPLParser.g4

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ valueExpression
310310
| extractFunction # extractFunctionCall
311311
| getFormatFunction # getFormatFunctionCall
312312
| timestampFunction # timestampFunctionCall
313+
| jsonObjectFunction # jsonObjectFunctionCall
313314
| LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr
314315
;
315316

@@ -324,6 +325,10 @@ positionFunction
324325
: positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS
325326
;
326327

328+
jsonObjectFunction
329+
: jsonObjectFunctionName LT_PRTHS functionArg COMMA functionArg (COMMA functionArg COMMA functionArg)* RT_PRTHS
330+
;
331+
327332
booleanExpression
328333
: booleanFunctionCall
329334
;
@@ -419,6 +424,7 @@ evalFunctionName
419424
| flowControlFunctionName
420425
| systemFunctionName
421426
| positionFunctionName
427+
| jsonObjectFunctionName
422428
;
423429

424430
functionArgs
@@ -700,6 +706,10 @@ positionFunctionName
700706
: POSITION
701707
;
702708

709+
jsonObjectFunctionName
710+
: JSON_OBJECT
711+
;
712+
703713
// operators
704714
comparisonOperator
705715
: EQUAL

0 commit comments

Comments
 (0)