Skip to content

Commit

Permalink
NIFI-12517 Updated isJson function to improve space handling
Browse files Browse the repository at this point in the history 
- The isJson Expression Language function returns true regardless of leading or trailing spaces

This closes apache#8165

Signed-off-by: David Handermann <exceptionfactory@apache.org>
  • Loading branch information
@dan-s1 @exceptionfactory
dan-s1 authored and exceptionfactory committed Dec 30, 2023
1 parent e15aecc commit 3e62e1e
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 38 deletions.
17 changes: 9 additions & 8 deletions ...a/org/apache/nifi/attribute/expression/language/evaluation/functions/IsJsonEvaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.apache.nifi.attribute.expression.language.evaluation.functions;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.attribute.expression.language.EvaluationContext;
import org.apache.nifi.attribute.expression.language.evaluation.BooleanEvaluator;
import org.apache.nifi.attribute.expression.language.evaluation.BooleanQueryResult;
Expand All @@ -37,13 +36,15 @@ public IsJsonEvaluator(Evaluator<String> subject) {
@Override
public QueryResult<Boolean> evaluate(EvaluationContext evaluationContext) {
final String subjectValue = subject.evaluate(evaluationContext).getValue();
if (StringUtils.isNotBlank(subjectValue)
&& (isPossibleJsonArray(subjectValue) || isPossibleJsonObject(subjectValue))) {
try {
MAPPER.readTree(subjectValue);
return new BooleanQueryResult(true);
} catch (IOException ignored) {
//IOException ignored
if (subjectValue != null) {
final String trimmedSubjectValue = subjectValue.trim();
if (isPossibleJsonArray(trimmedSubjectValue) || isPossibleJsonObject(trimmedSubjectValue)) {
try {
MAPPER.readTree(trimmedSubjectValue);
return new BooleanQueryResult(true);
} catch (IOException ignored) {
//IOException ignored
}
}
}
return new BooleanQueryResult(false);
Expand Down
2 changes: 2 additions & 0 deletions ...ssion-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2498,6 +2498,7 @@ public void testUuidsWithNamespace() {
void testIsJson() {
final Map<String, String> attributes = new HashMap<>();
attributes.put("jsonObj", "{\"name\":\"John\", \"age\":30, \"car\":null}");
attributes.put("jsonObjWithLeadingAndTrailingingSpaces", "\n{\"name\":\"John\", \"age\":30, \"car\":null}\n");
attributes.put("jsonObjMissingStartingBrace", "\"name\":\"John\", \"age\":30, \"car\":null}");
attributes.put("jsonObjMissingEndingBrace", "{\"name\":\"John\", \"age\":30, \"car\":null");
attributes.put("jsonArray", "[\"Ford\", \"BMW\", \"Fiat\"]");
Expand All @@ -2512,6 +2513,7 @@ void testIsJson() {
attributes.put("nullAttr", "null");

verifyEquals("${jsonObj:isJson()}", attributes, true);
verifyEquals("${jsonObjWithLeadingAndTrailingingSpaces:isJson()}", attributes, true);
verifyEquals("${jsonObjMissingStartingBrace:isJson()}", attributes, false);
verifyEquals("${jsonObjMissingEndingBrace:isJson()}", attributes, false);
verifyEquals("${jsonArray:isJson()}", attributes, true);
Expand Down
11 changes: 7 additions & 4 deletions ...andard-processors/src/main/java/org/apache/nifi/processors/standard/AttributesToJSON.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,11 +324,14 @@ private Map<String, Object> getFormattedAttributes(Map<String, Object> flowFileA
Map<String, Object> formattedAttributes = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : flowFileAttributes.entrySet()) {
String value = (String) entry.getValue();
if (StringUtils.isNotBlank(value) && (isPossibleJsonArray(value) || isPossibleJsonObject(value))) {
formattedAttributes.put(entry.getKey(), OBJECT_MAPPER.readTree(value));
} else {
formattedAttributes.put(entry.getKey(), value);
if (value != null) {
final String trimmedValue = value.trim();
if (isPossibleJsonArray(trimmedValue) || isPossibleJsonObject(trimmedValue)) {
formattedAttributes.put(entry.getKey(), OBJECT_MAPPER.readTree(trimmedValue));
continue;
}
}
formattedAttributes.put(entry.getKey(), value);
}

return formattedAttributes;
Expand Down
72 changes: 46 additions & 26 deletions ...rd-processors/src/test/java/org/apache/nifi/processors/standard/TestAttributesToJSON.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.nifi.processors.standard;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
Expand All @@ -42,6 +43,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNull;
Expand All @@ -54,6 +56,8 @@ public class TestAttributesToJSON {
private static final String TEST_ATTRIBUTE_KEY = "TestAttribute";
private static final String TEST_ATTRIBUTE_VALUE = "TestValue";

private static final ObjectMapper MAPPER = new ObjectMapper();

@Test
public void testInvalidUserSuppliedAttributeList() {
final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToJSON());
Expand Down Expand Up @@ -109,8 +113,7 @@ public void testNullValueForEmptyAttribute() throws Exception {
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);

assertNull(val.get(NON_PRESENT_ATTRIBUTE_KEY));
}
Expand Down Expand Up @@ -139,8 +142,7 @@ public void testEmptyStringValueForEmptyAttribute() throws Exception {
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);

assertEquals(val.get(NON_PRESENT_ATTRIBUTE_KEY), "");
}
Expand Down Expand Up @@ -188,8 +190,7 @@ public void testAttributes_emptyListUserSpecifiedAttributes() throws Exception {
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);
assertTrue(val.get(TEST_ATTRIBUTE_KEY).equals(TEST_ATTRIBUTE_VALUE));
}

Expand Down Expand Up @@ -234,8 +235,7 @@ public void testAttribute_singleUserDefinedAttribute() throws Exception {
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);
assertTrue(val.get(TEST_ATTRIBUTE_KEY).equals(TEST_ATTRIBUTE_VALUE));
assertTrue(val.size() == 1);
}
Expand All @@ -262,8 +262,7 @@ public void testAttribute_singleUserDefinedAttributeWithWhiteSpace() throws Exce
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);
assertTrue(val.get(TEST_ATTRIBUTE_KEY).equals(TEST_ATTRIBUTE_VALUE));
assertTrue(val.size() == 1);
}
Expand All @@ -290,8 +289,7 @@ public void testAttribute_singleNonExistingUserDefinedAttribute() throws Excepti
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);

//If a Attribute is requested but does not exist then it is placed in the JSON with an empty string
assertTrue(val.get("NonExistingAttribute").equals(""));
Expand Down Expand Up @@ -320,8 +318,7 @@ public void testAttribute_noIncludeCoreAttributesUserDefined() throws IOExceptio
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);
assertEquals(TEST_ATTRIBUTE_VALUE, val.get(TEST_ATTRIBUTE_KEY));
assertEquals(TEST_ATTRIBUTE_VALUE, val.get(CoreAttributes.PATH.key()));
assertEquals(2, val.size());
Expand Down Expand Up @@ -349,8 +346,7 @@ public void testAttribute_noIncludeCoreAttributesRegex() throws IOException {
String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(json, HashMap.class);
Map<String, String> val = MAPPER.readValue(json, HashMap.class);
assertEquals(TEST_ATTRIBUTE_VALUE, val.get(CoreAttributes.PATH.key()));
assertEquals(1, val.size());
}
Expand All @@ -372,8 +368,7 @@ public void testAttribute_noIncludeCoreAttributesContent() throws IOException {
testRunner.assertTransferCount(AttributesToJSON.REL_SUCCESS, 1);
testRunner.assertTransferCount(AttributesToJSON.REL_FAILURE, 0);

ObjectMapper mapper = new ObjectMapper();
Map<String, String> val = mapper.readValue(testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS).get(0).toByteArray(), HashMap.class);
Map<String, String> val = MAPPER.readValue(testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS).get(0).toByteArray(), HashMap.class);
assertEquals(TEST_ATTRIBUTE_VALUE, val.get(TEST_ATTRIBUTE_KEY));
assertEquals(1, val.size());
}
Expand All @@ -399,7 +394,7 @@ public void testAttribute_includeCoreAttributesContent() throws IOException {

assertEquals(AttributesToJSON.APPLICATION_JSON, flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));

Map<String, String> val = new ObjectMapper().readValue(flowFile.toByteArray(), HashMap.class);
Map<String, String> val = MAPPER.readValue(flowFile.toByteArray(), HashMap.class);
assertEquals(3, val.size());
Set<String> coreAttributes = Arrays.stream(CoreAttributes.values()).map(CoreAttributes::key).collect(Collectors.toSet());
val.keySet().forEach(k -> assertTrue(coreAttributes.contains(k)));
Expand All @@ -425,7 +420,7 @@ public void testAttribute_includeCoreAttributesAttribute() throws IOException {

assertNull(flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));

Map<String, String> val = new ObjectMapper().readValue(flowFile.getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME), HashMap.class);
Map<String, String> val = MAPPER.readValue(flowFile.getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME), HashMap.class);
assertEquals(3, val.size());
Set<String> coreAttributes = Arrays.stream(CoreAttributes.values()).map(CoreAttributes::key).collect(Collectors.toSet());
val.keySet().forEach(k -> assertTrue(coreAttributes.contains(k)));
Expand Down Expand Up @@ -455,7 +450,7 @@ public void testAttributesRegex() throws IOException {

MockFlowFile flowFile = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS).get(0);

Map<String, String> val = new ObjectMapper().readValue(flowFile.getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME), HashMap.class);
Map<String, String> val = MAPPER.readValue(flowFile.getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME), HashMap.class);
assertTrue(val.keySet().contains("delimited.header.column.1"));
assertTrue(val.keySet().contains("delimited.header.column.2"));
assertTrue(val.keySet().contains("delimited.header.column.3"));
Expand Down Expand Up @@ -485,7 +480,7 @@ public void testAttributeWithNestedJsonOutputAsJsonInContent(String nestedJson,
List<MockFlowFile> flowFilesForRelationship = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS);
MockFlowFile flowFile = flowFilesForRelationship.get(0);
assertEquals(AttributesToJSON.APPLICATION_JSON, flowFile.getAttribute(CoreAttributes.MIME_TYPE.key()));
Map<String, Object> val = new ObjectMapper().readValue(flowFile.toByteArray(), Map.class);
Map<String, Object> val = MAPPER.readValue(flowFile.toByteArray(), Map.class);
assertInstanceOf(expectedClass, val.get(TEST_ATTRIBUTE_KEY));
}

Expand Down Expand Up @@ -516,8 +511,7 @@ public void testAttributeWithNestedJsonOutputAsJsonInAttribute(String nestedJson

String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);
ObjectMapper mapper = new ObjectMapper();
Map<String, Object> val = mapper.readValue(json, Map.class);
Map<String, Object> val = MAPPER.readValue(json, Map.class);
assertInstanceOf(expectedClass, val.get(TEST_ATTRIBUTE_KEY));
}

Expand Down Expand Up @@ -560,8 +554,34 @@ public void testAttributeWithNestedJsonOutputAsStringInAttribute(String nestedJs

String json = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS)
.get(0).getAttribute(AttributesToJSON.JSON_ATTRIBUTE_NAME);
ObjectMapper mapper = new ObjectMapper();
Map<String, Object> val = mapper.readValue(json, Map.class);
Map<String, Object> val = MAPPER.readValue(json, Map.class);
assertInstanceOf(String.class, val.get(TEST_ATTRIBUTE_KEY));
}

@Test
public void testAttributeWithJsonBetweenLeadingAndTrailingSpaces() throws JsonProcessingException {
final String jsonBetweenLeadingAndTrailingSpaces = "\n{\n" +
" \"fruit\": \"Apple\",\n" +
" \"size\": \"Large\",\n" +
" \"color\": \"Red\"\n" +
"}\n";
assertDoesNotThrow(() -> MAPPER.readValue(jsonBetweenLeadingAndTrailingSpaces, Map.class));

final TestRunner testRunner = TestRunners.newTestRunner(new AttributesToJSON());
testRunner.setProperty(AttributesToJSON.DESTINATION, AttributesToJSON.DESTINATION_CONTENT);
testRunner.setProperty(AttributesToJSON.JSON_HANDLING_STRATEGY,
AttributesToJSON.JsonHandlingStrategy.NESTED.getValue());

ProcessSession session = testRunner.getProcessSessionFactory().createSession();
FlowFile ff = session.create();
ff = session.putAttribute(ff, TEST_ATTRIBUTE_KEY, jsonBetweenLeadingAndTrailingSpaces);
testRunner.enqueue(ff);
testRunner.run();

testRunner.assertTransferCount(AttributesToJSON.REL_FAILURE, 0);
testRunner.assertTransferCount(AttributesToJSON.REL_SUCCESS, 1);
MockFlowFile result = testRunner.getFlowFilesForRelationship(AttributesToJSON.REL_SUCCESS).get(0);
Map<String, Object> attributes = MAPPER.readValue(result.getContent(), Map.class);
assertInstanceOf(Map.class, attributes.get(TEST_ATTRIBUTE_KEY));
}
}

0 comments on commit 3e62e1e

Please sign in to comment.