[Ingest Processor] Add ignore_missing param to the uri_parts inge…

…st processor. (#95068)
elastic · Apr 13, 2023 · 9071d11 · 9071d11
1 parent 2c8e298
commit 9071d11
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 7 deletions.
diff --git a/docs/changelog/95068.yaml b/docs/changelog/95068.yaml
@@ -0,0 +1,6 @@
+pr: 95068
+summary: "[Ingest Processor] Add `ignore_missing` param to the `uri_parts` ingest\
+  \ processor"
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/docs/reference/ingest/processors/uri-parts.asciidoc b/docs/reference/ingest/processors/uri-parts.asciidoc
@@ -21,6 +21,7 @@ unparsed URI to `<target_field>.original`.
 | `remove_if_successful` | no    | false   | If `true`, the processor removes
 the `field` after parsing the URI string. If parsing fails, the processor does not
 remove the `field`.
+| `ignore_missing`   | no        | `false`          | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 
 include::common-options.asciidoc[]
 |======

diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java
@@ -29,13 +29,23 @@ public class UriPartsProcessor extends AbstractProcessor {
     private final String targetField;
     private final boolean removeIfSuccessful;
     private final boolean keepOriginal;
-
-    UriPartsProcessor(String tag, String description, String field, String targetField, boolean removeIfSuccessful, boolean keepOriginal) {
+    private final boolean ignoreMissing;
+
+    UriPartsProcessor(
+        String tag,
+        String description,
+        String field,
+        String targetField,
+        boolean removeIfSuccessful,
+        boolean keepOriginal,
+        boolean ignoreMissing
+    ) {
         super(tag, description);
         this.field = field;
         this.targetField = targetField;
         this.removeIfSuccessful = removeIfSuccessful;
         this.keepOriginal = keepOriginal;
+        this.ignoreMissing = ignoreMissing;
     }
 
     public String getField() {
@@ -54,10 +64,17 @@ public boolean getKeepOriginal() {
         return keepOriginal;
     }
 
+    public Object getIgnoreMissing() {
+        return ignoreMissing;
+    }
+
     @Override
     public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
-        String value = ingestDocument.getFieldValue(field, String.class);
+        String value = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
 
+        if (ignoreMissing && null == value) {
+            return ingestDocument;
+        }
         var uriParts = apply(value);
         if (keepOriginal) {
             uriParts.put("original", value);
@@ -165,7 +182,8 @@ public UriPartsProcessor create(
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", "url");
             boolean removeIfSuccessful = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "remove_if_successful", false);
             boolean keepOriginal = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "keep_original", true);
-            return new UriPartsProcessor(processorTag, description, field, targetField, removeIfSuccessful, keepOriginal);
+            boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            return new UriPartsProcessor(processorTag, description, field, targetField, removeIfSuccessful, keepOriginal, ignoreMissing);
         }
     }
 }
diff --git a/...t-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java b/...t-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java
@@ -40,13 +40,17 @@ public void testCreate() throws Exception {
         boolean keepOriginal = randomBoolean();
         config.put("keep_original", keepOriginal);
 
+        boolean ignoreMissing = randomBoolean();
+        config.put("ignore_missing", ignoreMissing);
+
         String processorTag = randomAlphaOfLength(10);
         UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config);
         assertThat(uriPartsProcessor.getTag(), equalTo(processorTag));
         assertThat(uriPartsProcessor.getField(), equalTo(field));
         assertThat(uriPartsProcessor.getTargetField(), equalTo(targetField));
         assertThat(uriPartsProcessor.getRemoveIfSuccessful(), equalTo(removeIfSuccessful));
         assertThat(uriPartsProcessor.getKeepOriginal(), equalTo(keepOriginal));
+        assertThat(uriPartsProcessor.getIgnoreMissing(), equalTo(ignoreMissing));
     }
 
     public void testCreateNoFieldPresent() throws Exception {

diff --git a/...s/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java b/...s/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java
@@ -17,6 +17,7 @@
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasSize;
 
 public class UriPartsProcessorTests extends ESTestCase {
 
@@ -182,7 +183,7 @@ public void testUrlWithCharactersNotToleratedByUri() throws Exception {
 
     public void testRemoveIfSuccessfulDoesNotRemoveTargetField() throws Exception {
         String field = "field";
-        UriPartsProcessor processor = new UriPartsProcessor(null, null, field, field, true, false);
+        UriPartsProcessor processor = new UriPartsProcessor(null, null, field, field, true, false, false);
 
         Map<String, Object> source = new HashMap<>();
         source.put(field, "http://www.google.com");
@@ -198,7 +199,7 @@ public void testRemoveIfSuccessfulDoesNotRemoveTargetField() throws Exception {
 
     public void testInvalidUri() {
         String uri = "not:\\/_a_valid_uri";
-        UriPartsProcessor processor = new UriPartsProcessor(null, null, "field", "url", true, false);
+        UriPartsProcessor processor = new UriPartsProcessor(null, null, "field", "url", true, false, false);
 
         Map<String, Object> source = new HashMap<>();
         source.put("field", uri);
@@ -208,13 +209,50 @@ public void testInvalidUri() {
         assertThat(e.getMessage(), containsString("unable to parse URI [" + uri + "]"));
     }
 
+    public void testNullValue() {
+        Map<String, Object> source = new HashMap<>();
+        source.put("field", null);
+        IngestDocument input = TestIngestDocument.withDefaultVersion(source);
+
+        UriPartsProcessor processor = new UriPartsProcessor(null, null, "field", "url", true, false, false);
+
+        expectThrows(NullPointerException.class, () -> processor.execute(input));
+    }
+
+    public void testMissingField() {
+        Map<String, Object> source = new HashMap<>();
+        IngestDocument input = TestIngestDocument.withDefaultVersion(source);
+
+        UriPartsProcessor processor = new UriPartsProcessor(null, null, "field", "url", true, false, false);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(input));
+        assertThat(e.getMessage(), containsString("field [field] not present as part of path [field]"));
+    }
+
+    public void testIgnoreMissingField() throws Exception {
+        Map<String, Object> source = new HashMap<>();
+        // Adding a random field, so we can check the doc is leaved unchanged.
+        source.put(randomIdentifier(), randomIdentifier());
+        IngestDocument input = TestIngestDocument.withDefaultVersion(source);
+        Map<String, Object> expectedSourceAndMetadata = Map.copyOf(input.getSourceAndMetadata());
+
+        UriPartsProcessor processor = new UriPartsProcessor(null, null, "field", "url", true, false, true);
+        IngestDocument output = processor.execute(input);
+
+        assertThat(output.getSourceAndMetadata().entrySet(), hasSize(expectedSourceAndMetadata.size()));
+
+        for (Map.Entry<String, Object> entry : expectedSourceAndMetadata.entrySet()) {
+            assertThat(output.getSourceAndMetadata(), hasEntry(entry.getKey(), entry.getValue()));
+        }
+    }
+
     private void testUriParsing(String uri, Map<String, Object> expectedValues) throws Exception {
         testUriParsing(false, false, uri, expectedValues);
     }
 
     private void testUriParsing(boolean keepOriginal, boolean removeIfSuccessful, String uri, Map<String, Object> expectedValues)
         throws Exception {
-        UriPartsProcessor processor = new UriPartsProcessor(null, null, "field", "url", removeIfSuccessful, keepOriginal);
+        UriPartsProcessor processor = new UriPartsProcessor(null, null, "field", "url", removeIfSuccessful, keepOriginal, false);
 
         Map<String, Object> source = new HashMap<>();
         source.put("field", uri);