Skip to content

Commit 7cd146b

Browse files
danhermanndanhermann
committed
URI parts ingest processor (elastic#65150)
1 parent 427bb64 commit 7cd146b

File tree

5 files changed

+461
-0
lines changed

5 files changed

+461
-0
lines changed

‎x-pack/plugin/ingest/build.gradle

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
apply plugin: 'elasticsearch.esplugin'
21+
apply plugin: 'elasticsearch.internal-cluster-test'
22+
esplugin {
23+
name 'x-pack-ingest'
24+
description 'Elasticsearch Expanded Pack Plugin - Ingest'
25+
classname 'org.elasticsearch.xpack.ingest.IngestPlugin'
26+
extendedPlugins = ['x-pack-core']
27+
}
28+
archivesBaseName = 'x-pack-ingest'
29+
30+
dependencies {
31+
compileOnly project(path: xpackModule('core'), configuration: 'default')
32+
testImplementation project(path: xpackModule('core'), configuration: 'testArtifacts')
33+
testImplementation project(path: ':modules:ingest-common')
34+
testImplementation project(path: ':modules:lang-mustache')
35+
testImplementation project(path: ':modules:geo')
36+
testImplementation project(path: xpackModule('monitoring'), configuration: 'testArtifacts')
37+
}
38+
39+
addQaCheckDependencies()
40+
41+
testingConventions.enabled = false

‎x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.ingest;
8+
9+
import org.elasticsearch.ingest.Processor;
10+
import org.elasticsearch.plugins.Plugin;
11+
12+
import java.util.Map;
13+
14+
public class IngestPlugin extends Plugin implements org.elasticsearch.plugins.IngestPlugin {
15+
16+
@Override
17+
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
18+
return Map.of(UriPartsProcessor.TYPE, new UriPartsProcessor.Factory());
19+
}
20+
}

‎x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/UriPartsProcessor.java

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.ingest;
8+
9+
import org.elasticsearch.ingest.AbstractProcessor;
10+
import org.elasticsearch.ingest.ConfigurationUtils;
11+
import org.elasticsearch.ingest.IngestDocument;
12+
import org.elasticsearch.ingest.Processor;
13+
14+
import java.net.URI;
15+
import java.net.URISyntaxException;
16+
import java.util.HashMap;
17+
import java.util.Map;
18+
19+
public class UriPartsProcessor extends AbstractProcessor {
20+
21+
public static final String TYPE = "uri_parts";
22+
23+
private final String field;
24+
private final String targetField;
25+
private final boolean removeIfSuccessful;
26+
private final boolean keepOriginal;
27+
28+
UriPartsProcessor(String tag, String description, String field, String targetField, boolean removeIfSuccessful, boolean keepOriginal) {
29+
super(tag, description);
30+
this.field = field;
31+
this.targetField = targetField;
32+
this.removeIfSuccessful = removeIfSuccessful;
33+
this.keepOriginal = keepOriginal;
34+
}
35+
36+
public String getField() {
37+
return field;
38+
}
39+
40+
public String getTargetField() {
41+
return targetField;
42+
}
43+
44+
public boolean getRemoveIfSuccessful() {
45+
return removeIfSuccessful;
46+
}
47+
48+
public boolean getKeepOriginal() {
49+
return keepOriginal;
50+
}
51+
52+
@Override
53+
public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
54+
String value = ingestDocument.getFieldValue(field, String.class);
55+
56+
URI uri;
57+
try {
58+
uri = new URI(value);
59+
} catch (URISyntaxException e) {
60+
throw new IllegalArgumentException("unable to parse URI [" + value + "]");
61+
}
62+
var uriParts = new HashMap<String, Object>();
63+
uriParts.put("domain", uri.getHost());
64+
if (uri.getFragment() != null) {
65+
uriParts.put("fragment", uri.getFragment());
66+
}
67+
if (keepOriginal) {
68+
uriParts.put("original", value);
69+
}
70+
final String path = uri.getPath();
71+
if (path != null) {
72+
uriParts.put("path", path);
73+
if (path.contains(".")) {
74+
int periodIndex = path.lastIndexOf('.');
75+
uriParts.put("extension", periodIndex < path.length() ? path.substring(periodIndex + 1) : "");
76+
}
77+
}
78+
if (uri.getPort() != -1) {
79+
uriParts.put("port", uri.getPort());
80+
}
81+
if (uri.getQuery() != null) {
82+
uriParts.put("query", uri.getQuery());
83+
}
84+
uriParts.put("scheme", uri.getScheme());
85+
final String userInfo = uri.getUserInfo();
86+
if (userInfo != null) {
87+
uriParts.put("user_info", userInfo);
88+
if (userInfo.contains(":")) {
89+
int colonIndex = userInfo.indexOf(":");
90+
uriParts.put("username", userInfo.substring(0, colonIndex));
91+
uriParts.put("password", colonIndex < userInfo.length() ? userInfo.substring(colonIndex + 1) : "");
92+
}
93+
}
94+
95+
if (removeIfSuccessful && targetField.equals(field) == false) {
96+
ingestDocument.removeField(field);
97+
}
98+
ingestDocument.setFieldValue(targetField, uriParts);
99+
return ingestDocument;
100+
}
101+
102+
@Override
103+
public String getType() {
104+
return TYPE;
105+
}
106+
107+
public static final class Factory implements Processor.Factory {
108+
109+
@Override
110+
public UriPartsProcessor create(
111+
Map<String, Processor.Factory> registry,
112+
String processorTag,
113+
String description,
114+
Map<String, Object> config
115+
) throws Exception {
116+
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
117+
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", "url");
118+
boolean removeIfSuccessful = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "remove_if_successful", false);
119+
boolean keepOriginal = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "keep_original", true);
120+
return new UriPartsProcessor(processorTag, description, field, targetField, removeIfSuccessful, keepOriginal);
121+
}
122+
}
123+
}

‎x-pack/plugin/ingest/src/test/java/org/elasticsearch/xpack/ingest/UriPartsProcessorFactoryTests.java

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.ingest;
8+
9+
import org.elasticsearch.ElasticsearchParseException;
10+
import org.elasticsearch.test.ESTestCase;
11+
import org.junit.Before;
12+
13+
import java.util.HashMap;
14+
import java.util.Map;
15+
16+
import static org.hamcrest.CoreMatchers.equalTo;
17+
18+
public class UriPartsProcessorFactoryTests extends ESTestCase {
19+
20+
private UriPartsProcessor.Factory factory;
21+
22+
@Before
23+
public void init() {
24+
factory = new UriPartsProcessor.Factory();
25+
}
26+
27+
public void testCreate() throws Exception {
28+
Map<String, Object> config = new HashMap<>();
29+
String field = randomAlphaOfLength(6);
30+
config.put("field", field);
31+
String targetField = "url";
32+
if (randomBoolean()) {
33+
targetField = randomAlphaOfLength(6);
34+
config.put("target_field", targetField);
35+
}
36+
boolean removeIfSuccessful = randomBoolean();
37+
config.put("remove_if_successful", removeIfSuccessful);
38+
boolean keepOriginal = randomBoolean();
39+
config.put("keep_original", keepOriginal);
40+
41+
String processorTag = randomAlphaOfLength(10);
42+
UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config);
43+
assertThat(uriPartsProcessor.getTag(), equalTo(processorTag));
44+
assertThat(uriPartsProcessor.getField(), equalTo(field));
45+
assertThat(uriPartsProcessor.getTargetField(), equalTo(targetField));
46+
assertThat(uriPartsProcessor.getRemoveIfSuccessful(), equalTo(removeIfSuccessful));
47+
assertThat(uriPartsProcessor.getKeepOriginal(), equalTo(keepOriginal));
48+
}
49+
50+
public void testCreateNoFieldPresent() throws Exception {
51+
Map<String, Object> config = new HashMap<>();
52+
config.put("value", "value1");
53+
try {
54+
factory.create(null, null, null, config);
55+
fail("factory create should have failed");
56+
} catch (ElasticsearchParseException e) {
57+
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
58+
}
59+
}
60+
61+
public void testCreateNullField() throws Exception {
62+
Map<String, Object> config = new HashMap<>();
63+
config.put("field", null);
64+
try {
65+
factory.create(null, null, null, config);
66+
fail("factory create should have failed");
67+
} catch (ElasticsearchParseException e) {
68+
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
69+
}
70+
}
71+
}

0 commit comments

Comments
 (0)