Skip to content

Commit 00c70b8

Browse files
benwtrentbenwtrent
committed
[ML] adds support for non-numeric mapped types and mapping overrides
1 parent a87b139 commit 00c70b8

File tree

20 files changed

+587
-86
lines changed

20 files changed

+587
-86
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import org.elasticsearch.common.xcontent.XContentParser;
2929

3030
import java.io.IOException;
31+
import java.util.Collections;
32+
import java.util.Map;
3133
import java.util.Objects;
3234

3335
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
@@ -39,6 +41,7 @@ public class DataFrameTransformConfig implements ToXContentObject {
3941
public static final ParseField SOURCE = new ParseField("source");
4042
public static final ParseField DEST = new ParseField("dest");
4143
public static final ParseField QUERY = new ParseField("query");
44+
public static final ParseField MAPPING_OVERRIDE = new ParseField("mapping_override");
4245
// types of transforms
4346
public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
4447

@@ -47,6 +50,7 @@ public class DataFrameTransformConfig implements ToXContentObject {
4750
private final String dest;
4851
private final QueryConfig queryConfig;
4952
private final PivotConfig pivotConfig;
53+
private final Map<String, String> mappingOverride;
5054

5155
public static final ConstructingObjectParser<DataFrameTransformConfig, String> PARSER =
5256
new ConstructingObjectParser<>("data_frame_transform", true,
@@ -56,7 +60,9 @@ public class DataFrameTransformConfig implements ToXContentObject {
5660
String dest = (String) args[2];
5761
QueryConfig queryConfig = (QueryConfig) args[3];
5862
PivotConfig pivotConfig = (PivotConfig) args[4];
59-
return new DataFrameTransformConfig(id, source, dest, queryConfig, pivotConfig);
63+
@SuppressWarnings("unchecked")
64+
Map<String, String> mappingOverrides = (Map<String, String>) args[5];
65+
return new DataFrameTransformConfig(id, source, dest, queryConfig, pivotConfig, mappingOverrides);
6066
});
6167

6268
static {
@@ -65,23 +71,25 @@ public class DataFrameTransformConfig implements ToXContentObject {
6571
PARSER.declareString(constructorArg(), DEST);
6672
PARSER.declareObject(optionalConstructorArg(), (p, c) -> QueryConfig.fromXContent(p), QUERY);
6773
PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM);
74+
PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), MAPPING_OVERRIDE);
6875
}
6976

7077
public static DataFrameTransformConfig fromXContent(final XContentParser parser) {
7178
return PARSER.apply(parser, null);
7279
}
7380

74-
7581
public DataFrameTransformConfig(final String id,
7682
final String source,
7783
final String dest,
7884
final QueryConfig queryConfig,
79-
final PivotConfig pivotConfig) {
85+
final PivotConfig pivotConfig,
86+
final Map<String, String> mappingOverride) {
8087
this.id = Objects.requireNonNull(id);
8188
this.source = Objects.requireNonNull(source);
8289
this.dest = Objects.requireNonNull(dest);
8390
this.queryConfig = queryConfig;
8491
this.pivotConfig = pivotConfig;
92+
this.mappingOverride = mappingOverride == null ? null : Collections.unmodifiableMap(mappingOverride);
8593
}
8694

8795
public String getId() {
@@ -104,6 +112,10 @@ public QueryConfig getQueryConfig() {
104112
return queryConfig;
105113
}
106114

115+
public Map<String, String> getMappingOverrides() {
116+
return mappingOverride;
117+
}
118+
107119
public boolean isValid() {
108120
if (queryConfig != null && queryConfig.isValid() == false) {
109121
return false;
@@ -128,6 +140,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
128140
if (pivotConfig != null) {
129141
builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
130142
}
143+
if (mappingOverride != null) {
144+
builder.field(MAPPING_OVERRIDE.getPreferredName(), mappingOverride);
145+
}
131146
builder.endObject();
132147
return builder;
133148
}
@@ -148,12 +163,13 @@ public boolean equals(Object other) {
148163
&& Objects.equals(this.source, that.source)
149164
&& Objects.equals(this.dest, that.dest)
150165
&& Objects.equals(this.queryConfig, that.queryConfig)
151-
&& Objects.equals(this.pivotConfig, that.pivotConfig);
166+
&& Objects.equals(this.pivotConfig, that.pivotConfig)
167+
&& Objects.equals(this.mappingOverride, that.mappingOverride);
152168
}
153169

154170
@Override
155171
public int hashCode() {
156-
return Objects.hash(id, source, dest, queryConfig, pivotConfig);
172+
return Objects.hash(id, source, dest, queryConfig, pivotConfig, mappingOverride);
157173
}
158174

159175
@Override

‎client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,12 @@ public void testCreateDelete() throws IOException {
8484
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
8585

8686
String id = "test-crud";
87-
DataFrameTransformConfig transform = new DataFrameTransformConfig(id, sourceIndex, "pivot-dest", queryConfig, pivotConfig);
87+
DataFrameTransformConfig transform = new DataFrameTransformConfig(id,
88+
sourceIndex,
89+
"pivot-dest",
90+
queryConfig,
91+
pivotConfig,
92+
Collections.emptyMap());
8893

8994
DataFrameClient client = highLevelClient().dataFrame();
9095
AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,
@@ -114,7 +119,12 @@ public void testStartStop() throws IOException {
114119
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
115120

116121
String id = "test-stop-start";
117-
DataFrameTransformConfig transform = new DataFrameTransformConfig(id, sourceIndex, "pivot-dest", queryConfig, pivotConfig);
122+
DataFrameTransformConfig transform = new DataFrameTransformConfig(id,
123+
sourceIndex,
124+
"pivot-dest",
125+
queryConfig,
126+
pivotConfig,
127+
Collections.emptyMap());
118128

119129
DataFrameClient client = highLevelClient().dataFrame();
120130
AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,

‎client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,29 @@
2828

2929
import java.io.IOException;
3030
import java.util.Collections;
31+
import java.util.HashMap;
32+
import java.util.Map;
3133
import java.util.function.Predicate;
3234

3335
public class DataFrameTransformConfigTests extends AbstractXContentTestCase<DataFrameTransformConfig> {
3436

3537
public static DataFrameTransformConfig randomDataFrameTransformConfig() {
3638
return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
37-
randomAlphaOfLengthBetween(1, 10), QueryConfigTests.randomQueryConfig(), PivotConfigTests.randomPivotConfig());
39+
randomAlphaOfLengthBetween(1, 10), QueryConfigTests.randomQueryConfig(), PivotConfigTests.randomPivotConfig(),
40+
randomNullableStringMap());
3841
}
3942

43+
public static Map<String, String> randomNullableStringMap() {
44+
Map<String, String> stringStringMap = null;
45+
if (randomBoolean()) {
46+
stringStringMap = new HashMap<>();
47+
int kvCount = randomInt(10);
48+
for (int i = 0; i < kvCount; i++) {
49+
stringStringMap.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
50+
}
51+
}
52+
return stringStringMap;
53+
}
4054
@Override
4155
protected DataFrameTransformConfig createTestInstance() {
4256
return randomDataFrameTransformConfig();

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,9 @@
4949
import java.io.IOException;
5050
import java.util.ArrayList;
5151
import java.util.Collections;
52+
import java.util.HashMap;
5253
import java.util.List;
54+
import java.util.Map;
5355
import java.util.concurrent.CountDownLatch;
5456
import java.util.concurrent.TimeUnit;
5557

@@ -115,6 +117,10 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
115117
AggregationBuilders.avg("avg_rating").field("stars")); // <1>
116118
AggregationConfig aggConfig = new AggregationConfig(aggBuilder);
117119
// end::put-data-frame-transform-agg-config
120+
// tag::put-data-frame-transform-mapping-override
121+
Map<String, String> mappingOverride = new HashMap<>();
122+
mappingOverride.put("avg_rating", "keyword"); // <1>
123+
// end::put-data-frame-transform-mapping-override
118124
// tag::put-data-frame-transform-pivot-config
119125
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
120126
// end::put-data-frame-transform-pivot-config
@@ -124,7 +130,8 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
124130
"source-index", // <2>
125131
"pivot-destination", // <3>
126132
queryConfig, // <4>
127-
pivotConfig); // <5>
133+
pivotConfig, // <5>
134+
mappingOverride); // <6>
128135
// end::put-data-frame-transform-config
129136

130137
{
@@ -144,7 +151,7 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
144151
{
145152
DataFrameTransformConfig configWithDifferentId = new DataFrameTransformConfig("reviewer-avg-rating2",
146153
transformConfig.getSource(), transformConfig.getDestination(), transformConfig.getQueryConfig(),
147-
transformConfig.getPivotConfig());
154+
transformConfig.getPivotConfig(), null);
148155
PutDataFrameTransformRequest request = new PutDataFrameTransformRequest(configWithDifferentId);
149156

150157
// tag::put-data-frame-transform-execute-listener
@@ -189,7 +196,7 @@ public void testStartStop() throws IOException, InterruptedException {
189196
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
190197

191198
DataFrameTransformConfig transformConfig = new DataFrameTransformConfig("mega-transform",
192-
"source-data", "pivot-dest", queryConfig, pivotConfig);
199+
"source-data", "pivot-dest", queryConfig, pivotConfig, null);
193200

194201
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
195202
transformsToClean.add(transformConfig.getId());
@@ -306,9 +313,9 @@ public void testDeleteDataFrameTransform() throws IOException, InterruptedExcept
306313
PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
307314

308315
DataFrameTransformConfig transformConfig1 = new DataFrameTransformConfig("mega-transform",
309-
"source-data", "pivot-dest", queryConfig, pivotConfig);
316+
"source-data", "pivot-dest", queryConfig, pivotConfig, null);
310317
DataFrameTransformConfig transformConfig2 = new DataFrameTransformConfig("mega-transform2",
311-
"source-data", "pivot-dest2", queryConfig, pivotConfig);
318+
"source-data", "pivot-dest2", queryConfig, pivotConfig, null);
312319

313320
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig1), RequestOptions.DEFAULT);
314321
client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig2), RequestOptions.DEFAULT);

‎docs/java-rest/high-level/dataframe/put_data_frame.asciidoc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ include-tagged::{doc-tests-file}[{api}-config]
3636
<3> The destination index
3737
<4> Optionally a QueryConfig
3838
<5> The PivotConfig
39+
<6> The provided `Map<String, String>` values to override deduced destination index mappings.
3940

4041
[id="{upid}-{api}-query-config"]
4142
==== QueryConfig
@@ -84,6 +85,19 @@ include-tagged::{doc-tests-file}[{api}-agg-config]
8485
--------------------------------------------------
8586
<1> Aggregate the average star rating
8687

88+
==== Overriding Deduced Destination Mapping Field Types
89+
90+
When creating the destination index for the {dataframe-transform}, a best
91+
effort mapping is created. This option allows overriding specific mapped
92+
field types.
93+
94+
["source","java",subs="attributes,callouts,macros"]
95+
--------------------------------------------------
96+
include-tagged::{doc-tests-file}[{api}-mapping-override]
97+
--------------------------------------------------
98+
<1> Override the deduced mapping for field `avg_rating` to be a type of
99+
`keyword`
100+
87101
include::../execution.asciidoc[]
88102

89103
[id="{upid}-{api}-response"]

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameMessages.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public class DataFrameMessages {
4545
"Failed to create composite aggregation from pivot function";
4646
public static final String DATA_FRAME_TRANSFORM_CONFIGURATION_INVALID =
4747
"Data frame transform configuration [{0}] has invalid elements";
48-
48+
public static final String DATA_FRAME_UNABLE_TO_GATHER_FIELD_MAPPINGS = "Failed to gather field mappings for index [{0}]";
4949
public static final String LOG_DATA_FRAME_TRANSFORM_CONFIGURATION_BAD_QUERY =
5050
"Failed to parse query for data frame transform";
5151
public static final String LOG_DATA_FRAME_TRANSFORM_CONFIGURATION_BAD_GROUP_BY =

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformConfig.java

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
3939
public static final String NAME = "data_frame_transform_config";
4040
public static final ParseField HEADERS = new ParseField("headers");
4141
public static final ParseField QUERY = new ParseField("query");
42+
public static final ParseField MAPPING_OVERRIDES = new ParseField("mapping_override");
4243

4344
// types of transforms
4445
public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
@@ -56,6 +57,7 @@ public class DataFrameTransformConfig extends AbstractDiffable<DataFrameTransfor
5657

5758
private final QueryConfig queryConfig;
5859
private final PivotConfig pivotConfig;
60+
private final Map<String, String> mappingOverrides;
5961

6062
private static ConstructingObjectParser<DataFrameTransformConfig, String> createParser(boolean lenient) {
6163
ConstructingObjectParser<DataFrameTransformConfig, String> parser = new ConstructingObjectParser<>(NAME, lenient,
@@ -93,7 +95,10 @@ private static ConstructingObjectParser<DataFrameTransformConfig, String> create
9395
}
9496

9597
PivotConfig pivotConfig = (PivotConfig) args[6];
96-
return new DataFrameTransformConfig(id, source, dest, headers, queryConfig, pivotConfig);
98+
99+
@SuppressWarnings("unchecked")
100+
Map<String, String> mappingOverrides = (Map<String, String>) args[7];
101+
return new DataFrameTransformConfig(id, source, dest, headers, queryConfig, pivotConfig, mappingOverrides);
97102
});
98103

99104
parser.declareString(optionalConstructorArg(), DataFrameField.ID);
@@ -104,6 +109,7 @@ private static ConstructingObjectParser<DataFrameTransformConfig, String> create
104109
parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS);
105110
parser.declareObject(optionalConstructorArg(), (p, c) -> QueryConfig.fromXContent(p, lenient), QUERY);
106111
parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM);
112+
parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), MAPPING_OVERRIDES);
107113

108114
return parser;
109115
}
@@ -117,13 +123,15 @@ public DataFrameTransformConfig(final String id,
117123
final String dest,
118124
final Map<String, String> headers,
119125
final QueryConfig queryConfig,
120-
final PivotConfig pivotConfig) {
126+
final PivotConfig pivotConfig,
127+
final Map<String, String> mappingOverrides) {
121128
this.id = ExceptionsHelper.requireNonNull(id, DataFrameField.ID.getPreferredName());
122129
this.source = ExceptionsHelper.requireNonNull(source, DataFrameField.SOURCE.getPreferredName());
123130
this.dest = ExceptionsHelper.requireNonNull(dest, DataFrameField.DESTINATION.getPreferredName());
124131
this.queryConfig = ExceptionsHelper.requireNonNull(queryConfig, QUERY.getPreferredName());
125132
this.setHeaders(headers == null ? Collections.emptyMap() : headers);
126133
this.pivotConfig = pivotConfig;
134+
this.mappingOverrides = mappingOverrides == null ? Collections.emptyMap() : Collections.unmodifiableMap(mappingOverrides);
127135

128136
// at least one function must be defined
129137
if (this.pivotConfig == null) {
@@ -138,6 +146,7 @@ public DataFrameTransformConfig(final StreamInput in) throws IOException {
138146
setHeaders(in.readMap(StreamInput::readString, StreamInput::readString));
139147
queryConfig = in.readOptionalWriteable(QueryConfig::new);
140148
pivotConfig = in.readOptionalWriteable(PivotConfig::new);
149+
mappingOverrides = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString));
141150
}
142151

143152
public String getId() {
@@ -168,6 +177,10 @@ public QueryConfig getQueryConfig() {
168177
return queryConfig;
169178
}
170179

180+
public Map<String, String> getMappingOverrides() {
181+
return mappingOverrides;
182+
}
183+
171184
public boolean isValid() {
172185
// collect validation results from all child objects
173186
if (queryConfig != null && queryConfig.isValid() == false) {
@@ -189,6 +202,7 @@ public void writeTo(final StreamOutput out) throws IOException {
189202
out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString);
190203
out.writeOptionalWriteable(queryConfig);
191204
out.writeOptionalWriteable(pivotConfig);
205+
out.writeMap(mappingOverrides, StreamOutput::writeString, StreamOutput::writeString);
192206
}
193207

194208
@Override
@@ -209,6 +223,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
209223
if (headers.isEmpty() == false && params.paramAsBoolean(DataFrameField.FOR_INTERNAL_STORAGE, false) == true) {
210224
builder.field(HEADERS.getPreferredName(), headers);
211225
}
226+
if (mappingOverrides.isEmpty() == false) {
227+
builder.field(MAPPING_OVERRIDES.getPreferredName(), mappingOverrides);
228+
}
212229

213230
builder.endObject();
214231
return builder;
@@ -231,12 +248,13 @@ public boolean equals(Object other) {
231248
&& Objects.equals(this.dest, that.dest)
232249
&& Objects.equals(this.headers, that.headers)
233250
&& Objects.equals(this.queryConfig, that.queryConfig)
234-
&& Objects.equals(this.pivotConfig, that.pivotConfig);
251+
&& Objects.equals(this.pivotConfig, that.pivotConfig)
252+
&& Objects.equals(this.mappingOverrides, that.mappingOverrides);
235253
}
236254

237255
@Override
238256
public int hashCode() {
239-
return Objects.hash(id, source, dest, headers, queryConfig, pivotConfig);
257+
return Objects.hash(id, source, dest, headers, queryConfig, pivotConfig, mappingOverrides);
240258
}
241259

242260
@Override

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/dataframe/action/PreviewDataFrameTransformActionRequestTests.java

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import org.junit.Before;
2020

2121
import java.io.IOException;
22+
import java.util.HashMap;
23+
import java.util.Map;
2224

2325
import static java.util.Collections.emptyList;
2426

@@ -62,8 +64,17 @@ protected boolean supportsUnknownFields() {
6264

6365
@Override
6466
protected Request createTestInstance() {
67+
Map<String, String> mappingOverrides = null;
68+
if (randomBoolean()) {
69+
mappingOverrides = new HashMap<>();
70+
int kvCount = randomInt(10);
71+
for (int i = 0; i < kvCount; i++) {
72+
mappingOverrides.put(randomAlphaOfLength(10), randomAlphaOfLength(10));
73+
}
74+
}
6575
DataFrameTransformConfig config = new DataFrameTransformConfig("transform-preview", randomAlphaOfLength(10),
66-
"unused-transform-preview-index", null, QueryConfigTests.randomQueryConfig(), PivotConfigTests.randomPivotConfig());
76+
"unused-transform-preview-index", null, QueryConfigTests.randomQueryConfig(),
77+
PivotConfigTests.randomPivotConfig(), mappingOverrides);
6778
return new Request(config);
6879
}
6980

0 commit comments

Comments
 (0)