Skip to content

[ML] Transition to typeless (mapping) APIs #39256

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Feb 28, 2019
Merged

[ML] Transition to typeless (mapping) APIs #39256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
bfbd2a9
Remove type from API calls
davidkyle Feb 20, 2019
6d9ff65
Remove type from yml tests
davidkyle Feb 20, 2019
676d2e9
Remove type from test
davidkyle Feb 21, 2019
c6c7cd0
More test fixes
davidkyle Feb 21, 2019
be0684c
Mappings merged with template mappings on index create must be of the…
davidkyle Feb 21, 2019
28a0999
Use constant
davidkyle Feb 21, 2019
78c0ec4
Remove metaindex type
davidkyle Feb 21, 2019
1532fa9
Check type before updating mapping
davidkyle Feb 21, 2019
895a1e8
more test fixes
davidkyle Feb 21, 2019
bf4dabe
explain need for mapping type when mappings are merged
davidkyle Feb 21, 2019
1ac2127
use id not type in error message
davidkyle Feb 21, 2019
54bd51a
fix update mapping on upgrade test
davidkyle Feb 21, 2019
edf0251
Typeless audit mappings
davidkyle Feb 22, 2019
7f3dc8b
Add _doc type to template mappings
davidkyle Feb 25, 2019
b2a474d
Parameterise type in results mapping
davidkyle Feb 25, 2019
ddadba4
And the annotations index
davidkyle Feb 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.io.IOException;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;

public final class MlMetaIndex {
/**
Expand All @@ -21,14 +22,12 @@ public final class MlMetaIndex {
*/
public static final String INDEX_NAME = ".ml-meta";

public static final String TYPE = "doc";

private MlMetaIndex() {}

public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
ElasticsearchMappings.addDefaultMapping(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
Expand Down
5 changes: 3 additions & 2 deletions ...lugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.SortedMap;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;

Expand Down Expand Up @@ -71,7 +72,7 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c

CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, annotationsMapping);
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
Expand Down Expand Up @@ -111,7 +112,7 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
public static XContentBuilder annotationsMapping() throws IOException {
XContentBuilder builder = jsonBuilder()
.startObject()
.startObject(ElasticsearchMappings.DOC_TYPE);
.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Annotation.ANNOTATION.getPreferredName())
Expand Down
105 changes: 59 additions & 46 deletions .../src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.CheckedBiFunction;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
Expand Down Expand Up @@ -62,6 +62,7 @@
import java.util.Map;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;

Expand All @@ -86,8 +87,6 @@
*/
public class ElasticsearchMappings {

public static final String DOC_TYPE = "doc";

/**
* String constants used in mappings
*/
Expand Down Expand Up @@ -137,7 +136,7 @@ private ElasticsearchMappings() {
public static XContentBuilder configMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
Expand Down Expand Up @@ -420,14 +419,14 @@ public static void addMetaInformation(XContentBuilder builder) throws IOExceptio
.endObject();
}

public static XContentBuilder resultsMapping() throws IOException {
return resultsMapping(Collections.emptyList());
public static XContentBuilder resultsMapping(String mappingType) throws IOException {
return resultsMapping(mappingType, Collections.emptyList());
}

public static XContentBuilder resultsMapping(Collection<String> extraTermFields) throws IOException {
public static XContentBuilder resultsMapping(String mappingType, Collection<String> extraTermFields) throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(mappingType);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
Expand Down Expand Up @@ -456,11 +455,12 @@ public static XContentBuilder resultsMapping(Collection<String> extraTermFields)

// end properties
builder.endObject();
// end mapping
// end type
builder.endObject();
// end doc
// end mapping
builder.endObject();


return builder;
}

Expand Down Expand Up @@ -575,18 +575,25 @@ private static void addResultsMapping(XContentBuilder builder) throws IOExceptio
addModelSizeStatsFieldsToMapping(builder);
}

public static XContentBuilder termFieldsMapping(String type, Collection<String> termFields) {
/**
* Generate a keyword mapping for {@code termFields} for the default type
* {@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME}
*
* If the returned mapping is used in index creation and the new index has a matching template
* then the mapping type ({@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME})
* must match the mapping type of the template otherwise the mappings will not be merged correctly.
*
* @param termFields Fields to generate mapping for
* @return The mapping
*/
public static XContentBuilder termFieldsMapping(Collection<String> termFields) {
try {
XContentBuilder builder = jsonBuilder().startObject();
if (type != null) {
builder.startObject(type);
}
builder.startObject(SINGLE_MAPPING_NAME);
builder.startObject(PROPERTIES);
addTermFields(builder, termFields);
builder.endObject();
if (type != null) {
builder.endObject();
}
builder.endObject();
return builder.endObject();
} catch (IOException e) {
throw new RuntimeException(e);
Expand Down Expand Up @@ -872,7 +879,7 @@ private static void addCategoryDefinitionMapping(XContentBuilder builder) throws
public static XContentBuilder stateMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.field(ENABLED, false);
builder.endObject();
Expand Down Expand Up @@ -960,46 +967,47 @@ private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) th
}

public static XContentBuilder auditMessageMapping() throws IOException {
XContentBuilder builder = jsonBuilder().startObject()
.startObject(AuditMessage.TYPE.getPreferredName());
XContentBuilder builder = jsonBuilder().startObject();
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.LEVEL.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.MESSAGE.getPreferredName())
.field(TYPE, TEXT)
.startObject(FIELDS)
.startObject(RAW)
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject()
.startObject(AuditMessage.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(AuditMessage.NODE_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.LEVEL.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.MESSAGE.getPreferredName())
.field(TYPE, TEXT)
.startObject(FIELDS)
.startObject(RAW)
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject();
.endObject()
.startObject(AuditMessage.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(AuditMessage.NODE_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject()
.endObject();

return builder;
}

static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) throws IOException {
List<String> indicesToUpdate = new ArrayList<>();

ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> currentMapping = state.metaData().findMappings(concreteIndices,
new String[] {DOC_TYPE}, MapperPlugin.NOOP_FIELD_FILTER);
new String[0], MapperPlugin.NOOP_FIELD_FILTER);

for (String index : concreteIndices) {
ImmutableOpenMap<String, MappingMetaData> innerMap = currentMapping.get(index);
if (innerMap != null) {
MappingMetaData metaData = innerMap.get(DOC_TYPE);
MappingMetaData metaData = innerMap.valuesIt().next();
try {
@SuppressWarnings("unchecked")
Map<String, Object> meta = (Map<String, Object>) metaData.sourceAsMap().get("_meta");
Expand Down Expand Up @@ -1038,7 +1046,8 @@ static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndic
return indicesToUpdate.toArray(new String[indicesToUpdate.size()]);
}

public static void addDocMappingIfMissing(String alias, CheckedSupplier<XContentBuilder, IOException> mappingSupplier,
public static void addDocMappingIfMissing(String alias,
CheckedBiFunction<String, Collection<String>, XContentBuilder, IOException> mappingSupplier,
Client client, ClusterState state, ActionListener<Boolean> listener) {
AliasOrIndex aliasOrIndex = state.metaData().getAliasAndIndexLookup().get(alias);
if (aliasOrIndex == null) {
Expand All @@ -1058,9 +1067,13 @@ public static void addDocMappingIfMissing(String alias, CheckedSupplier<XContent
}

if (indicesThatRequireAnUpdate.length > 0) {
try (XContentBuilder mapping = mappingSupplier.get()) {
// Use the mapping type of the first index in the update
IndexMetaData indexMetaData = state.metaData().index(indicesThatRequireAnUpdate[0]);
String mappingType = indexMetaData.mapping().type();

try (XContentBuilder mapping = mappingSupplier.apply(mappingType, Collections.emptyList())) {
PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate);
putMappingRequest.type(DOC_TYPE);
putMappingRequest.type(mappingType);
putMappingRequest.source(mapping);
executeAsyncWithOrigin(client, ML_ORIGIN, PutMappingAction.INSTANCE, putMappingRequest,
ActionListener.wrap(response -> {
Expand Down
2 changes: 1 addition & 1 deletion ...plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import java.util.Objects;

public class AuditMessage implements ToXContentObject, Writeable {
public static final ParseField TYPE = new ParseField("audit_message");
private static final ParseField TYPE = new ParseField("audit_message");

public static final ParseField MESSAGE = new ParseField("message");
public static final ParseField LEVEL = new ParseField("level");
Expand Down
11 changes: 7 additions & 4 deletions ...test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;


public class ElasticsearchMappingsTests extends ESTestCase {

Expand Down Expand Up @@ -117,11 +119,12 @@ private void compareFields(Set<String> expected, Set<String> reserved) {
@SuppressWarnings("unchecked")
public void testTermFieldMapping() throws IOException {

XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(null, Arrays.asList("apple", "strawberry",
XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(Arrays.asList("apple", "strawberry",
AnomalyRecord.BUCKET_SPAN.getPreferredName()));

XContentParser parser = createParser(builder);
Map<String, Object> properties = (Map<String, Object>) parser.map().get(ElasticsearchMappings.PROPERTIES);
Map<String, Object> mapping = (Map<String, Object>) parser.map().get(SINGLE_MAPPING_NAME);
Map<String, Object> properties = (Map<String, Object>) mapping.get(ElasticsearchMappings.PROPERTIES);

Map<String, Object> instanceMapping = (Map<String, Object>) properties.get("apple");
assertNotNull(instanceMapping);
Expand Down Expand Up @@ -217,7 +220,7 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object>
}
mapping.put("_meta", meta);

indexMetaData.putMapping(new MappingMetaData(ElasticsearchMappings.DOC_TYPE, mapping));
indexMetaData.putMapping(new MappingMetaData("_doc", mapping));

metaDataBuilder.put(indexMetaData);
}
Expand All @@ -230,7 +233,7 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object>

private Set<String> collectResultsDocFieldNames() throws IOException {
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
return collectFieldNames(ElasticsearchMappings.resultsMapping());
return collectFieldNames(ElasticsearchMappings.resultsMapping("_doc"));
}

private Set<String> collectConfigDocFieldNames() throws IOException {
Expand Down
2 changes: 1 addition & 1 deletion ...-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public void testDeleteExpiredData() throws Exception {

// Update snapshot timestamp to force it out of snapshot retention window
String snapshotUpdate = "{ \"timestamp\": " + oneDayAgo + "}";
UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), "doc", snapshotDocId);
UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), snapshotDocId);
updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON);
client().execute(UpdateAction.INSTANCE, updateSnapshotRequest).get();

Expand Down
Loading