Skip to content

Don't persist type information to translog #47229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Oct 15, 2019
Merged

Don't persist type information to translog #47229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
89f5db5
Don't store types in translog
romseygeek Sep 27, 2019
0a870b7
Merge remote-tracking branch 'origin/master' into types-removal/translog
romseygeek Sep 27, 2019
ea1acca
Fix wire format for bwc
romseygeek Sep 30, 2019
d0905b0
Merge remote-tracking branch 'origin/master' into types-removal/translog
romseygeek Oct 11, 2019
1658274
test fix
romseygeek Oct 12, 2019
52e27e0
test fix
romseygeek Oct 12, 2019
04b0569
bwc assertions
romseygeek Oct 12, 2019
d29a84c
feedback
romseygeek Oct 14, 2019
56e49a7
Merge branch 'master' into types-removal/translog
elasticmachine Oct 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .../lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -544,10 +544,9 @@ private static Response prepareRamIndex(Request request,
try (RAMDirectory ramDirectory = new RAMDirectory()) {
try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(defaultAnalyzer))) {
String index = indexService.index().getName();
String type = indexService.mapperService().documentMapper().type();
BytesReference document = request.contextSetup.document;
XContentType xContentType = request.contextSetup.xContentType;
SourceToParse sourceToParse = new SourceToParse(index, type, "_id", document, xContentType);
SourceToParse sourceToParse = new SourceToParse(index, "_id", document, xContentType);
ParsedDocument parsedDocument = indexService.mapperService().documentMapper().parse(sourceToParse);
indexWriter.addDocuments(parsedDocument.docs());
try (IndexReader indexReader = DirectoryReader.open(indexWriter)) {
Expand Down
12 changes: 6 additions & 6 deletions ...pper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public void testDefaults() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc1 = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc1 = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 10)
Expand All @@ -84,7 +84,7 @@ public void testDefaults() throws Exception {
assertThat(fields[0], Matchers.instanceOf(FeatureField.class));
FeatureField featureField1 = (FeatureField) fields[0];

ParsedDocument doc2 = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc2 = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 12)
Expand All @@ -108,7 +108,7 @@ public void testNegativeScoreImpact() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc1 = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc1 = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 10)
Expand All @@ -120,7 +120,7 @@ public void testNegativeScoreImpact() throws Exception {
assertThat(fields[0], Matchers.instanceOf(FeatureField.class));
FeatureField featureField1 = (FeatureField) fields[0];

ParsedDocument doc2 = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc2 = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 12)
Expand All @@ -145,7 +145,7 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
assertEquals(mapping, mapper.mappingSource().toString());

MapperParsingException e = expectThrows(MapperParsingException.class,
() -> mapper.parse(new SourceToParse("test", "type", "1", BytesReference
() -> mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", Arrays.asList(10, 20))
Expand All @@ -155,7 +155,7 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
e.getCause().getMessage());

e = expectThrows(MapperParsingException.class,
() -> mapper.parse(new SourceToParse("test", "type", "1", BytesReference
() -> mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.startArray("foo")
Expand Down
6 changes: 3 additions & 3 deletions ...per-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public void testDefaults() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc1 = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc1 = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.startObject("field")
Expand Down Expand Up @@ -95,7 +95,7 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
assertEquals(mapping, mapper.mappingSource().toString());

MapperParsingException e = expectThrows(MapperParsingException.class,
() -> mapper.parse(new SourceToParse("test", "type", "1", BytesReference
() -> mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.startObject("field")
Expand All @@ -107,7 +107,7 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
"START_ARRAY", e.getCause().getMessage());

e = expectThrows(MapperParsingException.class,
() -> mapper.parse(new SourceToParse("test", "type", "1", BytesReference
() -> mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.startArray("foo")
Expand Down
20 changes: 10 additions & 10 deletions ...pper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public void testDefaults() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 123)
Expand Down Expand Up @@ -115,7 +115,7 @@ public void testNotIndexed() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 123)
Expand All @@ -139,7 +139,7 @@ public void testNoDocValues() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 123)
Expand All @@ -163,7 +163,7 @@ public void testStore() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", 123)
Expand Down Expand Up @@ -192,7 +192,7 @@ public void testCoerce() throws Exception {

assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "123")
Expand All @@ -216,7 +216,7 @@ public void testCoerce() throws Exception {

assertEquals(mapping, mapper2.mappingSource().toString());

ThrowingRunnable runnable = () -> mapper2.parse(new SourceToParse("test", "type", "1", BytesReference
ThrowingRunnable runnable = () -> mapper2.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", "123")
Expand Down Expand Up @@ -245,7 +245,7 @@ private void doTestIgnoreMalformed(String value, String exceptionMessageContains

assertEquals(mapping, mapper.mappingSource().toString());

ThrowingRunnable runnable = () -> mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ThrowingRunnable runnable = () -> mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", value)
Expand All @@ -261,7 +261,7 @@ private void doTestIgnoreMalformed(String value, String exceptionMessageContains

DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping));

ParsedDocument doc = mapper2.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc = mapper2.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("field", value)
Expand All @@ -286,7 +286,7 @@ public void testNullValue() throws IOException {
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());

ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.nullField("field")
Expand All @@ -308,7 +308,7 @@ public void testNullValue() throws IOException {
mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());

doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference
doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.nullField("field")
Expand Down
6 changes: 3 additions & 3 deletions ...-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public void testIndexing() throws IOException {
.mapperService()
.documentMapperParser()
.parse("_doc", new CompressedXContent(mapping));
ParsedDocument doc = mapper.parse(new SourceToParse("test", "_doc", "1", BytesReference
ParsedDocument doc = mapper.parse(new SourceToParse("test", "1", BytesReference
.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("a_field", "new york city")
Expand Down Expand Up @@ -273,7 +273,7 @@ public void testMultiFields() throws IOException {
}

ParsedDocument doc = mapperService.documentMapper()
.parse(new SourceToParse("test", "_doc", "1",
.parse(new SourceToParse("test", "1",
BytesReference.bytes(
XContentFactory.jsonBuilder()
.startObject()
Expand Down Expand Up @@ -740,7 +740,7 @@ private void documentParsingTestCase(Collection<String> values) throws IOExcepti
}
builder.endObject();
final ParsedDocument parsedDocument = defaultMapper.parse(
new SourceToParse("test", "_doc", "1", BytesReference.bytes(builder), XContentType.JSON));
new SourceToParse("test", "1", BytesReference.bytes(builder), XContentType.JSON));


final Set<Matcher<IndexableField>> rootFieldMatchers = values.stream()
Expand Down
2 changes: 1 addition & 1 deletion ...apper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ private SourceToParse createDocument(String fieldValue) throws Exception {
.field("test", fieldValue)
.endObject());

return new SourceToParse("test", "person", "1", request, XContentType.JSON);
return new SourceToParse("test", "1", request, XContentType.JSON);
}

private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse request) {
Expand Down
26 changes: 13 additions & 13 deletions ...s/parent-join/src/test/java/org/elasticsearch/join/mapper/ParentJoinFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,20 +63,20 @@ public void testSingleLevel() throws Exception {
assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(service.mapperService()));

// Doc without join
ParsedDocument doc = docMapper.parse(new SourceToParse("test", "type", "0",
ParsedDocument doc = docMapper.parse(new SourceToParse("test", "0",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), XContentType.JSON));
assertNull(doc.rootDoc().getBinaryValue("join_field"));

// Doc parent
doc = docMapper.parse(new SourceToParse("test", "type", "1",
doc = docMapper.parse(new SourceToParse("test", "1",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.field("join_field", "parent")
.endObject()), XContentType.JSON));
assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
assertEquals("parent", doc.rootDoc().getBinaryValue("join_field").utf8ToString());

// Doc child
doc = docMapper.parse(new SourceToParse("test", "type", "2",
doc = docMapper.parse(new SourceToParse("test", "2",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.startObject("join_field")
.field("name", "child")
Expand All @@ -88,7 +88,7 @@ public void testSingleLevel() throws Exception {

// Unknown join name
MapperException exc = expectThrows(MapperParsingException.class,
() -> docMapper.parse(new SourceToParse("test", "type", "1",
() -> docMapper.parse(new SourceToParse("test", "1",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.field("join_field", "unknown")
.endObject()), XContentType.JSON)));
Expand All @@ -109,7 +109,7 @@ public void testParentIdSpecifiedAsNumber() throws Exception {
IndexService service = createIndex("test");
DocumentMapper docMapper = service.mapperService().merge("type", new CompressedXContent(mapping),
MapperService.MergeReason.MAPPING_UPDATE);
ParsedDocument doc = docMapper.parse(new SourceToParse("test", "type", "2",
ParsedDocument doc = docMapper.parse(new SourceToParse("test", "2",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.startObject("join_field")
.field("name", "child")
Expand All @@ -118,7 +118,7 @@ public void testParentIdSpecifiedAsNumber() throws Exception {
.endObject()), XContentType.JSON, "1"));
assertEquals("1", doc.rootDoc().getBinaryValue("join_field#parent").utf8ToString());
assertEquals("child", doc.rootDoc().getBinaryValue("join_field").utf8ToString());
doc = docMapper.parse(new SourceToParse("test", "type", "2",
doc = docMapper.parse(new SourceToParse("test", "2",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.startObject("join_field")
.field("name", "child")
Expand Down Expand Up @@ -147,12 +147,12 @@ public void testMultipleLevels() throws Exception {
assertTrue(docMapper.mappers().getMapper("join_field") == ParentJoinFieldMapper.getMapper(service.mapperService()));

// Doc without join
ParsedDocument doc = docMapper.parse(new SourceToParse("test", "type", "0",
ParsedDocument doc = docMapper.parse(new SourceToParse("test", "0",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), XContentType.JSON));
assertNull(doc.rootDoc().getBinaryValue("join_field"));

// Doc parent
doc = docMapper.parse(new SourceToParse("test", "type", "1",
doc = docMapper.parse(new SourceToParse("test", "1",
BytesReference.bytes(XContentFactory.jsonBuilder()
.startObject()
.field("join_field", "parent")
Expand All @@ -161,7 +161,7 @@ public void testMultipleLevels() throws Exception {
assertEquals("parent", doc.rootDoc().getBinaryValue("join_field").utf8ToString());

// Doc child
doc = docMapper.parse(new SourceToParse("test", "type", "2",
doc = docMapper.parse(new SourceToParse("test", "2",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.startObject("join_field")
.field("name", "child")
Expand All @@ -174,15 +174,15 @@ public void testMultipleLevels() throws Exception {

// Doc child missing parent
MapperException exc = expectThrows(MapperParsingException.class,
() -> docMapper.parse(new SourceToParse("test", "type", "2",
() -> docMapper.parse(new SourceToParse("test", "2",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.field("join_field", "child")
.endObject()), XContentType.JSON, "1")));
assertThat(exc.getRootCause().getMessage(), containsString("[parent] is missing for join field [join_field]"));

// Doc child missing routing
exc = expectThrows(MapperParsingException.class,
() -> docMapper.parse(new SourceToParse("test", "type", "2",
() -> docMapper.parse(new SourceToParse("test", "2",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.startObject("join_field")
.field("name", "child")
Expand All @@ -192,7 +192,7 @@ public void testMultipleLevels() throws Exception {
assertThat(exc.getRootCause().getMessage(), containsString("[routing] is missing for join field [join_field]"));

// Doc grand_child
doc = docMapper.parse(new SourceToParse("test", "type", "3",
doc = docMapper.parse(new SourceToParse("test", "3",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.startObject("join_field")
.field("name", "grand_child")
Expand All @@ -204,7 +204,7 @@ public void testMultipleLevels() throws Exception {

// Unknown join name
exc = expectThrows(MapperParsingException.class,
() -> docMapper.parse(new SourceToParse("test", "type", "1",
() -> docMapper.parse(new SourceToParse("test", "1",
BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.field("join_field", "unknown")
.endObject()), XContentType.JSON)));
Expand Down
2 changes: 1 addition & 1 deletion modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
String type = mapperService.documentMapper().type();
docMapper = mapperService.documentMapper(type);
for (BytesReference document : documents) {
docs.add(docMapper.parse(new SourceToParse(context.index().getName(), type, "_temp_id", document, documentXContentType)));
docs.add(docMapper.parse(new SourceToParse(context.index().getName(), "_temp_id", document, documentXContentType)));
}

FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer();
Expand Down
Loading