diff --git a/benchmarks/src/main/java/org/opensearch/benchmark/index/mapper/FlatObjectMappingBenchmark.java b/benchmarks/src/main/java/org/opensearch/benchmark/index/mapper/FlatObjectMappingBenchmark.java index 1bf310b28563a..32250edf0b0c6 100644 --- a/benchmarks/src/main/java/org/opensearch/benchmark/index/mapper/FlatObjectMappingBenchmark.java +++ b/benchmarks/src/main/java/org/opensearch/benchmark/index/mapper/FlatObjectMappingBenchmark.java @@ -41,8 +41,6 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; -import org.opensearch.search.fetch.subphase.highlight.HighlightBuilder; -import org.opensearch.search.sort.SortOrder; import java.io.IOException; import java.net.URISyntaxException; @@ -54,8 +52,8 @@ @State(Scope.Thread) @Fork(1) -@Warmup(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 100, time = 1, timeUnit = TimeUnit.SECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) public class FlatObjectMappingBenchmark { @@ -87,7 +85,7 @@ public void tearDown() throws Exception { @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void CreateDynamicIndex(MyState state) throws IOException, URISyntaxException { + public void CreateDynamicIndex(MyState state) throws IOException { GetDynamicIndex(state, "demo-dynamic-test"); DeleteIndex(state, "demo-dynamic-test"); } @@ -99,7 +97,7 @@ public void CreateDynamicIndex(MyState state) throws IOException, URISyntaxExcep @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void CreateFlatObjectIndex(MyState state) throws IOException, URISyntaxException { + public void CreateFlatObjectIndex(MyState state) throws IOException { GetFlatObjectIndex(state, "demo-flat-object-test", "host"); DeleteIndex(state, "demo-flat-object-test"); } @@ -112,10 +110,10 @@ public void CreateFlatObjectIndex(MyState state) throws IOException, URISyntaxEx @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void indexDynamicMapping(MyState state) throws IOException, URISyntaxException { + public void indexDynamicMapping(MyState state) throws IOException { GetDynamicIndex(state, "demo-dynamic-test1"); String doc = - "{ \"message\": \"[5592:1:0309/123054.737712:ERROR:child_process_sandbox_support_impl_linux.cc(79)] FontService unique font name matching request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 3383 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; + "{ \"message\": \"[1234:1:0309/123054.737712:ERROR: request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 1234 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; UploadDoc(state, "demo-dynamic-test1", doc); DeleteIndex(state, "demo-dynamic-test1"); } @@ -130,7 +128,7 @@ public void indexDynamicMapping(MyState state) throws IOException, URISyntaxExce public void indexFlatObjectMapping(MyState state) throws IOException, URISyntaxException { GetFlatObjectIndex(state, "demo-flat-object-test1", "host"); String doc = - "{ \"message\": \"[5592:1:0309/123054.737712:ERROR:child_process_sandbox_support_impl_linux.cc(79)] FontService unique font name matching request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 3383 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; + "{ \"message\": \"[1234:1:0309/123054.737712:ERROR: request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 1234 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; UploadDoc(state, "demo-flat-object-test1", doc); DeleteIndex(state, "demo-flat-object-test1"); } @@ -146,7 +144,7 @@ public void searchDynamicMapping(MyState state) throws IOException { String indexName = "demo-dynamic-test2"; GetDynamicIndex(state, indexName); String doc = - "{ \"message\": \"[5592:1:0309/123054.737712:ERROR:child_process_sandbox_support_impl_linux.cc(79)] FontService unique font name matching request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 3383 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; + "{ \"message\": \"[1234:1:0309/123054.737712:ERROR: request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 1234 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; UploadDoc(state, indexName, doc); SearchDoc(state, indexName, "host.hostname", "bionic", "@timestamp", "message"); DeleteIndex(state, indexName); @@ -162,9 +160,9 @@ public void searchDynamicMapping(MyState state) throws IOException { public void searchFlatObjectMapping(MyState state) throws IOException { GetFlatObjectIndex(state, "demo-flat-object-test2", "host"); String doc = - "{ \"message\": \"[5592:1:0309/123054.737712:ERROR:child_process_sandbox_support_impl_linux.cc(79)] FontService unique font name matching request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 3383 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; + "{ \"message\": \"[1234:1:0309/123054.737712:ERROR: request did not receive a response.\", \"fileset\": { \"name\": \"syslog\" }, \"process\": { \"name\": \"org.gnome.Shell.desktop\", \"pid\": 1234 }, \"@timestamp\": \"2020-03-09T18:00:54.000+05:30\", \"host\": { \"hostname\": \"bionic\", \"name\": \"bionic\" } }"; UploadDoc(state, "demo-flat-object-test2", doc); - SearchDoc(state, "demo-flat-object-test2", "host", "name", "@timestamp", "message"); + SearchDoc(state, "demo-flat-object-test2", "host.hostname", "name", "@timestamp", "message"); DeleteIndex(state, "demo-flat-object-test2"); } @@ -174,27 +172,23 @@ public void searchFlatObjectMapping(MyState state) throws IOException { * search for document and delete index * Caught exceptions with the number of fields over 1000 */ - // @Benchmark - // @BenchmarkMode(Mode.AverageTime) - // @OutputTimeUnit(TimeUnit.MILLISECONDS) - // public void searchDynamicMappingWithOneHundredNestedJSON(MyState state) throws IOException { - // - // String indexName = "demo-dynamic-test3"; - // GetDynamicIndex(state, indexName); - // String doc = GenerateRandomJson(); - // Map searchValueAndPath = findNestedValueAndPath(doc,99, "field0"); - // String searchValue = searchValueAndPath.get("value"); - // String searchFieldName = searchValueAndPath.get("path"); - // UploadDoc(state, indexName, doc); - // SearchDoc(state,indexName,searchFieldName,searchValue,searchValue ,searchFieldName ); - // DeleteIndex(state, indexName); - // } + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void searchDynamicMappingWithOneHundredNestedJSON(MyState state) throws IOException { + String indexName = "demo-dynamic-test3"; + GetDynamicIndex(state, indexName); + String doc = GenerateRandomJson(10, "nested"); + Map searchValueAndPath = findNestedValueAndPath(doc, 26, ""); + String searchValue = searchValueAndPath.get("value"); + String searchFieldName = searchValueAndPath.get("path"); + UploadDoc(state, indexName, doc); + SearchDoc(state, indexName, searchFieldName, searchValue, searchFieldName, searchFieldName); + DeleteIndex(state, indexName); + } /** - * FlatObjectIndex: - * create index, upload a nested document in 100 levels, and each level with 10 fields, - * search for document and delete index - * works fine and able to return document + * debug search in dotpath */ @Benchmark @BenchmarkMode(Mode.AverageTime) @@ -202,10 +196,10 @@ public void searchFlatObjectMapping(MyState state) throws IOException { public void searchFlatObjectMappingInValueWithOneHundredNestedJSON(MyState state) throws IOException { String indexName = "demo-flat-object-test4"; GetFlatObjectIndex(state, indexName, "nested0"); - String doc = GenerateRandomJson(100, "nested"); - Map searchValueAndPath = findNestedValueAndPath(doc, 6, "nested0"); + String doc = GenerateRandomJson(10, "nested"); + Map searchValueAndPath = findNestedValueAndPath(doc, 26, ""); String SearchRandomWord = searchValueAndPath.get("value"); - String SearchRandomPath = "nested0._value"; + String SearchRandomPath = searchValueAndPath.get("path"); String searchFieldName = "nested0"; UploadDoc(state, indexName, doc); SearchDoc(state, indexName, SearchRandomPath, SearchRandomWord, searchFieldName, searchFieldName); @@ -277,20 +271,15 @@ private static void SearchDoc( sourceBuilder.query(QueryBuilders.matchQuery(searchFieldName, searchText)); sourceBuilder.from(0); sourceBuilder.size(10); - sourceBuilder.sort(sortFieldName, SortOrder.DESC); - sourceBuilder.highlighter(new HighlightBuilder().field(highlightFieldName)); SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.source(sourceBuilder); SearchResponse SearchResponse = state.client.search(searchRequest, RequestOptions.DEFAULT); if (!SearchResponse.status().toString().equals("OK")) { - System.out.println("the number of hit is: " + SearchResponse.getHits().getTotalHits().value); - System.out.println("SearchResponse: " + SearchResponse.toString()); - } - - SearchHits hits = SearchResponse.getHits(); - long totalHits = hits.getTotalHits().value; - if (totalHits == 0) { - throw new IOException("No hit is found"); + SearchHits hits = SearchResponse.getHits(); + long totalHits = hits.getTotalHits().value; + if (totalHits == 0) { + throw new IOException("No hit is found"); + } } } @@ -298,14 +287,14 @@ private static String GenerateRandomJson(int numberOfNestedLevel, String subObje JSONObject json = new JSONObject(); Random random = new Random(); - // Create 100 nested levels + // Create nested levels for (int i = 0; i < numberOfNestedLevel; i++) { JSONObject nestedObject = new JSONObject(); // Add 10 fields to each nested level for (int j = 0; j < 10; j++) { - String field = "field" + j; + String field = "field" + i + j; String value = generateRandomString(random); nestedObject.put(field, value); } @@ -337,31 +326,37 @@ private static Map findNestedValueAndPath(String randomJsonStrin String targetKey = "field" + levelNumber; Map result = new HashMap<>(); Iterator keys = jsonObject.keys(); - StringBuilder path = new StringBuilder(); while (keys.hasNext()) { String key = keys.next(); - if (path.length() == 0) { - path.append(currentPath); - } - Object value = jsonObject.get(key); if (key.equals(targetKey)) { result.put("value", value.toString()); - result.put("path", key); - System.out.println("value is " + value.toString()); - System.out.println("path is " + path.toString()); - break; + if (currentPath.length() == 0) { + currentPath = key; + } + result.put("path", currentPath + "." + key); + return result; } - if (value instanceof JSONObject) { - path.append("." + key); + if (value instanceof JSONObject) { + if (currentPath.length() == 0) { + currentPath = key; + } else { + if (currentPath.contains(".") && currentPath.split("\\.").length > 1) { + int pathLength = currentPath.split("\\.").length; + currentPath = "nested0." + key; + } else { + currentPath = currentPath + "." + key; + } - Map nestedResult = findNestedValueAndPath(value.toString(), levelNumber, path.toString()); + } + Map nestedResult = findNestedValueAndPath(value.toString(), levelNumber, currentPath); if (!nestedResult.isEmpty()) { return nestedResult; } } } + return result; } diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java index 755bf509ab9f5..a4b78c0ee575a 100644 --- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java +++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java @@ -38,11 +38,10 @@ import java.io.IOException; import java.nio.CharBuffer; import java.util.ArrayList; -import java.util.logging.Logger; /** * JsonToStringParser is the main parser class to transform JSON into stringFields in a XContentParser - * returns XContentParser with 3 string fields + * returns XContentParser with one parent field and subfields * fieldName, fieldName._value, fieldName._valueAndPath * @opensearch.internal */ @@ -60,12 +59,11 @@ public class JsonToStringXContentParser extends AbstractXContentParser { private NamedXContentRegistry xContentRegistry; private DeprecationHandler deprecationHandler; - /** - * logging function - * To removed after draft PR - */ - private static final Logger logger = Logger.getLogger((JsonToStringXContentParser.class.getName())); + private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; + private static final String VALUE_SUFFIX = "._value"; + private static final String DOT_SYMBOL = "."; + private static final String EQUAL_SYMBOL = "="; public JsonToStringXContentParser( NamedXContentRegistry xContentRegistry, @@ -85,11 +83,10 @@ public XContentParser parseObject() throws IOException { builder.startObject(); parseToken(); builder.field(this.fieldTypeName, keyList); - builder.field(this.fieldTypeName + "._value", valueList); - builder.field(this.fieldTypeName + "._valueAndPath", valueAndPathList); + builder.field(this.fieldTypeName + VALUE_SUFFIX, valueList); + builder.field(this.fieldTypeName + VALUE_AND_PATH_SUFFIX, valueAndPathList); builder.endObject(); String jString = XContentHelper.convertToJson(BytesReference.bytes(builder), false, XContentType.JSON); - logger.info("Before createParser, jString: " + jString + "\n"); return JsonXContent.jsonXContent.createParser(this.xContentRegistry, this.deprecationHandler, String.valueOf(jString)); } @@ -98,60 +95,38 @@ private void parseToken() throws IOException { while (this.parser.nextToken() != Token.END_OBJECT) { currentFieldName = this.parser.currentName(); - - logger.info("currentFieldName: " + currentFieldName + "\n"); StringBuilder parsedFields = new StringBuilder(); StringBuilder path = new StringBuilder(fieldTypeName); if (this.parser.nextToken() == Token.START_OBJECT) { - /** - * for nested Json, make a copy of parser, then parse the entire Json as string. - * for example: - * {"grandpa": { - * "dad": { - * "son": "me" - * } } - * the JSON object would be read as three string fields for "grandpa" would be - * grandpa: {"dad","son"} -- the parent string field contains the keys only. - * grandpa._value: { "{dad: {son: me}}} ,"{son: me}","me"} -- the _value sub string field contains the values only. - * grandpa._pathAndValue: { "grandpa={"dad: {son: me}}}","grandpa.dad={son: me}}", "grandpa.dad.son=me"} - * -- the _pathAndValue sub string field contains the "path=Value" format. - */ // TODO: to convert the entire JsonObject as string without changing the tokenizer position. - path.append("." + currentFieldName); + path.append(DOT_SYMBOL + currentFieldName); parsedFields.append(this.parser.toString()); this.keyList.add(currentFieldName); this.valueList.add(parsedFields.toString()); - this.valueAndPathList.add(path + "=" + parsedFields.toString()); + this.valueAndPathList.add(path + EQUAL_SYMBOL + parsedFields.toString()); parseToken(); } else { - path.append("." + currentFieldName); + path.append(DOT_SYMBOL + currentFieldName); parseValue(currentFieldName, parsedFields); this.keyList.add(currentFieldName); this.valueList.add(parsedFields.toString()); - this.valueAndPathList.add(path + "=" + parsedFields.toString()); + this.valueAndPathList.add(path + EQUAL_SYMBOL + parsedFields.toString()); } } } private void parseValue(String currentFieldName, StringBuilder parsedFields) throws IOException { - logger.info("this.parser.currentToken(): " + this.parser.currentToken() + "\n"); switch (this.parser.currentToken()) { case VALUE_STRING: parsedFields.append(this.parser.textOrNull()); - logger.info("currentFieldName and parsedFields :" + currentFieldName + " " + parsedFields.toString() + "\n"); break; // Handle other token types as needed - // ToDo, what do we do, if encountered these fields? - // should never get to START_OBJECT case START_OBJECT: throw new IOException("Unsupported token type"); case FIELD_NAME: - // should never get to FIELD_NAME - logger.info("token is FIELD_NAME: " + this.parser.currentName() + "\n"); break; case VALUE_EMBEDDED_OBJECT: - logger.info("token is VALUE_EMBEDDED_OBJECT: " + this.parser.objectText() + "\n"); break; default: throw new IOException("Unsupported token type [" + parser.currentToken() + "]"); diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index 3636567e815b2..76784a17c16e2 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -14,12 +14,20 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; +import org.opensearch.OpenSearchException; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Iterators; import org.opensearch.common.lucene.Lucene; +import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.xcontent.DeprecationHandler; import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.common.xcontent.XContentParser; @@ -29,34 +37,27 @@ import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.opensearch.index.query.QueryShardContext; -import org.opensearch.index.similarity.SimilarityProvider; +import org.opensearch.index.query.QueryShardException; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.function.BiFunction; import java.util.function.Supplier; -import java.util.logging.Logger; +import static org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; /** * A field mapper for flat-objects. This mapper accepts JSON object and treat as string fields in one index. * @opensearch.internal */ -public final class FlatObjectFieldMapper extends ParametrizedFieldMapper { - /** - * logging function: - * To remove after draft PR - */ - - private static final Logger logger = Logger.getLogger((FlatObjectFieldMapper.class.getName())); +public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper { /** * A flat-object mapping contains one parent field itself and two substring fields, @@ -66,10 +67,11 @@ public final class FlatObjectFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "flat-object"; private static final String VALUE_AND_PATH_SUFFIX = "._valueAndPath"; private static final String VALUE_SUFFIX = "._value"; + private static final String DOT_SYMBOL = "."; + private static final String EQUAL_SYMBOL = "="; /** - * Default parameters, similar to keyword - * In flat-object, three fields are treated as keyword fields with the same parameters + * In flat-object field mapper, field type is similar to keyword field type * Cannot be tokenized, can OmitNorms, and can setIndexOption. * @opensearch.internal */ @@ -82,12 +84,16 @@ public static class Defaults { FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); FIELD_TYPE.freeze(); } + + } + + @Override + public MappedFieldType keyedFieldType(String key) { + return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key); } /** - * The flat-object field for the field mapper - * - * @opensearch.internal + * FlatObjectFieldType is the parent field type. */ public static class FlatObjectField extends Field { @@ -102,58 +108,21 @@ private static FlatObjectFieldMapper toType(FieldMapper in) { } /** - * The builder for the flat-object field mapper - * Set the same parameters from keywordFieldMapper.Builder + * The builder for the flat-object field mapper using default parameters as + * indexed: flat-object field mapper is default to be indexed. + * hasDocValues: to store index and support efficient access to individual field values. + * stored: the original value of the field is not stored in the index. + * nullValue: not accept null value + * ignoreAbove: exclude values that exceed the maximum length from the indexing process. * @opensearch.internal */ - public static class Builder extends ParametrizedFieldMapper.Builder { - - private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); - - private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); - private final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); - - private final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null) - .acceptsNull(); - - private final Parameter eagerGlobalOrdinals = Parameter.boolParam( - "eager_global_ordinals", - true, - m -> toType(m).eagerGlobalOrdinals, - false - ); - private final Parameter ignoreAbove = Parameter.intParam( - "ignore_above", - true, - m -> toType(m).ignoreAbove, - Integer.MAX_VALUE - ); - - private final Parameter indexOptions = Parameter.restrictedStringParam( - "index_options", - false, - m -> toType(m).indexOptions, - "docs", - "freqs" - ); - private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); - private final Parameter similarity = TextParams.similarity(m -> toType(m).similarity); - - private final Parameter normalizer = Parameter.stringParam("normalizer", false, m -> toType(m).normalizerName, "default"); - - private final Parameter splitQueriesOnWhitespace = Parameter.boolParam( - "split_queries_on_whitespace", - true, - m -> toType(m).splitQueriesOnWhitespace, - false - ); - - private final Parameter> meta = Parameter.metaParam(); - private final Parameter boost = Parameter.boostParam(); + public static class Builder extends FieldMapper.Builder { + private final IndexAnalyzers indexAnalyzers; - public Builder(String name, IndexAnalyzers indexAnalyzers) { - super(name); + Builder(String name, IndexAnalyzers indexAnalyzers) { + super(name, Defaults.FIELD_TYPE); + builder = this; this.indexAnalyzers = indexAnalyzers; } @@ -161,131 +130,50 @@ public Builder(String name) { this(name, null); } - public Builder ignoreAbove(int ignoreAbove) { - this.ignoreAbove.setValue(ignoreAbove); - return this; - } - - Builder normalizer(String normalizerName) { - this.normalizer.setValue(normalizerName); - return this; - } - - Builder nullValue(String nullValue) { - this.nullValue.setValue(nullValue); - return this; - } - - public Builder docValues(boolean hasDocValues) { - this.hasDocValues.setValue(hasDocValues); - return this; - } - - public Builder index(boolean index) { - return this; - } - - public Builder store(boolean store) { - this.stored.setValue(store); - return this; - } - - @Override - protected List> getParameters() { - return Arrays.asList( - indexed, - hasDocValues, - stored, - nullValue, - eagerGlobalOrdinals, - ignoreAbove, - indexOptions, - hasNorms, - similarity, - normalizer, - splitQueriesOnWhitespace, - boost, - meta - ); - } - - /** - * FlatObjectFieldType is the parent field type. the parent field enables KEYWORD_ANALYZER, - * allows normalizer and splitQueriesOnWhitespace - */ private FlatObjectFieldType buildFlatObjectFieldType(BuilderContext context, FieldType fieldType) { NamedAnalyzer normalizer = Lucene.KEYWORD_ANALYZER; - NamedAnalyzer searchAnalyzer = Lucene.KEYWORD_ANALYZER; - String normalizerName = this.normalizer.getValue(); - if (Objects.equals(normalizerName, "default") == false) { - assert indexAnalyzers != null; - normalizer = indexAnalyzers.getNormalizer(normalizerName); - if (normalizer == null) { - throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]"); - } - if (splitQueriesOnWhitespace.getValue()) { - searchAnalyzer = indexAnalyzers.getWhitespaceNormalizer(normalizerName); - } else { - searchAnalyzer = normalizer; - } - } else if (splitQueriesOnWhitespace.getValue()) { - searchAnalyzer = Lucene.WHITESPACE_ANALYZER; - } - return new FlatObjectFieldType(buildFullName(context), fieldType, normalizer, searchAnalyzer, this); + return new FlatObjectFieldType(buildFullName(context), fieldType); } /** - * ValueFieldMapper is the sub field type for values in the Json. + * ValueFieldMapper is the subfield type for values in the Json. * use a keywordfieldtype */ private ValueFieldMapper buildValueFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { String fullName = buildFullName(context); FieldType vft = new FieldType(fieldType); - vft.setOmitNorms(this.hasNorms.getValue() == false); - KeywordFieldMapper.KeywordFieldType valueFieldType = new KeywordFieldMapper.KeywordFieldType(fullName + "._value", vft); - // TODO: revisit analyzer object + KeywordFieldMapper.KeywordFieldType valueFieldType = new KeywordFieldMapper.KeywordFieldType(fullName + VALUE_SUFFIX, vft); fft.setValueFieldType(valueFieldType); return new ValueFieldMapper(vft, valueFieldType); - } /** - * ValueAndPathFieldMapper is the sub field type for path=value format in the Json. + * ValueAndPathFieldMapper is the subfield type for path=value format in the Json. * also use a keywordfieldtype */ private ValueAndPathFieldMapper buildValueAndPathFieldMapper(BuilderContext context, FieldType fieldType, FlatObjectFieldType fft) { - String fullName = buildFullName(context); FieldType vft = new FieldType(fieldType); - vft.setOmitNorms(this.hasNorms.getValue() == false); KeywordFieldMapper.KeywordFieldType ValueAndPathFieldType = new KeywordFieldMapper.KeywordFieldType( - fullName + "._valueAndPath", + fullName + VALUE_AND_PATH_SUFFIX, vft ); - // TODO: revisit analyzer object fft.setValueAndPathFieldType(ValueAndPathFieldType); return new ValueAndPathFieldMapper(vft, ValueAndPathFieldType); } - /** - * FlatObjectFieldMapper builds the FLatObjectFieldMapper itself, and also build the two sub fieldMappers: - * ValueFieldMapper and ValueAndPathFieldMapper - */ @Override public FlatObjectFieldMapper build(BuilderContext context) { FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); - fieldtype.setOmitNorms(this.hasNorms.getValue() == false); - fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); - fieldtype.setStored(this.stored.getValue()); FlatObjectFieldType fft = buildFlatObjectFieldType(context, fieldtype); return new FlatObjectFieldMapper( name, - fieldtype, + Defaults.FIELD_TYPE, fft, buildValueFieldMapper(context, fieldtype, fft), buildValueAndPathFieldMapper(context, fieldtype, fft), - multiFieldsBuilder.build(this, context), - copyTo.build(), + MultiFields.empty(), + CopyTo.empty(), this ); } @@ -294,8 +182,24 @@ public FlatObjectFieldMapper build(BuilderContext context) { public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers())); /** - * Field type for flat-object fields - * one flat-object fields contains its own fieldType, one valueFieldType and one valueAndPathFieldType + * Creates a new TypeParser for flatObjectFieldMapper that does not use ParameterizedFieldMapper + */ + public static class TypeParser implements Mapper.TypeParser { + private final BiFunction builderFunction; + + public TypeParser(BiFunction builderFunction) { + this.builderFunction = builderFunction; + } + + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = builderFunction.apply(name, parserContext); + return builder; + } + } + + /** + * flat-object fields type contains its own fieldType, one valueFieldType and one valueAndPathFieldType * @opensearch.internal */ public static final class FlatObjectFieldType extends StringFieldType { @@ -307,30 +211,8 @@ public static final class FlatObjectFieldType extends StringFieldType { private KeywordFieldMapper.KeywordFieldType valueAndPathFieldType; - public FlatObjectFieldType( - String name, - FieldType fieldType, - NamedAnalyzer normalizer, - NamedAnalyzer searchAnalyzer, - Builder builder - ) { - super( - name, - fieldType.indexOptions() != IndexOptions.NONE, - fieldType.stored(), - builder.hasDocValues.getValue(), - new TextSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, searchAnalyzer), - builder.meta.getValue() - ); - setEagerGlobalOrdinals(builder.eagerGlobalOrdinals.getValue()); - setIndexAnalyzer(normalizer); - setBoost(builder.boost.getValue()); - this.ignoreAbove = builder.ignoreAbove.getValue(); - this.nullValue = builder.nullValue.getValue(); - } - public FlatObjectFieldType(String name, boolean isSearchable, boolean hasDocValues, Map meta) { - super(name, isSearchable, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + super(name, isSearchable, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; @@ -345,7 +227,7 @@ public FlatObjectFieldType(String name, FieldType fieldType) { name, fieldType.indexOptions() != IndexOptions.NONE, false, - false, + true, new TextSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); @@ -384,6 +266,13 @@ NamedAnalyzer normalizer() { return indexAnalyzer(); } + /** + * + * Fielddata is an in-memory data structure that is used for aggregations, sorting, and scripting. + * @param fullyQualifiedIndexName the name of the index this field-data is build for + * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields + * @return IndexFieldData.Builder + */ @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { failIfNoDocValues(); @@ -449,6 +338,134 @@ protected BytesRef indexedValueForSearch(Object value) { return getTextSearchInfo().getSearchAnalyzer().normalize(name(), value.toString()); } + /** + * redirect term query with rewrite value to rewriteSearchValue and directSubFieldName + */ + @Override + public Query termQuery(Object value, @Nullable QueryShardContext context) { + + String searchValueString = ((BytesRef) value).utf8ToString(); + String directSubFieldName = directSubfield(); + String rewriteSearchValue = rewriteValue(searchValueString); + + failIfNotIndexed(); + Query query; + query = new TermQuery(new Term(directSubFieldName, indexedValueForSearch(rewriteSearchValue))); + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + return query; + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + failIfNotIndexed(); + String directedSearchFieldName = directSubfield(); + BytesRef[] bytesRefs = new BytesRef[values.size()]; + for (int i = 0; i < bytesRefs.length; i++) { + String rewriteValues = rewriteValue(((BytesRef) values.get(i)).utf8ToString()); + + bytesRefs[i] = indexedValueForSearch(new BytesRef(rewriteValues)); + + } + + return new TermInSetQuery(directedSearchFieldName, bytesRefs); + } + + /** + * To direch search fields, if a dot path was used in search query, + * then direct to flatObjectFieldName._valueAndPath subfield, + * else, direct to flatObjectFieldName._value subfield. + * @return directedSubFieldName + */ + public String directSubfield() { + if (name().contains(DOT_SYMBOL)) { + String[] dotPathList = name().split("\\."); + return dotPathList[0] + VALUE_AND_PATH_SUFFIX; + } else { + return this.valueFieldType.name(); + } + } + + /** + * If the search key is assigned with value, + * the dot path was used in search query, then + * rewrite the searchValueString as the format "dotpath=value", + * @return rewriteSearchValue + */ + public String rewriteValue(String searchValueString) { + if (!name().contains(DOT_SYMBOL)) { + return searchValueString; + } else { + String rewriteSearchValue = new StringBuilder().append(name()).append(EQUAL_SYMBOL).append(searchValueString).toString(); + return rewriteSearchValue; + } + + } + + @Override + public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { + String directSubfield = directSubfield(); + String rewriteValue = rewriteValue(value); + + if (context.allowExpensiveQueries() == false) { + throw new OpenSearchException( + "[prefix] queries cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false. For optimised prefix queries on text " + + "fields please enable [index_prefixes]." + ); + } + failIfNotIndexed(); + if (method == null) { + method = MultiTermQuery.CONSTANT_SCORE_REWRITE; + } + if (caseInsensitive) { + return AutomatonQueries.caseInsensitivePrefixQuery((new Term(directSubfield, indexedValueForSearch(rewriteValue))), method); + } + return new PrefixQuery(new Term(directSubfield, indexedValueForSearch(rewriteValue)), method); + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { + String directSubfield = directSubfield(); + String rewriteUpperTerm = rewriteValue(((BytesRef) upperTerm).utf8ToString()); + String rewriteLowerTerm = rewriteValue(((BytesRef) lowerTerm).utf8ToString()); + if (context.allowExpensiveQueries() == false) { + throw new OpenSearchException( + "[range] queries on [text] or [keyword] fields cannot be executed when '" + + ALLOW_EXPENSIVE_QUERIES.getKey() + + "' is set to false." + ); + } + failIfNotIndexed(); + return new TermRangeQuery( + directSubfield, + lowerTerm == null ? null : indexedValueForSearch(rewriteLowerTerm), + upperTerm == null ? null : indexedValueForSearch(rewriteUpperTerm), + includeLower, + includeUpper + ); + } + + /** + * if there is dot path. query the field name in flatObject parent field. + * else query in _field_names system field + */ + @Override + public Query existsQuery(QueryShardContext context) { + String searchKey; + String searchField; + if (name().contains(DOT_SYMBOL)) { + searchKey = name().split("\\.")[0]; + searchField = name(); + } else { + searchKey = FieldNamesFieldMapper.NAME; + searchField = name(); + } + return new TermQuery(new Term(searchKey, indexedValueForSearch(searchField))); + } + @Override public Query wildcardQuery( String value, @@ -458,27 +475,23 @@ public Query wildcardQuery( ) { // flat-object field types are always normalized, so ignore case sensitivity and force normalize the wildcard // query text - return super.wildcardQuery(value, method, caseInsensitve, true, context); + throw new QueryShardException( + context, + "Can only use wildcard queries on keyword and text fields - not on [" + + name() + + "] which is of type [" + + "flat-object" + + "]" + ); } } - private final boolean indexed; - private final boolean hasDocValues; - private final String nullValue; - private final boolean eagerGlobalOrdinals; - private final int ignoreAbove; - private final String indexOptions; - private final FieldType fieldType; - private final SimilarityProvider similarity; - private final String normalizerName; - private final boolean splitQueriesOnWhitespace; private final ValueFieldMapper valueFieldMapper; private final ValueAndPathFieldMapper valueAndPathFieldMapper; - private final IndexAnalyzers indexAnalyzers; - protected FlatObjectFieldMapper( + FlatObjectFieldMapper( String simpleName, FieldType fieldType, FlatObjectFieldType mappedFieldType, @@ -488,29 +501,13 @@ protected FlatObjectFieldMapper( CopyTo copyTo, Builder builder ) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, fieldType, mappedFieldType, copyTo); assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; - this.indexed = builder.indexed.getValue(); - this.hasDocValues = builder.hasDocValues.getValue(); - this.nullValue = builder.nullValue.getValue(); - this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue(); - this.ignoreAbove = builder.ignoreAbove.getValue(); - this.indexOptions = builder.indexOptions.getValue(); this.fieldType = fieldType; - this.similarity = builder.similarity.getValue(); - this.normalizerName = builder.normalizer.getValue(); - this.splitQueriesOnWhitespace = builder.splitQueriesOnWhitespace.getValue(); this.indexAnalyzers = builder.indexAnalyzers; this.valueFieldMapper = valueFieldMapper; this.valueAndPathFieldMapper = valueAndPathFieldMapper; - } - - /** - * TODO: Placeholder, this is used at keywordfieldmapper, considering to remove ignoreAbove - * Values that have more chars than the return value of this method will - * be skipped at parsing time. */ - public int ignoreAbove() { - return ignoreAbove; + this.mappedFieldType = mappedFieldType; } @Override @@ -518,6 +515,11 @@ protected FlatObjectFieldMapper clone() { return (FlatObjectFieldMapper) super.clone(); } + @Override + protected void mergeOptions(FieldMapper other, List conflicts) { + + } + @Override public FlatObjectFieldType fieldType() { return (FlatObjectFieldType) super.fieldType(); @@ -530,7 +532,7 @@ protected void parseCreateField(ParseContext context) throws IOException { if (context.externalValueSet()) { value = context.externalValue().toString(); - ParseValueAddFields(context, value, fieldType().name()); + parseValueAddFields(context, value, fieldType().name()); } else { JsonToStringXContentParser JsonToStringParser = new JsonToStringXContentParser( NamedXContentRegistry.EMPTY, @@ -548,12 +550,10 @@ protected void parseCreateField(ParseContext context) throws IOException { switch (currentToken) { case FIELD_NAME: fieldName = parser.currentName(); - logger.info("fieldName: " + fieldName); break; case VALUE_STRING: value = parser.textOrNull(); - logger.info("value: " + value); - ParseValueAddFields(context, value, fieldName); + parseValueAddFields(context, value, fieldName); break; } @@ -580,10 +580,7 @@ public Iterator iterator() { return concat; } - private void ParseValueAddFields(ParseContext context, String value, String fieldName) throws IOException { - if (value == null || value.length() > ignoreAbove) { - return; - } + private void parseValueAddFields(ParseContext context, String value, String fieldName) throws IOException { NamedAnalyzer normalizer = fieldType().normalizer(); if (normalizer != null) { @@ -592,18 +589,16 @@ private void ParseValueAddFields(ParseContext context, String value, String fiel String[] valueTypeList = fieldName.split("\\._"); String valueType = "._" + valueTypeList[valueTypeList.length - 1]; - logger.info("valueType: " + valueType); + /** * the JsonToStringXContentParser returns XContentParser with 3 string fields * fieldName, fieldName._value, fieldName._valueAndPath */ if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { - logger.info("FlatObjectField name is " + fieldType().name()); - logger.info("FlatObjectField value is " + value); // convert to utf8 only once before feeding postings/dv/stored fields - final BytesRef binaryValue = new BytesRef(value); + final BytesRef binaryValue = new BytesRef(fieldType().name() + DOT_SYMBOL + value); Field field = new FlatObjectField(fieldType().name(), binaryValue, fieldType); if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { @@ -621,18 +616,15 @@ private void ParseValueAddFields(ParseContext context, String value, String fiel } if (valueType.equals(VALUE_SUFFIX)) { if (valueFieldMapper != null) { - logger.info("valueFieldMapper value is " + value); valueFieldMapper.addField(context, value); } } if (valueType.equals(VALUE_AND_PATH_SUFFIX)) { if (valueAndPathFieldMapper != null) { - logger.info("valueAndPathFieldMapper value is " + value); valueAndPathFieldMapper.addField(context, value); } } - // TODo: to revisit if flat-object needs docValues. if (fieldType().hasDocValues()) { if (context.doc().getField(fieldType().name()) == null || !context.doc().getFields(fieldType().name()).equals(field)) { context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); @@ -679,12 +671,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public ParametrizedFieldMapper.Builder getMergeBuilder() { - return new Builder(simpleName(), indexAnalyzers).init(this); - } - - // TODO Further simplify the code by new KeyWordFieldMapper to be ValueAndPathFieldMapper and ValueFieldMapper private static final class ValueAndPathFieldMapper extends FieldMapper { protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldMapper.KeywordFieldType mappedFieldType) { @@ -692,11 +678,9 @@ protected ValueAndPathFieldMapper(FieldType fieldType, KeywordFieldMapper.Keywor } void addField(ParseContext context, String value) { - // context.doc().add(new Field(fieldType().name(), value, fieldType)); final BytesRef binaryValue = new BytesRef(value); if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { Field field = new KeywordFieldMapper.KeywordField(fieldType().name(), binaryValue, fieldType); - // Field field = new (fieldType().name()+VALUE_AND_PATH_SUFFIX, binaryValue, fieldType); context.doc().add(field); @@ -725,6 +709,7 @@ protected String contentType() { public String toString() { return fieldType().toString(); } + } private static final class ValueFieldMapper extends FieldMapper { diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java index e413a73940011..c2005904c3c8c 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesModule.java +++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java @@ -51,6 +51,7 @@ import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.FieldAliasMapper; import org.opensearch.index.mapper.FieldNamesFieldMapper; +import org.opensearch.index.mapper.FlatObjectFieldMapper; import org.opensearch.index.mapper.GeoPointFieldMapper; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.IgnoredFieldMapper; @@ -162,6 +163,7 @@ public static Map getMappers(List mappe mappers.put(CompletionFieldMapper.CONTENT_TYPE, CompletionFieldMapper.PARSER); mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser()); mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); + mappers.put(FlatObjectFieldMapper.CONTENT_TYPE, FlatObjectFieldMapper.PARSER); for (MapperPlugin mapperPlugin : mapperPlugins) { for (Map.Entry entry : mapperPlugin.getMappers().entrySet()) { diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java new file mode 100644 index 0000000000000..c1a506481fd55 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java @@ -0,0 +1,113 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.mapper; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.apache.lucene.util.BytesRef; +import org.opensearch.common.Strings; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.json.JsonXContent; + +import java.io.IOException; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.StringContains.containsString; + +public class FlatObjectFieldMapperTests extends MapperServiceTestCase { + private static final String FIELD_TYPE = "flat-object"; + + // @Override + public FlatObjectFieldMapper.Builder newBuilder() { + return new FlatObjectFieldMapper.Builder("flat-object"); + } + + public final void testExistsQueryDocValuesDisabledWithNorms() throws IOException { + MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); })); + assertParseMinimalWarnings(); + } + + public void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", FIELD_TYPE); + } + + /** + * Writes a sample value for the field to the provided {@link XContentBuilder}. + * @param builder builder + */ + + protected void writeFieldValue(XContentBuilder builder) throws IOException { + builder.startObject(); + builder.field("foo", "bar"); + builder.endObject(); + } + + public final void testEmptyName() { + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping(b -> { + b.startObject(""); + minimalMapping(b); + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("name cannot be empty string")); + assertParseMinimalWarnings(); + } + + public void testMinimalToMaximal() throws IOException { + XContentBuilder orig = JsonXContent.contentBuilder().startObject(); + createMapperService(fieldMapping(this::minimalMapping)).documentMapper().mapping().toXContent(orig, ToXContent.EMPTY_PARAMS); + orig.endObject(); + XContentBuilder parsedFromOrig = JsonXContent.contentBuilder().startObject(); + createMapperService(orig).documentMapper().mapping().toXContent(parsedFromOrig, ToXContent.EMPTY_PARAMS); + parsedFromOrig.endObject(); + assertEquals(Strings.toString(orig), Strings.toString(parsedFromOrig)); + assertParseMaximalWarnings(); + } + + public void testDefaults() throws Exception { + XContentBuilder mapping = fieldMapping(this::minimalMapping); + DocumentMapper mapper = createDocumentMapper(mapping); + assertEquals(Strings.toString(mapping), mapper.mappingSource().toString()); + + String json = Strings.toString( + XContentFactory.jsonBuilder().startObject().startObject("field").field("foo", "bar").endObject().endObject() + ); + + ParsedDocument doc = mapper.parse(source(json)); + IndexableField[] fields = doc.rootDoc().getFields("field"); + assertEquals(4, fields.length); + assertEquals(new BytesRef("field.foo"), fields[0].binaryValue()); + + IndexableFieldType fieldType = fields[0].fieldType(); + assertFalse(fieldType.tokenized()); + assertFalse(fieldType.stored()); + assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS)); + assertEquals(DocValuesType.NONE, fieldType.docValuesType()); + + } + + public void testNullValue() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.nullField("field")))); + assertThat(e.getMessage(), containsString("object mapping for [_doc] tried to parse field [field] as object")); + + } + + protected void assertParseMinimalWarnings() { + // Most mappers don't emit any warnings + } + + protected void assertParseMaximalWarnings() { + // Most mappers don't emit any warnings + } + +}