elastic · jpountz · Aug 26, 2016 · Aug 25, 2016
diff --git a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -41,28 +41,19 @@
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParentFieldMapper;
-import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
-import org.elasticsearch.index.mapper.TTLFieldMapper;
-import org.elasticsearch.index.mapper.TimestampFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.index.shard.AbstractIndexShardComponent;
 import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.ParentFieldSubFetchPhase;
-import org.elasticsearch.search.lookup.LeafSearchLookup;
-import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -218,41 +209,14 @@ private GetResult innerGetLoadFromStoredFields(String type, String id, String[]
             fields.put(ParentFieldMapper.NAME, new GetField(ParentFieldMapper.NAME, Collections.singletonList(parentId)));
         }
 
-        // now, go and do the script thingy if needed
-
         if (gFields != null && gFields.length > 0) {
-            SearchLookup searchLookup = null;
             for (String field : gFields) {
-                Object value = null;
                 FieldMapper fieldMapper = docMapper.mappers().smartNameFieldMapper(field);
                 if (fieldMapper == null) {
                     if (docMapper.objectMappers().get(field) != null) {
                         // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
                         throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
                     }
-                } else if (!fieldMapper.fieldType().stored() && !fieldMapper.isGenerated()) {
-                    if (searchLookup == null) {
-                        searchLookup = new SearchLookup(mapperService, null, new String[]{type});
-                        LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(docIdAndVersion.context);
-                        searchLookup.source().setSource(source);
-                        leafSearchLookup.setDocument(docIdAndVersion.docId);
-                    }
-
-                    List<Object> values = searchLookup.source().extractRawValues(field);
-                    if (values.isEmpty() == false) {
-                        value = values;
-                    }
-                }
-
-                if (value != null) {
-                    if (fields == null) {
-                        fields = new HashMap<>(2);
-                    }
-                    if (value instanceof List) {
-                        fields.put(field, new GetField(field, (List) value));
-                    } else {
-                        fields.put(field, new GetField(field, Collections.singletonList(value)));
-                    }
                 }
             }
         }

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java
@@ -291,8 +291,4 @@ protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
         super.doMerge(mergeWith, updateAllTypes);
     }
 
-    @Override
-    public boolean isGenerated() {
-        return true;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -660,14 +660,4 @@ public List<String> copyToFields() {
         }
     }
 
-    /**
-     * Fields might not be available before indexing, for example _all, token_count,...
-     * When get is called and these fields are requested, this case needs special treatment.
-     *
-     * @return If the field is available before indexing or not.
-     */
-    public boolean isGenerated() {
-        return false;
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java
@@ -289,8 +289,4 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         return builder;
     }
 
-    @Override
-    public boolean isGenerated() {
-        return true;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/LegacyTokenCountFieldMapper.java
@@ -187,9 +187,4 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         builder.field("analyzer", analyzer());
     }
 
-    @Override
-    public boolean isGenerated() {
-        return true;
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java
@@ -182,9 +182,4 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         builder.field("analyzer", analyzer());
     }
 
-    @Override
-    public boolean isGenerated() {
-        return true;
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -35,6 +35,8 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.get.GetField;
 import org.elasticsearch.index.get.GetResult;
@@ -44,6 +46,7 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.StringFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
@@ -55,8 +58,10 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
@@ -191,9 +196,11 @@ private static Fields addGeneratedTermVectors(IndexShard indexShard, Engine.GetR
         }
 
         /* generate term vectors from fetched document fields */
+        String[] getFields = validFields.toArray(new String[validFields.size() + 1]);
+        getFields[getFields.length - 1] = SourceFieldMapper.NAME;
         GetResult getResult = indexShard.getService().get(
-                get, request.id(), request.type(), validFields.toArray(Strings.EMPTY_ARRAY), null);
-        Fields generatedTermVectors = generateTermVectors(indexShard, getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer(), validFields);
+                get, request.id(), request.type(), getFields, null);
+        Fields generatedTermVectors = generateTermVectors(indexShard, getResult.sourceAsMap(), getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer(), validFields);
 
         /* merge with existing Fields */
         if (termVectorsByField == null) {
@@ -227,17 +234,31 @@ private static Set<String> getFieldsToGenerate(Map<String, String> perAnalyzerFi
         return selectedFields;
     }
 
-    private static Fields generateTermVectors(IndexShard indexShard, Collection<GetField> getFields, boolean withOffsets, @Nullable Map<String, String> perFieldAnalyzer, Set<String> fields) throws IOException {
-        /* store document in memory index */
-        MemoryIndex index = new MemoryIndex(withOffsets);
+    private static Fields generateTermVectors(IndexShard indexShard, Map<String, Object> source, Collection<GetField> getFields, boolean withOffsets, @Nullable Map<String, String> perFieldAnalyzer, Set<String> fields) throws IOException {
+        Map<String, Collection<Object>> values = new HashMap<>();
         for (GetField getField : getFields) {
             String field = getField.getName();
-            if (fields.contains(field) == false) {
-                // some fields are returned even when not asked for, eg. _timestamp
-                continue;
+            if (fields.contains(field)) { // some fields are returned even when not asked for, eg. _timestamp
+                values.put(field, getField.getValues());
             }
+        }
+        if (source != null) {
+            for (String field : fields) {
+                if (values.containsKey(field) == false) {
+                    List<Object> v = XContentMapValues.extractRawValues(field, source);
+                    if (v.isEmpty() == false) {
+                        values.put(field, v);
+                    }
+                }
+            }
+        }
+
+        /* store document in memory index */
+        MemoryIndex index = new MemoryIndex(withOffsets);
+        for (Map.Entry<String, Collection<Object>> entry : values.entrySet()) {
+            String field = entry.getKey();
             Analyzer analyzer = getAnalyzerAtField(indexShard, field, perFieldAnalyzer);
-            for (Object text : getField.getValues()) {
+            for (Object text : entry.getValue()) {
                 index.addField(field, text.toString(), analyzer);
             }
         }
@@ -270,7 +291,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect
             String[] values = doc.getValues(field.name());
             getFields.add(new GetField(field.name(), Arrays.asList((Object[]) values)));
         }
-        return generateTermVectors(indexShard, getFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
+        return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true).v2(), getFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
     private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) {

diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java
@@ -149,21 +149,21 @@ public void testBulkUpdateSimple() throws Exception {
         assertThat(bulkResponse.getItems()[2].getResponse().getId(), equalTo("3"));
         assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(2L));
 
-        GetResponse getResponse = client().prepareGet().setIndex("test").setType("type1").setId("1").setFields("field").execute()
+        GetResponse getResponse = client().prepareGet().setIndex("test").setType("type1").setId("1").execute()
                 .actionGet();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getVersion(), equalTo(2L));
-        assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(2L));
+        assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(2L));
 
-        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").setFields("field").execute().actionGet();
+        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getVersion(), equalTo(2L));
-        assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(3L));
+        assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(3L));
 
-        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("3").setFields("field1").execute().actionGet();
+        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("3").execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getVersion(), equalTo(2L));
-        assertThat(getResponse.getField("field1").getValue().toString(), equalTo("test"));
+        assertThat(getResponse.getSource().get("field1").toString(), equalTo("test"));
 
         bulkResponse = client()
                 .prepareBulk()
@@ -185,18 +185,18 @@ public void testBulkUpdateSimple() throws Exception {
         assertThat(bulkResponse.getItems()[2].getResponse().getIndex(), equalTo("test"));
         assertThat(bulkResponse.getItems()[2].getResponse().getVersion(), equalTo(3L));
 
-        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("6").setFields("field").execute().actionGet();
+        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("6").execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getVersion(), equalTo(1L));
-        assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(0L));
+        assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(0L));
 
-        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("7").setFields("field").execute().actionGet();
+        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("7").execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(false));
 
-        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").setFields("field").execute().actionGet();
+        getResponse = client().prepareGet().setIndex("test").setType("type1").setId("2").execute().actionGet();
         assertThat(getResponse.isExists(), equalTo(true));
         assertThat(getResponse.getVersion(), equalTo(3L));
-        assertThat(((Number) getResponse.getField("field").getValue()).longValue(), equalTo(4L));
+        assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(4L));
     }
 
     public void testBulkVersioning() throws Exception {
@@ -325,11 +325,11 @@ public void testBulkUpdateLargerVolume() throws Exception {
             assertThat(((UpdateResponse) response.getItems()[i].getResponse()).getGetResult().field("counter").getValue(), equalTo(1));
 
             for (int j = 0; j < 5; j++) {
-                GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).setFields("counter").execute()
+                GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).execute()
                         .actionGet();
                 assertThat(getResponse.isExists(), equalTo(true));
                 assertThat(getResponse.getVersion(), equalTo(1L));
-                assertThat(((Number) getResponse.getField("counter").getValue()).longValue(), equalTo(1L));
+                assertThat(((Number) getResponse.getSource().get("counter")).longValue(), equalTo(1L));
             }
         }
 

diff --git a/core/src/test/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/AliasedIndexDocumentActionsIT.java
@@ -37,7 +37,9 @@ protected void createIndex() {
             // ignore
         }
         logger.info("--> creating index test");
-        client().admin().indices().create(createIndexRequest("test1").alias(new Alias("test"))).actionGet();
+        client().admin().indices().create(createIndexRequest("test1")
+                .mapping("type1", "name", "type=keyword,store=true")
+                .alias(new Alias("test"))).actionGet();
     }
 
     @Override

diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
 import java.io.IOException;
 
@@ -50,7 +51,7 @@
  */
 public class DocumentActionsIT extends ESIntegTestCase {
     protected void createIndex() {
-        createIndex(getConcreteIndexName());
+        ElasticsearchAssertions.assertAcked(prepareCreate(getConcreteIndexName()).addMapping("type1", "name", "type=keyword,store=true"));
     }
 
     protected String getConcreteIndexName() {

diff --git a/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java b/core/src/test/java/org/elasticsearch/explain/ExplainActionIT.java
@@ -114,7 +114,9 @@ public void testSimple() throws Exception {
     }
 
     public void testExplainWithFields() throws Exception {
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
+        assertAcked(prepareCreate("test")
+                .addMapping("test", "obj1.field1", "type=keyword,store=true", "obj1.field2", "type=keyword,store=true")
+                .addAlias(new Alias("alias")));
         ensureGreen("test");
 
         client().prepareIndex("test", "test", "1")

diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -61,6 +61,7 @@ public class GetActionIT extends ESIntegTestCase {
 
     public void testSimpleGet() {
         assertAcked(prepareCreate("test")
+                .addMapping("type1", "field1", "type=keyword,store=true", "field2", "type=keyword,store=true")
                 .setSettings(Settings.builder().put("index.refresh_interval", -1))
                 .addAlias(new Alias("alias")));
         ensureGreen();
@@ -107,15 +108,15 @@ public void testSimpleGet() {
         assertThat(response.getSourceAsMap().get("field1").toString(), equalTo("value1"));
         assertThat(response.getSourceAsMap().get("field2").toString(), equalTo("value2"));
 
-        logger.info("--> realtime fetch of field (requires fetching parsing source)");
+        logger.info("--> realtime fetch of field");
         response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
         assertThat(response.getSourceAsBytes(), nullValue());
         assertThat(response.getField("field1").getValues().get(0).toString(), equalTo("value1"));
         assertThat(response.getField("field2"), nullValue());
 
-        logger.info("--> realtime fetch of field & source (requires fetching parsing source)");
+        logger.info("--> realtime fetch of field & source");
         response = client().prepareGet(indexOrAlias(), "type1", "1").setFields("field1").setFetchSource("field1", null).get();
         assertThat(response.isExists(), equalTo(true));
         assertThat(response.getIndex(), equalTo("test"));
@@ -189,6 +190,7 @@ private static String indexOrAlias() {
 
     public void testSimpleMultiGet() throws Exception {
         assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+                .addMapping("type1", "field", "type=keyword,store=true")
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
         ensureGreen();
 
@@ -530,7 +532,7 @@ public void testMultiGetWithVersion() throws Exception {
     public void testGetFieldsMetaData() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("parent")
-                .addMapping("my-type1", "_parent", "type=parent")
+                .addMapping("my-type1", "_parent", "type=parent", "field1", "type=keyword,store=true")
                 .addAlias(new Alias("alias"))
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));