Index optimizer updates

oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java

@@ -674,20 +674,24 @@ private void improveIndexes(NodeStore store) {
             return;
         }
         TabularData slow = stats.getSlowQueries();
+
         @SuppressWarnings("unchecked")
-        Collection<CompositeData> coll = (Collection<CompositeData>) slow.values();
+        Collection<CompositeData> coll = new ArrayList<>((Collection<CompositeData>) slow.values());
+
+        // Find inefficient queries and add to collection for index diff generation
+        coll.addAll(findInefficientQueries(stats));
+
         if (coll.isEmpty()) {
             return;
         }
         boolean changed = false;
         for (CompositeData cd : coll) {
             String language = (String) cd.get("language");
             String statement = (String) cd.get("statement");
+
             log.info("language {} statement {}", language, statement);
             String indexDef = IndexDefinitionGenerator.generateIndexDefinition(language, statement);
-            if (indexDef != null) {
-                changed |= DiffIndexUpdater.applyIndexDefinition(store, rootState, builder, indexDef);
-            }
+            changed |= DiffIndexUpdater.applyIndexDefinition(store, rootState, builder, indexDef);
         }
         if (changed) {
             stats.resetStats();
@@ -699,6 +703,29 @@ private void improveIndexes(NodeStore store) {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    private List<CompositeData> findInefficientQueries(final QueryStatsMBean stats) {
+        final TabularData popularQueries = stats.getPopularQueries();
+        final List<CompositeData> inefficientQueries = new ArrayList<>();
+
+        for (CompositeData queryData : (Collection<? extends CompositeData>) popularQueries.values()) {
+            final Long rowsRead = (Long) queryData.get("rowsRead");
+            final Long rowsScanned = (Long) queryData.get("rowsScanned");
+
+            int readEfficiency = 100;
+
+            if (rowsScanned > 0) {
+                readEfficiency = (int) ((rowsRead * 100f) / rowsScanned);
+            }
+
+            if (readEfficiency < 30) {
+                inefficientQueries.add(queryData);
+            }
+        }
+
+        return inefficientQueries;
+    }
+
     private void clearLease() throws CommitFailedException {
         NodeState root = store.getRoot();
         NodeState async = root.getChildNode(ASYNC);

oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/optimizer/IndexConfigGenerator.java

@@ -32,6 +32,7 @@
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.commons.PathUtils;
 import org.apache.jackrabbit.oak.core.ImmutableRoot;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
 import org.apache.jackrabbit.oak.plugins.index.optimizer.IndexDefinitionBuilder.IndexRule;
 import org.apache.jackrabbit.oak.plugins.index.optimizer.IndexDefinitionBuilder.PropertyRule;
 import org.apache.jackrabbit.oak.query.ExecutionContext;
@@ -55,8 +56,8 @@
 
 class IndexConfigGenerator {
 
-    private QueryEngine queryEngine;
-    private IndexDefinitionBuilder builder = new IndexDefinitionBuilder();
+    private final QueryEngine queryEngine;
+    private final IndexDefinitionBuilder builder = new IndexDefinitionBuilder();
     private final Set<String> propsWithFulltextConstraints = new HashSet<>();
 
     public IndexConfigGenerator() {
@@ -112,12 +113,21 @@ public NodeState getIndexConfig() {
     private void processFilter(Filter filter, List<OrderEntry> sortOrder) {
         addPathRestrictions(filter);
         IndexRule rule = processNodeTypeConstraint(filter);
+        processTags(filter);
         processFulltextConstraints(filter, rule);
         processPropertyRestrictions(filter, rule);
         processSortConditions(sortOrder, rule);
         processPureNodeTypeConstraints(filter, rule);
     }
 
+    private void processTags(Filter filter) {
+        PropertyRestriction indexTag = filter.getPropertyRestriction(IndexConstants.INDEX_TAG_OPTION);
+
+        if (indexTag != null && indexTag.first != null) {
+            builder.tags(indexTag.first.getValue(Type.STRING));
+        }
+    }
+
     private void addPathRestrictions(Filter filter) {
         if (!filter.getPath().isEmpty() && !"/".equals(filter.getPath())) {
             String path = filter.getPath().replaceAll("\\s", "");
@@ -210,7 +220,6 @@ private void processSortConditions(List<OrderEntry> sortOrder, IndexRule rule) {
         }
     }
 
-
     /**
      * Returns if the propertyName is a function. If it is, it will be in Polish notation.
      *

oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/optimizer/IndexDefinitionBuilder.java

@@ -44,7 +44,7 @@ public class IndexDefinitionBuilder {
     private final NodeBuilder indexRule;
     private NodeBuilder aggregateBuilder;
 
-    public IndexDefinitionBuilder(){
+    public IndexDefinitionBuilder() {
         builder.setProperty("compatVersion", 2);
         builder.setProperty("async", "async");
         builder.setProperty("type", "lucene");
@@ -73,6 +73,11 @@ public IndexDefinitionBuilder excludedPaths(String ... paths){
         return this;
     }
 
+    public IndexDefinitionBuilder tags(String... tags) {
+        builder.setProperty(IndexConstants.INDEX_TAGS, Arrays.asList(tags), Type.STRINGS);
+        return this;
+    }
+
     public NodeState build(){
         return builder.getNodeState();
     }

oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/optimizer/IndexDefinitionGeneratorTest.java

@@ -24,7 +24,7 @@ public class IndexDefinitionGeneratorTest {
 
     @Test
     public void test() {
-        String def = IndexDefinitionGenerator.generateIndexDefinition("xpath", "/jcr:root/content//element(*, acme:test)[@test=1]");
+        String def = IndexDefinitionGenerator.generateIndexDefinition("xpath", "/jcr:root/content//element(*, acme:test)[@test=1]  option (index tag testTag)");
         assertEquals("{\n"
                 + "  \"index\": {\n"
                 + "    \"compatVersion\": 2,\n"
@@ -34,6 +34,7 @@ public void test() {
                 + "    \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n"
                 + "    \"evaluatePathRestrictions\": true,\n"
                 + "    \"type\": \"lucene\",\n"
+                + "    \"tags\": [\"testTag\"],\n"
                 + "    \"indexRules\": {\n"
                 + "      \"jcr:primaryType\": \"nam:nt:unstructured\",\n"
                 + "      \"acme:test\": {\n"