OAK-11260 Speed up Lucene cost estimation

oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/Profiler.java

@@ -389,6 +389,9 @@ private void processList(List<Object[]> list) {
             boolean packageCounts = false;
             for (int j = 0, i = 0; i < dump.length && j < depth; i++) {
                 String el = dump[i].toString();
+                if (el.startsWith("app//")) {
+                    el = el.substring("app//".length());
+                }
                 if (!el.equals(last) && !startsWithAny(el, ignoreLines)) {
                     last = el;
                     buff.append("at ").append(el).append(LINE_SEPARATOR);

oak-core/src/test/java/org/apache/jackrabbit/oak/query/SQL2OptimiseQueryTest.java

@@ -35,6 +35,8 @@
 
 import java.text.ParseException;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.jcr.RepositoryException;
 
@@ -66,6 +68,33 @@ public boolean isSql2Optimisation() {
         }
     };
 
+    @Test
+    public void limitUnionSize() throws ParseException {
+        String query = "SELECT * FROM [nt:base]\n"
+                + "WHERE (CONTAINS(*, '1') AND ([jcr:uuid] LIKE '2' OR [jcr:uuid] LIKE '3'))\n"
+                + "  AND ((CONTAINS(*, '4') AND ([jcr:uuid] LIKE '5' OR [jcr:uuid] LIKE '6'))\n"
+                + "       OR (CONTAINS(*, '0a') AND ([jcr:uuid] LIKE '0b' OR [jcr:uuid] LIKE '0c'))\n"
+                + "       OR (CONTAINS(*, '1a') AND ([jcr:uuid] LIKE '1b' OR [jcr:uuid] LIKE '1c'))\n"
+                + "       OR (CONTAINS(*, '2a') AND ([jcr:uuid] LIKE '2b' OR [jcr:uuid] LIKE '2c'))\n"
+                + "       OR (CONTAINS(*, '3a') AND ([jcr:uuid] LIKE '3b' OR [jcr:uuid] LIKE '3c'))\n"
+                + "       OR (CONTAINS(*, '4a') AND ([jcr:uuid] LIKE '4b' OR [jcr:uuid] LIKE '4c'))\n"
+                + "       OR (CONTAINS(*, '5a') AND ([jcr:uuid] LIKE '5b' OR [jcr:uuid] LIKE '5c'))\n"
+                + "       OR (CONTAINS(*, '6a') AND ([jcr:uuid] LIKE '6b' OR [jcr:uuid] LIKE '6c'))\n"
+                + "       OR (CONTAINS(*, '7a') AND ([jcr:uuid] LIKE '7b' OR [jcr:uuid] LIKE '7c'))\n"
+                + "       OR (CONTAINS(*, '8a') AND ([jcr:uuid] LIKE '8b' OR [jcr:uuid] LIKE '8c'))\n"
+                + "       OR (CONTAINS(*, '9a') AND ([jcr:uuid] LIKE '9b' OR [jcr:uuid] LIKE '9c'))\n"
+                + "       OR (CONTAINS(*, 'ea') AND ([jcr:uuid] LIKE 'eb' OR [jcr:uuid] LIKE 'ec')))\n"
+                + "  AND ((CONTAINS(*, '10') AND ([jcr:uuid] LIKE '11' OR [jcr:uuid] LIKE '12'))\n"
+                + "       OR (CONTAINS(*, '13') AND ([jcr:uuid] LIKE '14' OR [jcr:uuid] LIKE '15')))";
+        SQL2Parser parser = SQL2ParserTest.createTestSQL2Parser(
+                getMappings(), getNodeTypes(), qeSettings);
+        Query original;
+        original = parser.parse(query, false);
+        assertNotNull(original);
+        String alternative = original.buildAlternativeQuery().toString();
+        assertEquals(60825, alternative.length());
+    }
+
     /**
      * checks the {@code Query#optimise()} calls for the conversion from OR to UNION from a query
      * POV; ensuring that it returns always the same, expected resultset.

oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexLookupUtil.java

@@ -27,11 +27,17 @@
 import org.apache.jackrabbit.oak.plugins.index.search.IndexLookup;
 import org.apache.jackrabbit.oak.spi.query.Filter;
 import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
 import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
 
 class LuceneIndexLookupUtil {
+
+    private static final Logger LOG = LoggerFactory.getLogger(LuceneIndexLookupUtil.class);
+
     static final Predicate<NodeState> LUCENE_INDEX_DEFINITION_PREDICATE =
             state -> TYPE_LUCENE.equals(state.getString(TYPE_PROPERTY_NAME));
 
@@ -45,6 +51,14 @@ private LuceneIndexLookupUtil() {
     public static String getOldFullTextIndexPath(NodeState root, Filter filter, IndexTracker tracker) {
         Collection<String> indexPaths = getLuceneIndexLookup(root).collectIndexNodePaths(filter, false);
         for (String path : indexPaths) {
+            NodeState node = NodeStateUtils.getNode(root, path);
+            if (IndexDefinition.determineIndexFormatVersion(node) != IndexFormatVersion.V1) {
+                // shortcut to avoid reading the index definition if not needed
+                continue;
+            }
+            if (LOG.isDebugEnabled()) {
+                LOG.debug("Old V1 lucene index found at {}", path);
+            }
             IndexDefinition indexDefinition = tracker.getIndexDefinition(path);
             if (indexDefinition != null && indexDefinition.isFullTextEnabled()
                     && indexDefinition.getVersion() == IndexFormatVersion.V1) {

oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/ManyIndexesAndUnionPlanTest.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.jcr.query;
+
+import javax.jcr.Node;
+import javax.jcr.NodeIterator;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.Row;
+import javax.jcr.query.RowIterator;
+
+import org.apache.jackrabbit.core.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexFormatVersion;
+
+public class ManyIndexesAndUnionPlanTest extends AbstractQueryTest {
+
+    public void testResultSize() throws Exception {
+        createIndexes();
+        createData();
+        doTestResultSize(10);
+    }
+
+    private void createIndexes() throws RepositoryException, InterruptedException {
+        Session session = superuser;
+        Node index = session.getRootNode().getNode("oak:index");
+        for (int i = 0; i < 10; i++) {
+            Node lucene = index.addNode("lucene" + i, "oak:QueryIndexDefinition");
+            lucene.setProperty("type", "lucene");
+            lucene.setProperty("async", "async");
+            lucene.setProperty(FulltextIndexConstants.COMPAT_MODE, IndexFormatVersion.V1.getVersion());
+        }
+        session.save();
+    }
+
+    private void createData() throws RepositoryException {
+        Session session = superuser;
+        for (int i = 0; i < 10; i++) {
+            Node n = testRootNode.addNode("node" + i);
+            n.setProperty("text", "Hello World");
+        }
+        session.save();
+    }
+
+    private void doTestResultSize(int expected) throws RepositoryException {
+        Session session = superuser;
+        QueryManager qm = session.getWorkspace().getQueryManager();
+
+        String xpath;
+        xpath = "/jcr:root//*[jcr:contains(@text, 'Hello World')]";
+
+        Query q;
+        long result;
+        NodeIterator it;
+        StringBuilder buff;
+
+        q = qm.createQuery(xpath, "xpath");
+        it = q.execute().getNodes();
+        result = it.getSize();
+        assertTrue("size: " + result + " expected around " + expected, 
+                result > expected - 50 && 
+                result < expected + 50);
+        buff = new StringBuilder();
+        while (it.hasNext()) {
+            Node n = it.nextNode();
+            buff.append(n.getPath()).append('\n');
+        }
+        for (int j = 0; j < 1; j++) {
+            for (int i = 0; i < 1; i++) {
+                q = qm.createQuery("explain " + xpath, "xpath");
+                RowIterator rit = q.execute().getRows();
+                Row r = rit.nextRow();
+                assertTrue(r.toString().indexOf("luceneGlobal") >= 0);
+            }
+        }
+    }
+
+}

oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java

@@ -1829,7 +1829,7 @@ private static void markAsNtUnstructured(NodeBuilder nb) {
         nb.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
     }
 
-    protected static IndexFormatVersion determineIndexFormatVersion(NodeState defn) {
+    public static IndexFormatVersion determineIndexFormatVersion(NodeState defn) {
         //Compat mode version if specified has highest priority
         if (defn.hasProperty(COMPAT_MODE)) {
             return versionFrom(defn.getProperty(COMPAT_MODE));