Reduce IndexOrDocValuesQuery DV penalty from 8x to 6x

lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexOrDocValuesQueryBenchmark.java

@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KeywordField;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.IOUtils;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Benchmarks the points-vs-DV decision in IndexOrDocValuesQuery. Monotonic timestamps are close to
+ * a best case for BKD, so these results are conservative for DV relative to real workloads.
+ */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 3, time = 3)
+@Measurement(iterations = 5, time = 5)
+@Fork(value = 2, warmups = 1)
+public class IndexOrDocValuesQueryBenchmark {
+
+  private Directory dir;
+  private IndexReader reader;
+  private IndexSearcher searcher;
+  private Path path;
+
+  private BooleanQuery crossover10Query;
+  private BooleanQuery crossover20Query;
+  private BooleanQuery crossover30Query;
+  private BooleanQuery dvFavorableQuery;
+  private BooleanQuery pointsFavorableQuery;
+
+  @Param({"1000000", "10000000"})
+  public int docCount;
+
+  @Setup(Level.Trial)
+  public void setup() throws Exception {
+    path = Files.createTempDirectory("idvqBench");
+    dir = MMapDirectory.open(path);
+
+    IndexWriterConfig iwc = new IndexWriterConfig();
+    IndexWriter w = new IndexWriter(dir, iwc);
+
+    int numBuckets = 100;
+    for (int i = 0; i < docCount; i++) {
+      Document doc = new Document();
+      doc.add(new LongField("timestamp", i, Field.Store.NO));
+      doc.add(new KeywordField("bucket", "b" + (i % numBuckets), Field.Store.NO));
+      w.addDocument(doc);
+    }
+    w.forceMerge(1);
+    reader = DirectoryReader.open(w);
+    w.close();
+    searcher = new IndexSearcher(reader);
+    searcher.setQueryCache(null);
+
+    Query range80 = LongField.newRangeQuery("timestamp", 0, docCount * 4L / 5);
+
+    crossover10Query = buildConjunction(range80, 10, numBuckets);
+    crossover20Query = buildConjunction(range80, 20, numBuckets);
+    crossover30Query = buildConjunction(range80, 30, numBuckets);
+
+    // DV favorable: 1% lead + 80% range. Both 8x and 4x choose DV.
+    dvFavorableQuery =
+        new BooleanQuery.Builder()
+            .add(new TermQuery(new Term("bucket", "b0")), Occur.FILTER)
+            .add(range80, Occur.FILTER)
+            .build();
+
+    // Points favorable: 50% lead + 5% range. Both choose points.
+    Query range5 = LongField.newRangeQuery("timestamp", 0, docCount / 20);
+    BooleanQuery.Builder lead50 = new BooleanQuery.Builder();
+    for (int i = 0; i < 50; i++) {
+      lead50.add(new TermQuery(new Term("bucket", "b" + i)), Occur.SHOULD);
+    }
+    pointsFavorableQuery =
+        new BooleanQuery.Builder()
+            .add(lead50.build(), Occur.FILTER)
+            .add(range5, Occur.FILTER)
+            .build();
+  }
+
+  private static BooleanQuery buildConjunction(Query range, int leadBuckets, int totalBuckets) {
+    BooleanQuery.Builder lead = new BooleanQuery.Builder();
+    for (int i = 0; i < leadBuckets; i++) {
+      lead.add(new TermQuery(new Term("bucket", "b" + i)), Occur.SHOULD);
+    }
+    return new BooleanQuery.Builder()
+        .add(lead.build(), Occur.FILTER)
+        .add(range, Occur.FILTER)
+        .build();
+  }
+
+  @TearDown(Level.Trial)
+  public void tearDown() throws Exception {
+    reader.close();
+    dir.close();
+    IOUtils.rm(path);
+  }
+
+  @Benchmark
+  public TopDocs crossover10() throws IOException {
+    return searcher.search(crossover10Query, 10);
+  }
+
+  @Benchmark
+  public TopDocs crossover20() throws IOException {
+    return searcher.search(crossover20Query, 10);
+  }
+
+  @Benchmark
+  public TopDocs crossover30() throws IOException {
+    return searcher.search(crossover30Query, 10);
+  }
+
+  @Benchmark
+  public TopDocs dvFavorable() throws IOException {
+    return searcher.search(dvFavorableQuery, 10);
+  }
+
+  @Benchmark
+  public TopDocs pointsFavorable() throws IOException {
+    return searcher.search(pointsFavorableQuery, 10);
+  }
+}

lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java

@@ -184,8 +184,10 @@ public Scorer get(long leadCost) throws IOException {
             // At equal costs, doc values tend to be worse than points since they
             // still need to perform one comparison per document while points can
             // do much better than that given how values are organized. So we give
-            // an arbitrary 8x penalty to doc values.
-            final long threshold = cost() >>> 3;
+            // a 6x penalty to doc values. (Reduced from 8x: DocValuesSkipper
+            // enables block-level skipping that makes DV competitive with points
+            // for selective leads in conjunctions.)
+            final long threshold = cost() / 6;
             if (threshold <= leadCost) {
               return indexScorerSupplier.get(leadCost);
             } else {