Override LIMIT value as a new scan limit if it is smaller than filter…

…ed count all's result (#7468)

* Override limit value as scan limit if filtered_count_all_result > scan limit

* Add tests

* Ignore LIMIT value for top-k sort query

Signed-off-by: Misiu Godfrey <misiu.godfrey@kraken.mapd.com>

QueryEngine/RelAlgExecutor.cpp

@@ -3788,10 +3788,19 @@ ExecutionResult RelAlgExecutor::executeWorkUnit(
       } else if (!eo.just_explain) {
         const auto filter_count_all = getFilteredCountAll(ra_exe_unit, true, co, eo);
         if (filter_count_all) {
-          VLOG(1) << "Pre-flight counts query returns " << *filter_count_all;
           ra_exe_unit.scan_limit = std::max(*filter_count_all, size_t(1));
           VLOG(1) << "Set a new scan limit from filtered_count_all: "
                   << ra_exe_unit.scan_limit;
+          auto const has_limit_value = ra_exe_unit.sort_info.limit.has_value();
+          auto const top_k_sort_query =
+              has_limit_value && !ra_exe_unit.sort_info.order_entries.empty();
+          // top-k sort query needs to get a global result before sorting, so we cannot
+          // apply LIMIT value at this point
+          if (has_limit_value && !top_k_sort_query &&
+              ra_exe_unit.scan_limit > ra_exe_unit.sort_info.limit.value()) {
+            ra_exe_unit.scan_limit = ra_exe_unit.sort_info.limit.value();
+            VLOG(1) << "Override scan limit to LIMIT value: " << ra_exe_unit.scan_limit;
+          }
         }
       }
     }

Tests/ExecuteTest.cpp

@@ -9729,6 +9729,25 @@ TEST_F(Select, PerDeviceCardinalityShardedTable) {
   run_ddl_statement("DROP TABLE IF EXISTS pdc_test;");
 }
 
+TEST_F(Select, FilteredCountallWithLimit) {
+  ScopeGuard reset = [old_val = g_preflight_count_query_threshold] {
+    g_preflight_count_query_threshold = old_val;
+    run_ddl_statement("DROP TABLE IF EXISTS fcwl_test;");
+  };
+  g_preflight_count_query_threshold = 10;
+  run_ddl_statement("DROP TABLE IF EXISTS fcwl_test;");
+  run_ddl_statement("CREATE TABLE fcwl_test (v INT) WITH (FRAGMENT_SIZE=20);");
+  for (size_t i = 0; i < 40; i++) {
+    run_multiple_agg("INSERT INTO fcwl_test VALUES(1);", ExecutorDeviceType::CPU);
+  }
+  for (auto dt : {ExecutorDeviceType::CPU, ExecutorDeviceType::GPU}) {
+    SKIP_NO_GPU();
+    std::shared_ptr<ResultSet> res =
+        run_multiple_agg("SELECT v FROM fcwl_test LIMIT 11", dt);
+    EXPECT_EQ(res->entryCount(), size_t(11));
+  }
+}
+
 TEST_F(Select, UnsupportedNodes) {
   for (auto dt : {ExecutorDeviceType::CPU, ExecutorDeviceType::GPU}) {
     SKIP_NO_GPU();