Increase domain compaction limits for hive and delta

docs/src/main/sphinx/connector/delta-lake.rst

@@ -1067,7 +1067,7 @@ connector.
         can compromise performance. For optimization in that situation, Trino
         can compact the large predicates. If necessary, adjust the threshold to
         ensure a balance between performance and predicate pushdown.
-      - ``100``
+      - ``1000``
     * - ``delta.max-outstanding-splits``
       - The target number of buffered splits for each table scan in a query,
         before the scheduler tries to pause.

plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConfig.java

@@ -52,7 +52,7 @@ public class DeltaLakeConfig
     private long metadataCacheMaxSize = 1000;
     private DataSize dataFileCacheSize = DEFAULT_DATA_FILE_CACHE_SIZE;
     private Duration dataFileCacheTtl = new Duration(30, TimeUnit.MINUTES);
-    private int domainCompactionThreshold = 100;
+    private int domainCompactionThreshold = 1000;
     private int maxOutstandingSplits = 1_000;
     private int maxSplitsPerSecond = Integer.MAX_VALUE;
     private int maxInitialSplits = 200;

plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeConfig.java

@@ -43,7 +43,7 @@ public void testDefaults()
                 .setDataFileCacheTtl(new Duration(30, MINUTES))
                 .setMetadataCacheTtl(new Duration(5, TimeUnit.MINUTES))
                 .setMetadataCacheMaxSize(1000)
-                .setDomainCompactionThreshold(100)
+                .setDomainCompactionThreshold(1000)
                 .setMaxSplitsPerSecond(Integer.MAX_VALUE)
                 .setMaxOutstandingSplits(1_000)
                 .setMaxInitialSplits(200)

plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConfig.java

@@ -81,7 +81,7 @@ public class HiveConfig
     private int splitLoaderConcurrency = 64;
     private Integer maxSplitsPerSecond;
     private DataSize maxInitialSplitSize;
-    private int domainCompactionThreshold = 100;
+    private int domainCompactionThreshold = 1000;
     private boolean forceLocalScheduling;
     private boolean recursiveDirWalkerEnabled;
     private boolean ignoreAbsentPartitions;

plugin/trino-hive/src/main/java/io/trino/plugin/hive/HivePageSourceProvider.java

@@ -150,9 +150,6 @@ public ConnectorPageSource createPageSource(
 
         Configuration configuration = hdfsEnvironment.getConfiguration(new HdfsContext(session), new Path(hiveSplit.getPath()));
 
-        TupleDomain<HiveColumnHandle> simplifiedDynamicFilter = dynamicFilter
-                .getCurrentPredicate()
-                .transformKeys(HiveColumnHandle.class::cast).simplify(domainCompactionThreshold);
         Optional<ConnectorPageSource> pageSource = createHivePageSource(
                 pageSourceFactories,
                 cursorProviders,
@@ -164,7 +161,9 @@ public ConnectorPageSource createPageSource(
                 hiveSplit.getLength(),
                 hiveSplit.getEstimatedFileSize(),
                 hiveSplit.getSchema(),
-                hiveTable.getCompactEffectivePredicate().intersect(simplifiedDynamicFilter),
+                hiveTable.getCompactEffectivePredicate().intersect(
+                                dynamicFilter.getCurrentPredicate().transformKeys(HiveColumnHandle.class::cast))
+                        .simplify(domainCompactionThreshold),
                 hiveColumns,
                 typeManager,
                 hiveSplit.getBucketConversion(),

plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java

@@ -53,7 +53,7 @@ public void testDefaults()
                 .setMaxInitialSplitSize(DataSize.of(32, Unit.MEGABYTE))
                 .setSplitLoaderConcurrency(64)
                 .setMaxSplitsPerSecond(null)
-                .setDomainCompactionThreshold(100)
+                .setDomainCompactionThreshold(1000)
                 .setTargetMaxFileSize(DataSize.of(1, GIGABYTE))
                 .setForceLocalScheduling(false)
                 .setMaxConcurrentFileSystemOperations(20)