Use caching hive metastore and change defaults

jkhaliqi · Mar 1, 2023 · 365cba7 · 365cba7
1 parent 0263bdd
commit 365cba7
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 10 deletions.
diff --git a/presto-hudi/src/main/java/com/facebook/presto/hudi/HudiConfig.java b/presto-hudi/src/main/java/com/facebook/presto/hudi/HudiConfig.java
@@ -32,8 +32,8 @@ public class HudiConfig
     private DataSize standardSplitWeightSize = new DataSize(128, MEGABYTE);
     private double minimumAssignedSplitWeight = 0.05;
     private int maxOutstandingSplits = 1000;
-    private int splitLoaderParallelism = 2;
-    private int splitGeneratorParallelism = 8;
+    private int splitLoaderParallelism = 4;
+    private int splitGeneratorParallelism = 4;
 
     public boolean isMetadataTableEnabled()
     {

diff --git a/presto-hudi/src/main/java/com/facebook/presto/hudi/HudiMetadataFactory.java b/presto-hudi/src/main/java/com/facebook/presto/hudi/HudiMetadataFactory.java
@@ -16,6 +16,8 @@
 
 import com.facebook.presto.common.type.TypeManager;
 import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.MetastoreClientConfig;
+import com.facebook.presto.hive.metastore.CachingHiveMetastore;
 import com.facebook.presto.hive.metastore.ExtendedHiveMetastore;
 import com.facebook.presto.spi.connector.ConnectorMetadata;
 
@@ -28,20 +30,30 @@ public class HudiMetadataFactory
     private final ExtendedHiveMetastore metastore;
     private final HdfsEnvironment hdfsEnvironment;
     private final TypeManager typeManager;
+    private final long perTransactionCacheMaximumSize;
+    private final boolean metastoreImpersonationEnabled;
+    private final int metastorePartitionCacheMaxColumnCount;
 
     @Inject
     public HudiMetadataFactory(
             ExtendedHiveMetastore metastore,
             HdfsEnvironment hdfsEnvironment,
-            TypeManager typeManager)
+            TypeManager typeManager,
+            MetastoreClientConfig metastoreClientConfig)
     {
         this.metastore = requireNonNull(metastore, "metastore is null");
         this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
         this.typeManager = requireNonNull(typeManager, "typeManager is null");
+        this.perTransactionCacheMaximumSize = metastoreClientConfig.getPerTransactionMetastoreCacheMaximumSize();
+        this.metastoreImpersonationEnabled = metastoreClientConfig.isMetastoreImpersonationEnabled();
+        this.metastorePartitionCacheMaxColumnCount = metastoreClientConfig.getPartitionCacheColumnCountLimit();
     }
 
     public ConnectorMetadata create()
     {
-        return new HudiMetadata(metastore, hdfsEnvironment, typeManager);
+        return new HudiMetadata(
+                CachingHiveMetastore.memoizeMetastore(metastore, metastoreImpersonationEnabled, perTransactionCacheMaximumSize, metastorePartitionCacheMaxColumnCount),
+                hdfsEnvironment,
+                typeManager);
     }
 }
diff --git a/presto-hudi/src/test/java/com/facebook/presto/hudi/TestHudiConfig.java b/presto-hudi/src/test/java/com/facebook/presto/hudi/TestHudiConfig.java
@@ -36,8 +36,8 @@ public void testDefaults()
                 .setStandardSplitWeightSize(new DataSize(128, MEGABYTE))
                 .setMinimumAssignedSplitWeight(0.05)
                 .setMaxOutstandingSplits(1000)
-                .setSplitLoaderParallelism(2)
-                .setSplitGeneratorParallelism(8));
+                .setSplitLoaderParallelism(4)
+                .setSplitGeneratorParallelism(4));
     }
 
     @Test
@@ -49,8 +49,8 @@ public void testExplicitPropertyMappings()
                 .put("hudi.standard-split-weight-size", "500MB")
                 .put("hudi.minimum-assigned-split-weight", "0.1")
                 .put("hudi.max-outstanding-splits", "300")
-                .put("hudi.split-loader-parallelism", "6")
-                .put("hudi.split-generator-parallelism", "4")
+                .put("hudi.split-loader-parallelism", "2")
+                .put("hudi.split-generator-parallelism", "8")
                 .build();
 
         HudiConfig expected = new HudiConfig()
@@ -59,8 +59,8 @@ public void testExplicitPropertyMappings()
                 .setStandardSplitWeightSize(new DataSize(500, MEGABYTE))
                 .setMinimumAssignedSplitWeight(0.1)
                 .setMaxOutstandingSplits(300)
-                .setSplitLoaderParallelism(6)
-                .setSplitGeneratorParallelism(4);
+                .setSplitLoaderParallelism(2)
+                .setSplitGeneratorParallelism(8);
 
         assertFullMapping(properties, expected);
     }