[apache#3403] bug(hive-catalog): add hive catalog property list-all-…

…tabless (apache#3403)
mygrsun · May 31, 2024 · ee95672 · ee95672
1 parent b7080c5
commit ee95672
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 3 deletions.
diff --git a/...talog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java b/...talog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogOperations.java
@@ -6,9 +6,12 @@
 
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS;
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.CLIENT_POOL_SIZE;
+import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.LIST_ALL_TABLES;
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.METASTORE_URIS;
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.PRINCIPAL;
+import static com.datastrato.gravitino.catalog.hive.HiveTable.ICEBERG_TABLE_TYPE_VALUE;
 import static com.datastrato.gravitino.catalog.hive.HiveTable.SUPPORT_TABLE_TYPES;
+import static com.datastrato.gravitino.catalog.hive.HiveTable.TABLE_TYPE_PROP;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.COMMENT;
 import static com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TABLE_TYPE;
 import static com.datastrato.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
@@ -97,6 +100,7 @@ public class HiveCatalogOperations implements CatalogOperations, SupportsSchemas
   private ScheduledThreadPoolExecutor checkTgtExecutor;
   private String kerberosRealm;
   private ProxyPlugin proxyPlugin;
+  boolean listAllTables = true;
 
   // Map that maintains the mapping of keys in Gravitino to that in Hive, for example, users
   // will only need to set the configuration 'METASTORE_URL' in Gravitino and Gravitino will change
@@ -148,6 +152,8 @@ public void initialize(
 
     this.clientPool =
         new CachedClientPool(getClientPoolSize(conf), hiveConf, getCacheEvictionInterval(conf));
+
+    this.listAllTables = getListAllTables(conf);
   }
 
   private void initKerberosIfNecessary(Map<String, String> conf, Configuration hadoopConf) {
@@ -265,7 +271,11 @@ long getCacheEvictionInterval(Map<String, String> conf) {
             .catalogPropertiesMetadata()
             .getOrDefault(conf, CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS);
   }
-
+  boolean getListAllTables(Map<String, String> conf) {
+    return (boolean)
+            propertiesMetadata
+                    .catalogPropertiesMetadata().getOrDefault(conf, LIST_ALL_TABLES);
+  }
   /** Closes the Hive catalog and releases the associated client pool. */
   @Override
   public void close() {
@@ -521,8 +531,22 @@ public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaExcep
       return clientPool.run(
           c ->
               c.getTableObjectsByName(schemaIdent.name(), allTables).stream()
-                  .filter(tb -> SUPPORT_TABLE_TYPES.contains(tb.getTableType()))
-                  .map(tb -> NameIdentifier.of(namespace, tb.getTableName()))
+                  .filter(tb -> {
+                    boolean isSupportTable = SUPPORT_TABLE_TYPES.contains(tb.getTableType());
+                    if (!isSupportTable) {
+                      return false;
+                    }
+                    if (!listAllTables) {
+                      Map<String, String> parameters = tb.getParameters();
+                      if (parameters != null) {
+                        boolean isIcebergTable = ICEBERG_TABLE_TYPE_VALUE.equalsIgnoreCase(parameters.get(TABLE_TYPE_PROP));
+                        if (isIcebergTable) {
+                          return false;
+                        }
+                      }
+                    }
+                    return true;
+                  }).map(tb -> NameIdentifier.of(namespace, tb.getTableName()))
                   .toArray(NameIdentifier[]::new));
     } catch (UnknownDBException e) {
       throw new NoSuchSchemaException(

diff --git a/...g-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java b/...g-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveCatalogPropertiesMeta.java
@@ -36,6 +36,10 @@ public class HiveCatalogPropertiesMeta extends BaseCatalogPropertiesMetadata {
 
   public static final String FETCH_TIMEOUT_SEC = "kerberos.keytab-fetch-timeout-sec";
 
+  public static final String LIST_ALL_TABLES = "list-all-tables";
+
+  public static final boolean DEFAULT_LIST_ALL_TABLES = true;
+
   private static final Map<String, PropertyEntry<?>> HIVE_CATALOG_PROPERTY_ENTRIES =
       ImmutableMap.<String, PropertyEntry<?>>builder()
           .put(
@@ -88,6 +92,16 @@ public class HiveCatalogPropertiesMeta extends BaseCatalogPropertiesMetadata {
               FETCH_TIMEOUT_SEC,
               PropertyEntry.integerOptionalPropertyEntry(
                   FETCH_TIMEOUT_SEC, "The timeout to fetch key tab", true, 60, false))
+            .put(
+                    LIST_ALL_TABLES,
+                    PropertyEntry.booleanPropertyEntry(
+                            LIST_ALL_TABLES,
+                            "list all tables with iceberg tables",
+                            false,
+                            false,
+                            DEFAULT_LIST_ALL_TABLES,
+                            false,
+                            false))
           .putAll(BASIC_CATALOG_PROPERTY_ENTRIES)
           .build();
 

diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/HiveTable.java
@@ -62,6 +62,8 @@ public class HiveTable extends BaseTable {
   // A set of supported Hive table types.
   public static final Set<String> SUPPORT_TABLE_TYPES =
       Sets.newHashSet(MANAGED_TABLE.name(), EXTERNAL_TABLE.name());
+  public static final String ICEBERG_TABLE_TYPE_VALUE = "ICEBERG";
+  public static final String TABLE_TYPE_PROP = "table_type";
   private String schemaName;
   private CachedClientPool clientPool;
   private StorageDescriptor sd;

diff --git a/...g-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java b/...g-hive/src/test/java/com/datastrato/gravitino/catalog/hive/TestHiveCatalogOperations.java
@@ -11,6 +11,7 @@
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.FETCH_TIMEOUT_SEC;
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.IMPERSONATION_ENABLE;
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.KEY_TAB_URI;
+import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.LIST_ALL_TABLES;
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.METASTORE_URIS;
 import static com.datastrato.gravitino.catalog.hive.HiveCatalogPropertiesMeta.PRINCIPAL;
 import static com.datastrato.gravitino.catalog.hive.TestHiveCatalog.HIVE_PROPERTIES_METADATA;
@@ -73,6 +74,8 @@ void testPropertyMeta() {
     Assertions.assertTrue(propertyEntryMap.containsKey(BaseCatalog.CATALOG_OPERATION_IMPL));
     Assertions.assertTrue(propertyEntryMap.containsKey(CLIENT_POOL_SIZE));
     Assertions.assertTrue(propertyEntryMap.containsKey(IMPERSONATION_ENABLE));
+    Assertions.assertTrue(propertyEntryMap.containsKey(LIST_ALL_TABLES));
+
 
     Assertions.assertTrue(propertyEntryMap.get(METASTORE_URIS).isRequired());
     Assertions.assertFalse(propertyEntryMap.get(Catalog.PROPERTY_PACKAGE).isRequired());

diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md
@@ -39,6 +39,7 @@ The Hive catalog supports creating, updating, and deleting databases and tables
 | `kerberos.keytab-uri`                    | The uri of key tab for the catalog. Now supported protocols are `https`, `http`, `ftp`, `file`.                                                                                                                                                                         | (none)        | required if you use kerberos | 0.4.0         |
 | `kerberos.check-interval-sec`            | The interval to check validness of the principal                                                                                                                                                                                                                        | 60            | No                           | 0.4.0         |
 | `kerberos.keytab-fetch-timeout-sec`      | The timeout to fetch key tab                                                                                                                                                                                                                                            | 60            | No                           | 0.4.0         |
+| `list-all-tables`                        | List all tables with iceberg tables
 
 When you use the Gravitino with Trino. You can pass the Trino Hive connector configuration using prefix `trino.bypass.`. For example, using `trino.bypass.hive.config.resources` to pass the `hive.config.resources` to the Gravitino Hive catalog in Trino runtime.