[apache#3188] feat(core): Clean user,group and role entity in Relatio…

…nalGarbageCollector (apache#3405) ### What changes were proposed in this pull request? support clean user, group and role entity in RelationalGarbageCollector ### Why are the changes needed? Fix: apache#3188 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? ut
diqiu50 · Jun 13, 2024 · ee5d499 · ee5d499
1 parent d0db37e
commit ee5d499
Show file tree

Hide file tree

Showing 14 changed files with 785 additions and 9 deletions.
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/JDBCBackend.java
@@ -224,12 +224,20 @@ public int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeLin
         return TopicMetaService.getInstance()
             .deleteTopicMetasByLegacyTimeLine(
                 legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
-
-      case COLUMN:
       case USER:
+        return UserMetaService.getInstance()
+            .deleteUserMetasByLegacyTimeLine(
+                legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
       case GROUP:
-      case AUDIT:
+        return GroupMetaService.getInstance()
+            .deleteGroupMetasByLegacyTimeLine(
+                legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
       case ROLE:
+        return RoleMetaService.getInstance()
+            .deleteRoleMetasByLegacyTimeLine(
+                legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
+      case COLUMN:
+      case AUDIT:
         return 0;
         // TODO: Implement hard delete logic for these entity types.
 

diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupMetaMapper.java
@@ -7,6 +7,7 @@
 
 import com.datastrato.gravitino.storage.relational.po.GroupPO;
 import java.util.List;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Select;
@@ -132,4 +133,11 @@ Integer updateGroupMeta(
           + " WHERE re.role_id = #{roleId}"
           + " AND gr.deleted_at = 0 AND re.deleted_at = 0")
   List<GroupPO> listGroupsByRoleId(@Param("roleId") Long roleId);
+
+  @Delete(
+      "DELETE FROM "
+          + GROUP_TABLE_NAME
+          + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
+  Integer deleteGroupMetasByLegacyTimeLine(
+      @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
 }
diff --git a/.../src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java b/.../src/main/java/com/datastrato/gravitino/storage/relational/mapper/GroupRoleRelMapper.java
@@ -7,6 +7,7 @@
 
 import com.datastrato.gravitino.storage.relational.po.GroupRoleRelPO;
 import java.util.List;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Update;
@@ -109,4 +110,11 @@ void softDeleteGroupRoleRelByGroupAndRoles(
           + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
           + " WHERE role_id = #{roleId} AND deleted_at = 0")
   void softDeleteGroupRoleRelByRoleId(Long roleId);
+
+  @Delete(
+      "DELETE FROM "
+          + GROUP_ROLE_RELATION_TABLE_NAME
+          + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
+  Integer deleteGroupRoleRelMetasByLegacyTimeLine(
+      @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/RoleMetaMapper.java
@@ -7,6 +7,7 @@
 
 import com.datastrato.gravitino.storage.relational.po.RolePO;
 import java.util.List;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Select;
@@ -161,4 +162,11 @@ Long selectRoleIdByMetalakeIdAndName(
           + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
           + " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
   void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId);
+
+  @Delete(
+      "DELETE FROM "
+          + ROLE_TABLE_NAME
+          + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
+  Integer deleteRoleMetasByLegacyTimeLine(
+      @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserMetaMapper.java
@@ -7,6 +7,7 @@
 
 import com.datastrato.gravitino.storage.relational.po.UserPO;
 import java.util.List;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Select;
@@ -132,4 +133,11 @@ Integer updateUserMeta(
           + " WHERE re.role_id = #{roleId}"
           + " AND us.deleted_at = 0 AND re.deleted_at = 0")
   List<UserPO> listUsersByRoleId(@Param("roleId") Long roleId);
+
+  @Delete(
+      "DELETE FROM "
+          + USER_TABLE_NAME
+          + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
+  Integer deleteUserMetasByLegacyTimeLine(
+      @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/mapper/UserRoleRelMapper.java
@@ -7,6 +7,7 @@
 
 import com.datastrato.gravitino.storage.relational.po.UserRoleRelPO;
 import java.util.List;
+import org.apache.ibatis.annotations.Delete;
 import org.apache.ibatis.annotations.Insert;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Update;
@@ -109,4 +110,11 @@ void softDeleteUserRoleRelByUserAndRoles(
           + " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
           + " WHERE role_id = #{roleId} AND deleted_at = 0")
   void softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId);
+
+  @Delete(
+      "DELETE FROM "
+          + USER_ROLE_RELATION_TABLE_NAME
+          + " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
+  Integer deleteUserRoleRelMetasByLegacyTimeLine(
+      @Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/GroupMetaService.java
@@ -210,4 +210,24 @@ public <E extends Entity & HasIdentifier> GroupEntity updateGroup(
     }
     return newEntity;
   }
+
+  public int deleteGroupMetasByLegacyTimeLine(long legacyTimeLine, int limit) {
+    int[] groupDeletedCount = new int[] {0};
+    int[] groupRoleRelDeletedCount = new int[] {0};
+
+    SessionUtils.doMultipleWithCommit(
+        () ->
+            groupDeletedCount[0] =
+                SessionUtils.doWithoutCommitAndFetchResult(
+                    GroupMetaMapper.class,
+                    mapper -> mapper.deleteGroupMetasByLegacyTimeLine(legacyTimeLine, limit)),
+        () ->
+            groupRoleRelDeletedCount[0] =
+                SessionUtils.doWithoutCommitAndFetchResult(
+                    GroupRoleRelMapper.class,
+                    mapper ->
+                        mapper.deleteGroupRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)));
+
+    return groupDeletedCount[0] + groupRoleRelDeletedCount[0];
+  }
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/RoleMetaService.java
@@ -123,4 +123,30 @@ public boolean deleteRole(NameIdentifier identifier) {
                 GroupRoleRelMapper.class, mapper -> mapper.softDeleteGroupRoleRelByRoleId(roleId)));
     return true;
   }
+
+  public int deleteRoleMetasByLegacyTimeLine(long legacyTimeLine, int limit) {
+    int[] roleDeletedCount = new int[] {0};
+    int[] userRoleRelDeletedCount = new int[] {0};
+    int[] groupRoleRelDeletedCount = new int[] {0};
+
+    SessionUtils.doMultipleWithCommit(
+        () ->
+            roleDeletedCount[0] =
+                SessionUtils.doWithoutCommitAndFetchResult(
+                    RoleMetaMapper.class,
+                    mapper -> mapper.deleteRoleMetasByLegacyTimeLine(legacyTimeLine, limit)),
+        () ->
+            userRoleRelDeletedCount[0] =
+                SessionUtils.doWithoutCommitAndFetchResult(
+                    UserRoleRelMapper.class,
+                    mapper -> mapper.deleteUserRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)),
+        () ->
+            groupRoleRelDeletedCount[0] =
+                SessionUtils.doWithoutCommitAndFetchResult(
+                    GroupRoleRelMapper.class,
+                    mapper ->
+                        mapper.deleteGroupRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)));
+
+    return roleDeletedCount[0] + userRoleRelDeletedCount[0] + groupRoleRelDeletedCount[0];
+  }
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/service/UserMetaService.java
@@ -207,4 +207,24 @@ public <E extends Entity & HasIdentifier> UserEntity updateUser(
     }
     return newEntity;
   }
+
+  public int deleteUserMetasByLegacyTimeLine(long legacyTimeLine, int limit) {
+    int[] userDeletedCount = new int[] {0};
+    int[] userRoleRelDeletedCount = new int[] {0};
+
+    SessionUtils.doMultipleWithCommit(
+        () ->
+            userDeletedCount[0] =
+                SessionUtils.doWithoutCommitAndFetchResult(
+                    UserMetaMapper.class,
+                    mapper -> mapper.deleteUserMetasByLegacyTimeLine(legacyTimeLine, limit)),
+        () ->
+            userRoleRelDeletedCount[0] =
+                SessionUtils.doWithoutCommitAndFetchResult(
+                    UserRoleRelMapper.class,
+                    mapper ->
+                        mapper.deleteUserRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)));
+
+    return userDeletedCount[0] + userRoleRelDeletedCount[0];
+  }
 }
diff --git a/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java b/core/src/main/java/com/datastrato/gravitino/storage/relational/utils/SessionUtils.java
@@ -63,6 +63,21 @@ public static <T, R> R doWithCommitAndFetchResult(Class<T> mapperClazz, Function
     }
   }
 
+  /**
+   * This method is used to perform a database operation without a commit and fetch the result. If
+   * the operation fails, will throw the RuntimeException.
+   *
+   * @param mapperClazz mapper class to be used for the operation
+   * @param func the operation to be performed with the mapper
+   * @return the result of the operation
+   * @param <T> the type of the mapper
+   * @param <R> the type of the result
+   */
+  public static <T, R> R doWithoutCommitAndFetchResult(Class<T> mapperClazz, Function<T, R> func) {
+    T mapper = SqlSessions.getMapper(mapperClazz);
+    return func.apply(mapper);
+  }
+
   /**
    * This method is used to perform a database operation without a commit. If the operation fails,
    * will throw the RuntimeException.
@@ -72,12 +87,8 @@ public static <T, R> R doWithCommitAndFetchResult(Class<T> mapperClazz, Function
    * @param <T> the type of the mapper
    */
   public static <T> void doWithoutCommit(Class<T> mapperClazz, Consumer<T> consumer) {
-    try {
-      T mapper = SqlSessions.getMapper(mapperClazz);
-      consumer.accept(mapper);
-    } catch (Throwable t) {
-      throw t;
-    }
+    T mapper = SqlSessions.getMapper(mapperClazz);
+    consumer.accept(mapper);
   }
 
   /**