Skip to content

Commit

Permalink
[#3188] feat(core): Clean user,group and role entity in RelationalGar…
Browse files Browse the repository at this point in the history
…bageCollector (#3504)

### What changes were proposed in this pull request?

support clean user, group and role entity in RelationalGarbageCollector

### Why are the changes needed?

Fix: #3188 

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

ut

Co-authored-by: lwyang <yangliwei@xiaomi.com>
  • Loading branch information
jerryshao and lw-yang authored May 22, 2024
1 parent 121f2fe commit 13cc345
Show file tree
Hide file tree
Showing 14 changed files with 785 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -224,12 +224,20 @@ public int hardDeleteLegacyData(Entity.EntityType entityType, long legacyTimeLin
return TopicMetaService.getInstance()
.deleteTopicMetasByLegacyTimeLine(
legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);

case COLUMN:
case USER:
return UserMetaService.getInstance()
.deleteUserMetasByLegacyTimeLine(
legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
case GROUP:
case AUDIT:
return GroupMetaService.getInstance()
.deleteGroupMetasByLegacyTimeLine(
legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
case ROLE:
return RoleMetaService.getInstance()
.deleteRoleMetasByLegacyTimeLine(
legacyTimeLine, GARBAGE_COLLECTOR_SINGLE_DELETION_LIMIT);
case COLUMN:
case AUDIT:
return 0;
// TODO: Implement hard delete logic for these entity types.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import com.datastrato.gravitino.storage.relational.po.GroupPO;
import java.util.List;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
Expand Down Expand Up @@ -132,4 +133,11 @@ Integer updateGroupMeta(
+ " WHERE re.role_id = #{roleId}"
+ " AND gr.deleted_at = 0 AND re.deleted_at = 0")
List<GroupPO> listGroupsByRoleId(@Param("roleId") Long roleId);

@Delete(
"DELETE FROM "
+ GROUP_TABLE_NAME
+ " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
Integer deleteGroupMetasByLegacyTimeLine(
@Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import com.datastrato.gravitino.storage.relational.po.GroupRoleRelPO;
import java.util.List;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
Expand Down Expand Up @@ -109,4 +110,11 @@ void softDeleteGroupRoleRelByGroupAndRoles(
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " WHERE role_id = #{roleId} AND deleted_at = 0")
void softDeleteGroupRoleRelByRoleId(Long roleId);

@Delete(
"DELETE FROM "
+ GROUP_ROLE_RELATION_TABLE_NAME
+ " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
Integer deleteGroupRoleRelMetasByLegacyTimeLine(
@Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import com.datastrato.gravitino.storage.relational.po.RolePO;
import java.util.List;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
Expand Down Expand Up @@ -161,4 +162,11 @@ Long selectRoleIdByMetalakeIdAndName(
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " WHERE metalake_id = #{metalakeId} AND deleted_at = 0")
void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId);

@Delete(
"DELETE FROM "
+ ROLE_TABLE_NAME
+ " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
Integer deleteRoleMetasByLegacyTimeLine(
@Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import com.datastrato.gravitino.storage.relational.po.UserPO;
import java.util.List;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
Expand Down Expand Up @@ -132,4 +133,11 @@ Integer updateUserMeta(
+ " WHERE re.role_id = #{roleId}"
+ " AND us.deleted_at = 0 AND re.deleted_at = 0")
List<UserPO> listUsersByRoleId(@Param("roleId") Long roleId);

@Delete(
"DELETE FROM "
+ USER_TABLE_NAME
+ " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
Integer deleteUserMetasByLegacyTimeLine(
@Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import com.datastrato.gravitino.storage.relational.po.UserRoleRelPO;
import java.util.List;
import org.apache.ibatis.annotations.Delete;
import org.apache.ibatis.annotations.Insert;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
Expand Down Expand Up @@ -109,4 +110,11 @@ void softDeleteUserRoleRelByUserAndRoles(
+ " SET deleted_at = UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000.0"
+ " WHERE role_id = #{roleId} AND deleted_at = 0")
void softDeleteUserRoleRelByRoleId(@Param("roleId") Long roleId);

@Delete(
"DELETE FROM "
+ USER_ROLE_RELATION_TABLE_NAME
+ " WHERE deleted_at > 0 AND deleted_at < #{legacyTimeLine} LIMIT #{limit}")
Integer deleteUserRoleRelMetasByLegacyTimeLine(
@Param("legacyTimeLine") Long legacyTimeLine, @Param("limit") int limit);
}
Original file line number Diff line number Diff line change
Expand Up @@ -210,4 +210,24 @@ public <E extends Entity & HasIdentifier> GroupEntity updateGroup(
}
return newEntity;
}

public int deleteGroupMetasByLegacyTimeLine(long legacyTimeLine, int limit) {
int[] groupDeletedCount = new int[] {0};
int[] groupRoleRelDeletedCount = new int[] {0};

SessionUtils.doMultipleWithCommit(
() ->
groupDeletedCount[0] =
SessionUtils.doWithoutCommitAndFetchResult(
GroupMetaMapper.class,
mapper -> mapper.deleteGroupMetasByLegacyTimeLine(legacyTimeLine, limit)),
() ->
groupRoleRelDeletedCount[0] =
SessionUtils.doWithoutCommitAndFetchResult(
GroupRoleRelMapper.class,
mapper ->
mapper.deleteGroupRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)));

return groupDeletedCount[0] + groupRoleRelDeletedCount[0];
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -123,4 +123,30 @@ public boolean deleteRole(NameIdentifier identifier) {
GroupRoleRelMapper.class, mapper -> mapper.softDeleteGroupRoleRelByRoleId(roleId)));
return true;
}

public int deleteRoleMetasByLegacyTimeLine(long legacyTimeLine, int limit) {
int[] roleDeletedCount = new int[] {0};
int[] userRoleRelDeletedCount = new int[] {0};
int[] groupRoleRelDeletedCount = new int[] {0};

SessionUtils.doMultipleWithCommit(
() ->
roleDeletedCount[0] =
SessionUtils.doWithoutCommitAndFetchResult(
RoleMetaMapper.class,
mapper -> mapper.deleteRoleMetasByLegacyTimeLine(legacyTimeLine, limit)),
() ->
userRoleRelDeletedCount[0] =
SessionUtils.doWithoutCommitAndFetchResult(
UserRoleRelMapper.class,
mapper -> mapper.deleteUserRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)),
() ->
groupRoleRelDeletedCount[0] =
SessionUtils.doWithoutCommitAndFetchResult(
GroupRoleRelMapper.class,
mapper ->
mapper.deleteGroupRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)));

return roleDeletedCount[0] + userRoleRelDeletedCount[0] + groupRoleRelDeletedCount[0];
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -207,4 +207,24 @@ public <E extends Entity & HasIdentifier> UserEntity updateUser(
}
return newEntity;
}

public int deleteUserMetasByLegacyTimeLine(long legacyTimeLine, int limit) {
int[] userDeletedCount = new int[] {0};
int[] userRoleRelDeletedCount = new int[] {0};

SessionUtils.doMultipleWithCommit(
() ->
userDeletedCount[0] =
SessionUtils.doWithoutCommitAndFetchResult(
UserMetaMapper.class,
mapper -> mapper.deleteUserMetasByLegacyTimeLine(legacyTimeLine, limit)),
() ->
userRoleRelDeletedCount[0] =
SessionUtils.doWithoutCommitAndFetchResult(
UserRoleRelMapper.class,
mapper ->
mapper.deleteUserRoleRelMetasByLegacyTimeLine(legacyTimeLine, limit)));

return userDeletedCount[0] + userRoleRelDeletedCount[0];
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,21 @@ public static <T, R> R doWithCommitAndFetchResult(Class<T> mapperClazz, Function
}
}

/**
* This method is used to perform a database operation without a commit and fetch the result. If
* the operation fails, will throw the RuntimeException.
*
* @param mapperClazz mapper class to be used for the operation
* @param func the operation to be performed with the mapper
* @return the result of the operation
* @param <T> the type of the mapper
* @param <R> the type of the result
*/
public static <T, R> R doWithoutCommitAndFetchResult(Class<T> mapperClazz, Function<T, R> func) {
T mapper = SqlSessions.getMapper(mapperClazz);
return func.apply(mapper);
}

/**
* This method is used to perform a database operation without a commit. If the operation fails,
* will throw the RuntimeException.
Expand All @@ -72,12 +87,8 @@ public static <T, R> R doWithCommitAndFetchResult(Class<T> mapperClazz, Function
* @param <T> the type of the mapper
*/
public static <T> void doWithoutCommit(Class<T> mapperClazz, Consumer<T> consumer) {
try {
T mapper = SqlSessions.getMapper(mapperClazz);
consumer.accept(mapper);
} catch (Throwable t) {
throw t;
}
T mapper = SqlSessions.getMapper(mapperClazz);
consumer.accept(mapper);
}

/**
Expand Down
Loading

0 comments on commit 13cc345

Please sign in to comment.