Skip to content

HBASE-22673 Avoid to expose protobuf stuff in Hbck interface #372

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 19, 2019
Merged

HBASE-22673 Avoid to expose protobuf stuff in Hbck interface #372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 2 additions & 16 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseHbck.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.concurrent.Callable;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.yetus.audience.InterfaceAudience;
Expand All @@ -33,7 +34,6 @@

import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetTableStateResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.HbckService.BlockingInterface;
Expand Down Expand Up @@ -156,7 +156,7 @@ public MasterProtos.BypassProcedureResponse call() throws Exception {
}

@Override
public List<Long> scheduleServerCrashProcedure(List<HBaseProtos.ServerName> serverNames)
public List<Long> scheduleServerCrashProcedures(List<ServerName> serverNames)
throws IOException {
try {
MasterProtos.ScheduleServerCrashProcedureResponse response =
Expand All @@ -171,18 +171,4 @@ public List<Long> scheduleServerCrashProcedure(List<HBaseProtos.ServerName> serv
throw new IOException(se);
}
}

@Override
public Map<String, MasterProtos.RegionErrorType>
getFailedSplitMergeLegacyRegions(List<TableName> tableNames) throws IOException {
try {
MasterProtos.GetFailedSplitMergeLegacyRegionsResponse response =
this.hbck.getFailedSplitMergeLegacyRegions(rpcControllerFactory.newController(),
RequestConverter.toGetFailedSplitMergeLegacyRegionsRequest(tableNames));
return response.getErrorsMap();
} catch (ServiceException se) {
LOG.debug("get failed split/merge legacy regions failed", se);
throw new IOException(se);
}
}
}
26 changes: 14 additions & 12 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/client/Hbck.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,15 @@
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.yetus.audience.InterfaceAudience;

import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos;

Expand Down Expand Up @@ -105,18 +109,16 @@ default List<Long> unassigns(List<String> encodedRegionNames) throws IOException
List<Boolean> bypassProcedure(List<Long> pids, long waitTime, boolean override, boolean recursive)
throws IOException;

List<Long> scheduleServerCrashProcedure(List<HBaseProtos.ServerName> serverNames)
throws IOException;

/**
* This method is to get the regions which left by failed split/merge procedures for a certain
* table. There are two kinds of region this method will return. One is orphan regions left on FS,
* which left because split/merge procedure crashed before updating meta. And the other one is
* unassigned split daughter region or merged region, which left because split/merge procedure
* crashed before assignment.
* @param tableName table to check
* @return Map of problematic regions
* Use {@link #scheduleServerCrashProcedures(List)} instead.
* @deprecated since 2.2.1. Will removed in 3.0.0.
*/
Map<String, MasterProtos.RegionErrorType>
getFailedSplitMergeLegacyRegions(List<TableName> tableName) throws IOException;
@Deprecated
default List<Long> scheduleServerCrashProcedure(List<HBaseProtos.ServerName> serverNames)
throws IOException {
return scheduleServerCrashProcedures(
serverNames.stream().map(ProtobufUtil::toServerName).collect(Collectors.toList()));
}

List<Long> scheduleServerCrashProcedures(List<ServerName> serverNames) throws IOException;
}
16 changes: 4 additions & 12 deletions hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1887,19 +1887,11 @@ public static MasterProtos.UnassignsRequest toUnassignRegionsRequest(
}

public static MasterProtos.ScheduleServerCrashProcedureRequest
toScheduleServerCrashProcedureRequest(List<HBaseProtos.ServerName> serverNames) {
MasterProtos.ScheduleServerCrashProcedureRequest.Builder b =
toScheduleServerCrashProcedureRequest(List<ServerName> serverNames) {
MasterProtos.ScheduleServerCrashProcedureRequest.Builder builder =
MasterProtos.ScheduleServerCrashProcedureRequest.newBuilder();
return b.addAllServerName(serverNames).build();
}

public static MasterProtos.GetFailedSplitMergeLegacyRegionsRequest
toGetFailedSplitMergeLegacyRegionsRequest(List<TableName> tableNames) {
MasterProtos.GetFailedSplitMergeLegacyRegionsRequest.Builder b =
MasterProtos.GetFailedSplitMergeLegacyRegionsRequest.newBuilder();
List<HBaseProtos.TableName> protoTableNames = tableNames.stream()
.map(tableName -> ProtobufUtil.toProtoTableName(tableName)).collect(Collectors.toList());
return b.addAllTable(protoTableNames).build();
serverNames.stream().map(ProtobufUtil::toServerName).forEach(sn -> builder.addServerName(sn));
return builder.build();
}

private static List<RegionSpecifier> toEncodedRegionNameRegionSpecifiers(
Expand Down
16 changes: 0 additions & 16 deletions hbase-protocol-shaded/src/main/protobuf/Master.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1108,19 +1108,6 @@ message ScheduleServerCrashProcedureResponse {
repeated uint64 pid = 1;
}

message GetFailedSplitMergeLegacyRegionsRequest {
repeated TableName table = 1;
}

enum RegionErrorType {
DAUGHTER_MERGED_REGION_NOT_ONLINE = 0;
ORPHAN_REGION_ON_FS = 1;
}

message GetFailedSplitMergeLegacyRegionsResponse {
map<string, RegionErrorType> errors = 1;
}

service HbckService {
/** Update state of the table in meta only*/
rpc SetTableStateInMeta(SetTableStateInMetaRequest)
Expand Down Expand Up @@ -1151,7 +1138,4 @@ service HbckService {
/** Schedule a ServerCrashProcedure to help recover a crash server */
rpc ScheduleServerCrashProcedure(ScheduleServerCrashProcedureRequest)
returns(ScheduleServerCrashProcedureResponse);

rpc getFailedSplitMergeLegacyRegions(GetFailedSplitMergeLegacyRegionsRequest)
returns(GetFailedSplitMergeLegacyRegionsResponse);
}
157 changes: 0 additions & 157 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2528,163 +2528,6 @@ public MasterProtos.ScheduleServerCrashProcedureResponse scheduleServerCrashProc
}
}

@Override
public MasterProtos.GetFailedSplitMergeLegacyRegionsResponse getFailedSplitMergeLegacyRegions(
RpcController controller, MasterProtos.GetFailedSplitMergeLegacyRegionsRequest request)
throws ServiceException {
List<HBaseProtos.TableName> tables = request.getTableList();

Map<String, MasterProtos.RegionErrorType> errorRegions = new HashMap<>();
try {
for (HBaseProtos.TableName tableName : tables) {
errorRegions.putAll(getFailedSplitMergeLegacyRegions(ProtobufUtil.toTableName(tableName)));
}
} catch (IOException e) {
throw new ServiceException(e);
}
return MasterProtos.GetFailedSplitMergeLegacyRegionsResponse.newBuilder()
.putAllErrors(errorRegions).build();
}

private Map<String, MasterProtos.RegionErrorType>
getFailedSplitMergeLegacyRegions(TableName tableName) throws IOException {
if (!MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
throw new IOException("table " + tableName.getNameAsString() + " doesn't exist");
}
if (!MetaTableAccessor.getTableState(master.getConnection(), tableName).isEnabled()) {
throw new IOException(
"table " + tableName.getNameAsString() + " is not enabled yet");
}
final Map<String, MasterProtos.RegionErrorType> problemRegions = new HashMap<>();

// Case 1. find orphan region on fs
// orphan regions may due to a failed split region procedure, which daughter regions are created
// then the procedure is aborted. Or merged region is created then the procedure is aborted.
List<String> orphanRegions = findOrphanRegionOnFS(tableName);
orphanRegions.stream().forEach(
region -> problemRegions.put(region, MasterProtos.RegionErrorType.ORPHAN_REGION_ON_FS));

// Case 2. find unassigned daughter regions or merged regions
List<String> unassignedDaughterOrMergedRegions =
findUnassignedDaughterOrMergedRegions(tableName);
unassignedDaughterOrMergedRegions.stream().forEach(region -> problemRegions.put(region,
MasterProtos.RegionErrorType.DAUGHTER_MERGED_REGION_NOT_ONLINE));

// if these regions in problemRegions are currently handled by SplitTableRegionProcedure or
// MergeTableRegionsProcedure, we should remove them from this map
master.getProcedures().stream().filter(p -> !(p.isFinished() || p.isBypass())).forEach(p -> {
if (p instanceof SplitTableRegionProcedure) {
problemRegions
.remove(((SplitTableRegionProcedure) p).getDaughterOneRI().getRegionNameAsString());
problemRegions
.remove(((SplitTableRegionProcedure) p).getDaughterTwoRI().getRegionNameAsString());
} else if (p instanceof MergeTableRegionsProcedure) {
problemRegions
.remove(((MergeTableRegionsProcedure) p).getMergedRegion().getRegionNameAsString());
}
});

// check if regions are still problematic now
checkRegionStillProblematic(problemRegions, tableName);
return problemRegions;
}


private void checkRegionStillProblematic(
Map<String, MasterProtos.RegionErrorType> problemRegions, TableName tableName)
throws IOException {
Iterator<Map.Entry<String, MasterProtos.RegionErrorType>> iterator =
problemRegions.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, MasterProtos.RegionErrorType> entry = iterator.next();
Result r = MetaTableAccessor.getRegionResult(master.getConnection(),
Bytes.toBytesBinary(entry.getKey()));
switch (entry.getValue()) {
case ORPHAN_REGION_ON_FS:
// region is build for this directory, it is not a problematic region any more
if (r != null) {
iterator.remove();
}
break;
case DAUGHTER_MERGED_REGION_NOT_ONLINE:
RegionState.State state = RegionStateStore.getRegionState(r, 0);
if (!state.matches(RegionState.State.CLOSED, RegionState.State.SPLITTING_NEW,
RegionState.State.MERGED)) {
iterator.remove();
}
break;
default:
throw new IOException("there should be no problematic region of this type");
}
}
}

private List<String> findUnassignedDaughterOrMergedRegions(TableName tableName)
throws IOException {
Set<String> checkRegions = new HashSet<>();
Map<String, RegionState.State> regionStates = new HashMap<>();
Map<String, RegionInfo> regionInfos = new HashMap<>();

MetaTableAccessor.scanMeta(master.getConnection(), tableName,
MetaTableAccessor.QueryType.REGION, Integer.MAX_VALUE, r -> {
RegionInfo regionInfo = MetaTableAccessor.getRegionInfo(r);
regionInfos.put(regionInfo.getRegionNameAsString(), regionInfo);
RegionState.State state = RegionStateStore.getRegionState(r, 0);
regionStates.put(regionInfo.getEncodedName(), state);
if (regionInfo.isSplitParent()) {
PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(r);
checkRegions.add(daughters.getFirst().getRegionNameAsString());
checkRegions.add(daughters.getSecond().getRegionNameAsString());
} else if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null
|| r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER) != null) {
checkRegions.add(regionInfo.getRegionNameAsString());
}
return true;
});

// find unassigned merged or split daughter region
return checkRegions.stream().map(regionName -> regionInfos.get(regionName))
.filter(regionInfo -> !regionInfo.isSplitParent())
.filter(regionInfo -> !regionStates.get(regionInfo.getEncodedName())
.matches(RegionState.State.OPEN))
.map(regionInfo -> regionInfo.getRegionNameAsString()).collect(Collectors.toList());
}

private List<String> findOrphanRegionOnFS(TableName tableName) throws IOException {
// get available regions from meta, merged region should be consider available
HashSet<String> regionsInMeta = new HashSet<>();
MetaTableAccessor.scanMeta(master.getConnection(), tableName,
MetaTableAccessor.QueryType.REGION, Integer.MAX_VALUE, r -> {
RegionInfo regionInfo = MetaTableAccessor.getRegionInfo(r);
regionsInMeta.add(regionInfo.getEncodedName());
if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null
|| r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER) != null) {
PairOfSameType<RegionInfo> mergedRegions = MetaTableAccessor.getMergeRegions(r);
regionsInMeta.add(mergedRegions.getFirst().getEncodedName());
regionsInMeta.add(mergedRegions.getSecond().getEncodedName());
}
return true;
});
// get regionInfo from fs
Path tableDir = FSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
FileStatus[] regions =
master.getFileSystem().listStatus(tableDir, path -> !path.getName().startsWith("."));
HashMap<String, String> regionNames = new HashMap<>();
for (FileStatus region : regions) {
RegionInfo regionInfo =
HRegionFileSystem.loadRegionInfoFileContent(master.getFileSystem(), region.getPath());
regionNames.put(regionInfo.getEncodedName(), regionInfo.getRegionNameAsString());
}
Iterator<Map.Entry<String, String>> regionIterator = regionNames.entrySet().iterator();
while (regionIterator.hasNext()) {
Map.Entry<String, String> region = regionIterator.next();
if (regionsInMeta.contains(region.getKey())) {
regionIterator.remove();
}
}
return new ArrayList<>(regionNames.values());
}

@Override
public SwitchRpcThrottleResponse switchRpcThrottle(RpcController controller,
SwitchRpcThrottleRequest request) throws ServiceException {
Expand Down
Loading