Skip to content

Commit

Permalink
Refactor issue 4228 (alibaba#4229)
Browse files Browse the repository at this point in the history
* refactor: adjusts conformance protocol layer objects

* feat: repair problems

* refactor: reafctor consistency model
  • Loading branch information
chuntaojun authored Nov 16, 2020
1 parent c183b77 commit bcd62d5
Show file tree
Hide file tree
Showing 34 changed files with 411 additions and 3,301 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@
import com.alibaba.nacos.consistency.SerializeFactory;
import com.alibaba.nacos.consistency.Serializer;
import com.alibaba.nacos.consistency.cp.CPProtocol;
import com.alibaba.nacos.consistency.cp.LogProcessor4CP;
import com.alibaba.nacos.consistency.entity.GetRequest;
import com.alibaba.nacos.consistency.entity.Log;
import com.alibaba.nacos.consistency.cp.RequestProcessor4CP;
import com.alibaba.nacos.consistency.entity.ReadRequest;
import com.alibaba.nacos.consistency.entity.Response;
import com.alibaba.nacos.consistency.entity.WriteRequest;
import com.alibaba.nacos.consistency.exception.ConsistencyException;
import com.alibaba.nacos.consistency.snapshot.SnapshotOperation;
import com.alibaba.nacos.core.cluster.ServerMemberManager;
Expand Down Expand Up @@ -142,7 +142,7 @@
@Conditional(ConditionDistributedEmbedStorage.class)
@Component
@SuppressWarnings({"unchecked"})
public class DistributedDatabaseOperateImpl extends LogProcessor4CP implements BaseDatabaseOperate {
public class DistributedDatabaseOperateImpl extends RequestProcessor4CP implements BaseDatabaseOperate {

/**
* The data import operation is dedicated key, which ACTS as an identifier.
Expand Down Expand Up @@ -227,7 +227,7 @@ public <R> R queryOne(String sql, Class<R> cls) {
.containsExtendInfo(Constants.EXTEND_NEED_READ_UNTIL_HAVE_DATA);

Response response = innerRead(
GetRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
ReadRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
if (response.getSuccess()) {
return serializer.deserialize(response.getData().toByteArray(), cls);
}
Expand All @@ -251,7 +251,7 @@ public <R> R queryOne(String sql, Object[] args, Class<R> cls) {
.containsExtendInfo(Constants.EXTEND_NEED_READ_UNTIL_HAVE_DATA);

Response response = innerRead(
GetRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
ReadRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
if (response.getSuccess()) {
return serializer.deserialize(response.getData().toByteArray(), cls);
}
Expand All @@ -275,7 +275,7 @@ public <R> R queryOne(String sql, Object[] args, RowMapper<R> mapper) {
.containsExtendInfo(Constants.EXTEND_NEED_READ_UNTIL_HAVE_DATA);

Response response = innerRead(
GetRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
ReadRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
if (response.getSuccess()) {
return serializer.deserialize(response.getData().toByteArray(),
ClassUtils.resolveGenericTypeByInterface(mapper.getClass()));
Expand All @@ -300,7 +300,7 @@ public <R> List<R> queryMany(String sql, Object[] args, RowMapper<R> mapper) {
.containsExtendInfo(Constants.EXTEND_NEED_READ_UNTIL_HAVE_DATA);

Response response = innerRead(
GetRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
ReadRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
if (response.getSuccess()) {
return serializer.deserialize(response.getData().toByteArray(), List.class);
}
Expand All @@ -324,7 +324,7 @@ public <R> List<R> queryMany(String sql, Object[] args, Class<R> rClass) {
.containsExtendInfo(Constants.EXTEND_NEED_READ_UNTIL_HAVE_DATA);

Response response = innerRead(
GetRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
ReadRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
if (response.getSuccess()) {
return serializer.deserialize(response.getData().toByteArray(), List.class);
}
Expand All @@ -348,7 +348,7 @@ public List<Map<String, Object>> queryMany(String sql, Object[] args) {
.containsExtendInfo(Constants.EXTEND_NEED_READ_UNTIL_HAVE_DATA);

Response response = innerRead(
GetRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
ReadRequest.newBuilder().setGroup(group()).setData(ByteString.copyFrom(data)).build(), blockRead);
if (response.getSuccess()) {
return serializer.deserialize(response.getData().toByteArray(), List.class);
}
Expand All @@ -362,12 +362,12 @@ public List<Map<String, Object>> queryMany(String sql, Object[] args) {
/**
* In some business situations, you need to avoid the timeout issue, so blockRead is used to determine this.
*
* @param request {@link GetRequest}
* @param request {@link ReadRequest}
* @param blockRead is async read operation
* @return {@link Response}
* @throws Exception Exception
*/
private Response innerRead(GetRequest request, boolean blockRead) throws Exception {
private Response innerRead(ReadRequest request, boolean blockRead) throws Exception {
if (blockRead) {
return (Response) protocol.aGetData(request).join();
}
Expand All @@ -390,7 +390,7 @@ public CompletableFuture<RestResult<String>> dataImport(File file) {
if (submit) {
List<ModifyRequest> requests = batchUpdate.stream().map(ModifyRequest::new)
.collect(Collectors.toList());
CompletableFuture<Response> future = protocol.submitAsync(Log.newBuilder().setGroup(group())
CompletableFuture<Response> future = protocol.submitAsync(WriteRequest.newBuilder().setGroup(group())
.setData(ByteString.copyFrom(serializer.serialize(requests)))
.putExtendInfo(DATA_IMPORT_KEY, Boolean.TRUE.toString()).build());
futures.add(future);
Expand Down Expand Up @@ -427,19 +427,19 @@ public Boolean update(List<ModifyRequest> sqlContext, BiConsumer<Boolean, Throwa
final String key =
System.currentTimeMillis() + "-" + group() + "-" + memberManager.getSelf().getAddress() + "-"
+ MD5Utils.md5Hex(sqlContext.toString(), Constants.ENCODE);
Log log = Log.newBuilder().setGroup(group()).setKey(key)
WriteRequest request = WriteRequest.newBuilder().setGroup(group()).setKey(key)
.setData(ByteString.copyFrom(serializer.serialize(sqlContext)))
.putAllExtendInfo(EmbeddedStorageContextUtils.getCurrentExtendInfo())
.setType(sqlContext.getClass().getCanonicalName()).build();
if (Objects.isNull(consumer)) {
Response response = this.protocol.submit(log);
Response response = this.protocol.submit(request);
if (response.getSuccess()) {
return true;
}
LogUtil.DEFAULT_LOG.error("execute sql modify operation failed : {}", response.getErrMsg());
return false;
} else {
this.protocol.submitAsync(log).whenComplete((BiConsumer<Response, Throwable>) (response, ex) -> {
this.protocol.submitAsync(request).whenComplete((BiConsumer<Response, Throwable>) (response, ex) -> {
String errMsg = Objects.isNull(ex) ? response.getErrMsg() : ExceptionUtil.getCause(ex).getMessage();
consumer.accept(response.getSuccess(),
StringUtils.isBlank(errMsg) ? null : new NJdbcException(errMsg));
Expand All @@ -462,7 +462,7 @@ public List<SnapshotOperation> loadSnapshotOperate() {

@SuppressWarnings("all")
@Override
public Response onRequest(final GetRequest request) {
public Response onRequest(final ReadRequest request) {
final SelectRequest selectRequest = serializer
.deserialize(request.getData().toByteArray(), SelectRequest.class);

Expand Down Expand Up @@ -511,7 +511,7 @@ public Response onRequest(final GetRequest request) {
}

@Override
public Response onApply(Log log) {
public Response onApply(WriteRequest log) {
LoggerUtils.printIfDebugEnabled(LogUtil.DEFAULT_LOG, "onApply info : log : {}", log);
final ByteString byteString = log.getData();
Preconditions.checkArgument(byteString != null, "Log.getData() must not null");
Expand Down
52 changes: 26 additions & 26 deletions consistency/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,32 +30,32 @@
</extensions>
<plugins>
<!-- Grpc coding plug-in-->
<!-- <plugin>-->
<!-- <groupId>org.xolstice.maven.plugins</groupId>-->
<!-- <artifactId>protobuf-maven-plugin</artifactId>-->
<!-- <version>0.5.0</version>-->
<!-- <configuration>-->
<!-- <protocArtifact>com.google.protobuf:protoc:${protobuf-java.version}:exe:${os.detected.classifier}</protocArtifact>-->
<!-- <pluginId>grpc-java</pluginId>-->
<!-- <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc-java.version}:exe:${os.detected.classifier}</pluginArtifact>-->
<!-- </configuration>-->
<!-- <executions>-->
<!-- <execution>-->
<!-- <goals>-->
<!-- <goal>compile</goal>-->
<!-- <goal>compile-custom</goal>-->
<!-- </goals>-->
<!-- </execution>-->
<!-- </executions>-->
<!-- </plugin>-->
<!-- <plugin>-->
<!-- <groupId>org.apache.maven.plugins</groupId>-->
<!-- <artifactId>maven-compiler-plugin</artifactId>-->
<!-- <configuration>-->
<!-- <source>8</source>-->
<!-- <target>8</target>-->
<!-- </configuration>-->
<!-- </plugin>-->
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<version>0.5.0</version>
<configuration>
<protocArtifact>com.google.protobuf:protoc:${protobuf-java.version}:exe:${os.detected.classifier}</protocArtifact>
<pluginId>grpc-java</pluginId>
<pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc-java.version}:exe:${os.detected.classifier}</pluginArtifact>
</configuration>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>compile-custom</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
</plugins>
</build>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@
/**
* Consistent protocol related configuration objects.
*
* <p>{@link LogProcessor} : The consistency protocol provides services for all businesses, but each business only cares
* <p>{@link RequestProcessor} : The consistency protocol provides services for all businesses, but each business only cares
* about the transaction information belonging to that business, and the transaction processing between the various
* services should not block each other. Therefore, the LogProcessor is abstracted to implement the parallel processing
* of transactions of different services. Corresponding LogProcessor sub-interface: LogProcessor4AP or LogProcessor4CP,
* different consistency protocols will actively discover the corresponding LogProcessor
*
* @author <a href="mailto:liaochuntao@live.com">liaochuntao</a>
*/
public interface Config<L extends LogProcessor> extends Serializable {
public interface Config<L extends RequestProcessor> extends Serializable {

/**
* Set the cluster node information to initialize,like [ip:port, ip:port, ip:port].
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@

package com.alibaba.nacos.consistency;

import com.alibaba.nacos.consistency.entity.GetRequest;
import com.alibaba.nacos.consistency.entity.Log;
import com.alibaba.nacos.consistency.entity.ReadRequest;
import com.alibaba.nacos.consistency.entity.Response;
import com.alibaba.nacos.consistency.entity.WriteRequest;

import java.util.Collection;
import java.util.Set;
Expand All @@ -37,7 +37,7 @@
*
* @author <a href="mailto:liaochuntao@live.com">liaochuntao</a>
*/
public interface ConsistencyProtocol<T extends Config, P extends LogProcessor> extends CommandOperations {
public interface ConsistencyProtocol<T extends Config, P extends RequestProcessor> extends CommandOperations {

/**
* Consistency protocol initialization: perform initialization operations based on the incoming.
Expand All @@ -50,7 +50,7 @@ public interface ConsistencyProtocol<T extends Config, P extends LogProcessor> e
/**
* Add a log handler.
*
* @param processors {@link LogProcessor}
* @param processors {@link RequestProcessor}
*/
void addLogProcessors(Collection<P> processors);

Expand All @@ -69,35 +69,35 @@ public interface ConsistencyProtocol<T extends Config, P extends LogProcessor> e
* @return data {@link Response}
* @throws Exception {@link Exception}
*/
Response getData(GetRequest request) throws Exception;
Response getData(ReadRequest request) throws Exception;

/**
* Get data asynchronously.
*
* @param request request
* @return data {@link CompletableFuture}
*/
CompletableFuture<Response> aGetData(GetRequest request);
CompletableFuture<Response> aGetData(ReadRequest request);

/**
* Data operation, returning submission results synchronously.
* 同步数据提交,在 Datum 中已携带相应的数据操作信息
*
* @param data {@link Log}
* @param request {@link com.alibaba.nacos.consistency.entity.WriteRequest}
* @return submit operation result {@link Response}
* @throws Exception {@link Exception}
*/
Response submit(Log data) throws Exception;
Response submit(WriteRequest request) throws Exception;

/**
* Data submission operation, returning submission results asynchronously.
* 异步数据提交,在 Datum中已携带相应的数据操作信息,返回一个Future,自行操作,提交发生的异常会在CompleteFuture中
*
* @param data {@link Log}
* @param request {@link com.alibaba.nacos.consistency.entity.WriteRequest}
* @return {@link CompletableFuture} submit result
* @throws Exception when submit throw Exception
*/
CompletableFuture<Response> submitAsync(Log data);
CompletableFuture<Response> submitAsync(WriteRequest request);

/**
* New member list .
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright 1999-2018 Alibaba Group Holding Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.alibaba.nacos.consistency;

import com.alibaba.nacos.consistency.entity.GetRequest;
import com.alibaba.nacos.consistency.entity.Log;
import com.alibaba.nacos.consistency.entity.ReadRequest;
import com.alibaba.nacos.consistency.entity.WriteRequest;
import com.alibaba.nacos.consistency.exception.ConsistencyException;
import com.google.protobuf.Message;

/**
* protobuf message utils.
*
* @author <a href="mailto:liaochuntao@live.com">liaochuntao</a>
*/
public class ProtoMessageUtil {

/**
* Converts the byte array to a specific Protobuf object.
* Internally, the protobuf new and old objects are compatible.
*
* @param bytes An array of bytes
* @return Message
*/
public static Message parse(byte[] bytes) {
Message result;
try {
result = WriteRequest.parseFrom(bytes);
return result;
} catch (Throwable ignore) {
}
try {
result = ReadRequest.parseFrom(bytes);
return result;
} catch (Throwable ignore) {
}

// old consistency entity, will be @Deprecated in future
try {
Log log = Log.parseFrom(bytes);
return convertToWriteRequest(log);
} catch (Throwable ignore) {
}

try {
GetRequest request = GetRequest.parseFrom(bytes);
return convertToReadRequest(request);
} catch (Throwable ignore) {
}

throw new ConsistencyException("The current array cannot be serialized to the corresponding object");
}

/**
* convert Log to WriteRequest.
*
* @param log log
* @return {@link WriteRequest}
*/
public static WriteRequest convertToWriteRequest(Log log) {
return WriteRequest.newBuilder().setKey(log.getKey()).setGroup(log.getGroup())
.setData(log.getData())
.setType(log.getType())
.setOperation(log.getOperation())
.putAllExtendInfo(log.getExtendInfoMap())
.build();
}

/**
* convert Log to ReadRequest.
*
* @param request request
* @return {@link ReadRequest}
*/
public static ReadRequest convertToReadRequest(GetRequest request) {
return ReadRequest.newBuilder()
.setGroup(request.getGroup())
.setData(request.getData())
.putAllExtendInfo(request.getExtendInfoMap())
.build();
}
}
Loading

0 comments on commit bcd62d5

Please sign in to comment.