Skip to content

HDFS-15947. Replace deprecated protobuf APIs #2856

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 5, 2021
Merged

HDFS-15947. Replace deprecated protobuf APIs #2856

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL

std::string buf;

int size = msg->ByteSize();
const auto size = msg->ByteSizeLong();
buf.reserve(pbio::CodedOutputStream::VarintSize32(size) + size);
pbio::StringOutputStream ss(&buf);
pbio::CodedOutputStream os(&ss);
Expand All @@ -68,9 +68,9 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL
return buf;
}

int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) {
size_t size = msg->ByteSize();
return ::google::protobuf::io::CodedOutputStream::VarintSize32(size) + size;
size_t DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) {
const auto size = msg->ByteSizeLong();
return ::google::protobuf::io::CodedOutputStream::VarintSize64(size) + size;
}

std::shared_ptr<std::string> GetRandomClientName() {
Expand Down
2 changes: 1 addition & 1 deletion hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ Status ToStatus(const boost::system::error_code &ec);

// Determine size of buffer that needs to be allocated in order to serialize msg
// in delimited format
int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg);
size_t DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg);

// Construct msg from the input held in the CodedInputStream
// return false on failure, otherwise return true
Expand Down
8 changes: 4 additions & 4 deletions hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ static const int kNoRetry = -1;
static void AddHeadersToPacket(std::string *res,
std::initializer_list<const pb::MessageLite *> headers,
const std::string *payload) {
int len = 0;
size_t len = 0;
std::for_each(
headers.begin(), headers.end(),
[&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); });
Expand All @@ -68,7 +68,7 @@ static void AddHeadersToPacket(std::string *res,

std::for_each(
headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) {
buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(v->ByteSizeLong(), buf);
buf = v->SerializeWithCachedSizesToArray(buf);
});

Expand All @@ -78,13 +78,13 @@ static void AddHeadersToPacket(std::string *res,
}

static void ConstructPayload(std::string *res, const pb::MessageLite *header) {
int len = DelimitedPBMessageSize(header);
const auto len = DelimitedPBMessageSize(header);
res->reserve(len);
pbio::StringOutputStream ss(res);
pbio::CodedOutputStream os(&ss);
uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
assert(buf);
buf = pbio::CodedOutputStream::WriteVarint32ToArray(header->ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(header->ByteSizeLong(), buf);
buf = header->SerializeWithCachedSizesToArray(buf);
}

Expand Down
4 changes: 2 additions & 2 deletions ...roject/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ using namespace ::std::placeholders;
static void AddHeadersToPacket(
std::string *res, std::initializer_list<const pb::MessageLite *> headers,
const std::string *payload) {
int len = 0;
size_t len = 0;
std::for_each(
headers.begin(), headers.end(),
[&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); });
Expand All @@ -57,7 +57,7 @@ static void AddHeadersToPacket(

std::for_each(
headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) {
buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(v->ByteSizeLong(), buf);
buf = v->SerializeWithCachedSizesToArray(buf);
});

Expand Down
6 changes: 3 additions & 3 deletions ...ect/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ static inline string ToDelimitedString(const pb::MessageLite *msg) {
res.reserve(hdfs::DelimitedPBMessageSize(msg));
pbio::StringOutputStream os(&res);
pbio::CodedOutputStream out(&os);
out.WriteVarint32(msg->ByteSize());
out.WriteVarint64(msg->ByteSizeLong());
msg->SerializeToCodedStream(&out);
return res;
}
Expand All @@ -141,9 +141,9 @@ static inline std::pair<error_code, string> ProducePacket(
*reinterpret_cast<unsigned *>(prefix) =
htonl(data.size() + checksum.size() + sizeof(int32_t));
*reinterpret_cast<short *>(prefix + sizeof(int32_t)) =
htons(proto.ByteSize());
htons(static_cast<uint16_t>(proto.ByteSizeLong()));
std::string payload(prefix, sizeof(prefix));
payload.reserve(payload.size() + proto.ByteSize() + checksum.size() +
payload.reserve(payload.size() + proto.ByteSizeLong() + checksum.size() +
data.size());
proto.AppendToString(&payload);
payload += checksum;
Expand Down
10 changes: 5 additions & 5 deletions ...hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,9 @@ class SharedConnectionEngine : public RpcEngine {
static inline std::pair<boost::system::error_code, string> RpcResponse(
const RpcResponseHeaderProto &h, const std::string &data,
const boost::system::error_code &ec = boost::system::error_code()) {
uint32_t payload_length =
pbio::CodedOutputStream::VarintSize32(h.ByteSize()) +
pbio::CodedOutputStream::VarintSize32(data.size()) + h.ByteSize() +
const auto payload_length =
pbio::CodedOutputStream::VarintSize64(h.ByteSizeLong()) +
pbio::CodedOutputStream::VarintSize64(data.size()) + h.ByteSizeLong() +
data.size();

std::string res;
Expand All @@ -99,9 +99,9 @@ static inline std::pair<boost::system::error_code, string> RpcResponse(

buf = pbio::CodedOutputStream::WriteLittleEndian32ToArray(
htonl(payload_length), buf);
buf = pbio::CodedOutputStream::WriteVarint32ToArray(h.ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(h.ByteSizeLong(), buf);
buf = h.SerializeWithCachedSizesToArray(buf);
buf = pbio::CodedOutputStream::WriteVarint32ToArray(data.size(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(data.size(), buf);
buf = pbio::CodedOutputStream::WriteStringToArray(data, buf);

return std::make_pair(ec, std::move(res));
Expand Down