Skip to content

Commit

Permalink
Add xls_value proto representation
Browse files Browse the repository at this point in the history 
This adds a representation of xls::Value as a proto to enable reasonably fast serialization and deserialization of values.

PiperOrigin-RevId: 579011778
  • Loading branch information
@allight @copybara-github
allight authored and copybara-github committed Nov 2, 2023
1 parent c341405 commit 276584a
Show file tree
Hide file tree
Showing 5 changed files with 549 additions and 5 deletions.
29 changes: 28 additions & 1 deletion xls/ir/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -813,14 +813,18 @@ cc_library(
srcs = ["value.cc"],
hdrs = ["value.h"],
deps = [
":bit_push_buffer",
":bits",
":bits_ops",
":format_preference",
":xls_type_cc_proto",
":xls_value_cc_proto",
"//xls/common:math_util",
"//xls/common/logging",
"//xls/common/status:ret_check",
"//xls/common/status:status_macros",
"@com_google_absl//absl/algorithm:container",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
Expand Down Expand Up @@ -849,11 +853,16 @@ cc_test(
srcs = ["value_test.cc"],
deps = [
":bits",
":ir",
":ir_parser",
":value",
":xls_value_cc_proto",
"//xls/common:xls_gunit",
"//xls/common:xls_gunit_main",
"//xls/common/status:matchers",
"@com_google_absl//absl/status",
"@com_google_absl//absl/types:span",
"@com_google_fuzztest//fuzztest",
"@com_google_protobuf//:protobuf",
],
)

Expand Down Expand Up @@ -1240,6 +1249,24 @@ cc_test(
],
)

proto_library(
name = "xls_value_proto",
srcs = ["xls_value.proto"],
visibility = ["//xls:xls_users"],
)

cc_proto_library(
name = "xls_value_cc_proto",
visibility = ["//xls:xls_users"],
deps = [":xls_value_proto"],
)

py_proto_library(
name = "xls_value_py_pb2",
visibility = ["//xls:xls_users"],
deps = [":xls_value_proto"],
)

proto_library(
name = "xls_type_proto",
srcs = ["xls_type.proto"],
Expand Down
112 changes: 112 additions & 0 deletions xls/ir/value.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,29 @@

#include "xls/ir/value.h"

#include <cstdint>
#include <cstring>
#include <string>
#include <utility>
#include <variant>
#include <vector>

#include "absl/algorithm/container.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/types/span.h"
#include "xls/common/logging/logging.h"
#include "xls/common/math_util.h"
#include "xls/common/status/ret_check.h"
#include "xls/common/status/status_macros.h"
#include "xls/ir/bit_push_buffer.h"
#include "xls/ir/bits.h"
#include "xls/ir/bits_ops.h"
#include "xls/ir/format_preference.h"
#include "xls/ir/xls_value.pb.h"

namespace xls {

Expand Down Expand Up @@ -312,6 +322,108 @@ bool Value::SameTypeAs(const Value& other) const {
XLS_LOG(FATAL) << "Invalid value encountered: " << ValueKindToString(kind());
}

absl::StatusOr<ValueProto> Value::AsProto() const {
ValueProto v;
switch (kind()) {
case ValueKind::kBits: {
ValueProto::Bits* proto_bits = v.mutable_bits();
proto_bits->set_bit_count(bits().bit_count());
std::vector<uint8_t> bytes = bits().ToBytes();
proto_bits->mutable_data()->resize(bytes.size());
memcpy(proto_bits->mutable_data()->data(), bytes.data(), bytes.size());
break;
}
case ValueKind::kTuple: {
ValueProto::Tuple* tuple = v.mutable_tuple();
for (const Value& e : elements()) {
XLS_ASSIGN_OR_RETURN(*tuple->add_elements(), e.AsProto());
}
break;
}
case ValueKind::kArray: {
ValueProto::Array* array = v.mutable_array();
for (const Value& e : elements()) {
XLS_ASSIGN_OR_RETURN(*array->add_elements(), e.AsProto());
}
break;
}
case ValueKind::kToken:
v.mutable_token();
break;
case ValueKind::kInvalid:
return absl::InternalError("Cannot serialize an invalid value");
}
return v;
}

namespace {
absl::Span<const uint8_t> StrToU8Span(const std::string& s) {
return absl::Span<const uint8_t>(reinterpret_cast<const uint8_t*>(s.data()),
s.size());
}
} // namespace

absl::StatusOr<Value> Value::FromProto(const ValueProto& proto,
int64_t max_bit_size) {
switch (proto.variant_case()) {
case ValueProto::kBits: {
if (proto.bits().bit_count() <= 0) {
return absl::InvalidArgumentError(
absl::StrFormat("Bit count must be at least 1 but was %d",
proto.bits().bit_count()));
}
if (proto.bits().bit_count() >= max_bit_size) {
return absl::InternalError(absl::StrFormat(
"Bit value is too large with %d bits", proto.bits().bit_count()));
}
std::string data_copy;
absl::Span<const uint8_t> data;
int64_t byte_size = CeilOfRatio(proto.bits().bit_count(), int64_t{8});
if (byte_size == proto.bits().data().size()) {
data = StrToU8Span(proto.bits().data());
} else {
data_copy = proto.bits().data();
data_copy.resize(byte_size, '\0');
data = StrToU8Span(data_copy);
}
return Value(Bits::FromBytes(data, proto.bits().bit_count()));
}
case ValueProto::kTuple: {
std::vector<Value> values;
values.reserve(proto.tuple().elements_size());
for (const ValueProto& e : proto.tuple().elements()) {
XLS_ASSIGN_OR_RETURN(Value element, Value::FromProto(e));
values.push_back(element);
}
return Value::TupleOwned(std::move(values));
}
case ValueProto::kToken: {
return Value::Token();
}
case ValueProto::kArray: {
if (proto.array().elements_size() == 0) {
return absl::InvalidArgumentError("empty arrays are not supported.");
}
std::vector<Value> values;
values.reserve(proto.array().elements_size());
for (const ValueProto& e : proto.array().elements()) {
XLS_ASSIGN_OR_RETURN(Value element, Value::FromProto(e));
values.push_back(element);
if (values.size() != 1 && !values.front().SameTypeAs(values.back())) {
return absl::InvalidArgumentError(absl::StrFormat(
"Multiple different value types in array. Value %s does not "
"match %s",
values.back().ToString(), values.front().ToString()));
}
}
return Value::ArrayOwned(std::move(values));
}
case ValueProto::VARIANT_NOT_SET:
return absl::InvalidArgumentError(
absl::StrFormat("Unpopulated value proto: %v", proto.DebugString()));
}
}

absl::StatusOr<TypeProto> Value::TypeAsProto() const {
TypeProto proto;
switch (kind()) {
Expand Down
16 changes: 16 additions & 0 deletions xls/ir/value.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#define XLS_IR_VALUE_H_

#include <cstddef>
#include <cstdint>
#include <ostream>
#include <string>
#include <utility>
Expand All @@ -27,6 +28,7 @@
#include "xls/ir/bits.h"
#include "xls/ir/format_preference.h"
#include "xls/ir/xls_type.pb.h"
#include "xls/ir/xls_value.pb.h"

namespace xls {

Expand Down Expand Up @@ -112,6 +114,17 @@ class Value {
explicit Value(Bits bits)
: kind_(ValueKind::kBits), payload_(std::move(bits)) {}

// Convert a ValueProto back to a value.
//
// If any bits element has a bit-count of more than max_bit_size the
// conversion will fail. Default to rejecting 1GiB or larger bits values.
// This limit is not for any correctness reason (if you want to have gigabytes
// of binary data that's fine) but just to change weird OOM crashes when fed
// with fuzzed/corrupted data into more debuggable Status errors. Set this to
// std::number_limits<int64_t>::max() to disable the check.
static absl::StatusOr<Value> FromProto(
const ValueProto& proto, int64_t max_bit_size = int64_t{1} << 33);

// Serializes the contents of this value as bits in the buffer.
void FlattenTo(BitPushBuffer* buffer) const;

Expand Down Expand Up @@ -157,6 +170,9 @@ class Value {
// Returns the type of the Value as a type proto.
absl::StatusOr<TypeProto> TypeAsProto() const;

// Returns this Value as a ValueProto.
absl::StatusOr<ValueProto> AsProto() const;

// Returns true if 'other' has the same type as this Value.
bool SameTypeAs(const Value& other) const;

Expand Down
Loading

0 comments on commit 276584a

Please sign in to comment.