Skip to content

Proto3 field presence #268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 8 commits into from
Closed

Proto3 field presence #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2f50f19
Update protobuf pregenerated files
roblabla Sep 20, 2021
aa498a0
Update grpcio-tools to latest version
roblabla Sep 21, 2021
fa4f526
Implement proto3 field presence
roblabla Sep 21, 2021
583709e
Fix to_dict with None optional fields.
roblabla Sep 21, 2021
6ba2fff
Add test with optional enum
roblabla Sep 30, 2021
e215052
Properly support optional enums
roblabla Sep 30, 2021
ae0dd45
Add tests for 64-bit ints and floats
roblabla Sep 30, 2021
9a39e19
Support field presence for in64 types
roblabla Sep 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
255 changes: 134 additions & 121 deletions poetry.lock
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ python-dateutil = "^2.8"
asv = "^0.4.2"
black = "^20.8b1"
bpython = "^0.19"
grpcio-tools = "^1.30.0"
grpcio-tools = "^1.40.0"
jinja2 = "^2.11.2"
mypy = "^0.770"
poethepoet = ">=0.9.0"
Expand Down
100 changes: 62 additions & 38 deletions src/betterproto/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,11 @@ def dataclass_field(
map_types: Optional[Tuple[str, str]] = None,
group: Optional[str] = None,
wraps: Optional[str] = None,
optional: bool = False,
) -> dataclasses.Field:
"""Creates a dataclass field with attached protobuf metadata."""
return dataclasses.field(
default=PLACEHOLDER,
default=None if optional else PLACEHOLDER,
metadata={
"betterproto": FieldMetadata(number, proto_type, map_types, group, wraps)
},
Expand All @@ -174,74 +175,74 @@ def dataclass_field(
# out at runtime. The generated dataclass variables are still typed correctly.


def enum_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_ENUM, group=group)
def enum_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_ENUM, group=group, optional=optional)


def bool_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_BOOL, group=group)
def bool_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_BOOL, group=group, optional=optional)


def int32_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_INT32, group=group)
def int32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_INT32, group=group, optional=optional)


def int64_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_INT64, group=group)
def int64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_INT64, group=group, optional=optional)


def uint32_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_UINT32, group=group)
def uint32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_UINT32, group=group, optional=optional)


def uint64_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_UINT64, group=group)
def uint64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_UINT64, group=group, optional=optional)


def sint32_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_SINT32, group=group)
def sint32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_SINT32, group=group, optional=optional)


def sint64_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_SINT64, group=group)
def sint64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_SINT64, group=group, optional=optional)


def float_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_FLOAT, group=group)
def float_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_FLOAT, group=group, optional=optional)


def double_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_DOUBLE, group=group)
def double_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_DOUBLE, group=group, optional=optional)


def fixed32_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_FIXED32, group=group)
def fixed32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_FIXED32, group=group, optional=optional)


def fixed64_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_FIXED64, group=group)
def fixed64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_FIXED64, group=group, optional=optional)


def sfixed32_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_SFIXED32, group=group)
def sfixed32_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_SFIXED32, group=group, optional=optional)


def sfixed64_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_SFIXED64, group=group)
def sfixed64_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_SFIXED64, group=group, optional=optional)


def string_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_STRING, group=group)
def string_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_STRING, group=group, optional=optional)


def bytes_field(number: int, group: Optional[str] = None) -> Any:
return dataclass_field(number, TYPE_BYTES, group=group)
def bytes_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any:
return dataclass_field(number, TYPE_BYTES, group=group, optional=optional)


def message_field(
number: int, group: Optional[str] = None, wraps: Optional[str] = None
number: int, group: Optional[str] = None, wraps: Optional[str] = None, optional: bool = False
) -> Any:
return dataclass_field(number, TYPE_MESSAGE, group=group, wraps=wraps)
return dataclass_field(number, TYPE_MESSAGE, group=group, wraps=wraps, optional=optional)


def map_field(
Expand Down Expand Up @@ -701,12 +702,16 @@ def __bytes__(self) -> bytes:

if value is None:
# Optional items should be skipped. This is used for the Google
# wrapper types.
# wrapper types and proto3 field presence/optional fields.
continue

# Being selected in a a group means this field is the one that is
# currently set in a `oneof` group, so it must be serialized even
# if the value is the default zero value.
#
# Note that proto3 field presence/optional fields are put in a
# synthetic single-item oneof by protoc, which helps us ensure we
# send the value even if the value is the default zero value.
selected_in_group = (
meta.group and self._group_current[meta.group] == field_name
)
Expand Down Expand Up @@ -829,8 +834,9 @@ def _get_field_default_gen(cls, field: dataclasses.Field) -> Any:
# This is some kind of list (repeated) field.
return list
elif t.__origin__ is Union and t.__args__[1] is type(None):
# This is an optional (wrapped) field. For setting the default we
# really don't care what kind of field it is.
# This is an optional field (either wrapped, or using proto3
# field presence). For setting the default we really don't care
# what kind of field it is.
return type(None)
else:
return t
Expand Down Expand Up @@ -1004,6 +1010,7 @@ def to_dict(
defaults = self._betterproto.default_gen
for field_name, meta in self._betterproto.meta_by_field_name.items():
field_is_repeated = defaults[field_name] is list
field_is_optional = defaults[field_name] is type(None)
value = getattr(self, field_name)
cased_name = casing(field_name).rstrip("_") # type: ignore
if meta.proto_type == TYPE_MESSAGE:
Expand Down Expand Up @@ -1041,6 +1048,8 @@ def to_dict(
]
if value or include_default_values:
output[cased_name] = value
elif value is None:
output[cased_name] = None
elif (
value._serialized_on_wire
or include_default_values
Expand All @@ -1066,13 +1075,17 @@ def to_dict(
if meta.proto_type in INT_64_TYPES:
if field_is_repeated:
output[cased_name] = [str(n) for n in value]
elif value is None:
output[cased_name] = value
else:
output[cased_name] = str(value)
elif meta.proto_type == TYPE_BYTES:
if field_is_repeated:
output[cased_name] = [
b64encode(b).decode("utf8") for b in value
]
elif value is None:
output[cased_name] = None
else:
output[cased_name] = b64encode(value).decode("utf8")
elif meta.proto_type == TYPE_ENUM:
Expand All @@ -1085,6 +1098,11 @@ def to_dict(
else:
# transparently upgrade single value to repeated
output[cased_name] = [enum_class(value).name]
elif value is None:
output[cased_name] = None
elif field_is_optional:
enum_class = field_types[field_name].__args__[0]
output[cased_name] = enum_class(value).name
else:
enum_class = field_types[field_name] # noqa
output[cased_name] = enum_class(value).name
Expand Down Expand Up @@ -1122,6 +1140,9 @@ def from_dict(self: T, value: Dict[str, Any]) -> T:
if value[key] is not None:
if meta.proto_type == TYPE_MESSAGE:
v = getattr(self, field_name)
if value[key] is None and self._get_field_default(key) == None:
# Setting an optional value to None.
setattr(self, field_name, None)
if isinstance(v, list):
cls = self._betterproto.cls_by_field[field_name]
if cls == datetime:
Expand All @@ -1141,6 +1162,9 @@ def from_dict(self: T, value: Dict[str, Any]) -> T:
setattr(self, field_name, v)
elif meta.wraps:
setattr(self, field_name, value[key])
elif v is None:
cls = self._betterproto.cls_by_field[field_name]
setattr(self, field_name, cls().from_dict(value[key]))
else:
# NOTE: `from_dict` mutates the underlying message, so no
# assignment here is necessary.
Expand Down
Loading