Skip to content

Commit

Permalink
Compare Python objects instead of proto objects (feast-dev#2227)
Browse files Browse the repository at this point in the history 
* Compare Python objects instead of proto objects

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Remove unnecessary helper method

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Fix docstring test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Add docstring to RepoContents

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Lint

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Update usage test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Set cache ttl to 1 second in tests for local feature server tests

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Add FCO test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Add properties to feature service

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Lint

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Remove logic that converts Registry to RepoContents

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Always initialize registry

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Move diffing methods from Registry into FcoDiff.py

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Fix unit test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Put registry initialization back in repo_operations.py

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Fix usage test

Signed-off-by: Felix Wang <wangfelix98@gmail.com>

* Switch from hardcoded names to enum

Signed-off-by: Felix Wang <wangfelix98@gmail.com>
  • Loading branch information
@felixwang9817
felixwang9817 authored Jan 25, 2022
1 parent 3894b8d commit 3dcec6d
Show file tree
Hide file tree
Showing 8 changed files with 408 additions and 264 deletions.
202 changes: 175 additions & 27 deletions sdk/python/feast/diff/FcoDiff.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from dataclasses import dataclass
from typing import Generic, Iterable, List, Set, Tuple, TypeVar
from typing import Any, Dict, Generic, Iterable, List, Set, Tuple, TypeVar

from feast.base_feature_view import BaseFeatureView
from feast.diff.property_diff import PropertyDiff, TransitionType
Expand All @@ -16,23 +16,28 @@
from feast.protos.feast.core.RequestFeatureView_pb2 import (
RequestFeatureView as RequestFeatureViewProto,
)
from feast.registry import FeastObjectType, Registry
from feast.repo_contents import RepoContents

FcoProto = TypeVar(
"FcoProto",
EntityProto,
FeatureViewProto,
FeatureServiceProto,
OnDemandFeatureViewProto,
RequestFeatureViewProto,
)
FEAST_OBJECT_TYPE_TO_STR = {
FeastObjectType.ENTITY: "entity",
FeastObjectType.FEATURE_VIEW: "feature view",
FeastObjectType.ON_DEMAND_FEATURE_VIEW: "on demand feature view",
FeastObjectType.REQUEST_FEATURE_VIEW: "request feature view",
FeastObjectType.FEATURE_SERVICE: "feature service",
}

FEAST_OBJECT_TYPES = FEAST_OBJECT_TYPE_TO_STR.keys()

Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)


@dataclass
class FcoDiff(Generic[FcoProto]):
class FcoDiff(Generic[Fco]):
name: str
fco_type: str
current_fco: FcoProto
new_fco: FcoProto
current_fco: Fco
new_fco: Fco
fco_property_diffs: List[PropertyDiff]
transition_type: TransitionType

Expand All @@ -48,20 +53,28 @@ def add_fco_diff(self, fco_diff: FcoDiff):
self.fco_diffs.append(fco_diff)


Fco = TypeVar("Fco", Entity, BaseFeatureView, FeatureService)


def tag_objects_for_keep_delete_add(
def tag_objects_for_keep_delete_update_add(
existing_objs: Iterable[Fco], desired_objs: Iterable[Fco]
) -> Tuple[Set[Fco], Set[Fco], Set[Fco]]:
) -> Tuple[Set[Fco], Set[Fco], Set[Fco], Set[Fco]]:
existing_obj_names = {e.name for e in existing_objs}
desired_obj_names = {e.name for e in desired_objs}

objs_to_add = {e for e in desired_objs if e.name not in existing_obj_names}
objs_to_keep = {e for e in desired_objs if e.name in existing_obj_names}
objs_to_update = {e for e in desired_objs if e.name in existing_obj_names}
objs_to_keep = {e for e in existing_objs if e.name in desired_obj_names}
objs_to_delete = {e for e in existing_objs if e.name not in desired_obj_names}

return objs_to_keep, objs_to_delete, objs_to_add
return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add


FcoProto = TypeVar(
"FcoProto",
EntityProto,
FeatureViewProto,
FeatureServiceProto,
OnDemandFeatureViewProto,
RequestFeatureViewProto,
)


def tag_proto_objects_for_keep_delete_add(
Expand All @@ -80,23 +93,158 @@ def tag_proto_objects_for_keep_delete_add(
FIELDS_TO_IGNORE = {"project"}


def diff_between(current: FcoProto, new: FcoProto, object_type: str) -> FcoDiff:
assert current.DESCRIPTOR.full_name == new.DESCRIPTOR.full_name
def diff_registry_objects(current: Fco, new: Fco, object_type: str) -> FcoDiff:
current_proto = current.to_proto()
new_proto = new.to_proto()
assert current_proto.DESCRIPTOR.full_name == new_proto.DESCRIPTOR.full_name
property_diffs = []
transition: TransitionType = TransitionType.UNCHANGED
if current.spec != new.spec:
for _field in current.spec.DESCRIPTOR.fields:
if current_proto.spec != new_proto.spec:
for _field in current_proto.spec.DESCRIPTOR.fields:
if _field.name in FIELDS_TO_IGNORE:
continue
if getattr(current.spec, _field.name) != getattr(new.spec, _field.name):
if getattr(current_proto.spec, _field.name) != getattr(
new_proto.spec, _field.name
):
transition = TransitionType.UPDATE
property_diffs.append(
PropertyDiff(
_field.name,
getattr(current.spec, _field.name),
getattr(new.spec, _field.name),
getattr(current_proto.spec, _field.name),
getattr(new_proto.spec, _field.name),
)
)
return FcoDiff(
new.spec.name, object_type, current, new, property_diffs, transition,
name=new_proto.spec.name,
fco_type=object_type,
current_fco=current,
new_fco=new,
fco_property_diffs=property_diffs,
transition_type=transition,
)


def extract_objects_for_keep_delete_update_add(
registry: Registry, current_project: str, desired_repo_contents: RepoContents,
) -> Tuple[
Dict[FeastObjectType, Set[Fco]],
Dict[FeastObjectType, Set[Fco]],
Dict[FeastObjectType, Set[Fco]],
Dict[FeastObjectType, Set[Fco]],
]:
"""
Returns the objects in the registry that must be modified to achieve the desired repo state.
Args:
registry: The registry storing the current repo state.
current_project: The Feast project whose objects should be compared.
desired_repo_contents: The desired repo state.
"""
objs_to_keep = {}
objs_to_delete = {}
objs_to_update = {}
objs_to_add = {}

registry_object_type_to_objects: Dict[FeastObjectType, List[Any]]
registry_object_type_to_objects = {
FeastObjectType.ENTITY: registry.list_entities(project=current_project),
FeastObjectType.FEATURE_VIEW: registry.list_feature_views(
project=current_project
),
FeastObjectType.ON_DEMAND_FEATURE_VIEW: registry.list_on_demand_feature_views(
project=current_project
),
FeastObjectType.REQUEST_FEATURE_VIEW: registry.list_request_feature_views(
project=current_project
),
FeastObjectType.FEATURE_SERVICE: registry.list_feature_services(
project=current_project
),
}
registry_object_type_to_repo_contents: Dict[FeastObjectType, Set[Any]]
registry_object_type_to_repo_contents = {
FeastObjectType.ENTITY: desired_repo_contents.entities,
FeastObjectType.FEATURE_VIEW: desired_repo_contents.feature_views,
FeastObjectType.ON_DEMAND_FEATURE_VIEW: desired_repo_contents.on_demand_feature_views,
FeastObjectType.REQUEST_FEATURE_VIEW: desired_repo_contents.request_feature_views,
FeastObjectType.FEATURE_SERVICE: desired_repo_contents.feature_services,
}

for object_type in FEAST_OBJECT_TYPES:
(
to_keep,
to_delete,
to_update,
to_add,
) = tag_objects_for_keep_delete_update_add(
registry_object_type_to_objects[object_type],
registry_object_type_to_repo_contents[object_type],
)

objs_to_keep[object_type] = to_keep
objs_to_delete[object_type] = to_delete
objs_to_update[object_type] = to_update
objs_to_add[object_type] = to_add

return objs_to_keep, objs_to_delete, objs_to_update, objs_to_add


def diff_between(
registry: Registry, current_project: str, desired_repo_contents: RepoContents,
) -> RegistryDiff:
"""
Returns the difference between the current and desired repo states.
Args:
registry: The registry storing the current repo state.
current_project: The Feast project for which the diff is being computed.
desired_repo_contents: The desired repo state.
"""
diff = RegistryDiff()

(
objs_to_keep,
objs_to_delete,
objs_to_update,
objs_to_add,
) = extract_objects_for_keep_delete_update_add(
registry, current_project, desired_repo_contents
)

for object_type in FEAST_OBJECT_TYPES:
objects_to_keep = objs_to_keep[object_type]
objects_to_delete = objs_to_delete[object_type]
objects_to_update = objs_to_update[object_type]
objects_to_add = objs_to_add[object_type]

for e in objects_to_add:
diff.add_fco_diff(
FcoDiff(
name=e.name,
fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
current_fco=None,
new_fco=e,
fco_property_diffs=[],
transition_type=TransitionType.CREATE,
)
)
for e in objects_to_delete:
diff.add_fco_diff(
FcoDiff(
name=e.name,
fco_type=FEAST_OBJECT_TYPE_TO_STR[object_type],
current_fco=e,
new_fco=None,
fco_property_diffs=[],
transition_type=TransitionType.DELETE,
)
)
for e in objects_to_update:
current_obj = [_e for _e in objects_to_keep if _e.name == e.name][0]
diff.add_fco_diff(
diff_registry_objects(
current_obj, e, FEAST_OBJECT_TYPE_TO_STR[object_type]
)
)

return diff
76 changes: 63 additions & 13 deletions sdk/python/feast/feature_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ class FeatureService:
Services.
"""

name: str
feature_view_projections: List[FeatureViewProjection]
tags: Dict[str, str]
description: Optional[str] = None
created_timestamp: Optional[datetime] = None
last_updated_timestamp: Optional[datetime] = None
_name: str
_feature_view_projections: List[FeatureViewProjection]
_tags: Dict[str, str]
_description: Optional[str] = None
_created_timestamp: Optional[datetime] = None
_last_updated_timestamp: Optional[datetime] = None

@log_exceptions
def __init__(
Expand All @@ -51,22 +51,22 @@ def __init__(
Raises:
ValueError: If one of the specified features is not a valid type.
"""
self.name = name
self.feature_view_projections = []
self._name = name
self._feature_view_projections = []

for feature_grouping in features:
if isinstance(feature_grouping, BaseFeatureView):
self.feature_view_projections.append(feature_grouping.projection)
self._feature_view_projections.append(feature_grouping.projection)
else:
raise ValueError(
"The FeatureService {fs_name} has been provided with an invalid type"
f'{type(feature_grouping)} as part of the "features" argument.)'
)

self.tags = tags or {}
self.description = description
self.created_timestamp = None
self.last_updated_timestamp = None
self._tags = tags or {}
self._description = description
self._created_timestamp = None
self._last_updated_timestamp = None

def __repr__(self):
items = (f"{k} = {v}" for k, v in self.__dict__.items())
Expand All @@ -93,6 +93,56 @@ def __eq__(self, other):

return True

@property
def name(self) -> str:
return self._name

@name.setter
def name(self, name: str):
self._name = name

@property
def feature_view_projections(self) -> List[FeatureViewProjection]:
return self._feature_view_projections

@feature_view_projections.setter
def feature_view_projections(
self, feature_view_projections: List[FeatureViewProjection]
):
self._feature_view_projections = feature_view_projections

@property
def tags(self) -> Dict[str, str]:
return self._tags

@tags.setter
def tags(self, tags: Dict[str, str]):
self._tags = tags

@property
def description(self) -> Optional[str]:
return self._description

@description.setter
def description(self, description: str):
self._description = description

@property
def created_timestamp(self) -> Optional[datetime]:
return self._created_timestamp

@created_timestamp.setter
def created_timestamp(self, created_timestamp: datetime):
self._created_timestamp = created_timestamp

@property
def last_updated_timestamp(self) -> Optional[datetime]:
return self._last_updated_timestamp

@last_updated_timestamp.setter
def last_updated_timestamp(self, last_updated_timestamp: datetime):
self._last_updated_timestamp = last_updated_timestamp

@staticmethod
def from_proto(feature_service_proto: FeatureServiceProto):
"""
Expand Down
Loading

0 comments on commit 3dcec6d

Please sign in to comment.