Fixes large payload runtime exception in Datastore (issue 1633) (feas…

…t-dev#2181) * Fixes runtime exception when feature values are larger than 1500 bytes in Datastore. Datastore indexes values as well as keys so large payloads are disallowed. This change clarifies that values should not be indexed. It avoids google.api_core.exceptions.InvalidArgument: 400 The value of property _ is longer than 1500 bytes. Signed-off-by: Pamela Toman <ptoman@paloaltonetworks.com> * Linted Signed-off-by: Pamela Toman <ptoman@paloaltonetworks.com>
Unacademy · Jan 4, 2022 · ecdf15e · ecdf15e
1 parent 94c780b
commit ecdf15e
Showing 1 changed file with 11 additions and 11 deletions.
diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py
@@ -196,18 +196,18 @@ def _write_minibatch(
                 key=key, exclude_from_indexes=("created_ts", "event_ts", "values")
             )
 
-            entity.update(
-                dict(
-                    key=entity_key.SerializeToString(),
-                    values={k: v.SerializeToString() for k, v in features.items()},
-                    event_ts=utils.make_tzaware(timestamp),
-                    created_ts=(
-                        utils.make_tzaware(created_ts)
-                        if created_ts is not None
-                        else None
-                    ),
-                )
+            content_entity = datastore.Entity(
+                exclude_from_indexes=tuple(features.keys())
             )
+            for k, v in features.items():
+                content_entity[k] = v.SerializeToString()
+            entity["key"] = entity_key.SerializeToString()
+            entity["values"] = content_entity
+            entity["event_ts"] = utils.make_tzaware(timestamp)
+            entity["created_ts"] = (
+                utils.make_tzaware(created_ts) if created_ts is not None else None
+            )
+
             entities.append(entity)
         with client.transaction():
             client.put_multi(entities)