Add reuse entries option (contentful#32)

* Add reuse entries option

* Make resource cache locale aware

Author: dlitvakb

CHANGELOG.md

@@ -1,6 +1,8 @@
 # CHANGELOG
 
 ## Unreleased
+### Added
+* Added support to reuse entries. This is a performance improvement, which is disabled by default due to backwards compatibility. All users are highly encouraged to enable it and test it in their applications. Inspired by @rezonant in [contentful/contentful.rb#164]((https://github.com/contentful/contentful.rb/pull/164).
 
 ## v1.8.0
 ### Added

Makefile

@@ -56,6 +56,7 @@ test-all:
 coverage:
 	coverage run --source contentful setup.py test
 	coverage report -m
+	flake8 contentful
 
 watch:
 	fswatch -d -e contentful/__pycache__ -e tests/__pycache__ contentful tests | xargs -n1 make coverage

README.rst

@@ -172,6 +172,8 @@ Client Configuration Options
 
 ``content_type_cache``: (optional) Boolean determining wether to store a Cache of the Content Types in order to properly coerce Entry fields, defaults to True.
 
+``reuse_entries``: (optional) Boolean, when True, reuse hydrated Entry and Asset objects within the same request when possible. Can result in a large speed increase and better handles cyclical object graphs. This can be a good alternative to max_include_resolution_depth if your content model contains (or can contain) circular references.
+
 ``proxy_host``: (optional) URL for Proxy, defaults to None.
 
 ``proxy_port``: (optional) Port for Proxy, defaults to None.
@@ -184,7 +186,7 @@ Client Configuration Options
 
 ``max_rate_limit_wait``: (optional) Timeout (in seconds) for waiting for retry after RateLimitError, defaults to 60.
 
-``max_include_resolution_depth``: (optional) Maximum include resolution level for Resources, defaults to 20 (max include level * 2).
+``max_include_resolution_depth``: (optional) Maximum include resolution level for Resources, defaults to 20 (max include level * 2). Note that when `reuse_entries` is enabled, the max include resolution depth only affects deep chains of unique objects (ie, not simple circular references).
 
 ``application_name``: (optional) User application name, defaults to None.
 

_docs/index.rst

@@ -169,6 +169,8 @@ Client Configuration Options
 
 ``content_type_cache``: (optional) Boolean determining wether to store a Cache of the Content Types in order to properly coerce Entry fields, defaults to True.
 
+``reuse_entries``: (optional) Boolean, when True, reuse hydrated Entry and Asset objects within the same request when possible. Can result in a large speed increase and better handles cyclical object graphs. This can be a good alternative to max_include_resolution_depth if your content model contains (or can contain) circular references.
+
 ``proxy_host``: (optional) URL for Proxy, defaults to None.
 
 ``proxy_port``: (optional) Port for Proxy, defaults to None.
@@ -181,7 +183,7 @@ Client Configuration Options
 
 ``max_rate_limit_wait``: (optional) Timeout (in seconds) for waiting for retry after RateLimitError, defaults to 60.
 
-``max_include_resolution_depth``: (optional) Maximum include resolution level for Resources, defaults to 20 (max include level * 2).
+``max_include_resolution_depth``: (optional) Maximum include resolution level for Resources, defaults to 20 (max include level * 2).Note that when `reuse_entries` is enabled, the max include resolution depth only affects deep chains of unique objects (ie, not simple circular references).
 
 ``application_name``: (optional) User application name, defaults to None.
 

contentful/client.py

@@ -1,7 +1,7 @@
 import requests
 import platform
 from re import sub
-from .utils import ConfigurationException, NotSupportedException
+from .utils import ConfigurationException
 from .utils import retry_request, string_class
 from .errors import get_error, RateLimitExceededError, EntryNotFoundError
 from .resource_builder import ResourceBuilder
@@ -49,6 +49,9 @@ class Client(object):
     :param content_type_cache: (optional) Boolean determining wether to
         store a Cache of the Content Types in order to properly coerce
         Entry fields, defaults to True.
+    :param reuse_entries: (optional) Boolean determining wether to reuse
+        hydrated Entry and Asset objects within the same request when possible.
+        Defaults to False
     :param proxy_host: (optional) URL for Proxy, defaults to None.
     :param proxy_port: (optional) Port for Proxy, defaults to None.
     :param proxy_username: (optional) Username for Proxy, defaults to None.
@@ -89,6 +92,7 @@ def __init__(
             gzip_encoded=True,
             raise_errors=True,
             content_type_cache=True,
+            reuse_entries=False,
             proxy_host=None,
             proxy_port=None,
             proxy_username=None,
@@ -112,6 +116,7 @@ def __init__(
         self.gzip_encoded = gzip_encoded
         self.raise_errors = raise_errors
         self.content_type_cache = content_type_cache
+        self.reuse_entries = reuse_entries
         self.proxy_host = proxy_host
         self.proxy_port = proxy_port
         self.proxy_username = proxy_username
@@ -566,7 +571,8 @@ def _get(self, url, query=None):
             self.default_locale,
             localized,
             response.json(),
-            max_depth=self.max_include_resolution_depth
+            max_depth=self.max_include_resolution_depth,
+            reuse_entries=self.reuse_entries
         ).build()
 
     def _has_proxy(self):

contentful/entry.py

@@ -21,12 +21,13 @@ class Entry(FieldsResource):
     API Reference: https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/entries
     """
 
-    def _coerce(self, field_id, value, localized, includes, errors):
+    def _coerce(self, field_id, value, localized, includes, errors, resources=None):
         if is_link(value) and not unresolvable(value, errors):
             return self._build_nested_resource(
                 value,
                 localized,
-                includes
+                includes,
+                resources=resources
             )
         elif is_link_array(value):
             items = []
@@ -37,7 +38,8 @@ def _coerce(self, field_id, value, localized, includes, errors):
                     self._build_nested_resource(
                         link,
                         localized,
-                        includes
+                        includes,
+                        resources=resources
                     )
                 )
 
@@ -56,33 +58,48 @@ def _coerce(self, field_id, value, localized, includes, errors):
             value,
             localized,
             includes,
-            errors
+            errors,
+            resources
         )
 
-    def _build_nested_resource(self, value, localized, includes):
+    def _build_nested_resource(self, value, localized, includes, resources=None):
         # Maximum include Depth is 10 in the API, but we raise it to 20 (default),
         # in case one of the included items has a reference in an upper level,
         # so we can keep the include chain for that object as well
-        # Any included object after the 20th level of depth will be just a Link
-        if self._depth < self._max_depth:
-            resource = resource_for_link(value, includes)
+        # Any included object after the 20th level of depth will be just a Link.
+        # When using reuse_entries, this is not the case if the entry was previously
+        # cached.
 
+        resource = resource_for_link(
+            value,
+            includes,
+            resources=resources,
+            locale=self.sys.get('locale', '*')
+        )
+
+        if isinstance(resource, FieldsResource):  # Resource comes from instance cache
+            return resource
+
+        if self._depth < self._max_depth:
             if resource is not None:
                 return self._resolve_include(
                     resource,
                     localized,
-                    includes
+                    includes,
+                    resources=resources
                 )
 
         return self._build_link(value)
 
-    def _resolve_include(self, resource, localized, includes):
+    def _resolve_include(self, resource, localized, includes, resources=None):
         from .resource_builder import ResourceBuilder
         return ResourceBuilder(
             self.default_locale,
             localized,
             resource,
             includes,
+            reuse_entries=bool(resources),
+            resources=resources,
             depth=self._depth + 1,
             max_depth=self._max_depth
         ).build()

contentful/resource.py

@@ -32,6 +32,7 @@ def __init__(
             includes=None,
             errors=None,
             localized=False,
+            resources=None,
             depth=0,
             max_depth=20):
         self.raw = item
@@ -40,6 +41,14 @@ def __init__(
         self._max_depth = max_depth
         self.sys = self._hydrate_sys(item)
 
+        if resources is not None and 'sys' in item:
+            cache_key = "{0}:{1}:{2}".format(
+                item['sys']['type'],
+                item['sys']['id'],
+                item['sys'].get('locale', '*')
+            )
+            resources[cache_key] = self
+
     def _hydrate_sys(self, item):
         sys = {}
         for k, v in item.get('sys', {}).items():
@@ -81,18 +90,20 @@ def __init__(
             includes=None,
             errors=None,
             localized=False,
+            resources=None,
             **kwargs):
         super(FieldsResource, self).__init__(
             item,
             includes=includes,
             errors=errors,
             localized=localized,
+            resources=resources,
             **kwargs
         )
 
-        self._fields = self._hydrate_fields(item, localized, includes, errors)
+        self._fields = self._hydrate_fields(item, localized, includes, errors, resources=resources)
 
-    def _hydrate_fields(self, item, localized, includes, errors):
+    def _hydrate_fields(self, item, localized, includes, errors, resources=None):
         if 'fields' not in item:
             return {}
 
@@ -105,12 +116,12 @@ def _hydrate_fields(self, item, localized, includes, errors):
         locale = self._locale()
         fields = {locale: {}}
         if localized:
-            self._hydrate_localized_entry(fields, item, includes, errors)
+            self._hydrate_localized_entry(fields, item, includes, errors, resources)
         else:
-            self._hydrate_non_localized_entry(fields, item, includes, errors)
+            self._hydrate_non_localized_entry(fields, item, includes, errors, resources)
         return fields
 
-    def _hydrate_localized_entry(self, fields, item, includes, errors):
+    def _hydrate_localized_entry(self, fields, item, includes, errors, resources=None):
         for k, locales in item['fields'].items():
             for locale, v in locales.items():
                 if locale not in fields:
@@ -120,20 +131,22 @@ def _hydrate_localized_entry(self, fields, item, includes, errors):
                     v,
                     True,
                     includes,
-                    errors
+                    errors,
+                    resources=resources
                 )
 
-    def _hydrate_non_localized_entry(self, fields, item, includes, errors):
+    def _hydrate_non_localized_entry(self, fields, item, includes, errors, resources=None):
         for k, v in item['fields'].items():
             fields[self._locale()][snake_case(k)] = self._coerce(
                 snake_case(k),
                 v,
                 False,
                 includes,
-                errors
+                errors,
+                resources=resources
             )
 
-    def _coerce(self, field_id, value, localized, includes, errors):
+    def _coerce(self, field_id, value, localized, includes, errors, resources=None):
         return value
 
     def fields(self, locale=None):

contentful/resource_builder.py

@@ -1,12 +1,11 @@
-# Classes imported here are meant to be used via globals() on build
-from .array import Array  # noqa: F401
-from .entry import Entry  # noqa: F401
-from .asset import Asset  # noqa: F401
-from .space import Space  # noqa: F401
-from .content_type import ContentType  # noqa: F401
-from .deleted_asset import DeletedAsset  # noqa: F401
-from .deleted_entry import DeletedEntry  # noqa: F401
-from .locale import Locale  # noqa: F401
+from .array import Array
+from .entry import Entry
+from .asset import Asset
+from .space import Space
+from .content_type import ContentType
+from .deleted_asset import DeletedAsset
+from .deleted_entry import DeletedEntry
+from .locale import Locale
 from .sync_page import SyncPage
 from .utils import unresolvable
 
@@ -31,15 +30,22 @@ def __init__(
             localized,
             json,
             includes_for_single=None,
+            reuse_entries=False,
+            resources=None,
             depth=0,
             max_depth=20):
         self.default_locale = default_locale
         self.localized = localized
         self.json = json
         self.includes_for_single = includes_for_single
+        self.reuse_entries = reuse_entries
         self.depth = depth
         self.max_depth = max_depth
 
+        if resources is None:
+            resources = {} if self.reuse_entries else None
+        self.resources = resources
+
     def build(self):
         """Creates the objects from the JSON response"""
 
@@ -78,26 +84,41 @@ def _build_item(self, item, includes=None, errors=None):
         if errors is None:
             errors = []
 
-        buildables = [
-            'Entry',
-            'Asset',
-            'ContentType',
-            'Space',
-            'DeletedEntry',
-            'DeletedAsset',
-            'Locale'
-        ]
+        buildables = {
+            'Entry': Entry,
+            'Asset': Asset,
+            'ContentType': ContentType,
+            'Space': Space,
+            'DeletedEntry': DeletedEntry,
+            'DeletedAsset': DeletedAsset,
+            'Locale': Locale
+        }
+
+        resource = self._resource_from_cache(item) if self.reuse_entries else None
+        if resource is not None:
+            return resource
+
         if item['sys']['type'] in buildables:
-            return globals()[item['sys']['type']](
+            return buildables[item['sys']['type']](
                 item,
                 default_locale=self.default_locale,
                 localized=self.localized,
                 includes=includes,
                 errors=errors,
+                resources=self.resources,
                 depth=self.depth,
                 max_depth=self.max_depth
             )
 
+    def _resource_from_cache(self, item):
+        cache_key = "{0}:{1}:{2}".format(
+            item['sys']['type'],
+            item['sys']['id'],
+            item['sys'].get('locale', '*')
+        )
+        if self.resources and cache_key in self.resources:
+            return self.resources[cache_key]
+
     def _includes(self):
         includes = list(self.json['items'])
         for e in ['Entry', 'Asset']:

contentful/utils.py

@@ -115,9 +115,18 @@ def unresolvable(item, errors):
     return False
 
 
-def resource_for_link(link, includes):
+def resource_for_link(link, includes, resources=None, locale=None):
     """Returns the resource that matches the link"""
 
+    if resources is not None:
+        cache_key = "{0}:{1}:{2}".format(
+            link['sys']['linkType'],
+            link['sys']['id'],
+            locale
+        )
+        if cache_key in resources:
+            return resources[cache_key]
+
     for i in includes:
         if (i['sys']['id'] == link['sys']['id'] and
                 i['sys']['type'] == link['sys']['linkType']):
@@ -165,5 +174,6 @@ def wrapper(url, query=None):
                     )
                     log.debug(retry_message)
                     time.sleep(reset_time * uniform(1.0, 1.2))
-            raise exception
+            if exception is not None:
+                raise exception
         return wrapper