actually turn on ndb global cache with global_cache_policy

🤦

atproto_firehose.py

@@ -27,6 +27,7 @@
     cache_policy,
     create_task,
     global_cache,
+    global_cache_policy,
     global_cache_timeout_policy,
     report_exception,
     USER_AGENT,
@@ -69,6 +70,7 @@ def _load_dids():
     global atproto_dids, atproto_loaded_at, bridged_dids, bridged_loaded_at
 
     with ndb_client.context(cache_policy=cache_policy, global_cache=global_cache,
+                            global_cache_policy=global_cache_policy,
                             global_cache_timeout_policy=global_cache_timeout_policy):
         if not DEBUG:
             Timer(STORE_CURSOR_FREQ.total_seconds(), _load_dids).start()
@@ -99,6 +101,7 @@ def subscriber():
         try:
             with ndb_client.context(
                     cache_policy=cache_policy, global_cache=global_cache,
+                    global_cache_policy=global_cache_policy,
                     global_cache_timeout_policy=global_cache_timeout_policy):
                 subscribe()
 
@@ -261,6 +264,7 @@ def handler():
         try:
             with ndb_client.context(
                     cache_policy=cache_policy, global_cache=global_cache,
+                    global_cache_policy=global_cache_policy,
                     global_cache_timeout_policy=global_cache_timeout_policy):
                 handle()
 

atproto_hub.py

@@ -20,7 +20,7 @@
 # all protocols
 import activitypub, atproto, web
 import atproto_firehose
-from common import cache_policy, global_cache, global_cache_timeout_policy, USER_AGENT
+import common
 import models
 
 # as of 2024-07-10
@@ -44,9 +44,10 @@
     app.wsgi_app, client=appengine_config.ndb_client,
     # limited context-local cache. avoid full one due to this bug:
     # https://github.com/googleapis/python-ndb/issues/888
-    cache_policy=cache_policy,
-    global_cache=global_cache,
-    global_cache_timeout_policy=global_cache_timeout_policy)
+    cache_policy=common.cache_policy,
+    global_cache=common.global_cache,
+    global_cache_policy=common.global_cache_policy,
+    global_cache_timeout_policy=common.global_cache_timeout_policy)
 
 
 @app.get('/liveness_check')
@@ -114,7 +115,7 @@ def atproto_admin():
 if 'GAE_INSTANCE' in os.environ:  # prod
     def request_crawl():
         bgs = lexrpc.client.Client(f'https://{os.environ["BGS_HOST"]}',
-                                   headers={'User-Agent': USER_AGENT})
+                                   headers={'User-Agent': common.USER_AGENT})
         resp = bgs.com.atproto.sync.requestCrawl({'hostname': os.environ['PDS_HOST']})
         logger.info(resp)
         Timer(5 * 60, request_crawl).start()

common.py

@@ -407,6 +407,10 @@ def cache_policy(key):
     return key and key.kind == 'Object' and key.name.startswith('did:')
 
 
+def global_cache_policy(key):
+    return True
+
+
 PROFILE_ID_RE = re.compile(
     fr"""
       /users?/[^/]+$ |

flask_app.py

@@ -16,7 +16,7 @@
     flask_util,
 )
 
-from common import cache_policy, global_cache, global_cache_timeout_policy
+import common
 
 logger = logging.getLogger(__name__)
 # logging.getLogger('lexrpc').setLevel(logging.INFO)
@@ -45,9 +45,10 @@
     app.wsgi_app, client=appengine_config.ndb_client,
     # limited context-local cache. avoid full one due to this bug:
     # https://github.com/googleapis/python-ndb/issues/888
-    cache_policy=cache_policy,
-    global_cache=global_cache,
-    global_cache_timeout_policy=global_cache_timeout_policy)
+    cache_policy=common.cache_policy,
+    global_cache=common.global_cache,
+    global_cache_policy=common.global_cache_policy,
+    global_cache_timeout_policy=common.global_cache_timeout_policy)
 
 # deregister XRPC methods we don't support
 for nsid in (

router.py

@@ -11,7 +11,7 @@
 
 # all protocols
 import activitypub, atproto, web
-from common import cache_policy, global_cache, global_cache_timeout_policy
+import common
 import models
 import protocol
 
@@ -26,9 +26,10 @@
     app.wsgi_app, client=appengine_config.ndb_client,
     # limited context-local cache. avoid full one due to this bug:
     # https://github.com/googleapis/python-ndb/issues/888
-    cache_policy=cache_policy,
-    global_cache=global_cache,
-    global_cache_timeout_policy=global_cache_timeout_policy)
+    cache_policy=common.cache_policy,
+    global_cache=common.global_cache,
+    global_cache_policy=common.global_cache_policy,
+    global_cache_timeout_policy=common.global_cache_timeout_policy)
 
 app.add_url_rule('/queue/poll-feed', view_func=web.poll_feed_task, methods=['POST'])
 app.add_url_rule('/queue/receive', view_func=protocol.receive_task, methods=['POST'])

tests/testutil.py

@@ -34,7 +34,7 @@
 
 # other modules are imported _after_ Fake etc classes is defined so that it's in
 # PROTOCOLS when URL routes are registered.
-from common import add, global_cache_timeout_policy, long_to_base64, TASKS_LOCATION
+from common import add, long_to_base64, TASKS_LOCATION
 import ids
 import models
 from models import KEY_BITS, Object, PROTOCOLS, Target, User
@@ -287,7 +287,8 @@ def setUp(self):
         self.ndb_context = ndb_client.context(
             cache_policy=common.cache_policy,
             global_cache=_InProcessGlobalCache(),
-            global_cache_timeout_policy=global_cache_timeout_policy)
+            global_cache_policy=common.global_cache_policy,
+            global_cache_timeout_policy=common.global_cache_timeout_policy)
         self.ndb_context.__enter__()
 
         util.now = lambda **kwargs: testutil.NOW