Add generic scroll/size parameters to scan calls

BuloZB · Jun 21, 2023 · 947165f · 947165f
1 parent 3eb7214
commit 947165f
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 8 deletions.
diff --git a/aleph.env.tmpl b/aleph.env.tmpl
@@ -70,13 +70,13 @@ ALEPH_OAUTH_SECRET=
 # To use an external ElasticSearch service:
 # ALEPH_ELASTICSEARCH_URI=
 
-# 'scroll' parameter used on ES scan() calls for xref operations
+# 'scroll' parameter used on ES scan() calls
 # (how long a consistent view of the index should be maintained for scrolled search)
-# ALEPH_XREF_SCROLL = 5m
+# ALEPH_ELASTICSEARCH_SCROLL = 5m
 
-# 'size' parameter used on ES scan() calls for xref operations
+# 'size' parameter used on ES scan() calls
 # size (per shard) of the batch send for each iteration of a scan
-# ALEPH_XREF_SCROLL_SIZE = 1000
+# ALEPH_ELASTICSEARCH_SCROLL_SIZE = 1000
 
 # To use an external Postgres service:
 # ALEPH_DATABASE_URI=postgresql://<username>:<password>@<host>/<database>

diff --git a/aleph/index/entities.py b/aleph/index/entities.py
@@ -82,6 +82,8 @@ def iter_entities(
         timeout=MAX_TIMEOUT,
         request_timeout=MAX_REQUEST_TIMEOUT,
         preserve_order=preserve_order,
+        scroll=SETTINGS.ELASTICSEARCH_SCROLL,
+        size=SETTINGS.ELASTICSEARCH_SCROLL_SIZE,
     ):
         entity = unpack_result(res)
         if entity is not None:

diff --git a/aleph/index/xref.py b/aleph/index/xref.py
@@ -94,8 +94,8 @@ def iter_matches(collection, authz):
         es,
         index=xref_index(),
         query=query,
-        scroll=SETTINGS.XREF_SCROLL,
-        size=SETTINGS.XREF_SCROLL_SIZE,
+        scroll=SETTINGS.ELASTICSEARCH_SCROLL,
+        size=SETTINGS.ELASTICSEARCH_SCROLL_SIZE,
     ):
         yield unpack_result(res)
 

diff --git a/aleph/settings.py b/aleph/settings.py
@@ -184,8 +184,8 @@ def __init__(self) -> None:
             "ALEPH_ELASTICSEARCH_URI", "http://localhost:9200"
         )
         self.ELASTICSEARCH_TIMEOUT = env.to_int("ELASTICSEARCH_TIMEOUT", 60)
-        self.XREF_SCROLL = env.get("ALEPH_XREF_SCROLL", "5m")
-        self.XREF_SCROLL_SIZE = env.get("ALEPH_XREF_SCROLL_SIZE", "1000")
+        self.ELASTICSEARCH_SCROLL = env.get("ALEPH_ELASTICSEARCH_SCROLL", "5m")
+        self.ELASTICSEARCH_SCROLL_SIZE = env.get("ALEPH_ELASTICSEARCH_SCROLL_SIZE", "1000")
 
         # Number of replicas to maintain. '2' means 3 overall copies.
         self.INDEX_REPLICAS = env.to_int("ALEPH_INDEX_REPLICAS", 0)