Merge pull request #3166 from alephdata/release/3.14.2

Release/3.14.2 to develop

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.14.1
+current_version = 3.14.2
 tag_name = {new_version}
 commit = True
 tag = True

.tx/config

@@ -1,14 +1,14 @@
 [main]
 host = https://www.transifex.com
 
-[aleph-api.messages-pot]
+[o:aleph:p:aleph-api:r:messages-pot]
 file_filter = aleph/translations/<lang>/LC_MESSAGES/aleph.po
 minimum_perc = 0
 source_file = aleph/translations/messages.pot
 source_lang = en
 type = PO
 
-[aleph-ui.messages-json]
+[o:aleph:p:aleph-ui:r:messages-json]
 file_filter = ui/i18n/translations/raw/<lang>.json
 minimum_perc = 0
 source_file = ui/i18n/messages.json

CONTRIBUTING.md

@@ -34,7 +34,7 @@ If, after reviewing the list (or selecting an issue to work on) you'd like to re
 
 ### Forking and Branching
 
-All new development work needs to happen on a branch. If this is your first time, you'll not have the ability to create branches directly from Aleph so will need to fork the repo. Once you've done this create an issue branch and commit all you changes to this.
+All new development work needs to happen on a branch. If this is your first time, you'll not have the ability to create branches directly from Aleph so will need to fork the repo. Once you've done this create an issue branch and commit all you changes to this. Make sure you set up [commit signing](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) beforehand.
 
 ### Pull Requests
 
@@ -54,7 +54,7 @@ Aleph uses two main branches. The **develop** branch is for any new work that ne
 
 ### Code Formatting and Linting
 
-We use automatic code formatters and linters to keep our code consistent and to spot bugs and style issues early in the process. For Python code, we use [black](https://black.readthedocs.io/en/stable/) and [flake8](https://flake8.pycqa.org/en/latest/). Our JavaScript code base uses [Prettier](https://prettier.io) and [ESLint](https://eslint.org/).
+We use automatic code formatters and linters to keep our code consistent and to spot bugs and style issues early in the process. For Python code, we use  [black](https://black.readthedocs.io/en/stable/) and [ruff](https://beta.ruff.rs/). Our JavaScript code base uses [Prettier](https://prettier.io) and [ESLint](https://eslint.org/).
 
 When you open a pull request, your changes are automatically checked for consistency with our code formatting and linting rules. You can also run the checks in your local development environment. If you aren’t familiar with these tools yet, don’t worry. You can open a pull request even if your changes don’t pass these checks.
 

Makefile

@@ -28,7 +28,7 @@ test-ui:
 	$(UIDOCKER) npm run test
 
 lint:
-	flake8 aleph/
+	ruff check .
 
 lint-ui:
 	$(UIDOCKER) npm run lint

aleph.env.tmpl

@@ -48,10 +48,10 @@ ALEPH_OAUTH_SECRET=
 # ALEPH_OAUTH_METADATA_URL=https://accounts.google.com/.well-known/openid-configuration
 #
 # KeyCloak/OCCRP:
-# ALEPH_OAUTH_METADATA_URL=https://secure.occrp.org/auth/realms/general/.well-known/openid-configuration 
+# ALEPH_OAUTH_METADATA_URL=https://secure.occrp.org/auth/realms/general/.well-known/openid-configuration
 #
 # Azure:
-# ALEPH_OAUTH_METADATA_URL=https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration 
+# ALEPH_OAUTH_METADATA_URL=https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration
 #
 # Amazon Cognito:
 # ALEPH_OAUTH_METADATA_URL=https://cognito-idp.{region}.amazonaws.com/{userPoolId}/.well-known/openid-configuration
@@ -70,6 +70,14 @@ ALEPH_OAUTH_SECRET=
 # To use an external ElasticSearch service:
 # ALEPH_ELASTICSEARCH_URI=
 
+# 'scroll' parameter used on ES scan() calls on xref operations
+# (how long a consistent view of the index should be maintained for scrolled search)
+# ALEPH_XREF_SCROLL = 5m
+
+# 'size' parameter used on ES scan() calls on xref operations
+# size (per shard) of the batch send for each iteration of a scan
+# ALEPH_XREF_SCROLL_SIZE = 1000
+
 # To use an external Postgres service:
 # ALEPH_DATABASE_URI=postgresql://<username>:<password>@<host>/<database>
 

aleph/index/entities.py

@@ -64,6 +64,8 @@ def iter_entities(
     excludes=None,
     filters=None,
     sort=None,
+    es_scroll="5m",
+    es_scroll_size=1000,
 ):
     """Scan all entities matching the given criteria."""
     query = {
@@ -82,6 +84,8 @@ def iter_entities(
         timeout=MAX_TIMEOUT,
         request_timeout=MAX_REQUEST_TIMEOUT,
         preserve_order=preserve_order,
+        scroll=es_scroll,
+        size=es_scroll_size,
     ):
         entity = unpack_result(res)
         if entity is not None:

aleph/logic/expand.py

@@ -117,7 +117,7 @@ def entity_tags(proxy, authz, prop_types=DEFAULT_TAGS):
 
     type_names = [t.name for t in prop_types]
     log.debug("Tags[%s]: %s values", type_names, len(values))
-    for (type_, value) in values:
+    for type_, value in values:
         key = type_.node_id(value)
         lookup[key] = (type_, value)
         # Determine which indexes may contain further mentions (only things).

aleph/logic/html.py

@@ -53,7 +53,7 @@ def sanitize_html(html_text, base_url, encoding=None):
                 doc.make_links_absolute(base_url)
             except TypeError:
                 pass
-        for (el, attr, href, _) in doc.iterlinks():
+        for el, attr, href, _ in doc.iterlinks():
             if el.tag == "a":
                 el.set("target", "_blank")
                 el.set("rel", "nofollow noreferrer external noopener")

aleph/logic/matching.py

@@ -60,19 +60,19 @@ def match_query(proxy, collection_ids=None, query=None):
         query["bool"]["filter"].append({"terms": {"collection_id": collection_ids}})
 
     filters = set()
-    for (prop, value) in proxy.itervalues():
+    for prop, value in proxy.itervalues():
         specificity = prop.specificity(value)
         if specificity > 0:
             filters.add((prop.type, value, specificity))
 
     filters = sorted(filters, key=lambda p: p[2], reverse=True)
     required = []
-    for (type_, value, _) in filters:
+    for type_, value, _ in filters:
         if type_ in REQUIRED and len(required) <= MAX_CLAUSES:
             required.extend(_make_queries(type_, value))
 
     scoring = []
-    for (type_, value, _) in filters:
+    for type_, value, _ in filters:
         clauses = len(required) + len(scoring)
         if type_ not in REQUIRED and clauses <= MAX_CLAUSES:
             scoring.extend(_make_queries(type_, value))

aleph/logic/profiles.py

@@ -94,7 +94,7 @@ def profile_fragments(collection, aggregator, entity_id=None):
     aggregator.delete(origin=ORIGIN)
     writer = aggregator.bulk()
     profile_id = None
-    for (profile_id, entity_id) in EntitySet.all_profiles(
+    for profile_id, entity_id in EntitySet.all_profiles(
         collection.id, entity_id=entity_id
     ):
         data = {"id": entity_id, "schema": Entity.THING, "profile_id": profile_id}
@@ -129,11 +129,11 @@ def pairwise_judgements(pairs, collection_id):
     q = q.filter(db.tuple_(left.entity_id, right.entity_id).in_(pairs))
 
     judgements = {}
-    for (left, right) in pairs:
+    for left, right in pairs:
         judgements[(left, right)] = Judgement.NO_JUDGEMENT
         judgements[(right, left)] = Judgement.NO_JUDGEMENT
 
-    for (left, right) in q.all():
+    for left, right in q.all():
         judgement = left.judgement + right.judgement
         judgements[(left.entity_id, right.entity_id)] = judgement
         judgements[(right.entity_id, left.entity_id)] = judgement

aleph/logic/xref.py

@@ -192,7 +192,12 @@ def _query_entities(collection):
     """Generate matches for indexing."""
     log.info("[%s] Generating entity-based xref...", collection)
     matchable = [s for s in model if s.matchable]
-    for proxy in iter_proxies(collection_id=collection.id, schemata=matchable):
+    for proxy in iter_proxies(
+        collection_id=collection.id,
+        schemata=matchable,
+        es_scroll=SETTINGS.XREF_SCROLL,
+        es_scroll_size=SETTINGS.XREF_SCROLL_SIZE,
+    ):
         yield from _query_item(proxy)
 
 

aleph/search/parser.py

@@ -64,7 +64,7 @@ def sorts(self):
 
     @property
     def items(self):
-        for (key, value) in self.args.items(multi=True):
+        for key, value in self.args.items(multi=True):
             if key in ("offset", "limit", "next_limit"):
                 continue
             value = sanitize_text(value, encoding="utf-8")

aleph/search/query.py

@@ -201,7 +201,7 @@ def get_sort(self):
             return self.SORT_DEFAULT
 
         sort_fields = ["_score"]
-        for (field, direction) in self.parser.sorts:
+        for field, direction in self.parser.sorts:
             field = self.SORT_FIELDS.get(field, field)
             type_ = get_field_type(field)
             config = {"order": direction, "missing": "_last"}

aleph/settings.py

@@ -184,6 +184,8 @@ def __init__(self) -> None:
             "ALEPH_ELASTICSEARCH_URI", "http://localhost:9200"
         )
         self.ELASTICSEARCH_TIMEOUT = env.to_int("ELASTICSEARCH_TIMEOUT", 60)
+        self.XREF_SCROLL = env.get("ALEPH_XREF_SCROLL", "5m")
+        self.XREF_SCROLL_SIZE = env.get("ALEPH_XREF_SCROLL_SIZE", "1000")
 
         # Number of replicas to maintain. '2' means 3 overall copies.
         self.INDEX_REPLICAS = env.to_int("ALEPH_INDEX_REPLICAS", 0)

aleph/tests/util.py

@@ -67,7 +67,6 @@ class TestResponse(response_class, JsonResponseMixin):
 
 
 class TestCase(unittest.TestCase):
-
     # Expose faker since it should be easy to use
     fake = Factory.create()
 

aleph/translations/ar/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2022 ORGANIZATION
+# Copyright (C) 2023 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
 # 
 # Translators:
 # Eman Alqaisi <eman@arij.net>, 2020
@@ -13,10 +13,10 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-12-12 11:06+0100\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Mohammed AlKawmani <komani@arij.net>, 2020\n"
-"Language-Team: Arabic (https://www.transifex.com/aleph/teams/76591/ar/)\n"
+"Language-Team: Arabic (https://app.transifex.com/aleph/teams/76591/ar/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

aleph/translations/bs/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2022 ORGANIZATION
+# Copyright (C) 2023 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
 # 
 # Translators:
 # pudo <friedrich@pudo.org>, 2018
@@ -14,10 +14,10 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-12-12 11:06+0100\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Tajna Biscevic <tajna@occrp.org>, 2019\n"
-"Language-Team: Bosnian (https://www.transifex.com/aleph/teams/76591/bs/)\n"
+"Language-Team: Bosnian (https://app.transifex.com/aleph/teams/76591/bs/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

aleph/translations/de/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2022 ORGANIZATION
+# Copyright (C) 2023 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
 # 
 # Translators:
 # pudo <friedrich@pudo.org>, 2020
@@ -11,10 +11,10 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-12-12 11:06+0100\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: pudo <friedrich@pudo.org>, 2020\n"
-"Language-Team: German (https://www.transifex.com/aleph/teams/76591/de/)\n"
+"Language-Team: German (https://app.transifex.com/aleph/teams/76591/de/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

aleph/translations/es/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2022 ORGANIZATION
+# Copyright (C) 2023 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
 # 
 # Translators:
 # John Ospina <jmospina@gmail.com>, 2018
@@ -13,10 +13,10 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-12-12 11:06+0100\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Nathan Jaccard <nathan@occrp.org>, 2020\n"
-"Language-Team: Spanish (https://www.transifex.com/aleph/teams/76591/es/)\n"
+"Language-Team: Spanish (https://app.transifex.com/aleph/teams/76591/es/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

aleph/translations/fr/LC_MESSAGES/aleph.po

@@ -1,7 +1,7 @@
 # Translations template for PROJECT.
-# Copyright (C) 2022 ORGANIZATION
+# Copyright (C) 2023 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
 # 
 # Translators:
 # Mathieu Morey <mathieu@datactivi.st>, 2020
@@ -12,10 +12,10 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-12-12 11:06+0100\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
 "Last-Translator: Jean-Philippe Menotti, 2022\n"
-"Language-Team: French (https://www.transifex.com/aleph/teams/76591/fr/)\n"
+"Language-Team: French (https://app.transifex.com/aleph/teams/76591/fr/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

aleph/translations/messages.pot

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2023-05-11 10:13+0200\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

aleph/translations/nb/LC_MESSAGES/aleph.po

@@ -1,16 +1,16 @@
 # Translations template for PROJECT.
-# Copyright (C) 2022 ORGANIZATION
+# Copyright (C) 2023 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
 # 
 #, fuzzy
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-12-12 11:06+0100\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
-"Language-Team: Norwegian Bokmål (https://www.transifex.com/aleph/teams/76591/nb/)\n"
+"Language-Team: Norwegian Bokmål (https://app.transifex.com/aleph/teams/76591/nb/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"

aleph/translations/nl/LC_MESSAGES/aleph.po

@@ -1,16 +1,16 @@
 # Translations template for PROJECT.
-# Copyright (C) 2022 ORGANIZATION
+# Copyright (C) 2023 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
-# FIRST AUTHOR <EMAIL@ADDRESS>, 2022.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2023.
 # 
 #, fuzzy
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
 "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
-"POT-Creation-Date: 2022-12-12 11:06+0100\n"
+"POT-Creation-Date: 2023-06-23 12:45+0200\n"
 "PO-Revision-Date: 2018-03-15 07:37+0000\n"
-"Language-Team: Dutch (https://www.transifex.com/aleph/teams/76591/nl/)\n"
+"Language-Team: Dutch (https://app.transifex.com/aleph/teams/76591/nl/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"