From 84fd6f620cb3cd265d35339f8fd0a04ba8e7f846 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 15:26:52 +0000 Subject: [PATCH 01/11] Bump followthemoney from 3.5.8 to 3.5.9 Bumps [followthemoney](https://github.com/alephdata/followthemoney) from 3.5.8 to 3.5.9. - [Release notes](https://github.com/alephdata/followthemoney/releases) - [Commits](https://github.com/alephdata/followthemoney/compare/v3.5.8...v3.5.9) --- updated-dependencies: - dependency-name: followthemoney dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c73e9997..fe6019911 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ banal==1.0.6 normality==2.5.0 pantomime==0.6.1 -followthemoney==3.5.8 +followthemoney==3.5.9 followthemoney-store[postgresql]==3.0.6 servicelayer[google,amazon]==1.22.1 languagecodes==1.1.1 From 74f91a15514fde81cb0b88baa7cffd840bdd1f33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 15:22:56 +0000 Subject: [PATCH 02/11] Bump google-cloud-vision from 3.5.0 to 3.7.2 Bumps [google-cloud-vision](https://github.com/googleapis/google-cloud-python) from 3.5.0 to 3.7.2. - [Release notes](https://github.com/googleapis/google-cloud-python/releases) - [Changelog](https://github.com/googleapis/google-cloud-python/blob/main/packages/google-cloud-documentai/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-cloud-python/compare/google-cloud-vision-v3.5.0...google-cloud-vision-v3.7.2) --- updated-dependencies: - dependency-name: google-cloud-vision dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c73e9997..c1796ebcb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ servicelayer[google,amazon]==1.22.1 languagecodes==1.1.1 countrytagger==0.1.2 pyicu==2.12 -google-cloud-vision==3.5.0 +google-cloud-vision==3.7.2 tesserocr==2.6.2 spacy==3.6.1 fingerprints==1.1.1 From b12e3e840175a9b140ece2e5f98c3b7ce352bb1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 15:22:59 +0000 Subject: [PATCH 03/11] Bump icalendar from 5.0.11 to 5.0.12 Bumps [icalendar](https://github.com/collective/icalendar) from 5.0.11 to 5.0.12. - [Release notes](https://github.com/collective/icalendar/releases) - [Changelog](https://github.com/collective/icalendar/blob/master/CHANGES.rst) - [Commits](https://github.com/collective/icalendar/compare/v5.0.11...v5.0.12) --- updated-dependencies: - dependency-name: icalendar dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c73e9997..3015cbb66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ olefile==0.47 Pillow==10.1.0 vobject==0.9.6.1 msglite==0.30.0 -icalendar==5.0.11 +icalendar==5.0.12 cryptography==41.0.7 requests[security]==2.31.0 From dff0438d2d156339730aee09ba0a901e6b1b392c Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Mon, 15 Apr 2024 16:26:25 +0200 Subject: [PATCH 04/11] feat: Introduce a setting to disable sending ProcessingExceptions to Sentry --- ingestors/manager.py | 3 ++- ingestors/settings.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ingestors/manager.py b/ingestors/manager.py index fb5cc126f..59cb35d1d 100644 --- a/ingestors/manager.py +++ b/ingestors/manager.py @@ -219,7 +219,8 @@ def ingest(self, file_path, entity, **kwargs): log.exception(f"[{repr(entity)}] Failed to process: {pexc}") INGESTIONS_FAILED.labels(ingestor=ingestor_name).inc() entity.set("processingError", stringify(pexc)) - capture_exception(pexc) + if settings.SENTRY_CAPTURE_PROCESSING_EXCEPTIONS: + capture_exception(pexc) finally: self.finalize(entity) diff --git a/ingestors/settings.py b/ingestors/settings.py index 4768e91e9..c239de456 100644 --- a/ingestors/settings.py +++ b/ingestors/settings.py @@ -50,3 +50,10 @@ # Also store cached values in the SQL database sls.TAGS_DATABASE_URI = fts.DATABASE_URI + +# ProcessingException is thrown whenever something goes wrong wiht +# parsing a file. Enable this with care, it can easily eat up the +# Sentry quota of events. +SENTRY_CAPTURE_PROCESSING_EXCEPTIONS = env.to_bool( + "SENTRY_CAPTURE_PROCESSING_EXCEPTIONS", False +) From c6c59289661e14d92c5e8da9701612118da715da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 15:16:20 +0000 Subject: [PATCH 05/11] Bump sentry-sdk from 1.39.1 to 2.0.1 Bumps [sentry-sdk](https://github.com/getsentry/sentry-python) from 1.39.1 to 2.0.1. - [Release notes](https://github.com/getsentry/sentry-python/releases) - [Changelog](https://github.com/getsentry/sentry-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-python/compare/1.39.1...2.0.1) --- updated-dependencies: - dependency-name: sentry-sdk dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c73e9997..42eb26126 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,4 +38,4 @@ cryptography==41.0.7 requests[security]==2.31.0 pymupdf==1.21.1 -sentry_sdk==1.39.1 +sentry_sdk==2.0.1 From 0fb565de4c8e426ab92a2a0f377a6ff4794a1fdc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 15:16:23 +0000 Subject: [PATCH 06/11] Bump rarfile from 4.1 to 4.2 Bumps [rarfile](https://github.com/markokr/rarfile) from 4.1 to 4.2. - [Release notes](https://github.com/markokr/rarfile/releases) - [Changelog](https://github.com/markokr/rarfile/blob/master/doc/news.rst) - [Commits](https://github.com/markokr/rarfile/compare/v4.1...v4.2) --- updated-dependencies: - dependency-name: rarfile dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c73e9997..bc187a58d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ click==8.1.6 dbf==0.99.9 pymediainfo==6.1.0 python-magic==0.4.27 -rarfile==4.1 +rarfile==4.2 xlrd==2.0.1 openpyxl==3.1.2 odfpy==1.4.1 From 8f3c65f0ec8d79adcc6742858f59700a3035d000 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 15:16:36 +0000 Subject: [PATCH 07/11] Bump servicelayer[amazon,google] from 1.22.1 to 1.22.2 Bumps [servicelayer[amazon,google]](https://github.com/alephdata/servicelayer) from 1.22.1 to 1.22.2. - [Release notes](https://github.com/alephdata/servicelayer/releases) - [Commits](https://github.com/alephdata/servicelayer/compare/v1.22.1...v1.22.2) --- updated-dependencies: - dependency-name: servicelayer[amazon,google] dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c73e9997..8f3de4006 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ normality==2.5.0 pantomime==0.6.1 followthemoney==3.5.8 followthemoney-store[postgresql]==3.0.6 -servicelayer[google,amazon]==1.22.1 +servicelayer[google,amazon]==1.22.2 languagecodes==1.1.1 countrytagger==0.1.2 pyicu==2.12 From cc4ac7c038b7ba63d5359b9fb697502b59bfaab1 Mon Sep 17 00:00:00 2001 From: Christian Stefanescu Date: Fri, 3 May 2024 11:53:51 +0300 Subject: [PATCH 08/11] Fix formatting --- ingestors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ingestors/__init__.py b/ingestors/__init__.py index 9c0e68517..c96f6acc8 100644 --- a/ingestors/__init__.py +++ b/ingestors/__init__.py @@ -1,4 +1,5 @@ """Provides a set of ingestors based on different file types.""" + import logging __version__ = "3.21.0-rc1" From 6b48037c4523abdff18539c034b7ff3a949d6a16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 09:10:27 +0000 Subject: [PATCH 09/11] Bump the dev-dependencies group with 4 updates Bumps the dev-dependencies group with 4 updates: [pytest](https://github.com/pytest-dev/pytest), [pytest-cov](https://github.com/pytest-dev/pytest-cov), [black](https://github.com/psf/black) and [ruff](https://github.com/astral-sh/ruff). Updates `pytest` from 7.4.4 to 8.2.0 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.4...8.2.0) Updates `pytest-cov` from 4.1.0 to 5.0.0 - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v4.1.0...v5.0.0) Updates `black` from 23.12.1 to 24.4.2 - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.12.1...24.4.2) Updates `ruff` from 0.1.9 to 0.4.2 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/v0.1.9...v0.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major dependency-group: dev-dependencies - dependency-name: pytest-cov dependency-type: direct:production update-type: version-update:semver-major dependency-group: dev-dependencies - dependency-name: black dependency-type: direct:development update-type: version-update:semver-major dependency-group: dev-dependencies - dependency-name: ruff dependency-type: direct:development update-type: version-update:semver-minor dependency-group: dev-dependencies ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 4 ++-- requirements.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6940d3799..643ceda81 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ bump2version==1.0.1 -black==23.12.1 -ruff==0.1.9 +black==24.4.2 +ruff==0.4.2 diff --git a/requirements.txt b/requirements.txt index 3c73e9997..5e60a2d7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,8 +14,8 @@ fingerprints==1.1.1 fasttext==0.9.2 # Development -pytest==7.4.4 -pytest-cov==4.1.0 +pytest==8.2.0 +pytest-cov==5.0.0 click==8.1.6 # File format support From d635426025596330b7f915cef2c4b69945379aed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 10:28:33 +0000 Subject: [PATCH 10/11] Bump followthemoney-store[postgresql] from 3.0.6 to 3.1.0 Bumps [followthemoney-store[postgresql]](https://github.com/alephdata/followthemoney-store) from 3.0.6 to 3.1.0. - [Release notes](https://github.com/alephdata/followthemoney-store/releases) - [Commits](https://github.com/alephdata/followthemoney-store/compare/3.0.6...3.1.0) --- updated-dependencies: - dependency-name: followthemoney-store[postgresql] dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2806ebbd2..932e2f14f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ banal==1.0.6 normality==2.5.0 pantomime==0.6.1 followthemoney==3.5.8 -followthemoney-store[postgresql]==3.0.6 +followthemoney-store[postgresql]==3.1.0 servicelayer[google,amazon]==1.22.2 languagecodes==1.1.1 countrytagger==0.1.2 From a324fccab60ccfe1c8c1bed529e2c050bd6a82b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 10:34:51 +0000 Subject: [PATCH 11/11] Bump click from 8.1.6 to 8.1.7 Bumps [click](https://github.com/pallets/click) from 8.1.6 to 8.1.7. - [Release notes](https://github.com/pallets/click/releases) - [Changelog](https://github.com/pallets/click/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/click/compare/8.1.6...8.1.7) --- updated-dependencies: - dependency-name: click dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a9e450a27..740043bae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ fasttext==0.9.2 # Development pytest==8.2.0 pytest-cov==5.0.0 -click==8.1.6 +click==8.1.7 # File format support dbf==0.99.9