Merge pull request #36 from RudolfCardinal/email-text-extraction

Support text extraction from eml files and other text extraction improvements

Author: martinburchell

.github/scripts/install_test_python_packages.sh

@@ -10,3 +10,4 @@ ${PYTHON} -m pip install xlrd
 ${PYTHON} -m pip install dogpile.cache==0.9.2  # Later versions incompatible
 ${PYTHON} -m pip install pytest
 ${PYTHON} -m pip install xhtml2pdf weasyprint pdfkit  # For PDF tests
+${PYTHON} -m pip install faker==13.3.1 faker-file'[common]'==0.18.3

.pre-commit-config.yaml

@@ -18,10 +18,14 @@ repos:
     rev: 5.0.4
     hooks:
     -   id: flake8
+        additional_dependencies:
+        - flake8-builtins==2.5.0
 -   repo: https://github.com/asottile/yesqa
     rev: v1.5.0
     hooks:
     -   id: yesqa
+        additional_dependencies:
+        - flake8-builtins==2.5.0
 -   repo: https://github.com/pre-commit/pygrep-hooks
     rev: v1.9.0
     hooks:

cardinal_pythonlib/bulk_email/main.py

@@ -57,7 +57,7 @@
     Recipient,
     SendAttempt,
 )
-from cardinal_pythonlib.email.sendmail import (
+from cardinal_pythonlib.email_utils.sendmail import (
     CONTENT_TYPE_HTML,
     CONTENT_TYPE_TEXT,
     is_email_valid,

cardinal_pythonlib/bulk_email/models.py

@@ -63,7 +63,7 @@
     USERNAME_MAX_LENGTH,
 )
 from cardinal_pythonlib.colander_utils import EMAIL_ADDRESS_MAX_LEN
-from cardinal_pythonlib.email.sendmail import (
+from cardinal_pythonlib.email_utils.sendmail import (
     ASCII,
     CONTENT_TYPE_TEXT,
     is_email_valid,

cardinal_pythonlib/django/fields/jsonclassfield.py

@@ -130,7 +130,7 @@ def my_decoder_hook(d: Dict) -> Any:
 # noinspection PyUnresolvedReferences
 from django.db.models import TextField
 
-from cardinal_pythonlib.json.serialize import json_decode, json_encode
+from cardinal_pythonlib.json_utils.serialize import json_decode, json_encode
 
 
 # =============================================================================

cardinal_pythonlib/django/function_cache.py

@@ -36,7 +36,7 @@
 from django.core.cache import cache  # default cache
 
 from cardinal_pythonlib.logs import get_brace_style_log_with_null_handler
-from cardinal_pythonlib.json.serialize import json_encode
+from cardinal_pythonlib.json_utils.serialize import json_encode
 
 log = get_brace_style_log_with_null_handler(__name__)
 

cardinal_pythonlib/django/middleware.py

@@ -28,7 +28,7 @@
 
 import logging
 import os
-from re import compile
+import re
 import sys
 from typing import Optional
 
@@ -107,9 +107,9 @@ def process_exception(
 Modified according to: https://djangosnippets.org/snippets/2845/
 """
 
-# EXEMPT_URLS = [compile(settings.LOGIN_URL.lstrip('/'))]
+# EXEMPT_URLS = [re.compile(settings.LOGIN_URL.lstrip('/'))]
 # if hasattr(settings, 'LOGIN_EXEMPT_URLS'):
-#     EXEMPT_URLS += [compile(expr) for expr in settings.LOGIN_EXEMPT_URLS]
+#     EXEMPT_URLS += [re.compile(expr) for expr in settings.LOGIN_EXEMPT_URLS]
 #
 #
 # class LoginRequiredMiddleware:
@@ -166,10 +166,10 @@ def process_exception(
 # 3. RNC; composite of those patterns.
 # -----------------------------------------------------------------------------
 
-EXEMPT_URLS = [compile(settings.LOGIN_URL.lstrip("/"))]
+EXEMPT_URLS = [re.compile(settings.LOGIN_URL.lstrip("/"))]
 if hasattr(settings, "LOGIN_EXEMPT_URLS"):
     EXEMPT_URLS += [
-        compile(expr.lstrip("/")) for expr in settings.LOGIN_EXEMPT_URLS
+        re.compile(expr.lstrip("/")) for expr in settings.LOGIN_EXEMPT_URLS
     ]
 
 

cardinal_pythonlib/email/__init__.py …rdinal_pythonlib/email_utils/__init__.pycardinal_pythonlib/email/__init__.py renamed to cardinal_pythonlib/email_utils/__init__.py cardinal_pythonlib/email/__init__.py renamed to cardinal_pythonlib/email_utils/__init__.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# cardinal_pythonlib/email/__init__.py
+# cardinal_pythonlib/email_utils/__init__.py
 
 """
 ===============================================================================

cardinal_pythonlib/email/mailboxpurge.py …al_pythonlib/email_utils/mailboxpurge.pycardinal_pythonlib/email/mailboxpurge.py renamed to cardinal_pythonlib/email_utils/mailboxpurge.py cardinal_pythonlib/email/mailboxpurge.py renamed to cardinal_pythonlib/email_utils/mailboxpurge.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# cardinal_pythonlib/email/mailboxpurge.py
+# cardinal_pythonlib/email_utils/mailboxpurge.py
 
 """
 Remove all binary attachments from email messages

cardinal_pythonlib/email/sendmail.py …rdinal_pythonlib/email_utils/sendmail.pycardinal_pythonlib/email/sendmail.py renamed to cardinal_pythonlib/email_utils/sendmail.py cardinal_pythonlib/email/sendmail.py renamed to cardinal_pythonlib/email_utils/sendmail.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# cardinal_pythonlib/email/sendmail.py
+# cardinal_pythonlib/email_utils/sendmail.py
 
 """
 ===============================================================================