caches + docs

celery_project/celery.py

@@ -19,6 +19,6 @@
 app.autodiscover_tasks()
 
 
-@app.task(bind=True)
-def debug_task(self):
-    print("Request: {0!r}".format(self.request))
+# @app.task(bind=True)
+# def debug_task(self):
+#     print("Request: {0!r}".format(self.request))

celery_project/settings.py

@@ -136,8 +136,13 @@
     'CELERY_ENABLED': True,
     'MAX_RETRIES': 3,
     'BATCH_SIZE': 20,
+    'BATCH_DELIVERY_TIMEOUT': 30,
     'MESSAGE_ID_ENABLED': True,
     'DEFAULT_PRIORITY': 'medium',
+    'BACKENDS': {
+        'default': 'django.core.mail.backends.smtp.EmailBackend',
+        'ses': 'django_ses.SESBackend'
+    }
 }
 WSGI_APPLICATION = "celery_project.wsgi.application"
 
@@ -193,3 +198,13 @@
         'width': 1000
     }
 }
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': 'redis://127.0.0.1:6379/1',  # Use the appropriate Redis server URL
+        'OPTIONS': {
+            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
+        }
+    }
+}

celery_project/tasks.py

@@ -2,6 +2,6 @@
 
 from celery import shared_task
 
-@shared_task
-def sample(*args, **kwargs):
-    print(f"HELLO {random.randint(1,100)}")
+# @shared_task
+# def sample(*args, **kwargs):
+#     print(f"HELLO {random.randint(1,100)}")

celery_project/tests/test_00_integration.py

@@ -61,7 +61,7 @@ def get_html_message_for_recipient(recipient_email, messages):
         recipient = get_recipients(mid)[0]
         if recipient == recipient_email:
             return get_message(mid)['HTML'].replace('\n', '').replace('\t', '').replace('\r', '').strip()
-    return 
+    return
 
 
 def get_recipients(message_id, type='To'):
@@ -147,7 +147,8 @@ def test_send_many(settings, cleanup_messages, template):
                                       gender='other',
                                       is_blocked=True)
 
-    emails = send_many(recipients=[john, marry, ben], template=template, context={'test_var': 'test_value'}, backend='smtp')
+    emails = send_many(recipients=[john, marry, ben], template=template, context={'test_var': 'test_value'},
+                       backend='smtp')
 
     _send_bulk(emails, uses_multiprocessing=False)
 
@@ -168,8 +169,8 @@ def test_send_many(settings, cleanup_messages, template):
 
     assert sorted(recipients) == sorted([john.email, marry.email])
 
-    assert all([info['Subject'] == 'test_subject' for info in message_infos])
-    assert all([info['Text'] == 'test_content' for info in message_infos])
+    assert sorted([info['Subject'] for info in message_infos]) == sorted(['test_subject', 'DE test_subject'])
+    assert sorted([info['Text'] for info in message_infos]) == sorted(['test_content', 'DE test_content'])
 
     assert (john_msg := get_html_message_for_recipient('john@gmail.com', messages)).count('John') > 0
     assert john_msg.count('Doe') > 0
@@ -185,18 +186,16 @@ def test_send_many(settings, cleanup_messages, template):
 
     placeholder.save()
 
-    emails = send_many(recipients=[john, marry, ben], template=template, context={'test_var': 'test_value'}, backend='smtp')
+    emails = send_many(recipients=[john, marry, ben], template=template, context={'test_var': 'test_value'},
+                       backend='smtp')
 
     _send_bulk(emails, uses_multiprocessing=False)
 
     messages, count = get_all_messages()
     assert count == 4
-    message_id = messages[0]['ID']
-
-    message_info = get_message(message_id)
-
-    assert message_info['HTML'].count('test_value') == 2
-
-    assert message_info['HTML'].count('#test_var#') == 0
 
+    assert get_html_message_for_recipient('john@gmail.com', messages).count('test_value') == 2
+    assert get_html_message_for_recipient('marry@gmail.com', messages).count('test_value') == 1
 
+    assert get_html_message_for_recipient('john@gmail.com', messages).count('#test_var#') == 0
+    assert get_html_message_for_recipient('marry@gmail.com', messages).count('#test_var#') == 0

celery_project/tests/test_01_mail.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import timedelta
 from unittest.mock import patch
 from zoneinfo import ZoneInfo
@@ -170,14 +171,15 @@ def test_send_email(template, recipient):
                        bcc=bcc)
     assert email_model.template.language == 'en'
 
-    with pytest.raises(ValidationError):
-        send(sender=sender,
-             recipients=recipients,
-             template=template,
-             priority='medium',
-             commit=True,
-             context=context,
-             language='es')
+    email = send(sender=sender,
+                 recipients=recipients,
+                 template=template,
+                 priority='medium',
+                 commit=True,
+                 context=context,
+                 language='es')
+
+    assert email.template.language == 'en'
 
     with pytest.raises(ValidationError):
         nv_recipients = [*recipients, 'not_valid']
@@ -432,6 +434,57 @@ def test_send_many(template):
     assert list(emails[0].attachments.values_list('name', flat=True)) == ['test.txt', 'new.txt']
 
 
+@pytest.mark.django_db
+def test_internalization(caplog, template):
+    en_recipient = EmailAddress.objects.create(
+        email='en@gmail.com',
+        first_name='EN',
+        preferred_language='en'
+    )
+
+    de_recipient = EmailAddress.objects.create(
+        email='de@gmail.com',
+        first_name='DE',
+        preferred_language='de'
+    )
+
+    ua_recipient = EmailAddress.objects.create(
+        email='ua@gmail.com',
+        first_name='UA',
+        preferred_language='ua'
+    )
+
+    nullable_recipient = EmailAddress.objects.create(
+        email='nullable@gmail.com',
+        first_name='NL',
+    )
+
+    with caplog.at_level(logging.WARNING):
+        emails = send_many(
+            recipients=[en_recipient, de_recipient, ua_recipient, nullable_recipient],
+            template=template)
+
+        assert 'Language "ua" is not found in LANGUAGES configuration.' in caplog.text
+
+    assert len(emails) == 4
+    assert emails[0].template.language == 'en'
+    assert emails[1].template.language == 'de'
+    assert emails[2].template.language == 'en'
+    assert emails[3].template.language == 'en'
+
+    caplog.clear()
+
+    with caplog.at_level(logging.WARNING):
+        emails = send_many(
+            recipients=[en_recipient, de_recipient, ua_recipient, nullable_recipient],
+            template=template,
+            language='en')
+
+        assert 'Language "ua" is not found in LANGUAGES configuration.' not in caplog.text
+
+    assert all([email.template.language == 'en' for email in emails])
+
+
 def test_split_batches(settings):
     settings.POST_OFFICE['BATCH_SIZE'] = 2
     assert split_into_batches([1, 2, 3, 4, 5, 6, 7]) == [[1, 2], [3, 4], [5, 6], [7]]
@@ -524,7 +577,3 @@ def test_errors(settings, template):
     with pytest.raises(InterfaceError):
         # Connection already closed
         _send_bulk([email], uses_multiprocessing=True)
-
-
-
-

celery_project/views.py

@@ -4,7 +4,6 @@
 
 from django.template import loader
 
-from celery_project.tasks import sample
 from django.http import HttpResponse
 from post_office import mail
 from post_office.models import EmailMergeModel, EmailAddress
@@ -97,7 +96,6 @@ def send_many(request):
             sender='Mykhailo.Poienko@uibk.ac.at',
             template='nice_email',
             context={'id': 228, 'shirts': 100, 'all': 10, 'shoes': 75},
-            language='en',
             attachments={'new_test.txt': f},
         )
     return redirect('home')

docs/source/celery.rst

@@ -0,0 +1,76 @@
+Integration with Celery
+===============================
+
+If your project runs in a Celery enabled environment, you can use its worker to send out queued emails.
+This setup has a big advantage that emails are sent immediately after they are added to the queue.
+The delivery is performed asynchronously in a separate task to prevent blocking request/response-cycle.
+
+.. warning::
+    Current version of post_office uses Django ORM ``select_for_update(skip_locked=True)`` method in celery task
+    for locking sent emails. Not all database backends support it.
+
+    "Using select_for_update() on backends which do not support SELECT ... FOR UPDATE (such as SQLite) will have no effect.
+    SELECT ... FOR UPDATE will not be added to the query, and an error isn’t raised if select_for_update() is used in autocommit mode."
+    (read more in `Django QuerySet documentation <https://docs.djangoproject.com/en/5.1/ref/models/querysets/>`_)
+
+You should `configure celery <https://docs.celeryq.dev/en/latest/userguide/application.html>`_ so that you ``celery.py``
+setup invokes `autodiscover_tasks <https://docs.celeryq.dev/en/latest/reference/celery.html#celery.Celery.autodiscover_tasks>`_
+
+Celery must also be enabled in post_office configurations in ``settings.py``:
+
+.. code-block:: python
+
+    POST_OFFICE = {
+    # other settings
+    'CELERY_ENABLED': True,
+    }
+
+Now you can start celery worker:
+
+.. code-block::
+
+    python -m celery -A your_project worker -l info --concurrency=5
+
+Adjust number of concurrent processes to meet your needs.
+
+You should see something like this:
+
+.. code-block::
+
+         -------------- celery@mykhailo-Latitude-5540 v5.4.0 (opalescent)
+    --- ***** -----
+    -- ******* ---- Linux-6.8.0-45-generic-x86_64-with-glibc2.39 2024-10-08 13:36:49
+    - *** --- * ---
+    - ** ---------- [config]
+    - ** ---------- .> app:         celery_project:0x7e00fa50c710
+    - ** ---------- .> transport:   redis://localhost:6379//
+    - ** ---------- .> results:     redis://localhost:6379/
+    - *** --- * --- .> concurrency: 5 (prefork)
+    -- ******* ---- .> task events: OFF (enable -E to monitor tasks in this worker)
+    --- ***** -----
+     -------------- [queues]
+                    .> celery           exchange=celery(direct) key=celery
+
+
+    [tasks]
+      . post_office.tasks.cleanup_mail
+      . post_office.tasks.send_queued_mail
+
+In case of a temporary delivery failure, we might want retrying to send those emails by a periodic task.
+This can be scheduled with a simple `Celery beat configuration <https://docs.celeryq.dev/en/latest/userguide/periodic-tasks.html#entries>`_,
+for instance through
+
+.. code-block:: python
+
+    app.conf.beat_schedule = {
+    'send-queued-mail': {
+        'task': 'post_office.tasks.send_queued_mail',
+        'schedule': 600.0,
+        },
+    }
+
+The email queue now will be processed every 10 minutes.
+If you are using `Django Celery Beat <https://django-celery-beat.readthedocs.io/en/latest/>`_, then use the Django-Admin backend and add a periodic tasks for ``post_office.tasks.send_queued_mail``.
+
+Depending on your policy, you may also want to remove expired emails from the queue.
+This can be done by adding another periodic tasks for ``post_office.tasks.cleanup_mail``, which may run once a week or month.

docs/source/conf.py

@@ -18,7 +18,7 @@
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
-extensions = []
+extensions = ["sphinx.ext.autosectionlabel"]
 
 templates_path = ['_templates']
 exclude_patterns = []

docs/source/index.rst

@@ -32,5 +32,6 @@ post_office provides a set of powerful features, such as:
    settings
    celery
    uwsgi
+   signals