Feature/update-celery-config

CHANGELOG.md

@@ -36,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     - `use`: Point your active configuration to a new configuration file
 - The `merlin server` command no longer modifies the `~/.merlin/app.yaml` file by default. Instead, it modifies the `./merlin_server/app.yaml` file.
 - Dropped support for Python 3.7
+- Celery settings have been updated to try to improve resiliency
 - Ported all distributed tests of the integration test suite to pytest
   - There is now a `commands/` directory and a `workflows/` directory under the integration suite to house these tests
   - Removed the "Distributed-tests" GitHub action as these tests will now be run under "Integration-tests"

docs/user_guide/configuration/index.md

@@ -79,8 +79,8 @@ Merlin's default Celery configurations are as follows:
     broker_read_url: None
     broker_write_url: None
     broker_transport: None
-    broker_transport_options: {'visibility_timeout': 86400, 'max_connections': 100}
-    broker_connection_timeout: 4
+    broker_transport_options: {'visibility_timeout': 86400, 'max_connections': 100, 'socket_timeout': 300, 'retry_policy': {'timeout': 600}}
+    broker_connection_timeout: 60
     broker_connection_retry: True
     broker_connection_retry_on_startup: None
     broker_connection_max_retries: 100
@@ -89,6 +89,7 @@ Merlin's default Celery configurations are as follows:
     broker_heartbeat: 120
     broker_heartbeat_checkrate: 3.0
     broker_login_method: None
+    broker_native_delayed_delivery_queue_type: quorum
     broker_pool_limit: 0
     broker_use_ssl: <set in the broker section of app.yaml>
     broker_host: <set in the broker section of app.yaml>
@@ -122,6 +123,10 @@ Merlin's default Celery configurations are as follows:
     azureblockblob_base_path: 
     azureblockblob_connection_timeout: 20
     azureblockblob_read_timeout: 120
+    gcs_bucket: None
+    gcs_project: None
+    gcs_base_path: 
+    gcs_ttl: 0
     control_queue_ttl: 300.0
     control_queue_expires: 10.0
     control_exchange: celery  # DO NOT MODIFY
@@ -145,10 +150,10 @@ Merlin's default Celery configurations are as follows:
     redis_username: <set in results_backend section of app.yaml>
     redis_password: <set in results_backend section of app.yaml>
     redis_port: <set in results_backend section of app.yaml>
-    redis_socket_timeout: 120.0
-    redis_socket_connect_timeout: None
-    redis_retry_on_timeout: False
-    redis_socket_keepalive: False
+    redis_socket_timeout: 300
+    redis_socket_connect_timeout: 300
+    redis_retry_on_timeout: True
+    redis_socket_keepalive: True
     result_backend: <set in results_backend section of app.yaml>
     result_cache_max: -1
     result_compression: None
@@ -162,9 +167,9 @@ Merlin's default Celery configurations are as follows:
     result_chord_retry_interval: 1.0
     result_chord_join_timeout: 3.0
     result_backend_max_sleep_between_retries_ms: 10000
-    result_backend_max_retries: inf
+    result_backend_max_retries: 20
     result_backend_base_sleep_between_retries_ms: 10
-    result_backend_always_retry: False
+    result_backend_always_retry: True
     elasticsearch_retry_on_timeout: None
     elasticsearch_max_retries: None
     elasticsearch_timeout: None
@@ -179,6 +184,7 @@ Merlin's default Celery configurations are as follows:
     database_short_lived_sessions: False
     database_table_schemas: None
     database_table_names: None
+    database_create_tables_at_setup: True
     task_acks_late: True  # DO NOT MODIFY
     task_acks_on_failure_or_timeout: True  # DO NOT MODIFY
     task_always_eager: False  # DO NOT MODIFY
@@ -198,7 +204,7 @@ Merlin's default Celery configurations are as follows:
     task_store_eager_result: False
     task_protocol: 2  # DO NOT MODIFY
     task_publish_retry: True
-    task_publish_retry_policy: {'interval_start': 10, 'interval_step': 10, 'interval_max': 60}
+    task_publish_retry_policy: {'interval_start': 10, 'interval_step': 10, 'interval_max': 300}
     task_queues: None  # DO NOT MODIFY
     task_queue_max_priority: 10  # DO NOT MODIFY
     task_reject_on_worker_lost: True  # DO NOT MODIFY
@@ -214,6 +220,8 @@ Merlin's default Celery configurations are as follows:
     worker_agent: None  # DO NOT MODIFY
     worker_autoscaler: celery.worker.autoscale:Autoscaler  # DO NOT MODIFY
     worker_cancel_long_running_tasks_on_connection_loss: True
+    worker_soft_shutdown_timeout: 0.0
+    worker_enable_soft_shutdown_on_idle: False
     worker_concurrency: None  # DO NOT MODIFY; this will be set on a worker-by-worker basis that you can customize in your spec file
     worker_consumer: celery.worker.consumer:Consumer  # DO NOT MODIFY
     worker_direct: False  # DO NOT MODIFY
@@ -238,6 +246,7 @@ Merlin's default Celery configurations are as follows:
     worker_task_log_format: [%(asctime)s: %(levelname)s] [%(task_name)s(%(task_id)s)] %(message)s
     worker_timer: None
     worker_timer_precision: 1.0
+    worker_detect_quorum_queues: True
     deprecated_settings: set()
     visibility_timeout: 86400
     ```

merlin/config/celeryconfig.py

@@ -10,23 +10,40 @@
 
 
 DICT = {
+    # -------- SERIALIZER SETTINGS --------
     "task_serializer": "pickle",
     "accept_content": ["pickle"],
     "result_serializer": "pickle",
+    # ----------- TASK SETTINGS -----------
     "task_acks_late": True,
     "task_reject_on_worker_lost": True,
     "task_publish_retry_policy": {
         "interval_start": 10,
         "interval_step": 10,
-        "interval_max": 60,
+        "interval_max": 300,
     },
-    "redis_max_connections": 100000,
+    "task_default_queue": "merlin",
+    # ---------- BROKER SETTINGS ----------
     "broker_transport_options": {
         "visibility_timeout": 60 * 60 * 24,
         "max_connections": 100,
+        "socket_timeout": 300,
+        "retry_policy": {
+            "timeout": 600,
+        },
     },
+    "broker_connection_timeout": 60,
     "broker_pool_limit": 0,
-    "task_default_queue": "merlin",
+    # --------- BACKEND SETTINGS ----------
+    "result_backend_always_retry": True,
+    "result_backend_max_retries": 20,
+    # ---------- REDIS SETTINGS -----------
+    "redis_max_connections": 100000,
+    "redis_retry_on_timeout": True,
+    "redis_socket_connect_timeout": 300,
+    "redis_socket_timeout": 300,
+    "redis_socket_keepalive": True,
+    # ---------- WORKER SETTINGS ----------
     "worker_log_color": True,
     "worker_log_format": FORMATS["DEFAULT"],
     "worker_task_log_format": FORMATS["WORKER"],