CV3-47 implement use celery to delete archived data

- configure celery beat
- implement use celery to delete archived data that is over 30 days old
[Delivers CV3-47]

Author: Mireille Niwemuhuza

config.py

@@ -1,4 +1,5 @@
 import os
+from celery.schedules import crontab
 basedir = os.path.abspath(os.path.dirname(__file__))
 
 
@@ -18,6 +19,13 @@ class Config:
     # Celery configuration
     CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL')
     CELERY_RESULT_BACKEND = os.getenv('CELERY_RESULT_BACKEND')
+    CELERY_IMPORTS = ["services.data_deletion.clean_archived_data"]
+    CELERYBEAT_SCHEDULE = {
+        'clean_archived_data': {
+            'task': 'clean_archived_data.delete_archived_data',
+            'schedule': crontab(hour=23, minute=00)
+        },
+    }
 
     @staticmethod
     def init_app(app):

docker/dev/start_redis.sh

@@ -4,4 +4,4 @@
 #done
 cd /app
 export $(cat .env | xargs)
-celery worker -A cworker.celery --loglevel=info
+celery worker -A cworker.celery --loglevel=info & celery beat -A cworker.celery --schedule=/tmp/celerybeat-schedule --loglevel=info --pidfile=/tmp/celerybeat.pid

services/data_deletion/clean_archived_data.py

@@ -0,0 +1,22 @@
+from helpers.database import database_uri
+from sqlalchemy import create_engine, MetaData, and_
+from datetime import timedelta, datetime
+import celery
+
+
+@celery.task(name='clean_archived_data.delete_archived_data')
+def delete_archived_data():
+    database_engine = create_engine(database_uri)
+    metadata = MetaData()
+    metadata.reflect(bind=database_engine)
+    print('deleting archived data...')
+
+    for table in reversed(metadata.sorted_tables):
+        try:
+            now = datetime.now()
+            delta = now - timedelta(days=30)
+            statement = table.delete().where(
+                and_(table.c.date_updated < delta, table.c.state == 'archived'))
+            database_engine.execute(statement)
+        except AttributeError:
+            continue