Add Document Analysis Row

.github/workflows/backend.yml

@@ -89,26 +89,26 @@ jobs:
           echo "$ci_env"
 
       - name: Build the Stack
-        run:  docker-compose -f test.yml build
+        run:  docker compose -f test.yml build
 
       - name: Run DB Migrations
-        run:  docker-compose -f test.yml run --rm django python manage.py migrate
+        run:  docker compose -f test.yml run --rm django python manage.py migrate
 
       - name: Collect Static Files
-        run: docker-compose -f test.yml run --rm django python manage.py collectstatic
+        run: docker compose -f test.yml run --rm django python manage.py collectstatic
 
       - name: Verify Docker Containers
         run: |
-          docker-compose -f test.yml ps
+          docker compose -f test.yml ps
 
       - name: Inspect Docker Network
         run: |
-          docker network inspect $(docker-compose -f test.yml ps -q | xargs docker inspect --format='{{range .NetworkSettings.Networks}}{{.NetworkID}}{{end}}' | uniq)
+          docker network inspect $(docker compose -f test.yml ps -q | xargs docker inspect --format='{{range .NetworkSettings.Networks}}{{.NetworkID}}{{end}}' | uniq)
 
       - name: Capture Docker Compose Logs
         if: failure()
         run: |
-          docker-compose -f test.yml logs --no-color > docker-compose-logs.txt
+          docker compose -f test.yml logs --no-color > docker-compose-logs.txt
 
       - name: Upload Docker Compose Logs
         if: failure()
@@ -119,7 +119,7 @@ jobs:
 
       - name: Build Pytest Coverage File
         run: |
-          docker-compose -f test.yml run django coverage run -m pytest --cov-report=xml --cov
+          docker compose -f test.yml run django coverage run -m pytest --cov-report=xml --cov
 
       - name: Upload Coverage Reports to Codecov
         env:
@@ -128,7 +128,7 @@ jobs:
         run:  |
               # use bash variable expression to get the substring
               ci_env=`bash <(curl -s https://codecov.io/env)`
-              docker-compose -f test.yml run $ci_env django /bin/codecov -v -t ${CODECOV_TOKEN} -R . -f coverage.xml -C ${COMMIT_SHA}
+              docker compose -f test.yml run $ci_env django /bin/codecov -v -t ${CODECOV_TOKEN} -R . -f coverage.xml -C ${COMMIT_SHA}
 
       - name: Tear down the Stack
-        run:  docker-compose -f test.yml down
+        run:  docker compose -f test.yml down

.gitignore

@@ -171,3 +171,4 @@ GitHub.sublime-settings
 # File directories
 /staticfiles/
 /uploadfiles/
+/user_*/

config/graphql/mutations.py

@@ -1408,13 +1408,16 @@ def mutate(root, info, corpus_id, analyzer_id):
         try:
 
             corpus_pk = from_global_id(corpus_id)[1]
+            analyzer_pk = from_global_id(analyzer_id)[1]
 
             obj = Analysis.objects.create(
-                analyzer_id=analyzer_id,
+                analyzer_id=analyzer_pk,
                 analyzed_corpus_id=corpus_pk,
                 creator=info.context.user,
             )
 
+            logger.info(f"StartCorpusAnalysisMutation - retrieved analysis: {obj}")
+
             transaction.on_commit(
                 lambda: start_analysis.s(
                     analysis_id=obj.id,

config/settings/base.py

@@ -395,7 +395,7 @@
     # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-timezone
     CELERY_TIMEZONE = TIME_ZONE
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_url
-CELERY_BROKER_URL = env("CELERY_BROKER_URL")
+CELERY_BROKER_URL = env("REDIS_URL")
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-result_backend
 CELERY_RESULT_BACKEND = CELERY_BROKER_URL
 # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-accept_content
@@ -415,6 +415,7 @@
 CELERY_WORKER_MAX_MEMORY_PER_CHILD = 14240000  # 14 GB (thousands of kilobytes)
 CELERY_MAX_TASKS_PER_CHILD = 4
 CELERY_PREFETCH_MULTIPLIER = 1
+CELERY_RESULT_BACKEND_MAX_RETRIES = 10
 # django-rest-framework
 # -------------------------------------------------------------------------------
 # django-rest-framework - https://www.django-rest-framework.org/api-guide/settings/

config/settings/local.py

@@ -43,9 +43,12 @@
 
 # Celery
 # ------------------------------------------------------------------------------
-
-# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-eager-propagates
-CELERY_TASK_EAGER_PROPAGATES = True
+CELERY_TASK_EAGER_PROPAGATES = (
+    True  # If this is True, eagerly executed tasks will propagate exceptions
+)
+CELERY_BROKER_URL = "memory://"
+CELERY_RESULT_BACKEND = "cache"
+CELERY_CACHE_BACKEND = "memory"
 
 # SECURITY
 # ------------------------------------------------------------------------------

config/settings/production.py

@@ -27,7 +27,7 @@
         "LOCATION": env("REDIS_URL"),
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            # Mimicing memcache behavior.
+            # Mimicking memcache behavior.
             # https://github.com/jazzband/django-redis#memcached-exceptions-behavior
             "IGNORE_EXCEPTIONS": True,
         },

config/settings/test.py

@@ -37,5 +37,15 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
 EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
 
-# Your stuff...
+# Celery
 # ------------------------------------------------------------------------------
+CELERY_TASK_ALWAYS_EAGER = True
+CELERY_TASK_EAGER_PROPAGATES = True
+CELERY_BROKER_URL = "memory://"
+CELERY_RESULT_BACKEND = "cache"
+CELERY_CACHE_BACKEND = "memory"
+
+# Ensure these settings are applied
+print("Test settings loaded. CELERY_TASK_ALWAYS_EAGER:", CELERY_TASK_ALWAYS_EAGER)
+print("CELERY_BROKER_URL:", CELERY_BROKER_URL)
+print("CELERY_RESULT_BACKEND:", CELERY_RESULT_BACKEND)

opencontractserver/analyzer/migrations/0003_alter_analysis_user_lock_alter_analyzer_user_lock_and_more.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.2.14 on 2024-07-30 04:29
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("analyzer", "0002_initial"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="analysis",
+            name="user_lock",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="locked_%(class)s_objects",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="analyzer",
+            name="user_lock",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="locked_%(class)s_objects",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="gremlinengine",
+            name="user_lock",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="locked_%(class)s_objects",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

opencontractserver/analyzer/migrations/0004_analyzer_task_name_alter_analyzer_host_gremlin.py

@@ -0,0 +1,34 @@
+# Generated by Django 4.2.14 on 2024-08-02 04:56
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzer", "0003_alter_analysis_user_lock_alter_analyzer_user_lock_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="analyzer",
+            name="task_name",
+            field=models.CharField(
+                blank=True,
+                default="opencontractserver.tasks.data_extract_tasks.oc_llama_index_doc_query",
+                max_length=1024,
+                null=True,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="analyzer",
+            name="host_gremlin",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="analyzer.gremlinengine",
+            ),
+        ),
+    ]

opencontractserver/analyzer/migrations/0005_analyzer_one_field_null_constraint_and_more.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.2.14 on 2024-08-02 05:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzer", "0004_analyzer_task_name_alter_analyzer_host_gremlin"),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name="analyzer",
+            constraint=models.CheckConstraint(
+                check=models.Q(
+                    models.Q(
+                        ("host_gremlin__isnull", True), ("task_name__isnull", False)
+                    ),
+                    models.Q(
+                        ("host_gremlin__isnull", False), ("task_name__isnull", True)
+                    ),
+                    _connector="OR",
+                ),
+                name="one_field_null_constraint",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="analyzer",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("host_gremlin__isnull", False)),
+                fields=("host_gremlin",),
+                name="unique_host_gremlin_if_not_null",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="analyzer",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("task_name__isnull", False)),
+                fields=("task_name",),
+                name="unique_task_name_if_not_null",
+            ),
+        ),
+    ]

opencontractserver/analyzer/migrations/0006_alter_analyzer_task_name.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.14 on 2024-08-02 05:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("analyzer", "0005_analyzer_one_field_null_constraint_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="analyzer",
+            name="task_name",
+            field=models.CharField(
+                blank=True, default=None, max_length=1024, null=True
+            ),
+        ),
+    ]

opencontractserver/analyzer/models.py

@@ -80,21 +80,55 @@ class Meta:
             ("update_analyzer", "update analyzer"),
             ("remove_analyzer", "delete analyzer"),
         )
+        constraints = [
+            django.db.models.CheckConstraint(
+                check=(
+                    django.db.models.Q(
+                        host_gremlin__isnull=True, task_name__isnull=False
+                    )
+                    | django.db.models.Q(
+                        host_gremlin__isnull=False, task_name__isnull=True
+                    )
+                ),
+                name="one_field_null_constraint",
+            ),
+            django.db.models.UniqueConstraint(
+                fields=["host_gremlin"],
+                condition=django.db.models.Q(host_gremlin__isnull=False),
+                name="unique_host_gremlin_if_not_null",
+            ),
+            django.db.models.UniqueConstraint(
+                fields=["task_name"],
+                condition=django.db.models.Q(task_name__isnull=False),
+                name="unique_task_name_if_not_null",
+            ),
+        ]
 
     id = django.db.models.CharField(max_length=1024, primary_key=True)
 
     # Tracking information to tie this back to the OC Analyzer that was used to create it.
     manifest = NullableJSONField(default=jsonfield_default_value, null=True, blank=True)
     description = django.db.models.TextField(null=False, blank=True, default="")
-    host_gremlin = django.db.models.ForeignKey(
-        GremlinEngine, blank=False, null=False, on_delete=django.db.models.CASCADE
-    )
     disabled = django.db.models.BooleanField(default=False)
     is_public = django.db.models.BooleanField(default=True)
     icon = django.db.models.FileField(
         blank=True, upload_to=calculate_analyzer_icon_path
     )
 
+    host_gremlin = django.db.models.ForeignKey(
+        GremlinEngine,
+        on_delete=django.db.models.CASCADE,
+        null=True,
+        blank=True,
+    )
+
+    task_name = django.db.models.CharField(
+        max_length=1024,
+        null=True,
+        blank=True,
+        default=None,
+    )
+
 
 class AnalyzerUserObjectPermission(UserObjectPermissionBase):
     content_object = django.db.models.ForeignKey(