Added bg tasks and import concepts

docker-compose-dev.yml

@@ -8,6 +8,7 @@ services:
     volumes:
       - ./webapp/api/core:/home/api/core
       - ./webapp/api/api:/home/api/api
+      - ./webapp/run.sh:/home/run.sh
       - api-media:/home/api/media
       - api-static:/home/api/static
       - api-db:/home/api/db

webapp/api/api/admin.py

@@ -1,6 +1,7 @@
 from django.http import HttpResponse
 from io import StringIO
 import json
+from background_task import background
 
 from django.contrib import admin
 from .models import *
@@ -19,7 +20,6 @@
 admin.site.register(MetaTask)
 admin.site.register(MetaAnnotation)
 admin.site.register(Vocabulary)
-admin.site.register(ConceptDB)
 
 
 def download(modeladmin, request, queryset):
@@ -60,13 +60,47 @@ def download(modeladmin, request, queryset):
     response['Content-Disposition'] = 'attachment; filename={}'.format(f_name)
     return response
 
+
 class ProjectAnnotateEntitiesAdmin(admin.ModelAdmin):
     model = ProjectAnnotateEntities
     actions = [download]
 admin.site.register(ProjectAnnotateEntities, ProjectAnnotateEntitiesAdmin)
 
+
 class AnnotatedEntityAdmin(admin.ModelAdmin):
     list_display = ('user', 'project', 'document', 'entity', 'value', 'deleted', 'validated')
     list_filter = ('user', 'project', 'document', 'deleted', 'validated')
     model = AnnotatedEntity
 admin.site.register(AnnotatedEntity, AnnotatedEntityAdmin)
+
+@background(schedule=5)
+def _import_concepts(id):
+    from medcat.cdb import CDB
+    concept_db = ConceptDB.objects.get(id=id)
+    cdb = CDB()
+    cdb.load_dict(concept_db.cdb_file.path)
+
+    for cui in cdb.cui2pretty_name:
+        cnt = Concept.objects.filter(cui=cui).count()
+        if cnt == 0:
+            tui = cdb.cui2tui.get(cui, 'unk')
+            concept = Concept()
+            concept.pretty_name = cdb.cui2pretty_name.get(cui, '')
+            concept.cui = cui
+            concept.tui = tui
+            concept.semantic_type = cdb.tui2name.get(tui, '')
+            concept.desc = cdb.cui2desc.get(cui, '')
+            concept.synonyms = ",".join(cdb.cui2original_names.get(cui, []))
+            #concept.vocab = cdb.cui2ontos.get(cui, '')
+            concept.save()
+
+
+def import_concepts(modeladmin, request, queryset):
+    for concept_db in queryset:
+        _import_concepts(concept_db.id)
+
+class ConceptDBAdmin(admin.ModelAdmin):
+    model = ConceptDB
+    actions = [import_concepts]
+admin.site.register(ConceptDB, ConceptDBAdmin)
+

webapp/api/api/data_utils.py

@@ -4,7 +4,7 @@
 
 
 def text_from_csv(dataset):
-    df = pandas.read_csv(dataset.original_file.path)
+    df = pandas.read_csv(dataset.original_file.path, escapechar='\\')
 
     if 'text' not in df.columns:
         # TODO: Fix me

webapp/api/api/migrations/0004_auto_20190925_1450.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.5 on 2019-09-25 14:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_auto_20190924_1934'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='project',
+            name='validated_documents',
+            field=models.ManyToManyField(blank=True, default=None, to='api.Document'),
+        ),
+    ]

webapp/api/api/models.py

@@ -84,7 +84,7 @@ class Project(PolymorphicModel):
     create_time = models.DateTimeField(auto_now_add=True)
     members = models.ManyToManyField(settings.AUTH_USER_MODEL)
     dataset = models.ForeignKey('Dataset', on_delete=models.CASCADE)
-    validated_documents = models.ManyToManyField(Document)
+    validated_documents = models.ManyToManyField(Document, default=None, blank=True)
     cuis = models.TextField(default=None, blank=True)
     tuis = models.TextField(default=None, blank=True)
 

webapp/api/core/settings.py

@@ -38,9 +38,10 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
+    'rest_framework.authtoken',
     'django_filters',
+    'background_task',
     'api',
-    'rest_framework.authtoken',
 ]
 
 MIDDLEWARE = [
@@ -142,3 +143,6 @@
 STATICFILES_DIRS = [
     os.path.join(BASE_DIR, "..", "frontend", "dist", "static"),
 ]
+
+# BG TASKS
+MAX_RUN_TIME=60*60*10

webapp/requirements.txt

@@ -4,6 +4,7 @@ django-filter~=2.2
 django-filters~=0.2
 django-polymorphic~=2.1
 djangorestframework~=3.10
+django-background-tasks~=1.2
 numpy~=1.15
 pandas~=0.23
 medcat~=0.2

webapp/run.sh

@@ -7,4 +7,5 @@ python /home/api/manage.py makemigrations api --noinput
 python /home/api/manage.py migrate --noinput
 python /home/api/manage.py migrate api --noinput
 
+python /home/api/manage.py process_tasks &
 uwsgi --http :8000 --master --chdir /home/api/ --module core.wsgi