Merge pull request #61 from di2ag/yakaboskic-opentelemetry

Upgrading TRAPI and BioLink and adding in OpenTelemetry

Author: yakaboskic

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "gennifer"]
+	path = gennifer
+	url = git@github.com:di2ag/gennifer.git

Dockerfile renamed to chp_api/Dockerfile

@@ -3,12 +3,15 @@
 ###########
 
 # first stage of build to pull repos
-FROM python:3.8 as intermediate
+FROM python:3.9 as intermediate
 
 # set work directory
 WORKDIR /usr/src/chp_api
 
-RUN git clone --single-branch --branch gene_spec_pydantic-ghyde https://github.com/di2ag/gene-specificity.git
+RUN git clone --single-branch --branch master https://github.com/di2ag/gene-specificity.git
+
+# Upgrade pip
+RUN pip3 install --upgrade pip
 
 # Upgrade pip
 RUN pip3 install --upgrade pip
@@ -25,7 +28,7 @@ RUN cd gene-specificity && python3 setup.py bdist_wheel && cd dist && cp gene_sp
 #########
 
 #pull official base image
-FROM python:3.8
+FROM python:3.9
 
 # add app user
 RUN groupadd chp_api && useradd -ms /bin/bash -g chp_api chp_api
@@ -53,10 +56,13 @@ RUN pip3 install pydantic==1.10.12
 RUN pip3 install --no-cache /wheels/*
 
 # copy project
-COPY ./chp_api/chp_api $APP_HOME/chp_api
-COPY ./chp_api/manage.py $APP_HOME
-COPY ./chp_api/dispatcher $APP_HOME/dispatcher
-COPY ./gunicorn.config.py $APP_HOME
+COPY . .
+#COPY ./chp_api $APP_HOME/chp_api
+#COPY ./manage.py $APP_HOME
+#COPY ./dispatcher $APP_HOME/dispatcher
+#COPY ./gennifer $APP_HOME/gennifer
+#COPY ./chp_db_fixture.json.gz $APP_HOME
+#COPY ./gunicorn.config.py $APP_HOME
 
 # chown all the files to the app user
 RUN chown -R chp_api:chp_api $APP_HOME \

chp_api/chp_api/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

chp_api/chp_api/celery.py

@@ -0,0 +1,15 @@
+import os
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "chp_api.settings")
+app = Celery(
+        "chp_api",
+        include=['gennifer.tasks'],
+        )
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()
+app.conf.update({
+    "task_routes": {
+        "create_gennifer_task": {"queue": 'chp_api'}
+        }
+    })

chp_api/chp_api/serializers.py

@@ -0,0 +1,12 @@
+from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
+
+class ChpTokenObtainPairSerializer(TokenObtainPairSerializer):
+    @classmethod
+    def get_token(cls, user):
+        token = super().get_token(user)
+
+        # Add custom claims
+        token['email'] = user.email
+        token['username'] = user.username
+
+        return token

chp_api/chp_api/settings.py

@@ -10,9 +10,11 @@
 https://docs.djangoproject.com/en/3.0/ref/settings/
 """
 import os
-from importlib import import_module
 import environ as environ # type: ignore
 
+from importlib import import_module
+
+
 # Initialise environment variables
 env = environ.Env()
 environ.Env.read_env()
@@ -27,9 +29,33 @@
 REST_FRAMEWORK = {
     'DEFAULT_PARSER_CLASSES': [
         'rest_framework.parsers.JSONParser',
-    ]
+    ],
+    'DEFAULT_AUTHENTICATION_CLASSES': (
+        #'rest_framework_simplejwt.authentication.JWTAuthentication',
+        'oauth2_provider.contrib.rest_framework.OAuth2Authentication',
+        ),
+    'DEFAULT_PERMISSION_CLASSES': (
+        'rest_framework.permissions.IsAuthenticatedOrReadOnly',
+        )
 }
 
+AUTHENTICATION_BACKENDS = [
+    'oauth2_provider.backends.OAuth2Backend',
+    # Uncomment following if you want to access the admin
+    'django.contrib.auth.backends.ModelBackend',
+]
+
+OAUTH2_PROVIDER = {
+    # this is the list of available scopes
+    'SCOPES': {'read': 'Read scope', 'write': 'Write scope', 'groups': 'Access to your groups'}
+}
+
+# Cors stuff (must go before installed apps)
+CORS_ALLOWED_ORIGINS = [
+        'http://localhost',
+        'http://localhost:3000',
+        ]
+
 # Application definition
 INSTALLED_BASE_APPS = [
     'django.contrib.admin',
@@ -38,13 +64,20 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'corsheaders',
     'rest_framework',
+    'rest_framework_simplejwt',
+    'django_filters',
     'dispatcher.apps.DispatcherConfig',
     'django_extensions',
+    'users',
+    'oauth2_provider',
+    #'gennifer', # Need to make into CHP app
 ]
 
 INSTALLED_CHP_APPS = [
     'gene_specificity',
+    'gennifer',
     ]
 
 # CHP Versions
@@ -54,13 +87,16 @@
 INSTALLED_APPS = INSTALLED_BASE_APPS + INSTALLED_CHP_APPS
 
 MIDDLEWARE = [
+    'corsheaders.middleware.CorsMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'oauth2_provider.middleware.OAuth2TokenMiddleware',
 ]
 
 ROOT_URLCONF = 'chp_api.urls'
@@ -116,6 +152,10 @@
     },
 ]
 
+# Authorization
+AUTH_USER_MODEL='users.User'
+LOGIN_URL='/admin/login/'
+
 # Internationalization
 # https://docs.djangoproject.com/en/3.0/topics/i18n/
 LANGUAGE_CODE = 'en-us'
@@ -193,3 +233,21 @@
 if not DJANGO_SUPERUSER_PASSWORD:
     with open(env("DJANGO_SUPERUSER_PASSWORD_FILE"), 'r') as dsp_file:
         os.environ["DJANGO_SUPERUSER_PASSWORD"] = dsp_file.readline().strip()
+
+# Simple JWT Settings
+SIMPLE_JWT = {
+        "TOKEN_OBTAIN_SERIALIZER": "chp_api.serializers.ChpTokenObtainPairSerializer",
+        }
+
+# Celery Settings
+CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379")
+CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379")
+
+# Gennifer settings
+GENNIFER_ALGORITHM_URLS = [
+        "http://pidc:5000",
+        "http://grisli:5000",
+        "http://genie3:5000",
+        "http://grnboost2:5000",
+        "http://bkb-grn:5000",
+        ]

chp_api/chp_api/urls.py

@@ -17,9 +17,20 @@
 from django.urls import path, include
 from rest_framework.urlpatterns import format_suffix_patterns
 
+from rest_framework_simplejwt.views import (
+    TokenObtainPairView,
+    TokenRefreshView,
+)
+
+
 urlpatterns = [
         path('admin/', admin.site.urls),
         path('', include('dispatcher.urls')),
+        path('gennifer/api/', include('gennifer.urls')),
+        path('api/token/', TokenObtainPairView.as_view(), name='token_obtain_pair'),
+        path('api/token/refresh/', TokenRefreshView.as_view(), name='token_refresh'), 
+        path('o/', include('oauth2_provider.urls', namespace='oauth2_provider')),
+        path('users/', include('users.urls')),
         ]
 
 

chp_api/dispatcher/admin.py

@@ -1,7 +1,7 @@
 from django.contrib import admin
 
-from .models import App, ZenodoFile, DispatcherSettings
+from .models import App, ZenodoFile, DispatcherSetting
 
 admin.site.register(App)
 admin.site.register(ZenodoFile)
-admin.site.register(DispatcherSettings)
+admin.site.register(DispatcherSetting)

chp_api/dispatcher/base.py

@@ -12,8 +12,8 @@
 from importlib import import_module
 from collections import defaultdict
 
-from .models import Transaction, App, DispatcherSettings, Template, TemplateMatch
-from reasoner_pydantic import MetaKnowledgeGraph, Message, MetaEdge
+from .models import Transaction, App, DispatcherSetting, Template, TemplateMatch
+from reasoner_pydantic import MetaKnowledgeGraph, Message, MetaEdge, MetaNode
 from reasoner_pydantic.qgraph import QNode, QEdge
 
 # Setup logging
@@ -42,9 +42,27 @@ def __init__(self, request, trapi_version, biolink_version):
         #self.validator = TRAPISchemaValidator(self.trapi_version)
         self.logger = Logger()
 
+    def merge_meta_kg(self, metakg1, metakg2):
+        new_metakg = MetaKnowledgeGraph.parse_obj({"nodes": [], "edges": []})
+        # Merge nodes
+        new_metakg.nodes = metakg1.nodes
+        for n, v in metakg2.nodes.items():
+            if n in new_metakg.nodes:
+                id_prefixes = list(set.union(*[set(list(metakg1.nodes[n].id_prefixes)), set(list(metakg2.nodes[n].id_prefixes))]))
+                new_node = MetaNode.parse_obj({"id_prefixes": id_prefixes})
+                new_metakg.nodes[n] = new_node
+            else:
+                new_metakg.nodes[n] = MetaNode.parse_obj(v)
+        # Merge edges
+        for e in metakg1.edges:
+            new_metakg.edges.append(e)
+        for e in metakg2.edges:
+            new_metakg.edges.append(e)
+        return new_metakg
+
     def get_meta_knowledge_graph(self):
         # Get current trapi and biolink versions
-        dispatcher_settings = DispatcherSettings.load()
+        dispatcher_settings = DispatcherSetting.load()
         merged_meta_kg = None
         for app, app_name in zip(APPS, settings.INSTALLED_CHP_APPS):
             app_db_obj = App.objects.get(name=app_name)
@@ -59,7 +77,7 @@ def get_meta_knowledge_graph(self):
             if merged_meta_kg is None:
                 merged_meta_kg = meta_kg
             else:
-                merged_meta_kg.update(meta_kg)
+                merged_meta_kg = self.merge_meta_kg(merged_meta_kg, meta_kg)
         return merged_meta_kg
 
     def process_invalid_trapi(self, request):

chp_api/dispatcher/migrations/0008_dispatchersettings_sri_node_normalizer_baseurl.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.1 on 2023-05-29 22:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dispatcher', '0007_template_templatematch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dispatchersettings',
+            name='sri_node_normalizer_baseurl',
+            field=models.URLField(default='https://nodenormalization-sri.renci.org', max_length=128),
+        ),
+    ]