FastAPI Integration

app.py

@@ -1,4 +1,6 @@
 import logging
+
+from fastapi import FastAPI
 from api.healthcheck import Healthcheck
 from api.misc import IsDemoRest, IsManagedRest
 import graphene
@@ -15,21 +17,104 @@
     CognitionPrepareProject,
     CognitionParseMarkdownFile,
 )
+from fast_api.routes.organization import router as org_router
+from fast_api.routes.project import router as project_router
+from fast_api.routes.project_setting import router as project_setting_router
+from fast_api.routes.misc import router as misc_router
+from fast_api.routes.comment import router as comment_router
+from fast_api.routes.zero_shot import router as zero_shot_router
+from fast_api.routes.attribute import router as attribute_router
+from fast_api.routes.embedding import router as embedding_router
+from fast_api.routes.notification import router as notification_router
+from fast_api.routes.data_slices import router as data_slice_router
+from fast_api.routes.lookup_lists import router as lookup_lists_router
+from fast_api.routes.heuristic import router as heuristic_router
+from fast_api.routes.data_browser import router as data_browser_router
+from fast_api.routes.labeling import router as labeling_router
+from fast_api.routes.record_ide import router as record_ide_router
+from fast_api.routes.record import router as record_router
+from fast_api.routes.weak_supervision import router as weak_supervision_router
+from fast_api.routes.labeling_tasks import router as labeling_tasks_router
 from middleware.database_session import DatabaseSessionHandler
 from starlette.applications import Starlette
 from starlette.graphql import GraphQLApp
 from starlette.middleware import Middleware
-from starlette.routing import Route
+from starlette.routing import Route, Mount
 
 from graphql_api import schema
 from controller.task_queue.task_queue import init_task_queues
 from controller.project.manager import check_in_deletion_projects
+from route_prefix import (
+    PREFIX_ORGANIZATION,
+    PREFIX_PROJECT,
+    PREFIX_PROJECT_SETTING,
+    PREFIX_MISC,
+    PREFIX_COMMENT,
+    PREFIX_ZERO_SHOT,
+    PREFIX_ATTRIBUTE,
+    PREFIX_EMBEDDING,
+    PREFIX_NOTIFICATION,
+    PREFIX_DATA_SLICE,
+    PREFIX_LOOKUP_LISTS,
+    PREFIX_HEURISTIC,
+    PREFIX_DATA_BROWSER,
+    PREFIX_LABELING,
+    PREFIX_RECORD_IDE,
+    PREFIX_RECORD,
+    PREFIX_WEAK_SUPERVISION,
+    PREFIX_LABELING_TASKS,
+)
 from util import security, clean_up
 
-
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
+fastapi_app = FastAPI()
+
+fastapi_app.include_router(
+    org_router, prefix=PREFIX_ORGANIZATION, tags=["organization"]
+)
+fastapi_app.include_router(project_router, prefix=PREFIX_PROJECT, tags=["project"])
+fastapi_app.include_router(
+    project_setting_router, prefix=PREFIX_PROJECT_SETTING, tags=["project-setting"]
+)
+fastapi_app.include_router(misc_router, prefix=PREFIX_MISC, tags=["misc"])
+fastapi_app.include_router(comment_router, prefix=PREFIX_COMMENT, tags=["comment"])
+fastapi_app.include_router(
+    zero_shot_router, prefix=PREFIX_ZERO_SHOT, tags=["zero-shot"]
+)
+fastapi_app.include_router(
+    attribute_router, prefix=PREFIX_ATTRIBUTE, tags=["attribute"]
+)
+fastapi_app.include_router(
+    embedding_router, prefix=PREFIX_EMBEDDING, tags=["embedding"]
+)
+fastapi_app.include_router(
+    notification_router, prefix=PREFIX_NOTIFICATION, tags=["notification"]
+)
+fastapi_app.include_router(
+    data_slice_router, prefix=PREFIX_DATA_SLICE, tags=["data-slice"]
+)
+fastapi_app.include_router(
+    lookup_lists_router, prefix=PREFIX_LOOKUP_LISTS, tags=["lookup-lists"]
+)
+fastapi_app.include_router(
+    heuristic_router, prefix=PREFIX_HEURISTIC, tags=["heuristic"]
+)
+fastapi_app.include_router(
+    data_browser_router, prefix=PREFIX_DATA_BROWSER, tags=["data-browser"]
+)
+fastapi_app.include_router(labeling_router, prefix=PREFIX_LABELING, tags=["labeling"])
+fastapi_app.include_router(
+    record_ide_router, prefix=PREFIX_RECORD_IDE, tags=["record-ide"]
+),
+fastapi_app.include_router(record_router, prefix=PREFIX_RECORD, tags=["record"]),
+fastapi_app.include_router(
+    weak_supervision_router, prefix=PREFIX_WEAK_SUPERVISION, tags=["weak-supervision"]
+)
+fastapi_app.include_router(
+    labeling_tasks_router, prefix=PREFIX_LABELING_TASKS, tags=["labeling-tasks"]
+)
 
 routes = [
     Route(
@@ -64,6 +149,7 @@
     Route("/project", ProjectCreationFromWorkflow),
     Route("/is_managed", IsManagedRest),
     Route("/is_demo", IsDemoRest),
+    Mount("/api", app=fastapi_app, name="REST API"),
 ]
 
 middleware = [Middleware(DatabaseSessionHandler)]

controller/auth/kratos.py

@@ -1,4 +1,4 @@
-from typing import Union, Any
+from typing import Union, Any, List, Optional, Dict
 from requests import Response
 import os
 import requests
@@ -32,3 +32,56 @@ def resolve_user_name_by_id(user_id: str) -> str:
     if res.status_code == 200 and data["traits"]:
         return data["traits"]["name"]
     return None
+
+
+def resolve_all_user_ids(
+    relevant_ids: List[str], as_list: bool = True
+) -> List[Dict[str, str]]:
+    final = [] if as_list else {}
+    for id in relevant_ids:
+        r = requests.get(f"{KRATOS_ADMIN_URL}/identities/{id}").json()
+        d = {
+            "id": id,
+            "mail": None,
+            "firstName": None,
+            "lastName": None,
+        }
+        if "traits" in r:
+            traits = r["traits"]
+            d["mail"] = traits["email"]
+            d["firstName"] = traits["name"]["first"]
+            d["lastName"] = traits["name"]["last"]
+        if as_list:
+            final.append(d)
+        else:
+            final[id] = d
+    return final
+
+
+def expand_user_mail_name(
+    users: List[Dict[str, str]], user_id_key="id"
+) -> List[Dict[str, str]]:
+    final = []
+    for user in users:
+        r = requests.get(f"{KRATOS_ADMIN_URL}/identities/{user[user_id_key]}").json()
+        d = {
+            "mail": None,
+            "firstName": None,
+            "lastName": None,
+        }
+        if "traits" in r:
+            traits = r["traits"]
+            d["mail"] = traits["email"]
+            d["firstName"] = traits["name"]["first"]
+            d["lastName"] = traits["name"]["last"]
+        user = {**user, **d}
+        final.append(user)
+    return final
+
+
+def resolve_user_name_and_email_by_id(user_id: str) -> dict:
+    res: Response = requests.get("{}/identities/{}".format(KRATOS_ADMIN_URL, user_id))
+    data: Any = res.json()
+    if res.status_code == 200 and data["traits"]:
+        return data["traits"]["name"], data["traits"]["email"]
+    return None

controller/auth/manager.py

@@ -1,8 +1,9 @@
 from typing import Any, Dict, List, Set
 
+from fastapi import Request
 from graphene import ResolveInfo
 from controller.misc import config_service
-from exceptions.exceptions import NotAllowedInDemoError
+from exceptions.exceptions import NotAllowedInDemoError, ProjectAccessError
 import jwt
 from graphql import GraphQLError
 from controller.project import manager as project_manager
@@ -13,6 +14,8 @@
 from controller.misc import manager as misc_manager
 import sqlalchemy
 
+DEV_USER_ID = "59e8dfca-ce56-44df-a8c7-5f05c61da499"
+
 
 def get_organization_id_by_info(info) -> Organization:
     organization: Organization = get_user_by_info(info).organization
@@ -22,7 +25,11 @@ def get_organization_id_by_info(info) -> Organization:
 
 
 def get_user_by_info(info) -> User:
-    user_id: str = get_user_id_by_jwt_token(info.context["request"])
+    request = info.context["request"]
+    if request.url.hostname == "localhost" and request.url.port == 7051:
+        user_id = DEV_USER_ID
+    else:
+        user_id: str = get_user_id_by_jwt_token(request)
     return user_manager.get_or_create_user(user_id)
 
 
@@ -61,6 +68,13 @@ def get_user_id_by_jwt_token(request) -> str:
     return claims["session"]["identity"]["id"]
 
 
+def check_project_access_dep(request: Request, project_id: str):
+    if len(project_id) == 36:
+        check_project_access(request.state.info, project_id)
+    else:
+        raise ProjectAccessError
+
+
 def check_project_access(info, project_id: str) -> None:
     organization_id: str = get_organization_id_by_info(info).id
     project: Project = project_manager.get_project_with_orga_id(
@@ -110,14 +124,14 @@ def check_is_admin(request: Any) -> bool:
     return False
 
 
-def check_demo_access(info: ResolveInfo) -> None:
+def check_demo_access(info: Any) -> None:
     if not check_is_admin(info.context["request"]) and config_service.get_config_value(
         "is_demo"
     ):
         check_black_white(info)
 
 
-def check_black_white(info: ResolveInfo):
+def check_black_white(info: Any):
     black_white = misc_manager.get_black_white_demo()
     if str(info.parent_type) == "Mutation":
         if info.field_name not in black_white["mutations"]:

controller/comment/manager.py

@@ -47,6 +47,10 @@ def get_comment(xftype: str, user_id: str, comment_id: str) -> CommentData:
     return comments.get_as_json(comment_id, user_id)
 
 
+def get_comment_by_comment_id(user_id: str, comment_id: str) -> CommentData:
+    return comments.get_as_json(comment_id, user_id)
+
+
 def get_add_info(
     xftype: str,
     project_id: Optional[str] = None,
@@ -100,24 +104,24 @@ def update_comment(
     item = comments.get(comment_id)
 
     if not item:
-        raise ValueError(f"Can't find comment")
+        raise ValueError("Can't find comment")
 
     if user.role != enums.UserRoles.ENGINEER.value and user.id != item.created_by:
-        raise ValueError(f"Can't update comment")
+        raise ValueError("Can't update comment")
     comments.change(item, changes, with_commit=True)
     return item
 
 
 def delete_comment(comment_id: str, user_id: str) -> CommentData:
     item = comments.get(comment_id)
     if not item:
-        raise ValueError(f"Can't find comment")
+        raise ValueError("Can't find comment")
 
     if (
         item.created_by != user_id
         and user_manager.get_user_role_by_id(user_id) != enums.UserRoles.ENGINEER.value
     ):
-        raise ValueError(f"Can't delete comment")
+        raise ValueError("Can't delete comment")
     comments.remove(comment_id, with_commit=True)
 
 

controller/embedding/manager.py

@@ -9,7 +9,15 @@
 from . import connector
 from .terms import TERMS_INFO
 from controller.model_provider import manager as model_manager
-from submodules.model.business_objects import attribute, embedding, agreement, general
+from submodules.model.business_objects import (
+    attribute,
+    embedding,
+    agreement,
+    general,
+    project,
+)
+from submodules.model.util import sql_alchemy_to_dict
+from controller.embedding.connector import collection_on_qdrant
 
 
 def get_terms_info(
@@ -137,6 +145,62 @@ def get_embedding_name(
     return name
 
 
+EMBEDDING_SCHEMA_WHITELIST = [
+    "id",
+    "name",
+    "custom",
+    "type",
+    "state",
+    "progress",
+    "dimension",
+    "count",
+    "platform",
+    "model",
+    "filter_attributes",
+    "attribute_id",
+]
+
+
+def get_embedding_schema(project_id: str) -> List[Dict[str, Any]]:
+    embeddings = embedding.get_all_embeddings_by_project_id(project_id)
+    embedding_dict = sql_alchemy_to_dict(
+        embeddings, column_whitelist=EMBEDDING_SCHEMA_WHITELIST
+    )
+    number_records = len(project.get(project_id).records)
+    expanded_embeddings = []
+    for embed in embedding_dict:
+        count = embedding.get_tensor_count(embed["id"])
+        onQdrant = collection_on_qdrant(project_id, embed["id"])
+
+        embedding_item = embedding.get_tensor(embed["id"])
+        dimension = 0
+        if embedding_item is not None:
+            # distinguish between token and attribute embeddings
+            if type(embedding_item.data[0]) is list:
+                dimension = len(embedding_item.data[0])
+            else:
+                dimension = len(embedding_item.data)
+
+        if embed["state"] == "FINISHED":
+            progress = 1
+        elif embed["state"] == "INITIALIZING" or embed["state"] == "WAITING":
+            progress = 0.0
+        else:
+            progress = min(
+                0.1 + (count / number_records * 0.9),
+                0.99,
+            )
+        expanded_embed = {
+            **embed,
+            "progress": progress,
+            "count": count,
+            "dimension": dimension,
+            "onQdrant": onQdrant,
+        }
+        expanded_embeddings.append(expanded_embed)
+    return {"id": project_id, "embeddings": expanded_embeddings}
+
+
 def recreate_embeddings(
     project_id: str, embedding_ids: Optional[List[str]] = None, user_id: str = None
 ) -> None:

controller/knowledge_term/manager.py

@@ -6,7 +6,7 @@
     EntityAlreadyExistsException,
     EntityNotFoundException,
 )
-from submodules.model.business_objects import general, knowledge_term, knowledge_base
+from submodules.model.business_objects import knowledge_term, knowledge_base
 from util.notification import create_notification
 
 

controller/labeling_task/manager.py

@@ -1,6 +1,6 @@
 from submodules.model import LabelingTask
 from .util import resolve_attribute_information
-from submodules.model.business_objects import labeling_task, general, attribute
+from submodules.model.business_objects import labeling_task, attribute
 
 
 def get_labeling_task(project_id: str, labeling_task_id: str) -> LabelingTask: