LLM attribute calculation

business_objects/attribute.py

@@ -9,6 +9,7 @@
 from ..models import Attribute
 from ..session import session
 from submodules.model import enums
+from sqlalchemy import or_
 
 from ..util import prevent_sql_injection
 
@@ -109,7 +110,11 @@ def get_text_attributes(
     if state_filter is None:
         state_filter = DEFAULT_ATTRIBUTE_STATES_USEABLE
     query = session.query(Attribute).filter(
-        Attribute.project_id == project_id, Attribute.data_type == DataTypes.TEXT.value
+        Attribute.project_id == project_id,
+        or_(
+            Attribute.data_type == DataTypes.TEXT.value,
+            Attribute.data_type == DataTypes.LLM_RESPONSE.value,
+        ),
     )
     if state_filter:
         query = query.filter(Attribute.state.in_(state_filter))
@@ -140,7 +145,9 @@ def get_non_text_attributes(
     if not state_filter:
         state_filter = DEFAULT_ATTRIBUTE_STATES_USEABLE
     query = session.query(Attribute).filter(
-        Attribute.project_id == project_id, Attribute.data_type != DataTypes.TEXT.value
+        Attribute.project_id == project_id,
+        Attribute.data_type != DataTypes.TEXT.value,
+        Attribute.data_type != DataTypes.LLM_RESPONSE.value,
     )
     if state_filter:
         query = query.filter(Attribute.state.in_(state_filter))
@@ -218,6 +225,7 @@ def create(
     started_at: Optional[datetime] = None,
     finished_at: Optional[datetime] = None,
     with_commit: bool = False,
+    additional_config: Optional[Dict[str, Any]] = None,
 ) -> Attribute:
     attribute: Attribute = Attribute(
         project_id=project_id,
@@ -246,6 +254,9 @@ def create(
     if finished_at is not None:
         attribute.finished_at = finished_at
 
+    if additional_config is not None:
+        attribute.additional_config = additional_config
+
     general.add(attribute, with_commit)
     return attribute
 
@@ -264,6 +275,7 @@ def update(
     finished_at: Optional[datetime] = None,
     visibility: Optional[str] = None,
     progress: Optional[float] = None,
+    additional_config: Optional[Dict[str, Any]] = None,
 ) -> Attribute:
     attribute: Attribute = get(project_id, attribute_id)
     if data_type is not None:
@@ -292,6 +304,10 @@ def update(
     if finished_at is not None:
         attribute.finished_at = finished_at
 
+    if additional_config is not None:
+        attribute.additional_config = additional_config
+        flag_modified(attribute, "additional_config")
+
     general.flush_or_commit(with_commit)
     return attribute
 

business_objects/embedding.py

@@ -317,7 +317,10 @@ def __build_payload_selector(
         for attr, data_type in attributes_to_include.items():
             if payload_selector != "":
                 payload_selector += ","
-            if data_type != enums.DataTypes.TEXT.value:
+            if (
+                data_type != enums.DataTypes.TEXT.value
+                and data_type != enums.DataTypes.LLM_RESPONSE.value
+            ):
                 payload_selector += f"'{attr}', (r.\"data\"->>'{attr}')::{data_type}"
             else:
                 payload_selector += f"'{attr}', r.\"data\"->>'{attr}'"

business_objects/record.py

@@ -845,7 +845,7 @@ def get_first_no_text_column(project_id: str, record_id: str) -> str:
     (
         SELECT a.name
         FROM attribute a 
-        WHERE data_type NOT IN('{enums.DataTypes.TEXT.value}' , '{enums.DataTypes.CATEGORY.value}')
+        WHERE data_type NOT IN('{enums.DataTypes.TEXT.value}' , '{enums.DataTypes.CATEGORY.value}', '{enums.DataTypes.LLM_RESPONSE.value}')
             AND a.state IN ('{enums.AttributeState.AUTOMATICALLY_CREATED.value}','{enums.AttributeState.UPLOADED.value}','{enums.AttributeState.USABLE.value}')
             AND a.project_id = '{project_id}'
         ORDER BY a.relative_position

enums.py

@@ -8,6 +8,7 @@ class DataTypes(Enum):
     BOOLEAN = "BOOLEAN"
     CATEGORY = "CATEGORY"
     TEXT = "TEXT"
+    LLM_RESPONSE = "LLM_RESPONSE"
     EMBEDDING_LIST = "EMBEDDING_LIST"  # only for embeddings & default hidden
     UNKNOWN = "UNKNOWN"
 

models.py

@@ -482,6 +482,7 @@ class Attribute(Base):
     started_at = Column(DateTime, default=sql.func.now())
     finished_at = Column(DateTime)
     progress = Column(Float)
+    additional_config = Column(JSON, comment="used when data_type == LLM_RESPONSE")
 
     embeddings = parent_to_child_relationship(
         Tablenames.ATTRIBUTE,