Skip to content

Commit c5f6fbb

Browse files
JWittmeyerJWittmeyer
authored
Merge pull request #61 from code-kern-ai/llm-ac
LLM attribute calculation
2 parents fff256b + 6e0c976 commit c5f6fbb

File tree

30 files changed

+968
-66
lines changed

30 files changed

+968
-66
lines changed

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,4 @@
5050
"postcss": "^8.4.28",
5151
"tailwindcss": "^3.1.8"
5252
}
53-
}
53+
}

‎src/components/projects/projectId/attributes/attributeId/AttributeCalculations.tsx

+97-29
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@ import Statuses from "@/src/components/shared/statuses/Statuses";
22
import { selectAllLookupLists, setAllLookupLists } from "@/src/reduxStore/states/pages/lookup-lists";
33
import { selectAttributes, selectVisibleAttributeAC, setAllAttributes, setLabelingTasksAll, updateAttributeById } from "@/src/reduxStore/states/pages/settings";
44
import { selectProjectId } from "@/src/reduxStore/states/project"
5-
import { Attribute, AttributeState } from "@/src/types/components/projects/projectId/settings/data-schema";
5+
import { Attribute, AttributeState, LLMConfig } from "@/src/types/components/projects/projectId/settings/data-schema";
66
import { DataTypeEnum } from "@/src/types/shared/general";
7-
import { postProcessCurrentAttribute } from "@/src/util/components/projects/projectId/settings/attribute-calculation-helper";
7+
import { LLM_PROVIDER_OPTIONS, postProcessCurrentAttribute } from "@/src/util/components/projects/projectId/settings/attribute-calculation-helper";
88
import { ATTRIBUTES_VISIBILITY_STATES, DATA_TYPES, getTooltipVisibilityState } from "@/src/util/components/projects/projectId/settings/data-schema-helper";
99
import { copyToClipboard } from "@/submodules/javascript-functions/general";
1010
import { Editor } from "@monaco-editor/react";
1111
import { Tooltip } from "@nextui-org/react";
1212
import { IconAlertTriangleFilled, IconArrowLeft, IconCircleCheckFilled } from "@tabler/icons-react";
1313
import { useRouter } from "next/router";
14-
import { useCallback, useEffect, useState } from "react";
14+
import { useCallback, useEffect, useMemo, useState } from "react";
1515
import { useDispatch, useSelector } from "react-redux"
1616
import ExecutionContainer from "./ExecutionContainer";
1717
import { getPythonFunctionRegExMatch, toPythonFunctionName } from "@/submodules/javascript-functions/python-functions-parser";
@@ -24,7 +24,6 @@ import { TOOLTIPS_DICT } from "@/src/util/tooltip-constants";
2424
import { selectAllUsers, selectOrganizationId, setComments } from "@/src/reduxStore/states/general";
2525
import { CommentDataManager } from "@/src/util/classes/comments";
2626
import { CommentType } from "@/src/types/shared/comments";
27-
import { AttributeCodeLookup } from "@/src/util/classes/attribute-calculation";
2827
import KernDropdown from "@/submodules/react-components/components/KernDropdown";
2928
import { useWebsocket } from "@/submodules/react-components/hooks/web-socket/useWebsocket";
3029
import { postProcessLabelingTasksSchema } from "@/src/util/components/projects/projectId/settings/labeling-tasks-helper";
@@ -35,9 +34,16 @@ import { getLabelingTasksByProjectId, getProjectTokenization } from "@/src/servi
3534
import { getAttributeByAttributeId, updateAttribute } from "@/src/services/base/project-setting";
3635
import { Application, CurrentPage } from "@/submodules/react-components/hooks/web-socket/constants";
3736
import { VisitBricksButton } from "@/src/components/shared/bricks/VisitBricksButton";
37+
import LLMResponseConfig from "./LLMResponseConfig";
38+
import useDebounce from "@/submodules/react-components/hooks/useHooks/useDebounce";
39+
import useRefFor from "@/submodules/react-components/hooks/useRefFor";
40+
import { simpleDictCompare } from "@/submodules/javascript-functions/validations";
41+
import { LLM_CODE_TEMPLATE_EXAMPLES, LLM_CODE_TEMPLATE_OPTIONS } from "./LLM/llmTemplates";
3842

3943
const EDITOR_OPTIONS = { theme: 'vs-light', language: 'python', readOnly: false };
4044

45+
46+
4147
export default function AttributeCalculation() {
4248
const router = useRouter();
4349
const dispatch = useDispatch();
@@ -59,26 +65,44 @@ export default function AttributeCalculation() {
5965
const [attributeName, setAttributeName] = useState('');
6066
const [checkUnsavedChanges, setCheckUnsavedChanges] = useState(false);
6167
const [enableRunButton, setEnableButton] = useState(false);
68+
const [additionalConfigTmp, setAdditionalConfigTmp] = useState<LLMConfig>(null);
69+
70+
const currentAttributeRef = useRefFor(currentAttribute);
71+
const debouncedConfig = useDebounce(additionalConfigTmp, 1000);
72+
73+
const updateSourceCode = useCallback((value: string, attributeNameParam?: string) => {
74+
var regMatch: any = getPythonFunctionRegExMatch(value);
75+
if (!regMatch) {
76+
console.log("Can't find python function name -- seems wrong -- better dont save");
77+
return;
78+
}
79+
const finalSourceCode = value.replace(regMatch[0], 'def ac(record)');
80+
updateAttribute(projectId, currentAttribute.id, (res) => {
6281

82+
}, null, null, attributeNameParam, finalSourceCode);
83+
}, [projectId, currentAttribute]);
84+
85+
useEffect(() => setAdditionalConfigTmp(currentAttribute?.additionalConfig), [currentAttribute?.additionalConfig])
6386

6487
useEffect(() => {
6588
if (!projectId) return;
66-
if (!currentAttribute || attributes.length == 0) {
67-
getAttributes(projectId, ['ALL'], (res) => {
68-
dispatch(setAllAttributes(res));
69-
const currentAttribute = postProcessCurrentAttribute(attributes.find((attribute) => attribute.id === router.query.attributeId));
70-
setCurrentAttribute(currentAttribute);
71-
setEditorValue(currentAttribute?.sourceCodeToDisplay);
72-
});
73-
}
7489
if (lookupLists.length == 0) {
7590
getLookupListsByProjectId(projectId, (res) => {
7691
dispatch(setAllLookupLists(res));
7792
});
7893
}
7994
refetchLabelingTasksAndProcess();
8095
checkProjectTokenization();
81-
}, [projectId, attributes, currentAttribute]);
96+
}, [projectId, attributes]);
97+
98+
useEffect(() => {
99+
if (currentAttribute || !projectId) return;
100+
getAttributeByAttributeId(projectId, router.query.attributeId as string, (attribute) => {
101+
const currentAttribute = postProcessCurrentAttribute(attribute);
102+
setCurrentAttribute(currentAttribute);
103+
setEditorValue(currentAttribute?.sourceCodeToDisplay);
104+
});
105+
}, [projectId, currentAttribute, router.query.attributeId])
82106

83107
useEffect(() => {
84108
if (!attributes) return;
@@ -97,7 +121,7 @@ export default function AttributeCalculation() {
97121
setEditorOptions({ ...EDITOR_OPTIONS, readOnly: false });
98122
}
99123
setAttributeName(currentAttribute.name);
100-
}, [currentAttribute]);
124+
}, [currentAttribute, updateSourceCode]);
101125

102126
useEffect(() => {
103127
if (!projectId || allUsers.length == 0) return;
@@ -125,7 +149,21 @@ export default function AttributeCalculation() {
125149
spinner.unsubscribe();
126150
subscription.unsubscribe();
127151
}
128-
}, [editorValue, currentAttribute]);
152+
}, [editorValue, currentAttribute, updateSourceCode]);
153+
154+
155+
useEffect(() => {
156+
if (!currentAttributeRef.current || !currentAttributeRef.current.additionalConfig || simpleDictCompare(currentAttributeRef.current?.additionalConfig, debouncedConfig)) return;
157+
const attributeNew = { ...currentAttribute };
158+
attributeNew.additionalConfig = { ...debouncedConfig };
159+
updateAttribute(projectId, currentAttribute.id, (res) => {
160+
setCurrentAttribute(postProcessCurrentAttribute(attributeNew));
161+
dispatch(updateAttributeById(attributeNew));
162+
setEnableButton(true);
163+
}, null, null, null, null, null, debouncedConfig);
164+
165+
}, [debouncedConfig])
166+
129167

130168
function setUpCommentsRequests() {
131169
const requests = [];
@@ -207,17 +245,6 @@ export default function AttributeCalculation() {
207245
}
208246
}
209247

210-
function updateSourceCode(value: string, attributeNameParam?: string) {
211-
var regMatch: any = getPythonFunctionRegExMatch(value);
212-
if (!regMatch) {
213-
console.log("Can't find python function name -- seems wrong -- better dont save");
214-
return;
215-
}
216-
const finalSourceCode = value.replace(regMatch[0], 'def ac(record)');
217-
updateAttribute(projectId, currentAttribute.id, (res) => {
218-
219-
}, null, null, attributeNameParam, finalSourceCode);
220-
}
221248

222249
function checkProjectTokenization() {
223250
getProjectTokenization(projectId, (res) => {
@@ -269,9 +296,20 @@ export default function AttributeCalculation() {
269296
}
270297
}, [projectId, currentAttribute]);
271298

299+
const selectCodeTemplate = useCallback((option) => {
300+
if (!currentAttributeRef.current) return;
301+
updateSourceCode(LLM_CODE_TEMPLATE_EXAMPLES[option.value]);
302+
setEditorValue(LLM_CODE_TEMPLATE_EXAMPLES[option.value].replace('def ac(record)', 'def ' + currentAttributeRef.current.name + '(record)'));
303+
}, [updateSourceCode])
304+
272305
const orgId = useSelector(selectOrganizationId);
273306
useWebsocket(orgId, Application.REFINERY, CurrentPage.ATTRIBUTE_CALCULATION, handleWebsocketNotification, projectId);
274307

308+
const disabledOptions = useMemo(() => {
309+
if (!currentAttribute || currentAttribute.dataType == DataTypeEnum.LLM_RESPONSE) return undefined;
310+
return DATA_TYPES.map((e) => e.value == DataTypeEnum.LLM_RESPONSE);
311+
}, [currentAttribute?.dataType])
312+
275313
return (projectId && <div className={`bg-white p-4 overflow-y-auto min-h-full h-[calc(100vh-4rem)]`} onScroll={(e: any) => onScrollEvent(e)}>
276314
{currentAttribute && <div>
277315
<div className={`sticky z-50 h-12 ${isHeaderNormal ? 'top-1' : '-top-5'}`}>
@@ -315,11 +353,25 @@ export default function AttributeCalculation() {
315353

316354
<div className="text-sm leading-5 font-medium text-gray-700">Data type</div>
317355
<div className="flex flex-row items-center">
318-
<Tooltip color="invert" placement="right" content={currentAttribute.state == AttributeState.USABLE || currentAttribute.state == AttributeState.RUNNING ? TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.CANNOT_EDIT_DATATYPE : TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.EDIT_DATATYPE}>
356+
<Tooltip color="invert" placement="right" className="cursor-not-allowed" content={currentAttribute.state == AttributeState.USABLE || currentAttribute.state == AttributeState.RUNNING ? TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.CANNOT_EDIT_DATATYPE : TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.EDIT_DATATYPE}>
319357
<KernDropdown buttonName={currentAttribute.dataTypeName} options={DATA_TYPES} dropdownWidth="w-52"
320-
selectedOption={(option: any) => updateDataType(option)} disabled={currentAttribute.state == AttributeState.USABLE} dropdownClasses="z-30" />
358+
selectedOption={(option: any) => updateDataType(option)} disabledOptions={disabledOptions} disabled={currentAttribute.state == AttributeState.USABLE || currentAttribute.dataType == DataTypeEnum.LLM_RESPONSE} dropdownClasses="z-30" />
321359
</Tooltip>
322360
{currentAttribute.dataType == DataTypeEnum.EMBEDDING_LIST && <div className="text-gray-700 text-sm ml-3">Only useable for similarity search</div>}
361+
{currentAttribute.dataType == DataTypeEnum.LLM_RESPONSE && <div className="ml-3 flex flex-row flex-nowrap w-full items-center gap-x-2">
362+
<label className="block text-sm font-medium text-gray-900 whitespace-nowrap">Provider</label>
363+
<KernDropdown
364+
buttonName={additionalConfigTmp?.llmIdentifier ?? 'Select LLM provider'}
365+
options={LLM_PROVIDER_OPTIONS}
366+
dropdownWidth="w-52"
367+
selectedOption={(option) => setAdditionalConfigTmp(p => ({ ...p, llmIdentifier: option }))}
368+
disabled={currentAttribute.state == AttributeState.USABLE}
369+
/>
370+
<label className="block text-sm font-medium text-gray-900 whitespace-nowrap">Api Key</label>
371+
372+
<input type="text" disabled={currentAttribute.state == AttributeState.USABLE} value={additionalConfigTmp?.llmConfig.apiKey || ""} onInput={(e: any) => setAdditionalConfigTmp(p => ({ ...p, llmConfig: { ...additionalConfigTmp.llmConfig, apiKey: e.target.value } }))}
373+
className="h-8 text-sm border-gray-300 rounded-md placeholder-italic w-full border text-gray-700 pl-4 placeholder:text-gray-400 focus:outline-none focus:ring-2 focus:ring-gray-300 focus:ring-offset-2 focus:ring-offset-gray-100 disabled:opacity-50" />
374+
</div>}
323375
</div>
324376
<div className="text-sm leading-5 font-medium text-gray-700 inline-block">Attributes</div>
325377
<div className="flex flex-row items-center">
@@ -349,8 +401,24 @@ export default function AttributeCalculation() {
349401
))}
350402
</div>
351403
</div>
404+
{
405+
currentAttribute.dataType == DataTypeEnum.LLM_RESPONSE &&
406+
<LLMResponseConfig disabled={currentAttribute.state == AttributeState.USABLE} attributeId={currentAttribute?.id} fullLlmConfig={additionalConfigTmp} setFullLlmConfig={setAdditionalConfigTmp} apiKey={additionalConfigTmp?.llmConfig.apiKey} noPlayground={currentAttribute.state == AttributeState.USABLE} />
407+
}
352408
<div className="flex flex-row items-center justify-between my-3">
353-
<div className="text-sm leading-5 font-medium text-gray-700 inline-block mr-2">Editor</div>
409+
<div className="flex flex-row flex-nowrap items-center">
410+
<span className="text-sm leading-5 font-medium text-gray-700 inline-block mr-2">{currentAttribute.dataType == DataTypeEnum.LLM_RESPONSE ? 'Postprocessing' : 'Editor'}</span>
411+
{currentAttribute.dataType == DataTypeEnum.LLM_RESPONSE &&
412+
<Tooltip content={TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.LLM_POSTPROCESSING_CODE} color="invert" placement="right">
413+
<KernDropdown
414+
buttonName="Use code example"
415+
options={LLM_CODE_TEMPLATE_OPTIONS}
416+
dropdownWidth="w-52"
417+
disabled={currentAttribute.state == AttributeState.USABLE}
418+
selectedOption={selectCodeTemplate}
419+
/>
420+
</Tooltip>}
421+
</div>
354422
<div className="flex flex-row flex-nowrap">
355423
<VisitBricksButton urlExtension="generators" tooltipPlacement="left" size="small" />
356424
<Tooltip content={TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.AVAILABLE_LIBRARIES} placement="bottom" color="invert">

‎src/components/projects/projectId/attributes/attributeId/ExecutionContainer.tsx

+8-6
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import ViewRecordDetailsModal from "./ViewRecordDetailsModal";
1414
import { extendArrayElementsByUniqueId } from "@/submodules/javascript-functions/id-prep";
1515
import { getRecordByRecordId } from "@/src/services/base/project-setting";
1616
import { getSampleRecords } from "@/src/services/base/attribute";
17+
import { DataTypeEnum } from "@/src/types/shared/general";
1718

1819
export default function ExecutionContainer(props: ExecutionContainerProps) {
1920
const projectId = useSelector(selectProjectId);
@@ -56,34 +57,35 @@ export default function ExecutionContainer(props: ExecutionContainerProps) {
5657
dispatch(setModalStates(ModalEnum.VIEW_RECORD_DETAILS, { record: postProcessRecordByRecordId(res) }));
5758
});
5859
}
59-
6060
return (<div>
6161
<div className="mt-8 text-sm leading-5">
6262
<div className="text-gray-700 font-medium mr-2">
6363
Execution
6464
</div>
6565

6666
<div className="flex items-center">
67-
<div className="text-gray-500 font-normal">You can execute your attribute calculation
67+
<div className="text-gray-500 font-normal inline-flex flex-col gap-y-2 w-full">You can execute your attribute calculation
6868
on all records, or test-run it on 10 examples (which are sampled randomly). Test results are
6969
shown
70-
below after computation.</div>
70+
below after computation.{
71+
props.currentAttribute.dataType == DataTypeEnum.LLM_RESPONSE && <span className="italic">Note that LLM results are cached for the same set of settings to ensure that a critical error during the final execution doesn't lose the already calculated values</span>
72+
}</div>
7173
{requestedSomething && <div className="inline-block">
7274
<LoadingIcon color="indigo" />
7375
</div>}
7476

7577
<Tooltip content={TOOLTIPS_DICT.ATTRIBUTE_CALCULATION.EXECUTE_10_RECORDS} color="invert" placement="bottom" className="ml-auto">
7678
<button onClick={calculateUserAttributeSampleRecords}
7779
disabled={props.currentAttribute.state == AttributeState.USABLE || props.currentAttribute.state == AttributeState.RUNNING || requestedSomething || props.tokenizationProgress < 1 || props.checkUnsavedChanges}
78-
className={`bg-white text-gray-700 text-xs font-semibold px-4 py-2 rounded-md border border-gray-300 hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 disabled:opacity-50 disabled:cursor-not-allowed`}>
80+
className="bg-white text-gray-700 text-xs font-semibold px-4 py-2 rounded-md border whitespace-nowrap border-gray-300 hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 disabled:opacity-50 disabled:cursor-not-allowed">
7981
Run on 10
8082
</button>
8183
</Tooltip>
8284

8385
<Tooltip color="invert" placement="bottom" content={props.currentAttribute.state == AttributeState.USABLE ? 'Attribute is already in use' : requestedSomething ? 'Test is running' : checkIfAtLeastRunning ? 'Another attribute is running' : checkIfAtLeastQueued ? 'Another attribute is queued for execution' : props.tokenizationProgress < 1 ? 'Tokenization is in progress' : runOn10HasError ? 'Run on 10 records has an error' : 'Execute the attribute on all records'}>
8486
<button onClick={() => dispatch(setModalStates(ModalEnum.EXECUTE_ATTRIBUTE_CALCULATION, { open: true, requestedSomething: requestedSomething }))}
8587
disabled={props.currentAttribute.state == AttributeState.USABLE || props.currentAttribute.state == AttributeState.RUNNING || requestedSomething || checkIfAtLeastRunning || checkIfAtLeastQueued || props.tokenizationProgress < 1 || runOn10HasError || props.checkUnsavedChanges}
86-
className={`bg-indigo-700 text-white text-xs leading-4 font-semibold px-4 py-2 rounded-md cursor-pointer ml-3 hover:bg-indigo-800 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 disabled:opacity-50 disabled:cursor-not-allowed`}>
88+
className="bg-indigo-700 text-white text-xs leading-4 font-semibold px-4 py-2 rounded-md cursor-pointer ml-3 hover:bg-indigo-800 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 disabled:opacity-50 disabled:cursor-not-allowed">
8789
Run
8890
</button>
8991
</Tooltip>
@@ -99,7 +101,7 @@ export default function ExecutionContainer(props: ExecutionContainerProps) {
99101
<div key={record.id} className="divide-y divide-gray-200 bg-white">
100102
<div className="flex-shrink-0 border-b border-gray-200 shadow-sm flex justify-between items-center">
101103
<div className="flex items-center text-xs leading-5 text-gray-500 font-normal mx-4 my-3 text-justify">
102-
{record.value}
104+
{String(record.value)}
103105
</div>
104106
<div className="flex items-center justify-center mr-5 ml-auto">
105107
<button onClick={() => {

0 commit comments

Comments
 (0)