[backend] enforce user confidence level on create/delete/update/upser…

…t operations (#5697)

opencti-platform/opencti-front/src/private/components/common/containers/ContainerStixCoreObjectPopover.jsx

@@ -224,7 +224,6 @@ class ContainerStixCoreObjectPopover extends Component {
         }
       },
       onCompleted: () => {
-        this.submitDeleteMapping();
         this.handleCloseDelete();
         const newSelectedElements = R.omit([toId], selectedElements);
         setSelectedElements?.(newSelectedElements);

opencti-platform/opencti-front/src/private/components/common/files/workbench/WorkbenchFileContent.jsx

@@ -38,7 +38,7 @@ import StixItemLabels from '../../../../../components/StixItemLabels';
 import StixItemMarkings from '../../../../../components/StixItemMarkings';
 import SwitchField from '../../../../../components/SwitchField';
 import TextField from '../../../../../components/TextField';
-import { APP_BASE_PATH, commitMutation, MESSAGING$, QueryRenderer } from '../../../../../relay/environment';
+import { APP_BASE_PATH, commitMutation, handleError, MESSAGING$, QueryRenderer } from '../../../../../relay/environment';
 import { observableValue, resolveIdentityClass, resolveIdentityType, resolveLink, resolveLocationType, resolveThreatActorType } from '../../../../../utils/Entity';
 import { defaultKey, defaultValue } from '../../../../../utils/Graph';
 import useAttributes from '../../../../../utils/hooks/useAttributes';
@@ -807,6 +807,12 @@ const WorkbenchFileContentComponent = ({
                 history.push('/dashboard/data/import');
               }
             },
+            onError: (error) => {
+              handleError(error);
+              setSubmitting(false);
+              resetForm();
+              setDisplayValidate(false);
+            },
           });
         }, 2000);
       },

opencti-platform/opencti-front/src/private/components/data/tasks/TasksList.jsx

@@ -391,8 +391,9 @@ const TasksList = ({ data }) => {
               </Grid>
               <Button
                 style={{ position: 'absolute', right: 10, top: 10 }}
-                variant="contained"
-                color="secondary"
+                variant={task.errors.length > 0 ? 'contained' : 'outlined'}
+                color={'error'}
+                disabled={task.errors.length === 0}
                 onClick={() => handleOpenErrors(task.errors)}
                 size="small"
               >

opencti-platform/opencti-front/src/relay/environment.jsx

@@ -5,7 +5,6 @@ import { debounce } from 'rxjs/operators';
 import React, { Component } from 'react';
 import { commitLocalUpdate as CLU, commitMutation as CM, QueryRenderer as QR, requestSubscription as RS, fetchQuery as FQ } from 'react-relay';
 import * as PropTypes from 'prop-types';
-import { map, isEmpty, filter, pathOr, isNil } from 'ramda';
 import { urlMiddleware, RelayNetworkLayer } from 'react-relay-network-modern';
 import * as R from 'ramda';
 import uploadMiddleware from './uploadMiddleware';
@@ -29,7 +28,7 @@ export class ApplicationError extends Error {
 }
 
 // Network
-const isEmptyPath = isNil(window.BASE_PATH) || isEmpty(window.BASE_PATH);
+const isEmptyPath = R.isNil(window.BASE_PATH) || R.isEmpty(window.BASE_PATH);
 const contextPath = isEmptyPath || window.BASE_PATH === '/' ? '' : window.BASE_PATH;
 export const APP_BASE_PATH = isEmptyPath || contextPath.startsWith('/') ? contextPath : `/${contextPath}`;
 
@@ -89,10 +88,10 @@ QueryRenderer.propTypes = {
   query: PropTypes.object,
 };
 
-const buildErrorMessages = (error) => map(
+const buildErrorMessages = (error) => R.map(
   (e) => ({
     type: 'error',
-    text: pathOr(e.message, ['data', 'reason'], e),
+    text: R.pathOr(e.message, ['data', 'reason'], e),
   }),
   error.res.errors,
 );
@@ -126,11 +125,11 @@ export const commitMutation = ({
   onError: (error) => {
     if (setSubmitting) setSubmitting(false);
     if (error && error.res && error.res.errors) {
-      const authRequired = filter(
-        (e) => pathOr(e.message, ['data', 'type'], e) === 'authentication',
+      const authRequired = R.filter(
+        (e) => R.pathOr(e.message, ['data', 'type'], e) === 'authentication',
         error.res.errors,
       );
-      if (!isEmpty(authRequired)) {
+      if (!R.isEmpty(authRequired)) {
         MESSAGING$.notifyError('Unauthorized action, please refresh your browser');
       } else if (onError) {
         const messages = buildErrorMessages(error);
@@ -157,10 +156,10 @@ export const handleErrorInForm = (error, setErrors) => {
         formattedError.data.message || formattedError.data.reason,
     });
   } else {
-    const messages = map(
+    const messages = R.map(
       (e) => ({
         type: 'error',
-        text: pathOr(e.message, ['data', 'reason'], e),
+        text: R.pathOr(e.message, ['data', 'reason'], e),
       }),
       error.res.errors,
     );
@@ -170,10 +169,10 @@ export const handleErrorInForm = (error, setErrors) => {
 
 export const handleError = (error) => {
   if (error && error.res && error.res.errors) {
-    const messages = map(
+    const messages = R.map(
       (e) => ({
         type: 'error',
-        text: pathOr(e.message, ['data', 'message'], e),
+        text: R.pathOr(e.message, ['data', 'message'], e),
       }),
       error.res.errors,
     );

opencti-platform/opencti-graphql/src/database/file-storage.js

@@ -17,6 +17,7 @@ import { elDeleteFilesByIds } from './file-search';
 import { isAttachmentProcessorEnabled } from './engine';
 import { allFilesForPaths, deleteDocumentIndex, indexFileToDocument } from '../modules/internal/document/document-domain';
 import { truncate } from '../utils/mailData';
+import { controlUserConfidenceAgainstElement } from '../utils/confidence-level';
 
 // Minio configuration
 const clientEndpoint = conf.get('minio:endpoint');
@@ -369,9 +370,20 @@ export const upload = async (context, user, filePath, fileUpload, opts) => {
   };
   // Register in elastic
   await indexFileToDocument(file);
+
+  // confidence control on the context entity (like a report) if we want auto-enrichment
+  let isConfidenceMatch = true;
+  if (entity) {
+    // noThrow ; we do not want to fail here as it's an automatic process.
+    // we will simply not start the job
+    isConfidenceMatch = controlUserConfidenceAgainstElement(user, entity, true);
+  }
+  const isFilePathForImportEnrichment = filePath.startsWith('import/')
+    && !filePath.startsWith('import/pending')
+    && !filePath.startsWith('import/External-Reference');
+
   // Trigger a enrich job for import file if needed
-  if (!noTriggerImport && filePath.startsWith('import/') && !filePath.startsWith('import/pending')
-      && !filePath.startsWith('import/External-Reference')) {
+  if (!noTriggerImport && isConfidenceMatch && isFilePathForImportEnrichment) {
     await uploadJobImport(context, user, file.id, file.metaData.mimetype, file.metaData.entity_id);
   }
   return file;

opencti-platform/opencti-graphql/src/database/middleware.js

@@ -198,6 +198,12 @@ import { confidence } from '../schema/attribute-definition';
 import { ENTITY_TYPE_INDICATOR } from '../modules/indicator/indicator-types';
 import { FilterMode, FilterOperator } from '../generated/graphql';
 import { getMandatoryAttributesForSetting } from '../modules/entitySetting/entitySetting-attributeUtils';
+import {
+  controlCreateInputWithUserConfidence,
+  controlUserConfidenceAgainstElement,
+  controlUpsertInputWithUserConfidence,
+  adaptUpdateInputsConfidence
+} from '../utils/confidence-level';
 
 // region global variables
 const MAX_BATCH_SIZE = 300;
@@ -1851,7 +1857,7 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs
         const { value: targetsCreated } = meta[metaIndex];
         const targetCreated = R.head(targetsCreated);
         // If asking for a real change
-        if (currentValue?.standard_id !== targetCreated?.internal_id && currentValue?.id !== targetCreated?.internal_id) {
+        if (currentValue?.id !== targetCreated?.internal_id) {
           // Delete the current relation
           if (currentValue?.standard_id) {
             const currentRels = await listAllRelations(context, user, relType, { fromId: initial.id });
@@ -1982,12 +1988,15 @@ export const updateAttributeFromLoadedWithRefs = async (context, user, initial,
   if (!initial) {
     throw FunctionalError('Cant update undefined element');
   }
-  const updates = Array.isArray(inputs) ? inputs : [inputs];
+  // region confidence control
+  controlUserConfidenceAgainstElement(user, initial);
+  const newInputs = adaptUpdateInputsConfidence(user, inputs, initial);
+  // endregion
   const metaKeys = [...schemaRelationsRefDefinition.getStixNames(initial.entity_type), ...schemaRelationsRefDefinition.getInputNames(initial.entity_type)];
-  const meta = updates.filter((e) => metaKeys.includes(e.key));
+  const meta = newInputs.filter((e) => metaKeys.includes(e.key));
   const metaIds = R.uniq(meta.map((i) => i.value ?? []).flat());
   const metaDependencies = await elFindByIds(context, user, metaIds, { toMap: true, mapWithAllIds: true });
-  const revolvedInputs = updates.map((input) => {
+  const revolvedInputs = newInputs.map((input) => {
     if (metaKeys.includes(input.key)) {
       const resolvedValues = (input.value ?? []).map((refId) => metaDependencies[refId]).filter((o) => isNotEmptyField(o));
       return { ...input, value: resolvedValues };
@@ -2404,8 +2413,13 @@ const upsertElement = async (context, user, element, type, basePatch, opts = {})
     const fileImpact = { key: 'x_opencti_files', value: [...(element.x_opencti_files ?? []), convertedFile] };
     inputs.push(fileImpact);
   }
-  // If confidence is passed at creation, just compare confidence
-  const isConfidenceMatch = (updatePatch.confidence ?? 0) >= (element.confidence ?? 0);
+
+  // region confidence control / upsert
+  const { confidenceLevelToApply, isConfidenceMatch } = controlUpsertInputWithUserConfidence(user, updatePatch, element);
+  updatePatch.confidence = confidenceLevelToApply;
+  // note that if the existing data has no confidence (null) it will still be updated below, even if isConfidenceMatch = false
+  // endregion
+
   // -- Upsert attributes
   const attributes = Array.from(schemaAttributesDefinition.getAttributes(type).values());
   for (let attrIndex = 0; attrIndex < attributes.length; attrIndex += 1) {
@@ -2441,24 +2455,34 @@ const upsertElement = async (context, user, element, type, basePatch, opts = {})
       const isUpsertSynchro = (context.synchronizedUpsert || inputField === INPUT_GRANTED_REFS); // Granted Refs are always fully sync
       if (relDef.multiple) {
         const currentData = element[relDef.databaseName] ?? [];
+        const isCurrentWithData = isNotEmptyField(currentData);
         const targetData = (patchInputData ?? []).map((n) => n.internal_id);
         // If expected data is different from current data
         if (R.symmetricDifference(currentData, targetData).length > 0) {
           const diffTargets = (patchInputData ?? []).filter((target) => !currentData.includes(target.internal_id));
           // In full synchro, just replace everything
           if (isUpsertSynchro) {
             inputs.push({ key: inputField, value: patchInputData ?? [], operation: UPDATE_OPERATION_REPLACE });
-          } else if (isInputWithData && diffTargets.length > 0) {
-            // If data is provided and different from existing data, apply an add operation
+          } else if (
+            (isCurrentWithData && isInputWithData && diffTargets.length > 0 && isConfidenceMatch)
+            || (isInputWithData && !isCurrentWithData)
+          ) {
+            // If data is provided, different from existing data, and of higher confidence
+            // OR if existing data is empty and data is provided (even if lower confidence, it's better than nothing),
+            // --> apply an add operation
             inputs.push({ key: inputField, value: diffTargets, operation: UPDATE_OPERATION_ADD });
           }
         }
-      } else {
+      } else { // not multiple
+        // If expected data is different from current data...
         const currentData = element[relDef.databaseName];
-        const updatable = isUpsertSynchro || (isInputWithData && isEmptyField(currentData));
-        // If expected data is different from current data
-        // And data can be updated (complete a null value or forced through synchro upsert option
-        if (!R.equals(currentData, patchInputData) && updatable) {
+        const isInputDifferentFromCurrent = !R.equals(currentData, patchInputData);
+        // ... and data can be updated:
+        //   forced synchro
+        //   OR the field was null -> better than nothing !
+        //   OR the confidence matches -> new value is "better" than existing value
+        const updatable = isUpsertSynchro || (isInputWithData && isEmptyField(currentData)) || isConfidenceMatch;
+        if (isInputDifferentFromCurrent && updatable) {
           inputs.push({ key: inputField, value: [patchInputData] });
         }
       }
@@ -2627,10 +2651,17 @@ export const buildRelationData = async (context, user, input, opts = {}) => {
   };
 };
 
-export const createRelationRaw = async (context, user, input, opts = {}) => {
+export const createRelationRaw = async (context, user, rawInput, opts = {}) => {
   let lock;
   const { fromRule, locks = [] } = opts;
-  const { fromId, toId, relationship_type: relationshipType } = input;
+  const { fromId, toId, relationship_type: relationshipType } = rawInput;
+
+  // region confidence control
+  const input = structuredClone(rawInput);
+  const { confidenceLevelToApply } = controlCreateInputWithUserConfidence(user, input);
+  input.confidence = confidenceLevelToApply; // confidence of the new relation will be capped to user's confidence
+  // endregion
+
   // Pre-check before inputs resolution
   if (fromId === toId) {
     /* v8 ignore next */
@@ -2643,6 +2674,12 @@ export const createRelationRaw = async (context, user, input, opts = {}) => {
   // We need to check existing dependencies
   const resolvedInput = await inputResolveRefs(context, user, filledInput, relationshipType, entitySetting);
   const { from, to } = resolvedInput;
+
+  // when creating stix ref, we must check confidence on from side (this count has modifying this element itself)
+  if (isStixRefRelationship(relationshipType)) {
+    controlUserConfidenceAgainstElement(user, from);
+  }
+
   // check if user has "edit" access on from and to
   if (!validateUserAccessOperation(user, from, 'edit') || !validateUserAccessOperation(user, to, 'edit')) {
     throw ForbiddenAccess();
@@ -2742,10 +2779,8 @@ export const createRelationRaw = async (context, user, input, opts = {}) => {
         });
       }
       // TODO Handling merging relation when updating to prevent multiple relations finding
-      existingRelationship = R.head(filteredRelations);
-      // We can use the resolved input from/to to complete the element
-      existingRelationship.from = from;
-      existingRelationship.to = to;
+      // resolve all refs so we can upsert properly
+      existingRelationship = await storeLoadByIdWithRefs(context, user, R.head(filteredRelations).internal_id);
     }
     // endregion
     if (existingRelationship) {
@@ -3003,12 +3038,17 @@ const buildEntityData = async (context, user, input, type, opts = {}) => {
   };
 };
 
-const createEntityRaw = async (context, user, input, type, opts = {}) => {
-  // Region - Pre-Check
+const createEntityRaw = async (context, user, rawInput, type, opts = {}) => {
+  // region confidence control
+  const input = structuredClone(rawInput);
+  const { confidenceLevelToApply } = controlCreateInputWithUserConfidence(user, input);
+  input.confidence = confidenceLevelToApply; // confidence of new entity will be capped to user's confidence
+  // endregion
+  // region - Pre-Check
   const entitySetting = await getEntitySettingFromCache(context, type);
   const filledInput = fillDefaultValues(user, input, entitySetting);
   await validateEntityAndRelationCreation(context, user, filledInput, type, entitySetting, opts);
-  // Endregion
+  // endregion
   const { fromRule } = opts;
   // We need to check existing dependencies
   const resolvedInput = await inputResolveRefs(context, user, filledInput, type, entitySetting);
@@ -3168,6 +3208,13 @@ export const internalDeleteElementById = async (context, user, id, opts = {}) =>
   if (!element) {
     throw AlreadyDeletedError({ id });
   }
+  // region confidence control
+  controlUserConfidenceAgainstElement(user, element);
+  // when deleting stix ref, we must check confidence on from side (this count has modifying this element itself)
+  if (isStixRefRelationship(element.entity_type)) {
+    controlUserConfidenceAgainstElement(user, element.from);
+  }
+  // endregion
   // Prevent individual deletion if linked to a user
   if (element.entity_type === ENTITY_TYPE_IDENTITY_INDIVIDUAL && !isEmptyField(element.contact_information)) {
     const args = {

opencti-platform/opencti-graphql/src/domain/backgroundTask.js

@@ -13,6 +13,7 @@ import { ENTITY_TYPE_VOCABULARY } from '../modules/vocabulary/vocabulary-types';
 import { ENTITY_TYPE_NOTIFICATION } from '../modules/notification/notification-types';
 import { ENTITY_TYPE_CASE_TEMPLATE } from '../modules/case/case-template/case-template-types';
 import { ENTITY_TYPE_LABEL } from '../schema/stixMetaObject';
+import { adaptFiltersWithUserConfidence } from '../utils/confidence-level';
 
 export const DEFAULT_ALLOWED_TASK_ENTITY_TYPES = [
   ABSTRACT_STIX_CORE_OBJECT,
@@ -44,20 +45,22 @@ export const findAll = (context, user, args) => {
   return listEntities(context, user, [ENTITY_TYPE_BACKGROUND_TASK], args);
 };
 
-const buildQueryFilters = async (rawFilters, search, taskPosition) => {
+const buildQueryFilters = async (user, filters, search, taskPosition) => {
+  const inputFilters = filters ? JSON.parse(filters) : undefined;
+  const finalFilters = adaptFiltersWithUserConfidence(user, inputFilters);
   // Construct filters
   return {
     types: DEFAULT_ALLOWED_TASK_ENTITY_TYPES,
     first: MAX_TASK_ELEMENTS,
     orderMode: 'asc',
     orderBy: 'created_at',
     after: taskPosition,
-    filters: rawFilters ? JSON.parse(rawFilters) : undefined,
+    filters: finalFilters,
     search: search && search.length > 0 ? search : null,
   };
 };
 export const executeTaskQuery = async (context, user, filters, search, start = null) => {
-  const options = await buildQueryFilters(filters, search, start);
+  const options = await buildQueryFilters(user, filters, search, start);
   return elPaginate(context, user, READ_DATA_INDICES_WITHOUT_INFERRED, options);
 };
 

opencti-platform/opencti-graphql/src/domain/file.js

@@ -12,6 +12,7 @@ import { supportedMimeTypes } from '../modules/managerConfiguration/managerConfi
 import { SYSTEM_USER } from '../utils/access';
 import { isEmptyField, isNotEmptyField, READ_INDEX_FILES } from '../database/utils';
 import { getStats } from '../database/engine';
+import { controlUserConfidenceAgainstElement } from '../utils/confidence-level';
 
 export const buildOptionsFromFileManager = async (context) => {
   let importPaths = ['import/'];
@@ -71,6 +72,12 @@ export const askJobImport = async (context, user, args) => {
   const entityId = bypassEntityId || file.metaData.entity_id;
   const opts = { manual: true, connectorId, configuration, bypassValidation };
   const entity = await internalLoadById(context, user, entityId);
+
+  // This is a manual request for import, we have to check confidence and throw on error
+  if (entity) {
+    controlUserConfidenceAgainstElement(user, entity);
+  }
+
   const connectors = await uploadJobImport(context, user, file.id, file.metaData.mimetype, entityId, opts);
   const entityName = entityId ? extractEntityRepresentativeName(entity) : 'global';
   const entityType = entityId ? entity.entity_type : 'global';

opencti-platform/opencti-graphql/src/domain/group.js

@@ -129,6 +129,10 @@ export const groupEditField = async (context, user, groupId, input) => {
     message: `updates \`${input.map((i) => i.key).join(', ')}\` for group \`${element.name}\``,
     context_data: { id: groupId, entity_type: ENTITY_TYPE_GROUP, input }
   });
+  // on editing the group confidence level, all memebers might have changed their effective level
+  if (input.find((i) => i.key === 'group_confidence_level')) {
+    await groupSessionRefresh(context, user, groupId);
+  }
   return notify(BUS_TOPICS[ENTITY_TYPE_GROUP].EDIT_TOPIC, element, user);
 };