[backend] improve file indexing metrics performance (#7530)

opencti-platform/opencti-graphql/src/database/engine.js

@@ -2695,7 +2695,7 @@ export const elHistogramCount = async (context, user, indexName, options = {}) =
   });
 };
 export const elAggregationCount = async (context, user, indexName, options = {}) => {
-  const { field, types = null } = options;
+  const { field, types = null, weightField = 'i_inference_weight', normalizeLabel = true } = options;
   const isIdFields = field.endsWith('internal_id') || field.endsWith('.id');
   const body = await elQueryBodyBuilder(context, user, { ...options, noSize: true, noSort: true });
   body.size = 0;
@@ -2708,7 +2708,7 @@ export const elAggregationCount = async (context, user, indexName, options = {})
       aggs: {
         weight: {
           sum: {
-            field: 'i_inference_weight',
+            field: weightField,
             missing: 1,
           },
         },
@@ -2727,10 +2727,10 @@ export const elAggregationCount = async (context, user, indexName, options = {})
         let label = b.key;
         if (typeof label === 'number') {
           label = String(b.key);
-        } else if (!isIdFields) {
+        } else if (!isIdFields && normalizeLabel) {
           label = pascalize(b.key);
         }
-        return { label, value: b.weight.value };
+        return { label, value: b.weight.value, count: b.doc_count };
       });
     })
     .catch((err) => {

opencti-platform/opencti-graphql/src/domain/file.js

@@ -5,7 +5,7 @@
 import { buildContextDataForFile, completeContextDataForEntity, publishUserAction } from '../listener/UserActionListener';
 import { stixCoreObjectImportDelete } from './stixCoreObject';
 import { extractEntityRepresentativeName } from '../database/entity-representative';
-import { allFilesForPaths, allRemainingFilesCount } from '../modules/internal/document/document-domain';
+import { allFilesMimeTypeDistribution, allRemainingFilesCount } from '../modules/internal/document/document-domain';
 import { getManagerConfigurationFromCache } from '../modules/managerConfiguration/managerConfiguration-domain';
 import { supportedMimeTypes } from '../modules/managerConfiguration/managerConfiguration-utils';
 import { SYSTEM_USER } from '../utils/access';
@@ -45,22 +45,22 @@
   const metrics = await getStats([READ_INDEX_FILES]);
   const indexedFilesCount = metrics.docs.count;
   const fileOptions = await buildOptionsFromFileManager(context);
-  const potentialIndexingFiles = await allFilesForPaths(context, user, fileOptions.paths, fileOptions.opts);
+  const filesMimeTypesDistribution = await allFilesMimeTypeDistribution(context, user, fileOptions.paths, fileOptions.opts);
   const remainingFilesCount = await allRemainingFilesCount(context, user, fileOptions.paths, fileOptions.opts);
   const metricsByMimeType = [];
   supportedMimeTypes.forEach((mimeType) => {
-    const mimeTypeFiles = potentialIndexingFiles.filter((file) => file.metaData.mimetype?.startsWith(mimeType));
-    if (mimeTypeFiles.length > 0) {
+    const mimeTypeDistribution = filesMimeTypesDistribution.filter((dist) => dist.label.startsWith(mimeType));
+    if (mimeTypeDistribution.length > 0) {
       metricsByMimeType.push({
         mimeType,
-        count: mimeTypeFiles.length,
-        size: R.sum(mimeTypeFiles.map((file) => file.size)),
+        count: R.sum(mimeTypeDistribution.map((dist) => dist.count)),
+        size: R.sum(mimeTypeDistribution.map((dist) => dist.value)),
       });
     }
   });
   return {
     globalCount: indexedFilesCount + remainingFilesCount,
-    globalSize: R.sum(potentialIndexingFiles.map((file) => file.size)),
+    globalSize: R.sum(filesMimeTypesDistribution.map((dist) => dist.value)),
     metricsByMimeType,
   };
 };

opencti-platform/opencti-graphql/src/modules/internal/document/document-domain.ts

@@ -3,7 +3,7 @@
 import moment from 'moment';
 import { generateFileIndexId } from '../../../schema/identifier';
 import { ENTITY_TYPE_INTERNAL_FILE } from '../../../schema/internalObject';
-import { elCount, elDeleteInstances, elIndex } from '../../../database/engine';
+import { elAggregationCount, elCount, elDeleteInstances, elIndex } from '../../../database/engine';
 import { INDEX_INTERNAL_OBJECTS, isEmptyField, isNotEmptyField, READ_INDEX_INTERNAL_OBJECTS } from '../../../database/utils';
 import { type EntityOptions, type FilterGroupWithNested, internalLoadById, listAllEntities, listEntitiesPaginated, storeLoadById } from '../../../database/middleware-loader';
 import type { AuthContext, AuthUser } from '../../../types/user';
@@ -131,6 +131,20 @@
   return elCount(context, user, [READ_INDEX_INTERNAL_OBJECTS], remainingOpts);
 };
 
+export const allFilesMimeTypeDistribution = async (context: AuthContext, user: AuthUser, paths: string[], opts?: FilesOptions<BasicStoreEntityDocument>) => {
+  const findOpts: EntityOptions<BasicStoreEntityDocument> = {
+    filters: buildFileFilters(paths, opts),
+    noFiltersChecking: true // No associated model
+  };
+  return elAggregationCount(context, user, READ_INDEX_INTERNAL_OBJECTS, {
+    ...findOpts,
+    types: [ENTITY_TYPE_INTERNAL_FILE],
+    field: 'metaData.mimetype',
+    weightField: 'size',
+    normalizeLabel: false,
+  });
+};
+
 export const checkFileAccess = async (context: AuthContext, user: AuthUser, scope: string, { entity_id, filename, id }: { entity_id?: string, filename: string, id: string }) => {
   if (isEmptyField(entity_id)) {
     return true;