[api] Add migration to purge indicates indexing

opencti-platform/opencti-front/src/private/components/common/stix_domain_objects/StixDomainObjectKnowledge.js

@@ -269,7 +269,7 @@ class StixDomainObjectKnowledge extends Component {
             />
           </Grid>
           {stixDomainObjectType === 'Sector' && (
-            <Grid item={true} xs={6} style={{ marginBottom: 25 }}>
+            <Grid item={true} xs={6} style={{ height: 350 }}>
               <SectorTargetedOrganizations
                 sectorId={stixDomainObjectId}
                 link={link}

opencti-platform/opencti-graphql/src/database/elasticSearch.js

@@ -1131,7 +1131,7 @@ export const elPaginate = async (user, indexName, options = {}) => {
   const query = {
     index: indexName,
     ignore_throttled: IGNORE_THROTTLED,
-    _source_excludes: `${REL_INDEX_PREFIX}*`,
+    //_source_excludes: `${REL_INDEX_PREFIX}*`,
     track_total_hits: true,
     body,
   };

opencti-platform/opencti-graphql/src/migrations/1612381566895-clear_indicates_indexation.js

@@ -0,0 +1,41 @@
+import * as R from 'ramda';
+import { Promise } from 'bluebird';
+import { READ_DATA_INDICES } from '../database/utils';
+import { ENTITY_TYPE_INDICATOR } from '../schema/stixDomainObject';
+import { BULK_TIMEOUT, elBulk, elList, ES_MAX_CONCURRENCY, MAX_SPLIT } from '../database/elasticSearch';
+import { logger } from '../config/conf';
+import { SYSTEM_USER } from '../domain/user';
+import { ABSTRACT_STIX_CORE_OBJECT, ABSTRACT_STIX_CORE_RELATIONSHIP } from '../schema/general';
+
+export const up = async (next) => {
+  const start = new Date().getTime();
+  logger.info(`[MIGRATION] Cleaning indicates for all entities and relationships`);
+  const bulkOperations = [];
+  const callback = (entities) => {
+    const op = entities
+      .filter((n) => n.entity_type !== ENTITY_TYPE_INDICATOR)
+      .map((att) => {
+        return [{ update: { _index: att._index, _id: att.id } }, { doc: { 'rel_indicates.internal_id': null } }];
+      })
+      .flat();
+    bulkOperations.push(...op);
+  };
+  const filters = [{ key: 'rel_indicates.internal_id', values: ['EXISTS'] }];
+  const opts = { types: [ABSTRACT_STIX_CORE_OBJECT, ABSTRACT_STIX_CORE_RELATIONSHIP], filters, callback };
+  await elList(SYSTEM_USER, READ_DATA_INDICES, opts);
+  // Apply operations.
+  let currentProcessing = 0;
+  const groupsOfOperations = R.splitEvery(MAX_SPLIT, bulkOperations);
+  const concurrentUpdate = async (bulk) => {
+    await elBulk({ refresh: true, timeout: BULK_TIMEOUT, body: bulk });
+    currentProcessing += bulk.length;
+    logger.info(`[OPENCTI] Cleaning indicates indexation: ${currentProcessing} / ${bulkOperations.length}`);
+  };
+  await Promise.map(groupsOfOperations, concurrentUpdate, { concurrency: ES_MAX_CONCURRENCY });
+  logger.info(`[MIGRATION] Cleaning indicates done in ${new Date() - start} ms`);
+  next();
+};
+
+export const down = async (next) => {
+  next();
+};