Skip to content

Commit

Permalink
[api] Add migration to purge indicates indexing
Browse files Browse the repository at this point in the history
  • Loading branch information
Samuel Hassine committed Feb 3, 2021
1 parent 7a60041 commit 2a242ec
Show file tree
Hide file tree
Showing 3 changed files with 43 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ class StixDomainObjectKnowledge extends Component {
/>
</Grid>
{stixDomainObjectType === 'Sector' && (
<Grid item={true} xs={6} style={{ marginBottom: 25 }}>
<Grid item={true} xs={6} style={{ height: 350 }}>
<SectorTargetedOrganizations
sectorId={stixDomainObjectId}
link={link}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1131,7 +1131,7 @@ export const elPaginate = async (user, indexName, options = {}) => {
const query = {
index: indexName,
ignore_throttled: IGNORE_THROTTLED,
_source_excludes: `${REL_INDEX_PREFIX}*`,
//_source_excludes: `${REL_INDEX_PREFIX}*`,
track_total_hits: true,
body,
};
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import * as R from 'ramda';
import { Promise } from 'bluebird';
import { READ_DATA_INDICES } from '../database/utils';
import { ENTITY_TYPE_INDICATOR } from '../schema/stixDomainObject';
import { BULK_TIMEOUT, elBulk, elList, ES_MAX_CONCURRENCY, MAX_SPLIT } from '../database/elasticSearch';
import { logger } from '../config/conf';
import { SYSTEM_USER } from '../domain/user';
import { ABSTRACT_STIX_CORE_OBJECT, ABSTRACT_STIX_CORE_RELATIONSHIP } from '../schema/general';

export const up = async (next) => {
const start = new Date().getTime();
logger.info(`[MIGRATION] Cleaning indicates for all entities and relationships`);
const bulkOperations = [];
const callback = (entities) => {
const op = entities
.filter((n) => n.entity_type !== ENTITY_TYPE_INDICATOR)
.map((att) => {
return [{ update: { _index: att._index, _id: att.id } }, { doc: { 'rel_indicates.internal_id': null } }];
})
.flat();
bulkOperations.push(...op);
};
const filters = [{ key: 'rel_indicates.internal_id', values: ['EXISTS'] }];
const opts = { types: [ABSTRACT_STIX_CORE_OBJECT, ABSTRACT_STIX_CORE_RELATIONSHIP], filters, callback };
await elList(SYSTEM_USER, READ_DATA_INDICES, opts);
// Apply operations.
let currentProcessing = 0;
const groupsOfOperations = R.splitEvery(MAX_SPLIT, bulkOperations);
const concurrentUpdate = async (bulk) => {
await elBulk({ refresh: true, timeout: BULK_TIMEOUT, body: bulk });
currentProcessing += bulk.length;
logger.info(`[OPENCTI] Cleaning indicates indexation: ${currentProcessing} / ${bulkOperations.length}`);
};
await Promise.map(groupsOfOperations, concurrentUpdate, { concurrency: ES_MAX_CONCURRENCY });
logger.info(`[MIGRATION] Cleaning indicates done in ${new Date() - start} ms`);
next();
};

export const down = async (next) => {
next();
};

0 comments on commit 2a242ec

Please sign in to comment.