[backend] add scripts for easier tests development (#5796)

opencti-platform/opencti-graphql/package.json

@@ -19,7 +19,11 @@
     "start-cluster": "NODE_ENV=cluster node build/back.js",
     "serv": "node build/back.js",
     "insert:dev": "node build/script-insert-dataset.js",
-    "test:dev": "vitest watch --config vitest.config.dev.ts",
+    "test:dev:init": "INIT_TEST_PLATFORM=true vitest run --config vitest.config.dev.ts --testNamePattern=\"Should import creation succeed\" tests/02-integration/00-inject/loader-test.ts",
+    "test:dev:resume": "SKIP_CLEANUP_PLATFORM=true vitest run --config vitest.config.dev.ts",
+    "test:dev": "vitest run --config vitest.config.dev.ts",
+    "test:watch:resume": "SKIP_CLEANUP_PLATFORM=true vitest watch --config vitest.config.dev.ts",
+    "test:watch": "vitest watch --config vitest.config.dev.ts",
     "test": "vitest --silent run --config vitest.config.test.ts --coverage"
   },
   "pkg": {

opencti-platform/opencti-graphql/tests/02-integration/00-inject/loader-test.js → opencti-platform/opencti-graphql/tests/02-integration/00-inject/loader-test.ts

@@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest';
 import { ADMIN_USER, API_TOKEN, API_URI, FIVE_MINUTES, PYTHON_PATH, testContext } from '../../utils/testQuery';
 import { execChildPython } from '../../../src/python/pythonBridge';
 
-const importOpts = [API_URI, API_TOKEN, './tests/data/DATA-TEST-STIX2_v2.json'];
+const importOpts: string[] = [API_URI, API_TOKEN, './tests/data/DATA-TEST-STIX2_v2.json'];
 
 describe('Database provision', () => {
   it('Should import creation succeed', async () => {

opencti-platform/opencti-graphql/tests/02-integration/05-parser/csv-mapper-test.js

@@ -64,11 +64,8 @@ const mapData = async (fileName, mapper, user = ADMIN_USER) => {
  * /!\
  * To run those tests, we need the data injected by loader-test.
  * So if you want to run only this file follow the steps below:
- * - run 'yarn test:dev loader-test' to inject data in your DB,
- * - after test is done, stop the run,
- * - comment the line 'await elDeleteIndices(...);' in globalSetup.js (DO NOT COMMIT THIS CHANGES),
- * - comment the line 'await createTestUsers();' in globalSetup.js (DO NOT COMMIT THIS CHANGES),
- * - run 'yarn test:dev csv-mapper-test' should be ok
+ * - run 'yarn test:dev:init" to set up a seeded test DB
+ * - run 'yarn test:dev:resume csv-mapper-test'
  */
 describe('CSV-MAPPER', () => {
   let individual;

opencti-platform/opencti-graphql/tests/utils/globalSetup.ts

@@ -0,0 +1,142 @@
+// region static graphql modules, need to be imported before everything
+import '../../src/modules/index';
+// import managers
+import '../../src/manager/index';
+// endregion
+import { deleteBucket, initializeBucket } from '../../src/database/file-storage';
+import { deleteQueues } from '../../src/domain/connector';
+import { ADMIN_USER, createTestUsers, isPlatformAlive, testContext } from './testQuery';
+import { elDeleteIndices, elPlatformIndices, initializeSchema, searchEngineInit } from '../../src/database/engine';
+import { wait } from '../../src/database/utils';
+import { createRedisClient, shutdownRedisClients } from '../../src/database/redis';
+import { logApp } from '../../src/config/conf';
+import cacheManager from '../../src/manager/cacheManager';
+import { initializeAdminUser } from '../../src/config/providers';
+import { initDefaultNotifiers } from '../../src/modules/notifier/notifier-domain';
+import { initializeInternalQueues } from '../../src/database/rabbitmq';
+import { executionContext } from '../../src/utils/access';
+import { initializeData } from '../../src/database/data-initialization';
+import { shutdownModules, startModules } from '../../src/managers';
+
+/**
+ * Vitest setup is configurable with environment variables, as you can see in our package.json scripts
+ *   INIT_TEST_PLATFORM=1 > cleanup the test platform, removing elastic indices, and setup it again
+ *   SKIP_CLEANUP_PLATFORM=1 > skip cleanup, and directly start the platform
+ *
+ * run yarn test:dev-init-only to cleanup and reinit a test platform (it also provision the data)
+ * run yarn test:dev-no-cleanup <file-pattern> to run directly some tests without cleanup and init of the test platform
+ */
+
+const { INIT_TEST_PLATFORM, SKIP_CLEANUP_PLATFORM } = process.env;
+
+const initializePlatform = async () => {
+  const context = executionContext('platform_test_initialization');
+  logApp.info('[vitest-global-setup] initializing platform');
+  const stopTime = new Date().getTime();
+
+  await initializeInternalQueues();
+  await initializeBucket();
+  await initializeSchema();
+  await initializeData(context, true);
+  await initializeAdminUser(context);
+  await initDefaultNotifiers(context);
+  logApp.info(`[vitest-global-setup] Platform initialized in ${new Date().getTime() - stopTime} ms`);
+};
+
+const testPlatformStart = async () => {
+  const stopTime = new Date().getTime();
+  logApp.info('[vitest-global-setup] Starting platform');
+  try {
+    // Check all dependencies access
+    await searchEngineInit();
+    // Init the cache manager
+    await cacheManager.start();
+    // Init the platform default if it was cleaned up
+    if (!SKIP_CLEANUP_PLATFORM) {
+      await initializePlatform();
+    }
+    // Init the modules
+    await startModules();
+    logApp.info(`[vitest-global-setup] Platform started in ${new Date().getTime() - stopTime} ms`);
+  } catch (e) {
+    logApp.error(e);
+    process.exit(1);
+  }
+};
+
+const testPlatformStop = async () => {
+  logApp.info('[vitest-global-setup] stopping platform');
+  const stopTime = new Date().getTime();
+  // Shutdown the cache manager
+  await cacheManager.shutdown();
+  // Destroy the modules
+  await shutdownModules();
+  // Shutdown the redis clients
+  await shutdownRedisClients();
+  logApp.info(`[vitest-global-setup] Platform stopped in ${new Date().getTime() - stopTime} ms`);
+};
+
+const platformClean = async () => {
+  logApp.info('[vitest-global-setup] cleaning up platform');
+  const stopTime = new Date().getTime();
+  // Delete the bucket
+  await deleteBucket();
+  // Delete all rabbitmq queues
+  await deleteQueues(testContext, ADMIN_USER);
+  // Remove all elastic indices
+  const indices = await elPlatformIndices();
+  await elDeleteIndices(indices.map((i: { index: number }) => i.index));
+  // Delete redis streams
+  const testRedisClient = createRedisClient('reset');
+  await testRedisClient.del('stream.opencti');
+  testRedisClient.disconnect();
+  logApp.info(`[vitest-global-setup] Platform cleaned up in ${new Date().getTime() - stopTime} ms`);
+};
+
+const waitPlatformIsAlive = async (): Promise<true> => {
+  const isAlive = await isPlatformAlive();
+  if (!isAlive) {
+    logApp.info('[vitest-global-setup] ping platform ...');
+    await wait(1000);
+    return waitPlatformIsAlive();
+  }
+  logApp.info('[vitest-global-setup] platform is alive');
+  return true;
+};
+
+export async function setup() {
+  // cleanup and setup a seeded platform, with all the tests users, ready to run some tests.
+  if (INIT_TEST_PLATFORM) {
+    logApp.info('[vitest-global-setup] only running test platform initialization');
+    const stopTime = new Date().getTime();
+    await platformClean();
+    await testPlatformStart();
+    await waitPlatformIsAlive();
+    logApp.info('[vitest-global-setup] creating test users');
+    await createTestUsers();
+    logApp.info(`[vitest-global-setup] Test Platform initialization done in ${new Date().getTime() - stopTime} ms`);
+    return;
+  }
+
+  if (!SKIP_CLEANUP_PLATFORM) {
+    // Platform cleanup before executing tests
+    logApp.info('[vitest-global-setup] Cleaning up test platform...');
+    await platformClean();
+  } else {
+    logApp.info('[vitest-global-setup] !!! skipping platform cleanup and setup - database state is the same as your last run !!!');
+  }
+  // Start the platform
+  await testPlatformStart();
+
+  // setup tests users
+  if (!SKIP_CLEANUP_PLATFORM) {
+    await waitPlatformIsAlive();
+    logApp.info('[vitest-global-setup] Creating test users...');
+    await createTestUsers();
+  }
+}
+
+export async function teardown() {
+  // Stop the platform
+  await testPlatformStop();
+}

opencti-platform/opencti-graphql/tests/utils/testQuery.ts

@@ -473,6 +473,20 @@ export const buildStandardUser = (allowedMarkings: markingType[], allMarkings?:
   };
 };
 
+// TODO: use a real healthcheck query
+const HEALTHCHECK_QUERY = `
+  query {
+    about {
+      version
+    }
+  }
+`;
+
+export const isPlatformAlive = async () => {
+  const { data } = await adminQuery(HEALTHCHECK_QUERY, { });
+  return !!data?.about.version;
+};
+
 const serverFromUser = new ApolloServer<AuthContext>({
   schema: createSchema(),
   introspection: true,

opencti-platform/opencti-graphql/vitest.config.test.ts

@@ -10,7 +10,7 @@ export const buildTestConfig = (include: string[]) => defineConfig({
     include,
     testTimeout: 1200000,
     teardownTimeout: 20000,
-    globalSetup: ['./tests/utils/globalSetup.js'],
+    globalSetup: ['./tests/utils/globalSetup.ts'],
     setupFiles: ['./tests/utils/testSetup.js'],
     coverage: {
       provider: 'v8',
@@ -41,4 +41,3 @@ export const buildTestConfig = (include: string[]) => defineConfig({
 });
 
 export default buildTestConfig(['tests/**/*-test.{js,mjs,cjs,ts,mts,cts,jsx,tsx}']);
-// export default buildTestConfig(['tests/(02)-*/**/(loader|filterGroup|workspace)*-test.{js,mjs,cjs,ts,mts,cts,jsx,tsx}']);