diff --git a/packages/@n8n/config/src/configs/logging.config.ts b/packages/@n8n/config/src/configs/logging.config.ts index 94e464222361a..ef4661c1159a2 100644 --- a/packages/@n8n/config/src/configs/logging.config.ts +++ b/packages/@n8n/config/src/configs/logging.config.ts @@ -7,6 +7,7 @@ export const LOG_SCOPES = [ 'external-secrets', 'license', 'multi-main-setup', + 'pruning', 'pubsub', 'redis', 'scaling', diff --git a/packages/cli/src/commands/start.ts b/packages/cli/src/commands/start.ts index 041b6a87411b4..bb8b56d32b437 100644 --- a/packages/cli/src/commands/start.ts +++ b/packages/cli/src/commands/start.ts @@ -27,7 +27,7 @@ import { Subscriber } from '@/scaling/pubsub/subscriber.service'; import { Server } from '@/server'; import { OrchestrationService } from '@/services/orchestration.service'; import { OwnershipService } from '@/services/ownership.service'; -import { PruningService } from '@/services/pruning.service'; +import { PruningService } from '@/services/pruning/pruning.service'; import { UrlService } from '@/services/url.service'; import { WaitTracker } from '@/wait-tracker'; import { WorkflowRunner } from '@/workflow-runner'; diff --git a/packages/cli/src/databases/repositories/execution.repository.ts b/packages/cli/src/databases/repositories/execution.repository.ts index 39f5e92cad21b..3b93d15ca00b2 100644 --- a/packages/cli/src/databases/repositories/execution.repository.ts +++ b/packages/cli/src/databases/repositories/execution.repository.ts @@ -513,7 +513,7 @@ export class ExecutionRepository extends Repository { .execute(); } - async hardDeleteSoftDeletedExecutions() { + async findSoftDeletedExecutions() { const date = new Date(); date.setHours(date.getHours() - this.globalConfig.pruning.hardDeleteBuffer); diff --git a/packages/cli/src/services/pruning/__tests__/pruning.service.test.ts b/packages/cli/src/services/pruning/__tests__/pruning.service.test.ts new file mode 100644 index 0000000000000..5ea26ad540f40 --- /dev/null +++ b/packages/cli/src/services/pruning/__tests__/pruning.service.test.ts @@ -0,0 +1,213 @@ +import type { GlobalConfig } from '@n8n/config'; +import { mock } from 'jest-mock-extended'; +import type { InstanceSettings } from 'n8n-core'; + +import type { MultiMainSetup } from '@/scaling/multi-main-setup.ee'; +import type { OrchestrationService } from '@/services/orchestration.service'; +import { mockLogger } from '@test/mocking'; + +import { PruningService } from '../pruning.service'; + +describe('PruningService', () => { + describe('init', () => { + it('should start pruning if leader', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: true }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock(), + ); + const startPruningSpy = jest.spyOn(pruningService, 'startPruning'); + + pruningService.init(); + + expect(startPruningSpy).toHaveBeenCalled(); + }); + + it('should not start pruning if follower', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: false }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock(), + ); + const startPruningSpy = jest.spyOn(pruningService, 'startPruning'); + + pruningService.init(); + + expect(startPruningSpy).not.toHaveBeenCalled(); + }); + + it('should register leadership events if multi-main setup is enabled', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: true }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock({ on: jest.fn() }), + }), + mock(), + ); + + pruningService.init(); + + // @ts-expect-error Private method + expect(pruningService.orchestrationService.multiMainSetup.on).toHaveBeenCalledWith( + 'leader-takeover', + expect.any(Function), + ); + + // @ts-expect-error Private method + expect(pruningService.orchestrationService.multiMainSetup.on).toHaveBeenCalledWith( + 'leader-stepdown', + expect.any(Function), + ); + }); + }); + + describe('isEnabled', () => { + it('should return `true` based on config if leader main', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: true, instanceType: 'main' }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock({ pruning: { isEnabled: true } }), + ); + + // @ts-expect-error Private method + const isEnabled = pruningService.isEnabled(); + + expect(isEnabled).toBe(true); + }); + + it('should return `false` based on config if leader main', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: true, instanceType: 'main' }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock({ pruning: { isEnabled: false } }), + ); + + // @ts-expect-error Private method + const isEnabled = pruningService.isEnabled(); + + expect(isEnabled).toBe(false); + }); + + it('should return `false` if non-main even if enabled', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: false, instanceType: 'worker' }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock({ pruning: { isEnabled: true } }), + ); + + // @ts-expect-error Private method + const isEnabled = pruningService.isEnabled(); + + expect(isEnabled).toBe(false); + }); + + it('should return `false` if follower main even if enabled', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: false, isFollower: true, instanceType: 'main' }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock({ pruning: { isEnabled: true }, multiMainSetup: { enabled: true } }), + ); + + // @ts-expect-error Private method + const isEnabled = pruningService.isEnabled(); + + expect(isEnabled).toBe(false); + }); + }); + + describe('startPruning', () => { + it('should not start pruning if service is disabled', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: true, instanceType: 'main' }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock({ pruning: { isEnabled: false } }), + ); + + // @ts-expect-error Private method + const setSoftDeletionInterval = jest.spyOn(pruningService, 'setSoftDeletionInterval'); + + // @ts-expect-error Private method + const scheduleHardDeletion = jest.spyOn(pruningService, 'scheduleHardDeletion'); + + pruningService.startPruning(); + + expect(setSoftDeletionInterval).not.toHaveBeenCalled(); + expect(scheduleHardDeletion).not.toHaveBeenCalled(); + }); + + it('should start pruning if service is enabled', () => { + const pruningService = new PruningService( + mockLogger(), + mock({ isLeader: true, instanceType: 'main' }), + mock(), + mock(), + mock({ + isMultiMainSetupEnabled: true, + multiMainSetup: mock(), + }), + mock({ pruning: { isEnabled: true } }), + ); + + const setSoftDeletionInterval = jest + // @ts-expect-error Private method + .spyOn(pruningService, 'setSoftDeletionInterval') + .mockImplementation(); + + const scheduleHardDeletion = jest + // @ts-expect-error Private method + .spyOn(pruningService, 'scheduleHardDeletion') + .mockImplementation(); + + pruningService.startPruning(); + + expect(setSoftDeletionInterval).toHaveBeenCalled(); + expect(scheduleHardDeletion).toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/cli/src/services/pruning.service.ts b/packages/cli/src/services/pruning/pruning.service.ts similarity index 75% rename from packages/cli/src/services/pruning.service.ts rename to packages/cli/src/services/pruning/pruning.service.ts index 943f8d30ca3eb..7314015091db7 100644 --- a/packages/cli/src/services/pruning.service.ts +++ b/packages/cli/src/services/pruning/pruning.service.ts @@ -3,12 +3,12 @@ import { BinaryDataService, InstanceSettings } from 'n8n-core'; import { jsonStringify } from 'n8n-workflow'; import { Service } from 'typedi'; -import { inTest, TIME } from '@/constants'; +import { TIME } from '@/constants'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; import { OnShutdown } from '@/decorators/on-shutdown'; import { Logger } from '@/logging/logger.service'; -import { OrchestrationService } from './orchestration.service'; +import { OrchestrationService } from '../orchestration.service'; @Service() export class PruningService { @@ -32,7 +32,9 @@ export class PruningService { private readonly binaryDataService: BinaryDataService, private readonly orchestrationService: OrchestrationService, private readonly globalConfig: GlobalConfig, - ) {} + ) { + this.logger = this.logger.scoped('pruning'); + } /** * @important Requires `OrchestrationService` to be initialized. @@ -49,9 +51,9 @@ export class PruningService { } } - private isPruningEnabled() { + private isEnabled() { const { instanceType, isFollower } = this.instanceSettings; - if (!this.globalConfig.pruning.isEnabled || inTest || instanceType !== 'main') { + if (!this.globalConfig.pruning.isEnabled || instanceType !== 'main') { return false; } @@ -66,23 +68,23 @@ export class PruningService { * @important Call this method only after DB migrations have completed. */ startPruning() { - if (!this.isPruningEnabled()) return; + if (!this.isEnabled()) return; if (this.isShuttingDown) { - this.logger.warn('[Pruning] Cannot start pruning while shutting down'); + this.logger.warn('Cannot start pruning while shutting down'); return; } - this.logger.debug('[Pruning] Starting soft-deletion and hard-deletion timers'); + this.logger.debug('Starting soft-deletion and hard-deletion timers'); this.setSoftDeletionInterval(); this.scheduleHardDeletion(); } stopPruning() { - if (!this.isPruningEnabled()) return; + if (!this.isEnabled()) return; - this.logger.debug('[Pruning] Removing soft-deletion and hard-deletion timers'); + this.logger.debug('Removing soft-deletion and hard-deletion timers'); clearInterval(this.softDeletionInterval); clearTimeout(this.hardDeletionTimeout); @@ -96,7 +98,7 @@ export class PruningService { this.rates.softDeletion, ); - this.logger.debug(`[Pruning] Soft-deletion scheduled every ${when}`); + this.logger.debug(`Soft-deletion scheduled every ${when}`); } private scheduleHardDeletion(rateMs = this.rates.hardDeletion) { @@ -113,27 +115,27 @@ export class PruningService { ? error.message : jsonStringify(error, { replaceCircularRefs: true }); - this.logger.error('[Pruning] Failed to hard-delete executions', { errorMessage }); + this.logger.error('Failed to hard-delete executions', { errorMessage }); }); }, rateMs); - this.logger.debug(`[Pruning] Hard-deletion scheduled for next ${when}`); + this.logger.debug(`Hard-deletion scheduled for next ${when}`); } /** * Mark executions as deleted based on age and count, in a pruning cycle. */ async softDeleteOnPruningCycle() { - this.logger.debug('[Pruning] Starting soft-deletion of executions'); + this.logger.debug('Starting soft-deletion of executions'); const result = await this.executionRepository.softDeletePrunableExecutions(); if (result.affected === 0) { - this.logger.debug('[Pruning] Found no executions to soft-delete'); + this.logger.debug('Found no executions to soft-delete'); return; } - this.logger.debug('[Pruning] Soft-deleted executions', { count: result.affected }); + this.logger.debug('Soft-deleted executions', { count: result.affected }); } @OnShutdown() @@ -147,26 +149,26 @@ export class PruningService { * @return Delay in ms after which the next cycle should be started */ private async hardDeleteOnPruningCycle() { - const ids = await this.executionRepository.hardDeleteSoftDeletedExecutions(); + const ids = await this.executionRepository.findSoftDeletedExecutions(); const executionIds = ids.map((o) => o.executionId); if (executionIds.length === 0) { - this.logger.debug('[Pruning] Found no executions to hard-delete'); + this.logger.debug('Found no executions to hard-delete'); return this.rates.hardDeletion; } try { - this.logger.debug('[Pruning] Starting hard-deletion of executions', { executionIds }); + this.logger.debug('Starting hard-deletion of executions', { executionIds }); await this.binaryDataService.deleteMany(ids); await this.executionRepository.deleteByIds(executionIds); - this.logger.debug('[Pruning] Hard-deleted executions', { executionIds }); + this.logger.debug('Hard-deleted executions', { executionIds }); } catch (error) { - this.logger.error('[Pruning] Failed to hard-delete executions', { + this.logger.error('Failed to hard-delete executions', { executionIds, error: error instanceof Error ? error.message : `${error}`, }); diff --git a/packages/cli/test/integration/pruning.service.test.ts b/packages/cli/test/integration/pruning.service.test.ts index 7ade1d3fe5aaf..76ee1079036ff 100644 --- a/packages/cli/test/integration/pruning.service.test.ts +++ b/packages/cli/test/integration/pruning.service.test.ts @@ -8,8 +8,7 @@ import { TIME } from '@/constants'; import type { ExecutionEntity } from '@/databases/entities/execution-entity'; import type { WorkflowEntity } from '@/databases/entities/workflow-entity'; import { ExecutionRepository } from '@/databases/repositories/execution.repository'; -import { Logger } from '@/logging/logger.service'; -import { PruningService } from '@/services/pruning.service'; +import { PruningService } from '@/services/pruning/pruning.service'; import { annotateExecution, @@ -18,7 +17,7 @@ import { } from './shared/db/executions'; import { createWorkflow } from './shared/db/workflows'; import * as testDb from './shared/test-db'; -import { mockInstance } from '../shared/mocking'; +import { mockInstance, mockLogger } from '../shared/mocking'; describe('softDeleteOnPruningCycle()', () => { let pruningService: PruningService; @@ -35,7 +34,7 @@ describe('softDeleteOnPruningCycle()', () => { globalConfig = Container.get(GlobalConfig); pruningService = new PruningService( - mockInstance(Logger), + mockLogger(), instanceSettings, Container.get(ExecutionRepository), mockInstance(BinaryDataService),