Skip to content

Commit dcb2b8c

Browse files
NinjeneerNinjeneer
committed
feat(jobs): restart scans
1 parent 50256b3 commit dcb2b8c

File tree

19 files changed

+190
-35
lines changed

19 files changed

+190
-35
lines changed

‎.vscode/settings.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"editor.codeActionsOnSave": {
33
"source.fixAll.eslint": true
44
},
5-
"jest.autoRun": {}
5+
"jest.autoRun": {},
66
}

‎docker-compose.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ services:
77
target: dev
88
context: ./
99

10-
image: ninjeneer/vuln-request-service
10+
image: ninjeneer/vuln-scanner-request-service
1111
ports:
1212
- 3000:3000
1313
volumes:
@@ -21,14 +21,26 @@ services:
2121
target: dev
2222
context: ./
2323

24-
image: ninjeneer/vuln-report-service
24+
image: ninjeneer/vuln-scanner-report-service
2525
ports:
2626
- 3001:3000
2727
volumes:
2828
- .:/app
2929
networks:
3030
- vulnscanner
3131

32+
jobs-service:
33+
build:
34+
dockerfile: ./jobs.dockerfile
35+
target: dev
36+
context: ./
37+
38+
image: ninjeneer/vuln-scanner-jobs-service
39+
volumes:
40+
- .:/app
41+
networks:
42+
- vulnscanner
43+
3244
networks:
3345
vulnscanner:
3446
name: vulnscanner

‎jobs.dockerfile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM node:alpine as builder
2+
WORKDIR /app
3+
COPY --chown=node:node package.json yarn.lock ./
4+
RUN yarn install --frozen-lockfile
5+
COPY --chown=node:node . .
6+
RUN yarn build
7+
8+
9+
# Dev stage
10+
FROM builder as dev
11+
WORKDIR /app
12+
EXPOSE 3000
13+
CMD ["yarn", "dev:jobs"]
14+
15+
16+
# Prod stage
17+
FROM node:alpine as prod
18+
WORKDIR /app
19+
COPY --from=builder /app/node_modules ./dist/node_modules
20+
COPY --from=builder /app/dist/src ./dist/src
21+
CMD ["node", "dist/src/services/jobs/index.js"]

‎package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,9 @@
44
"description": "",
55
"main": "src/index.ts",
66
"scripts": {
7-
"dev:docker": "docker-compose -f docker-compose.yml up",
8-
"dev:docker-down": "docker-compose -f docker-compose.yml down",
97
"dev:requests": "nodemon src/services/requests/server.ts",
108
"dev:reports": "nodemon src/services/reports/server.ts",
9+
"dev:jobs": "nodemon src/services/jobs/index.ts",
1110
"build": "swc src -d dist/src",
1211
"test": "jest test"
1312
},
@@ -20,6 +19,7 @@
2019
"@swc/register": "^0.1.10",
2120
"@types/jest": "^29.2.4",
2221
"@types/jsonwebtoken": "^9.0.1",
22+
"@types/node-cron": "^3.0.7",
2323
"@types/uuid": "^9.0.0",
2424
"jest": "^29.3.1",
2525
"nodemon": "^2.0.20",
@@ -33,6 +33,7 @@
3333
"fastify": "^4.10.2",
3434
"jsonwebtoken": "^9.0.0",
3535
"mongoose": "^7.0.3",
36+
"node-cron": "^3.0.2",
3637
"uuid": "^9.0.0",
3738
"zod": "^3.19.1"
3839
}

‎src/models/probe.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export type Probe = {
33
status: ProbeStatus;
44
scanId: string;
55
name: string
6+
settings: Record<string, any>
67
}
78

89
export enum ProbeStatus {

‎src/services/jobs/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import initializeJobs from './scansStarter'
2+
3+
initializeJobs()

‎src/services/jobs/scansStarter.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import { getScan, getScansWithProbes, listenScans } from "../../storage/scan.storage";
2+
import cron from 'node-cron'
3+
import { restartScan } from "../requests/scanService";
4+
5+
const cronMapping: Record<string, cron.ScheduledTask> = {}
6+
7+
// Keep in memory scans periodicities to avoid restarting job on scan update
8+
const periodicityMapping = {}
9+
10+
const init = async () => {
11+
console.log(`[JOBS][SCAN][INIT] Initializating scan jobs...`)
12+
const scans = await getScansWithProbes()
13+
for (const scan of scans) {
14+
if (scan.periodicity !== 'ONCE') {
15+
const scanTask = cron.schedule(scan.periodicity, () => restartScan(scan))
16+
cronMapping[scan.id] = scanTask
17+
cronMapping[scan.id].start()
18+
}
19+
periodicityMapping[scan.id] = scan.periodicity
20+
}
21+
console.log(`[JOBS][SCAN][INIT] Initialized ${Object.values(cronMapping).length} scans`)
22+
23+
listenScans(async (change) => {
24+
if (change.eventType === 'UPDATE' || change.eventType === 'INSERT') {
25+
const isUpdate = change.eventType === 'UPDATE'
26+
const changeScan = change.new
27+
if (isUpdate && changeScan.periodicity === periodicityMapping[changeScan.id]) {
28+
// The periodicity has not been edited
29+
return
30+
}
31+
32+
if (changeScan.periodicity === 'ONCE') {
33+
periodicityMapping[changeScan.id] = changeScan.periodicity
34+
if (isUpdate) {
35+
cronMapping[changeScan.id].stop()
36+
delete cronMapping[changeScan.id]
37+
}
38+
return
39+
}
40+
41+
console.log(`[JOBS][SCAN][${change.eventType.toUpperCase()}] ${isUpdate ? 'Restarting' : 'Starting'} scan ${change.new.id} job...`)
42+
const scan = await getScan(change.new.id)
43+
if (isUpdate && cronMapping[scan.id]) {
44+
cronMapping[scan.id].stop()
45+
}
46+
cronMapping[scan.id] = cron.schedule(scan.periodicity, () => restartScan(scan))
47+
cronMapping[scan.id].start()
48+
periodicityMapping[scan.id] = scan.periodicity
49+
console.log({ old: periodicityMapping[scan.id], new: scan.periodicity })
50+
console.log(`[JOBS][SCAN][${change.eventType.toUpperCase()}] ${isUpdate ? 'Restarted' : 'Started'} scan ${change.new.id} job`)
51+
}
52+
})
53+
}
54+
55+
export default init

‎src/services/reports/probeResultService.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,14 @@ const onProbeResult = async (probeId: string, resultId: string): Promise<boolean
3030
throw new ProbeDoesNotExist();
3131
}
3232
await updateProbe(probeId, { status: ProbeStatus.FINISHED });
33-
await createProbeResult({ probeId, resultId })
34-
33+
3534
const scan = await getScan(probe.scanId);
3635
if (!scan) {
3736
throw new ScanDoesNotExist(probe.scanId);
3837
}
38+
39+
await createProbeResult({ probeId, resultId, reportId: scan.currentReportId })
40+
3941
if (isScanFinished(scan)) {
4042
console.log(`[RESULT][${scan.id}] Scan ${scan.id} is finished`)
4143

@@ -51,7 +53,8 @@ const onProbeResult = async (probeId: string, resultId: string): Promise<boolean
5153
status: ScanStatus.FINISHED,
5254
notification: true,
5355
lastReportId: savedReport.id,
54-
currentReportId: null
56+
currentReportId: null,
57+
lastRun: new Date()
5558
})
5659
console.log(`[REPORT][${scan.id}] Created report !`)
5760
}

‎src/services/reports/reportService.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { NoProbeResultsForReport, ReportDoesNotExist, ScanDoesNotExist } from "../../exceptions/exceptions";
22
import { ProbeResult } from "../../models/probe";
33
import { Report } from "../../models/report";
4-
import { getProbeResultsByCurrentReportId, getProbeResultsByLastReportId } from "../../storage/probe.storage";
4+
import { getProbeResultsByReportId } from "../../storage/probe.storage";
55
import { getScan } from "../../storage/scan.storage";
66
import { getResultsByIds } from "../../storage/mongo/mongoProbe.storage";
77
import * as reportMongoStorage from "../../storage/mongo/mongoReport.storage"
@@ -20,8 +20,7 @@ export const buildReport = async (reportId: string, isRebuild?: boolean): Promis
2020
throw new ReportDoesNotExist(reportId);
2121
}
2222

23-
const getProbesMethod = isRebuild ? getProbeResultsByLastReportId : getProbeResultsByCurrentReportId
24-
const probeResults = await getProbesMethod(reportId);
23+
const probeResults = await getProbeResultsByReportId(reportId);
2524
if (!probeResults) {
2625
throw new NoProbeResultsForReport(reportId);
2726
}

‎src/services/requests/scanService.ts‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,25 @@
11
import { CreateScanRequest, UpdateScanRequest } from "../requests/validators/scanRequest";
22
import { v4 as uuidv4 } from 'uuid';
33
import * as scanStorage from "../../storage/scan.storage";
4-
import { ScanStatus, ScanRequestResponse } from "../../models/scan";
5-
import { ProbeStatus } from "../../models/probe";
4+
import { ScanStatus, ScanRequestResponse, ScanWithProbes } from "../../models/scan";
5+
import { Probe, ProbeStatus } from "../../models/probe";
66
import { Scan } from '../../models/scan'
77
import { Report, SupabaseReport } from '../../models/report'
88
import { publishProbeRequest } from "../../storage/awsSqsQueue";
9-
import { saveReport } from "../../storage/mongo/mongoReport.storage";
109
import { createReport } from "../../storage/report.storage";
1110
import { ScanDoesNotExist } from "../../exceptions/exceptions";
12-
import { deleteProbes } from "../../storage/probe.storage";
11+
import { deleteProbes, updateProbesByScanId } from "../../storage/probe.storage";
1312

1413

1514
export const requestScan = async (scanRequest: CreateScanRequest): Promise<ScanRequestResponse> => {
1615
const newScanId = uuidv4();
1716
console.log(`[REQUEST][${newScanId}] Received scan request ${newScanId}`)
1817

1918
// Assing uids to probes
20-
const probes = scanRequest.probes.map((probe) => {
19+
const probes: Partial<Probe>[] = scanRequest.probes.map((probe) => {
2120
return {
2221
...probe,
23-
uid: uuidv4()
22+
id: uuidv4()
2423
}
2524
})
2625

@@ -40,18 +39,19 @@ export const requestScan = async (scanRequest: CreateScanRequest): Promise<ScanR
4039

4140
console.log(`[REQUEST][${newScanId}] Saving probe start data...`)
4241
await scanStorage.saveProbesStartData(probes.map((probe) => ({
43-
id: probe.uid,
42+
id: probe.id,
4443
status: ProbeStatus.PENDING,
4544
scanId: newScanId,
46-
name: probe.name
45+
name: probe.name,
46+
settings: probe.settings
4747
})))
4848
console.log(`[REQUEST][${newScanId}] Probe start data saved !`)
4949

5050

5151
console.log(`[REQUEST][${newScanId}] Publishing request to Queue...`)
5252
await publishProbeRequest(probes.map((probe) => ({
5353
context: {
54-
id: probe.uid,
54+
id: probe.id,
5555
name: probe.name,
5656
target: scanRequest.target
5757
},
@@ -97,7 +97,8 @@ export const updateScan = async (scanId: string, scanWithProbes: UpdateScanReque
9797
id: uuidv4(),
9898
status: ProbeStatus.PENDING,
9999
scanId,
100-
name: probe.name
100+
name: probe.name,
101+
settings: probe.settings
101102
})))
102103
console.log(`[REQUEST][SCAN][UPDATE][${scanId}] Probes created successfully`)
103104
}
@@ -115,4 +116,24 @@ export const updateScan = async (scanId: string, scanWithProbes: UpdateScanReque
115116
const updatedScan = await scanStorage.updateScan(scanId, payload)
116117
console.log(`[REQUEST][SCAN][UPDATE][${scanId}] Scan updated`)
117118
return updatedScan
119+
}
120+
121+
export const restartScan = async (scan: ScanWithProbes): Promise<void> => {
122+
console.log(`[REQUEST][SCAN][RESTART][${scan.id}] Restarting scan...`)
123+
const report = await setupScanNewReport(scan)
124+
await scanStorage.updateScan(scan.id, { currentReportId: report.id})
125+
126+
console.log(`[REQUEST][SCAN][RESTART][${scan.id}] Updating probes...`)
127+
await updateProbesByScanId(scan.id, { status: ProbeStatus.PENDING })
128+
129+
console.log(`[REQUEST][SCAN][RESTART][${scan.id}] Publishing request to Queue...`)
130+
await publishProbeRequest(scan.probes.map((probe) => ({
131+
context: {
132+
id: probe.id,
133+
name: probe.name,
134+
target: scan.target
135+
},
136+
settings: probe.settings
137+
})));
138+
console.log(`[REQUEST][SCAN][RESTART][${scan.id}] Published request to Queue !`)
118139
}

0 commit comments

Comments
 (0)