diff --git a/apps/api/index.ts b/apps/api/index.ts index 06da10b9b..f77224936 100644 --- a/apps/api/index.ts +++ b/apps/api/index.ts @@ -6,6 +6,7 @@ import cors from 'cors'; import type { NextFunction, Request, Response } from 'express'; import express from 'express'; import promBundle from 'express-prom-bundle'; +import robots from 'express-robots-txt'; import pinoHttp from 'pino-http'; import { v4 as uuidv4 } from 'uuid'; import { getDependencyContainer } from './dependency_container'; @@ -19,6 +20,13 @@ if (process.env.GLOBAL_AGENT_HTTP_PROXY) { require('@supaglue/schemas/proxy.bootstrap'); } +app.use( + robots({ + UserAgent: '*', + Disallow: '/', + }) +); + app.use( promBundle({ includeMethod: true, diff --git a/apps/api/package.json b/apps/api/package.json index fafebf612..93ff2d74a 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -19,6 +19,7 @@ "express": ">=5.0.0-beta.1", "express-openapi-validator": "^5.0.2", "express-prom-bundle": "^6.6.0", + "express-robots-txt": "^1.0.0", "json-schema-traverse": "^1.0.0", "jsonwebtoken": "^9.0.2", "jwks-rsa": "^3.1.0", diff --git a/apps/api/types/express-robots-txt.d.ts b/apps/api/types/express-robots-txt.d.ts new file mode 100644 index 000000000..fe13b4467 --- /dev/null +++ b/apps/api/types/express-robots-txt.d.ts @@ -0,0 +1 @@ +declare module 'express-robots-txt'; diff --git a/yarn.lock b/yarn.lock index 45d3cb85c..bf9e7ef3e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8505,6 +8505,7 @@ __metadata: express: ">=5.0.0-beta.1" express-openapi-validator: ^5.0.2 express-prom-bundle: ^6.6.0 + express-robots-txt: ^1.0.0 json-schema-traverse: ^1.0.0 jsonwebtoken: ^9.0.2 jwks-rsa: ^3.1.0 @@ -12608,6 +12609,15 @@ __metadata: languageName: node linkType: hard +"express-robots-txt@npm:^1.0.0": + version: 1.0.0 + resolution: "express-robots-txt@npm:1.0.0" + peerDependencies: + express: ^4.12.1 + checksum: 8ee6e2a076bb7c06e59c01201ce3a789e31b1d1e938792d835538bf89337f3957baaab62ed5d045bb22724edebaaabcd2cf8974afb5fe3d056555a79da83846b + languageName: node + linkType: hard + "express@npm:>=5.0.0-beta.1": version: 5.0.0-beta.1 resolution: "express@npm:5.0.0-beta.1"