chore: prevent robots from crawling API (#1982)

apps/api/index.ts

@@ -6,6 +6,7 @@ import cors from 'cors';
 import type { NextFunction, Request, Response } from 'express';
 import express from 'express';
 import promBundle from 'express-prom-bundle';
+import robots from 'express-robots-txt';
 import pinoHttp from 'pino-http';
 import { v4 as uuidv4 } from 'uuid';
 import { getDependencyContainer } from './dependency_container';
@@ -19,6 +20,13 @@ if (process.env.GLOBAL_AGENT_HTTP_PROXY) {
   require('@supaglue/schemas/proxy.bootstrap');
 }
 
+app.use(
+  robots({
+    UserAgent: '*',
+    Disallow: '/',
+  })
+);
+
 app.use(
   promBundle({
     includeMethod: true,

apps/api/package.json

@@ -19,6 +19,7 @@
     "express": ">=5.0.0-beta.1",
     "express-openapi-validator": "^5.0.2",
     "express-prom-bundle": "^6.6.0",
+    "express-robots-txt": "^1.0.0",
     "json-schema-traverse": "^1.0.0",
     "jsonwebtoken": "^9.0.2",
     "jwks-rsa": "^3.1.0",

apps/api/types/express-robots-txt.d.ts

@@ -0,0 +1 @@
+declare module 'express-robots-txt';

yarn.lock

@@ -8505,6 +8505,7 @@ __metadata:
     express: ">=5.0.0-beta.1"
     express-openapi-validator: ^5.0.2
     express-prom-bundle: ^6.6.0
+    express-robots-txt: ^1.0.0
     json-schema-traverse: ^1.0.0
     jsonwebtoken: ^9.0.2
     jwks-rsa: ^3.1.0
@@ -12608,6 +12609,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"express-robots-txt@npm:^1.0.0":
+  version: 1.0.0
+  resolution: "express-robots-txt@npm:1.0.0"
+  peerDependencies:
+    express: ^4.12.1
+  checksum: 8ee6e2a076bb7c06e59c01201ce3a789e31b1d1e938792d835538bf89337f3957baaab62ed5d045bb22724edebaaabcd2cf8974afb5fe3d056555a79da83846b
+  languageName: node
+  linkType: hard
+
 "express@npm:>=5.0.0-beta.1":
   version: 5.0.0-beta.1
   resolution: "express@npm:5.0.0-beta.1"