feat: custom connect http server (apify#285)

Author: szmarczak

README.md

@@ -244,6 +244,61 @@ server.listen(() => {
 });
 ```
 
+## Routing CONNECT to another HTTP server
+
+While `customResponseFunction` enables custom handling methods such as `GET` and `POST`, many HTTP clients rely on `CONNECT` tunnels.
+It's possible to route those requests differently using the `customConnectServer` option. It accepts an instance of Node.js HTTP server.
+
+```javascript
+const http = require('http');
+const ProxyChain = require('proxy-chain');
+
+const exampleServer = http.createServer((request, response) => {
+    response.end('Hello from a custom server!');
+});
+
+const server = new ProxyChain.Server({
+    port: 8000,
+    prepareRequestFunction: ({ request, username, password, hostname, port, isHttp }) => {
+        if (request.url.toLowerCase() === 'example.com:80') {
+            return {
+                customConnectServer: exampleServer,
+            };
+        }
+
+        return {};
+    },
+});
+
+server.listen(() => {
+  console.log(`Proxy server is listening on port ${server.port}`);
+});
+```
+
+In the example above, all CONNECT tunnels to `example.com` are overridden.
+This is an unsecure server, so it accepts only `http:` requests.
+
+In order to intercept `https:` requests, `https.createServer` should be used instead, along with a self signed certificate.
+
+```javascript
+const https = require('https');
+const fs = require('fs');
+const key = fs.readFileSync('./test/ssl.key');
+const cert = fs.readFileSync('./test/ssl.crt');
+
+const exampleServer = https.createServer({
+    key,
+    cert,
+}, (request, response) => {
+    response.end('Hello from a custom server!');
+});
+```
+
+```diff
+-if (request.url.toLowerCase() === 'example.com:80') {
++if (request.url.toLowerCase() === 'example.com:443') {
+```
+
 ## Closing the server
 
 To shut down the proxy server, call the `close([destroyConnections], [callback])` function. For example:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "proxy-chain",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "description": "Node.js implementation of a proxy server (think Squid) with support for SSL, authentication, upstream proxy chaining, and protocol tunneling.",
   "main": "dist/index.js",
   "keywords": [
@@ -47,7 +47,7 @@
     "@apify/eslint-config-ts": "^0.2.3",
     "@apify/tsconfig": "^0.1.0",
     "@types/jest": "^28.1.2",
-    "@types/node": "^18.0.0",
+    "@types/node": "^18.8.3",
     "@typescript-eslint/eslint-plugin": "5.29.0",
     "@typescript-eslint/parser": "5.29.0",
     "basic-auth": "^2.0.1",

src/custom_connect.ts

@@ -0,0 +1,27 @@
+import net from 'net';
+import type http from 'http';
+import { promisify } from 'util';
+
+const asyncWrite = promisify(net.Socket.prototype.write);
+
+export const customConnect = async (socket: net.Socket, server: http.Server): Promise<void> => {
+    // `countTargetBytes(socket, socket)` is incorrect here since `socket` is not a target.
+    // We would have to create a new stream and pipe traffic through that,
+    // however this would also increase CPU usage.
+    // Also, counting bytes here is not correct since we don't know how the response is generated
+    // (whether any additional sockets are used).
+
+    await asyncWrite.call(socket, 'HTTP/1.1 200 Connection Established\r\n\r\n');
+    server.emit('connection', socket);
+
+    return new Promise((resolve) => {
+        if (socket.destroyed) {
+            resolve();
+            return;
+        }
+
+        socket.once('close', () => {
+            resolve();
+        });
+    });
+};

src/server.ts

@@ -17,6 +17,7 @@ import { handleCustomResponse, HandlerOpts as CustomResponseOpts } from './custo
 import { Socket } from './socket';
 import { normalizeUrlPort } from './utils/normalize_url_port';
 import { badGatewayStatusCodes } from './statuses';
+import { customConnect } from './custom_connect';
 
 // TODO:
 // - Implement this requirement from rfc7230
@@ -46,7 +47,8 @@ type HandlerOpts = {
     trgParsed: URL | null;
     upstreamProxyUrlParsed: URL | null;
     isHttp: boolean;
-    customResponseFunction: CustomResponseOpts['customResponseFunction'] | null;
+    customResponseFunction?: CustomResponseOpts['customResponseFunction'] | null;
+    customConnectServer?: http.Server | null;
     localAddress?: string;
     ipFamily?: number;
     dnsLookup?: typeof dns['lookup'];
@@ -64,6 +66,7 @@ export type PrepareRequestFunctionOpts = {
 
 export type PrepareRequestFunctionResult = {
     customResponseFunction?: CustomResponseOpts['customResponseFunction'];
+    customConnectServer?: http.Server | null;
     requestAuthentication?: boolean;
     failMsg?: string;
     upstreamProxyUrl?: string | null;
@@ -274,6 +277,11 @@ export class Server extends EventEmitter {
 
             const data = { request, sourceSocket: socket, head, handlerOpts: handlerOpts as ChainOpts, server: this, isPlain: false };
 
+            if (handlerOpts.customConnectServer) {
+                socket.unshift(head); // See chain.ts for why we do this
+                return await customConnect(socket, handlerOpts.customConnectServer);
+            }
+
             if (handlerOpts.upstreamProxyUrlParsed) {
                 this.log(socket.proxyChainId, `Using HandlerTunnelChain => ${request.url}`);
                 return await chain(data);
@@ -301,6 +309,7 @@ export class Server extends EventEmitter {
             isHttp: false,
             srcResponse: null,
             customResponseFunction: null,
+            customConnectServer: null,
         };
 
         this.log((request.socket as Socket).proxyChainId, `!!! Handling ${request.method} ${request.url} HTTP/${request.httpVersion}`);
@@ -404,6 +413,7 @@ export class Server extends EventEmitter {
         handlerOpts.localAddress = funcResult.localAddress;
         handlerOpts.ipFamily = funcResult.ipFamily;
         handlerOpts.dnsLookup = funcResult.dnsLookup;
+        handlerOpts.customConnectServer = funcResult.customConnectServer;
 
         // If not authenticated, request client to authenticate
         if (funcResult.requestAuthentication) {

test/server.js

@@ -1332,6 +1332,53 @@ it('supports localAddress', async () => {
     }
 });
 
+it('supports custom CONNECT server handler', async () => {
+    const server = new Server({
+        port: 0,
+        prepareRequestFunction: () => {
+            const customConnectServer = http.createServer((_request, response) => {
+                response.end('Hello, world!');
+            });
+
+            return {
+                customConnectServer,
+            };
+        },
+    });
+
+    await server.listen();
+
+    try {
+        const response = await new Promise((resolve, reject) => {
+            http.request(`http://127.0.0.1:${server.port}`, {
+                method: 'CONNECT',
+                path: 'example.com:80',
+                headers: {
+                    host: 'example.com:80',
+                },
+            }).on('connect', (connectResponse, socket, head) => {
+                http.request('http://example.com', {
+                    createConnection: () => socket,
+                }, (res) => {
+                    const buffer = [];
+
+                    res.on('data', (chunk) => {
+                        buffer.push(chunk);
+                    });
+
+                    res.on('end', () => {
+                        resolve(Buffer.concat(buffer).toString());
+                    });
+                }).on('error', reject).end();
+            }).on('error', reject).end();
+        });
+
+        expect(response).to.be.equal('Hello, world!');
+    } finally {
+        await server.close();
+    }
+});
+
 it('supports pre-response CONNECT payload', (done) => {
     const plain = net.createServer((socket) => {
         socket.pipe(socket);