feat(retry-on-timeout): Implement retry on timeout when creating new client

packages/pg-pool/README.md

@@ -356,6 +356,28 @@ maxUses = rebalanceWindowSeconds * totalRequestsPerSecond / numAppInstances / po
    7200 =        1800            *          1000          /        10       /    25
 ```
 
+## Retry on timeout
+
+It is possible to configure the pool to retry connections that timeout. 
+This is useful in cases where you have DNS issues or other transient network problems that cause the pool to fail to
+connect to the database.
+
+This can be done by setting the `connectionTimeoutMillis` option, plus passing the `retryOnTimeout`, `maxRetries`, and
+`retryDelay` options to the pool constructor.
+
+```js
+const Pool = require('pg-pool')
+const pool = new Pool({
+  connectionTimeoutMillis: 1000, // timeout for acquiring a connection
+  retryOnTimeout: true, // retry on timeout
+  maxRetries: 5, // maximum number of retries
+  retryDelay: 1_000, // delay between retries in milliseconds
+})
+```
+The connection will be retried up to `maxRetries` times if it times out, with a delay of `retryDelay` milliseconds between each retry.
+The existing error (connection timeout) will still be thrown if the maximum number of retries is reached.
+
+
 ## tests
 
 To run tests clone the repo, `npm i` in the working dir, and then run `npm test`

packages/pg-pool/index.js

@@ -91,6 +91,9 @@ class Pool extends EventEmitter {
     this.options.maxUses = this.options.maxUses || Infinity
     this.options.allowExitOnIdle = this.options.allowExitOnIdle || false
     this.options.maxLifetimeSeconds = this.options.maxLifetimeSeconds || 0
+    this.options.retryOnTimeout = this.options.retryOnTimeout || false
+    this.options.maxRetries = this.options.maxRetries || 3
+    this.options.retryDelay = this.options.retryDelay || 3_000
     this.log = this.options.log || function () {}
     this.Client = this.options.Client || Client || require('pg').Client
     this.Promise = this.options.Promise || global.Promise
@@ -229,7 +232,7 @@ class Pool extends EventEmitter {
     return result
   }
 
-  newClient(pendingItem) {
+  newClient(pendingItem, retryAttempt = 0) {
     const client = new this.Client(this.options)
     this._clients.push(client)
     const idleListener = makeIdleListener(this, client)
@@ -258,6 +261,18 @@ class Pool extends EventEmitter {
         this.log('client failed to connect', err)
         // remove the dead client from our list of clients
         this._clients = this._clients.filter((c) => c !== client)
+
+        // Retry on timeout if enabled
+        if (timeoutHit && this.options.retryOnTimeout && retryAttempt < this.options.maxRetries) {
+          this.log(`Connection timeout, retry ${retryAttempt + 1}/${this.options.maxRetries}`)
+
+          // delay the retry to avoid tight loops
+          setTimeout(() => {
+            this.newClient(pendingItem, retryAttempt + 1)
+          }, this.options.retryDelay)
+
+          return
+        }
         if (timeoutHit) {
           err = new Error('Connection terminated due to connection timeout', { cause: err })
         }

packages/pg-pool/test/retry-on-timeout.js

@@ -0,0 +1,124 @@
+'use strict'
+
+const assert = require('assert')
+const Pool = require('../')
+const expect = require('expect.js')
+
+let connectionAttempts = 0
+
+// Test Client that simulates always timeout
+class TimeoutClient {
+  constructor() {}
+
+  connect(cb) {
+    connectionAttempts++
+    setTimeout(() => {
+      const err = new Error('timeout')
+      err.code = 'ETIMEDOUT'
+      cb(err)
+    }, 10)
+  }
+
+  end() {
+    // No-op for event handling
+  }
+
+  on(event, listener) {
+    // No-op for event handling
+  }
+
+  removeListener(event, listener) {
+    // No-op for event handling
+  }
+}
+
+// Test Client that simulates two timeout then connects successfully
+class TimeoutTwiceClient extends TimeoutClient {
+  constructor() {
+    super()
+    this.timeout = true
+    this.ended = false
+  }
+
+  connect(cb) {
+    if (connectionAttempts++ > 1) {
+      this.timeout = false
+    }
+    if (this.timeout) {
+      setTimeout(() => {
+        const err = new Error('timeout')
+        err.code = 'ETIMEDOUT'
+        cb(err)
+      }, 10)
+    } else {
+      cb()
+    }
+  }
+
+  end() {
+    this.ended = true
+  }
+}
+
+describe('retry on timeout', () => {
+  beforeEach(() => {
+    connectionAttempts = 0
+  })
+
+  it('should retry when client connection times out', function (done) {
+    const pool = new Pool({
+      Client: TimeoutTwiceClient,
+      max: 1,
+      connectionTimeoutMillis: 5,
+      retryOnTimeout: true,
+      maxRetries: 3,
+      retryDelay: 15,
+    })
+
+    pool.connect((err, client, release) => {
+      expect(err).to.be(undefined)
+      expect(client).to.be.an(TimeoutTwiceClient)
+      assert.equal(connectionAttempts, 3, 'should have tried 3 times')
+      release()
+      done()
+    })
+  })
+
+  it('should fail after max retries', function (done) {
+    const pool = new Pool({
+      Client: TimeoutClient,
+      max: 1,
+      connectionTimeoutMillis: 5,
+      retryOnTimeout: true,
+      maxRetries: 3,
+      retryDelay: 15,
+    })
+
+    pool.connect((err, client, release) => {
+      expect(err).to.be.an(Error)
+      expect(err.message).to.equal('Connection terminated due to connection timeout')
+      expect(client).to.be(undefined)
+      assert.equal(connectionAttempts, 4, 'should have tried 4 times (first attempt + 3 retries)')
+      release()
+      done()
+    })
+  })
+
+  it('should not retry when retryOnTimeout is false', function (done) {
+    const pool = new Pool({
+      Client: TimeoutClient,
+      max: 1,
+      connectionTimeoutMillis: 5,
+      retryOnTimeout: false,
+    })
+
+    pool.connect((err, client, release) => {
+      expect(err).to.be.an(Error)
+      expect(err.message).to.equal('Connection terminated due to connection timeout')
+      expect(client).to.be(undefined)
+      assert.equal(connectionAttempts, 1, 'should have tried only once without retries')
+      release()
+      done()
+    })
+  })
+})