RUST-1785 Make ExceededTimeLimit a read-retryable error

src/error.rs

@@ -16,8 +16,8 @@ use crate::{bson::Document, options::ServerAddress, sdam::TopologyVersion};
 const RECOVERING_CODES: [i32; 5] = [11600, 11602, 13436, 189, 91];
 const NOTWRITABLEPRIMARY_CODES: [i32; 3] = [10107, 13435, 10058];
 const SHUTTING_DOWN_CODES: [i32; 2] = [11600, 91];
-const RETRYABLE_READ_CODES: [i32; 12] = [
-    11600, 11602, 10107, 13435, 13436, 189, 91, 7, 6, 89, 9001, 134,
+const RETRYABLE_READ_CODES: [i32; 13] = [
+    11600, 11602, 10107, 13435, 13436, 189, 91, 7, 6, 89, 9001, 134, 262,
 ];
 const RETRYABLE_WRITE_CODES: [i32; 12] = [
     11600, 11602, 10107, 13435, 13436, 189, 91, 7, 6, 89, 9001, 262,

src/test/spec/json/retryable-reads/README.rst

@@ -232,10 +232,80 @@ This test requires MongoDB 4.2.9+ for ``blockConnection`` support in the failpoi
 
 9. Disable the failpoint.
 
+Retrying Reads in a Sharded Cluster
+===================================
+
+These tests will be used to ensure drivers properly retry reads on a different
+mongos.
+
+Retryable Reads Are Retried on a Different mongos if One is Available
+---------------------------------------------------------------------
+
+This test MUST be executed against a sharded cluster that has at least two
+mongos instances.
+
+1. Ensure that a test is run against a sharded cluster that has at least two
+   mongoses. If there are more than two mongoses in the cluster, pick two to
+   test against.
+
+2. Create a client per mongos using the direct connection, and configure the
+   following fail points on each mongos::
+
+     {
+         configureFailPoint: "failCommand",
+         mode: { times: 1 },
+         data: {
+             failCommands: ["find"],
+             errorCode: 6,
+             closeConnection: true
+         }
+     }
+
+3. Create a client with ``retryReads=true`` that connects to the cluster,
+   providing the two selected mongoses as seeds.
+
+4. Enable command monitoring, and execute a ``find`` command that is
+   supposed to fail on both mongoses.
+
+5. Asserts that there were failed command events from each mongos.
+
+6. Disable the fail points.
+
+
+Retryable Reads Are Retried on the Same mongos if No Others are Available
+-------------------------------------------------------------------------
+
+1. Ensure that a test is run against a sharded cluster. If there are multiple
+   mongoses in the cluster, pick one to test against.
+
+2. Create a client that connects to the mongos using the direct connection,
+   and configure the following fail point on the mongos::
+
+     {
+         configureFailPoint: "failCommand",
+         mode: { times: 1 },
+         data: {
+             failCommands: ["find"],
+             errorCode: 6,
+             closeConnection: true
+         }
+     }
+
+3. Create a client with ``retryReads=true`` that connects to the cluster,
+   providing the selected mongos as the seed.
+
+4. Enable command monitoring, and execute a ``find`` command.
+
+5. Asserts that there was a failed command and a successful command event.
+
+6. Disable the fail point.
+
 
 Changelog
 =========
 
+:2023-08-26 Add prose tests for retrying in a sharded cluster.
+
 :2022-04-22: Clarifications to ``serverless`` and ``useMultipleMongoses``.
 
 :2022-01-10: Create legacy and unified subdirectories for new unified tests

src/test/spec/json/retryable-reads/unified/exceededTimeLimit.json

@@ -0,0 +1,147 @@
+{
+  "description": "ExceededTimeLimit is a retryable read",
+  "schemaVersion": "1.3",
+  "runOnRequirements": [
+    {
+      "minServerVersion": "4.0",
+      "topologies": [
+        "single",
+        "replicaset"
+      ]
+    },
+    {
+      "minServerVersion": "4.1.7",
+      "topologies": [
+        "sharded",
+        "load-balanced"
+      ]
+    }
+  ],
+  "createEntities": [
+    {
+      "client": {
+        "id": "client0",
+        "useMultipleMongoses": false,
+        "observeEvents": [
+          "commandStartedEvent"
+        ]
+      }
+    },
+    {
+      "database": {
+        "id": "database0",
+        "client": "client0",
+        "databaseName": "retryable-reads-tests"
+      }
+    },
+    {
+      "collection": {
+        "id": "collection0",
+        "database": "database0",
+        "collectionName": "exceededtimelimit-test"
+      }
+    }
+  ],
+  "initialData": [
+    {
+      "collectionName": "exceededtimelimit-test",
+      "databaseName": "retryable-reads-tests",
+      "documents": [
+        {
+          "_id": 1,
+          "x": 11
+        },
+        {
+          "_id": 2,
+          "x": 22
+        },
+        {
+          "_id": 3,
+          "x": 33
+        }
+      ]
+    }
+  ],
+  "tests": [
+    {
+      "description": "Find succeeds on second attempt after ExceededTimeLimit",
+      "operations": [
+        {
+          "name": "failPoint",
+          "object": "testRunner",
+          "arguments": {
+            "client": "client0",
+            "failPoint": {
+              "configureFailPoint": "failCommand",
+              "mode": {
+                "times": 1
+              },
+              "data": {
+                "failCommands": [
+                  "find"
+                ],
+                "errorCode": 262
+              }
+            }
+          }
+        },
+        {
+          "name": "find",
+          "arguments": {
+            "filter": {
+              "_id": {
+                "$gt": 1
+              }
+            }
+          },
+          "object": "collection0",
+          "expectResult": [
+            {
+              "_id": 2,
+              "x": 22
+            },
+            {
+              "_id": 3,
+              "x": 33
+            }
+          ]
+        }
+      ],
+      "expectEvents": [
+        {
+          "client": "client0",
+          "events": [
+            {
+              "commandStartedEvent": {
+                "command": {
+                  "find": "exceededtimelimit-test",
+                  "filter": {
+                    "_id": {
+                      "$gt": 1
+                    }
+                  }
+                },
+                "commandName": "find",
+                "databaseName": "retryable-reads-tests"
+              }
+            },
+            {
+              "commandStartedEvent": {
+                "command": {
+                  "find": "exceededtimelimit-test",
+                  "filter": {
+                    "_id": {
+                      "$gt": 1
+                    }
+                  }
+                },
+                "commandName": "find",
+                "databaseName": "retryable-reads-tests"
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

src/test/spec/json/retryable-reads/unified/exceededTimeLimit.yml

@@ -0,0 +1,68 @@
+description: "ExceededTimeLimit is a retryable read"
+
+schemaVersion: "1.3"
+
+runOnRequirements:
+  - minServerVersion: "4.0"
+    topologies: [single, replicaset]
+  - minServerVersion: "4.1.7"
+    topologies: [sharded, load-balanced]
+
+createEntities:
+  - client:
+      id: &client0 client0
+      # Ensure the `configureFailpoint` and `find` commands are run on the same mongos
+      useMultipleMongoses: false
+      observeEvents: [ commandStartedEvent ]
+  - database:
+      id: &database0 database0
+      client: *client0
+      databaseName: &database0Name "retryable-reads-tests"
+  - collection:
+      id: &collection0 collection0
+      database: *database0
+      collectionName: &collection0Name "exceededtimelimit-test"
+
+initialData:
+  - collectionName: *collection0Name
+    databaseName: *database0Name
+    documents:
+      - { _id: 1, x: 11 }
+      - { _id: 2, x: 22 }
+      - { _id: 3, x: 33 }
+
+tests:
+  - description: "Find succeeds on second attempt after ExceededTimeLimit"
+    operations:
+      - name: failPoint
+        object: testRunner
+        arguments:
+          client: *client0
+          failPoint:
+            configureFailPoint: failCommand
+            mode: { times: 1 }
+            data:
+              failCommands: [ "find" ]
+              errorCode: 262 # ExceededTimeLimit
+      - name: find
+        arguments:
+          filter: { _id: { $gt: 1 } }
+        object: *collection0
+        expectResult:
+          - { _id: 2, x: 22 }
+          - { _id: 3, x: 33 }
+    expectEvents:
+      - client: *client0
+        events:
+          - commandStartedEvent:
+              command:
+                find: *collection0Name
+                filter: { _id: { $gt: 1 } }
+              commandName: find
+              databaseName: *database0Name
+          - commandStartedEvent:
+              command:
+                find: *collection0Name
+                filter: { _id: { $gt: 1 } }
+              commandName: find
+              databaseName: *database0Name