Handle rejection in FollowerChecker (#84484)

In #83576 we moved cluster coordination activities to a threadpool which
rejects actions on shutdown. #84483 is a test failure caused by a
missing rejection handler, which this commit addresses.

Closes #84483

Author: DaveCTurner

server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.monitor.NodeHealthService;
 import org.elasticsearch.monitor.StatusInfo;
@@ -358,9 +359,20 @@ public void handleException(TransportException exp) {
         }
 
         void failNode(String reason) {
-            transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION).execute(new Runnable() {
+            transportService.getThreadPool().executor(Names.CLUSTER_COORDINATION).execute(new AbstractRunnable() {
+
                 @Override
-                public void run() {
+                public void onRejection(Exception e) {
+                    logger.debug(new ParameterizedMessage("rejected task to fail node [{}] with reason [{}]", discoveryNode, reason), e);
+                    if (e instanceof EsRejectedExecutionException esRejectedExecutionException) {
+                        assert esRejectedExecutionException.isExecutorShutdown();
+                    } else {
+                        assert false : e;
+                    }
+                }
+
+                @Override
+                protected void doRun() {
                     synchronized (mutex) {
                         if (running() == false) {
                             logger.trace("{} no longer running, not marking faulty", FollowerChecker.this);
@@ -373,6 +385,15 @@ public void run() {
                     onNodeFailure.accept(discoveryNode, reason);
                 }
 
+                @Override
+                public void onFailure(Exception e) {
+                    assert false : e;
+                    logger.error(
+                        new ParameterizedMessage("unexpected failure when failing node [{}] with reason [{}]", discoveryNode, reason),
+                        e
+                    );
+                }
+
                 @Override
                 public String toString() {
                     return "detected failure of " + discoveryNode;