Add progress for retry connections (celery#5915)

This will show current retry progress so it will clear confusion about how many retries will be tried for connecting to broker.
Closes celery#4556

Author: marfgold1

celery/worker/consumer/consumer.py

@@ -51,7 +51,7 @@
 """
 
 CONNECTION_RETRY_STEP = """\
-Trying again {when}...\
+Trying again {when}... ({retries}/{max_retries})\
 """
 
 CONNECTION_ERROR = """\
@@ -421,8 +421,11 @@ def ensure_connected(self, conn):
         def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP):
             if getattr(conn, 'alt', None) and interval == 0:
                 next_step = CONNECTION_FAILOVER
-            error(CONNECTION_ERROR, conn.as_uri(), exc,
-                  next_step.format(when=humanize_seconds(interval, 'in', ' ')))
+            next_step = next_step.format(
+                when=humanize_seconds(interval, 'in', ' '),
+                retries=int(interval / 2),
+                max_retries=self.app.conf.broker_connection_max_retries)
+            error(CONNECTION_ERROR, conn.as_uri(), exc, next_step)
 
         # remember that the connection is lazy, it won't establish
         # until needed.

t/unit/worker/test_consumer.py

@@ -264,6 +264,22 @@ def test_connect_error_handler(self):
         errback = conn.ensure_connection.call_args[0][0]
         errback(Mock(), 0)
 
+    @patch('celery.worker.consumer.consumer.error')
+    def test_connect_error_handler_progress(self, error):
+        self.app.conf.broker_connection_retry = True
+        self.app.conf.broker_connection_max_retries = 3
+        self.app._connection = _amqp_connection()
+        conn = self.app._connection.return_value
+        c = self.get_consumer()
+        assert c.connect()
+        errback = conn.ensure_connection.call_args[0][0]
+        errback(Mock(), 2)
+        assert error.call_args[0][3] == 'Trying again in 2.00 seconds... (1/3)'
+        errback(Mock(), 4)
+        assert error.call_args[0][3] == 'Trying again in 4.00 seconds... (2/3)'
+        errback(Mock(), 6)
+        assert error.call_args[0][3] == 'Trying again in 6.00 seconds... (3/3)'
+
 
 class test_Heart: