revise the retry method

Author: cuauty

opencompass/models/bailing_api_oc.py

@@ -3,11 +3,11 @@
 import os
 import socket
 import time
-import traceback
 from typing import Dict, List, Optional, Union
 
 import requests
 from requests.adapters import HTTPAdapter
+from requests.exceptions import ConnectionError
 from urllib3.connection import HTTPConnection
 
 try:
@@ -104,7 +104,7 @@ def __init__(
     def generate(
         self,
         inputs: Union[List[str], PromptList],
-        max_out_len: int = 4096,
+        max_out_len: int = 11264,
     ) -> List[str]:
         """Generate results given a list of inputs.
 
@@ -128,7 +128,7 @@ def generate(
                 ): i
                 for i, input in enumerate(inputs)
             }
-            results = [''] * len(inputs)
+            results = [''] * len(inputs)  # set empty str in advance
             for future in concurrent.futures.as_completed(future_to_m):
                 m = future_to_m[future]  # noqa F841
                 resp = future.result()
@@ -198,34 +198,29 @@ def _generate(
             'messages':
             messages,
             'max_tokens':
-            max(
-                max_out_len if max_out_len else 4096,
-                self.max_seq_len if self.max_seq_len else 4096,
-            ),
+            11264,
         }
         request.update(self.generation_kwargs)
-        try:
-            retry_num = 0
-            while retry_num < self.retry:
+
+        retry_num = 0
+        while retry_num < self.retry:
+            try:
                 response = self._infer_result(request, sess)
-                if response.status_code == 200:
-                    break  # success
-                elif response.status_code == 426:
-                    retry_num += 1  # retry
-                elif response.status_code in [302, 429, 500, 504]:
-                    time.sleep(BAILING_RETRY_DELAY)
-                    retry_num += 1  # retry
-                else:
-                    raise ValueError(f'Status code = {response.status_code}')
+            except ConnectionError:
+                time.sleep(BAILING_RETRY_DELAY)
+                retry_num += 1  # retry
+            if response.status_code == 200:
+                break  # success
+            elif response.status_code == 426:
+                retry_num += 1  # retry
+            elif response.status_code in [302, 429, 500, 504]:
+                time.sleep(BAILING_RETRY_DELAY)
+                retry_num += 1  # retry
             else:
-                raise ValueError(
-                    f'Exceed the maximal retry times. Last status code '
-                    f'= {response.status_code}')
-        except Exception as e:
-            self.logger.error(f'Fail to inference request={request}; '
-                              f'model_name={self.path};  error={e}, '
-                              f'stack:{traceback.format_exc()}')
-            raise e
+                raise ValueError(f'Status code = {response.status_code}')
+        else:
+            # Exceed the maximal retry times, return empty str
+            return ''
         return response
 
     # @retry(stop_max_attempt_number=3, wait_fixed=16000)  # ms