Regroup curl code lists

Author: EgorkaZ

ydb/core/fq/libs/init/init.cpp

@@ -195,7 +195,7 @@ void Init(
 
     if (protoConfig.GetPrivateApi().GetEnabled()) {
         const auto& s3readConfig = protoConfig.GetReadActorsFactoryConfig().GetS3ReadActorFactoryConfig();
-        auto s3HttpRetryPolicy = NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.MaxTime = TDuration::Max(), .ExtendedRetriedCodes = {CURLE_GOT_NOTHING}});
+        auto s3HttpRetryPolicy = NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.MaxTime = TDuration::Max(), .RetriedCurlCodes = NYql::FqRetriedCurlCodes()});
         NYql::NDq::TS3ReadActorFactoryConfig readActorFactoryCfg;
         if (const ui64 rowsInBatch = s3readConfig.GetRowsInBatch()) {
             readActorFactoryCfg.RowsInBatch = rowsInBatch;

ydb/core/kqp/compute_actor/kqp_compute_actor.cpp

@@ -77,7 +77,7 @@ NYql::NDq::IDqAsyncIoFactory::TPtr CreateKqpAsyncIoFactory(
     RegisterSequencerActorFactory(*factory, counters);
 
     if (federatedQuerySetup) {
-        auto s3HttpRetryPolicy = NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.ExtendedRetriedCodes = {CURLE_GOT_NOTHING}});
+        auto s3HttpRetryPolicy = NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.RetriedCurlCodes = NYql::FqRetriedCurlCodes()});
         RegisterS3ReadActorFactory(*factory, federatedQuerySetup->CredentialsFactory, federatedQuerySetup->HttpGateway, s3HttpRetryPolicy);
         RegisterS3WriteActorFactory(*factory,  federatedQuerySetup->CredentialsFactory, federatedQuerySetup->HttpGateway, s3HttpRetryPolicy);
 

ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.cpp

@@ -2,37 +2,50 @@
 
 namespace NYql {
 
+std::unordered_set<CURLcode> YqlRetriedCurlCodes() {
+    return {
+        CURLE_COULDNT_CONNECT,
+        CURLE_WEIRD_SERVER_REPLY,
+        CURLE_WRITE_ERROR,
+        CURLE_READ_ERROR,
+        CURLE_OPERATION_TIMEDOUT,
+        CURLE_SSL_CONNECT_ERROR,
+        CURLE_BAD_DOWNLOAD_RESUME,
+        CURLE_SEND_ERROR,
+        CURLE_RECV_ERROR,
+        CURLE_NO_CONNECTION_AVAILABLE
+    };
+}
+
+std::unordered_set<CURLcode> FqRetriedCurlCodes() {
+    return {
+        CURLE_COULDNT_CONNECT,
+        CURLE_WEIRD_SERVER_REPLY,
+        CURLE_WRITE_ERROR,
+        CURLE_READ_ERROR,
+        CURLE_OPERATION_TIMEDOUT,
+        CURLE_SSL_CONNECT_ERROR,
+        CURLE_BAD_DOWNLOAD_RESUME,
+        CURLE_SEND_ERROR,
+        CURLE_RECV_ERROR,
+        CURLE_NO_CONNECTION_AVAILABLE,
+        CURLE_GOT_NOTHING
+    };
+}
+
 IHTTPGateway::TRetryPolicy::TPtr GetHTTPDefaultRetryPolicy(THttpRetryPolicyOptions&& options) {
     auto maxTime = options.MaxTime;
     auto maxRetries = options.MaxRetries;
     if (!maxTime) {
         maxTime = TDuration::Minutes(5);
     }
     return IHTTPGateway::TRetryPolicy::GetExponentialBackoffPolicy([options = std::move(options)](CURLcode curlCode, long httpCode) {
-
-        switch (curlCode) {
-            case CURLE_OK:
-                // look to http code
-                break;
-            case CURLE_COULDNT_CONNECT:
-            case CURLE_WEIRD_SERVER_REPLY:
-            case CURLE_WRITE_ERROR:
-            case CURLE_READ_ERROR:
-            case CURLE_OPERATION_TIMEDOUT:
-            case CURLE_SSL_CONNECT_ERROR:
-            case CURLE_BAD_DOWNLOAD_RESUME:
-            case CURLE_SEND_ERROR:
-            case CURLE_RECV_ERROR:
-            case CURLE_NO_CONNECTION_AVAILABLE:
-                // retry small number of known errors
-                return ERetryErrorClass::ShortRetry;
-            default:
-                if (options.ExtendedRetriedCodes.contains(curlCode)) {
-                    // retry explicitly enumerated codes
-                    return ERetryErrorClass::ShortRetry;
-                }
-                // do not retry others
-                return ERetryErrorClass::NoRetry;
+        if (curlCode == CURLE_OK) {
+            // pass
+        } else if (options.RetriedCurlCodes.contains(curlCode)) {
+            return ERetryErrorClass::ShortRetry;
+        } else {
+            return ERetryErrorClass::NoRetry;
         }
 
         switch (httpCode) {

ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.h

@@ -7,14 +7,17 @@
 
 namespace NYql {
 
+std::unordered_set<CURLcode> YqlRetriedCurlCodes();
+std::unordered_set<CURLcode> FqRetriedCurlCodes();
+
 struct THttpRetryPolicyOptions {
     TDuration MaxTime = TDuration::Zero(); // Zero means default maxTime
     size_t MaxRetries = std::numeric_limits<size_t>::max();
-    std::unordered_set<CURLcode> ExtendedRetriedCodes;
+    std::unordered_set<CURLcode> RetriedCurlCodes = YqlRetriedCurlCodes();
 };
 
-IHTTPGateway::TRetryPolicy::TPtr GetHTTPDefaultRetryPolicy(THttpRetryPolicyOptions&& options);
+IHTTPGateway::TRetryPolicy::TPtr GetHTTPDefaultRetryPolicy(THttpRetryPolicyOptions&& options = {});
 
-IHTTPGateway::TRetryPolicy::TPtr GetHTTPDefaultRetryPolicy(TDuration maxTime = TDuration::Zero(), size_t maxRetries = std::numeric_limits<size_t>::max()); // Zero means default maxTime
+IHTTPGateway::TRetryPolicy::TPtr GetHTTPDefaultRetryPolicy(TDuration maxTime, size_t maxRetries = std::numeric_limits<size_t>::max()); // Zero means default maxTime
 
 }

ydb/library/yql/providers/s3/actors/yql_s3_applicator_actor.cpp

@@ -231,7 +231,7 @@ class TS3ApplicatorActor : public NActors::TActorBootstrapped<TS3ApplicatorActor
     , CredentialsFactory(credentialsFactory)
     , ExternalEffect(externalEffect)
     , ActorSystem(NActors::TActivationContext::ActorSystem())
-    , RetryPolicy(NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.MaxRetries = 3, .ExtendedRetriedCodes = {CURLE_GOT_NOTHING}}))
+    , RetryPolicy(NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.MaxRetries = 3, .RetriedCurlCodes = NYql::FqRetriedCurlCodes()}))
     , RetryCount(GLOBAL_RETRY_LIMIT) {
         // ^^^ 3 retries in HTTP GW per operation
         // up to 100 retries at app level for all operations ^^^