fix(integration): Add more auto retries for reuqest limit errors (#2570)

## Context

Many dead jobs are raised by integration with an
`SSS_REQUEST_LIMIT_EXCEEDED` error code.
Theses jobs should be automatically retried without ending up in the
dead queue.

## Description

This PR adds the logic to dig in the error message of integration
response looking for the error code. When this error code is
`SSS_REQUEST_LIMIT_EXCEEDED` it now raises a new `RequestLimitError` and
related jobs are now able to reprocess up to 10 times

app/jobs/integrations/aggregator/credit_notes/create_job.rb

@@ -7,6 +7,7 @@ class CreateJob < ApplicationJob
         queue_as 'integrations'
 
         retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+        retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
         def perform(credit_note:)
           result = Integrations::Aggregator::CreditNotes::CreateService.call(credit_note:)

app/jobs/integrations/aggregator/fetch_items_job.rb

@@ -6,6 +6,7 @@ class FetchItemsJob < ApplicationJob
       queue_as 'integrations'
 
       retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+      retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
       def perform(integration:)
         result = Integrations::Aggregator::ItemsService.call(integration:)

app/jobs/integrations/aggregator/fetch_tax_items_job.rb

@@ -6,6 +6,7 @@ class FetchTaxItemsJob < ApplicationJob
       queue_as 'integrations'
 
       retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+      retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
       def perform(integration:)
         result = Integrations::Aggregator::TaxItemsService.call(integration:)

app/jobs/integrations/aggregator/invoices/create_job.rb

@@ -7,6 +7,7 @@ class CreateJob < ApplicationJob
         queue_as 'integrations'
 
         retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+        retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
         def perform(invoice:)
           result = Integrations::Aggregator::Invoices::CreateService.call(invoice:)

app/jobs/integrations/aggregator/payments/create_job.rb

@@ -7,6 +7,7 @@ class CreateJob < ApplicationJob
         queue_as 'integrations'
 
         retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 5
+        retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
         def perform(payment:)
           result = Integrations::Aggregator::Payments::CreateService.call(payment:)

app/jobs/integrations/aggregator/perform_sync_job.rb

@@ -6,6 +6,7 @@ class PerformSyncJob < ApplicationJob
       queue_as 'integrations'
 
       retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+      retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
       def perform(integration:, sync_tax_items: false)
         sync_result = Integrations::Aggregator::SyncService.call(integration:)

app/jobs/integrations/aggregator/sales_orders/create_job.rb

@@ -7,6 +7,7 @@ class CreateJob < ApplicationJob
         queue_as 'integrations'
 
         retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+        retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
         def perform(invoice:)
           result = Integrations::Aggregator::SalesOrders::CreateService.call(invoice:)

app/jobs/integrations/aggregator/send_restlet_endpoint_job.rb

@@ -6,6 +6,7 @@ class SendRestletEndpointJob < ApplicationJob
       queue_as 'integrations'
 
       retry_on LagoHttpClient::HttpError, wait: :polynomially_longer, attempts: 3
+      retry_on RequestLimitError, wait: :polynomially_longer, attempts: 10
 
       def perform(integration:)
         result = Integrations::Aggregator::SendRestletEndpointService.call(integration:)

app/services/integrations/aggregator/base_service.rb

@@ -6,6 +6,7 @@ module Integrations
   module Aggregator
     class BaseService < BaseService
       BASE_URL = 'https://api.nango.dev/'
+      REQUEST_LIMIT_ERROR_CODE = 'SSS_REQUEST_LIMIT_EXCEEDED'
 
       def initialize(integration:, options: {})
         @integration = integration
@@ -101,6 +102,14 @@ def message(error)
           json.dig('error', 'payload', 'message').presence ||
           json.dig('error', 'message')
       end
+
+      def request_limit_error?(http_error)
+        return false unless http_error.error_code.to_i == 500
+
+        http_error.json_message.dig('error', 'payload', 'error', 'code') == REQUEST_LIMIT_ERROR_CODE
+      rescue JSON::ParserError
+        false
+      end
     end
   end
 end

app/services/integrations/aggregator/contacts/create_service.rb

@@ -27,6 +27,8 @@ def call
 
           result
         rescue LagoHttpClient::HttpError => e
+          raise RequestLimitError(e) if request_limit_error?(e)
+
           code = code(e)
           message = message(e)
 

app/services/integrations/aggregator/contacts/update_service.rb

@@ -26,6 +26,8 @@ def call
 
           result
         rescue LagoHttpClient::HttpError => e
+          raise RequestLimitError(e) if request_limit_error?(e)
+
           code = code(e)
           message = message(e)
 

app/services/integrations/aggregator/credit_notes/create_service.rb

@@ -40,6 +40,8 @@ def call
 
           result
         rescue LagoHttpClient::HttpError => e
+          raise RequestLimitError(e) if request_limit_error?(e)
+
           code = code(e)
           message = message(e)
 

app/services/integrations/aggregator/invoices/create_service.rb

@@ -34,6 +34,8 @@ def call
 
           result
         rescue LagoHttpClient::HttpError => e
+          raise RequestLimitError(e) if request_limit_error?(e)
+
           code = code(e)
           message = message(e)
 

app/services/integrations/aggregator/payments/create_service.rb

@@ -40,6 +40,8 @@ def call
 
           result
         rescue LagoHttpClient::HttpError => e
+          raise RequestLimitError(e) if request_limit_error?(e)
+
           code = code(e)
           message = message(e)
 

app/services/integrations/aggregator/request_limit_error.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Integrations
+  module Aggregator
+    class RequestLimitError < StandardError
+      def initialize(http_error)
+        @http_error = http_error
+        super(http_error.message)
+      end
+
+      attr_reader :http_error
+    end
+  end
+end

app/services/integrations/aggregator/sales_orders/create_service.rb

@@ -26,6 +26,8 @@ def call
 
           result
         rescue LagoHttpClient::HttpError => e
+          raise RequestLimitError(e) if request_limit_error?(e)
+
           code = code(e)
           message = message(e)
 

app/services/integrations/aggregator/taxes/invoices/create_draft_service.rb

@@ -20,6 +20,8 @@ def call
 
             result
           rescue LagoHttpClient::HttpError => e
+            raise RequestLimitError(e) if request_limit_error?(e)
+
             code = code(e)
             message = message(e)
 

app/services/integrations/aggregator/taxes/invoices/create_service.rb

@@ -24,6 +24,8 @@ def call
 
             result
           rescue LagoHttpClient::HttpError => e
+            raise RequestLimitError(e) if request_limit_error?(e)
+
             code = code(e)
             message = message(e)
 

app/services/integrations/aggregator/taxes/invoices/negate_service.rb

@@ -20,6 +20,8 @@ def call
 
             result
           rescue LagoHttpClient::HttpError => e
+            raise RequestLimitError(e) if request_limit_error?(e)
+
             code = code(e)
             message = message(e)
 

app/services/integrations/aggregator/taxes/invoices/void_service.rb

@@ -20,6 +20,8 @@ def call
 
             result
           rescue LagoHttpClient::HttpError => e
+            raise RequestLimitError(e) if request_limit_error?(e)
+
             code = code(e)
             message = message(e)