Raise logging level for 'invalid_index_name_exception' ES exceptions to ERROR

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.1.4
+  - Log an error -- not a warning -- when ES raises an invalid\_index\_name\_exception.
+
 ## 9.1.3
   - Improve plugin behavior when Elasticsearch is down on startup #758
 

lib/logstash/outputs/elasticsearch/common.rb

@@ -213,14 +213,7 @@ def submit(actions)
           @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
           next
         elsif DOC_DLQ_CODES.include?(status)
-          action_event = action[2]
-          # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
-          if @dlq_writer
-            # TODO: Change this to send a map with { :status => status, :action => action } in the future
-            @dlq_writer.write(action_event, "Could not index event to Elasticsearch. status: #{status}, action: #{action}, response: #{response}")
-          else
-            @logger.warn "Could not index event to Elasticsearch.", status: status, action: action, response: response
-          end
+          handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:non_retryable_failures)
           next
         else
@@ -234,6 +227,22 @@ def submit(actions)
       actions_to_retry
     end
 
+    def handle_dlq_status(message, action, status, response)
+      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
+      if @dlq_writer
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
+      else
+        error_type = response.fetch('index', {}).fetch('error', {})['type']
+        if 'invalid_index_name_exception' == error_type
+          level = :error
+        else
+          level = :warn
+        end
+        @logger.send level, message, status: status, action: action, response: response
+      end
+    end
+
     # Determine the correct value for the 'type' field for the given event
     DEFAULT_EVENT_TYPE_ES6="doc".freeze
     DEFAULT_EVENT_TYPE_ES7="_doc".freeze

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.1.3'
+  s.version         = '9.1.4'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/unit/outputs/elasticsearch_spec.rb

@@ -17,7 +17,7 @@
       allow(subject.client.pool).to receive(:start_resurrectionist)
       allow(subject.client.pool).to receive(:start_sniffer)
       allow(subject.client.pool).to receive(:healthcheck!)
-      allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
+      allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
       subject.client.pool.adapter.manticore.respond_with(:body => "{}")
     end
   end
@@ -358,6 +358,7 @@
       let(:options) { super.merge("action" => "index") }
 
       it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
+        allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
         action, params, event_data = subject.event_action_tuple(event)
         expect(params).not_to include({:_retry_on_conflict => num_retries})
       end
@@ -498,4 +499,60 @@
       end
     end
   end
+
+  context 'handling elasticsearch document-level status meant for the DLQ' do
+    let(:options) { { "manage_template" => false } }
+    let(:logger)  { subject.instance_variable_get(:@logger) }
+
+    context 'when @dlq_writer is nil' do
+      before { subject.instance_variable_set '@dlq_writer', nil }
+
+      context 'resorting to previous behaviour of logging the error' do
+        context 'getting an invalid_index_name_exception' do
+          it 'should log at ERROR level' do
+            expect(logger).to receive(:error).with(/Could not index/, hash_including(:status, :action, :response))
+            mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
+            subject.handle_dlq_status("Could not index event to Elasticsearch.",
+              [:action, :params, :event], :some_status, mock_response)
+          end
+        end
+
+        context 'when getting any other exception' do
+          it 'should log at WARN level' do
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
+            subject.handle_dlq_status("Could not index event to Elasticsearch.",
+              [:action, :params, :event], :some_status, mock_response)
+          end
+        end
+
+        context 'when the response does not include [error]' do
+          it 'should not fail, but just log a warning' do
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            mock_response = { 'index' => {} }
+            expect do
+              subject.handle_dlq_status("Could not index event to Elasticsearch.",
+                [:action, :params, :event], :some_status, mock_response)
+            end.to_not raise_error
+          end
+        end
+      end
+    end
+
+    # DLQ writer always nil, no matter what I try here. So mocking it all the way
+    context 'when DLQ is enabled' do
+      let(:dlq_writer) { double('DLQ writer') }
+      before { subject.instance_variable_set('@dlq_writer', dlq_writer) }
+
+      # Note: This is not quite the desired behaviour.
+      # We should still log when sending to the DLQ.
+      # This shall be solved by another issue, however: logstash-output-elasticsearch#772
+      it 'should send the event to the DLQ instead, and not log' do
+        expect(dlq_writer).to receive(:write).with(:event, /Could not index/)
+        mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
+        subject.handle_dlq_status("Could not index event to Elasticsearch.",
+          [:action, :params, :event], :some_status, mock_response)
+      end
+    end
+  end
 end