Add more sniffing tests

Author: andrewvc

lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -130,7 +130,7 @@ def sniff!
     ES2_SNIFF_RE_URL  = /([^\/]*)?\/?([^:]*):([0-9]+)/
     # Sniffs and returns the results. Does not update internal URLs!
     def check_sniff
-      url, resp = perform_request('GET', '_nodes')
+      url, resp = perform_request(:get, '_nodes')
       parsed = LogStash::Json.load(resp.body)
       parsed['nodes'].map do |id,info|
         # TODO Make sure this works with shield. Does that listed

spec/fixtures/5x_node_resp.json

@@ -0,0 +1,2 @@
+
+{"cluster_name":"elasticsearch","nodes":{"vW8V2o_KSnOa-i97FACWFw":{"name":"Blockbuster","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9200","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Blockbuster"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19048,"mlockall":false},"jvm":{"pid":19048,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781724873,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9300","[::1]:9300","127.0.0.1:9300"],"publish_address":"127.0.0.1:9300","profiles":{}},"http":{"bound_address":["[fe80::1]:9200","[::1]:9200","127.0.0.1:9200"],"publish_address":"127.0.0.1:9200","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}},"BIAIepXSTYufETY06CUpYw":{"name":"Spyne","transport_address":"127.0.0.1:9301","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9201","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Spyne"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19029,"mlockall":false},"jvm":{"pid":19029,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781692985,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9301","[::1]:9301","127.0.0.1:9301"],"publish_address":"127.0.0.1:9301","profiles":{}},"http":{"bound_address":["[fe80::1]:9201","[::1]:9201","127.0.0.1:9201"],"publish_address":"127.0.0.1:9201","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}}}}

spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -1,5 +1,6 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
+require "json"
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
@@ -38,6 +39,31 @@
         expect(subject.sniffer_alive?).to eql(true)
       end
     end
+
+    describe "check sniff" do
+      context "with a good sniff result" do
+        let(:sniff_resp_path) { File.dirname(__FILE__) + '/../../../../fixtures/5x_node_resp.json' }
+        let(:sniff_resp) { double("resp") }
+        let(:sniff_resp_body) { File.open(sniff_resp_path).read }
+        
+        before do
+          allow(subject).to receive(:perform_request).
+                              with(:get, '_nodes').
+                              and_return([double('url'), sniff_resp])
+          allow(sniff_resp).to receive(:body).and_return(sniff_resp_body)
+        end
+        
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return the correct sniff URL list" do
+          url_strs = subject.check_sniff.map(&:to_s)
+          expect(url_strs).to include("http://127.0.0.1:9200")
+          expect(url_strs).to include("http://127.0.0.1:9201")
+        end
+      end
+    end
   end
 
   describe "closing" do