diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d83396..8f1d663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# 3.0.0-beta3 + - Return empty result when IP lookup fails for location field (#70) + # 3.0.0-beta2 - Internal: Actually include the vendored jars diff --git a/lib/logstash/filters/geoip.rb b/lib/logstash/filters/geoip.rb index faa31aa..26145be 100644 --- a/lib/logstash/filters/geoip.rb +++ b/lib/logstash/filters/geoip.rb @@ -119,6 +119,9 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base # to having multiple caches for different instances at different points in the pipeline, that would just increase the # number of cache misses and waste memory. config :lru_cache_size, :validate => :number, :default => 1000 + + # Tags the event on failure to look up geo information. This can be used in later analysis. + config :tag_on_failure, :validate => :array, :default => ["_geoip_lookup_failure"] public def register @@ -150,69 +153,84 @@ def filter(event) begin ip = event[@source] ip = ip.first if ip.is_a? Array + geo_data_hash = Hash.new ip_address = InetAddress.getByName(ip) response = @parser.city(ip_address) - country = response.getCountry() - subdivision = response.getMostSpecificSubdivision() - city = response.getCity() - postal = response.getPostal() - location = response.getLocation() - - geo_data_hash = Hash.new() - - @fields.each do |field| - case field - when "city_name" - geo_data_hash["city_name"] = city.getName() - when "country_name" - geo_data_hash["country_name"] = country.getName() - when "continent_code" - geo_data_hash["continent_code"] = response.getContinent().getCode() - when "continent_name" - geo_data_hash["continent_name"] = response.getContinent().getName() - when "country_code2" - geo_data_hash["country_code2"] = country.getIsoCode() - when "country_code3" - geo_data_hash["country_code3"] = country.getIsoCode() - when "ip" - geo_data_hash["ip"] = ip_address.getHostAddress() - when "postal_code" - geo_data_hash["postal_code"] = postal.getCode() - when "dma_code" - geo_data_hash["dma_code"] = location.getMetroCode() - when "region_name" - geo_data_hash["region_name"] = subdivision.getName() - when "region_code" - geo_data_hash["region_code"] = subdivision.getIsoCode() - when "timezone" - geo_data_hash["timezone"] = location.getTimeZone() - when "location" - geo_data_hash["location"] = [ location.getLongitude(), location.getLatitude() ] - when "latitude" - geo_data_hash["latitude"] = location.getLatitude() - when "longitude" - geo_data_hash["longitude"] = location.getLongitude() - else - raise Exception.new("[#{field}] is not a supported field option.") - end - end - + populate_geo_data(response, ip_address, geo_data_hash) rescue com.maxmind.geoip2.exception.AddressNotFoundException => e @logger.debug("IP not found!", :exception => e, :field => @source, :event => event) - event[@target] = {} - return rescue java.net.UnknownHostException => e @logger.error("IP Field contained invalid IP address or hostname", :exception => e, :field => @source, :event => event) - event[@target] = {} - return rescue Exception => e @logger.error("Unknown error while looking up GeoIP data", :exception => e, :field => @source, :event => event) - event[@target] = {} - return + # Dont' swallow this, bubble up for unknown issue + raise e end event[@target] = geo_data_hash + if geo_data_hash.empty? + tag_unsuccessful_lookup(event) + return + end + filter_matched(event) end # def filter + + def populate_geo_data(response, ip_address, geo_data_hash) + country = response.getCountry() + subdivision = response.getMostSpecificSubdivision() + city = response.getCity() + postal = response.getPostal() + location = response.getLocation() + + # if location is empty, there is no point populating geo data + # and most likely all other fields are empty as well + if location.getLatitude().nil? && location.getLongitude().nil? + return + end + + @fields.each do |field| + case field + when "city_name" + geo_data_hash["city_name"] = city.getName() + when "country_name" + geo_data_hash["country_name"] = country.getName() + when "continent_code" + geo_data_hash["continent_code"] = response.getContinent().getCode() + when "continent_name" + geo_data_hash["continent_name"] = response.getContinent().getName() + when "country_code2" + geo_data_hash["country_code2"] = country.getIsoCode() + when "country_code3" + geo_data_hash["country_code3"] = country.getIsoCode() + when "ip" + geo_data_hash["ip"] = ip_address.getHostAddress() + when "postal_code" + geo_data_hash["postal_code"] = postal.getCode() + when "dma_code" + geo_data_hash["dma_code"] = location.getMetroCode() + when "region_name" + geo_data_hash["region_name"] = subdivision.getName() + when "region_code" + geo_data_hash["region_code"] = subdivision.getIsoCode() + when "timezone" + geo_data_hash["timezone"] = location.getTimeZone() + when "location" + geo_data_hash["location"] = [ location.getLongitude(), location.getLatitude() ] + when "latitude" + geo_data_hash["latitude"] = location.getLatitude() + when "longitude" + geo_data_hash["longitude"] = location.getLongitude() + else + raise Exception.new("[#{field}] is not a supported field option.") + end + end + end + + def tag_unsuccessful_lookup(event) + @logger.debug? && @logger.debug("IP #{event[@source]} was not found in the database", :event => event) + @tag_on_failure.each{|tag| event.tag(tag)} + end + end # class LogStash::Filters::GeoIP diff --git a/logstash-filter-geoip.gemspec b/logstash-filter-geoip.gemspec index f58f2e7..c26e65e 100644 --- a/logstash-filter-geoip.gemspec +++ b/logstash-filter-geoip.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-geoip' - s.version = '3.0.0.beta2' + s.version = '3.0.0.beta3' s.licenses = ['Apache License (2.0)'] s.summary = "$summary" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/filters/geoip_spec.rb b/spec/filters/geoip_spec.rb index 5157f15..dee8f74 100644 --- a/spec/filters/geoip_spec.rb +++ b/spec/filters/geoip_spec.rb @@ -173,8 +173,8 @@ expect(event["geoip"]).to eq({}) end - it "should not have added any tags" do - expect(event["tags"]).to be_nil + it "should add failure tags" do + expect(event["tags"]).to include("_geoip_lookup_failure") end end @@ -186,6 +186,33 @@ expect(event["geoip"]).to eq({}) end end + + context "when a IP is not found in the DB" do + let(:ipstring) { "113.208.89.21" } + + it "should set the target field to an empty hash" do + expect(event["geoip"]).to eq({}) + expect(event["tags"]).to include("_geoip_lookup_failure") + end + end + + context "when IP is IPv6 format for localhost" do + let(:ipstring) { "::1" } + + it "should set the target field to an empty hash" do + expect(event["geoip"]).to eq({}) + end + end + + context "when IP is IPv6 format" do + let(:ipstring) { "2607:f0d0:1002:51::4" } + + it "should set the target field to an empty hash" do + expect(event["geoip"]).not_to be_empty + expect(event["geoip"]["city_name"]).not_to be_nil + end + end + end context "should return the correct source field in the logging message" do