Use empty result when location is nil (logstash-plugins#73)

* Use empty result when location is nil. 
* Add tags when this happens.
* Adds new tests for IPv6 

Fix logstash-plugins#70

CHANGELOG.md

@@ -1,3 +1,6 @@
+# 3.0.0-beta3
+ - Return empty result when IP lookup fails for location field (#70)
+
 # 3.0.0-beta2
  - Internal: Actually include the vendored jars
 

lib/logstash/filters/geoip.rb

@@ -119,6 +119,9 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
   # to having multiple caches for different instances at different points in the pipeline, that would just increase the
   # number of cache misses and waste memory.
   config :lru_cache_size, :validate => :number, :default => 1000
+
+  # Tags the event on failure to look up geo information. This can be used in later analysis.
+  config :tag_on_failure, :validate => :array, :default => ["_geoip_lookup_failure"]
 
   public
   def register
@@ -150,69 +153,84 @@ def filter(event)
     begin
       ip = event[@source]
       ip = ip.first if ip.is_a? Array
+      geo_data_hash = Hash.new
       ip_address = InetAddress.getByName(ip)
       response = @parser.city(ip_address)
-      country = response.getCountry()
-      subdivision = response.getMostSpecificSubdivision()
-      city = response.getCity()
-      postal = response.getPostal()
-      location = response.getLocation()
-
-      geo_data_hash = Hash.new()
-
-      @fields.each do |field|
-        case field
-        when "city_name"
-          geo_data_hash["city_name"] = city.getName()
-        when "country_name"
-          geo_data_hash["country_name"] = country.getName()
-        when "continent_code"
-          geo_data_hash["continent_code"] = response.getContinent().getCode()
-        when "continent_name"
-          geo_data_hash["continent_name"] = response.getContinent().getName()
-        when "country_code2"
-          geo_data_hash["country_code2"] = country.getIsoCode()
-        when "country_code3"
-          geo_data_hash["country_code3"] = country.getIsoCode()
-        when "ip"
-          geo_data_hash["ip"] = ip_address.getHostAddress()
-        when "postal_code"
-          geo_data_hash["postal_code"] = postal.getCode()
-        when "dma_code"
-          geo_data_hash["dma_code"] = location.getMetroCode()
-        when "region_name"
-          geo_data_hash["region_name"] = subdivision.getName()
-        when "region_code"
-          geo_data_hash["region_code"] = subdivision.getIsoCode()
-        when "timezone"
-          geo_data_hash["timezone"] = location.getTimeZone()
-        when "location"
-          geo_data_hash["location"] = [ location.getLongitude(), location.getLatitude() ]
-        when "latitude"
-          geo_data_hash["latitude"] = location.getLatitude()
-        when "longitude"
-          geo_data_hash["longitude"] = location.getLongitude()
-        else
-          raise Exception.new("[#{field}] is not a supported field option.")
-        end
-      end
-
+      populate_geo_data(response, ip_address, geo_data_hash)
     rescue com.maxmind.geoip2.exception.AddressNotFoundException => e
       @logger.debug("IP not found!", :exception => e, :field => @source, :event => event)
-      event[@target] = {}
-      return
     rescue java.net.UnknownHostException => e
       @logger.error("IP Field contained invalid IP address or hostname", :exception => e, :field => @source, :event => event)
-      event[@target] = {}
-      return
     rescue Exception => e
       @logger.error("Unknown error while looking up GeoIP data", :exception => e, :field => @source, :event => event)
-      event[@target] = {}
-      return
+      # Dont' swallow this, bubble up for unknown issue
+      raise e
     end
 
     event[@target] = geo_data_hash
 
+    if geo_data_hash.empty?
+      tag_unsuccessful_lookup(event)
+      return
+    end
+
     filter_matched(event)
   end # def filter
+
+  def populate_geo_data(response, ip_address, geo_data_hash)
+    country = response.getCountry()
+    subdivision = response.getMostSpecificSubdivision()
+    city = response.getCity()
+    postal = response.getPostal()
+    location = response.getLocation()
+
+    # if location is empty, there is no point populating geo data
+    # and most likely all other fields are empty as well
+    if location.getLatitude().nil? && location.getLongitude().nil?
+      return
+    end
+
+    @fields.each do |field|
+      case field
+      when "city_name"
+        geo_data_hash["city_name"] = city.getName()
+      when "country_name"
+        geo_data_hash["country_name"] = country.getName()
+      when "continent_code"
+        geo_data_hash["continent_code"] = response.getContinent().getCode()
+      when "continent_name"
+        geo_data_hash["continent_name"] = response.getContinent().getName()
+      when "country_code2"
+        geo_data_hash["country_code2"] = country.getIsoCode()
+      when "country_code3"
+        geo_data_hash["country_code3"] = country.getIsoCode()
+      when "ip"
+        geo_data_hash["ip"] = ip_address.getHostAddress()
+      when "postal_code"
+        geo_data_hash["postal_code"] = postal.getCode()
+      when "dma_code"
+        geo_data_hash["dma_code"] = location.getMetroCode()
+      when "region_name"
+        geo_data_hash["region_name"] = subdivision.getName()
+      when "region_code"
+        geo_data_hash["region_code"] = subdivision.getIsoCode()
+      when "timezone"
+        geo_data_hash["timezone"] = location.getTimeZone()
+      when "location"
+        geo_data_hash["location"] = [ location.getLongitude(), location.getLatitude() ]
+      when "latitude"
+        geo_data_hash["latitude"] = location.getLatitude()
+      when "longitude"
+        geo_data_hash["longitude"] = location.getLongitude()
+      else
+        raise Exception.new("[#{field}] is not a supported field option.")
+      end
+    end
+  end
+
+  def tag_unsuccessful_lookup(event)
+    @logger.debug? && @logger.debug("IP #{event[@source]} was not found in the database", :event => event)
+    @tag_on_failure.each{|tag| event.tag(tag)}
+  end
+
 end # class LogStash::Filters::GeoIP

logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '3.0.0.beta2'
+  s.version         = '3.0.0.beta3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "$summary"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/filters/geoip_spec.rb

@@ -173,8 +173,8 @@
           expect(event["geoip"]).to eq({})
         end
 
-        it "should not have added any tags" do
-          expect(event["tags"]).to be_nil
+        it "should add failure tags" do
+          expect(event["tags"]).to include("_geoip_lookup_failure")
         end
       end
 
@@ -186,6 +186,33 @@
           expect(event["geoip"]).to eq({})
         end
       end
+
+      context "when a IP is not found in the DB" do
+        let(:ipstring) { "113.208.89.21" }
+
+        it "should set the target field to an empty hash" do
+          expect(event["geoip"]).to eq({})
+          expect(event["tags"]).to include("_geoip_lookup_failure")
+        end
+      end
+
+      context "when IP is IPv6 format for localhost" do
+        let(:ipstring) { "::1" }
+
+        it "should set the target field to an empty hash" do
+          expect(event["geoip"]).to eq({})
+        end
+      end
+
+      context "when IP is IPv6 format" do
+        let(:ipstring) { "2607:f0d0:1002:51::4" }
+
+        it "should set the target field to an empty hash" do
+          expect(event["geoip"]).not_to be_empty
+          expect(event["geoip"]["city_name"]).not_to be_nil
+        end
+      end
+
     end
 
     context "should return the correct source field in the logging message" do