port deep parameter logging from Badiapp/grape_logging (aserafin#37)

* port deep parameter logging from Badiapp/grape_logging; add some specs

* fix initializer for ParameterFilter

* PR feedback

.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+.rspec

grape_logging.gemspec

@@ -23,4 +23,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.8'
   spec.add_development_dependency 'rake', '~> 10.0'
-end
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'pry-byebug', '~> 3.4.2'
+end

lib/grape_logging.rb

@@ -12,3 +12,4 @@
 require 'grape_logging/reporters/logger_reporter'
 require 'grape_logging/timings'
 require 'grape_logging/middleware/request_logger'
+require 'grape_logging/util/parameter_filter'

lib/grape_logging/loggers/filter_parameters.rb

@@ -1,23 +1,35 @@
 module GrapeLogging
   module Loggers
     class FilterParameters < GrapeLogging::Loggers::Base
-      def initialize(filter_parameters = nil, replacement = '[FILTERED]')
+      AD_PARAMS = 'action_dispatch.request.parameters'.freeze
+
+      def initialize(filter_parameters = nil, replacement = nil, exceptions = %w(controller action format))
         @filter_parameters = filter_parameters || (defined?(Rails.application) ? Rails.application.config.filter_parameters : [])
-        @replacement = replacement
+        @replacement = replacement || '[FILTERED]'
+        @exceptions = exceptions
       end
 
       def parameters(request, _)
-        { params: replace_parameters(request.params.clone) }
+        { params: safe_parameters(request) }
       end
 
       private
-      def replace_parameters(parameters)
-        @filter_parameters.each do |parameter_name|
-          if parameters.key?(parameter_name.to_s)
-            parameters[parameter_name.to_s] = @replacement
-          end
+
+      def parameter_filter
+        @parameter_filter ||= ParameterFilter.new(@replacement, @filter_parameters)
+      end
+
+      def safe_parameters(request)
+        # Now this logger can work also over Rails requests
+        if request.params.empty?
+          clean_parameters(request.env[AD_PARAMS] || {})
+        else
+          clean_parameters(request.params)
         end
-        parameters
+      end
+
+      def clean_parameters(parameters)
+        parameter_filter.filter(parameters).reject{ |key, _value| @exceptions.include?(key) }
       end
     end
   end

lib/grape_logging/util/parameter_filter.rb

@@ -0,0 +1,90 @@
+if defined?(Rails.application)
+  class ParameterFilter < ActionDispatch::Http::ParameterFilter
+    def initialize(_replacement, filter_parameters)
+      super(filter_parameters)
+    end
+  end
+else
+  #
+  # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/http/parameter_filter.rb
+  # we could depend on Rails specifically, but that would us way to hefty!
+  #
+  class ParameterFilter
+    def initialize(replacement, filters = [])
+      @replacement = replacement
+      @filters = filters
+    end
+
+    def filter(params)
+      compiled_filter.call(params)
+    end
+
+    private
+
+    def compiled_filter
+      @compiled_filter ||= CompiledFilter.compile(@replacement, @filters)
+    end
+
+    class CompiledFilter # :nodoc:
+      def self.compile(replacement, filters)
+        return lambda { |params| params.dup } if filters.empty?
+
+        strings, regexps, blocks = [], [], []
+
+        filters.each do |item|
+          case item
+          when Proc
+            blocks << item
+          when Regexp
+            regexps << item
+          else
+            strings << Regexp.escape(item.to_s)
+          end
+        end
+
+        deep_regexps, regexps = regexps.partition { |r| r.to_s.include?("\\.".freeze) }
+        deep_strings, strings = strings.partition { |s| s.include?("\\.".freeze) }
+
+        regexps << Regexp.new(strings.join('|'.freeze), true) unless strings.empty?
+        deep_regexps << Regexp.new(deep_strings.join('|'.freeze), true) unless deep_strings.empty?
+
+        new replacement, regexps, deep_regexps, blocks
+      end
+
+      attr_reader :regexps, :deep_regexps, :blocks
+
+      def initialize(replacement, regexps, deep_regexps, blocks)
+        @replacement = replacement
+        @regexps = regexps
+        @deep_regexps = deep_regexps.any? ? deep_regexps : nil
+        @blocks  = blocks
+      end
+
+      def call(original_params, parents = [])
+        filtered_params = {}
+
+        original_params.each do |key, value|
+          parents.push(key) if deep_regexps
+          if regexps.any? { |r| key =~ r }
+            value = @replacement
+          elsif deep_regexps && (joined = parents.join('.')) && deep_regexps.any? { |r| joined =~ r }
+            value = @replacement
+          elsif value.is_a?(Hash)
+            value = call(value, parents)
+          elsif value.is_a?(Array)
+            value = value.map { |v| v.is_a?(Hash) ? call(v, parents) : v }
+          elsif blocks.any?
+            key = key.dup if key.duplicable?
+            value = value.dup if value.duplicable?
+            blocks.each { |b| b.call(key, value) }
+          end
+          parents.pop if deep_regexps
+
+          filtered_params[key] = value
+        end
+
+        filtered_params
+      end
+    end
+  end
+end

spec/lib/grape_logging/loggers/client_env_spec.rb

@@ -0,0 +1,49 @@
+require 'ostruct'
+
+describe GrapeLogging::Loggers::ClientEnv do
+  let(:ip) { '10.0.0.1' }
+  let(:user_agent) { 'user agent' }
+  let(:forwarded_for) { "forwarded for" }
+  let(:remote_addr) { "remote address" }
+
+  context 'forwarded for' do
+    let(:mock_request) do
+      OpenStruct.new(env: {
+        "HTTP_X_FORWARDED_FOR" => forwarded_for
+      })
+    end
+
+    it 'sets the ip key' do
+      expect(subject.parameters(mock_request, nil)).to eq(ip: forwarded_for, ua: nil)
+    end
+
+    it 'prefers the forwarded_for over the remote_addr' do
+      mock_request.env['REMOTE_ADDR'] = remote_addr
+      expect(subject.parameters(mock_request, nil)).to eq(ip: forwarded_for, ua: nil)
+    end
+  end
+
+  context 'remote address' do
+    let(:mock_request) do
+      OpenStruct.new(env: {
+        "REMOTE_ADDR" => remote_addr
+      })
+    end
+
+    it 'sets the ip key' do
+      expect(subject.parameters(mock_request, nil)).to eq(ip: remote_addr, ua: nil)
+    end
+  end
+
+  context 'user agent' do
+    let(:mock_request) do
+      OpenStruct.new(env: {
+        "HTTP_USER_AGENT" => user_agent
+      })
+    end
+
+    it 'sets the ua key' do
+      expect(subject.parameters(mock_request, nil)).to eq(ip: nil, ua: user_agent)
+    end
+  end
+end

spec/lib/grape_logging/loggers/filter_parameters_spec.rb

@@ -0,0 +1,75 @@
+require 'ostruct'
+
+describe GrapeLogging::Loggers::FilterParameters do
+  let(:filtered_parameters) { %w[one four] }
+
+  let(:mock_request) do
+    OpenStruct.new(params: {
+      this_one: 'this one',
+      that_one: 'one',
+      two: 'two',
+      three: 'three',
+      four: 'four'
+    })
+  end
+
+  let(:mock_request_with_deep_nesting) do
+    deep_clone = lambda { Marshal.load Marshal.dump mock_request.params }
+    OpenStruct.new(
+      params: deep_clone.call.merge(
+        five: deep_clone.call.merge(
+          deep_clone.call.merge({six: {seven: 'seven', eight: 'eight', one: 'another one'}})
+        )
+      )
+    )
+  end
+
+  let(:subject) do
+    GrapeLogging::Loggers::FilterParameters.new filtered_parameters, replacement
+  end
+
+  let(:replacement) { nil }
+
+  shared_examples 'filtering' do
+    it 'filters out sensitive parameters' do
+      expect(subject.parameters(mock_request, nil)).to eq(params: {
+        this_one: subject.instance_variable_get('@replacement'),
+        that_one: subject.instance_variable_get('@replacement'),
+        two: 'two',
+        three: 'three',
+        four: subject.instance_variable_get('@replacement'),
+      })
+    end
+
+    it 'deeply filters out sensitive parameters' do
+      expect(subject.parameters(mock_request_with_deep_nesting, nil)).to eq(params: {
+        this_one: subject.instance_variable_get('@replacement'),
+        that_one: subject.instance_variable_get('@replacement'),
+        two: 'two',
+        three: 'three',
+        four: subject.instance_variable_get('@replacement'),
+        five: {
+          this_one: subject.instance_variable_get('@replacement'),
+          that_one: subject.instance_variable_get('@replacement'),
+          two: 'two',
+          three: 'three',
+          four: subject.instance_variable_get('@replacement'),
+          six: {
+            seven: 'seven',
+            eight: 'eight',
+            one: subject.instance_variable_get('@replacement'),
+          },
+        },
+      })
+    end
+  end
+
+  context 'with default replacement' do
+    it_behaves_like 'filtering'
+  end
+
+  context 'with custom replacement' do
+    let(:replacement) { 'CUSTOM_REPLACEMENT' }
+    it_behaves_like 'filtering'
+  end
+end

spec/lib/grape_logging/loggers/request_headers_spec.rb

@@ -0,0 +1,13 @@
+require 'ostruct'
+
+describe GrapeLogging::Loggers::RequestHeaders do
+  let(:mock_request) do
+    OpenStruct.new(env: {HTTP_REFERER: 'http://example.com', HTTP_ACCEPT: 'text/plain'})
+  end
+
+  it 'strips HTTP_ from the parameter' do
+    expect(subject.parameters(mock_request, nil)).to eq({
+      headers: {'Referer' => 'http://example.com', 'Accept' => 'text/plain'}
+    })
+  end
+end

spec/lib/grape_logging/loggers/response_spec.rb

@@ -0,0 +1,27 @@
+require 'ostruct'
+
+describe GrapeLogging::Loggers::Response do
+  context 'with a parseable JSON body' do
+    let(:response) do
+      OpenStruct.new(body: [%q{{"one": "two", "three": {"four": 5}}}])
+    end
+
+    it 'returns an array of parseable JSON objects' do
+      expect(subject.parameters(nil, response)).to eq({
+        response: [response.body.first.dup]
+      })
+    end
+  end
+
+  context 'with a body that is not parseable JSON' do
+    let(:response) do
+      OpenStruct.new(body: "this is a body")
+    end
+
+    it 'just returns the body' do
+      expect(subject.parameters(nil, response)).to eq({
+        response: response.body.dup
+      })
+    end
+  end
+end