diff --git a/lib/cucumber/configuration.rb b/lib/cucumber/configuration.rb
index 7b550531b4..58042dbd5b 100644
--- a/lib/cucumber/configuration.rb
+++ b/lib/cucumber/configuration.rb
@@ -26,6 +26,8 @@ def self.default
     # @method on_event
     def_instance_delegator :event_bus, :on, :on_event
 
+    attr_writer :total_cases
+
     # @private
     def notify(message, *args)
       event_bus.send(message, *args)
@@ -67,6 +69,10 @@ def retry_attempts
       @options[:retry]
     end
 
+    def total_cases
+      @total_cases ||= 0
+    end
+
     def guess?
       @options[:guess]
     end
diff --git a/lib/cucumber/filters/retry.rb b/lib/cucumber/filters/retry.rb
index db4f63b0ee..05b59ca11c 100644
--- a/lib/cucumber/filters/retry.rb
+++ b/lib/cucumber/filters/retry.rb
@@ -8,6 +8,7 @@ module Filters
     class Retry < Core::Filter.new(:configuration)
 
       def test_case(test_case)
+        configuration.total_cases += 1
         configuration.on_event(:test_case_finished) do |event|
           next unless retry_required?(test_case, event)
 
diff --git a/lib/cucumber/runtime.rb b/lib/cucumber/runtime.rb
index 714275b08d..4f5a5386ab 100644
--- a/lib/cucumber/runtime.rb
+++ b/lib/cucumber/runtime.rb
@@ -229,6 +229,8 @@ def legacy_formatter?(factory)
     def failure?
       if @configuration.wip?
         summary_report.test_cases.total_passed > 0
+      elsif @configuration.retry_attempts > 0
+        summary_report.test_cases.total_passed != @configuration.total_cases
       else
         summary_report.test_cases.total_failed > 0 || summary_report.test_steps.total_failed > 0 ||
           (@configuration.strict? && (summary_report.test_steps.total_undefined > 0 || summary_report.test_steps.total_pending > 0))
@@ -248,6 +250,7 @@ def filters
         filters << Cucumber::Core::Test::LocationsFilter.new(filespecs.locations)
         filters << Filters::Randomizer.new(@configuration.seed) if @configuration.randomize?
         filters << Filters::Quit.new
+
         filters << Filters::Retry.new(@configuration)
         # TODO: can we just use RbLanguages's step definitions directly?
         step_match_search = StepMatchSearch.new(@support_code.ruby.method(:step_matches), @configuration)