From 709d7b3703d4396cc0378a6c86a4476dfd3945d7 Mon Sep 17 00:00:00 2001
From: Xavier Shay <xavier@rhnh.net>
Date: Sat, 25 Jan 2014 15:06:51 -0800
Subject: [PATCH] Execute pending examples and mark as failed if they succeed.

This is a backwards incompatible change that makes the behaviour of
pending consisent with the current behaviour for when it is called
inside an example with a block.

The old "never run this example" behaviour is still available via the
"x" family of methods, the `:skip` metadata option, or the new `skip`
method.

Implements #1208.
---
 Changelog.md                                  |  7 ++
 features/command_line/format_option.feature   |  2 +-
 features/hooks/around_hooks.feature           |  1 +
 features/pending/pending_examples.feature     | 82 ++++++++++++++++--
 lib/rspec/core/example.rb                     | 59 +++++++++----
 lib/rspec/core/example_group.rb               | 53 +++++++++---
 lib/rspec/core/pending.rb                     | 33 ++++++--
 spec/rspec/core/example_group_spec.rb         | 83 +++++++++++++++----
 spec/rspec/core/example_spec.rb               | 64 +++++++++++++-
 .../core/formatters/base_formatter_spec.rb    |  1 -
 .../documentation_formatter_spec.rb           |  2 +-
 .../core/formatters/json_formatter_spec.rb    | 19 ++++-
 spec/rspec/core/pending_example_spec.rb       | 49 ++++++++++-
 spec/support/in_sub_process.rb                |  4 +-
 14 files changed, 393 insertions(+), 66 deletions(-)

diff --git a/Changelog.md b/Changelog.md
index 598fcc0356..a6f2932f24 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -24,6 +24,11 @@ Breaking Changes for 3.0.0:
 * Remove support for deprecated `--configure` CLI option. (Myron Marston)
 * Remove support for deprecated `RSpec::Core::RakeTask#spec_opts=`.
   (Myron Marston)
+* An example group level `pending` block or `:pending` metadata now behaves the
+  same as a `pending` block inside an example: it will be executed and cause
+  a failure if it passes, otherwise it will be pending if it fails. The old
+  "never run" behaviour is still used when prefixing example/it/specify with an
+  `x`, or via a new `skip` method or `:skip` metadata option. (Xavier Shay)
 
 Enhancements:
 
@@ -41,6 +46,8 @@ Enhancements:
   pending. (Myron Marston)
 * Add `fdescribe` and `fcontext` as shortcuts to focus an example group.
   (Myron Marston)
+* Add `skip` method to not run any more of an example. (This behaviour is
+  identical to the old `pending` behaviour from RSpec 2.) (Xavier Shay)
 
 Bug Fixes:
 
diff --git a/features/command_line/format_option.feature b/features/command_line/format_option.feature
index f7aa352931..016dc1649f 100644
--- a/features/command_line/format_option.feature
+++ b/features/command_line/format_option.feature
@@ -34,7 +34,7 @@ Feature: --format option
         end
 
         it "does something that is pending", :pending => true do
-          expect(5).to be > 3
+          expect(5).to be < 3
         end
       end
       """
diff --git a/features/hooks/around_hooks.feature b/features/hooks/around_hooks.feature
index 524743327d..b05387f3e7 100644
--- a/features/hooks/around_hooks.feature
+++ b/features/hooks/around_hooks.feature
@@ -230,6 +230,7 @@ Feature: around hooks
 
         it "should be detected as pending" do
           pending
+          fail
         end
       end
       """
diff --git a/features/pending/pending_examples.feature b/features/pending/pending_examples.feature
index 9e59137c93..ab152698d8 100644
--- a/features/pending/pending_examples.feature
+++ b/features/pending/pending_examples.feature
@@ -77,7 +77,78 @@ Feature: pending examples
     And the output should contain "Expected pending 'something else getting finished' to fail. No Error was raised."
     And the output should contain "pending_with_passing_block_spec.rb:3"
 
-  Scenario: temporarily pending by prefixing `it`, `specify`, or `example` with an x
+  Scenario: pending for an example that is currently passing
+    Given a file named "pending_with_passing_block_spec.rb" with:
+      """ruby
+      describe "an example" do
+        pending("something else getting finished") do
+          expect(1).to eq(1)
+        end
+      end
+      """
+    When I run `rspec pending_with_passing_block_spec.rb`
+    Then the exit status should not be 0
+    And the output should contain "1 example, 1 failure"
+    And the output should contain "FIXED"
+    And the output should contain "Expected pending 'No reason given' to fail. No Error was raised."
+    And the output should contain "pending_with_passing_block_spec.rb:2"
+
+  Scenario: pending for an example that is currently passing with a reason
+    Given a file named "pending_with_passing_block_spec.rb" with:
+      """ruby
+      describe "an example" do
+        example("something else getting finished", :pending => 'unimplemented') do
+          expect(1).to eq(1)
+        end
+      end
+      """
+    When I run `rspec pending_with_passing_block_spec.rb`
+    Then the exit status should not be 0
+    And the output should contain "1 example, 1 failure"
+    And the output should contain "FIXED"
+    And the output should contain "Expected pending 'unimplemented' to fail. No Error was raised."
+    And the output should contain "pending_with_passing_block_spec.rb:2"
+
+  Scenario: skipping using `skip`
+    Given a file named "skipped_spec.rb" with:
+      """ruby
+      describe "an example" do
+        skip "is skipped" do
+        end
+      end
+      """
+    When I run `rspec skipped_spec.rb`
+    Then the exit status should be 0
+    And the output should contain "1 example, 0 failures, 1 pending"
+    And the output should contain:
+      """
+      Pending:
+        an example is skipped
+          # No reason given
+          # ./skipped_spec.rb:2
+      """
+
+  Scenario: skipping using `skip` inside an example
+    Given a file named "skipped_spec.rb" with:
+      """ruby
+      describe "an example" do
+        it "is skipped" do
+          skip
+        end
+      end
+      """
+    When I run `rspec skipped_spec.rb`
+    Then the exit status should be 0
+    And the output should contain "1 example, 0 failures, 1 pending"
+    And the output should contain:
+      """
+      Pending:
+        an example is skipped
+          # No reason given
+          # ./skipped_spec.rb:2
+      """
+
+  Scenario: temporarily skipping by prefixing `it`, `specify`, or `example` with an x
     Given a file named "temporarily_pending_spec.rb" with:
       """ruby
       describe "an example" do
@@ -98,13 +169,13 @@ Feature: pending examples
       """
       Pending:
         an example is pending using xit
-          # Temporarily disabled with xit
+          # Temporarily skipped with xit
           # ./temporarily_pending_spec.rb:2
         an example is pending using xspecify
-          # Temporarily disabled with xspecify
+          # Temporarily skipped with xspecify
           # ./temporarily_pending_spec.rb:5
         an example is pending using xexample
-          # Temporarily disabled with xexample
+          # Temporarily skipped with xexample
           # ./temporarily_pending_spec.rb:8
       """
 
@@ -117,6 +188,7 @@ Feature: pending examples
         end
         specify do
           pending
+          fail
         end
       end
       """
@@ -138,7 +210,7 @@ Feature: pending examples
           expect(3+4).to eq(7)
         end
         pending do
-          expect("string".reverse).to eq("gnirts")
+          fail
         end
       end
       """
diff --git a/lib/rspec/core/example.rb b/lib/rspec/core/example.rb
index 71e2b2ba3d..ef7d429962 100644
--- a/lib/rspec/core/example.rb
+++ b/lib/rspec/core/example.rb
@@ -41,7 +41,13 @@ def self.delegate_to_metadata(*keys)
         keys.each { |key| define_method(key) { @metadata[key] } }
       end
 
-      delegate_to_metadata :full_description, :execution_result, :file_path, :pending, :location
+      delegate_to_metadata \
+        :execution_result,
+        :file_path,
+        :full_description,
+        :location,
+        :pending,
+        :skip
 
       # Returns the string submitted to `example` or its aliases (e.g.
       # `specify`, `it`, etc).  If no string is submitted (e.g. `it { is_expected.to
@@ -49,7 +55,9 @@ def self.delegate_to_metadata(*keys)
       # there is one, otherwise returns a message including the location of the
       # example.
       def description
-        description = metadata[:description].to_s.empty? ? "example at #{location}" : metadata[:description]
+        description = metadata[:description].to_s.empty? ?
+          "example at #{location}" :
+          metadata[:description]
         RSpec.configuration.format_docstrings_block.call(description)
       end
 
@@ -84,7 +92,6 @@ def initialize(example_group_class, description, metadata, example_block=nil)
         @example_group_class, @options, @example_block = example_group_class, metadata, example_block
         @metadata  = @example_group_class.metadata.for_example(description, metadata)
         @example_group_instance = @exception = nil
-        @pending_declared_in_example = false
         @clock = RSpec::Core::Time
       end
 
@@ -100,6 +107,7 @@ def example_group
       end
 
       alias_method :pending?, :pending
+      alias_method :skipped?, :skip
 
       # @api private
       # instance_evals the block passed to the constructor in the context of
@@ -112,13 +120,21 @@ def run(example_group_instance, reporter)
         start(reporter)
 
         begin
-          unless pending || RSpec.configuration.dry_run?
+          unless skipped? || RSpec.configuration.dry_run?
             with_around_each_hooks do
               begin
                 run_before_each
                 @example_group_instance.instance_exec(self, &@example_block)
-              rescue Pending::PendingDeclaredInExample => e
-                @pending_declared_in_example = e.message
+
+                result = metadata[:execution_result]
+
+                if result[:pending_fixed] == false
+                  result[:pending_fixed] = true
+                  raise Pending::PendingExampleFixedError
+                end
+              rescue Pending::SkipDeclaredInExample
+                # no-op, required metadata has already been set by the `skip`
+                # method.
               rescue Exception => e
                 set_exception(e)
               ensure
@@ -257,16 +273,18 @@ def start(reporter)
       end
 
       def finish(reporter)
-        if @exception
+        pending_message = metadata[:execution_result][:pending_message]
+
+        if pending_message && !metadata[:execution_result][:pending_fixed]
+          record_finished 'pending', :pending_message => pending_message
+          reporter.example_pending self
+          true
+        elsif @exception
           record_finished 'failed', :exception => @exception
           reporter.example_failed self
           false
-        elsif @pending_declared_in_example
-          record_finished 'pending', :pending_message => @pending_declared_in_example
-          reporter.example_pending self
-          true
-        elsif pending
-          record_finished 'pending', :pending_message => String === pending ? pending : Pending::NO_REASON_GIVEN
+        elsif skipped?
+          record_finished 'pending', :pending_message => skip_message
           reporter.example_pending self
           true
         else
@@ -278,7 +296,11 @@ def finish(reporter)
 
       def record_finished(status, results={})
         finished_at = clock.now
-        record results.merge(:status => status, :finished_at => finished_at, :run_time => (finished_at - execution_result[:started_at]).to_f)
+        record results.merge(
+          :status      => status,
+          :finished_at => finished_at,
+          :run_time    => (finished_at - execution_result[:started_at]).to_f
+        )
       end
 
       def run_before_each
@@ -301,7 +323,6 @@ def verify_mocks
         if metadata[:execution_result][:pending_fixed]
           metadata[:execution_result][:pending_fixed] = false
           metadata[:pending] = true
-          @pending_declared_in_example = metadata[:execution_result][:pending_message]
           @exception = nil
         else
           set_exception(e, :dont_print)
@@ -321,6 +342,14 @@ def assign_generated_description
       def record(results={})
         execution_result.update(results)
       end
+
+      def skip_message
+        if String === skip
+          skip
+        else
+          Pending::NO_REASON_GIVEN
+        end
+      end
     end
   end
 end
diff --git a/lib/rspec/core/example_group.rb b/lib/rspec/core/example_group.rb
index 8bd02a253f..130e69ddf9 100644
--- a/lib/rspec/core/example_group.rb
+++ b/lib/rspec/core/example_group.rb
@@ -59,13 +59,41 @@ def self.define_example_method(name, extra_options={})
           define_method(name) do |*all_args, &block|
             desc, *args = *all_args
             options = Metadata.build_hash_from(args)
-            options.update(:pending => RSpec::Core::Pending::NOT_YET_IMPLEMENTED) unless block
+            options.update(:skip => RSpec::Core::Pending::NOT_YET_IMPLEMENTED) unless block
             options.update(extra_options)
+
+            # Metadata inheritance normally happens in `Example#initialize`,
+            # but for `:pending` specifically we need it earlier.
+            pending_metadata = options[:pending] || metadata[:pending]
+
+            if pending_metadata
+              options, block = ExampleGroup.pending_metadata_and_block_for(
+                options.merge(:pending => pending_metadata),
+                block
+              )
+            end
+
             examples << RSpec::Core::Example.new(self, desc, options, block)
             examples.last
           end
         end
 
+        def pending_metadata_and_block_for(options, block)
+          if String === options[:pending]
+            reason = options[:pending]
+          else
+            options[:pending] = true
+            reason = RSpec::Core::Pending::NO_REASON_GIVEN
+          end
+
+          # This will fail if no block is provided, which is effectively the
+          # same as failing the example so it will be marked correctly as
+          # pending.
+          callback = Proc.new { pending(reason) { instance_exec(&block) } }
+
+          return options, callback
+        end
+
         # Defines an example within a group.
         # @example
         #   example do
@@ -100,18 +128,21 @@ def self.define_example_method(name, extra_options={})
         # @see example
         define_example_method :fit,     :focused => true, :focus => true
 
-        # Shortcut to define an example with :pending => true
+        # Shortcut to define an example with :skip => 'Temporarily skipped with xexample'
         # @see example
-        define_example_method :pending,  :pending => true
-        # Shortcut to define an example with :pending => 'Temporarily disabled with xexample'
+        define_example_method :xexample, :skip => 'Temporarily skipped with xexample'
+        # Shortcut to define an example with :skip => 'Temporarily skipped with xit'
         # @see example
-        define_example_method :xexample, :pending => 'Temporarily disabled with xexample'
-        # Shortcut to define an example with :pending => 'Temporarily disabled with xit'
+        define_example_method :xit,      :skip => 'Temporarily skipped with xit'
+        # Shortcut to define an example with :skip => 'Temporarily skipped with xspecify'
         # @see example
-        define_example_method :xit,      :pending => 'Temporarily disabled with xit'
-        # Shortcut to define an example with :pending => 'Temporarily disabled with xspecify'
+        define_example_method :xspecify, :skip => 'Temporarily skipped with xspecify'
+        # Shortcut to define an example with :skip => true
         # @see example
-        define_example_method :xspecify, :pending => 'Temporarily disabled with xspecify'
+        define_example_method :skip,     :skip => true
+        # Shortcut to define an example with :pending => true
+        # @see example
+        define_example_method :pending,  :pending => true
 
         # Works like `alias_method :name, :example` with the added benefit of
         # assigning default metadata to the generated example.
@@ -270,11 +301,11 @@ def self.example_group(*args, &example_group_block)
 
       # Shortcut to temporarily make an example group pending.
       # @see example_group
-      alias_example_group_to :xdescribe, :pending => "Temporarily disabled with xdescribe"
+      alias_example_group_to :xdescribe, :skip => "Temporarily skipped with xdescribe"
 
       # Shortcut to temporarily make an example group pending.
       # @see example_group
-      alias_example_group_to :xcontext,  :pending => "Temporarily disabled with xcontext"
+      alias_example_group_to :xcontext,  :skip => "Temporarily skipped with xcontext"
 
       # Shortcut to define an example group with `:focus` => true
       # @see example_group
diff --git a/lib/rspec/core/pending.rb b/lib/rspec/core/pending.rb
index 00057d6300..6b9d882388 100644
--- a/lib/rspec/core/pending.rb
+++ b/lib/rspec/core/pending.rb
@@ -1,7 +1,7 @@
 module RSpec
   module Core
     module Pending
-      class PendingDeclaredInExample < StandardError; end
+      class SkipDeclaredInExample < StandardError; end
 
       # If Test::Unit is loaed, we'll use its error as baseclass, so that Test::Unit
       # will report unmet RSpec expectations as failures rather than errors.
@@ -71,29 +71,30 @@ class PendingExampleFixedError < StandardError; end
       #       it "does something", :pending => "something else getting finished" do
       #         # ...
       #       end
-      def pending(*args)
+      def pending(*args, &block)
         current_example = RSpec.current_example
 
         return self.class.before(:each) { pending(*args) } unless current_example
 
         options = args.last.is_a?(Hash) ? args.pop : {}
-        message = args.first || NO_REASON_GIVEN
 
         if options[:unless] || (options.has_key?(:if) && !options[:if])
           return block_given? ? yield : nil
         end
 
+        set_message! current_example, args
+
         current_example.metadata[:pending] = true
-        current_example.metadata[:execution_result][:pending_message] = message
-        current_example.execution_result[:pending_fixed] = false
+
         if block_given?
           begin
             no_failure = false
-            yield
+            block.call
             no_failure = true
             current_example.metadata[:pending] = false
           rescue Exception => e
             current_example.execution_result[:exception] = e
+            raise
           end
 
           if no_failure
@@ -101,7 +102,25 @@ def pending(*args)
             raise PendingExampleFixedError.new
           end
         end
-        raise PendingDeclaredInExample.new(message)
+      end
+
+      def skip(*args)
+        current_example = RSpec.current_example
+
+        return self.class.before(:each) { skip(*args) } unless current_example
+
+        set_message! current_example, args
+
+        current_example.metadata[:skip] = true
+
+        raise SkipDeclaredInExample
+      end
+
+      def set_message!(example, args)
+        message = args.first || NO_REASON_GIVEN
+
+        example.metadata[:execution_result][:pending_message] = message
+        example.execution_result[:pending_fixed] = false
       end
     end
   end
diff --git a/spec/rspec/core/example_group_spec.rb b/spec/rspec/core/example_group_spec.rb
index af0972f1a5..6340bd7e1f 100644
--- a/spec/rspec/core/example_group_spec.rb
+++ b/spec/rspec/core/example_group_spec.rb
@@ -888,26 +888,48 @@ def define_and_run_group(define_outer_example = false)
       end
     end
 
-    %w[pending xit xspecify xexample].each do |method_name|
-      describe "::#{method_name}" do
-        before do
-          @group = ExampleGroup.describe
-          @group.send(method_name, "is pending") { }
-        end
+    describe ".pending" do
+      let(:group) { ExampleGroup.describe { pending { fail } } }
 
-        it "generates a pending example" do
-          @group.run
-          expect(@group.examples.first).to be_pending
-        end
+      it "generates a pending example" do
+        group.run
+        expect(group.examples.first).to be_pending
+      end
+
+      it "sets the pending message" do
+        group.run
+        expect(group.examples.first.metadata[:execution_result][:pending_message]).to eq(RSpec::Core::Pending::NO_REASON_GIVEN)
+      end
+    end
+
+    describe ".skip" do
+      let(:group) { ExampleGroup.describe { skip("skip this") { } } }
+
+      it "generates a skipped example" do
+        group.run
+        expect(group.examples.first).to be_skipped
+      end
 
-        it "sets the pending message", :if => method_name == 'pending' do
-          @group.run
-          expect(@group.examples.first.metadata[:execution_result][:pending_message]).to eq(RSpec::Core::Pending::NO_REASON_GIVEN)
+      it "sets the pending message" do
+        group.run
+        expect(group.examples.first.metadata[:execution_result][:pending_message]).to eq(RSpec::Core::Pending::NO_REASON_GIVEN)
+      end
+    end
+
+    %w[xit xspecify xexample].each do |method_name|
+      describe ".#{method_name}" do
+        let(:group) { ExampleGroup.describe.tap {|x|
+          x.send(method_name, "is pending") { }
+        }}
+
+        it "generates a skipped example" do
+          group.run
+          expect(group.examples.first).to be_skipped
         end
 
-        it "sets the pending message", :unless => method_name == 'pending' do
-          @group.run
-          expect(@group.examples.first.metadata[:execution_result][:pending_message]).to eq("Temporarily disabled with #{method_name}")
+        it "sets the pending message" do
+          group.run
+          expect(group.examples.first.metadata[:execution_result][:pending_message]).to eq("Temporarily skipped with #{method_name}")
         end
       end
     end
@@ -930,7 +952,7 @@ def extract_execution_results(group)
           expect(extract_execution_results(group)).to match([
             a_hash_including(
               :status => "pending",
-              :pending_message => "Temporarily disabled with #{method_name}"
+              :pending_message => "Temporarily skipped with #{method_name}"
             )
           ] * 2)
         end
@@ -967,6 +989,33 @@ def executed_examples_of(group)
       end
     end
 
+    describe "setting pending metadata in parent" do
+      def extract_execution_results(group)
+        group.examples.map do |ex|
+          ex.metadata.fetch(:execution_result)
+        end
+      end
+
+      it 'marks every example as pending' do
+        group = ExampleGroup.describe(:pending => true) do
+          it("passes") { }
+          it("fails", :pending => 'unimplemented')  { fail }
+        end
+        group.run
+
+        expect(extract_execution_results(group)).to match([
+          a_hash_including(
+            :status => "failed",
+            :pending_message => "No reason given"
+          ),
+          a_hash_including(
+            :status => "pending",
+            :pending_message => "unimplemented"
+          )
+        ])
+      end
+    end
+
     describe "adding examples" do
 
       it "allows adding an example using 'it'" do
diff --git a/spec/rspec/core/example_spec.rb b/spec/rspec/core/example_spec.rb
index 4ad94f8580..ee6a1aef74 100644
--- a/spec/rspec/core/example_spec.rb
+++ b/spec/rspec/core/example_spec.rb
@@ -119,7 +119,7 @@ def assert(val)
         end
 
         it "uses the file and line number of the example if no matcher ran" do
-          example = example_group.example { pending; expect(4).to eq(5) }
+          example = example_group.example { pending; fail }
           example_group.run
           expect(example.description).to match(/example at #{relative_path(__FILE__)}:#{__LINE__ - 2}/)
         end
@@ -491,6 +491,68 @@ def run_and_capture_reported_message(group)
     end
   end
 
+  describe "#skip" do
+    context "in the example" do
+      it "sets the example to skipped" do
+        group = RSpec::Core::ExampleGroup.describe do
+          example { skip }
+        end
+        group.run
+        expect(group.examples.first).to be_skipped
+      end
+
+      it "allows post-example processing in around hooks (see https://github.com/rspec/rspec-core/issues/322)" do
+        blah = nil
+        group = RSpec::Core::ExampleGroup.describe do
+          around do |example|
+            example.run
+            blah = :success
+          end
+          example { skip }
+        end
+        group.run
+        expect(blah).to be(:success)
+      end
+    end
+
+    context "in before(:each)" do
+      it "sets each example to skipped" do
+        group = RSpec::Core::ExampleGroup.describe do
+          before(:each) { skip }
+          example {}
+          example {}
+        end
+        group.run
+        expect(group.examples.first).to be_skipped
+        expect(group.examples.last).to be_skipped
+      end
+    end
+
+    context "in before(:all)" do
+      it "sets each example to pending" do
+        group = RSpec::Core::ExampleGroup.describe do
+          before(:all) { skip }
+          example {}
+          example {}
+        end
+        group.run
+        expect(group.examples.first).to be_skipped
+        expect(group.examples.last).to be_skipped
+      end
+    end
+
+    context "in around(:each)" do
+      it "sets the example to skipped" do
+        group = RSpec::Core::ExampleGroup.describe do
+          around(:each) { skip }
+          example {}
+        end
+        group.run
+        expect(group.examples.first).to be_skipped
+      end
+    end
+  end
+
   describe "timing" do
     it "uses RSpec::Core::Time as to not be affected by changes to time in examples" do
       reporter = double(:reporter).as_null_object
diff --git a/spec/rspec/core/formatters/base_formatter_spec.rb b/spec/rspec/core/formatters/base_formatter_spec.rb
index 730d5f9eda..f7e62a42c6 100644
--- a/spec/rspec/core/formatters/base_formatter_spec.rb
+++ b/spec/rspec/core/formatters/base_formatter_spec.rb
@@ -52,7 +52,6 @@
       end
 
       it "doesn't hang when file exists" do
-        pending("This issue still exists on JRuby, but should be resolved shortly: https://github.com/rspec/rspec-core/issues/295", :if => RUBY_PLATFORM == 'java')
         exception = double(:Exception, :backtrace => [ "#{__FILE__}:#{__LINE__}"])
 
         example = double(:Example, :file_path => __FILE__)
diff --git a/spec/rspec/core/formatters/documentation_formatter_spec.rb b/spec/rspec/core/formatters/documentation_formatter_spec.rb
index bb529e3f08..94cc3fda5f 100644
--- a/spec/rspec/core/formatters/documentation_formatter_spec.rb
+++ b/spec/rspec/core/formatters/documentation_formatter_spec.rb
@@ -58,7 +58,7 @@ module RSpec::Core::Formatters
       group = RSpec::Core::ExampleGroup.describe(" root ")
       context1 = group.describe(" nested ")
       context1.example(" example 1 ") {}
-      context1.example(" example 2 ", :pending => true){}
+      context1.example(" example 2 ", :pending => true){ fail }
       context1.example(" example 3 ") { fail }
 
       group.run(reporter)
diff --git a/spec/rspec/core/formatters/json_formatter_spec.rb b/spec/rspec/core/formatters/json_formatter_spec.rb
index cb21820508..53d52773f9 100644
--- a/spec/rspec/core/formatters/json_formatter_spec.rb
+++ b/spec/rspec/core/formatters/json_formatter_spec.rb
@@ -18,7 +18,7 @@
     group = RSpec::Core::ExampleGroup.describe("one apiece") do
       it("succeeds") { expect(1).to eq 1 }
       it("fails") { fail "eek" }
-      it("pends") { pending "world peace" }
+      it("pends") { pending "world peace"; fail "eek" }
     end
     succeeding_line = __LINE__ - 4
     failing_line = __LINE__ - 4
@@ -31,7 +31,9 @@
     end
 
     # grab the actual backtrace -- kind of a cheat
-    failing_backtrace = formatter.output_hash[:examples][1][:exception][:backtrace]
+    examples = formatter.output_hash[:examples]
+    failing_backtrace = examples[1][:exception][:backtrace]
+    pending_backtrace = examples[2][:exception][:backtrace]
     this_file = relative_path(__FILE__)
 
     expected = {
@@ -51,7 +53,11 @@
           :file_path => this_file,
           :line_number => failing_line,
           :run_time => formatter.output_hash[:examples][1][:run_time],
-          :exception => {:class => "RuntimeError", :message => "eek", :backtrace => failing_backtrace}
+          :exception => {
+            :class     => "RuntimeError",
+            :message   => "eek",
+            :backtrace => failing_backtrace
+          }
         },
         {
           :description => "pends",
@@ -59,7 +65,12 @@
           :status => "pending",
           :file_path => this_file,
           :line_number => pending_line,
-          :run_time => formatter.output_hash[:examples][2][:run_time]
+          :run_time => formatter.output_hash[:examples][2][:run_time],
+          :exception => {
+            :class     => "RuntimeError",
+            :message   => "eek",
+            :backtrace => pending_backtrace
+          }
         },
       ],
       :summary => {
diff --git a/spec/rspec/core/pending_example_spec.rb b/spec/rspec/core/pending_example_spec.rb
index 5db564e492..fbfd4588b8 100644
--- a/spec/rspec/core/pending_example_spec.rb
+++ b/spec/rspec/core/pending_example_spec.rb
@@ -11,10 +11,21 @@
     end
   end
 
+  matcher :be_skipped_with do |message|
+    match do |example|
+      example.skipped? && example.metadata[:execution_result][:pending_message] == message
+    end
+
+    failure_message_for_should do |example|
+      "expected: example skipped with #{message.inspect}\n     got: #{example.metadata[:execution_result][:pending_message].inspect}"
+    end
+  end
+
   context "declared pending with metadata" do
     it "uses the value assigned to :pending as the message" do
       group = RSpec::Core::ExampleGroup.describe('group') do
         example "example", :pending => 'just because' do
+          fail
         end
       end
       example = group.examples.first
@@ -25,6 +36,7 @@
     it "sets the message to 'No reason given' if :pending => true" do
       group = RSpec::Core::ExampleGroup.describe('group') do
         example "example", :pending => true do
+          fail
         end
       end
       example = group.examples.first
@@ -40,7 +52,7 @@
       end
       example = group.examples.first
       example.run(group.new, double.as_null_object)
-      expect(example).to be_pending_with('Not yet implemented')
+      expect(example).to be_skipped_with('Not yet implemented')
     end
   end
 
@@ -49,12 +61,30 @@
       group = RSpec::Core::ExampleGroup.describe('group') do
         it "does something" do
           pending
+          fail
         end
       end
       example = group.examples.first
       example.run(group.new, double.as_null_object)
       expect(example).to be_pending_with(RSpec::Core::Pending::NO_REASON_GIVEN)
     end
+
+    it "fails when the rest of the example passes" do
+      called = false
+      group = RSpec::Core::ExampleGroup.describe('group') do
+        it "does something" do
+          pending
+          called = true
+        end
+      end
+
+      example = group.examples.first
+      example.run(group.new, double.as_null_object)
+      expect(called).to eq(true)
+      result = example.metadata[:execution_result]
+      expect(result[:pending_fixed]).to eq(true)
+      expect(result[:status]).to eq("failed")
+    end
   end
 
   context "with no docstring" do
@@ -184,6 +214,23 @@ def run_example(*pending_args)
       end
     end
 
+    context 'that fails due to both a failed message expectation and a standard failure' do
+      def run_example(*pending_args)
+        super(*pending_args) {
+          expect("foo").to receive("bar")
+          fail
+        }
+      end
+
+      it 'indicates it is pending with the given message' do
+        expect(run_example("just because")).to be_pending_with("just because")
+      end
+
+      it 'indicates the pending block was not fixed' do
+        expect(run_example.metadata[:execution_result][:pending_fixed]).to be false
+      end
+    end
+
     it 'does not verify or teardown mocks multiple times' do
       counts = Hash.new(0)
 
diff --git a/spec/support/in_sub_process.rb b/spec/support/in_sub_process.rb
index cb74e6b467..e3541ffe90 100644
--- a/spec/support/in_sub_process.rb
+++ b/spec/support/in_sub_process.rb
@@ -1,8 +1,8 @@
 module InSubProcess
   if RUBY_PLATFORM == 'java'
     def in_sub_process
-      pending "This spec requires forking to work properly, " +
-              "and JRuby does not support forking"
+      skip "This spec requires forking to work properly, " +
+           "and JRuby does not support forking"
     end
   else
     # Useful as a way to isolate a global change to a subprocess.