Support error comparison

README.md

@@ -108,6 +108,38 @@ class MyWidget
 end
 ```
 
+If either the control block or candidate block raises an error, Scientist compares the two observations' classes and messages using `==`. To override this behavior, use `compare_error` to define how to compare observed errors instead:
+
+```ruby
+class MyWidget
+  include Scientist
+
+  def slug_from_login(login)
+    science "slug_from_login" do |e|
+      e.use { User.slug_from_login login }         # returns String instance or ArgumentError
+      e.try { UserService.slug_from_login login }  # returns String instance or ArgumentError
+
+      compare_error_message_and_class = -> (control, candidate) do
+        control.class == candidate.class && 
+        control.message == candidate.message
+      end
+
+      compare_argument_errors = -> (control, candidate) do
+        control.class == ArgumentError &&
+        candidate.class == ArgumentError &&
+        control.message.start_with?("Input has invalid characters") &&
+        candidate.message.star_with?("Invalid characters in input") 
+      end
+
+      e.compare_error do |control, candidate|
+        compare_error_message_and_class.call(control, candidate) ||
+        compare_argument_errors.call(control, candidate)
+      end
+    end
+  end
+end
+```
+
 ### Adding context
 
 Results aren't very useful without some way to identify them. Use the `context` method to add to or retrieve the context for an experiment:
@@ -225,7 +257,7 @@ def admin?(user)
 end
 ```
 
-The ignore blocks are only called if the *values* don't match. If one observation raises an exception and the other doesn't, it's always considered a mismatch. If both observations raise different exceptions, that is also considered a mismatch.
+The ignore blocks are only called if the *values* don't match. Unless a `compare_error` comparator is defined, two cases are considered mismatches: a) one observation raising an exception and the other not, b) observations raising exceptions with different classes or messages.
 
 ### Enabling/disabling experiments
 

lib/scientist/experiment.rb

@@ -121,6 +121,16 @@ def compare(*args, &block)
     @_scientist_comparator = block
   end
 
+  # A block which compares two experimental errors.
+  #
+  # The block must take two arguments, the control Error and a candidate Error,
+  # and return true or false.
+  #
+  # Returns the block.
+  def compare_errors(*args, &block)
+    @_scientist_error_comparator = block
+  end
+
   # A Symbol-keyed Hash of extra experiment data.
   def context(context = nil)
     @_scientist_context ||= {}
@@ -164,13 +174,9 @@ def name
     "experiment"
   end
 
-  # Internal: compare two observations, using the configured compare block if present.
+  # Internal: compare two observations, using the configured compare and compare_errors lambdas if present.
   def observations_are_equivalent?(a, b)
-    if @_scientist_comparator
-      a.equivalent_to?(b, &@_scientist_comparator)
-    else
-      a.equivalent_to? b
-    end
+    a.equivalent_to? b, @_scientist_comparator, @_scientist_error_comparator
   rescue StandardError => ex
     raised :compare, ex
     false

lib/scientist/observation.rb

@@ -47,25 +47,34 @@ def cleaned_value
 
   # Is this observation equivalent to another?
   #
-  # other      - the other Observation in question
-  # comparator - an optional comparison block. This observation's value and the
-  #              other observation's value are yielded to this to determine
-  #              their equivalency. Block should return true/false.
+  # other            - the other Observation in question
+  # comparator       - an optional comparison proc. This observation's value and the
+  #                    other observation's value are passed to this to determine
+  #                    their equivalency. Proc should return true/false.
+  # error_comparator - an optional comparison proc. This observation's Error and the
+  #                    other observation's Error are passed to this to determine
+  #                    their equivalency. Proc should return true/false.
   #
   # Returns true if:
   #
   # * The values of the observation are equal (using `==`)
   # * The values of the observations are equal according to a comparison
-  #   block, if given
+  #   proc, if given
+  # * The exceptions raised by the obeservations are equal according to the
+  #   error comparison proc, if given.
   # * Both observations raised an exception with the same class and message.
   #
   # Returns false otherwise.
-  def equivalent_to?(other, &comparator)
+  def equivalent_to?(other, comparator=nil, error_comparator=nil)
     return false unless other.is_a?(Scientist::Observation)
 
     if raised? || other.raised?
-      return other.exception.class == exception.class &&
-        other.exception.message == exception.message
+      if error_comparator
+        return error_comparator.call(exception, other.exception)
+      else
+        return other.exception.class == exception.class &&
+          other.exception.message == exception.message
+      end
     end
 
     if comparator

test/scientist/experiment_test.rb

@@ -180,6 +180,18 @@ def @ex.publish(result)
     assert @ex.published_result.matched?
   end
 
+  it "compares errors with an error comparator block if provided" do
+    @ex.compare_errors { |a, b| a.class == b.class }
+    @ex.use { raise "foo" }
+    @ex.try { raise "bar" }
+
+    resulting_error = assert_raises RuntimeError do
+      @ex.run
+    end
+    assert_equal "foo", resulting_error.message
+    assert @ex.published_result.matched?
+  end
+
   it "knows how to compare two experiments" do
     a = Scientist::Observation.new(@ex, "a") { 1 }
     b = Scientist::Observation.new(@ex, "b") { 2 }

test/scientist/observation_test.rb

@@ -80,7 +80,7 @@
     refute x.equivalent_to?(y)
   end
 
-  FirstErrror = Class.new(StandardError)
+  FirstError = Class.new(StandardError)
   SecondError = Class.new(StandardError)
 
   it "compares exception classes" do
@@ -92,22 +92,46 @@
     refute x.equivalent_to?(y)
   end
 
-  it "compares values using a comparator block" do
+  it "compares values using a comparator proc" do
     a = Scientist::Observation.new("test", @experiment) { 1 }
     b = Scientist::Observation.new("test", @experiment) { "1" }
 
     refute a.equivalent_to?(b)
-    assert a.equivalent_to?(b) { |x, y| x.to_s == y.to_s }
+
+    compare_on_string = -> (x, y) { x.to_s == y.to_s }
+
+    assert a.equivalent_to?(b, compare_on_string)
 
     yielded = []
-    a.equivalent_to?(b) do |x, y|
+    compare_appends = -> (x, y) do
       yielded << x
       yielded << y
       true
     end
+    a.equivalent_to?(b, compare_appends)
+
     assert_equal [a.value, b.value], yielded
   end
 
+  it "compares exceptions using an error comparator proc" do
+    x = Scientist::Observation.new("test", @experiment) { raise FirstError, "error" }
+    y = Scientist::Observation.new("test", @experiment) { raise SecondError, "error" }
+    z = Scientist::Observation.new("test", @experiment) { raise FirstError, "ERROR" }
+
+    refute x.equivalent_to?(z)
+    refute x.equivalent_to?(y)
+
+    compare_on_class = -> (error, other_error) {
+      error.class == other_error.class
+    }
+    compare_on_message = -> (error, other_error) {
+      error.message == other_error.message
+    }
+
+    assert x.equivalent_to?(z, nil, compare_on_class)
+    assert x.equivalent_to?(y, nil, compare_on_message)
+  end
+
   describe "#cleaned_value" do
     it "returns the observation's value by default" do
       a = Scientist::Observation.new("test", @experiment) { 1 }