From f7036ca08fc4d322dff4c4cc6c9f5a39af3ca93f Mon Sep 17 00:00:00 2001
From: Mike Moore <mike@blowmage.com>
Date: Tue, 20 Oct 2015 17:00:34 -0600
Subject: [PATCH] Add retry to acceptance tests

The acceptance tests work against the production services. Sometimes the
services present their eventually consistent nature and return results that
are not yet correct. Try rerunning failed tests instead of rerunning the
entire test suite.
---
 acceptance/bigquery_helper.rb  | 10 ++++++++++
 acceptance/datastore_helper.rb | 10 ++++++++++
 acceptance/dns_helper.rb       | 10 ++++++++++
 acceptance/pubsub_helper.rb    | 10 ++++++++++
 acceptance/storage_helper.rb   | 10 ++++++++++
 5 files changed, 50 insertions(+)

diff --git a/acceptance/bigquery_helper.rb b/acceptance/bigquery_helper.rb
index ae4c97a9cbc0..e53298ceffa0 100644
--- a/acceptance/bigquery_helper.rb
+++ b/acceptance/bigquery_helper.rb
@@ -63,6 +63,16 @@ def setup
       addl.include? :bigquery
     end
   end
+
+  def self.run_one_method klass, method_name, reporter
+    result = nil
+    (1..3).each do |try|
+      result = Minitest.run_one_method(klass, method_name)
+      break if result.passed?
+      puts "Retrying #{klass}##{method_name} (#{try})"
+    end
+    reporter.record result
+  end
 end
 
 def clean_up_bigquery_datasets
diff --git a/acceptance/datastore_helper.rb b/acceptance/datastore_helper.rb
index 4b96beb316be..68388de603b9 100644
--- a/acceptance/datastore_helper.rb
+++ b/acceptance/datastore_helper.rb
@@ -35,5 +35,15 @@ def setup
     register_spec_type(self) do |desc, *addl|
       addl.include? :datastore
     end
+
+    def self.run_one_method klass, method_name, reporter
+      result = nil
+      (1..3).each do |try|
+        result = Minitest.run_one_method(klass, method_name)
+        break if result.passed?
+        puts "Retrying #{klass}##{method_name} (#{try})"
+      end
+      reporter.record result
+    end
   end
 end
diff --git a/acceptance/dns_helper.rb b/acceptance/dns_helper.rb
index 2ab227589d27..6724bbf5ba2e 100644
--- a/acceptance/dns_helper.rb
+++ b/acceptance/dns_helper.rb
@@ -66,6 +66,16 @@ def setup
         addl.include? :dns
       end
     end
+
+    def self.run_one_method klass, method_name, reporter
+      result = nil
+      (1..3).each do |try|
+        result = Minitest.run_one_method(klass, method_name)
+        break if result.passed?
+        puts "Retrying #{klass}##{method_name} (#{try})"
+      end
+      reporter.record result
+    end
   end
 
   def clean_up_dns_zones
diff --git a/acceptance/pubsub_helper.rb b/acceptance/pubsub_helper.rb
index 7e3365163d33..b7a6ed4ad5bc 100644
--- a/acceptance/pubsub_helper.rb
+++ b/acceptance/pubsub_helper.rb
@@ -53,6 +53,16 @@ def setup
     register_spec_type(self) do |desc, *addl|
       addl.include? :pubsub
     end
+
+    def self.run_one_method klass, method_name, reporter
+      result = nil
+      (1..3).each do |try|
+        result = Minitest.run_one_method(klass, method_name)
+        break if result.passed?
+        puts "Retrying #{klass}##{method_name} (#{try})"
+      end
+      reporter.record result
+    end
   end
 end
 
diff --git a/acceptance/storage_helper.rb b/acceptance/storage_helper.rb
index 22fd874cdf76..44bb9e2d5a62 100644
--- a/acceptance/storage_helper.rb
+++ b/acceptance/storage_helper.rb
@@ -66,6 +66,16 @@ def setup
       addl.include? :storage
     end
   end
+
+  def self.run_one_method klass, method_name, reporter
+    result = nil
+    (1..3).each do |try|
+      result = Minitest.run_one_method(klass, method_name)
+      break if result.passed?
+      puts "Retrying #{klass}##{method_name} (#{try})"
+    end
+    reporter.record result
+  end
 end
 
 # Create buckets to be shared with all the tests