Implement retries for MongoDB shell commands

  Retries for MongoDB shell commands could be useful
  in some cases:
  * unstable connectivity
  * when replica or sharding initialization is in process and
    you should wait some time till its end

Author: iberezovskiy

lib/puppet/provider/mongodb.rb

@@ -76,11 +76,27 @@ def self.get_conn_string
 
   # Mongo Command Wrapper
   def self.mongo_eval(cmd, db = 'admin')
+    retry_count = 10
+    retry_sleep = 3
     if mongorc_file
         cmd = mongorc_file + cmd
     end
 
-    out = mongo([db, '--quiet', '--host', get_conn_string, '--eval', cmd])
+    out = nil
+    retry_count.times do |n|
+      begin
+        out = mongo([db, '--quiet', '--host', get_conn_string, '--eval', cmd])
+      rescue => e
+        debug "Request failed: '#{e.message}' Retry: '#{n}'"
+        sleep retry_sleep
+        next
+      end
+      break
+    end
+
+    if !out
+     fail "Could not evalute MongoDB shell command: #{cmd}"
+    end
 
     out.gsub!(/ObjectId\(([^)]*)\)/, '\1')
     out