Added retries to oc debug commands while deploying storage.

Sometimes clusters don't seem to like making so many debug pods with the
same name. This should help reduce failures.

app/playbooks/deploy.d/container-storage/gather-facts.yml

@@ -14,6 +14,9 @@
       register: cluster_drives
       ignore_errors: true
       changed_when: false
+      retries: 10
+      delay: 1
+      until: cluster_drives is success
 
     - name: save discovered hosts
       set_stats:

app/playbooks/deploy.d/container-storage/local-storage.yml

@@ -9,6 +9,9 @@
       loop: "{{ stg_drives | json_query('[*].host') }}"
       register: drive_id_lkp
       changed_when: false
+      retries: 10
+      delay: 1
+      until: cluster_drives is success
 
     - name: save storage node drive id paths
       set_fact:

app/playbooks/deploy.d/wipefs/gather-facts.yml

@@ -14,6 +14,9 @@
       register: cluster_drives
       ignore_errors: true
       changed_when: false
+      retries: 10
+      delay: 1
+      until: cluster_drives is success
 
     - name: save discovered hosts
       set_stats:

app/playbooks/deploy.d/wipefs/wipe-host.yml

@@ -4,25 +4,40 @@
   register: vgs
   changed_when: false
   ignore_errors: true
+  retries: 10
+  delay: 1
+  until: cluster_drives is success
 
 - name: delete associated vgs
   shell: "oc debug -n default node/{{ outer.host }} -- chroot /host vgremove --force --noudevsync '{{ item.stdout }}'"
   changed_when: true
   when: item.stdout != ""
   loop: "{{ vgs.results }}"
+  retries: 10
+  delay: 1
+  until: cluster_drives is success
 
 - name: remove pvs
   shell: "oc debug -n default node/{{ outer.host }} -- chroot /host pvremove '/dev/{{ item }}'"
   loop: "{{ outer.drives }}"
   changed_when: true
   ignore_errors: true
+  retries: 10
+  delay: 1
+  until: cluster_drives is success
 
 - name: wipe filesystem
   shell: "oc debug -n default node/{{ outer.host }} -- chroot /host wipefs -a '/dev/{{ item }}'"
   loop: "{{ outer.drives }}"
   changed_when: true
+  retries: 10
+  delay: 1
+  until: cluster_drives is success
 
 - name: clear GPT and MBR structures
   shell: "oc debug -n default node/{{ outer.host }} -- chroot /host sgdisk --zap-all '/dev/{{ item }}'"
   loop: "{{ outer.drives }}"
   changed_when: true
+  retries: 10
+  delay: 1
+  until: cluster_drives is success