osc/rdma: Fix race in attachment/location related to MPI_Win_attach().

There are cases where proc A will try to look for a region from proc B.
However, B has not finished attaching its region, resulting in A giving
up too soon and returning with OMPI_ERR_RMA_RANGE. This patch adds a reasonable
retry for A to find this region to prevent this.

Signed-off-by: Austen Lauria <awlauria@us.ibm.com>

Author: awlauria

ompi/mca/osc/rdma/osc_rdma.h

@@ -111,6 +111,7 @@ struct ompi_osc_rdma_component_t {
 
     /** maximum count for network AMO usage */
     unsigned long network_amo_max_count;
+    unsigned long dyn_win_attach_retry_count;
 };
 typedef struct ompi_osc_rdma_component_t ompi_osc_rdma_component_t;
 

ompi/mca/osc/rdma/osc_rdma_component.c

@@ -289,6 +289,15 @@ static int ompi_osc_rdma_component_register (void)
                                             MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, 0, 0, OPAL_INFO_LVL_3,
                                             MCA_BASE_VAR_SCOPE_LOCAL, &mca_osc_rdma_component.network_amo_max_count);
 
+    mca_osc_rdma_component.dyn_win_attach_retry_count = 50;
+    (void) mca_base_component_var_register (&mca_osc_rdma_component.super.osc_version, "dyn_win_attach_retry_count",
+                                            "Number of times to retry searching for a dynamic window. This"
+                                            "is an attempt to prevent failures when a window did not attach"
+                                            "as quickly as expected. If an app is frequently failing with OMPI_ERR_RMA_RANGE"
+                                            "when using dynamic windows, increasing this value may help. (default: 50)",
+                                            MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, 0, 0, OPAL_INFO_LVL_3,
+                                            MCA_BASE_VAR_SCOPE_LOCAL, &mca_osc_rdma_component.dyn_win_attach_retry_count);
+
     /* register performance variables */
 
     (void) mca_base_component_pvar_register (&mca_osc_rdma_component.super.osc_version, "put_retry_count",
@@ -305,6 +314,7 @@ static int ompi_osc_rdma_component_register (void)
                                              ompi_osc_rdma_pvar_read, NULL, NULL,
                                              (void *) (intptr_t) offsetof (ompi_osc_rdma_module_t, get_retry_count));
 
+
     return OMPI_SUCCESS;
 }
 

ompi/mca/osc/rdma/osc_rdma_dynamic.c

@@ -460,27 +460,38 @@ int ompi_osc_rdma_find_dynamic_region (ompi_osc_rdma_module_t *module, ompi_osc_
     ompi_osc_rdma_peer_dynamic_t *dy_peer = (ompi_osc_rdma_peer_dynamic_t *) peer;
     intptr_t bound = (intptr_t) base + len;
     ompi_osc_rdma_region_t *regions;
-    int ret, region_count;
+    int ret, region_count = 0;
+    ompi_osc_rdma_state_t *peer_state = NULL;
 
     OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "locating dynamic memory region matching: {%" PRIx64 ", %" PRIx64 "}"
                      " (len %lu)", base, base + len, (unsigned long) len);
 
-    if (!ompi_osc_rdma_peer_local_state (peer)) {
-        ret = ompi_osc_rdma_refresh_dynamic_region (module, dy_peer);
-        if (OMPI_SUCCESS != ret) {
-            return ret;
+    unsigned long num_retries = mca_osc_rdma_component.dyn_win_attach_retry_count;
+    do {
+        if (!ompi_osc_rdma_peer_local_state (peer)) {
+            ret = ompi_osc_rdma_refresh_dynamic_region (module, dy_peer);
+            if (OMPI_SUCCESS != ret) {
+                return ret;
+            }
+            regions = dy_peer->regions;
+            region_count = dy_peer->region_count;
+        } else {
+            peer_state = (ompi_osc_rdma_state_t *) peer->state;
+            regions = (ompi_osc_rdma_region_t *) peer_state->regions;
+            region_count = peer_state->region_count;
         }
 
-        regions = dy_peer->regions;
-        region_count = dy_peer->region_count;
-    } else {
-        ompi_osc_rdma_state_t *peer_state = (ompi_osc_rdma_state_t *) peer->state;
-        regions = (ompi_osc_rdma_region_t *) peer_state->regions;
-        region_count = peer_state->region_count;
-    }
+        if(region_count > 0) {
+            *region = ompi_osc_rdma_find_region_containing (regions, 0, region_count - 1, (intptr_t) base,
+                                                            bound, module->region_size, NULL);
+        }
+        --num_retries;
+        OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_TRACE, "locating dynamic memory region matching: {%" PRIx64 ", %" PRIx64 "}"
+                        " (len %lu) (attempts left = %lu)", base, base + len, (unsigned long) len, num_retries);
+        opal_atomic_wmb ();
+    } while (!*region && (num_retries > 0));
 
-    *region = ompi_osc_rdma_find_region_containing (regions, 0, region_count - 1, (intptr_t) base, bound, module->region_size, NULL);
-    if (!*region) {
+    if(!*region) {
         return OMPI_ERR_RMA_RANGE;
     }