osc/rdma: Fix some bugs running with btl/tcp.

- Make sure peer->state_endpoint is set correctly.
- Fix double free of pending_op in ompi_osc_rdma_btl_fop() and ompi_osc_rdma_btl_op().

Cleanup/leaks:
- Don't parse ompi_osc_rdma_btl_alternate_names twice.
- free temp in allocate_state_shared().

Signed-off-by: Austen Lauria <awlauria@us.ibm.com>

Author: awlauria

ompi/mca/osc/rdma/osc_rdma_component.c

@@ -529,6 +529,7 @@ static int allocate_state_single (ompi_osc_rdma_module_t *module, void **base, s
         my_peer->state_handle = module->state_handle;
         my_peer->state_btl_index = my_peer->data_btl_index;
         my_peer->state_endpoint = my_peer->data_endpoint;
+        assert(my_peer -> state_endpoint != NULL);
     }
 
     if (MPI_WIN_FLAVOR_DYNAMIC != module->flavor) {
@@ -581,7 +582,7 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
     int my_rank = ompi_comm_rank (module->comm);
     int global_size = ompi_comm_size (module->comm);
     ompi_osc_rdma_region_t *state_region;
-    struct _local_data *temp;
+    struct _local_data *temp = NULL;
     char *data_file;
     int page_size = opal_getpagesize();
 
@@ -625,6 +626,7 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
     }
 
     do {
+        free(temp); // free from prior iteration. Should be initialized to NULL.
         temp = calloc (local_size, sizeof (temp[0]));
         if (NULL == temp) {
             ret = OMPI_ERR_OUT_OF_RESOURCE;
@@ -788,10 +790,9 @@ static int allocate_state_shared (ompi_osc_rdma_module_t *module, void **base, s
                     peer->state_handle = (mca_btl_base_registration_handle_t *) state_region->btl_handle_data;
                 }
                 peer->state = (osc_rdma_counter_t) ((uintptr_t) state_region->base + state_base + module->state_size * i);
-                if (i > 0) {
-                    peer->state_endpoint = local_leader->state_endpoint;
-                    peer->state_btl_index = local_leader->state_btl_index;
-                }
+                peer->state_endpoint = local_leader->data_endpoint; // data_endpoint initialized in ompi_osc_rdma_new_peer();
+                peer->state_btl_index = local_leader->data_btl_index;
+                assert(peer->state_endpoint != NULL);
             }
 
             if (my_rank == peer_rank) {
@@ -914,10 +915,8 @@ static void ompi_osc_rdma_ensure_local_add_procs (void)
 static int ompi_osc_rdma_query_alternate_btls (ompi_communicator_t *comm, ompi_osc_rdma_module_t *module)
 {
     mca_btl_base_selected_module_t *item;
-    char **btls_to_use = opal_argv_split (ompi_osc_rdma_btl_alternate_names, ',');
     int btls_found = 0;
-
-    btls_to_use = opal_argv_split (ompi_osc_rdma_btl_alternate_names, ',');
+    char **btls_to_use = opal_argv_split (ompi_osc_rdma_btl_alternate_names, ',');
     if (NULL == btls_to_use) {
         OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_INFO, "no alternate BTLs requested: %s", ompi_osc_rdma_btl_alternate_names);
         return OMPI_ERR_UNREACH;

ompi/mca/osc/rdma/osc_rdma_lock.h

@@ -87,12 +87,9 @@ static inline int ompi_osc_rdma_btl_fop (ompi_osc_rdma_module_t *module, uint8_t
     if (OPAL_SUCCESS != ret) {
         if (OPAL_LIKELY(1 == ret)) {
             *result = ((int64_t *) pending_op->op_buffer)[0];
-            ret = OMPI_SUCCESS;
             ompi_osc_rdma_atomic_complete (selected_btl, endpoint, pending_op->op_buffer,
-                                           pending_op->op_frag->handle, (void *) pending_op, NULL, OPAL_SUCCESS);
-        } else {
-            /* need to release here because ompi_osc_rdma_atomic_complete was not called */
-            OBJ_RELEASE(pending_op);
+                                           NULL, (void *) pending_op, NULL, OPAL_SUCCESS);
+            return OMPI_SUCCESS;
         }
     } else if (wait_for_completion) {
         while (!pending_op->op_complete) {
@@ -193,6 +190,7 @@ static inline int ompi_osc_rdma_btl_cswap (ompi_osc_rdma_module_t *module, uint8
 {
     ompi_osc_rdma_pending_op_t *pending_op;
     mca_btl_base_module_t *selected_btl = ompi_osc_rdma_selected_btl (module, btl_index);
+    assert(selected_btl != NULL);
     int ret;
 
     pending_op = OBJ_NEW(ompi_osc_rdma_pending_op_t);
@@ -227,8 +225,6 @@ static inline int ompi_osc_rdma_btl_cswap (ompi_osc_rdma_module_t *module, uint8
             ret = OMPI_SUCCESS;
         }
 
-        /* need to release here because ompi_osc_rdma_atomic_complete was not called */
-        OBJ_RELEASE(pending_op);
     } else {
         while (!pending_op->op_complete) {
             ompi_osc_rdma_progress (module);