Skip to content

Commit b7fb88c

Browse files
hppritchahppritcha
committed
MPI4: implement mpi_isendrecv and variant
Signed-off-by: Howard Pritchard <howardp@lanl.gov>
1 parent 4f6127e commit b7fb88c

23 files changed

+1046
-0
lines changed

‎ompi/communicator/comm_request.c

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
* of Tennessee Research Foundation. All rights
99
* reserved.
1010
* Copyright (c) 2016 IBM Corporation. All rights reserved.
11+
* Copyright (c) 2021 Triad National Security, LLC. All rights
12+
* reserved.
1113
* $COPYRIGHT$
1214
*
1315
* Additional copyrights may follow
@@ -30,6 +32,7 @@ typedef struct ompi_comm_request_item_t {
3032
opal_list_item_t super;
3133
ompi_comm_request_callback_fn_t callback;
3234
ompi_request_t *subreqs[OMPI_COMM_REQUEST_MAX_SUBREQ];
35+
ompi_status_public_t *statuses[OMPI_COMM_REQUEST_MAX_SUBREQ];
3336
int subreq_count;
3437
} ompi_comm_request_item_t;
3538
OBJ_CLASS_DECLARATION(ompi_comm_request_item_t);
@@ -87,6 +90,43 @@ int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_r
8790

8891
for (i = 0 ; i < subreq_count ; ++i) {
8992
request_item->subreqs[i] = subreqs[i];
93+
request_item->statuses[i] = NULL;
94+
}
95+
96+
request_item->subreq_count = subreq_count;
97+
98+
opal_list_append (&request->schedule, &request_item->super);
99+
100+
return OMPI_SUCCESS;
101+
}
102+
103+
int ompi_comm_request_schedule_append_w_statuses(ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
104+
ompi_request_t *subreqs[], ompi_status_public_t *statuses, int subreq_count)
105+
{
106+
ompi_comm_request_item_t *request_item;
107+
int i;
108+
109+
if (subreq_count > OMPI_COMM_REQUEST_MAX_SUBREQ) {
110+
return OMPI_ERR_BAD_PARAM;
111+
}
112+
113+
/*
114+
* treat NULL callback as an error as the statuses array will need to be freed in the callback
115+
*/
116+
if (NULL == callback) {
117+
return OMPI_ERR_BAD_PARAM;
118+
}
119+
120+
request_item = OBJ_NEW(ompi_comm_request_item_t);
121+
if (NULL == request_item) {
122+
return OMPI_ERR_OUT_OF_RESOURCE;
123+
}
124+
125+
request_item->callback = callback;
126+
127+
for (i = 0 ; i < subreq_count ; ++i) {
128+
request_item->subreqs[i] = subreqs[i];
129+
request_item->statuses[i] = &statuses[i];
90130
}
91131

92132
request_item->subreq_count = subreq_count;
@@ -100,6 +140,7 @@ static int ompi_comm_request_progress (void)
100140
{
101141
ompi_comm_request_t *request, *next;
102142
static opal_atomic_int32_t progressing = 0;
143+
ompi_status_public_t *status;
103144
int completed = 0;
104145

105146
/* don't allow re-entry */
@@ -125,6 +166,10 @@ static int ompi_comm_request_progress (void)
125166
* that it does some subreqs cleanup */
126167
request->super.req_status.MPI_ERROR = subreq->req_status.MPI_ERROR;
127168
}
169+
status = request_item->statuses[request_item->subreq_count-1];
170+
if (NULL != status) {
171+
OMPI_COPY_STATUS(status, subreq->req_status, false);
172+
}
128173
ompi_request_free (&subreq);
129174
request_item->subreq_count--;
130175
completed++;

‎ompi/communicator/comm_request.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
/*
33
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
44
* reseved.
5+
* Copyright (c) 2021 Triad National Security, LLC. All rights
6+
* reserved.
57
* $COPYRIGHT$
68
*
79
* Additional copyrights may follow
@@ -32,6 +34,8 @@ void ompi_comm_request_init (void);
3234
void ompi_comm_request_fini (void);
3335
int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
3436
ompi_request_t *subreqs[], int subreq_count);
37+
int ompi_comm_request_schedule_append_w_statuses(ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
38+
ompi_request_t *subreqs[], ompi_status_public_t *statuses, int subreq_count);
3539
void ompi_comm_request_start (ompi_comm_request_t *request);
3640
ompi_comm_request_t *ompi_comm_request_get (void);
3741
void ompi_comm_request_return (ompi_comm_request_t *request);

‎ompi/include/mpi.h.in

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1698,6 +1698,13 @@ OMPI_DECLSPEC int MPI_Irsend(const void *buf, int count, MPI_Datatype datatype,
16981698
int tag, MPI_Comm comm, MPI_Request *request);
16991699
OMPI_DECLSPEC int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest,
17001700
int tag, MPI_Comm comm, MPI_Request *request);
1701+
OMPI_DECLSPEC int MPI_Isendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
1702+
int dest, int sendtag, void *recvbuf, int recvcount,
1703+
MPI_Datatype recvtype, int source, int recvtag,
1704+
MPI_Comm comm, MPI_Request *request);
1705+
OMPI_DECLSPEC int MPI_Isendrecv_replace(void * buf, int count, MPI_Datatype datatype,
1706+
int dest, int sendtag, int source, int recvtag,
1707+
MPI_Comm comm, MPI_Request *request);
17011708
OMPI_DECLSPEC int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest,
17021709
int tag, MPI_Comm comm, MPI_Request *request);
17031710
OMPI_DECLSPEC int MPI_Is_thread_main(int *flag);
@@ -2430,6 +2437,13 @@ OMPI_DECLSPEC int PMPI_Irsend(const void *buf, int count, MPI_Datatype datatype
24302437
int tag, MPI_Comm comm, MPI_Request *request);
24312438
OMPI_DECLSPEC int PMPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest,
24322439
int tag, MPI_Comm comm, MPI_Request *request);
2440+
OMPI_DECLSPEC int PMPI_Isendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
2441+
int dest, int sendtag, void *recvbuf, int recvcount,
2442+
MPI_Datatype recvtype, int source, int recvtag,
2443+
MPI_Comm comm, MPI_Request *request);
2444+
OMPI_DECLSPEC int PMPI_Isendrecv_replace(void * buf, int count, MPI_Datatype datatype,
2445+
int dest, int sendtag, int source, int recvtag,
2446+
MPI_Comm comm, MPI_Request *request);
24332447
OMPI_DECLSPEC int PMPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest,
24342448
int tag, MPI_Comm comm, MPI_Request *request);
24352449
OMPI_DECLSPEC int PMPI_Precv_init(void* buf, int partitions, MPI_Count count,

‎ompi/mpi/c/Makefile.am

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,6 +283,8 @@ libmpi_c_mpi_la_SOURCES = \
283283
irsend.c \
284284
is_thread_main.c \
285285
isend.c \
286+
isendrecv.c \
287+
isendrecv_replace.c \
286288
issend.c \
287289
lookup_name.c \
288290
message_f2c.c \

‎ompi/mpi/c/isendrecv.c

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2+
/*
3+
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
4+
* University Research and Technology
5+
* Corporation. All rights reserved.
6+
* Copyright (c) 2004-2021 The University of Tennessee and The University
7+
* of Tennessee Research Foundation. All rights
8+
* reserved.
9+
* Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
10+
* University of Stuttgart. All rights reserved.
11+
* Copyright (c) 2004-2005 The Regents of the University of California.
12+
* All rights reserved.
13+
* Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved.
14+
* Copyright (c) 2013 Los Alamos National Security, LLC. All rights
15+
* reserved.
16+
* Copyright (c) 2015 Research Organization for Information Science
17+
* and Technology (RIST). All rights reserved.
18+
* Copyright (c) 2021 Nanook Consulting. All rights reserved.
19+
* Copyright (c) 2021 Triad National Security, LLC. All rights
20+
* reserved.
21+
* $COPYRIGHT$
22+
*
23+
* Additional copyrights may follow
24+
*
25+
* $HEADER$
26+
*/
27+
#include "ompi_config.h"
28+
29+
#include "ompi/mpi/c/bindings.h"
30+
#include "ompi/runtime/params.h"
31+
#include "ompi/communicator/communicator.h"
32+
#include "ompi/communicator/comm_request.h"
33+
#include "ompi/errhandler/errhandler.h"
34+
#include "ompi/mca/pml/pml.h"
35+
#include "ompi/request/request.h"
36+
#include "ompi/memchecker.h"
37+
#include "ompi/runtime/ompi_spc.h"
38+
39+
#if OMPI_BUILD_MPI_PROFILING
40+
#if OPAL_HAVE_WEAK_SYMBOLS
41+
#pragma weak MPI_Isendrecv = PMPI_Isendrecv
42+
#endif
43+
#define MPI_Isendrecv PMPI_Isendrecv
44+
#endif
45+
46+
static const char FUNC_NAME[] = "MPI_Isendrecv";
47+
48+
struct ompi_isendrecv_context_t {
49+
opal_object_t super;
50+
int nreqs;
51+
ompi_status_public_t statuses[2];
52+
};
53+
54+
typedef struct ompi_isendrecv_context_t ompi_isendrecv_context_t;
55+
OBJ_CLASS_INSTANCE(ompi_isendrecv_context_t, opal_object_t, NULL, NULL);
56+
57+
static int ompi_isendrecv_completer_func (ompi_comm_request_t *request)
58+
{
59+
ompi_isendrecv_context_t *context =
60+
(ompi_isendrecv_context_t *) request->context;
61+
62+
/*
63+
* Copy the status from the receive side of the sendrecv request?
64+
* But what if the send failed?
65+
*
66+
* Probably need to bring up in the MPI forum.
67+
*/
68+
69+
OMPI_COPY_STATUS(&request->super.req_status,
70+
context->statuses[0], false);
71+
72+
return OMPI_SUCCESS;
73+
}
74+
75+
76+
int MPI_Isendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
77+
int dest, int sendtag, void *recvbuf, int recvcount,
78+
MPI_Datatype recvtype, int source, int recvtag,
79+
MPI_Comm comm, MPI_Request *request)
80+
{
81+
ompi_isendrecv_context_t *context = NULL;
82+
ompi_comm_request_t *crequest;
83+
ompi_request_t *subreq[2];
84+
int rc = MPI_SUCCESS;
85+
int nreqs = 0;
86+
87+
SPC_RECORD(OMPI_SPC_ISENDRECV, 1);
88+
89+
MEMCHECKER(
90+
memchecker_datatype(sendtype);
91+
memchecker_datatype(recvtype);
92+
memchecker_call(&opal_memchecker_base_isdefined, sendbuf, sendcount, sendtype);
93+
memchecker_comm(comm);
94+
);
95+
96+
if ( MPI_PARAM_CHECK ) {
97+
OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
98+
OMPI_CHECK_DATATYPE_FOR_SEND(rc, sendtype, sendcount);
99+
OMPI_CHECK_DATATYPE_FOR_RECV(rc, recvtype, recvcount);
100+
OMPI_CHECK_USER_BUFFER(rc, sendbuf, sendtype, sendcount);
101+
OMPI_CHECK_USER_BUFFER(rc, recvbuf, recvtype, recvcount);
102+
103+
if (ompi_comm_invalid(comm)) {
104+
return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_COMM, FUNC_NAME);
105+
} else if (dest != MPI_PROC_NULL && ompi_comm_peer_invalid(comm, dest)) {
106+
rc = MPI_ERR_RANK;
107+
} else if (sendtag < 0 || sendtag > mca_pml.pml_max_tag) {
108+
rc = MPI_ERR_TAG;
109+
} else if (source != MPI_PROC_NULL && source != MPI_ANY_SOURCE && ompi_comm_peer_invalid(comm, source)) {
110+
rc = MPI_ERR_RANK;
111+
} else if (((recvtag < 0) && (recvtag != MPI_ANY_TAG)) || (recvtag > mca_pml.pml_max_tag)) {
112+
rc = MPI_ERR_TAG;
113+
} else if (request == NULL) {
114+
rc = MPI_ERR_REQUEST;
115+
}
116+
117+
OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
118+
}
119+
120+
crequest = ompi_comm_request_get ();
121+
if (NULL == crequest) {
122+
return OMPI_ERR_OUT_OF_RESOURCE;
123+
}
124+
125+
context = OBJ_NEW(ompi_isendrecv_context_t);
126+
if (NULL == context) {
127+
ompi_comm_request_return (crequest);
128+
return OMPI_ERR_OUT_OF_RESOURCE;
129+
}
130+
131+
crequest->context = &context->super;
132+
133+
if (source != MPI_PROC_NULL) { /* post recv */
134+
rc = MCA_PML_CALL(irecv(recvbuf, recvcount, recvtype,
135+
source, recvtag, comm, &subreq[nreqs++]));
136+
if (MPI_SUCCESS != rc) {
137+
ompi_comm_request_return (crequest);
138+
}
139+
OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
140+
}
141+
142+
if (dest != MPI_PROC_NULL) { /* send */
143+
rc = MCA_PML_CALL(isend(sendbuf, sendcount, sendtype, dest,
144+
sendtag, MCA_PML_BASE_SEND_STANDARD, comm, &subreq[nreqs++]));
145+
if (MPI_SUCCESS != rc) {
146+
ompi_comm_request_return (crequest);
147+
}
148+
OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
149+
}
150+
151+
/*
152+
* schedule the operation
153+
*/
154+
155+
context->nreqs = nreqs;
156+
assert(nreqs <= 2);
157+
158+
rc = ompi_comm_request_schedule_append_w_statuses(crequest, ompi_isendrecv_completer_func,
159+
subreq, context->statuses, nreqs);
160+
OMPI_ERRHANDLER_CHECK(rc, comm, rc, FUNC_NAME);
161+
162+
/* kick off the request */
163+
164+
ompi_comm_request_start (crequest);
165+
*request = &crequest->super;
166+
167+
return rc;
168+
}

0 commit comments

Comments
 (0)