Skip to content

Commit b603822

Browse files
mgdudamgduda
committed
Merge branch 'framework/mpas_use_mpi_f08' into develop (PR #1142)
This merge enables MPAS to make use of the more modern mpi_f08 module introduced in MPI-3.0. The top-level Makefile now tries to compile a test program that uses mpi_f08, and if that test is successful, the macro MPAS_USE_MPI_F08 is defined in the CPPFLAGS used in the build; otherwise, MPAS will make use of the mpi module as it previously did. With the use of the mpi_f08 module, certain MPI types are no longer integers in Fortran, but are derived types; e.g., MPI_Comm, MPI_Request, MPI_Datatype, and MPI_Info. However, in some instances, an integer-typed MPI type is still needed for interoperability, and the MPI standard permits this to be done through the mpi_val member of all MPI derived types, e.g., MPI_Comm % mpi_val. * framework/mpas_use_mpi_f08: Add logic in top-level Makefile to detect mpi_f08 module availability Allow MPAS to use either of 'mpi_f08' or 'mpi' modules
2 parents cbba5a4 + d82a0e1 commit b603822

13 files changed

+194
-25
lines changed

‎Makefile

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1257,11 +1257,42 @@ endif
12571257
exit 1; \
12581258
fi
12591259

1260+
1261+
mpi_f08_test:
1262+
@#
1263+
@# MPAS_MPI_F08 will be set to:
1264+
@# 0 if no mpi_f08 module support was detected
1265+
@# 1 if the MPI library provides an mpi_f08 module
1266+
@#
1267+
$(info Checking for mpi_f08 support...)
1268+
$(eval MPAS_MPI_F08 := $(shell $\
1269+
printf "program main\n$\
1270+
& use mpi_f08, only : MPI_Init, MPI_Comm\n$\
1271+
& integer :: ierr\n$\
1272+
& type (MPI_Comm) :: comm\n$\
1273+
& call MPI_Init(ierr)\n$\
1274+
end program main\n" | sed 's/&/ /' > mpi_f08.f90; $\
1275+
$\
1276+
$(FC) mpi_f08.f90 -o mpi_f08.x $(FFLAGS) $(LDFLAGS) > /dev/null 2>&1; $\
1277+
mpi_f08_status=$$?; $\
1278+
rm -f mpi_f08.f90 mpi_f08.x; $\
1279+
if [ $$mpi_f08_status -eq 0 ]; then $\
1280+
printf "1"; $\
1281+
else $\
1282+
printf "0"; $\
1283+
fi $\
1284+
))
1285+
$(if $(findstring 0,$(MPAS_MPI_F08)), $(eval MPI_F08_MESSAGE = "Using the mpi module."), )
1286+
$(if $(findstring 0,$(MPAS_MPI_F08)), $(info No working mpi_f08 module detected; using mpi module.))
1287+
$(if $(findstring 1,$(MPAS_MPI_F08)), $(eval override CPPFLAGS += -DMPAS_USE_MPI_F08), )
1288+
$(if $(findstring 1,$(MPAS_MPI_F08)), $(eval MPI_F08_MESSAGE = "Using the mpi_f08 module."), )
1289+
$(if $(findstring 1,$(MPAS_MPI_F08)), $(info mpi_f08 module detected.))
1290+
12601291
ifneq "$(PIO)" ""
1261-
MAIN_DEPS = openmp_test openacc_test pio_test
1292+
MAIN_DEPS = openmp_test openacc_test pio_test mpi_f08_test
12621293
override CPPFLAGS += "-DMPAS_PIO_SUPPORT"
12631294
else
1264-
MAIN_DEPS = openmp_test openacc_test
1295+
MAIN_DEPS = openmp_test openacc_test mpi_f08_test
12651296
IO_MESSAGE = "Using the SMIOL library."
12661297
override CPPFLAGS += "-DMPAS_SMIOL_SUPPORT"
12671298
endif
@@ -1300,6 +1331,7 @@ endif
13001331
@echo $(PRECISION_MESSAGE)
13011332
@echo $(DEBUG_MESSAGE)
13021333
@echo $(PARALLEL_MESSAGE)
1334+
@echo $(MPI_F08_MESSAGE)
13031335
@echo $(PAPI_MESSAGE)
13041336
@echo $(TAU_MESSAGE)
13051337
@echo $(OPENMP_MESSAGE)

‎src/driver/mpas_subdriver.F

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,11 @@ module mpas_subdriver
3939
contains
4040

4141

42-
subroutine mpas_init(corelist, domain_ptr, mpi_comm, namelistFileParam, streamsFileParam)
42+
subroutine mpas_init(corelist, domain_ptr, external_comm, namelistFileParam, streamsFileParam)
4343

44+
#ifdef MPAS_USE_MPI_F08
45+
use mpi_f08, only : MPI_Comm
46+
#endif
4447
use mpas_stream_manager, only : MPAS_stream_mgr_init, MPAS_build_stream_filename, MPAS_stream_mgr_validate_streams
4548
use iso_c_binding, only : c_char, c_loc, c_ptr, c_int
4649
use mpas_c_interfacing, only : mpas_f_to_c_string, mpas_c_to_f_string
@@ -53,7 +56,11 @@ subroutine mpas_init(corelist, domain_ptr, mpi_comm, namelistFileParam, streamsF
5356

5457
type (core_type), intent(inout), pointer :: corelist
5558
type (domain_type), intent(inout), pointer :: domain_ptr
56-
integer, intent(in), optional :: mpi_comm
59+
#ifdef MPAS_USE_MPI_F08
60+
type (MPI_Comm), intent(in), optional :: external_comm
61+
#else
62+
integer, intent(in), optional :: external_comm
63+
#endif
5764
character(len=*), intent(in), optional :: namelistFileParam
5865
character(len=*), intent(in), optional :: streamsFileParam
5966

@@ -192,7 +199,7 @@ end subroutine xml_stream_get_attributes
192199
!
193200
! Initialize infrastructure
194201
!
195-
call mpas_framework_init_phase1(domain_ptr % dminfo, mpi_comm=mpi_comm)
202+
call mpas_framework_init_phase1(domain_ptr % dminfo, external_comm=external_comm)
196203
197204
198205
#ifdef CORE_ATMOSPHERE
@@ -304,7 +311,11 @@ end subroutine xml_stream_get_attributes
304311
305312
call mpas_f_to_c_string(domain_ptr % streams_filename, c_filename)
306313
call mpas_f_to_c_string(mesh_stream, c_mesh_stream)
314+
#ifdef MPAS_USE_MPI_F08
315+
c_comm = domain_ptr % dminfo % comm % mpi_val
316+
#else
307317
c_comm = domain_ptr % dminfo % comm
318+
#endif
308319
call xml_stream_get_attributes(c_filename, c_mesh_stream, c_comm, &
309320
c_mesh_filename_temp, c_ref_time_temp, &
310321
c_filename_interval_temp, c_iotype, c_ierr)

‎src/framework/mpas_abort.F

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,15 @@ subroutine mpas_dmpar_global_abort(mesg, deferredAbort)!{{{
2929
use mpas_kind_types, only : StrKIND
3030
use mpas_io_units, only : mpas_new_unit
3131
use mpas_threading, only : mpas_threading_get_thread_num
32-
32+
3333
#ifdef _MPI
3434
#ifndef NOMPIMOD
35+
#ifdef MPAS_USE_MPI_F08
36+
use mpi_f08
37+
#else
3538
use mpi
3639
#endif
40+
#endif
3741
#endif
3842

3943
implicit none

‎src/framework/mpas_derived_types.F

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ module mpas_derived_types
3838
use smiolf, only : SMIOLf_context, SMIOLf_decomp, SMIOLf_file, SMIOL_offset_kind
3939
#endif
4040

41+
#ifdef MPAS_USE_MPI_F08
42+
use mpi_f08, only : MPI_Request, MPI_Comm, MPI_Info
43+
#endif
44+
4145
use ESMF
4246

4347
#include "mpas_attlist_types.inc"

‎src/framework/mpas_dmpar.F

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,12 @@ module mpas_dmpar
3131

3232
#ifdef _MPI
3333
#ifndef NOMPIMOD
34+
#ifdef MPAS_USE_MPI_F08
35+
use mpi_f08
36+
#else
3437
use mpi
3538
#endif
39+
#endif
3640
#endif
3741

3842
implicit none
@@ -42,16 +46,31 @@ module mpas_dmpar
4246
#ifdef NOMPIMOD
4347
include 'mpif.h'
4448
#endif
49+
#ifdef MPAS_USE_MPI_F08
50+
type (MPI_Datatype), parameter :: MPI_INTEGERKIND = MPI_INTEGER
51+
type (MPI_Datatype), parameter :: MPI_2INTEGERKIND = MPI_2INTEGER
52+
#else
4553
integer, parameter :: MPI_INTEGERKIND = MPI_INTEGER
4654
integer, parameter :: MPI_2INTEGERKIND = MPI_2INTEGER
55+
#endif
4756

4857
#ifdef SINGLE_PRECISION
58+
#ifdef MPAS_USE_MPI_F08
59+
type (MPI_Datatype), parameter :: MPI_REALKIND = MPI_REAL
60+
type (MPI_Datatype), parameter :: MPI_2REALKIND = MPI_2REAL
61+
#else
4962
integer, parameter :: MPI_REALKIND = MPI_REAL
5063
integer, parameter :: MPI_2REALKIND = MPI_2REAL
64+
#endif
65+
#else
66+
#ifdef MPAS_USE_MPI_F08
67+
type (MPI_Datatype), parameter :: MPI_REALKIND = MPI_DOUBLE_PRECISION
68+
type (MPI_Datatype), parameter :: MPI_2REALKIND = MPI_2DOUBLE_PRECISION
5169
#else
5270
integer, parameter :: MPI_REALKIND = MPI_DOUBLE_PRECISION
5371
integer, parameter :: MPI_2REALKIND = MPI_2DOUBLE_PRECISION
5472
#endif
73+
#endif
5574
#endif
5675

5776
integer, parameter, public :: IO_NODE = 0
@@ -232,12 +251,16 @@ module mpas_dmpar
232251
!> It also setups of the domain information structure.
233252
!
234253
!-----------------------------------------------------------------------
235-
subroutine mpas_dmpar_init(dminfo, mpi_comm)!{{{
254+
subroutine mpas_dmpar_init(dminfo, external_comm)!{{{
236255

237256
implicit none
238257

239258
type (dm_info), intent(inout) :: dminfo !< Input/Output: Domain information
240-
integer, intent(in), optional :: mpi_comm !< Input - Optional: externally-supplied MPI communicator
259+
#ifdef MPAS_USE_MPI_F08
260+
type (MPI_Comm), intent(in), optional :: external_comm !< Input - Optional: externally-supplied MPI communicator
261+
#else
262+
integer, intent(in), optional :: external_comm !< Input - Optional: externally-supplied MPI communicator
263+
#endif
241264

242265
#ifdef _MPI
243266
integer :: mpi_rank, mpi_size
@@ -246,13 +269,13 @@ subroutine mpas_dmpar_init(dminfo, mpi_comm)!{{{
246269
integer :: desiredThreadLevel, threadLevel
247270
#endif
248271

249-
if ( present(mpi_comm) ) then
272+
if ( present(external_comm) ) then
250273
dminfo % initialized_mpi = .false.
251274
#ifdef MPAS_OPENMP
252275
desiredThreadLevel = MPI_THREAD_FUNNELED
253276
call MPI_Query_thread(threadLevel, mpi_ierr)
254277
#endif
255-
call MPI_Comm_dup(mpi_comm, dminfo % comm, mpi_ierr)
278+
call MPI_Comm_dup(external_comm, dminfo % comm, mpi_ierr)
256279
else
257280
dminfo % initialized_mpi = .true.
258281
#ifdef MPAS_OPENMP
@@ -1540,7 +1563,12 @@ subroutine mpas_dmpar_get_exch_list(haloLayer, ownedListField, neededListField,
15401563
type (field0dInteger), pointer :: offsetCursor, ownedLimitCursor
15411564
integer :: nOwnedBlocks, nNeededBlocks
15421565
integer :: nOwnedList, nNeededList
1543-
integer :: mpi_ierr, mpi_rreq, mpi_sreq
1566+
integer :: mpi_ierr
1567+
#ifdef MPAS_USE_MPI_F08
1568+
type (MPI_Request) :: mpi_rreq, mpi_sreq
1569+
#else
1570+
integer :: mpi_rreq, mpi_sreq
1571+
#endif
15441572

15451573
type (hashtable) :: neededHash
15461574
integer :: nUniqueNeededList, threadNum
@@ -10073,7 +10101,11 @@ subroutine mpas_dmpar_exch_group_print_buffers(exchangeGroup)!{{{
1007310101
call mpas_log_write(' proc: $i', intArgs=(/commListPtr % procID/))
1007410102
call mpas_log_write(' size check: $i $i', intArgs=(/commListPtr % nlist, size( commListPtr % rbuffer )/))
1007510103
call mpas_log_write(' bufferOffset: $i', intArgs=(/commListPtr % bufferOffset/))
10104+
#ifdef MPAS_USE_MPI_F08
10105+
call mpas_log_write(' reqId: $i', intArgs=(/commListPtr % reqId % mpi_val/))
10106+
#else
1007610107
call mpas_log_write(' reqId: $i', intArgs=(/commListPtr % reqId/))
10108+
#endif
1007710109
call mpas_log_write(' ibuffer assc: $l', logicArgs=(/ associated( commListPtr % ibuffer ) /) )
1007810110
call mpas_log_write(' rbuffer assc: $l', logicArgs=(/ associated( commListPtr % rbuffer ) /) )
1007910111
call mpas_log_write(' next assc: $l', logicArgs=(/ associated( commListPtr % next ) /) )
@@ -10092,7 +10124,11 @@ subroutine mpas_dmpar_exch_group_print_buffers(exchangeGroup)!{{{
1009210124
call mpas_log_write(' proc: $i', intArgs=(/ commListPtr % procID /) )
1009310125
call mpas_log_write(' size check: $i $i', intArgs=(/ commListPtr % nlist, size( commListPtr % rbuffer ) /) )
1009410126
call mpas_log_write(' bufferOffset: $i', intArgs=(/ commListPtr % bufferOffset /) )
10127+
#ifdef MPAS_USE_MPI_F08
10128+
call mpas_log_write(' reqId: $i', intArgs=(/ commListPtr % reqId % mpi_val /) )
10129+
#else
1009510130
call mpas_log_write(' reqId: $i', intArgs=(/ commListPtr % reqId /) )
10131+
#endif
1009610132
call mpas_log_write(' ibuffer assc: $l', logicArgs=(/ associated( commListPtr % ibuffer ) /) )
1009710133
call mpas_log_write(' rbuffer assc: $l', logicArgs=(/ associated( commListPtr % rbuffer ) /) )
1009810134
call mpas_log_write(' next assc: $l', logicArgs=(/ associated( commListPtr % next ) /) )

‎src/framework/mpas_dmpar_types.inc

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,14 @@
77
integer, parameter :: MPAS_DMPAR_BUFFER_EXISTS = 6
88

99
type dm_info
10-
integer :: nprocs, my_proc_id, comm, info
10+
#ifdef MPAS_USE_MPI_F08
11+
type (MPI_Comm) :: comm
12+
type (MPI_Info) :: info
13+
#else
14+
integer :: comm
15+
integer :: info
16+
#endif
17+
integer :: nprocs, my_proc_id
1118
logical :: initialized_mpi
1219

1320
! Add variables specific to block decomposition. {{{
@@ -47,7 +54,11 @@
4754
integer :: bufferOffset
4855
real (kind=RKIND), dimension(:), pointer :: rbuffer => null()
4956
integer, dimension(:), pointer :: ibuffer => null()
57+
#ifdef MPAS_USE_MPI_F08
58+
type (MPI_Request) :: reqID
59+
#else
5060
integer :: reqID
61+
#endif
5162
type (mpas_communication_list), pointer :: next => null()
5263
integer :: commListSize
5364
logical :: received

‎src/framework/mpas_framework.F

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,23 @@ module mpas_framework
4242
!> MPI, the log unit numbers.
4343
!
4444
!-----------------------------------------------------------------------
45-
subroutine mpas_framework_init_phase1(dminfo, mpi_comm)!{{{
45+
subroutine mpas_framework_init_phase1(dminfo, external_comm)!{{{
46+
47+
#ifdef MPAS_USE_MPI_F08
48+
use mpi_f08, only : MPI_Comm
49+
#endif
4650

4751
implicit none
4852

4953
type (dm_info), pointer :: dminfo
50-
integer, intent(in), optional :: mpi_comm
54+
#ifdef MPAS_USE_MPI_F08
55+
type (MPI_Comm), intent(in), optional :: external_comm
56+
#else
57+
integer, intent(in), optional :: external_comm
58+
#endif
5159

5260
allocate(dminfo)
53-
call mpas_dmpar_init(dminfo, mpi_comm)
61+
call mpas_dmpar_init(dminfo, external_comm)
5462

5563
end subroutine mpas_framework_init_phase1!}}}
5664

‎src/framework/mpas_halo.F

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -485,16 +485,28 @@ end subroutine mpas_halo_exch_group_add_field
485485
!-----------------------------------------------------------------------
486486
subroutine mpas_halo_exch_group_full_halo_exch(domain, groupName, iErr)
487487

488+
#ifdef MPAS_USE_MPI_F08
489+
use mpi_f08
490+
#else
488491
use mpi
492+
#endif
489493
use mpas_derived_types, only : domain_type, mpas_halo_group, MPAS_HALO_REAL, MPAS_LOG_CRIT
490494
use mpas_pool_routines, only : mpas_pool_get_array
491495
use mpas_log, only : mpas_log_write
492496

493497
! Parameters
498+
#ifdef MPAS_USE_MPI_F08
499+
#ifdef SINGLE_PRECISION
500+
type (MPI_Datatype), parameter :: MPI_REALKIND = MPI_REAL
501+
#else
502+
type (MPI_Datatype), parameter :: MPI_REALKIND = MPI_DOUBLE_PRECISION
503+
#endif
504+
#else
494505
#ifdef SINGLE_PRECISION
495506
integer, parameter :: MPI_REALKIND = MPI_REAL
496507
#else
497508
integer, parameter :: MPI_REALKIND = MPI_DOUBLE_PRECISION
509+
#endif
498510
#endif
499511

500512
! Arguments
@@ -508,7 +520,12 @@ subroutine mpas_halo_exch_group_full_halo_exch(domain, groupName, iErr)
508520
integer :: i1, i2, j, iNeighbor, iReq
509521
integer :: iHalo, iEndp
510522
integer :: nHalos, nSendEndpts, nRecvEndpts
511-
integer :: rank, comm
523+
integer :: rank
524+
#ifdef MPAS_USE_MPI_F08
525+
type (MPI_Comm) :: comm
526+
#else
527+
integer :: comm
528+
#endif
512529
integer :: mpi_ierr
513530
type (mpas_halo_group), pointer :: group
514531
integer, dimension(:), pointer :: compactHaloInfo
@@ -550,7 +567,11 @@ subroutine mpas_halo_exch_group_full_halo_exch(domain, groupName, iErr)
550567
! the group; all fields should be using the same communicator, so this should not
551568
! be problematic
552569
!
570+
#ifdef MPAS_USE_MPI_F08
571+
comm % mpi_val = group % fields(1) % compactHaloInfo(7)
572+
#else
553573
comm = group % fields(1) % compactHaloInfo(7)
574+
#endif
554575
rank = group % fields(1) % compactHaloInfo(8)
555576

556577

@@ -992,7 +1013,11 @@ subroutine mpas_halo_compact_halo_info(domain, sendList, recvList, dimSizes, hal
9921013
! 7-8: Add MPI info
9931014
!
9941015
idx = 7
1016+
#ifdef MPAS_USE_MPI_F08
1017+
compactHaloInfo(idx) = domain % dminfo % comm % mpi_val
1018+
#else
9951019
compactHaloInfo(idx) = domain % dminfo % comm
1020+
#endif
9961021
idx = idx + 1
9971022
compactHaloInfo(idx) = domain % dminfo % my_proc_id
9981023
idx = idx + 1

‎src/framework/mpas_halo_types.inc

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,11 @@
4747
integer :: nGroupSendNeighbors = MPAS_HALO_INVALID ! Number of unique neighbors that we send to
4848
integer :: groupSendBufSize = MPAS_HALO_INVALID ! Total number of elements to be sent in a group exchange
4949
real (kind=RKIND), dimension(:), pointer :: sendBuf => null() ! Segmented buffer used for outgoing messages
50-
integer, dimension(:), pointer :: sendRequests => null() ! Used internally - MPI request IDs
50+
#ifdef MPAS_USE_MPI_F08
51+
type (MPI_Request), dimension(:), pointer :: sendRequests => null() ! Used internally - MPI request IDs
52+
#else
53+
integer, dimension(:), pointer :: sendRequests => null() ! Used internally - MPI request IDs
54+
#endif
5155
integer, dimension(:,:), pointer :: groupPackOffsets => null() ! Offsets into sendBuf for each neighbor and each field
5256
! dimensioned (nGroupSendNeighbors, nFields)
5357
integer, dimension(:), pointer :: groupSendNeighbors => null() ! List of neighbors we send to
@@ -60,7 +64,11 @@
6064
integer :: nGroupRecvNeighbors = MPAS_HALO_INVALID ! Number of unique neighbors that we recv from
6165
integer :: groupRecvBufSize = MPAS_HALO_INVALID ! Total number of elements to be recvd in a group exchange
6266
real (kind=RKIND), dimension(:), pointer :: recvBuf => null() ! Segmented buffer used for incoming messages
63-
integer, dimension(:), pointer :: recvRequests => null() ! Used internally - MPI request IDs
67+
#ifdef MPAS_USE_MPI_F08
68+
type (MPI_Request), dimension(:), pointer :: recvRequests => null() ! Used internally - MPI request IDs
69+
#else
70+
integer, dimension(:), pointer :: recvRequests => null() ! Used internally - MPI request IDs
71+
#endif
6472
integer, dimension(:,:), pointer :: groupUnpackOffsets => null() ! Offsets into recvBuf for each neighbor and each field
6573
! dimensioned (nGroupRecvNeighbors, nFields)
6674
integer, dimension(:), pointer :: groupRecvNeighbors => null() ! List of neighbors we recv from

0 commit comments

Comments
 (0)