Skip to content

Add MCA parameters to define the size of memcpy chunks. #6426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add MCA parameters to define the size of memcpy chunks. #6426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions opal/datatype/opal_datatype_copy.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2013 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -41,7 +41,15 @@
#define DO_DEBUG(INST)
#endif /* OPAL_ENABLE_DEBUG */

static size_t opal_datatype_memop_block_size = 128 * 1024;
size_t opal_datatype_memop_block_size = 128 * 1024;
size_t opal_datatype_cuda_memop_block_size = SIZE_MAX; /* or (size_t)-1 in pre C99 */

/* The MEM_OP_BLOCK_SIZE_CONST define how a large contiguous memcpy
* should be split. In some cases having a pipeline might allow for
* cache write-backs, but in general (and certainly in the case of
* CUDA devices) this should be set to the largest size_t value.
*/
#define MEM_OP_BLOCK_SIZE_CONST opal_datatype_memop_block_size

/**
* Non overlapping memory regions
Expand Down Expand Up @@ -72,6 +80,10 @@ static size_t opal_datatype_memop_block_size = 128 * 1024;
#include "opal_datatype_copy.h"

#if OPAL_CUDA_SUPPORT

#undef MEM_OP_BLOCK_SIZE_CONST
#define MEM_OP_BLOCK_SIZE_CONST opal_datatype_cuda_memop_block_size

#include "opal_datatype_cuda.h"

#undef MEM_OP_NAME
Expand All @@ -92,9 +104,12 @@ static size_t opal_datatype_memop_block_size = 128 * 1024;
fct = copy_function; \
} \
} while(0)
#else

#else /* OPAL_CUDA_SUPPORT */

#define SET_CUDA_COPY_FCT(cuda_device_bufs, fct, copy_function)
#endif

#endif /* OPAL_CUDA_SUPPORT */

int32_t opal_datatype_copy_content_same_ddt( const opal_datatype_t* datatype, int32_t count,
char* destination_base, char* source_base )
Expand Down
123 changes: 89 additions & 34 deletions opal/datatype/opal_datatype_copy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2017 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2009 Oak Ridge National Labs. All rights reserved.
Expand Down Expand Up @@ -35,18 +35,61 @@
#endif


#define _predefined_data DT_CONCAT(MEM_OP_NAME,_predefined_data)
#define _contiguous_loop DT_CONCAT(MEM_OP_NAME,_contiguous_loop)
#define _copy_content_same_ddt DT_CONCAT(MEM_OP_NAME,_copy_content_same_ddt)

static inline void _predefined_data( const dt_elem_desc_t* ELEM,
const opal_datatype_t* DATATYPE,
unsigned char* SOURCE_BASE,
size_t TOTAL_COUNT,
size_t COUNT,
unsigned char* SOURCE,
unsigned char* DESTINATION,
size_t* SPACE )
#if !defined(MEM_OP_BLOCK_SIZE_CONST)
#error
#endif

#if !defined(_memcpy_vector)

#define _memcpy_vector DT_CONCAT(MEM_OP_NAME,_memcpy_vector)
#define __OPAL_DATATYPE_DEFINE__memcpy_vector

static inline size_t
_memcpy_vector( unsigned char* dest, /* destination pointer of the copy */
unsigned char* source, /* source pointer of the copy */
size_t blength, /* size in bytes of each block */
size_t count, /* the number of blocks */
ptrdiff_t dstride, /* the stride at the destination of each block */
ptrdiff_t sstride ) /* the stride at the source of each block */
{
size_t _length = 0;
if( (blength == (size_t)(sstride)) && (sstride == dstride) ) {
_length = count * blength;
/* the extent and the size of the basic datatype are equals */
DO_DEBUG( opal_output( 0, "vector copy [*] %s( %p, %p, %" PRIsize_t " )\n",
STRINGIFY(MEM_OP_NAME), (void*)dest, (void*)source, _length ); );
MEM_OP( dest, source, _length );
} else {
for(size_t _i = 0; _i < count; _i++ ) {
/* the extent and the size of the basic datatype are equals */
DO_DEBUG( opal_output( 0, "vector copy [%" PRIsize_t "] %s( %p, %p, %" PRIsize_t " )\n",
_i, STRINGIFY(MEM_OP_NAME), (void*)dest, (void*)source, blength ); );
MEM_OP( dest, source, blength );
_length += blength;
source += sstride;
dest += dstride;
}
}
return _length;
}
#endif /* !defined(_memcpy_vector) */

#if !defined(_predefined_data)

#define _predefined_data DT_CONCAT(MEM_OP_NAME,_predefined_data)
#define __OPAL_DATATYPE_DEFINE__predefined_data

static inline
void _predefined_data( const dt_elem_desc_t* ELEM,
const opal_datatype_t* DATATYPE,
unsigned char* SOURCE_BASE,
size_t TOTAL_COUNT,
size_t COUNT,
unsigned char* SOURCE,
unsigned char* DESTINATION,
size_t* SPACE )
{
size_t _copy_count = (COUNT);
size_t _copy_blength;
Expand All @@ -57,38 +100,34 @@ static inline void _predefined_data( const dt_elem_desc_t* ELEM,
_copy_blength = opal_datatype_basicDatatypes[_elem->common.type]->size;

if( _copy_blength == (size_t)_elem->extent ) {
_copy_blength *= _copy_count;
OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
(DATATYPE), (TOTAL_COUNT) );
/* the extent and the size of the basic datatype are equals */
DO_DEBUG( opal_output( 0, "copy 1. %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, _copy_blength, *(SPACE) ); );
MEM_OP( _destination, _source, _copy_blength );
_source += _copy_blength;
_destination += _copy_blength;
} else {
for(size_t _i = 0; _i < _copy_count; _i++ ) {
OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
(DATATYPE), (TOTAL_COUNT) );
DO_DEBUG( opal_output( 0, "copy 2. %s( %p, %p, %lu ) => space %lu\n",
STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, (unsigned long)_copy_blength, (unsigned long)(*(SPACE) - (_i * _copy_blength)) ); );
MEM_OP( _destination, _source, _copy_blength );
_source += _elem->extent;
_destination += _elem->extent;
}
_copy_blength *= _copy_count;
}
*(SPACE) -= _copy_blength;
_copy_blength = _memcpy_vector( _destination, _source,
_copy_blength, _copy_count,
_elem->extent, _elem->extent );
*(SPACE) -= _copy_blength;
}
#endif /* !defined(_predefined_data) */

static inline void _contiguous_loop( const dt_elem_desc_t* ELEM,
const opal_datatype_t* DATATYPE,
unsigned char* SOURCE_BASE,
size_t TOTAL_COUNT,
size_t COUNT,
unsigned char* SOURCE,
unsigned char* DESTINATION,
size_t* SPACE )
#if !defined(_contiguous_loop)
#define _contiguous_loop DT_CONCAT(MEM_OP_NAME,_contiguous_loop)
#define __OPAL_DATATYPE_DEFINE__contiguous_loop

static inline
void _contiguous_loop( const dt_elem_desc_t* ELEM,
const opal_datatype_t* DATATYPE,
unsigned char* SOURCE_BASE,
size_t TOTAL_COUNT,
size_t COUNT,
unsigned char* SOURCE,
unsigned char* DESTINATION,
size_t* SPACE )
{
ddt_loop_desc_t *_loop = (ddt_loop_desc_t*)(ELEM);
ddt_endloop_desc_t* _end_loop = (ddt_endloop_desc_t*)((ELEM) + _loop->items);
Expand All @@ -115,6 +154,7 @@ static inline void _contiguous_loop( const dt_elem_desc_t* ELEM,
}
*(SPACE) -= _copy_loops;
}
#endif /* !defined(_contiguous_loop) */

static inline int32_t _copy_content_same_ddt( const opal_datatype_t* datatype, int32_t count,
char* destination_base, char* source_base )
Expand Down Expand Up @@ -146,7 +186,7 @@ static inline int32_t _copy_content_same_ddt( const opal_datatype_t* datatype, i
source += datatype->true_lb;
if( (ptrdiff_t)datatype->size == extent ) { /* all contiguous == no gaps around */
size_t total_length = iov_len_local;
size_t memop_chunk = opal_datatype_memop_block_size;
size_t memop_chunk = MEM_OP_BLOCK_SIZE_CONST;
while( total_length > 0 ) {
if( memop_chunk > total_length ) memop_chunk = total_length;
OPAL_DATATYPE_SAFEGUARD_POINTER( destination, memop_chunk,
Expand Down Expand Up @@ -251,3 +291,18 @@ static inline int32_t _copy_content_same_ddt( const opal_datatype_t* datatype, i
}
}
}

#if defined(__OPAL_DATATYPE_DEFINE__memcpy_vector)
#undef __OPAL_DATATYPE_DEFINE__memcpy_vector
#undef _memcpy_vector
#endif /* defined(__OPAL_DATATYPE_DEFINE__memcpy_vector) */

#if defined(__OPAL_DATATYPE_DEFINE__predefined_data)
#undef __OPAL_DATATYPE_DEFINE__predefined_data
#undef _predefined_data
#endif /* defined(__OPAL_DATATYPE_DEFINE__predefined_data) */

#if defined(__OPAL_DATATYPE_DEFINE__contiguous_loop)
#undef __OPAL_DATATYPE_DEFINE__contiguous_loop
#undef _contiguous_loop
#endif /* defined(__OPAL_DATATYPE_DEFINE__contiguous_loop) */
5 changes: 4 additions & 1 deletion opal/datatype/opal_datatype_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2018 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -515,5 +515,8 @@ extern bool opal_position_debug;
extern bool opal_copy_debug;
#endif /* OPAL_ENABLE_DEBUG */

extern size_t opal_datatype_memop_block_size;
extern size_t opal_datatype_cuda_memop_block_size;

END_C_DECLS
#endif /* OPAL_DATATYPE_INTERNAL_H_HAS_BEEN_INCLUDED */
59 changes: 40 additions & 19 deletions opal/datatype/opal_datatype_module.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2018 The University of Tennessee and The University
* Copyright (c) 2004-2019 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -157,39 +157,39 @@ OPAL_DECLSPEC const opal_datatype_t* opal_datatype_basicDatatypes[OPAL_DATATYPE_

int opal_datatype_register_params(void)
{
#if OPAL_ENABLE_DEBUG
int ret;
#if OPAL_ENABLE_DEBUG

ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_unpack_debug",
"Whether to output debugging information in the ddt unpack functions (nonzero = enabled)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_unpack_debug);
"Whether to output debugging information in the ddt unpack functions (nonzero = enabled)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_unpack_debug);
if (0 > ret) {
return ret;
return ret;
}

ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_pack_debug",
"Whether to output debugging information in the ddt pack functions (nonzero = enabled)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_pack_debug);
"Whether to output debugging information in the ddt pack functions (nonzero = enabled)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_pack_debug);
if (0 > ret) {
return ret;
return ret;
}

ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_position_debug",
"Non zero lead to output generated by the datatype position functions",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_position_debug);
"Non zero lead to output generated by the datatype position functions",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_position_debug);
if (0 > ret) {
return ret;
return ret;
}

ret = mca_base_var_register ("opal", "mpi", NULL, "ddt_copy_debug",
"Whether to output debugging information in the ddt copy functions (nonzero = enabled)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_copy_debug);
"Whether to output debugging information in the ddt copy functions (nonzero = enabled)",
MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_LOCAL, &opal_copy_debug);
if (0 > ret) {
return ret;
return ret;
}

ret = mca_base_var_register ("opal", "opal", NULL, "ddt_verbose",
Expand All @@ -208,12 +208,33 @@ int opal_datatype_register_params(void)
OPAL_INFO_LVL_8, MCA_BASE_VAR_SCOPE_LOCAL,
&opal_cuda_verbose);
if (0 > ret) {
return ret;
return ret;
}
#endif

#endif /* OPAL_ENABLE_DEBUG */

/* Define the segment size for the data copy operations */
ret = mca_base_var_register ("opal", "opal", NULL, "datatype_memcpy_block_size",
"Split all memory copies handled by the datatype engine",
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
&opal_datatype_memop_block_size);
if (0 > ret) {
return ret;
}

#if OPAL_CUDA_SUPPORT
ret = mca_base_var_register ("opal", "opal", NULL, "datatype_cuda_memcpy_block_size",
"Split all CUDA memory copies handled by the datatype engine",
MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL,
&opal_datatype_cuda_memop_block_size);
if (0 > ret) {
return ret;
}
#endif

return OPAL_SUCCESS;
}

Expand Down