Skip to content

Commit

Permalink
Perftest: replace rand() with getrandom() during MR buffer initializa…
Browse files Browse the repository at this point in the history
…tion

rand() has very poor performance in some OS.

ib_send_bw will spend a lot of time during MR initialization when
testing large packects in above scenario.

test has been done:
"""
\#define HUGE_MR_SIZE 2147483647
int main(int argc, char *argv[])
{
        char *a = malloc(HUGE_MR_SIZE * sizeof(char));
        unsigned int i;
        char *tmp = a;
        int ret;

        srand(time(NULL));
        if (a == NULL)
                exit(1);

        if (argc <= 1)
                goto fall_back;

        for (i = HUGE_MR_SIZE; i > 0;) {
                ret = getrandom(tmp, i, 0);
                if (ret < 0)
                        goto fall_back;
                tmp += ret;
                i -= ret;
        }
        goto out;

fall_back:
        for(i = 0; i < HUGE_MR_SIZE; i++)
                a[i] = (char)rand();
out:
        free(a);
        return 0;
}

time ./a.out
real    5m35.033s
user    5m33.546s
sys     0m0.918s

time ./a.out 1

real    0m6.454s
user    0m0.000s
sys     0m6.449s
"""

As shown in the test above, getrandom() has a much better performance,
so replace rand() with it.

Signed-off-by: Chengchang Tang <tangchengchang@huawei.com>
  • Loading branch information
Chengchang Tang authored and Chengchang Tang committed Nov 10, 2022
1 parent 8b66647 commit e4cd0f7
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 5 deletions.
1 change: 1 addition & 0 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ AC_PROG_LIBTOOL
AC_PROG_RANLIB
AC_HEADER_STDC
AC_CHECK_HEADERS([infiniband/verbs.h],,[AC_MSG_ERROR([ibverbs header files not found])])
AC_CHECK_HEADERS([sys/random.h],,)
AC_CHECK_LIB([ibverbs], [ibv_get_device_list], [], [AC_MSG_ERROR([libibverbs not found])])
AC_CHECK_LIB([rdmacm], [rdma_create_event_channel], [], AC_MSG_ERROR([librdmacm-devel not found]))
AC_CHECK_LIB([ibumad], [umad_init], [LIBUMAD=-libumad], AC_MSG_ERROR([libibumad not found]))
Expand Down
31 changes: 26 additions & 5 deletions src/perftest_resources.c
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#ifdef HAVE_SYS_RANDOM_H
#include <sys/random.h>
#endif
#ifdef HAVE_SRD
#include <infiniband/efadv.h>
#endif
Expand Down Expand Up @@ -1730,12 +1733,33 @@ int create_cqs(struct pingpong_context *ctx, struct perftest_parameters *user_pa
return ret;
}

static void random_data(char *buf, int buff_size)
{
int i;
#ifdef HAVE_SYS_RANDOM_H
char *tmp = buf;
int ret;

for(i = buff_size; i > 0;) {
ret = getrandom(tmp, i, 0);
if(ret < 0)
goto fall_back;
tmp += ret;
i -= ret;
}
return;
fall_back:
#endif
srand(time(NULL));
for (i = 0; i < buff_size; i++)
buf[i] = (char)rand();
}

/******************************************************************************
*
******************************************************************************/
int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *user_param, int qp_index)
{
int i;
int flags = IBV_ACCESS_LOCAL_WRITE;


Expand Down Expand Up @@ -1923,13 +1947,10 @@ int create_single_mr(struct pingpong_context *ctx, struct perftest_parameters *u
#ifdef HAVE_CUDA
if (!user_param->use_cuda) {
#endif
srand(time(NULL));
if (user_param->verb == WRITE && user_param->tst == LAT) {
memset(ctx->buf[qp_index], 0, ctx->buff_size);
} else {
for (i = 0; i < ctx->buff_size; i++) {
((char*)ctx->buf[qp_index])[i] = (char)rand();
}
random_data(ctx->buf[qp_index], ctx->buff_size);
}
#ifdef HAVE_CUDA
}
Expand Down

0 comments on commit e4cd0f7

Please sign in to comment.