Skip to content

Commit

Permalink
common: reduce stack usage by eliminating complex memmove_ inline fun…
Browse files Browse the repository at this point in the history 
…ction

Signed-off-by: Tomasz Gromadzki <tomasz.gromadzki@intel.com>
  • Loading branch information
@grom72
grom72 committed Oct 24, 2023
1 parent 87108b8 commit 749b513
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 66 deletions.
1 change: 1 addition & 0 deletions ChangeLog
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

This release:
- Significantly reduces the libpmem's stack usage.
- Make memmove_mov_* function non-inline to prevent huge stack consumption.

Tue Aug 8 2023 Oksana Sałyk <oksana.salyk@intel.com>

Expand Down
2 changes: 1 addition & 1 deletion src/libpmem2/x86_64/memcpy/memcpy_t_avx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ memmove_mov_avx_bw(char *dest, const char *src, size_t len,
memmove_small_avx(dest - len, src - len, len, flush);
}

static force_inline void
static void
memmove_mov_avx(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down
6 changes: 3 additions & 3 deletions src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: BSD-3-Clause
/* Copyright 2017-2020, Intel Corporation */
/* Copyright 2017-2023, Intel Corporation */

#include <immintrin.h>
#include <stddef.h>
Expand Down Expand Up @@ -254,7 +254,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b)
flush64b(dest + 0 * 64);
}

static force_inline void
static void
memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down Expand Up @@ -319,7 +319,7 @@ memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
memmove_small_avx512f(dest, src, len, flush);
}

static force_inline void
static void
memmove_mov_avx512f_bw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down
4 changes: 2 additions & 2 deletions src/libpmem2/x86_64/memcpy/memcpy_t_sse2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b)
flush64b(dest + 0 * 64);
}

static force_inline void
static void
memmove_mov_sse_fw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down Expand Up @@ -151,7 +151,7 @@ memmove_mov_sse_fw(char *dest, const char *src, size_t len,
memmove_small_sse2(dest, src, len, flush);
}

static force_inline void
static void
memmove_mov_sse_bw(char *dest, const char *src, size_t len,
flush_fn flush, flush64b_fn flush64b)
{
Expand Down
70 changes: 40 additions & 30 deletions src/stats/stack-usage-debug.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,10 @@
51344 memmove_movnt_avx512f_clflush : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
51344 memmove_movnt_avx512f_clflushopt : src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
50640 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
25344 memmove_mov_avx512f_fw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
25344 memmove_mov_avx512f_fw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
25344 memmove_mov_avx512f_bw : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
25344 memmove_mov_avx512f_bw : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c static
24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
24144 memmove_movnt_avx_noflush_wcbarrier : src/debug/libpmem2/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
24144 memmove_movnt_avx_noflush_nobarrier : src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c dynamic,bounded
Expand Down Expand Up @@ -79,16 +73,8 @@
19024 memset_mov_avx512f_clflushopt : src/debug/libpmem/memset_t_avx512f.su:memset_t_avx512f.c dynamic,bounded
19024 memset_mov_avx512f_clflushopt : src/debug/libpmem2/memset_t_avx512f.su:memset_t_avx512f.c dynamic,bounded
16480 obj_vg_check_no_undef : src/debug/libpmemobj/obj.su:obj.c static
13840 memmove_mov_avx_noflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_noflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_empty : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_empty : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clwb : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clwb : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflushopt : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13840 memmove_mov_avx_clflushopt : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
13792 memmove_mov_avx : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c static
13792 memmove_mov_avx : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c static
9808 memset_movnt_avx_noflush_wcbarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_noflush_wcbarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_noflush_nobarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
Expand All @@ -109,16 +95,6 @@
9808 memset_movnt_avx_clflushopt_nobarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_clflush_nobarrier : src/debug/libpmem/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9808 memset_movnt_avx_clflush_nobarrier : src/debug/libpmem2/memset_nt_avx.su:memset_nt_avx.c dynamic,bounded
9200 memmove_mov_sse2_noflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_noflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_empty : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_empty : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clwb : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clwb : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflushopt : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
9200 memmove_mov_sse2_clflushopt : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
8464 out_error : src/debug/libpmempool/out.su:out.c dynamic,bounded
8464 out_error : src/debug/libpmem/out.su:out.c dynamic,bounded
8464 out_error : src/debug/libpmemobj/out.su:out.c dynamic,bounded
Expand Down Expand Up @@ -165,6 +141,10 @@
5936 memset_mov_avx_clflush : src/debug/libpmem2/memset_t_avx.su:memset_t_avx.c dynamic,bounded
5936 memset_mov_avx_clflushopt : src/debug/libpmem/memset_t_avx.su:memset_t_avx.c dynamic,bounded
5936 memset_mov_avx_clflushopt : src/debug/libpmem2/memset_t_avx.su:memset_t_avx.c dynamic,bounded
4576 memmove_mov_sse_fw : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static
4576 memmove_mov_sse_fw : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static
4576 memmove_mov_sse_bw : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c static
4576 memmove_mov_sse_bw : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c static
4384 pool_params_parse : src/debug/libpmempool/pool.su:pool.c dynamic,bounded
4352 out_init : src/debug/libpmempool/out.su:out.c dynamic,bounded
4352 out_init : src/debug/libpmem/out.su:out.c dynamic,bounded
Expand Down Expand Up @@ -952,6 +932,26 @@
112 obj_open_common : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
112 obj_norep_memmove : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
112 obj_norep_memcpy : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
112 memmove_mov_sse2_noflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_noflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_empty : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_empty : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clwb : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clwb : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflush : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflush : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflushopt : src/debug/libpmem/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_sse2_clflushopt : src/debug/libpmem2/memcpy_t_sse2.su:memcpy_t_sse2.c dynamic,bounded
112 memmove_mov_avx512f_noflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_noflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_empty : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_empty : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clwb : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clwb : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflush : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflush : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflushopt : src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memmove_mov_avx512f_clflushopt : src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c dynamic,bounded
112 memblock_run_bitmap : src/debug/libpmemobj/memblock.su:memblock.c dynamic,bounded
112 list_remove_single : src/debug/libpmemobj/list.su:list.c dynamic,bounded
112 lane_boot : src/debug/libpmemobj/lane.su:lane.c static
Expand Down Expand Up @@ -1672,6 +1672,16 @@
64 obj_replica_init : src/debug/libpmemobj/obj.su:obj.c static
64 obj_pool_open : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
64 obj_ctl_init_and_load : src/debug/libpmemobj/obj.su:obj.c dynamic,bounded
64 memmove_mov_avx_noflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_noflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_empty : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_empty : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clwb : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clwb : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflush : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflush : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflushopt : src/debug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 memmove_mov_avx_clflushopt : src/debug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c dynamic,bounded
64 membuf_alloc : src/debug/libpmempool/membuf.su:membuf.c static
64 membuf_alloc : src/debug/libpmemobj/membuf.su:membuf.c static
64 membuf_alloc : src/debug/libpmem2/membuf.su:membuf.c static
Expand Down
Loading

0 comments on commit 749b513

Please sign in to comment.