common: reduce stack usage by eliminating complex inline function

Signed-off-by: Tomasz Gromadzki <tomasz.gromadzki@intel.com>

ChangeLog

@@ -1,7 +1,7 @@
 	* Unreleased *
 
 	This release:
-	- ...
+	- Make memmove_mov_* function on-inline to prevent huge stack consumption.
 
 Tue Aug 8 2023 Oksana Sałyk <oksana.salyk@intel.com>
 

src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2017-2020, Intel Corporation */
+/* Copyright 2017-2023, Intel Corporation */
 
 #include <immintrin.h>
 #include <stddef.h>
@@ -254,7 +254,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b)
 	flush64b(dest + 0 * 64);
 }
 
-static force_inline void
+static void
 memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
 		flush_fn flush, flush64b_fn flush64b)
 {
@@ -319,7 +319,7 @@ memmove_mov_avx512f_fw(char *dest, const char *src, size_t len,
 		memmove_small_avx512f(dest, src, len, flush);
 }
 
-static force_inline void
+static void
 memmove_mov_avx512f_bw(char *dest, const char *src, size_t len,
 		flush_fn flush, flush64b_fn flush64b)
 {

src/stat/stack-usage-debug.txt

@@ -8,16 +8,10 @@
 51344 src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c:memmove_movnt_avx512f_clflush dynamic,bounded  
 51344 src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c:memmove_movnt_avx512f_empty dynamic,bounded  
 51344 src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c:memmove_movnt_avx512f_noflush dynamic,bounded  
-50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded  
-50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded  
-50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded  
-50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded  
-50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded  
-50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded  
-50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded  
-50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded  
-50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded  
-50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded  
+25344 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static  
+25344 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static  
+25344 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static  
+25344 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static  
 24144 src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c:memmove_movnt_avx_clwb_wcbarrier dynamic,bounded  
 24144 src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c:memmove_movnt_avx_clflushopt_wcbarrier dynamic,bounded  
 24144 src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c:memmove_movnt_avx_clflush_wcbarrier dynamic,bounded  
@@ -955,12 +949,22 @@
 112 src/debug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_new static  
 112 src/debug/libpmem/mmap.su:mmap.c:util_range_split dynamic,bounded  
 112 src/debug/libpmem/mmap.su:mmap.c:util_range_register dynamic,bounded  
+112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded  
+112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded  
+112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded  
+112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded  
+112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded  
 112 src/debug/libpmem/file.su:file.c:util_file_open dynamic,bounded  
 112 src/debug/libpmem/file.su:file.c:util_file_zero dynamic,bounded  
 112 src/debug/libpmem2/vm_reservation.su:vm_reservation.c:pmem2_vm_reservation_new static  
 112 src/debug/libpmem2/usc_ndctl.su:usc_ndctl.c:pmem2_source_device_usc dynamic,bounded  
 112 src/debug/libpmem2/persist.su:persist.c:pmem2_flush_file_buffers static  
 112 src/debug/libpmem2/persist.su:persist.c:pmem2_log_flush dynamic,bounded  
+112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded  
+112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded  
+112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded  
+112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded  
+112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded  
 112 src/debug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:devdax_write dynamic,bounded  
 112 src/debug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:devdax_read dynamic,bounded  
 112 src/debug/libpmem2/deep_flush.su:deep_flush.c:pmem2_deep_flush dynamic,bounded  

src/stat/stack-usage-nondebug.txt

@@ -10,8 +10,6 @@
 8224 src/nondebug/libpmemobj/pmem2_utils_linux.su:pmem2_utils_linux.c:pmem2_get_type_from_stat static  
 8224 src/nondebug/libpmem2/pmem2_utils_linux.su:pmem2_utils_linux.c:pmem2_get_type_from_stat static  
 8224 src/nondebug/common/pmem2_utils_linux.su:pmem2_utils_linux.c:pmem2_get_type_from_stat static  
-7424 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static  
-7424 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static  
 4352 src/nondebug/libpmempool/pool.su:pool.c:pool_params_parse.isra.2.constprop dynamic,bounded  
 4272 src/nondebug/libpmem/region_namespace_ndctl.su:region_namespace_ndctl.c:ndctl_match_fsdax static  
 4272 src/nondebug/libpmem/region_namespace_ndctl.su:region_namespace_ndctl.c:ndctl_match_devdax static  
@@ -419,10 +417,14 @@
 128 src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflushopt static  
 128 src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflush static  
 128 src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_noflush static  
+128 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static  
+128 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static  
 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clwb static  
 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflushopt static  
 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflush static  
 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_noflush static  
+128 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static  
+128 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static  
 128 src/nondebug/libpmem2/map.su:map.c:pmem2_map_from_existing static  
 128 src/nondebug/libpmem2/badblocks_ndctl.su:badblocks_ndctl.c:pmem2_badblock_context_new static  
 128 src/nondebug/common/badblocks_ndctl.su:badblocks_ndctl.c:pmem2_badblock_context_new static  
@@ -715,10 +717,6 @@
 64 src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflushopt static  
 64 src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflush static  
 64 src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_noflush static  
-64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static  
-64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static  
-64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static  
-64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static  
 64 src/nondebug/libpmem/file.su:file.c:util_file_zero dynamic,bounded  
 64 src/nondebug/libpmem/file.su:file.c:util_file_map_whole dynamic,bounded  
 64 src/nondebug/libpmem2/usc_ndctl.su:usc_ndctl.c:pmem2_source_device_id static  
@@ -739,10 +737,6 @@
 64 src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflushopt static  
 64 src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflush static  
 64 src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_noflush static  
-64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static  
-64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static  
-64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static  
-64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static  
 64 src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:pmem2_source_check_op_size dynamic,bounded  
 64 src/nondebug/libpmem2/map.su:map.c:get_min_granularity static  
 64 src/nondebug/libpmem2/badblocks_ndctl.su:badblocks_ndctl.c:pmem2_badblock_clear static  
@@ -1492,6 +1486,11 @@
 16 src/nondebug/libpmem/mmap.su:mmap.c:util_range_rw static  
 16 src/nondebug/libpmem/mmap.su:mmap.c:util_range_ro static  
 16 src/nondebug/libpmem/mmap.su:mmap.c:util_unmap static  
+16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static  
+16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static  
+16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static  
+16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static  
+16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static  
 16 src/nondebug/libpmem/libpmem.su:libpmem.c:pmem_check_version static  
 16 src/nondebug/libpmem/libpmem.su:libpmem.c:libpmem_fini static  
 16 src/nondebug/libpmem/libpmem.su:libpmem.c:libpmem_init static  
@@ -1546,6 +1545,11 @@
 16 src/nondebug/libpmem2/out.su:out.c:out_fini static  
 16 src/nondebug/libpmem2/out.su:out.c:out_init static  
 16 src/nondebug/libpmem2/os_thread_posix.su:os_thread_posix.c:os_thread_self static  
+16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static  
+16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static  
+16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static  
+16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static  
+16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static  
 16 src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:signal_handler static  
 16 src/nondebug/libpmem2/map.su:map.c:pmem2_map_fini static  
 16 src/nondebug/libpmem2/map.su:map.c:pmem2_map_init static  
@@ -2145,6 +2149,16 @@
 8 src/nondebug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_delete static  
 8 src/nondebug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_reserve static  
 8 src/nondebug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_find_first_free_slot static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:pmem2_arch.h:flush_empty_nolog static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h:noflush64b static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h:noflush static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush64b_empty static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush_clwb_nolog static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush_clflushopt_nolog static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush_clflush_nolog static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:pmem_clwb static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:pmem_clflushopt static  
+8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:pmem_clflush static  
 8 src/nondebug/libpmem/libpmem.su:libpmem.c:pmem_errormsg static  
 8 src/nondebug/libpmem/init.su:init.c:flush_clwb static  
 8 src/nondebug/libpmem/init.su:init.c:flush_clflushopt static  
@@ -2258,6 +2272,16 @@
 8 src/nondebug/libpmem2/os_posix.su:os_posix.c:os_chmod static  
 8 src/nondebug/libpmem2/os_posix.su:os_posix.c:os_fdopen static  
 8 src/nondebug/libpmem2/os_posix.su:os_posix.c:os_fopen static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:pmem2_arch.h:flush_empty_nolog static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h:noflush64b static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h:noflush static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush64b_empty static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush_clwb_nolog static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush_clflushopt_nolog static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush_clflush_nolog static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:pmem_clwb static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:pmem_clflushopt static  
+8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:pmem_clflush static  
 8 src/nondebug/libpmem2/membuf.su:membuf.c:membuf_ptr_user_data static  
 8 src/nondebug/libpmem2/membuf.su:membuf.c:membuf_free static  
 8 src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:mcsafe_op_devdax_write static