Add JEMALLOC_COLD, and mark some functions cold.

This hints to the compiler that it should care more about space than CPU (among other things). In cases where the compiler lacks profile-guided information, this can be a substantial space savings. For now, we mark the mallctl or atexit driven profiling and stats functions that take up the most space.
Clcanny · Jan 4, 2021 · a9fa2de · a9fa2de
1 parent 5d8e70a
commit a9fa2de
Show file tree

Hide file tree

Showing 7 changed files with 36 additions and 1 deletion.
diff --git a/configure.ac b/configure.ac
@@ -914,6 +914,18 @@ if test "x${je_cv_fallthrough}" = "xyes" ; then
   JE_CXXFLAGS_ADD([-Wimplicit-fallthrough])
 fi
 
+dnl Check for cold attribute support.
+JE_CFLAGS_SAVE()
+JE_CFLAGS_ADD([-Werror])
+JE_CFLAGS_ADD([-herror_on_warning])
+JE_COMPILABLE([cold attribute], [],
+              [__attribute__((__cold__)) void foo();],
+              [je_cv_cold])
+JE_CFLAGS_RESTORE()
+if test "x${je_cv_cold}" = "xyes" ; then
+  AC_DEFINE([JEMALLOC_HAVE_ATTR_COLD], [ ])
+fi
+
 dnl Support optional additions to rpath.
 AC_ARG_WITH([rpath],
   [AS_HELP_STRING([--with-rpath=<rpath>], [Colon-separated rpath (ELF systems only)])],

diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in
@@ -16,6 +16,9 @@
 /* Defined if fallthrough attribute is supported. */
 #undef JEMALLOC_HAVE_ATTR_FALLTHROUGH
 
+/* Defined if cold attribute is supported. */
+#undef JEMALLOC_HAVE_ATTR_COLD
+
 /*
  * Define overrides for non-standard allocator-related functions if they are
  * present on the system.

diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in
@@ -85,6 +85,7 @@
 #  else
 #    define JEMALLOC_ALLOCATOR
 #  endif
+#  define JEMALLOC_COLD
 #elif defined(JEMALLOC_HAVE_ATTR)
 #  define JEMALLOC_ATTR(s) __attribute__((s))
 #  define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
@@ -120,6 +121,11 @@
 #  define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
 #  define JEMALLOC_RESTRICT_RETURN
 #  define JEMALLOC_ALLOCATOR
+#  ifdef JEMALLOC_HAVE_ATTR_COLD
+#    define JEMALLOC_COLD JEMALLOC_ATTR(__cold__)
+#  else
+#    define JEMALLOC_COLD
+#  endif
 #else
 #  define JEMALLOC_ATTR(s)
 #  define JEMALLOC_ALIGNED(s)
@@ -133,6 +139,7 @@
 #  define JEMALLOC_SECTION(s)
 #  define JEMALLOC_RESTRICT_RETURN
 #  define JEMALLOC_ALLOCATOR
+#  define JEMALLOC_COLD
 #endif
 
 #if defined(__APPLE__) && !defined(JEMALLOC_NO_RENAME)

diff --git a/src/malloc_io.c b/src/malloc_io.c
@@ -321,6 +321,7 @@ x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) {
 	return s;
 }
 
+JEMALLOC_COLD
 size_t
 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) {
 	size_t i;

diff --git a/src/prof_log.c b/src/prof_log.c
@@ -200,6 +200,7 @@ prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) {
 	}
 }
 
+JEMALLOC_COLD
 void
 prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info) {
 	cassert(config_prof);
@@ -411,6 +412,7 @@ prof_log_dummy_set(bool new_value) {
 	prof_log_dummy = new_value;
 }
 
+JEMALLOC_COLD
 bool
 prof_log_start(tsdn_t *tsdn, const char *filename) {
 	cassert(config_prof);
@@ -593,6 +595,7 @@ prof_log_emit_metadata(emitter_t *emitter) {
 }
 
 #define PROF_LOG_STOP_BUFSIZE PROF_DUMP_BUFSIZE
+JEMALLOC_COLD
 bool
 prof_log_stop(tsdn_t *tsdn) {
 	cassert(config_prof);
@@ -681,7 +684,9 @@ prof_log_stop(tsdn_t *tsdn) {
 }
 #undef PROF_LOG_STOP_BUFSIZE
 
-bool prof_log_init(tsd_t *tsd) {
+JEMALLOC_COLD
+bool
+prof_log_init(tsd_t *tsd) {
 	cassert(config_prof);
 	if (malloc_mutex_init(&log_mtx, "prof_log",
 	    WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) {

diff --git a/src/prof_recent.c b/src/prof_recent.c
@@ -528,6 +528,7 @@ prof_recent_alloc_dump_node(emitter_t *emitter, prof_recent_t *node) {
 }
 
 #define PROF_RECENT_PRINT_BUFSIZE 65536
+JEMALLOC_COLD
 void
 prof_recent_alloc_dump(tsd_t *tsd, write_cb_t *write_cb, void *cbopaque) {
 	cassert(config_prof);

diff --git a/src/stats.c b/src/stats.c
@@ -281,6 +281,7 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row,
 	header_##column_name.str_val = human ? human : #column_name;
 
 
+JEMALLOC_COLD
 static void
 stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) {
 	size_t page;
@@ -488,6 +489,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti
 	}
 }
 
+JEMALLOC_COLD
 static void
 stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
 	unsigned nbins, nlextents, j;
@@ -576,6 +578,7 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
 	}
 }
 
+JEMALLOC_COLD
 static void
 stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 	unsigned j;
@@ -841,6 +844,7 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptim
 	emitter_json_object_end(emitter); /* End "mutexes". */
 }
 
+JEMALLOC_COLD
 static void
 stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
     bool mutex, bool extents, bool hpa) {
@@ -1168,6 +1172,7 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
 	}
 }
 
+JEMALLOC_COLD
 static void
 stats_general_print(emitter_t *emitter) {
 	const char *cpv;
@@ -1445,6 +1450,7 @@ stats_general_print(emitter_t *emitter) {
 	emitter_json_object_end(emitter); /* Close "arenas" */
 }
 
+JEMALLOC_COLD
 static void
 stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
     bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) {