From 17bb9defccf6d1700ec026260c06d311501852b6 Mon Sep 17 00:00:00 2001 From: Nathan Daly Date: Tue, 8 Feb 2022 09:53:52 -0500 Subject: [PATCH] Instrument jl_gc_big_alloc() when called from generated code for Allocations Profiler (#44043) * Instrument jl_gc_big_alloc() when called from generated code for Allocations Profiler Add _maybe_record_alloc_to_profile() call into jl_gc_big_alloc() to allow us to instrument it in the Allocations Profiler. Followup to #43868 Finishes fixing #43688 (gets the stacks, though not the types, of big allocs). ----- - Rename jl_gc_big_alloc to jl_gc_big_alloc_inner, make it inlined - add jl_gc_pool_alloc_noinline that calls jl_gc_big_alloc_inner - replace all internal calls to jl_gc_big_alloc to call this instead - add a new jl_gc_pool_alloc that calls jl_gc_big_alloc_inner - This one is instrumented, and is meant to be called by generated code. Co-authored-by: Pete Vilter <7341+vilterp@users.noreply.github.com> --- src/array.c | 2 +- src/gc.c | 18 +++++++++++++++++- src/julia_internal.h | 4 ++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/array.c b/src/array.c index fcfe22ed454d45..728dbf40bd4e1c 100644 --- a/src/array.c +++ b/src/array.c @@ -507,7 +507,7 @@ JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len) else { if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); - s = jl_gc_big_alloc(ptls, allocsz); + s = jl_gc_big_alloc_noinline(ptls, allocsz); } jl_set_typeof(s, jl_string_type); maybe_record_alloc_to_profile(s, len, jl_string_type); diff --git a/src/gc.c b/src/gc.c index 745f5365510f96..b31dd61c385583 100644 --- a/src/gc.c +++ b/src/gc.c @@ -944,7 +944,7 @@ static void sweep_weak_refs(void) // big value list // Size includes the tag and the tag is not cleared!! -JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) +static inline jl_value_t *jl_gc_big_alloc_inner(jl_ptls_t ptls, size_t sz) { maybe_collect(ptls); size_t offs = offsetof(bigval_t, header); @@ -972,6 +972,22 @@ JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) return jl_valueof(&v->header); } +// Instrumented version of jl_gc_big_alloc_inner, called into by LLVM-generated code. +JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) +{ + jl_value_t *val = jl_gc_big_alloc_inner(ptls, sz); + + maybe_record_alloc_to_profile(val, sz, jl_gc_unknown_type_tag); + return val; +} + +// This wrapper exists only to prevent `jl_gc_big_alloc_inner` from being inlined into +// its callers. We provide an external-facing interface for callers, and inline `jl_gc_big_alloc_inner` +// into this. (See https://github.com/JuliaLang/julia/pull/43868 for more details.) +jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t sz) { + return jl_gc_big_alloc_inner(ptls, sz); +} + // Sweep list rooted at *pv, removing and freeing any unmarked objects. // Return pointer to last `next` field in the culled list. static bigval_t **sweep_big_list(int sweep_full, bigval_t **pv) JL_NOTSAFEPOINT diff --git a/src/julia_internal.h b/src/julia_internal.h index 67166cd6351653..ced9e063bbe724 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -231,7 +231,7 @@ JL_DLLEXPORT extern const char *jl_filename; jl_value_t *jl_gc_pool_alloc_noinline(jl_ptls_t ptls, int pool_offset, int osize); -JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz); +jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t allocsz); JL_DLLEXPORT int jl_gc_classify_pools(size_t sz, int *osize); extern uv_mutex_t gc_perm_lock; void *jl_gc_perm_alloc_nolock(size_t sz, int zero, @@ -365,7 +365,7 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) else { if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); - v = jl_gc_big_alloc(ptls, allocsz); + v = jl_gc_big_alloc_noinline(ptls, allocsz); } jl_set_typeof(v, ty); maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty);