From f7d46b81197b9879e1f572f9a4d3bfe3b8f850b9 Mon Sep 17 00:00:00 2001 From: Alex Lapenkou Date: Mon, 30 Aug 2021 14:05:56 -0700 Subject: [PATCH] Allow setting custom backtrace hook Existing backtrace implementations skip native stack frames from runtimes like Python. The hook allows to augment the backtraces to attribute allocations to native functions in heap profiles. --- Makefile.in | 1 + include/jemalloc/internal/prof_externs.h | 4 +- include/jemalloc/internal/prof_hook.h | 16 +++++++ include/jemalloc/internal/prof_structs.h | 3 +- include/jemalloc/internal/prof_sys.h | 1 + src/ctl.c | 36 +++++++++++++- src/prof.c | 16 +++++++ src/prof_sys.c | 44 +++++++++-------- test/analyze/prof_bias.c | 14 +++--- test/unit/prof_hook.c | 61 ++++++++++++++++++++++++ test/unit/prof_hook.sh | 6 +++ 11 files changed, 172 insertions(+), 30 deletions(-) create mode 100644 include/jemalloc/internal/prof_hook.h create mode 100644 test/unit/prof_hook.c create mode 100644 test/unit/prof_hook.sh diff --git a/Makefile.in b/Makefile.in index 51276ceb..a6f61ced 100644 --- a/Makefile.in +++ b/Makefile.in @@ -247,6 +247,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/prof_accum.c \ $(srcroot)test/unit/prof_active.c \ $(srcroot)test/unit/prof_gdump.c \ + $(srcroot)test/unit/prof_hook.c \ $(srcroot)test/unit/prof_idump.c \ $(srcroot)test/unit/prof_log.c \ $(srcroot)test/unit/prof_mdump.c \ diff --git a/include/jemalloc/internal/prof_externs.h b/include/jemalloc/internal/prof_externs.h index 671ac9b8..75d1d7a0 100644 --- a/include/jemalloc/internal/prof_externs.h +++ b/include/jemalloc/internal/prof_externs.h @@ -2,6 +2,7 @@ #define JEMALLOC_INTERNAL_PROF_EXTERNS_H #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/prof_hook.h" extern bool opt_prof; extern bool opt_prof_active; @@ -52,7 +53,8 @@ extern bool prof_booted; * otherwise difficult to guarantee that two allocations are reported as coming * from the exact same stack trace in the presence of an optimizing compiler. */ -extern void (* JET_MUTABLE prof_backtrace_hook)(prof_bt_t *bt); +void prof_backtrace_hook_set(prof_backtrace_hook_t hook); +prof_backtrace_hook_t prof_backtrace_hook_get(); /* Functions only accessed in prof_inlines.h */ prof_tdata_t *prof_tdata_init(tsd_t *tsd); diff --git a/include/jemalloc/internal/prof_hook.h b/include/jemalloc/internal/prof_hook.h new file mode 100644 index 00000000..277cd992 --- /dev/null +++ b/include/jemalloc/internal/prof_hook.h @@ -0,0 +1,16 @@ +#ifndef JEMALLOC_INTERNAL_PROF_HOOK_H +#define JEMALLOC_INTERNAL_PROF_HOOK_H + +/* + * The hooks types of which are declared in this file are experimental and + * undocumented, thus the typedefs are located in an 'internal' header. + */ + +/* + * A hook to mock out backtrace functionality. This can be handy, since it's + * otherwise difficult to guarantee that two allocations are reported as coming + * from the exact same stack trace in the presence of an optimizing compiler. + */ +typedef void (*prof_backtrace_hook_t)(void **, unsigned *, unsigned); + +#endif /* JEMALLOC_INTERNAL_PROF_HOOK_H */ diff --git a/include/jemalloc/internal/prof_structs.h b/include/jemalloc/internal/prof_structs.h index c2a111a9..dd22115f 100644 --- a/include/jemalloc/internal/prof_structs.h +++ b/include/jemalloc/internal/prof_structs.h @@ -16,7 +16,8 @@ struct prof_bt_s { #ifdef JEMALLOC_PROF_LIBGCC /* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ typedef struct { - prof_bt_t *bt; + void **vec; + unsigned *len; unsigned max; } prof_unwind_data_t; #endif diff --git a/include/jemalloc/internal/prof_sys.h b/include/jemalloc/internal/prof_sys.h index 6e4e811a..3d25a429 100644 --- a/include/jemalloc/internal/prof_sys.h +++ b/include/jemalloc/internal/prof_sys.h @@ -6,6 +6,7 @@ extern base_t *prof_base; void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(tsd_t *tsd, prof_bt_t *bt); +void prof_hooks_init(); void prof_unwind_init(); void prof_sys_thread_name_fetch(tsd_t *tsd); int prof_getpid(void); diff --git a/src/ctl.c b/src/ctl.c index 8717c96d..6bf1c946 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -305,6 +305,7 @@ CTL_PROTO(stats_retained) CTL_PROTO(stats_zero_reallocs) CTL_PROTO(experimental_hooks_install) CTL_PROTO(experimental_hooks_remove) +CTL_PROTO(experimental_hooks_prof_backtrace) CTL_PROTO(experimental_thread_activity_callback) CTL_PROTO(experimental_utilization_query) CTL_PROTO(experimental_utilization_batch_query) @@ -833,7 +834,8 @@ static const ctl_named_node_t stats_node[] = { static const ctl_named_node_t experimental_hooks_node[] = { {NAME("install"), CTL(experimental_hooks_install)}, - {NAME("remove"), CTL(experimental_hooks_remove)} + {NAME("remove"), CTL(experimental_hooks_remove)}, + {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)} }; static const ctl_named_node_t experimental_thread_node[] = { @@ -3328,6 +3330,38 @@ prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, return 0; } +static int +experimental_hooks_prof_backtrace_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + if (oldp == NULL && newp == NULL) { + ret = EINVAL; + goto label_return; + } + if (oldp != NULL) { + prof_backtrace_hook_t old_hook = + prof_backtrace_hook_get(); + READ(old_hook, prof_backtrace_hook_t); + } + if (newp != NULL) { + if (!opt_prof) { + ret = ENOENT; + goto label_return; + } + prof_backtrace_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL); + WRITE(new_hook, prof_backtrace_hook_t); + if (new_hook == NULL) { + ret = EINVAL; + goto label_return; + } + prof_backtrace_hook_set(new_hook); + } + ret = 0; +label_return: + return ret; +} + /******************************************************************************/ CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) diff --git a/src/prof.c b/src/prof.c index 67a7f71a..d0cae0e9 100644 --- a/src/prof.c +++ b/src/prof.c @@ -10,6 +10,7 @@ #include "jemalloc/internal/prof_recent.h" #include "jemalloc/internal/prof_stats.h" #include "jemalloc/internal/prof_sys.h" +#include "jemalloc/internal/prof_hook.h" #include "jemalloc/internal/thread_event.h" /* @@ -69,6 +70,9 @@ static malloc_mutex_t next_thr_uid_mtx; /* Do not dump any profiles until bootstrapping is complete. */ bool prof_booted = false; +/* Logically a prof_backtrace_hook_t. */ +atomic_p_t prof_backtrace_hook; + /******************************************************************************/ void @@ -518,6 +522,17 @@ prof_gdump_set(tsdn_t *tsdn, bool gdump) { return prof_gdump_old; } +void +prof_backtrace_hook_set(prof_backtrace_hook_t hook) { + atomic_store_p(&prof_backtrace_hook, hook, ATOMIC_RELEASE); +} + +prof_backtrace_hook_t +prof_backtrace_hook_get() { + return (prof_backtrace_hook_t)atomic_load_p(&prof_backtrace_hook, + ATOMIC_ACQUIRE); +} + void prof_boot0(void) { cassert(config_prof); @@ -657,6 +672,7 @@ prof_boot2(tsd_t *tsd, base_t *base) { } } + prof_hooks_init(); prof_unwind_init(); } prof_booted = true; diff --git a/src/prof_sys.c b/src/prof_sys.c index 6a5b2b16..1485e8b2 100644 --- a/src/prof_sys.c +++ b/src/prof_sys.c @@ -49,18 +49,18 @@ bt_init(prof_bt_t *bt, void **vec) { #ifdef JEMALLOC_PROF_LIBUNWIND static void -prof_backtrace_impl(prof_bt_t *bt) { +prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { int nframes; cassert(config_prof); - assert(bt->len == 0); - assert(bt->vec != NULL); + assert(*len == 0); + assert(vec != NULL); - nframes = unw_backtrace(bt->vec, PROF_BT_MAX); + nframes = unw_backtrace(vec, PROF_BT_MAX); if (nframes <= 0) { return; } - bt->len = nframes; + *len = nframes; } #elif (defined(JEMALLOC_PROF_LIBGCC)) static _Unwind_Reason_Code @@ -81,9 +81,9 @@ prof_unwind_callback(struct _Unwind_Context *context, void *arg) { if (ip == NULL) { return _URC_END_OF_STACK; } - data->bt->vec[data->bt->len] = ip; - data->bt->len++; - if (data->bt->len == data->max) { + data->vec[*data->len] = ip; + (*data->len)++; + if (*data->len == data->max) { return _URC_END_OF_STACK; } @@ -91,8 +91,8 @@ prof_unwind_callback(struct _Unwind_Context *context, void *arg) { } static void -prof_backtrace_impl(prof_bt_t *bt) { - prof_unwind_data_t data = {bt, PROF_BT_MAX}; +prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { + prof_unwind_data_t data = {vec, len, max_len}; cassert(config_prof); @@ -100,9 +100,9 @@ prof_backtrace_impl(prof_bt_t *bt) { } #elif (defined(JEMALLOC_PROF_GCC)) static void -prof_backtrace_impl(prof_bt_t *bt) { +prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { #define BT_FRAME(i) \ - if ((i) < PROF_BT_MAX) { \ + if ((i) < max_len) { \ void *p; \ if (__builtin_frame_address(i) == 0) { \ return; \ @@ -111,8 +111,8 @@ prof_backtrace_impl(prof_bt_t *bt) { if (p == NULL) { \ return; \ } \ - bt->vec[(i)] = p; \ - bt->len = (i) + 1; \ + vec[(i)] = p; \ + *len = (i) + 1; \ } else { \ return; \ } @@ -263,24 +263,28 @@ prof_backtrace_impl(prof_bt_t *bt) { } #else static void -prof_backtrace_impl(prof_bt_t *bt) { +prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { cassert(config_prof); not_reached(); } #endif - -void (* JET_MUTABLE prof_backtrace_hook)(prof_bt_t *bt) = &prof_backtrace_impl; - void prof_backtrace(tsd_t *tsd, prof_bt_t *bt) { cassert(config_prof); pre_reentrancy(tsd, NULL); - prof_backtrace_hook(bt); + prof_backtrace_hook_t prof_backtrace_hook = prof_backtrace_hook_get(); + prof_backtrace_hook(bt->vec, &bt->len, PROF_BT_MAX); post_reentrancy(tsd); } -void prof_unwind_init() { +void +prof_hooks_init() { + prof_backtrace_hook_set(&prof_backtrace_impl); +} + +void +prof_unwind_init() { #ifdef JEMALLOC_PROF_LIBGCC /* * Cause the backtracing machinery to allocate its internal diff --git a/test/analyze/prof_bias.c b/test/analyze/prof_bias.c index 0aae766b..4b960a66 100644 --- a/test/analyze/prof_bias.c +++ b/test/analyze/prof_bias.c @@ -24,12 +24,12 @@ */ static void -mock_backtrace(prof_bt_t *bt) { - bt->len = 4; - bt->vec[0] = (void *)0x111; - bt->vec[1] = (void *)0x222; - bt->vec[2] = (void *)0x333; - bt->vec[3] = (void *)0x444; +mock_backtrace(void **vec, unsigned *len, unsigned max_len) { + *len = 4; + vec[0] = (void *)0x111; + vec[1] = (void *)0x222; + vec[2] = (void *)0x333; + vec[3] = (void *)0x444; } static void @@ -50,7 +50,7 @@ main(void) { sizeof(lg_prof_sample)); assert(err == 0); - prof_backtrace_hook = &mock_backtrace; + prof_backtrace_hook_set(mock_backtrace); do_allocs(16, 32 * 1024 * 1024, /* do_frees */ true); do_allocs(32 * 1024* 1024, 16, /* do_frees */ true); do_allocs(16, 32 * 1024 * 1024, /* do_frees */ false); diff --git a/test/unit/prof_hook.c b/test/unit/prof_hook.c new file mode 100644 index 00000000..32d0e9ea --- /dev/null +++ b/test/unit/prof_hook.c @@ -0,0 +1,61 @@ +#include "test/jemalloc_test.h" + +bool mock_bt_hook_called = false; + +void +mock_bt_hook(void **vec, unsigned *len, unsigned max_len) { + *len = max_len; + for (unsigned i = 0; i < max_len; ++i) { + vec[i] = (void *)((uintptr_t)i); + } + mock_bt_hook_called = true; +} + +TEST_BEGIN(test_prof_backtrace_hook) { + + test_skip_if(!config_prof); + + mock_bt_hook_called = false; + + void *p0 = mallocx(1, 0); + assert_ptr_not_null(p0, "Failed to allocate"); + + expect_false(mock_bt_hook_called, "Called mock hook before it's set"); + + prof_backtrace_hook_t null_hook = NULL; + expect_d_eq(mallctl("experimental.hooks.prof_backtrace", + NULL, 0, (void *)&null_hook, sizeof(null_hook)), + EINVAL, "Incorrectly allowed NULL backtrace hook"); + + prof_backtrace_hook_t default_hook; + size_t default_hook_sz = sizeof(prof_backtrace_hook_t); + prof_backtrace_hook_t hook = &mock_bt_hook; + expect_d_eq(mallctl("experimental.hooks.prof_backtrace", + (void *)&default_hook, &default_hook_sz, (void *)&hook, + sizeof(hook)), 0, "Unexpected mallctl failure setting hook"); + + void *p1 = mallocx(1, 0); + assert_ptr_not_null(p1, "Failed to allocate"); + + expect_true(mock_bt_hook_called, "Didn't call mock hook"); + + prof_backtrace_hook_t current_hook; + size_t current_hook_sz = sizeof(prof_backtrace_hook_t); + expect_d_eq(mallctl("experimental.hooks.prof_backtrace", + (void *)¤t_hook, ¤t_hook_sz, (void *)&default_hook, + sizeof(default_hook)), 0, + "Unexpected mallctl failure resetting hook to default"); + + expect_ptr_eq(current_hook, hook, + "Hook returned by mallctl is not equal to mock hook"); + + dallocx(p1, 0); + dallocx(p0, 0); +} +TEST_END + +int +main(void) { + return test( + test_prof_backtrace_hook); +} diff --git a/test/unit/prof_hook.sh b/test/unit/prof_hook.sh new file mode 100644 index 00000000..d14cb8c5 --- /dev/null +++ b/test/unit/prof_hook.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,lg_prof_sample:0" +fi +