From a0734fd6ee326cd2059edbe4bca7092988a63684 Mon Sep 17 00:00:00 2001 From: Guangli Dai Date: Fri, 19 Aug 2022 12:17:10 -0700 Subject: [PATCH] Making jemalloc max stack depth a runtime option --- include/jemalloc/internal/prof_externs.h | 1 + include/jemalloc/internal/prof_structs.h | 2 +- include/jemalloc/internal/prof_types.h | 7 +- src/ctl.c | 3 + src/jemalloc.c | 3 + src/prof.c | 1 + src/prof_data.c | 7 +- src/prof_sys.c | 152 ++++++++++++++++++++++- src/stats.c | 1 + test/unit/mallctl.c | 1 + test/unit/prof_hook.c | 1 + test/unit/prof_hook.sh | 3 +- 12 files changed, 171 insertions(+), 11 deletions(-) diff --git a/include/jemalloc/internal/prof_externs.h b/include/jemalloc/internal/prof_externs.h index bdff1349..d1101561 100644 --- a/include/jemalloc/internal/prof_externs.h +++ b/include/jemalloc/internal/prof_externs.h @@ -7,6 +7,7 @@ extern bool opt_prof; extern bool opt_prof_active; extern bool opt_prof_thread_active_init; +extern unsigned opt_prof_bt_max; extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ extern bool opt_prof_gdump; /* High-water memory dumping. */ diff --git a/include/jemalloc/internal/prof_structs.h b/include/jemalloc/internal/prof_structs.h index dd22115f..9331fba4 100644 --- a/include/jemalloc/internal/prof_structs.h +++ b/include/jemalloc/internal/prof_structs.h @@ -202,7 +202,7 @@ struct prof_tdata_s { prof_cnt_t cnt_summed; /* Backtrace vector, used for calls to prof_backtrace(). */ - void *vec[PROF_BT_MAX]; + void **vec; }; typedef rb_tree(prof_tdata_t) prof_tdata_tree_t; diff --git a/include/jemalloc/internal/prof_types.h b/include/jemalloc/internal/prof_types.h index ba628654..87cbb4ab 100644 --- a/include/jemalloc/internal/prof_types.h +++ b/include/jemalloc/internal/prof_types.h @@ -23,7 +23,12 @@ typedef struct prof_recent_s prof_recent_t; * is based on __builtin_return_address() necessarily has a hard-coded number * of backtrace frame handlers, and should be kept in sync with this setting. */ -#define PROF_BT_MAX 128 +#ifdef JEMALLOC_PROF_GCC +# define PROF_BT_MAX_LIMIT 256 +#else +# define PROF_BT_MAX_LIMIT UINT_MAX +#endif +#define PROF_BT_MAX_DEFAULT 128 /* Initial hash table size. */ #define PROF_CKH_MINITEMS 64 diff --git a/src/ctl.c b/src/ctl.c index e942cb1a..6b03f986 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -142,6 +142,7 @@ CTL_PROTO(opt_prof) CTL_PROTO(opt_prof_prefix) CTL_PROTO(opt_prof_active) CTL_PROTO(opt_prof_thread_active_init) +CTL_PROTO(opt_prof_bt_max) CTL_PROTO(opt_lg_prof_sample) CTL_PROTO(opt_lg_prof_interval) CTL_PROTO(opt_prof_gdump) @@ -468,6 +469,7 @@ static const ctl_named_node_t opt_node[] = { {NAME("prof_prefix"), CTL(opt_prof_prefix)}, {NAME("prof_active"), CTL(opt_prof_active)}, {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, + {NAME("prof_bt_max"), CTL(opt_prof_bt_max)}, {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, {NAME("prof_gdump"), CTL(opt_prof_gdump)}, @@ -2205,6 +2207,7 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, opt_prof_thread_active_init, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_bt_max, opt_prof_bt_max, unsigned) CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) diff --git a/src/jemalloc.c b/src/jemalloc.c index 83d69dd0..a4761c9b 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1585,6 +1585,9 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, true) CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") + CONF_HANDLE_UNSIGNED(opt_prof_bt_max, "prof_bt_max", + 1, PROF_BT_MAX_LIMIT, CONF_CHECK_MIN, CONF_CHECK_MAX, + /* clip */ true) CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, "lg_prof_interval", -1, (sizeof(uint64_t) << 3) - 1) diff --git a/src/prof.c b/src/prof.c index 7a6d5d56..3deac0b5 100644 --- a/src/prof.c +++ b/src/prof.c @@ -26,6 +26,7 @@ bool opt_prof = false; bool opt_prof_active = true; bool opt_prof_thread_active_init = true; +unsigned opt_prof_bt_max = PROF_BT_MAX_DEFAULT; size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; bool opt_prof_gdump = false; diff --git a/src/prof_data.c b/src/prof_data.c index bfa55be1..f8b19594 100644 --- a/src/prof_data.c +++ b/src/prof_data.c @@ -1167,13 +1167,16 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, cassert(config_prof); /* Initialize an empty cache for this thread. */ - tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t), - sz_size2index(sizeof(prof_tdata_t)), false, NULL, true, + size_t tdata_sz = ALIGNMENT_CEILING(sizeof(prof_tdata_t), QUANTUM); + size_t total_sz = tdata_sz + sizeof(void *) * opt_prof_bt_max; + tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), + total_sz, sz_size2index(total_sz), false, NULL, true, arena_get(TSDN_NULL, 0, true), true); if (tdata == NULL) { return NULL; } + tdata->vec = (void **)((uintptr_t)tdata + tdata_sz); tdata->lock = prof_tdata_mutex_choose(thr_uid); tdata->thr_uid = thr_uid; tdata->thr_discrim = thr_discrim; diff --git a/src/prof_sys.c b/src/prof_sys.c index b5f1f5b2..99fa3a77 100644 --- a/src/prof_sys.c +++ b/src/prof_sys.c @@ -55,9 +55,9 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { cassert(config_prof); assert(*len == 0); assert(vec != NULL); - assert(max_len == PROF_BT_MAX); + assert(max_len <= PROF_BT_MAX_LIMIT); - nframes = unw_backtrace(vec, PROF_BT_MAX); + nframes = unw_backtrace(vec, max_len); if (nframes <= 0) { return; } @@ -97,13 +97,14 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { cassert(config_prof); assert(vec != NULL); - assert(max_len == PROF_BT_MAX); + assert(max_len <= PROF_BT_MAX_LIMIT); _Unwind_Backtrace(prof_unwind_callback, &data); } #elif (defined(JEMALLOC_PROF_GCC)) static void prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { +/* The input arg must be a constant for __builtin_return_address. */ #define BT_FRAME(i) \ if ((i) < max_len) { \ void *p; \ @@ -122,7 +123,7 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { cassert(config_prof); assert(vec != NULL); - assert(max_len == PROF_BT_MAX); + assert(max_len <= PROF_BT_MAX_LIMIT); BT_FRAME(0) BT_FRAME(1) @@ -264,6 +265,147 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) { BT_FRAME(125) BT_FRAME(126) BT_FRAME(127) + BT_FRAME(128) + BT_FRAME(129) + + BT_FRAME(130) + BT_FRAME(131) + BT_FRAME(132) + BT_FRAME(133) + BT_FRAME(134) + BT_FRAME(135) + BT_FRAME(136) + BT_FRAME(137) + BT_FRAME(138) + BT_FRAME(139) + + BT_FRAME(140) + BT_FRAME(141) + BT_FRAME(142) + BT_FRAME(143) + BT_FRAME(144) + BT_FRAME(145) + BT_FRAME(146) + BT_FRAME(147) + BT_FRAME(148) + BT_FRAME(149) + + BT_FRAME(150) + BT_FRAME(151) + BT_FRAME(152) + BT_FRAME(153) + BT_FRAME(154) + BT_FRAME(155) + BT_FRAME(156) + BT_FRAME(157) + BT_FRAME(158) + BT_FRAME(159) + + BT_FRAME(160) + BT_FRAME(161) + BT_FRAME(162) + BT_FRAME(163) + BT_FRAME(164) + BT_FRAME(165) + BT_FRAME(166) + BT_FRAME(167) + BT_FRAME(168) + BT_FRAME(169) + + BT_FRAME(170) + BT_FRAME(171) + BT_FRAME(172) + BT_FRAME(173) + BT_FRAME(174) + BT_FRAME(175) + BT_FRAME(176) + BT_FRAME(177) + BT_FRAME(178) + BT_FRAME(179) + + BT_FRAME(180) + BT_FRAME(181) + BT_FRAME(182) + BT_FRAME(183) + BT_FRAME(184) + BT_FRAME(185) + BT_FRAME(186) + BT_FRAME(187) + BT_FRAME(188) + BT_FRAME(189) + + BT_FRAME(190) + BT_FRAME(191) + BT_FRAME(192) + BT_FRAME(193) + BT_FRAME(194) + BT_FRAME(195) + BT_FRAME(196) + BT_FRAME(197) + BT_FRAME(198) + BT_FRAME(199) + + BT_FRAME(200) + BT_FRAME(201) + BT_FRAME(202) + BT_FRAME(203) + BT_FRAME(204) + BT_FRAME(205) + BT_FRAME(206) + BT_FRAME(207) + BT_FRAME(208) + BT_FRAME(209) + + BT_FRAME(210) + BT_FRAME(211) + BT_FRAME(212) + BT_FRAME(213) + BT_FRAME(214) + BT_FRAME(215) + BT_FRAME(216) + BT_FRAME(217) + BT_FRAME(218) + BT_FRAME(219) + + BT_FRAME(220) + BT_FRAME(221) + BT_FRAME(222) + BT_FRAME(223) + BT_FRAME(224) + BT_FRAME(225) + BT_FRAME(226) + BT_FRAME(227) + BT_FRAME(228) + BT_FRAME(229) + + BT_FRAME(230) + BT_FRAME(231) + BT_FRAME(232) + BT_FRAME(233) + BT_FRAME(234) + BT_FRAME(235) + BT_FRAME(236) + BT_FRAME(237) + BT_FRAME(238) + BT_FRAME(239) + + BT_FRAME(240) + BT_FRAME(241) + BT_FRAME(242) + BT_FRAME(243) + BT_FRAME(244) + BT_FRAME(245) + BT_FRAME(246) + BT_FRAME(247) + BT_FRAME(248) + BT_FRAME(249) + + BT_FRAME(250) + BT_FRAME(251) + BT_FRAME(252) + BT_FRAME(253) + BT_FRAME(254) + BT_FRAME(255) #undef BT_FRAME } #else @@ -281,7 +423,7 @@ prof_backtrace(tsd_t *tsd, prof_bt_t *bt) { assert(prof_backtrace_hook != NULL); pre_reentrancy(tsd, NULL); - prof_backtrace_hook(bt->vec, &bt->len, PROF_BT_MAX); + prof_backtrace_hook(bt->vec, &bt->len, opt_prof_bt_max); post_reentrancy(tsd); } diff --git a/src/stats.c b/src/stats.c index d150baef..5bb1a346 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1521,6 +1521,7 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_UNSIGNED("debug_double_free_max_scan") OPT_WRITE_CHAR_P("thp") OPT_WRITE_BOOL("prof") + OPT_WRITE_UNSIGNED("prof_bt_max") OPT_WRITE_CHAR_P("prof_prefix") OPT_WRITE_BOOL_MUTABLE("prof_active", "prof.active") OPT_WRITE_BOOL_MUTABLE("prof_thread_active_init", diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 62bd1a2d..14fe7993 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -314,6 +314,7 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(bool, prof, prof); TEST_MALLCTL_OPT(const char *, prof_prefix, prof); TEST_MALLCTL_OPT(bool, prof_active, prof); + TEST_MALLCTL_OPT(unsigned, prof_bt_max, prof); TEST_MALLCTL_OPT(ssize_t, lg_prof_sample, prof); TEST_MALLCTL_OPT(bool, prof_accum, prof); TEST_MALLCTL_OPT(ssize_t, lg_prof_interval, prof); diff --git a/test/unit/prof_hook.c b/test/unit/prof_hook.c index 6480d930..fc06d84e 100644 --- a/test/unit/prof_hook.c +++ b/test/unit/prof_hook.c @@ -129,6 +129,7 @@ TEST_END TEST_BEGIN(test_prof_dump_hook) { test_skip_if(!config_prof); + expect_u_eq(opt_prof_bt_max, 200, "Unexpected backtrace stack depth"); mock_dump_hook_called = false; diff --git a/test/unit/prof_hook.sh b/test/unit/prof_hook.sh index c7ebd8f9..48cd51a5 100644 --- a/test/unit/prof_hook.sh +++ b/test/unit/prof_hook.sh @@ -1,6 +1,5 @@ #!/bin/sh if [ "x${enable_prof}" = "x1" ] ; then - export MALLOC_CONF="prof:true,prof_active:true,lg_prof_sample:0" + export MALLOC_CONF="prof:true,prof_active:true,lg_prof_sample:0,prof_bt_max:200" fi -