From d82a164d0ddb5418de3b6a07dd302edddc347129 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Wed, 27 May 2020 14:31:00 -0700 Subject: [PATCH] Add thread.peak.[read|reset] mallctls. These can be used to track net allocator activity on a per-thread basis. --- Makefile.in | 1 + doc/jemalloc.xml.in | 36 ++++++++++ include/jemalloc/internal/peak_event.h | 24 +++++++ include/jemalloc/internal/thread_event.h | 10 +-- include/jemalloc/internal/tsd.h | 7 ++ .../projects/vc2015/jemalloc/jemalloc.vcxproj | 1 + .../vc2015/jemalloc/jemalloc.vcxproj.filters | 3 + .../projects/vc2017/jemalloc/jemalloc.vcxproj | 1 + .../vc2017/jemalloc/jemalloc.vcxproj.filters | 3 + src/ctl.c | 41 +++++++++++ src/peak_event.c | 67 ++++++++++++++++++ src/thread_event.c | 10 +++ test/unit/mallctl.c | 70 ++++++++++++++++++- 13 files changed, 269 insertions(+), 5 deletions(-) create mode 100644 include/jemalloc/internal/peak_event.h create mode 100644 src/peak_event.c diff --git a/Makefile.in b/Makefile.in index b211f889..2f3fea1e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -129,6 +129,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/pa.c \ $(srcroot)src/pa_extra.c \ $(srcroot)src/pages.c \ + $(srcroot)src/peak_event.c \ $(srcroot)src/prng.c \ $(srcroot)src/prof.c \ $(srcroot)src/prof_data.c \ diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 1baf1f6a..5ab84568 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1621,6 +1621,42 @@ malloc_conf = "xmalloc:true";]]> should not be modified by the application. + + + thread.peak.read + (uint64_t) + r- + [] + + Get an approximation of the maximum value of the + difference between the number of bytes allocated and the number of bytes + deallocated by the calling thread since the last call to thread.peak.reset, + or since the thread's creation if it has not called thread.peak.reset. + No guarantees are made about the quality of the approximation, but + jemalloc currently endeavors to maintain accuracy to within one hundred + kilobytes. + + + + + + thread.peak.reset + (void) + -- + [] + + Resets the counter for net bytes allocated in the calling + thread to zero. This affects subsequent calls to thread.peak.read, + but not the values returned by thread.allocated + or thread.deallocated. + + + thread.tcache.enabled diff --git a/include/jemalloc/internal/peak_event.h b/include/jemalloc/internal/peak_event.h new file mode 100644 index 00000000..b808ce04 --- /dev/null +++ b/include/jemalloc/internal/peak_event.h @@ -0,0 +1,24 @@ +#ifndef JEMALLOC_INTERNAL_PEAK_EVENT_H +#define JEMALLOC_INTERNAL_PEAK_EVENT_H + +/* + * While peak.h contains the simple helper struct that tracks state, this + * contains the allocator tie-ins (and knows about tsd, the event module, etc.). + */ + +/* Update the peak with current tsd state. */ +void peak_event_update(tsd_t *tsd); +/* Set current state to zero. */ +void peak_event_zero(tsd_t *tsd); +uint64_t peak_event_max(tsd_t *tsd); + +/* Manual hooks. */ +/* The activity-triggered hooks. */ +uint64_t peak_alloc_new_event_wait(tsd_t *tsd); +uint64_t peak_alloc_postponed_event_wait(tsd_t *tsd); +void peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed); +uint64_t peak_dalloc_new_event_wait(tsd_t *tsd); +uint64_t peak_dalloc_postponed_event_wait(tsd_t *tsd); +void peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed); + +#endif /* JEMALLOC_INTERNAL_PEAK_EVENT_H */ diff --git a/include/jemalloc/internal/thread_event.h b/include/jemalloc/internal/thread_event.h index 2fcaa88a..bca8a447 100644 --- a/include/jemalloc/internal/thread_event.h +++ b/include/jemalloc/internal/thread_event.h @@ -53,10 +53,12 @@ void tsd_te_init(tsd_t *tsd); * E(event, (condition), is_alloc_event) */ #define ITERATE_OVER_ALL_EVENTS \ - E(tcache_gc, (opt_tcache_gc_incr_bytes > 0), true) \ - E(prof_sample, (config_prof && opt_prof), true) \ - E(stats_interval, (opt_stats_interval >= 0), true) \ - E(tcache_gc_dalloc, (opt_tcache_gc_incr_bytes > 0), false) + E(tcache_gc, (opt_tcache_gc_incr_bytes > 0), true) \ + E(prof_sample, (config_prof && opt_prof), true) \ + E(stats_interval, (opt_stats_interval >= 0), true) \ + E(tcache_gc_dalloc, (opt_tcache_gc_incr_bytes > 0), false) \ + E(peak_alloc, config_stats, true) \ + E(peak_dalloc, config_stats, false) #define E(event, condition_unused, is_alloc_event_unused) \ C(event##_event_wait) diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 18bdb8fd..9408b2ca 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -5,6 +5,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/jemalloc_internal_externs.h" +#include "jemalloc/internal/peak.h" #include "jemalloc/internal/prof_types.h" #include "jemalloc/internal/ql.h" #include "jemalloc/internal/rtree_tsd.h" @@ -69,6 +70,8 @@ typedef ql_elm(tsd_t) tsd_link_t; O(prof_sample_last_event, uint64_t, uint64_t) \ O(stats_interval_event_wait, uint64_t, uint64_t) \ O(stats_interval_last_event, uint64_t, uint64_t) \ + O(peak_alloc_event_wait, uint64_t, uint64_t) \ + O(peak_dalloc_event_wait, uint64_t, uint64_t) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ O(prng_state, uint64_t, uint64_t) \ O(iarena, arena_t *, arena_t *) \ @@ -77,6 +80,7 @@ typedef ql_elm(tsd_t) tsd_link_t; O(binshards, tsd_binshards_t, tsd_binshards_t)\ O(tsd_link, tsd_link_t, tsd_link_t) \ O(in_hook, bool, bool) \ + O(peak, peak_t, peak_t) \ O(tcache_slow, tcache_slow_t, tcache_slow_t) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) @@ -95,6 +99,8 @@ typedef ql_elm(tsd_t) tsd_link_t; /* prof_sample_last_event */ 0, \ /* stats_interval_event_wait */ 0, \ /* stats_interval_last_event */ 0, \ + /* peak_alloc_event_wait */ 0, \ + /* peak_dalloc_event_wait */ 0, \ /* prof_tdata */ NULL, \ /* prng_state */ 0, \ /* iarena */ NULL, \ @@ -103,6 +109,7 @@ typedef ql_elm(tsd_t) tsd_link_t; /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ /* tsd_link */ {NULL}, \ /* in_hook */ false, \ + /* peak */ PEAK_INITIALIZER, \ /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \ /* rtree_ctx */ RTREE_CTX_ZERO_INITIALIZER, diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index 9f81e21d..d50fa884 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -70,6 +70,7 @@ + diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index 15fe7f08..94db8c0c 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -94,6 +94,9 @@ Source Files + + Source Files + Source Files diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index b5fccaed..337dcfe7 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -70,6 +70,7 @@ + diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters index 15fe7f08..94db8c0c 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -94,6 +94,9 @@ Source Files + + Source Files + Source Files diff --git a/src/ctl.c b/src/ctl.c index be8be10f..0bd38feb 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -9,6 +9,7 @@ #include "jemalloc/internal/inspect.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" +#include "jemalloc/internal/peak_event.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/util.h" @@ -61,6 +62,8 @@ CTL_PROTO(background_thread) CTL_PROTO(max_background_threads) CTL_PROTO(thread_tcache_enabled) CTL_PROTO(thread_tcache_flush) +CTL_PROTO(thread_peak_read) +CTL_PROTO(thread_peak_reset) CTL_PROTO(thread_prof_name) CTL_PROTO(thread_prof_active) CTL_PROTO(thread_arena) @@ -294,6 +297,11 @@ static const ctl_named_node_t thread_tcache_node[] = { {NAME("flush"), CTL(thread_tcache_flush)} }; +static const ctl_named_node_t thread_peak_node[] = { + {NAME("read"), CTL(thread_peak_read)}, + {NAME("reset"), CTL(thread_peak_reset)}, +}; + static const ctl_named_node_t thread_prof_node[] = { {NAME("name"), CTL(thread_prof_name)}, {NAME("active"), CTL(thread_prof_active)} @@ -306,6 +314,7 @@ static const ctl_named_node_t thread_node[] = { {NAME("deallocated"), CTL(thread_deallocated)}, {NAME("deallocatedp"), CTL(thread_deallocatedp)}, {NAME("tcache"), CHILD(named, thread_tcache)}, + {NAME("peak"), CHILD(named, thread_peak)}, {NAME("prof"), CHILD(named, thread_prof)}, {NAME("idle"), CTL(thread_idle)} }; @@ -1953,6 +1962,38 @@ label_return: return ret; } +static int +thread_peak_read_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { + int ret; + if (!config_stats) { + return ENOENT; + } + READONLY(); + peak_event_update(tsd); + uint64_t result = peak_event_max(tsd); + READ(result, uint64_t); + ret = 0; +label_return: + return ret; +} + +static int +thread_peak_reset_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { + int ret; + if (!config_stats) { + return ENOENT; + } + NEITHER_READ_NOR_WRITE(); + peak_event_zero(tsd); + ret = 0; +label_return: + return ret; +} + static int thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, diff --git a/src/peak_event.c b/src/peak_event.c new file mode 100644 index 00000000..ffb061bf --- /dev/null +++ b/src/peak_event.c @@ -0,0 +1,67 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/peak.h" +#include "jemalloc/internal/peak_event.h" + +/* + * Update every 100k by default. We're not exposing this as a configuration + * option for now; we don't want to bind ourselves too tightly to any particular + * performance requirements for small values, or guarantee that we'll even be + * able to provide fine-grained accuracy. + */ +#define PEAK_EVENT_WAIT (100 * 1024) + +/* Update the peak with current tsd state. */ +void +peak_event_update(tsd_t *tsd) { + uint64_t alloc = tsd_thread_allocated_get(tsd); + uint64_t dalloc = tsd_thread_deallocated_get(tsd); + peak_t *peak = tsd_peakp_get(tsd); + peak_update(peak, alloc, dalloc); +} + +/* Set current state to zero. */ +void +peak_event_zero(tsd_t *tsd) { + uint64_t alloc = tsd_thread_allocated_get(tsd); + uint64_t dalloc = tsd_thread_deallocated_get(tsd); + peak_t *peak = tsd_peakp_get(tsd); + peak_set_zero(peak, alloc, dalloc); +} + +uint64_t +peak_event_max(tsd_t *tsd) { + peak_t *peak = tsd_peakp_get(tsd); + return peak_max(peak); +} + +uint64_t +peak_alloc_new_event_wait(tsd_t *tsd) { + return PEAK_EVENT_WAIT; +} + +uint64_t +peak_alloc_postponed_event_wait(tsd_t *tsd) { + return TE_MIN_START_WAIT; +} + +void +peak_alloc_event_handler(tsd_t *tsd, uint64_t elapsed) { + peak_event_update(tsd); +} + +uint64_t +peak_dalloc_new_event_wait(tsd_t *tsd) { + return PEAK_EVENT_WAIT; +} + +uint64_t +peak_dalloc_postponed_event_wait(tsd_t *tsd) { + return TE_MIN_START_WAIT; +} + +void +peak_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed) { + peak_event_update(tsd); +} diff --git a/src/thread_event.c b/src/thread_event.c index 40c0487e..99a188dd 100644 --- a/src/thread_event.c +++ b/src/thread_event.c @@ -60,6 +60,16 @@ stats_interval_fetch_elapsed(tsd_t *tsd) { return last_event - last_stats_event; } +static uint64_t +peak_alloc_fetch_elapsed(tsd_t *tsd) { + return TE_INVALID_ELAPSED; +} + +static uint64_t +peak_dalloc_fetch_elapsed(tsd_t *tsd) { + return TE_INVALID_ELAPSED; +} + /* Per event facilities done. */ static bool diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index cc1d5313..10d809fb 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -955,6 +955,73 @@ TEST_BEGIN(test_thread_idle) { } TEST_END +TEST_BEGIN(test_thread_peak) { + test_skip_if(!config_stats); + + /* + * We don't commit to any stable amount of accuracy for peak tracking + * (in practice, when this test was written, we made sure to be within + * 100k). But 10MB is big for more or less any definition of big. + */ + size_t big_size = 10 * 1024 * 1024; + size_t small_size = 256; + + void *ptr; + int err; + size_t sz; + uint64_t peak; + sz = sizeof(uint64_t); + + err = mallctl("thread.peak.reset", NULL, NULL, NULL, 0); + expect_d_eq(err, 0, ""); + ptr = mallocx(SC_SMALL_MAXCLASS, 0); + err = mallctl("thread.peak.read", &peak, &sz, NULL, 0); + expect_d_eq(err, 0, ""); + expect_u64_eq(peak, SC_SMALL_MAXCLASS, "Missed an update"); + free(ptr); + err = mallctl("thread.peak.read", &peak, &sz, NULL, 0); + expect_d_eq(err, 0, ""); + expect_u64_eq(peak, SC_SMALL_MAXCLASS, "Freeing changed peak"); + ptr = mallocx(big_size, 0); + free(ptr); + /* + * The peak should have hit big_size in the last two lines, even though + * the net allocated bytes has since dropped back down to zero. We + * should have noticed the peak change without having down any mallctl + * calls while net allocated bytes was high. + */ + err = mallctl("thread.peak.read", &peak, &sz, NULL, 0); + expect_d_eq(err, 0, ""); + expect_u64_ge(peak, big_size, "Missed a peak change."); + + /* Allocate big_size, but using small allocations. */ + size_t nallocs = big_size / small_size; + void **ptrs = calloc(nallocs, sizeof(void *)); + err = mallctl("thread.peak.reset", NULL, NULL, NULL, 0); + expect_d_eq(err, 0, ""); + err = mallctl("thread.peak.read", &peak, &sz, NULL, 0); + expect_d_eq(err, 0, ""); + expect_u64_eq(0, peak, "Missed a reset."); + for (size_t i = 0; i < nallocs; i++) { + ptrs[i] = mallocx(small_size, 0); + } + for (size_t i = 0; i < nallocs; i++) { + free(ptrs[i]); + } + err = mallctl("thread.peak.read", &peak, &sz, NULL, 0); + expect_d_eq(err, 0, ""); + /* + * We don't guarantee exactness; make sure we're within 10% of the peak, + * though. + */ + expect_u64_ge(peak, nallocx(small_size, 0) * nallocs * 9 / 10, + "Missed some peak changes."); + expect_u64_le(peak, nallocx(small_size, 0) * nallocs * 11 / 10, + "Overcounted peak changes."); + free(ptrs); +} +TEST_END + int main(void) { return test( @@ -987,5 +1054,6 @@ main(void) { test_stats_arenas, test_hooks, test_hooks_exhaustion, - test_thread_idle); + test_thread_idle, + test_thread_peak); }