diff --git a/include/jemalloc/internal/prof_externs.h b/include/jemalloc/internal/prof_externs.h index d500f560..3d9fcfb8 100644 --- a/include/jemalloc/internal/prof_externs.h +++ b/include/jemalloc/internal/prof_externs.h @@ -102,7 +102,7 @@ void prof_postfork_child(tsdn_t *tsdn); /* Only accessed by thread event. */ uint64_t prof_sample_new_event_wait(tsd_t *tsd); uint64_t prof_sample_postponed_event_wait(tsd_t *tsd); -bool prof_idump_accum(tsdn_t *tsdn, uint64_t accumbytes); +void prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed); bool prof_log_start(tsdn_t *tsdn, const char *filename); bool prof_log_stop(tsdn_t *tsdn); diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index 4989fe1a..93bde22e 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -40,7 +40,7 @@ extern char opt_stats_interval_opts[stats_print_tot_num_options+1]; /* Only accessed by thread event. */ uint64_t stats_interval_new_event_wait(tsd_t *tsd); uint64_t stats_interval_postponed_event_wait(tsd_t *tsd); -bool stats_interval_accum(tsd_t *tsd, uint64_t bytes); +void stats_interval_event_handler(tsd_t *tsd, uint64_t elapsed); /* Implements je_malloc_stats_print. */ void stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts); diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 3be65286..6eca928c 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -56,9 +56,9 @@ void tcache_assert_initialized(tcache_t *tcache); /* Only accessed by thread event. */ uint64_t tcache_gc_new_event_wait(tsd_t *tsd); uint64_t tcache_gc_postponed_event_wait(tsd_t *tsd); +void tcache_gc_event_handler(tsd_t *tsd, uint64_t elapsed); uint64_t tcache_gc_dalloc_new_event_wait(tsd_t *tsd); uint64_t tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd); -void tcache_event_hard(tsd_t *tsd, tcache_slow_t *tcache_slow, - tcache_t *tcache); +void tcache_gc_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed); #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */ diff --git a/include/jemalloc/internal/thread_event.h b/include/jemalloc/internal/thread_event.h index 321baaac..3a848829 100644 --- a/include/jemalloc/internal/thread_event.h +++ b/include/jemalloc/internal/thread_event.h @@ -23,6 +23,12 @@ */ #define TE_MAX_INTERVAL ((uint64_t)(4U << 20)) +/* + * Invalid elapsed time, for situations where elapsed time is not needed. See + * comments in thread_event.c for more info. + */ +#define TE_INVALID_ELAPSED UINT64_MAX + typedef struct te_ctx_s { bool is_alloc; uint64_t *current; diff --git a/src/prof.c b/src/prof.c index 77aa44d4..c8da81da 100644 --- a/src/prof.c +++ b/src/prof.c @@ -50,7 +50,7 @@ bool opt_prof_accum = false; char opt_prof_prefix[PROF_DUMP_FILENAME_LEN]; bool opt_prof_experimental_use_sys_thread_name = false; -/* Accessed via prof_idump_accum(). */ +/* Accessed via prof_sample_event_handler(). */ static counter_accum_t prof_idump_accumulated; /* @@ -574,6 +574,18 @@ prof_sample_postponed_event_wait(tsd_t *tsd) { return prof_sample_new_event_wait(tsd); } +void +prof_sample_event_handler(tsd_t *tsd, uint64_t elapsed) { + cassert(config_prof); + assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED); + if (prof_interval == 0 || !prof_active_get_unlocked()) { + return; + } + if (counter_accum(tsd_tsdn(tsd), &prof_idump_accumulated, elapsed)) { + prof_idump(tsd_tsdn(tsd)); + } +} + int prof_getpid(void) { #ifdef _WIN32 @@ -658,17 +670,6 @@ prof_idump_accum_init(void) { return counter_accum_init(&prof_idump_accumulated, prof_interval); } -bool -prof_idump_accum(tsdn_t *tsdn, uint64_t accumbytes) { - cassert(config_prof); - - if (prof_interval == 0 || !prof_active_get_unlocked()) { - return false; - } - - return counter_accum(tsdn, &prof_idump_accumulated, accumbytes); -} - bool prof_dump_prefix_set(tsdn_t *tsdn, const char *prefix) { cassert(config_prof); diff --git a/src/stats.c b/src/stats.c index 16d4e88e..42e4a1ca 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1493,11 +1493,6 @@ stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { emitter_end(&emitter); } -bool -stats_interval_accum(tsd_t *tsd, uint64_t bytes) { - return counter_accum(tsd_tsdn(tsd), &stats_interval_accumulated, bytes); -} - uint64_t stats_interval_new_event_wait(tsd_t *tsd) { return stats_interval_accum_batch; @@ -1508,6 +1503,15 @@ stats_interval_postponed_event_wait(tsd_t *tsd) { return TE_MIN_START_WAIT; } +void +stats_interval_event_handler(tsd_t *tsd, uint64_t elapsed) { + assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED); + if (counter_accum(tsd_tsdn(tsd), &stats_interval_accumulated, + elapsed)) { + je_malloc_stats_print(NULL, NULL, opt_stats_interval_opts); + } +} + bool stats_boot(void) { uint64_t stats_interval; diff --git a/src/tcache.c b/src/tcache.c index 16c87cb0..f8188cb8 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -60,8 +60,14 @@ tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd) { return TE_MIN_START_WAIT; } -void -tcache_event_hard(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache) { +static void +tcache_event(tsd_t *tsd) { + tcache_t *tcache = tcache_get(tsd); + if (tcache == NULL) { + return; + } + + tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd); szind_t binind = tcache_slow->next_gc_bin; bool is_small = (binind < SC_NBINS); cache_bin_t *cache_bin = &tcache->bins[binind]; @@ -110,6 +116,18 @@ tcache_event_hard(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache) { } } +void +tcache_gc_event_handler(tsd_t *tsd, uint64_t elapsed) { + assert(elapsed == TE_INVALID_ELAPSED); + tcache_event(tsd); +} + +void +tcache_gc_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed) { + assert(elapsed == TE_INVALID_ELAPSED); + tcache_event(tsd); +} + void * tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, diff --git a/src/thread_event.c b/src/thread_event.c index 9de8376d..40c0487e 100644 --- a/src/thread_event.c +++ b/src/thread_event.c @@ -5,68 +5,59 @@ #include "jemalloc/internal/thread_event.h" /* - * Signatures for functions computing new / postponed event wait time. New + * Signatures for event specific functions. These functions should be defined + * by the modules owning each event. The signatures here verify that the + * definitions follow the right format. + * + * The first two are functions computing new / postponed event wait time. New * event wait time is the time till the next event if an event is currently * being triggered; postponed event wait time is the time till the next event * if an event should be triggered but needs to be postponed, e.g. when the TSD * is not nominal or during reentrancy. * - * These event wait time computation functions should be defined by the modules - * owning each event. The signatures here are used to verify that the - * definitions follow the right format. + * The third is the event handler function, which is called whenever an event + * is triggered. The parameter is the elapsed time since the last time an + * event of the same type was triggered. */ #define E(event, condition_unused, is_alloc_event_unused) \ uint64_t event##_new_event_wait(tsd_t *tsd); \ -uint64_t event##_postponed_event_wait(tsd_t *tsd); +uint64_t event##_postponed_event_wait(tsd_t *tsd); \ +void event##_event_handler(tsd_t *tsd, uint64_t elapsed); ITERATE_OVER_ALL_EVENTS #undef E -/* Event handler function signatures. */ +/* Signatures for internal functions fetching elapsed time. */ #define E(event, condition_unused, is_alloc_event_unused) \ -static void event##_event_handler(tsd_t *tsd); +static uint64_t event##_fetch_elapsed(tsd_t *tsd); ITERATE_OVER_ALL_EVENTS #undef E -/* Handler functions. */ -static void -tcache_gc_event(tsd_t *tsd) { - tcache_t *tcache = tcache_get(tsd); - if (tcache != NULL) { - tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd); - tcache_event_hard(tsd, tcache_slow, tcache); - } +static uint64_t +tcache_gc_fetch_elapsed(tsd_t *tsd) { + return TE_INVALID_ELAPSED; } -static void -tcache_gc_event_handler(tsd_t *tsd) { - tcache_gc_event(tsd); +static uint64_t +tcache_gc_dalloc_fetch_elapsed(tsd_t *tsd) { + return TE_INVALID_ELAPSED; } -static void -tcache_gc_dalloc_event_handler(tsd_t *tsd) { - tcache_gc_event(tsd); -} - -static void -prof_sample_event_handler(tsd_t *tsd) { +static uint64_t +prof_sample_fetch_elapsed(tsd_t *tsd) { uint64_t last_event = thread_allocated_last_event_get(tsd); uint64_t last_sample_event = prof_sample_last_event_get(tsd); prof_sample_last_event_set(tsd, last_event); - if (prof_idump_accum(tsd_tsdn(tsd), last_event - last_sample_event)) { - prof_idump(tsd_tsdn(tsd)); - } + return last_event - last_sample_event; } -static void -stats_interval_event_handler(tsd_t *tsd) { +static uint64_t +stats_interval_fetch_elapsed(tsd_t *tsd) { uint64_t last_event = thread_allocated_last_event_get(tsd); uint64_t last_stats_event = stats_interval_last_event_get(tsd); stats_interval_last_event_set(tsd, last_event); - if (stats_interval_accum(tsd, last_event - last_stats_event)) { - je_malloc_stats_print(NULL, NULL, opt_stats_interval_opts); - } + return last_event - last_stats_event; } /* Per event facilities done. */ @@ -286,7 +277,8 @@ te_event_trigger(tsd_t *tsd, te_ctx_t *ctx) { if (is_alloc == alloc_event && condition && \ is_##event##_triggered) { \ assert(allow_event_trigger); \ - event##_event_handler(tsd); \ + uint64_t elapsed = event##_fetch_elapsed(tsd); \ + event##_event_handler(tsd, elapsed); \ } ITERATE_OVER_ALL_EVENTS