diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index 40c4286c..d356181c 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -5,7 +5,6 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" -#include "jemalloc/internal/ticker.h" #include "jemalloc/internal/util.h" static inline bool @@ -27,17 +26,6 @@ tcache_enabled_set(tsd_t *tsd, bool enabled) { tsd_slow_update(tsd); } -JEMALLOC_ALWAYS_INLINE void -tcache_event(tsd_t *tsd, tcache_t *tcache) { - if (TCACHE_GC_INCR == 0) { - return; - } - - if (unlikely(ticker_tick(&tcache->gc_ticker))) { - tcache_event_hard(tsd, tcache); - } -} - JEMALLOC_ALWAYS_INLINE void * tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, szind_t binind, bool zero, bool slow_path) { @@ -171,8 +159,6 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); } - - tcache_event(tsd, tcache); } JEMALLOC_ALWAYS_INLINE void @@ -195,8 +181,6 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); } - - tcache_event(tsd, tcache); } JEMALLOC_ALWAYS_INLINE tcache_t * diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index 98d3ef70..38a82fe4 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -16,9 +16,6 @@ struct tcache_s { * together at the start of this struct. */ - /* Drives incremental GC. */ - ticker_t gc_ticker; - /* * The pointer stacks associated with bins follow as a contiguous array. * During tcache initialization, the avail pointer in each element of diff --git a/include/jemalloc/internal/thread_event.h b/include/jemalloc/internal/thread_event.h index 454c689b..33cbcbe7 100644 --- a/include/jemalloc/internal/thread_event.h +++ b/include/jemalloc/internal/thread_event.h @@ -4,42 +4,51 @@ #include "jemalloc/internal/tsd.h" /* - * Maximum threshold on thread_allocated_next_event_fast, so that there is no - * need to check overflow in malloc fast path. (The allocation size in malloc + * Maximum threshold on thread_(de)allocated_next_event_fast, so that there is + * no need to check overflow in malloc fast path. (The allocation size in malloc * fast path never exceeds SC_LOOKUP_MAXCLASS.) */ -#define THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX \ +#define THREAD_NEXT_EVENT_FAST_MAX \ (UINT64_MAX - SC_LOOKUP_MAXCLASS + 1U) /* * The max interval helps make sure that malloc stays on the fast path in the - * common case, i.e. thread_allocated < thread_allocated_next_event_fast. - * When thread_allocated is within an event's distance to - * THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX above, thread_allocated_next_event_fast - * is wrapped around and we fall back to the medium-fast path. The max interval - * makes sure that we're not staying on the fallback case for too long, even if - * there's no active event or if all active events have long wait times. + * common case, i.e. thread_allocated < thread_allocated_next_event_fast. When + * thread_allocated is within an event's distance to THREAD_NEXT_EVENT_FAST_MAX + * above, thread_allocated_next_event_fast is wrapped around and we fall back to + * the medium-fast path. The max interval makes sure that we're not staying on + * the fallback case for too long, even if there's no active event or if all + * active events have long wait times. */ #define THREAD_EVENT_MAX_INTERVAL ((uint64_t)(4U << 20)) +typedef struct event_ctx_s { + bool is_alloc; + uint64_t *current; + uint64_t *last_event; + uint64_t *next_event; + uint64_t *next_event_fast; +} event_ctx_t; + void thread_event_assert_invariants_debug(tsd_t *tsd); -void thread_event_trigger(tsd_t *tsd, bool delay_event); -void thread_event_rollback(tsd_t *tsd, size_t diff); -void thread_event_update(tsd_t *tsd); +void thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event); +void thread_alloc_event_rollback(tsd_t *tsd, size_t diff); +void thread_event_update(tsd_t *tsd, bool alloc_event); void thread_event_boot(); void thread_event_recompute_fast_threshold(tsd_t *tsd); void tsd_thread_event_init(tsd_t *tsd); /* * List of all events, in the following format: - * E(event, (condition)) + * E(event, (condition), is_alloc_event) */ #define ITERATE_OVER_ALL_EVENTS \ - E(tcache_gc, (TCACHE_GC_INCR_BYTES > 0)) \ - E(prof_sample, (config_prof && opt_prof)) \ - E(stats_interval, (opt_stats_interval >= 0)) + E(tcache_gc, (TCACHE_GC_INCR_BYTES > 0), true) \ + E(prof_sample, (config_prof && opt_prof), true) \ + E(stats_interval, (opt_stats_interval >= 0), true) \ + E(tcache_gc_dalloc, (TCACHE_GC_INCR_BYTES > 0), false) -#define E(event, condition) \ +#define E(event, condition_unused, is_alloc_event_unused) \ C(event##_event_wait) /* List of all thread event counters. */ @@ -83,9 +92,9 @@ ITERATE_OVER_ALL_COUNTERS #undef E /* - * Two malloc fastpath getters -- use the unsafe getters since tsd may be - * non-nominal, in which case the fast_threshold will be set to 0. This allows - * checking for events and tsd non-nominal in a single branch. + * The malloc and free fastpath getters -- use the unsafe getters since tsd may + * be non-nominal, in which case the fast_threshold will be set to 0. This + * allows checking for events and tsd non-nominal in a single branch. * * Note that these can only be used on the fastpath. */ @@ -97,42 +106,83 @@ thread_allocated_malloc_fastpath(tsd_t *tsd) { JEMALLOC_ALWAYS_INLINE uint64_t thread_allocated_next_event_malloc_fastpath(tsd_t *tsd) { uint64_t v = *tsd_thread_allocated_next_event_fastp_get_unsafe(tsd); - assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX); + assert(v <= THREAD_NEXT_EVENT_FAST_MAX); return v; } +JEMALLOC_ALWAYS_INLINE void +thread_event_free_fastpath_ctx(tsd_t *tsd, uint64_t *deallocated, + uint64_t *threshold, bool size_hint) { + if (!size_hint) { + *deallocated = tsd_thread_deallocated_get(tsd); + *threshold = tsd_thread_deallocated_next_event_fast_get(tsd); + } else { + /* Unsafe getters since this may happen before tsd_init. */ + *deallocated = *tsd_thread_deallocatedp_get_unsafe(tsd); + *threshold = + *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd); + } + assert(*threshold <= THREAD_NEXT_EVENT_FAST_MAX); +} + +JEMALLOC_ALWAYS_INLINE bool +event_ctx_is_alloc(event_ctx_t *ctx) { + return ctx->is_alloc; +} + +JEMALLOC_ALWAYS_INLINE uint64_t +event_ctx_current_bytes_get(event_ctx_t *ctx) { + return *ctx->current; +} + +JEMALLOC_ALWAYS_INLINE void +event_ctx_current_bytes_set(event_ctx_t *ctx, uint64_t v) { + *ctx->current = v; +} + +JEMALLOC_ALWAYS_INLINE uint64_t +event_ctx_last_event_get(event_ctx_t *ctx) { + return *ctx->last_event; +} + +JEMALLOC_ALWAYS_INLINE void +event_ctx_last_event_set(event_ctx_t *ctx, uint64_t v) { + *ctx->last_event = v; +} + /* Below 3 for next_event_fast. */ JEMALLOC_ALWAYS_INLINE uint64_t -thread_allocated_next_event_fast_get(tsd_t *tsd) { - uint64_t v = tsd_thread_allocated_next_event_fast_get(tsd); - assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX); +event_ctx_next_event_fast_get(event_ctx_t *ctx) { + uint64_t v = *ctx->next_event_fast; + assert(v <= THREAD_NEXT_EVENT_FAST_MAX); return v; } JEMALLOC_ALWAYS_INLINE void -thread_allocated_next_event_fast_set(tsd_t *tsd, uint64_t v) { - assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX); - *tsd_thread_allocated_next_event_fastp_get(tsd) = v; +event_ctx_next_event_fast_set(event_ctx_t *ctx, uint64_t v) { + assert(v <= THREAD_NEXT_EVENT_FAST_MAX); + *ctx->next_event_fast = v; } JEMALLOC_ALWAYS_INLINE void -thread_allocated_next_event_fast_set_non_nominal(tsd_t *tsd) { +thread_next_event_fast_set_non_nominal(tsd_t *tsd) { /* - * Set the fast threshold to zero when tsd is non-nominal. Use the + * Set the fast thresholds to zero when tsd is non-nominal. Use the * unsafe getter as this may get called during tsd init and clean up. */ *tsd_thread_allocated_next_event_fastp_get_unsafe(tsd) = 0; + *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) = 0; } /* For next_event. Setter also updates the fast threshold. */ JEMALLOC_ALWAYS_INLINE uint64_t -thread_allocated_next_event_get(tsd_t *tsd) { - return tsd_thread_allocated_next_event_get(tsd); +event_ctx_next_event_get(event_ctx_t *ctx) { + return *ctx->next_event; } JEMALLOC_ALWAYS_INLINE void -thread_allocated_next_event_set(tsd_t *tsd, uint64_t v) { - *tsd_thread_allocated_next_eventp_get(tsd) = v; +event_ctx_next_event_set(tsd_t *tsd, event_ctx_t *ctx, uint64_t v) { + *ctx->next_event = v; thread_event_recompute_fast_threshold(tsd); } @@ -145,8 +195,8 @@ thread_allocated_next_event_set(tsd_t *tsd, uint64_t v) { * at the end will restore the invariants), * (b) thread_##event##_event_update() (the thread_event_update() call at the * end will restore the invariants), or - * (c) thread_event_rollback() if the rollback falls below the last_event (the - * thread_event_update() call at the end will restore the invariants). + * (c) thread_alloc_event_rollback() if the rollback falls below the last_event + * (the thread_event_update() call at the end will restore the invariants). */ JEMALLOC_ALWAYS_INLINE void thread_event_assert_invariants(tsd_t *tsd) { @@ -156,22 +206,52 @@ thread_event_assert_invariants(tsd_t *tsd) { } JEMALLOC_ALWAYS_INLINE void -thread_event(tsd_t *tsd, size_t usize) { - thread_event_assert_invariants(tsd); - - uint64_t thread_allocated_before = thread_allocated_get(tsd); - thread_allocated_set(tsd, thread_allocated_before + usize); - - /* The subtraction is intentionally susceptible to underflow. */ - if (likely(usize < thread_allocated_next_event_get(tsd) - - thread_allocated_before)) { - thread_event_assert_invariants(tsd); +event_ctx_get(tsd_t *tsd, event_ctx_t *ctx, bool is_alloc) { + ctx->is_alloc = is_alloc; + if (is_alloc) { + ctx->current = tsd_thread_allocatedp_get(tsd); + ctx->last_event = tsd_thread_allocated_last_eventp_get(tsd); + ctx->next_event = tsd_thread_allocated_next_eventp_get(tsd); + ctx->next_event_fast = + tsd_thread_allocated_next_event_fastp_get(tsd); } else { - thread_event_trigger(tsd, false); + ctx->current = tsd_thread_deallocatedp_get(tsd); + ctx->last_event = tsd_thread_deallocated_last_eventp_get(tsd); + ctx->next_event = tsd_thread_deallocated_next_eventp_get(tsd); + ctx->next_event_fast = + tsd_thread_deallocated_next_event_fastp_get(tsd); } } -#define E(event, condition) \ +JEMALLOC_ALWAYS_INLINE void +thread_event_advance(tsd_t *tsd, size_t usize, bool is_alloc) { + thread_event_assert_invariants(tsd); + + event_ctx_t ctx; + event_ctx_get(tsd, &ctx, is_alloc); + + uint64_t bytes_before = event_ctx_current_bytes_get(&ctx); + event_ctx_current_bytes_set(&ctx, bytes_before + usize); + + /* The subtraction is intentionally susceptible to underflow. */ + if (likely(usize < event_ctx_next_event_get(&ctx) - bytes_before)) { + thread_event_assert_invariants(tsd); + } else { + thread_event_trigger(tsd, &ctx, false); + } +} + +JEMALLOC_ALWAYS_INLINE void +thread_dalloc_event(tsd_t *tsd, size_t usize) { + thread_event_advance(tsd, usize, false); +} + +JEMALLOC_ALWAYS_INLINE void +thread_alloc_event(tsd_t *tsd, size_t usize) { + thread_event_advance(tsd, usize, true); +} + +#define E(event, condition, is_alloc) \ JEMALLOC_ALWAYS_INLINE void \ thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) { \ thread_event_assert_invariants(tsd); \ @@ -188,7 +268,7 @@ thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) { \ event_wait = THREAD_EVENT_MAX_START_WAIT; \ } \ event##_event_wait_set(tsd, event_wait); \ - thread_event_update(tsd); \ + thread_event_update(tsd, is_alloc); \ } ITERATE_OVER_ALL_EVENTS diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index a62793aa..6868ce4b 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -81,10 +81,14 @@ typedef void (*test_callback_t)(int *); O(thread_allocated, uint64_t, uint64_t) \ O(thread_allocated_next_event_fast, uint64_t, uint64_t) \ O(thread_deallocated, uint64_t, uint64_t) \ + O(thread_deallocated_next_event_fast, uint64_t, uint64_t) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \ O(thread_allocated_last_event, uint64_t, uint64_t) \ O(thread_allocated_next_event, uint64_t, uint64_t) \ + O(thread_deallocated_last_event, uint64_t, uint64_t) \ + O(thread_deallocated_next_event, uint64_t, uint64_t) \ O(tcache_gc_event_wait, uint64_t, uint64_t) \ + O(tcache_gc_dalloc_event_wait, uint64_t, uint64_t) \ O(prof_sample_event_wait, uint64_t, uint64_t) \ O(prof_sample_last_event, uint64_t, uint64_t) \ O(stats_interval_event_wait, uint64_t, uint64_t) \ @@ -114,10 +118,14 @@ typedef void (*test_callback_t)(int *); /* thread_allocated */ 0, \ /* thread_allocated_next_event_fast */ 0, \ /* thread_deallocated */ 0, \ + /* thread_deallocated_next_event_fast */ 0, \ /* rtree_ctx */ RTREE_CTX_ZERO_INITIALIZER, \ /* thread_allocated_last_event */ 0, \ /* thread_allocated_next_event */ THREAD_EVENT_MIN_START_WAIT, \ + /* thread_deallocated_last_event */ 0, \ + /* thread_deallocated_next_event */ THREAD_EVENT_MIN_START_WAIT, \ /* tcache_gc_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ + /* tcache_gc_dalloc_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ /* prof_sample_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ /* prof_sample_last_event */ 0, \ /* stats_interval_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ diff --git a/src/jemalloc.c b/src/jemalloc.c index 5f11fc38..60565df7 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2154,7 +2154,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { dopts->arena_ind = 0; } - thread_event(tsd, usize); + thread_alloc_event(tsd, usize); /* * If dopts->alignment > 0, then ind is still 0, but usize was computed @@ -2181,7 +2181,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { } if (unlikely(allocation == NULL)) { - thread_event_rollback(tsd, usize); + thread_alloc_event_rollback(tsd, usize); prof_alloc_rollback(tsd, tctx, true); goto label_oom; } @@ -2191,7 +2191,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize, ind); if (unlikely(allocation == NULL)) { - thread_event_rollback(tsd, usize); + thread_alloc_event_rollback(tsd, usize); goto label_oom; } } @@ -2575,7 +2575,6 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { if (config_prof && opt_prof) { prof_free(tsd, ptr, usize, &alloc_ctx); } - *tsd_thread_deallocatedp_get(tsd) += usize; if (likely(!slow_path)) { idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, @@ -2584,6 +2583,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, true); } + thread_dalloc_event(tsd, usize); } JEMALLOC_ALWAYS_INLINE void @@ -2645,14 +2645,12 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { if (config_prof && opt_prof) { prof_free(tsd, ptr, usize, ctx); } - - *tsd_thread_deallocatedp_get(tsd) += usize; - if (likely(!slow_path)) { isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false); } else { isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true); } + thread_dalloc_event(tsd, usize); } JEMALLOC_NOINLINE @@ -2694,12 +2692,12 @@ free_default(void *ptr) { JEMALLOC_ALWAYS_INLINE bool free_fastpath(void *ptr, size_t size, bool size_hint) { tsd_t *tsd = tsd_get(false); - if (unlikely(!tsd || !tsd_fast(tsd))) { - return false; - } szind_t szind; if (!size_hint) { + if (unlikely(!tsd || !tsd_fast(tsd))) { + return false; + } bool slab; rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), @@ -2711,6 +2709,15 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) { } assert(szind != SC_NSIZES); } else { + /* + * The size hinted fastpath does not involve rtree lookup, thus + * can tolerate an uninitialized tsd. This allows the tsd_fast + * check to be folded into the branch testing fast_threshold + * (set to 0 when !tsd_fast). + */ + if (unlikely(!tsd)) { + return false; + } /* * Check for both sizes that are too large, and for sampled * objects. Sampled objects are always page-aligned. The @@ -2722,19 +2729,26 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) { } szind = sz_size2index_lookup(size); } + uint64_t deallocated, threshold; + thread_event_free_fastpath_ctx(tsd, &deallocated, &threshold, size_hint); - tcache_t *tcache = tsd_tcachep_get(tsd); - if (unlikely(ticker_trytick(&tcache->gc_ticker))) { + size_t usize = sz_index2size(szind); + uint64_t deallocated_after = deallocated + usize; + /* + * Check for events and tsd non-nominal (fast_threshold will be set to + * 0) in a single branch. + */ + if (unlikely(deallocated_after >= threshold)) { return false; } + tcache_t *tcache = tsd_tcachep_get(tsd); cache_bin_t *bin = tcache_small_bin_get(tcache, szind); if (!cache_bin_dalloc_easy(bin, ptr)) { return false; } - size_t usize = sz_index2size(szind); - *tsd_thread_deallocatedp_get(tsd) += usize; + *tsd_thread_deallocatedp_get(tsd) = deallocated_after; return true; } @@ -3144,11 +3158,11 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) { if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { goto label_oom; } - thread_event(tsd, usize); + thread_alloc_event(tsd, usize); p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, zero, tcache, arena, &alloc_ctx, &hook_args); if (unlikely(p == NULL)) { - thread_event_rollback(tsd, usize); + thread_alloc_event_rollback(tsd, usize); goto label_oom; } } else { @@ -3158,11 +3172,10 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) { goto label_oom; } usize = isalloc(tsd_tsdn(tsd), p); - thread_event(tsd, usize); + thread_alloc_event(tsd, usize); } assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); - - *tsd_thread_deallocatedp_get(tsd) += old_usize; + thread_dalloc_event(tsd, old_usize); UTRACE(ptr, size, p); check_entry_exit_locking(tsd_tsdn(tsd)); @@ -3337,7 +3350,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, usize_max = SC_LARGE_MAXCLASS; } } - thread_event(tsd, usize_max); + thread_alloc_event(tsd, usize_max); bool prof_active = prof_active_get_unlocked(); prof_tctx_t *tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); @@ -3350,7 +3363,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, extra, alignment, zero); } if (usize <= usize_max) { - thread_event_rollback(tsd, usize_max - usize); + thread_alloc_event_rollback(tsd, usize_max - usize); } else { /* * For downsizing request, usize_max can be less than usize. @@ -3359,7 +3372,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, * to xallocx(), the entire usize will be rolled back if it's * equal to the old usize. */ - thread_event(tsd, usize - usize_max); + thread_alloc_event(tsd, usize - usize_max); } /* @@ -3438,7 +3451,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { } else { usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, extra, alignment, zero); - thread_event(tsd, usize); + thread_alloc_event(tsd, usize); } /* @@ -3448,12 +3461,10 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { assert(iealloc(tsd_tsdn(tsd), ptr) == old_edata); if (unlikely(usize == old_usize)) { - thread_event_rollback(tsd, usize); + thread_alloc_event_rollback(tsd, usize); goto label_not_resized; } - - *tsd_thread_deallocatedp_get(tsd) += old_usize; - + thread_dalloc_event(tsd, old_usize); label_not_resized: if (unlikely(!tsd_fast(tsd))) { uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; diff --git a/src/tcache.c b/src/tcache.c index 2f4ca5a4..3d965126 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -437,8 +437,6 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { tcache->next_gc_bin = 0; tcache->arena = NULL; - ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR); - assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS); memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS)); diff --git a/src/thread_event.c b/src/thread_event.c index 6aedf161..5bdc4aef 100644 --- a/src/thread_event.c +++ b/src/thread_event.c @@ -12,14 +12,14 @@ static bool thread_event_active = false; /* TSD event init function signatures. */ -#define E(event, condition) \ +#define E(event, condition_unused, is_alloc_event_unused) \ static void tsd_thread_##event##_event_init(tsd_t *tsd); ITERATE_OVER_ALL_EVENTS #undef E /* Event handler function signatures. */ -#define E(event, condition) \ +#define E(event, condition_unused, is_alloc_event_unused) \ static void thread_##event##_event_handler(tsd_t *tsd); ITERATE_OVER_ALL_EVENTS @@ -32,6 +32,12 @@ tsd_thread_tcache_gc_event_init(tsd_t *tsd) { thread_tcache_gc_event_update(tsd, TCACHE_GC_INCR_BYTES); } +static void +tsd_thread_tcache_gc_dalloc_event_init(tsd_t *tsd) { + assert(TCACHE_GC_INCR_BYTES > 0); + thread_tcache_gc_dalloc_event_update(tsd, TCACHE_GC_INCR_BYTES); +} + static void tsd_thread_prof_sample_event_init(tsd_t *tsd) { assert(config_prof && opt_prof); @@ -46,17 +52,30 @@ tsd_thread_stats_interval_event_init(tsd_t *tsd) { } /* Handler functions. */ + static void -thread_tcache_gc_event_handler(tsd_t *tsd) { +tcache_gc_event(tsd_t *tsd) { assert(TCACHE_GC_INCR_BYTES > 0); - assert(tcache_gc_event_wait_get(tsd) == 0U); - tsd_thread_tcache_gc_event_init(tsd); tcache_t *tcache = tcache_get(tsd); if (tcache != NULL) { tcache_event_hard(tsd, tcache); } } +static void +thread_tcache_gc_event_handler(tsd_t *tsd) { + assert(tcache_gc_event_wait_get(tsd) == 0U); + tsd_thread_tcache_gc_event_init(tsd); + tcache_gc_event(tsd); +} + +static void +thread_tcache_gc_dalloc_event_handler(tsd_t *tsd) { + assert(tcache_gc_dalloc_event_wait_get(tsd) == 0U); + tsd_thread_tcache_gc_dalloc_event_init(tsd); + tcache_gc_event(tsd); +} + static void thread_prof_sample_event_handler(tsd_t *tsd) { assert(config_prof && opt_prof); @@ -96,12 +115,12 @@ thread_stats_interval_event_handler(tsd_t *tsd) { /* Per event facilities done. */ static uint64_t -thread_allocated_next_event_compute(tsd_t *tsd) { +thread_next_event_compute(tsd_t *tsd, bool is_alloc) { uint64_t wait = THREAD_EVENT_MAX_START_WAIT; bool no_event_on = true; -#define E(event, condition) \ - if (condition) { \ +#define E(event, condition, alloc_event) \ + if (is_alloc == alloc_event && condition) { \ no_event_on = false; \ uint64_t event_wait = \ event##_event_wait_get(tsd); \ @@ -119,15 +138,15 @@ thread_allocated_next_event_compute(tsd_t *tsd) { return wait; } -void -thread_event_assert_invariants_debug(tsd_t *tsd) { - uint64_t thread_allocated = thread_allocated_get(tsd); - uint64_t last_event = thread_allocated_last_event_get(tsd); - uint64_t next_event = thread_allocated_next_event_get(tsd); - uint64_t next_event_fast = thread_allocated_next_event_fast_get(tsd); +static void +thread_event_assert_invariants_impl(tsd_t *tsd, event_ctx_t *ctx) { + uint64_t current_bytes = event_ctx_current_bytes_get(ctx); + uint64_t last_event = event_ctx_last_event_get(ctx); + uint64_t next_event = event_ctx_next_event_get(ctx); + uint64_t next_event_fast = event_ctx_next_event_fast_get(ctx); assert(last_event != next_event); - if (next_event > THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX || + if (next_event > THREAD_NEXT_EVENT_FAST_MAX || !tsd_fast(tsd)) { assert(next_event_fast == 0U); } else { @@ -138,10 +157,9 @@ thread_event_assert_invariants_debug(tsd_t *tsd) { uint64_t interval = next_event - last_event; /* The subtraction is intentionally susceptible to underflow. */ - assert(thread_allocated - last_event < interval); - - uint64_t min_wait = thread_allocated_next_event_compute(tsd); - + assert(current_bytes - last_event < interval); + uint64_t min_wait = thread_next_event_compute(tsd, + event_ctx_is_alloc(ctx)); /* * next_event should have been pushed up only except when no event is * on and the TSD is just initialized. The last_event == 0U guard @@ -153,6 +171,16 @@ thread_event_assert_invariants_debug(tsd_t *tsd) { (interval < min_wait && interval == THREAD_EVENT_MAX_INTERVAL)); } +void +thread_event_assert_invariants_debug(tsd_t *tsd) { + event_ctx_t ctx; + event_ctx_get(tsd, &ctx, true); + thread_event_assert_invariants_impl(tsd, &ctx); + + event_ctx_get(tsd, &ctx, false); + thread_event_assert_invariants_impl(tsd, &ctx); +} + /* * Synchronization around the fast threshold in tsd -- * There are two threads to consider in the synchronization here: @@ -200,39 +228,50 @@ thread_event_assert_invariants_debug(tsd_t *tsd) { * of the owner thread's next_event_fast, but that's always safe (it just sends * it down the slow path earlier). */ +static void +event_ctx_next_event_fast_update(event_ctx_t *ctx) { + uint64_t next_event = event_ctx_next_event_get(ctx); + uint64_t next_event_fast = (next_event <= + THREAD_NEXT_EVENT_FAST_MAX) ? next_event : 0U; + event_ctx_next_event_fast_set(ctx, next_event_fast); +} + void thread_event_recompute_fast_threshold(tsd_t *tsd) { if (tsd_state_get(tsd) != tsd_state_nominal) { /* Check first because this is also called on purgatory. */ - thread_allocated_next_event_fast_set_non_nominal(tsd); + thread_next_event_fast_set_non_nominal(tsd); return; } - uint64_t next_event = thread_allocated_next_event_get(tsd); - uint64_t next_event_fast = (next_event <= - THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX) ? next_event : 0U; - thread_allocated_next_event_fast_set(tsd, next_event_fast); + + event_ctx_t ctx; + event_ctx_get(tsd, &ctx, true); + event_ctx_next_event_fast_update(&ctx); + event_ctx_get(tsd, &ctx, false); + event_ctx_next_event_fast_update(&ctx); atomic_fence(ATOMIC_SEQ_CST); if (tsd_state_get(tsd) != tsd_state_nominal) { - thread_allocated_next_event_fast_set_non_nominal(tsd); + thread_next_event_fast_set_non_nominal(tsd); } } static void -thread_event_adjust_thresholds_helper(tsd_t *tsd, uint64_t wait) { +thread_event_adjust_thresholds_helper(tsd_t *tsd, event_ctx_t *ctx, + uint64_t wait) { assert(wait <= THREAD_EVENT_MAX_START_WAIT); - uint64_t next_event = thread_allocated_last_event_get(tsd) + (wait <= + uint64_t next_event = event_ctx_last_event_get(ctx) + (wait <= THREAD_EVENT_MAX_INTERVAL ? wait : THREAD_EVENT_MAX_INTERVAL); - thread_allocated_next_event_set(tsd, next_event); + event_ctx_next_event_set(tsd, ctx, next_event); } static uint64_t thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes, - bool allow_event_trigger) { + bool is_alloc, bool allow_event_trigger) { uint64_t wait = THREAD_EVENT_MAX_START_WAIT; -#define E(event, condition) \ - if (condition) { \ +#define E(event, condition, alloc_event) \ + if (is_alloc == alloc_event && condition) { \ uint64_t event_wait = event##_event_wait_get(tsd); \ assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \ if (event_wait > accumbytes) { \ @@ -267,28 +306,30 @@ thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes, } void -thread_event_trigger(tsd_t *tsd, bool delay_event) { +thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event) { /* usize has already been added to thread_allocated. */ - uint64_t thread_allocated_after = thread_allocated_get(tsd); + uint64_t bytes_after = event_ctx_current_bytes_get(ctx); /* The subtraction is intentionally susceptible to underflow. */ - uint64_t accumbytes = thread_allocated_after - - thread_allocated_last_event_get(tsd); + uint64_t accumbytes = bytes_after - event_ctx_last_event_get(ctx); /* Make sure that accumbytes cannot overflow uint64_t. */ assert(THREAD_EVENT_MAX_INTERVAL <= UINT64_MAX - SC_LARGE_MAXCLASS + 1); - thread_allocated_last_event_set(tsd, thread_allocated_after); + event_ctx_last_event_set(ctx, bytes_after); bool allow_event_trigger = !delay_event && tsd_nominal(tsd) && tsd_reentrancy_level_get(tsd) == 0; + + bool is_alloc = ctx->is_alloc; uint64_t wait = thread_event_trigger_batch_update(tsd, accumbytes, - allow_event_trigger); - thread_event_adjust_thresholds_helper(tsd, wait); + is_alloc, allow_event_trigger); + thread_event_adjust_thresholds_helper(tsd, ctx, wait); thread_event_assert_invariants(tsd); -#define E(event, condition) \ - if (condition && event##_event_wait_get(tsd) == 0U) { \ +#define E(event, condition, alloc_event) \ + if (is_alloc == alloc_event && condition && \ + event##_event_wait_get(tsd) == 0U) { \ assert(allow_event_trigger); \ thread_##event##_event_handler(tsd); \ } @@ -300,19 +341,23 @@ thread_event_trigger(tsd_t *tsd, bool delay_event) { } void -thread_event_rollback(tsd_t *tsd, size_t diff) { +thread_alloc_event_rollback(tsd_t *tsd, size_t diff) { thread_event_assert_invariants(tsd); if (diff == 0U) { return; } - uint64_t thread_allocated = thread_allocated_get(tsd); + /* Rollback happens only on alloc events. */ + event_ctx_t ctx; + event_ctx_get(tsd, &ctx, true); + + uint64_t thread_allocated = event_ctx_current_bytes_get(&ctx); /* The subtraction is intentionally susceptible to underflow. */ uint64_t thread_allocated_rollback = thread_allocated - diff; - thread_allocated_set(tsd, thread_allocated_rollback); + event_ctx_current_bytes_set(&ctx, thread_allocated_rollback); - uint64_t last_event = thread_allocated_last_event_get(tsd); + uint64_t last_event = event_ctx_last_event_get(&ctx); /* Both subtractions are intentionally susceptible to underflow. */ if (thread_allocated_rollback - last_event <= thread_allocated - last_event) { @@ -320,14 +365,14 @@ thread_event_rollback(tsd_t *tsd, size_t diff) { return; } - thread_allocated_last_event_set(tsd, thread_allocated_rollback); + event_ctx_last_event_set(&ctx, thread_allocated_rollback); /* The subtraction is intentionally susceptible to underflow. */ uint64_t wait_diff = last_event - thread_allocated_rollback; assert(wait_diff <= diff); -#define E(event, condition) \ - if (condition) { \ +#define E(event, condition, alloc_event) \ + if (alloc_event == true && condition) { \ uint64_t event_wait = event##_event_wait_get(tsd); \ assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \ if (event_wait > 0U) { \ @@ -347,27 +392,29 @@ thread_event_rollback(tsd_t *tsd, size_t diff) { ITERATE_OVER_ALL_EVENTS #undef E - thread_event_update(tsd); + thread_event_update(tsd, true); } void -thread_event_update(tsd_t *tsd) { - uint64_t wait = thread_allocated_next_event_compute(tsd); - thread_event_adjust_thresholds_helper(tsd, wait); +thread_event_update(tsd_t *tsd, bool is_alloc) { + event_ctx_t ctx; + event_ctx_get(tsd, &ctx, is_alloc); - uint64_t last_event = thread_allocated_last_event_get(tsd); + uint64_t wait = thread_next_event_compute(tsd, is_alloc); + thread_event_adjust_thresholds_helper(tsd, &ctx, wait); + uint64_t last_event = event_ctx_last_event_get(&ctx); /* Both subtractions are intentionally susceptible to underflow. */ - if (thread_allocated_get(tsd) - last_event >= - thread_allocated_next_event_get(tsd) - last_event) { - thread_event_trigger(tsd, true); + if (event_ctx_current_bytes_get(&ctx) - last_event >= + event_ctx_next_event_get(&ctx) - last_event) { + thread_event_trigger(tsd, &ctx, true); } else { thread_event_assert_invariants(tsd); } } void thread_event_boot() { -#define E(event, condition) \ +#define E(event, condition, ignored) \ if (condition) { \ thread_event_active = true; \ } @@ -377,7 +424,7 @@ void thread_event_boot() { } void tsd_thread_event_init(tsd_t *tsd) { -#define E(event, condition) \ +#define E(event, condition, is_alloc_event_unused) \ if (condition) { \ tsd_thread_##event##_event_init(tsd); \ } diff --git a/src/tsd.c b/src/tsd.c index 940ff7d2..54e5b4af 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -119,7 +119,7 @@ tsd_force_recompute(tsdn_t *tsdn) { tsd_state_nominal_recompute, ATOMIC_RELAXED); /* See comments in thread_event_recompute_fast_threshold(). */ atomic_fence(ATOMIC_SEQ_CST); - thread_allocated_next_event_fast_set_non_nominal(remote_tsd); + thread_next_event_fast_set_non_nominal(remote_tsd); } malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock); } diff --git a/test/unit/thread_event.c b/test/unit/thread_event.c index f016cc5d..db2d637e 100644 --- a/test/unit/thread_event.c +++ b/test/unit/thread_event.c @@ -2,14 +2,18 @@ TEST_BEGIN(test_next_event_fast_roll_back) { tsd_t *tsd = tsd_fetch(); - thread_allocated_last_event_set(tsd, 0); - thread_allocated_set(tsd, - THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX - 8U); - thread_allocated_next_event_set(tsd, - THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX); -#define E(event, condition) \ - event##_event_wait_set(tsd, \ - THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX); + event_ctx_t ctx; + event_ctx_get(tsd, &ctx, true); + + event_ctx_last_event_set(&ctx, 0); + event_ctx_current_bytes_set(&ctx, + THREAD_NEXT_EVENT_FAST_MAX - 8U); + event_ctx_next_event_set(tsd, &ctx, + THREAD_NEXT_EVENT_FAST_MAX); +#define E(event, condition, is_alloc) \ + if (is_alloc && condition) { \ + event##_event_wait_set(tsd, THREAD_NEXT_EVENT_FAST_MAX);\ + } ITERATE_OVER_ALL_EVENTS #undef E void *p = malloc(16U); @@ -20,14 +24,20 @@ TEST_END TEST_BEGIN(test_next_event_fast_resume) { tsd_t *tsd = tsd_fetch(); - thread_allocated_last_event_set(tsd, 0); - thread_allocated_set(tsd, - THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 8U); - thread_allocated_next_event_set(tsd, - THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U); -#define E(event, condition) \ - event##_event_wait_set(tsd, \ - THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U); + + event_ctx_t ctx; + event_ctx_get(tsd, &ctx, true); + + event_ctx_last_event_set(&ctx, 0); + event_ctx_current_bytes_set(&ctx, + THREAD_NEXT_EVENT_FAST_MAX + 8U); + event_ctx_next_event_set(tsd, &ctx, + THREAD_NEXT_EVENT_FAST_MAX + 16U); +#define E(event, condition, is_alloc) \ + if (is_alloc && condition) { \ + event##_event_wait_set(tsd, \ + THREAD_NEXT_EVENT_FAST_MAX + 16U); \ + } ITERATE_OVER_ALL_EVENTS #undef E void *p = malloc(SC_LOOKUP_MAXCLASS); @@ -42,7 +52,7 @@ TEST_BEGIN(test_event_rollback) { size_t count = 10; uint64_t thread_allocated = thread_allocated_get(tsd); while (count-- != 0) { - thread_event_rollback(tsd, diff); + thread_alloc_event_rollback(tsd, diff); uint64_t thread_allocated_after = thread_allocated_get(tsd); assert_u64_eq(thread_allocated - thread_allocated_after, diff, "thread event counters are not properly rolled back");