project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement deallocation events.

Browse Source 
Make the event module to accept two event types, and pass around the event
context.  Use bytes-based events to trigger tcache GC on deallocation, and get
rid of the tcache ticker.
This commit is contained in:
Qi Wang 2020-01-28 21:12:06 -08:00 committed by Qi Wang
parent 536ea6858e
commit 97dd79db6c
9 changed files with 306 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/jemalloc/internal/tcache_inlines.h
View File

@ -5,7 +5,6 @@
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/util.h"
static inline bool
@ -27,17 +26,6 @@ tcache_enabled_set(tsd_t *tsd, bool enabled) {
tsd_slow_update(tsd);
}
JEMALLOC_ALWAYS_INLINE void
tcache_event(tsd_t *tsd, tcache_t *tcache) {
if (TCACHE_GC_INCR == 0) {
return;
}
if (unlikely(ticker_tick(&tcache->gc_ticker))) {
tcache_event_hard(tsd, tcache);
}
}
JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
size_t size, szind_t binind, bool zero, bool slow_path) {
@ -171,8 +159,6 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);
}
tcache_event(tsd, tcache);
}
JEMALLOC_ALWAYS_INLINE void
@ -195,8 +181,6 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);
}
tcache_event(tsd, tcache);
}
JEMALLOC_ALWAYS_INLINE tcache_t *

3
include/jemalloc/internal/tcache_structs.h
View File

@ -16,9 +16,6 @@ struct tcache_s {
* together at the start of this struct.
*/
/* Drives incremental GC. */
ticker_t gc_ticker;
/*
* The pointer stacks associated with bins follow as a contiguous array.
* During tcache initialization, the avail pointer in each element of

176
include/jemalloc/internal/thread_event.h
View File

@ -4,42 +4,51 @@
#include "jemalloc/internal/tsd.h"
/*
* Maximum threshold on thread_allocated_next_event_fast, so that there is no
* need to check overflow in malloc fast path. (The allocation size in malloc
* Maximum threshold on thread_(de)allocated_next_event_fast, so that there is
* no need to check overflow in malloc fast path. (The allocation size in malloc
* fast path never exceeds SC_LOOKUP_MAXCLASS.)
*/
#define THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX \
#define THREAD_NEXT_EVENT_FAST_MAX \
(UINT64_MAX - SC_LOOKUP_MAXCLASS + 1U)
/*
* The max interval helps make sure that malloc stays on the fast path in the
* common case, i.e. thread_allocated < thread_allocated_next_event_fast.
* When thread_allocated is within an event's distance to
* THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX above, thread_allocated_next_event_fast
* is wrapped around and we fall back to the medium-fast path. The max interval
* makes sure that we're not staying on the fallback case for too long, even if
* there's no active event or if all active events have long wait times.
* common case, i.e. thread_allocated < thread_allocated_next_event_fast. When
* thread_allocated is within an event's distance to THREAD_NEXT_EVENT_FAST_MAX
* above, thread_allocated_next_event_fast is wrapped around and we fall back to
* the medium-fast path. The max interval makes sure that we're not staying on
* the fallback case for too long, even if there's no active event or if all
* active events have long wait times.
*/
#define THREAD_EVENT_MAX_INTERVAL ((uint64_t)(4U << 20))
typedef struct event_ctx_s {
bool is_alloc;
uint64_t *current;
uint64_t *last_event;
uint64_t *next_event;
uint64_t *next_event_fast;
} event_ctx_t;
void thread_event_assert_invariants_debug(tsd_t *tsd);
void thread_event_trigger(tsd_t *tsd, bool delay_event);
void thread_event_rollback(tsd_t *tsd, size_t diff);
void thread_event_update(tsd_t *tsd);
void thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event);
void thread_alloc_event_rollback(tsd_t *tsd, size_t diff);
void thread_event_update(tsd_t *tsd, bool alloc_event);
void thread_event_boot();
void thread_event_recompute_fast_threshold(tsd_t *tsd);
void tsd_thread_event_init(tsd_t *tsd);
/*
* List of all events, in the following format:
* E(event, (condition))
* E(event, (condition), is_alloc_event)
*/
#define ITERATE_OVER_ALL_EVENTS \
E(tcache_gc, (TCACHE_GC_INCR_BYTES > 0)) \
E(prof_sample, (config_prof && opt_prof)) \
E(stats_interval, (opt_stats_interval >= 0))
E(tcache_gc, (TCACHE_GC_INCR_BYTES > 0), true) \
E(prof_sample, (config_prof && opt_prof), true) \
E(stats_interval, (opt_stats_interval >= 0), true) \
E(tcache_gc_dalloc, (TCACHE_GC_INCR_BYTES > 0), false)
#define E(event, condition) \
#define E(event, condition_unused, is_alloc_event_unused) \
C(event##_event_wait)
/* List of all thread event counters. */
@ -83,9 +92,9 @@ ITERATE_OVER_ALL_COUNTERS
#undef E
/*
* Two malloc fastpath getters -- use the unsafe getters since tsd may be
* non-nominal, in which case the fast_threshold will be set to 0. This allows
* checking for events and tsd non-nominal in a single branch.
* The malloc and free fastpath getters -- use the unsafe getters since tsd may
* be non-nominal, in which case the fast_threshold will be set to 0. This
* allows checking for events and tsd non-nominal in a single branch.
*
* Note that these can only be used on the fastpath.
*/
@ -97,42 +106,83 @@ thread_allocated_malloc_fastpath(tsd_t *tsd) {
JEMALLOC_ALWAYS_INLINE uint64_t
thread_allocated_next_event_malloc_fastpath(tsd_t *tsd) {
uint64_t v = *tsd_thread_allocated_next_event_fastp_get_unsafe(tsd);
assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
assert(v <= THREAD_NEXT_EVENT_FAST_MAX);
return v;
}
JEMALLOC_ALWAYS_INLINE void
thread_event_free_fastpath_ctx(tsd_t *tsd, uint64_t *deallocated,
uint64_t *threshold, bool size_hint) {
if (!size_hint) {
*deallocated = tsd_thread_deallocated_get(tsd);
*threshold = tsd_thread_deallocated_next_event_fast_get(tsd);
} else {
/* Unsafe getters since this may happen before tsd_init. */
*deallocated = *tsd_thread_deallocatedp_get_unsafe(tsd);
*threshold =
*tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd);
}
assert(*threshold <= THREAD_NEXT_EVENT_FAST_MAX);
}
JEMALLOC_ALWAYS_INLINE bool
event_ctx_is_alloc(event_ctx_t *ctx) {
return ctx->is_alloc;
}
JEMALLOC_ALWAYS_INLINE uint64_t
event_ctx_current_bytes_get(event_ctx_t *ctx) {
return *ctx->current;
}
JEMALLOC_ALWAYS_INLINE void
event_ctx_current_bytes_set(event_ctx_t *ctx, uint64_t v) {
*ctx->current = v;
}
JEMALLOC_ALWAYS_INLINE uint64_t
event_ctx_last_event_get(event_ctx_t *ctx) {
return *ctx->last_event;
}
JEMALLOC_ALWAYS_INLINE void
event_ctx_last_event_set(event_ctx_t *ctx, uint64_t v) {
*ctx->last_event = v;
}
/* Below 3 for next_event_fast. */
JEMALLOC_ALWAYS_INLINE uint64_t
thread_allocated_next_event_fast_get(tsd_t *tsd) {
uint64_t v = tsd_thread_allocated_next_event_fast_get(tsd);
assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
event_ctx_next_event_fast_get(event_ctx_t *ctx) {
uint64_t v = *ctx->next_event_fast;
assert(v <= THREAD_NEXT_EVENT_FAST_MAX);
return v;
}
JEMALLOC_ALWAYS_INLINE void
thread_allocated_next_event_fast_set(tsd_t *tsd, uint64_t v) {
assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
*tsd_thread_allocated_next_event_fastp_get(tsd) = v;
event_ctx_next_event_fast_set(event_ctx_t *ctx, uint64_t v) {
assert(v <= THREAD_NEXT_EVENT_FAST_MAX);
*ctx->next_event_fast = v;
}
JEMALLOC_ALWAYS_INLINE void
thread_allocated_next_event_fast_set_non_nominal(tsd_t *tsd) {
thread_next_event_fast_set_non_nominal(tsd_t *tsd) {
/*
* Set the fast threshold to zero when tsd is non-nominal. Use the
* Set the fast thresholds to zero when tsd is non-nominal. Use the
* unsafe getter as this may get called during tsd init and clean up.
*/
*tsd_thread_allocated_next_event_fastp_get_unsafe(tsd) = 0;
*tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) = 0;
}
/* For next_event. Setter also updates the fast threshold. */
JEMALLOC_ALWAYS_INLINE uint64_t
thread_allocated_next_event_get(tsd_t *tsd) {
return tsd_thread_allocated_next_event_get(tsd);
event_ctx_next_event_get(event_ctx_t *ctx) {
return *ctx->next_event;
}
JEMALLOC_ALWAYS_INLINE void
thread_allocated_next_event_set(tsd_t *tsd, uint64_t v) {
*tsd_thread_allocated_next_eventp_get(tsd) = v;
event_ctx_next_event_set(tsd_t *tsd, event_ctx_t *ctx, uint64_t v) {
*ctx->next_event = v;
thread_event_recompute_fast_threshold(tsd);
}
@ -145,8 +195,8 @@ thread_allocated_next_event_set(tsd_t *tsd, uint64_t v) {
* at the end will restore the invariants),
* (b) thread_##event##_event_update() (the thread_event_update() call at the
* end will restore the invariants), or
* (c) thread_event_rollback() if the rollback falls below the last_event (the
* thread_event_update() call at the end will restore the invariants).
* (c) thread_alloc_event_rollback() if the rollback falls below the last_event
* (the thread_event_update() call at the end will restore the invariants).
*/
JEMALLOC_ALWAYS_INLINE void
thread_event_assert_invariants(tsd_t *tsd) {
@ -156,22 +206,52 @@ thread_event_assert_invariants(tsd_t *tsd) {
}
JEMALLOC_ALWAYS_INLINE void
thread_event(tsd_t *tsd, size_t usize) {
thread_event_assert_invariants(tsd);
uint64_t thread_allocated_before = thread_allocated_get(tsd);
thread_allocated_set(tsd, thread_allocated_before + usize);
/* The subtraction is intentionally susceptible to underflow. */
if (likely(usize < thread_allocated_next_event_get(tsd) -
thread_allocated_before)) {
thread_event_assert_invariants(tsd);
event_ctx_get(tsd_t *tsd, event_ctx_t *ctx, bool is_alloc) {
ctx->is_alloc = is_alloc;
if (is_alloc) {
ctx->current = tsd_thread_allocatedp_get(tsd);
ctx->last_event = tsd_thread_allocated_last_eventp_get(tsd);
ctx->next_event = tsd_thread_allocated_next_eventp_get(tsd);
ctx->next_event_fast =
tsd_thread_allocated_next_event_fastp_get(tsd);
} else {
thread_event_trigger(tsd, false);
ctx->current = tsd_thread_deallocatedp_get(tsd);
ctx->last_event = tsd_thread_deallocated_last_eventp_get(tsd);
ctx->next_event = tsd_thread_deallocated_next_eventp_get(tsd);
ctx->next_event_fast =
tsd_thread_deallocated_next_event_fastp_get(tsd);
}
}
#define E(event, condition) \
JEMALLOC_ALWAYS_INLINE void
thread_event_advance(tsd_t *tsd, size_t usize, bool is_alloc) {
thread_event_assert_invariants(tsd);
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, is_alloc);
uint64_t bytes_before = event_ctx_current_bytes_get(&ctx);
event_ctx_current_bytes_set(&ctx, bytes_before + usize);
/* The subtraction is intentionally susceptible to underflow. */
if (likely(usize < event_ctx_next_event_get(&ctx) - bytes_before)) {
thread_event_assert_invariants(tsd);
} else {
thread_event_trigger(tsd, &ctx, false);
}
}
JEMALLOC_ALWAYS_INLINE void
thread_dalloc_event(tsd_t *tsd, size_t usize) {
thread_event_advance(tsd, usize, false);
}
JEMALLOC_ALWAYS_INLINE void
thread_alloc_event(tsd_t *tsd, size_t usize) {
thread_event_advance(tsd, usize, true);
}
#define E(event, condition, is_alloc) \
JEMALLOC_ALWAYS_INLINE void \
thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) { \
thread_event_assert_invariants(tsd); \
@ -188,7 +268,7 @@ thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) { \
event_wait = THREAD_EVENT_MAX_START_WAIT; \
} \
event##_event_wait_set(tsd, event_wait); \
thread_event_update(tsd); \
thread_event_update(tsd, is_alloc); \
}
ITERATE_OVER_ALL_EVENTS

8
include/jemalloc/internal/tsd.h
View File

@ -81,10 +81,14 @@ typedef void (*test_callback_t)(int *);
O(thread_allocated, uint64_t, uint64_t) \
O(thread_allocated_next_event_fast, uint64_t, uint64_t) \
O(thread_deallocated, uint64_t, uint64_t) \
O(thread_deallocated_next_event_fast, uint64_t, uint64_t) \
O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \
O(thread_allocated_last_event, uint64_t, uint64_t) \
O(thread_allocated_next_event, uint64_t, uint64_t) \
O(thread_deallocated_last_event, uint64_t, uint64_t) \
O(thread_deallocated_next_event, uint64_t, uint64_t) \
O(tcache_gc_event_wait, uint64_t, uint64_t) \
O(tcache_gc_dalloc_event_wait, uint64_t, uint64_t) \
O(prof_sample_event_wait, uint64_t, uint64_t) \
O(prof_sample_last_event, uint64_t, uint64_t) \
O(stats_interval_event_wait, uint64_t, uint64_t) \
@ -114,10 +118,14 @@ typedef void (*test_callback_t)(int *);
/* thread_allocated */ 0, \
/* thread_allocated_next_event_fast */ 0, \
/* thread_deallocated */ 0, \
/* thread_deallocated_next_event_fast */ 0, \
/* rtree_ctx */ RTREE_CTX_ZERO_INITIALIZER, \
/* thread_allocated_last_event */ 0, \
/* thread_allocated_next_event */ THREAD_EVENT_MIN_START_WAIT, \
/* thread_deallocated_last_event */ 0, \
/* thread_deallocated_next_event */ THREAD_EVENT_MIN_START_WAIT, \
/* tcache_gc_event_wait */ THREAD_EVENT_MIN_START_WAIT, \
/* tcache_gc_dalloc_event_wait */ THREAD_EVENT_MIN_START_WAIT, \
/* prof_sample_event_wait */ THREAD_EVENT_MIN_START_WAIT, \
/* prof_sample_last_event */ 0, \
/* stats_interval_event_wait */ THREAD_EVENT_MIN_START_WAIT, \

65
src/jemalloc.c
View File

@ -2154,7 +2154,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
dopts->arena_ind = 0;
}
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
/*
* If dopts->alignment > 0, then ind is still 0, but usize was computed
@ -2181,7 +2181,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
}
if (unlikely(allocation == NULL)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
prof_alloc_rollback(tsd, tctx, true);
goto label_oom;
}
@ -2191,7 +2191,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
ind);
if (unlikely(allocation == NULL)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
goto label_oom;
}
}
@ -2575,7 +2575,6 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
if (config_prof && opt_prof) {
prof_free(tsd, ptr, usize, &alloc_ctx);
}
*tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path)) {
idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
@ -2584,6 +2583,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
true);
}
thread_dalloc_event(tsd, usize);
}
JEMALLOC_ALWAYS_INLINE void
@ -2645,14 +2645,12 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
if (config_prof && opt_prof) {
prof_free(tsd, ptr, usize, ctx);
}
*tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path)) {
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
} else {
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true);
}
thread_dalloc_event(tsd, usize);
}
JEMALLOC_NOINLINE
@ -2694,12 +2692,12 @@ free_default(void *ptr) {
JEMALLOC_ALWAYS_INLINE
bool free_fastpath(void *ptr, size_t size, bool size_hint) {
tsd_t *tsd = tsd_get(false);
if (unlikely(!tsd || !tsd_fast(tsd))) {
return false;
}
szind_t szind;
if (!size_hint) {
if (unlikely(!tsd || !tsd_fast(tsd))) {
return false;
}
bool slab;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd),
@ -2711,6 +2709,15 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
}
assert(szind != SC_NSIZES);
} else {
/*
* The size hinted fastpath does not involve rtree lookup, thus
* can tolerate an uninitialized tsd. This allows the tsd_fast
* check to be folded into the branch testing fast_threshold
* (set to 0 when !tsd_fast).
*/
if (unlikely(!tsd)) {
return false;
}
/*
* Check for both sizes that are too large, and for sampled
* objects. Sampled objects are always page-aligned. The
@ -2722,19 +2729,26 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
}
szind = sz_size2index_lookup(size);
}
uint64_t deallocated, threshold;
thread_event_free_fastpath_ctx(tsd, &deallocated, &threshold, size_hint);
tcache_t *tcache = tsd_tcachep_get(tsd);
if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
size_t usize = sz_index2size(szind);
uint64_t deallocated_after = deallocated + usize;
/*
* Check for events and tsd non-nominal (fast_threshold will be set to
* 0) in a single branch.
*/
if (unlikely(deallocated_after >= threshold)) {
return false;
}
tcache_t *tcache = tsd_tcachep_get(tsd);
cache_bin_t *bin = tcache_small_bin_get(tcache, szind);
if (!cache_bin_dalloc_easy(bin, ptr)) {
return false;
}
size_t usize = sz_index2size(szind);
*tsd_thread_deallocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) = deallocated_after;
return true;
}
@ -3144,11 +3158,11 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
goto label_oom;
}
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
zero, tcache, arena, &alloc_ctx, &hook_args);
if (unlikely(p == NULL)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
goto label_oom;
}
} else {
@ -3158,11 +3172,10 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
goto label_oom;
}
usize = isalloc(tsd_tsdn(tsd), p);
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
}
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
*tsd_thread_deallocatedp_get(tsd) += old_usize;
thread_dalloc_event(tsd, old_usize);
UTRACE(ptr, size, p);
check_entry_exit_locking(tsd_tsdn(tsd));
@ -3337,7 +3350,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
usize_max = SC_LARGE_MAXCLASS;
}
}
thread_event(tsd, usize_max);
thread_alloc_event(tsd, usize_max);
bool prof_active = prof_active_get_unlocked();
prof_tctx_t *tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
@ -3350,7 +3363,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
extra, alignment, zero);
}
if (usize <= usize_max) {
thread_event_rollback(tsd, usize_max - usize);
thread_alloc_event_rollback(tsd, usize_max - usize);
} else {
/*
* For downsizing request, usize_max can be less than usize.
@ -3359,7 +3372,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
* to xallocx(), the entire usize will be rolled back if it's
* equal to the old usize.
*/
thread_event(tsd, usize - usize_max);
thread_alloc_event(tsd, usize - usize_max);
}
/*
@ -3438,7 +3451,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
} else {
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
extra, alignment, zero);
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
}
/*
@ -3448,12 +3461,10 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
assert(iealloc(tsd_tsdn(tsd), ptr) == old_edata);
if (unlikely(usize == old_usize)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
goto label_not_resized;
}
*tsd_thread_deallocatedp_get(tsd) += old_usize;
thread_dalloc_event(tsd, old_usize);
label_not_resized:
if (unlikely(!tsd_fast(tsd))) {
uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};

2
src/tcache.c
View File

@ -437,8 +437,6 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
tcache->next_gc_bin = 0;
tcache->arena = NULL;
ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS);
memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS));

161
src/thread_event.c
View File

@ -12,14 +12,14 @@
static bool thread_event_active = false;
/* TSD event init function signatures. */
#define E(event, condition) \
#define E(event, condition_unused, is_alloc_event_unused) \
static void tsd_thread_##event##_event_init(tsd_t *tsd);
ITERATE_OVER_ALL_EVENTS
#undef E
/* Event handler function signatures. */
#define E(event, condition) \
#define E(event, condition_unused, is_alloc_event_unused) \
static void thread_##event##_event_handler(tsd_t *tsd);
ITERATE_OVER_ALL_EVENTS
@ -32,6 +32,12 @@ tsd_thread_tcache_gc_event_init(tsd_t *tsd) {
thread_tcache_gc_event_update(tsd, TCACHE_GC_INCR_BYTES);
}
static void
tsd_thread_tcache_gc_dalloc_event_init(tsd_t *tsd) {
assert(TCACHE_GC_INCR_BYTES > 0);
thread_tcache_gc_dalloc_event_update(tsd, TCACHE_GC_INCR_BYTES);
}
static void
tsd_thread_prof_sample_event_init(tsd_t *tsd) {
assert(config_prof && opt_prof);
@ -46,17 +52,30 @@ tsd_thread_stats_interval_event_init(tsd_t *tsd) {
}
/* Handler functions. */
static void
thread_tcache_gc_event_handler(tsd_t *tsd) {
tcache_gc_event(tsd_t *tsd) {
assert(TCACHE_GC_INCR_BYTES > 0);
assert(tcache_gc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_event_init(tsd);
tcache_t *tcache = tcache_get(tsd);
if (tcache != NULL) {
tcache_event_hard(tsd, tcache);
}
}
static void
thread_tcache_gc_event_handler(tsd_t *tsd) {
assert(tcache_gc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_event_init(tsd);
tcache_gc_event(tsd);
}
static void
thread_tcache_gc_dalloc_event_handler(tsd_t *tsd) {
assert(tcache_gc_dalloc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_dalloc_event_init(tsd);
tcache_gc_event(tsd);
}
static void
thread_prof_sample_event_handler(tsd_t *tsd) {
assert(config_prof && opt_prof);
@ -96,12 +115,12 @@ thread_stats_interval_event_handler(tsd_t *tsd) {
/* Per event facilities done. */
static uint64_t
thread_allocated_next_event_compute(tsd_t *tsd) {
thread_next_event_compute(tsd_t *tsd, bool is_alloc) {
uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
bool no_event_on = true;
#define E(event, condition) \
if (condition) { \
#define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition) { \
no_event_on = false; \
uint64_t event_wait = \
event##_event_wait_get(tsd); \
@ -119,15 +138,15 @@ thread_allocated_next_event_compute(tsd_t *tsd) {
return wait;
}
void
thread_event_assert_invariants_debug(tsd_t *tsd) {
uint64_t thread_allocated = thread_allocated_get(tsd);
uint64_t last_event = thread_allocated_last_event_get(tsd);
uint64_t next_event = thread_allocated_next_event_get(tsd);
uint64_t next_event_fast = thread_allocated_next_event_fast_get(tsd);
static void
thread_event_assert_invariants_impl(tsd_t *tsd, event_ctx_t *ctx) {
uint64_t current_bytes = event_ctx_current_bytes_get(ctx);
uint64_t last_event = event_ctx_last_event_get(ctx);
uint64_t next_event = event_ctx_next_event_get(ctx);
uint64_t next_event_fast = event_ctx_next_event_fast_get(ctx);
assert(last_event != next_event);
if (next_event > THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX ||
if (next_event > THREAD_NEXT_EVENT_FAST_MAX ||
!tsd_fast(tsd)) {
assert(next_event_fast == 0U);
} else {
@ -138,10 +157,9 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
uint64_t interval = next_event - last_event;
/* The subtraction is intentionally susceptible to underflow. */
assert(thread_allocated - last_event < interval);
uint64_t min_wait = thread_allocated_next_event_compute(tsd);
assert(current_bytes - last_event < interval);
uint64_t min_wait = thread_next_event_compute(tsd,
event_ctx_is_alloc(ctx));
/*
* next_event should have been pushed up only except when no event is
* on and the TSD is just initialized. The last_event == 0U guard
@ -153,6 +171,16 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
(interval < min_wait && interval == THREAD_EVENT_MAX_INTERVAL));
}
void
thread_event_assert_invariants_debug(tsd_t *tsd) {
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
thread_event_assert_invariants_impl(tsd, &ctx);
event_ctx_get(tsd, &ctx, false);
thread_event_assert_invariants_impl(tsd, &ctx);
}
/*
* Synchronization around the fast threshold in tsd --
* There are two threads to consider in the synchronization here:
@ -200,39 +228,50 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
* of the owner thread's next_event_fast, but that's always safe (it just sends
* it down the slow path earlier).
*/
static void
event_ctx_next_event_fast_update(event_ctx_t *ctx) {
uint64_t next_event = event_ctx_next_event_get(ctx);
uint64_t next_event_fast = (next_event <=
THREAD_NEXT_EVENT_FAST_MAX) ? next_event : 0U;
event_ctx_next_event_fast_set(ctx, next_event_fast);
}
void
thread_event_recompute_fast_threshold(tsd_t *tsd) {
if (tsd_state_get(tsd) != tsd_state_nominal) {
/* Check first because this is also called on purgatory. */
thread_allocated_next_event_fast_set_non_nominal(tsd);
thread_next_event_fast_set_non_nominal(tsd);
return;
}
uint64_t next_event = thread_allocated_next_event_get(tsd);
uint64_t next_event_fast = (next_event <=
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX) ? next_event : 0U;
thread_allocated_next_event_fast_set(tsd, next_event_fast);
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
event_ctx_next_event_fast_update(&ctx);
event_ctx_get(tsd, &ctx, false);
event_ctx_next_event_fast_update(&ctx);
atomic_fence(ATOMIC_SEQ_CST);
if (tsd_state_get(tsd) != tsd_state_nominal) {
thread_allocated_next_event_fast_set_non_nominal(tsd);
thread_next_event_fast_set_non_nominal(tsd);
}
}
static void
thread_event_adjust_thresholds_helper(tsd_t *tsd, uint64_t wait) {
thread_event_adjust_thresholds_helper(tsd_t *tsd, event_ctx_t *ctx,
uint64_t wait) {
assert(wait <= THREAD_EVENT_MAX_START_WAIT);
uint64_t next_event = thread_allocated_last_event_get(tsd) + (wait <=
uint64_t next_event = event_ctx_last_event_get(ctx) + (wait <=
THREAD_EVENT_MAX_INTERVAL ? wait : THREAD_EVENT_MAX_INTERVAL);
thread_allocated_next_event_set(tsd, next_event);
event_ctx_next_event_set(tsd, ctx, next_event);
}
static uint64_t
thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
bool allow_event_trigger) {
bool is_alloc, bool allow_event_trigger) {
uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
#define E(event, condition) \
if (condition) { \
#define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition) { \
uint64_t event_wait = event##_event_wait_get(tsd); \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
if (event_wait > accumbytes) { \
@ -267,28 +306,30 @@ thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
}
void
thread_event_trigger(tsd_t *tsd, bool delay_event) {
thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event) {
/* usize has already been added to thread_allocated. */
uint64_t thread_allocated_after = thread_allocated_get(tsd);
uint64_t bytes_after = event_ctx_current_bytes_get(ctx);
/* The subtraction is intentionally susceptible to underflow. */
uint64_t accumbytes = thread_allocated_after -
thread_allocated_last_event_get(tsd);
uint64_t accumbytes = bytes_after - event_ctx_last_event_get(ctx);
/* Make sure that accumbytes cannot overflow uint64_t. */
assert(THREAD_EVENT_MAX_INTERVAL <= UINT64_MAX - SC_LARGE_MAXCLASS + 1);
thread_allocated_last_event_set(tsd, thread_allocated_after);
event_ctx_last_event_set(ctx, bytes_after);
bool allow_event_trigger = !delay_event && tsd_nominal(tsd) &&
tsd_reentrancy_level_get(tsd) == 0;
bool is_alloc = ctx->is_alloc;
uint64_t wait = thread_event_trigger_batch_update(tsd, accumbytes,
allow_event_trigger);
thread_event_adjust_thresholds_helper(tsd, wait);
is_alloc, allow_event_trigger);
thread_event_adjust_thresholds_helper(tsd, ctx, wait);
thread_event_assert_invariants(tsd);
#define E(event, condition) \
if (condition && event##_event_wait_get(tsd) == 0U) { \
#define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition && \
event##_event_wait_get(tsd) == 0U) { \
assert(allow_event_trigger); \
thread_##event##_event_handler(tsd); \
}
@ -300,19 +341,23 @@ thread_event_trigger(tsd_t *tsd, bool delay_event) {
}
void
thread_event_rollback(tsd_t *tsd, size_t diff) {
thread_alloc_event_rollback(tsd_t *tsd, size_t diff) {
thread_event_assert_invariants(tsd);
if (diff == 0U) {
return;
}
uint64_t thread_allocated = thread_allocated_get(tsd);
/* Rollback happens only on alloc events. */
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
uint64_t thread_allocated = event_ctx_current_bytes_get(&ctx);
/* The subtraction is intentionally susceptible to underflow. */
uint64_t thread_allocated_rollback = thread_allocated - diff;
thread_allocated_set(tsd, thread_allocated_rollback);
event_ctx_current_bytes_set(&ctx, thread_allocated_rollback);
uint64_t last_event = thread_allocated_last_event_get(tsd);
uint64_t last_event = event_ctx_last_event_get(&ctx);
/* Both subtractions are intentionally susceptible to underflow. */
if (thread_allocated_rollback - last_event <=
thread_allocated - last_event) {
@ -320,14 +365,14 @@ thread_event_rollback(tsd_t *tsd, size_t diff) {
return;
}
thread_allocated_last_event_set(tsd, thread_allocated_rollback);
event_ctx_last_event_set(&ctx, thread_allocated_rollback);
/* The subtraction is intentionally susceptible to underflow. */
uint64_t wait_diff = last_event - thread_allocated_rollback;
assert(wait_diff <= diff);
#define E(event, condition) \
if (condition) { \
#define E(event, condition, alloc_event) \
if (alloc_event == true && condition) { \
uint64_t event_wait = event##_event_wait_get(tsd); \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
if (event_wait > 0U) { \
@ -347,27 +392,29 @@ thread_event_rollback(tsd_t *tsd, size_t diff) {
ITERATE_OVER_ALL_EVENTS
#undef E
thread_event_update(tsd);
thread_event_update(tsd, true);
}
void
thread_event_update(tsd_t *tsd) {
uint64_t wait = thread_allocated_next_event_compute(tsd);
thread_event_adjust_thresholds_helper(tsd, wait);
thread_event_update(tsd_t *tsd, bool is_alloc) {
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, is_alloc);
uint64_t last_event = thread_allocated_last_event_get(tsd);
uint64_t wait = thread_next_event_compute(tsd, is_alloc);
thread_event_adjust_thresholds_helper(tsd, &ctx, wait);
uint64_t last_event = event_ctx_last_event_get(&ctx);
/* Both subtractions are intentionally susceptible to underflow. */
if (thread_allocated_get(tsd) - last_event >=
thread_allocated_next_event_get(tsd) - last_event) {
thread_event_trigger(tsd, true);
if (event_ctx_current_bytes_get(&ctx) - last_event >=
event_ctx_next_event_get(&ctx) - last_event) {
thread_event_trigger(tsd, &ctx, true);
} else {
thread_event_assert_invariants(tsd);
}
}
void thread_event_boot() {
#define E(event, condition) \
#define E(event, condition, ignored) \
if (condition) { \
thread_event_active = true; \
}
@ -377,7 +424,7 @@ void thread_event_boot() {
}
void tsd_thread_event_init(tsd_t *tsd) {
#define E(event, condition) \
#define E(event, condition, is_alloc_event_unused) \
if (condition) { \
tsd_thread_##event##_event_init(tsd); \
}

2
src/tsd.c
View File

@ -119,7 +119,7 @@ tsd_force_recompute(tsdn_t *tsdn) {
tsd_state_nominal_recompute, ATOMIC_RELAXED);
/* See comments in thread_event_recompute_fast_threshold(). */
atomic_fence(ATOMIC_SEQ_CST);
thread_allocated_next_event_fast_set_non_nominal(remote_tsd);
thread_next_event_fast_set_non_nominal(remote_tsd);
}
malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock);
}

44
test/unit/thread_event.c
View File

@ -2,14 +2,18 @@
TEST_BEGIN(test_next_event_fast_roll_back) {
tsd_t *tsd = tsd_fetch();
thread_allocated_last_event_set(tsd, 0);
thread_allocated_set(tsd,
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX - 8U);
thread_allocated_next_event_set(tsd,
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
#define E(event, condition) \
event##_event_wait_set(tsd, \
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
event_ctx_last_event_set(&ctx, 0);
event_ctx_current_bytes_set(&ctx,
THREAD_NEXT_EVENT_FAST_MAX - 8U);
event_ctx_next_event_set(tsd, &ctx,
THREAD_NEXT_EVENT_FAST_MAX);
#define E(event, condition, is_alloc) \
if (is_alloc && condition) { \
event##_event_wait_set(tsd, THREAD_NEXT_EVENT_FAST_MAX);\
}
ITERATE_OVER_ALL_EVENTS
#undef E
void *p = malloc(16U);
@ -20,14 +24,20 @@ TEST_END
TEST_BEGIN(test_next_event_fast_resume) {
tsd_t *tsd = tsd_fetch();
thread_allocated_last_event_set(tsd, 0);
thread_allocated_set(tsd,
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 8U);
thread_allocated_next_event_set(tsd,
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U);
#define E(event, condition) \
event##_event_wait_set(tsd, \
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U);
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
event_ctx_last_event_set(&ctx, 0);
event_ctx_current_bytes_set(&ctx,
THREAD_NEXT_EVENT_FAST_MAX + 8U);
event_ctx_next_event_set(tsd, &ctx,
THREAD_NEXT_EVENT_FAST_MAX + 16U);
#define E(event, condition, is_alloc) \
if (is_alloc && condition) { \
event##_event_wait_set(tsd, \
THREAD_NEXT_EVENT_FAST_MAX + 16U); \
}
ITERATE_OVER_ALL_EVENTS
#undef E
void *p = malloc(SC_LOOKUP_MAXCLASS);
@ -42,7 +52,7 @@ TEST_BEGIN(test_event_rollback) {
size_t count = 10;
uint64_t thread_allocated = thread_allocated_get(tsd);
while (count-- != 0) {
thread_event_rollback(tsd, diff);
thread_alloc_event_rollback(tsd, diff);
uint64_t thread_allocated_after = thread_allocated_get(tsd);
assert_u64_eq(thread_allocated - thread_allocated_after, diff,
"thread event counters are not properly rolled back");