project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement deallocation events.

Browse Source 
Make the event module to accept two event types, and pass around the event
context.  Use bytes-based events to trigger tcache GC on deallocation, and get
rid of the tcache ticker.
This commit is contained in:
Qi Wang
2020-01-28 21:12:06 -08:00
committed by Qi Wang
parent 536ea6858e
commit 97dd79db6c
9 changed files with 306 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/jemalloc.c
View File

@@ -2154,7 +2154,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
dopts->arena_ind = 0;
}
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
/*
* If dopts->alignment > 0, then ind is still 0, but usize was computed
@@ -2181,7 +2181,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
}
if (unlikely(allocation == NULL)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
prof_alloc_rollback(tsd, tctx, true);
goto label_oom;
}
@@ -2191,7 +2191,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
ind);
if (unlikely(allocation == NULL)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
goto label_oom;
}
}
@@ -2575,7 +2575,6 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
if (config_prof && opt_prof) {
prof_free(tsd, ptr, usize, &alloc_ctx);
}
*tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path)) {
idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
@@ -2584,6 +2583,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
true);
}
thread_dalloc_event(tsd, usize);
}
JEMALLOC_ALWAYS_INLINE void
@@ -2645,14 +2645,12 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
if (config_prof && opt_prof) {
prof_free(tsd, ptr, usize, ctx);
}
*tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path)) {
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
} else {
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true);
}
thread_dalloc_event(tsd, usize);
}
JEMALLOC_NOINLINE
@@ -2694,12 +2692,12 @@ free_default(void *ptr) {
JEMALLOC_ALWAYS_INLINE
bool free_fastpath(void *ptr, size_t size, bool size_hint) {
tsd_t *tsd = tsd_get(false);
if (unlikely(!tsd || !tsd_fast(tsd))) {
return false;
}
szind_t szind;
if (!size_hint) {
if (unlikely(!tsd || !tsd_fast(tsd))) {
return false;
}
bool slab;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd),
@@ -2711,6 +2709,15 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
}
assert(szind != SC_NSIZES);
} else {
/*
* The size hinted fastpath does not involve rtree lookup, thus
* can tolerate an uninitialized tsd. This allows the tsd_fast
* check to be folded into the branch testing fast_threshold
* (set to 0 when !tsd_fast).
*/
if (unlikely(!tsd)) {
return false;
}
/*
* Check for both sizes that are too large, and for sampled
* objects. Sampled objects are always page-aligned. The
@@ -2722,19 +2729,26 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
}
szind = sz_size2index_lookup(size);
}
uint64_t deallocated, threshold;
thread_event_free_fastpath_ctx(tsd, &deallocated, &threshold, size_hint);
tcache_t *tcache = tsd_tcachep_get(tsd);
if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
size_t usize = sz_index2size(szind);
uint64_t deallocated_after = deallocated + usize;
/*
* Check for events and tsd non-nominal (fast_threshold will be set to
* 0) in a single branch.
*/
if (unlikely(deallocated_after >= threshold)) {
return false;
}
tcache_t *tcache = tsd_tcachep_get(tsd);
cache_bin_t *bin = tcache_small_bin_get(tcache, szind);
if (!cache_bin_dalloc_easy(bin, ptr)) {
return false;
}
size_t usize = sz_index2size(szind);
*tsd_thread_deallocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) = deallocated_after;
return true;
}
@@ -3144,11 +3158,11 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
goto label_oom;
}
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
zero, tcache, arena, &alloc_ctx, &hook_args);
if (unlikely(p == NULL)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
goto label_oom;
}
} else {
@@ -3158,11 +3172,10 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
goto label_oom;
}
usize = isalloc(tsd_tsdn(tsd), p);
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
}
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
*tsd_thread_deallocatedp_get(tsd) += old_usize;
thread_dalloc_event(tsd, old_usize);
UTRACE(ptr, size, p);
check_entry_exit_locking(tsd_tsdn(tsd));
@@ -3337,7 +3350,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
usize_max = SC_LARGE_MAXCLASS;
}
}
thread_event(tsd, usize_max);
thread_alloc_event(tsd, usize_max);
bool prof_active = prof_active_get_unlocked();
prof_tctx_t *tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
@@ -3350,7 +3363,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
extra, alignment, zero);
}
if (usize <= usize_max) {
thread_event_rollback(tsd, usize_max - usize);
thread_alloc_event_rollback(tsd, usize_max - usize);
} else {
/*
* For downsizing request, usize_max can be less than usize.
@@ -3359,7 +3372,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
* to xallocx(), the entire usize will be rolled back if it's
* equal to the old usize.
*/
thread_event(tsd, usize - usize_max);
thread_alloc_event(tsd, usize - usize_max);
}
/*
@@ -3438,7 +3451,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
} else {
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
extra, alignment, zero);
thread_event(tsd, usize);
thread_alloc_event(tsd, usize);
}
/*
@@ -3448,12 +3461,10 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
assert(iealloc(tsd_tsdn(tsd), ptr) == old_edata);
if (unlikely(usize == old_usize)) {
thread_event_rollback(tsd, usize);
thread_alloc_event_rollback(tsd, usize);
goto label_not_resized;
}
*tsd_thread_deallocatedp_get(tsd) += old_usize;
thread_dalloc_event(tsd, old_usize);
label_not_resized:
if (unlikely(!tsd_fast(tsd))) {
uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};

2
src/tcache.c
View File

@@ -437,8 +437,6 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
tcache->next_gc_bin = 0;
tcache->arena = NULL;
ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS);
memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS));

161
src/thread_event.c
View File

@@ -12,14 +12,14 @@
static bool thread_event_active = false;
/* TSD event init function signatures. */
#define E(event, condition) \
#define E(event, condition_unused, is_alloc_event_unused) \
static void tsd_thread_##event##_event_init(tsd_t *tsd);
ITERATE_OVER_ALL_EVENTS
#undef E
/* Event handler function signatures. */
#define E(event, condition) \
#define E(event, condition_unused, is_alloc_event_unused) \
static void thread_##event##_event_handler(tsd_t *tsd);
ITERATE_OVER_ALL_EVENTS
@@ -32,6 +32,12 @@ tsd_thread_tcache_gc_event_init(tsd_t *tsd) {
thread_tcache_gc_event_update(tsd, TCACHE_GC_INCR_BYTES);
}
static void
tsd_thread_tcache_gc_dalloc_event_init(tsd_t *tsd) {
assert(TCACHE_GC_INCR_BYTES > 0);
thread_tcache_gc_dalloc_event_update(tsd, TCACHE_GC_INCR_BYTES);
}
static void
tsd_thread_prof_sample_event_init(tsd_t *tsd) {
assert(config_prof && opt_prof);
@@ -46,17 +52,30 @@ tsd_thread_stats_interval_event_init(tsd_t *tsd) {
}
/* Handler functions. */
static void
thread_tcache_gc_event_handler(tsd_t *tsd) {
tcache_gc_event(tsd_t *tsd) {
assert(TCACHE_GC_INCR_BYTES > 0);
assert(tcache_gc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_event_init(tsd);
tcache_t *tcache = tcache_get(tsd);
if (tcache != NULL) {
tcache_event_hard(tsd, tcache);
}
}
static void
thread_tcache_gc_event_handler(tsd_t *tsd) {
assert(tcache_gc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_event_init(tsd);
tcache_gc_event(tsd);
}
static void
thread_tcache_gc_dalloc_event_handler(tsd_t *tsd) {
assert(tcache_gc_dalloc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_dalloc_event_init(tsd);
tcache_gc_event(tsd);
}
static void
thread_prof_sample_event_handler(tsd_t *tsd) {
assert(config_prof && opt_prof);
@@ -96,12 +115,12 @@ thread_stats_interval_event_handler(tsd_t *tsd) {
/* Per event facilities done. */
static uint64_t
thread_allocated_next_event_compute(tsd_t *tsd) {
thread_next_event_compute(tsd_t *tsd, bool is_alloc) {
uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
bool no_event_on = true;
#define E(event, condition) \
if (condition) { \
#define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition) { \
no_event_on = false; \
uint64_t event_wait = \
event##_event_wait_get(tsd); \
@@ -119,15 +138,15 @@ thread_allocated_next_event_compute(tsd_t *tsd) {
return wait;
}
void
thread_event_assert_invariants_debug(tsd_t *tsd) {
uint64_t thread_allocated = thread_allocated_get(tsd);
uint64_t last_event = thread_allocated_last_event_get(tsd);
uint64_t next_event = thread_allocated_next_event_get(tsd);
uint64_t next_event_fast = thread_allocated_next_event_fast_get(tsd);
static void
thread_event_assert_invariants_impl(tsd_t *tsd, event_ctx_t *ctx) {
uint64_t current_bytes = event_ctx_current_bytes_get(ctx);
uint64_t last_event = event_ctx_last_event_get(ctx);
uint64_t next_event = event_ctx_next_event_get(ctx);
uint64_t next_event_fast = event_ctx_next_event_fast_get(ctx);
assert(last_event != next_event);
if (next_event > THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX ||
if (next_event > THREAD_NEXT_EVENT_FAST_MAX ||
!tsd_fast(tsd)) {
assert(next_event_fast == 0U);
} else {
@@ -138,10 +157,9 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
uint64_t interval = next_event - last_event;
/* The subtraction is intentionally susceptible to underflow. */
assert(thread_allocated - last_event < interval);
uint64_t min_wait = thread_allocated_next_event_compute(tsd);
assert(current_bytes - last_event < interval);
uint64_t min_wait = thread_next_event_compute(tsd,
event_ctx_is_alloc(ctx));
/*
* next_event should have been pushed up only except when no event is
* on and the TSD is just initialized. The last_event == 0U guard
@@ -153,6 +171,16 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
(interval < min_wait && interval == THREAD_EVENT_MAX_INTERVAL));
}
void
thread_event_assert_invariants_debug(tsd_t *tsd) {
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
thread_event_assert_invariants_impl(tsd, &ctx);
event_ctx_get(tsd, &ctx, false);
thread_event_assert_invariants_impl(tsd, &ctx);
}
/*
* Synchronization around the fast threshold in tsd --
* There are two threads to consider in the synchronization here:
@@ -200,39 +228,50 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
* of the owner thread's next_event_fast, but that's always safe (it just sends
* it down the slow path earlier).
*/
static void
event_ctx_next_event_fast_update(event_ctx_t *ctx) {
uint64_t next_event = event_ctx_next_event_get(ctx);
uint64_t next_event_fast = (next_event <=
THREAD_NEXT_EVENT_FAST_MAX) ? next_event : 0U;
event_ctx_next_event_fast_set(ctx, next_event_fast);
}
void
thread_event_recompute_fast_threshold(tsd_t *tsd) {
if (tsd_state_get(tsd) != tsd_state_nominal) {
/* Check first because this is also called on purgatory. */
thread_allocated_next_event_fast_set_non_nominal(tsd);
thread_next_event_fast_set_non_nominal(tsd);
return;
}
uint64_t next_event = thread_allocated_next_event_get(tsd);
uint64_t next_event_fast = (next_event <=
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX) ? next_event : 0U;
thread_allocated_next_event_fast_set(tsd, next_event_fast);
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
event_ctx_next_event_fast_update(&ctx);
event_ctx_get(tsd, &ctx, false);
event_ctx_next_event_fast_update(&ctx);
atomic_fence(ATOMIC_SEQ_CST);
if (tsd_state_get(tsd) != tsd_state_nominal) {
thread_allocated_next_event_fast_set_non_nominal(tsd);
thread_next_event_fast_set_non_nominal(tsd);
}
}
static void
thread_event_adjust_thresholds_helper(tsd_t *tsd, uint64_t wait) {
thread_event_adjust_thresholds_helper(tsd_t *tsd, event_ctx_t *ctx,
uint64_t wait) {
assert(wait <= THREAD_EVENT_MAX_START_WAIT);
uint64_t next_event = thread_allocated_last_event_get(tsd) + (wait <=
uint64_t next_event = event_ctx_last_event_get(ctx) + (wait <=
THREAD_EVENT_MAX_INTERVAL ? wait : THREAD_EVENT_MAX_INTERVAL);
thread_allocated_next_event_set(tsd, next_event);
event_ctx_next_event_set(tsd, ctx, next_event);
}
static uint64_t
thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
bool allow_event_trigger) {
bool is_alloc, bool allow_event_trigger) {
uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
#define E(event, condition) \
if (condition) { \
#define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition) { \
uint64_t event_wait = event##_event_wait_get(tsd); \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
if (event_wait > accumbytes) { \
@@ -267,28 +306,30 @@ thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
}
void
thread_event_trigger(tsd_t *tsd, bool delay_event) {
thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event) {
/* usize has already been added to thread_allocated. */
uint64_t thread_allocated_after = thread_allocated_get(tsd);
uint64_t bytes_after = event_ctx_current_bytes_get(ctx);
/* The subtraction is intentionally susceptible to underflow. */
uint64_t accumbytes = thread_allocated_after -
thread_allocated_last_event_get(tsd);
uint64_t accumbytes = bytes_after - event_ctx_last_event_get(ctx);
/* Make sure that accumbytes cannot overflow uint64_t. */
assert(THREAD_EVENT_MAX_INTERVAL <= UINT64_MAX - SC_LARGE_MAXCLASS + 1);
thread_allocated_last_event_set(tsd, thread_allocated_after);
event_ctx_last_event_set(ctx, bytes_after);
bool allow_event_trigger = !delay_event && tsd_nominal(tsd) &&
tsd_reentrancy_level_get(tsd) == 0;
bool is_alloc = ctx->is_alloc;
uint64_t wait = thread_event_trigger_batch_update(tsd, accumbytes,
allow_event_trigger);
thread_event_adjust_thresholds_helper(tsd, wait);
is_alloc, allow_event_trigger);
thread_event_adjust_thresholds_helper(tsd, ctx, wait);
thread_event_assert_invariants(tsd);
#define E(event, condition) \
if (condition && event##_event_wait_get(tsd) == 0U) { \
#define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition && \
event##_event_wait_get(tsd) == 0U) { \
assert(allow_event_trigger); \
thread_##event##_event_handler(tsd); \
}
@@ -300,19 +341,23 @@ thread_event_trigger(tsd_t *tsd, bool delay_event) {
}
void
thread_event_rollback(tsd_t *tsd, size_t diff) {
thread_alloc_event_rollback(tsd_t *tsd, size_t diff) {
thread_event_assert_invariants(tsd);
if (diff == 0U) {
return;
}
uint64_t thread_allocated = thread_allocated_get(tsd);
/* Rollback happens only on alloc events. */
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, true);
uint64_t thread_allocated = event_ctx_current_bytes_get(&ctx);
/* The subtraction is intentionally susceptible to underflow. */
uint64_t thread_allocated_rollback = thread_allocated - diff;
thread_allocated_set(tsd, thread_allocated_rollback);
event_ctx_current_bytes_set(&ctx, thread_allocated_rollback);
uint64_t last_event = thread_allocated_last_event_get(tsd);
uint64_t last_event = event_ctx_last_event_get(&ctx);
/* Both subtractions are intentionally susceptible to underflow. */
if (thread_allocated_rollback - last_event <=
thread_allocated - last_event) {
@@ -320,14 +365,14 @@ thread_event_rollback(tsd_t *tsd, size_t diff) {
return;
}
thread_allocated_last_event_set(tsd, thread_allocated_rollback);
event_ctx_last_event_set(&ctx, thread_allocated_rollback);
/* The subtraction is intentionally susceptible to underflow. */
uint64_t wait_diff = last_event - thread_allocated_rollback;
assert(wait_diff <= diff);
#define E(event, condition) \
if (condition) { \
#define E(event, condition, alloc_event) \
if (alloc_event == true && condition) { \
uint64_t event_wait = event##_event_wait_get(tsd); \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
if (event_wait > 0U) { \
@@ -347,27 +392,29 @@ thread_event_rollback(tsd_t *tsd, size_t diff) {
ITERATE_OVER_ALL_EVENTS
#undef E
thread_event_update(tsd);
thread_event_update(tsd, true);
}
void
thread_event_update(tsd_t *tsd) {
uint64_t wait = thread_allocated_next_event_compute(tsd);
thread_event_adjust_thresholds_helper(tsd, wait);
thread_event_update(tsd_t *tsd, bool is_alloc) {
event_ctx_t ctx;
event_ctx_get(tsd, &ctx, is_alloc);
uint64_t last_event = thread_allocated_last_event_get(tsd);
uint64_t wait = thread_next_event_compute(tsd, is_alloc);
thread_event_adjust_thresholds_helper(tsd, &ctx, wait);
uint64_t last_event = event_ctx_last_event_get(&ctx);
/* Both subtractions are intentionally susceptible to underflow. */
if (thread_allocated_get(tsd) - last_event >=
thread_allocated_next_event_get(tsd) - last_event) {
thread_event_trigger(tsd, true);
if (event_ctx_current_bytes_get(&ctx) - last_event >=
event_ctx_next_event_get(&ctx) - last_event) {
thread_event_trigger(tsd, &ctx, true);
} else {
thread_event_assert_invariants(tsd);
}
}
void thread_event_boot() {
#define E(event, condition) \
#define E(event, condition, ignored) \
if (condition) { \
thread_event_active = true; \
}
@@ -377,7 +424,7 @@ void thread_event_boot() {
}
void tsd_thread_event_init(tsd_t *tsd) {
#define E(event, condition) \
#define E(event, condition, is_alloc_event_unused) \
if (condition) { \
tsd_thread_##event##_event_init(tsd); \
}

2
src/tsd.c
View File

@@ -119,7 +119,7 @@ tsd_force_recompute(tsdn_t *tsdn) {
tsd_state_nominal_recompute, ATOMIC_RELAXED);
/* See comments in thread_event_recompute_fast_threshold(). */
atomic_fence(ATOMIC_SEQ_CST);
thread_allocated_next_event_fast_set_non_nominal(remote_tsd);
thread_next_event_fast_set_non_nominal(remote_tsd);
}
malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock);
}