project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Abbreviate thread-event to te.

Browse Source
This commit is contained in:
Qi Wang 2020-01-30 16:31:45 -08:00 committed by Qi Wang
parent 5e500523a0
commit e896522616
7 changed files with 201 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
include/jemalloc/internal/thread_event.h
View File

@ -3,39 +3,40 @@
#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/tsd.h"
/* "te" is short for "thread_event" */
/* /*
* Maximum threshold on thread_(de)allocated_next_event_fast, so that there is * Maximum threshold on thread_(de)allocated_next_event_fast, so that there is
* no need to check overflow in malloc fast path. (The allocation size in malloc * no need to check overflow in malloc fast path. (The allocation size in malloc
* fast path never exceeds SC_LOOKUP_MAXCLASS.) * fast path never exceeds SC_LOOKUP_MAXCLASS.)
*/ */
#define THREAD_NEXT_EVENT_FAST_MAX \ #define TE_NEXT_EVENT_FAST_MAX (UINT64_MAX - SC_LOOKUP_MAXCLASS + 1U)
(UINT64_MAX - SC_LOOKUP_MAXCLASS + 1U)
/* /*
* The max interval helps make sure that malloc stays on the fast path in the * The max interval helps make sure that malloc stays on the fast path in the
* common case, i.e. thread_allocated < thread_allocated_next_event_fast. When * common case, i.e. thread_allocated < thread_allocated_next_event_fast. When
* thread_allocated is within an event's distance to THREAD_NEXT_EVENT_FAST_MAX * thread_allocated is within an event's distance to TE_NEXT_EVENT_FAST_MAX
* above, thread_allocated_next_event_fast is wrapped around and we fall back to * above, thread_allocated_next_event_fast is wrapped around and we fall back to
* the medium-fast path. The max interval makes sure that we're not staying on * the medium-fast path. The max interval makes sure that we're not staying on
* the fallback case for too long, even if there's no active event or if all * the fallback case for too long, even if there's no active event or if all
* active events have long wait times. * active events have long wait times.
*/ */
#define THREAD_EVENT_MAX_INTERVAL ((uint64_t)(4U << 20)) #define TE_MAX_INTERVAL ((uint64_t)(4U << 20))
typedef struct event_ctx_s { typedef struct te_ctx_s {
bool is_alloc; bool is_alloc;
uint64_t *current; uint64_t *current;
uint64_t *last_event; uint64_t *last_event;
uint64_t *next_event; uint64_t *next_event;
uint64_t *next_event_fast; uint64_t *next_event_fast;
} event_ctx_t; } te_ctx_t;
void thread_event_assert_invariants_debug(tsd_t *tsd); void te_assert_invariants_debug(tsd_t *tsd);
void thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event); void te_event_trigger(tsd_t *tsd, te_ctx_t *ctx, bool delay_event);
void thread_alloc_event_rollback(tsd_t *tsd, size_t diff); void te_alloc_rollback(tsd_t *tsd, size_t diff);
void thread_event_update(tsd_t *tsd, bool alloc_event); void te_event_update(tsd_t *tsd, bool alloc_event);
void thread_event_recompute_fast_threshold(tsd_t *tsd); void te_recompute_fast_threshold(tsd_t *tsd);
void tsd_thread_event_init(tsd_t *tsd); void tsd_te_init(tsd_t *tsd);
/* /*
* List of all events, in the following format: * List of all events, in the following format:
@ -97,21 +98,16 @@ ITERATE_OVER_ALL_COUNTERS
* *
* Note that these can only be used on the fastpath. * Note that these can only be used on the fastpath.
*/ */
JEMALLOC_ALWAYS_INLINE uint64_t JEMALLOC_ALWAYS_INLINE void
thread_allocated_malloc_fastpath(tsd_t *tsd) { te_malloc_fastpath_ctx(tsd_t *tsd, uint64_t *allocated, uint64_t *threshold) {
return *tsd_thread_allocatedp_get_unsafe(tsd); *allocated = *tsd_thread_allocatedp_get_unsafe(tsd);
} *threshold = *tsd_thread_allocated_next_event_fastp_get_unsafe(tsd);
assert(*threshold <= TE_NEXT_EVENT_FAST_MAX);
JEMALLOC_ALWAYS_INLINE uint64_t
thread_allocated_next_event_malloc_fastpath(tsd_t *tsd) {
uint64_t v = *tsd_thread_allocated_next_event_fastp_get_unsafe(tsd);
assert(v <= THREAD_NEXT_EVENT_FAST_MAX);
return v;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
thread_event_free_fastpath_ctx(tsd_t *tsd, uint64_t *deallocated, te_free_fastpath_ctx(tsd_t *tsd, uint64_t *deallocated, uint64_t *threshold,
uint64_t *threshold, bool size_hint) { bool size_hint) {
if (!size_hint) { if (!size_hint) {
*deallocated = tsd_thread_deallocated_get(tsd); *deallocated = tsd_thread_deallocated_get(tsd);
*threshold = tsd_thread_deallocated_next_event_fast_get(tsd); *threshold = tsd_thread_deallocated_next_event_fast_get(tsd);
@ -121,50 +117,50 @@ thread_event_free_fastpath_ctx(tsd_t *tsd, uint64_t *deallocated,
*threshold = *threshold =
*tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd); *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd);
} }
assert(*threshold <= THREAD_NEXT_EVENT_FAST_MAX); assert(*threshold <= TE_NEXT_EVENT_FAST_MAX);
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
event_ctx_is_alloc(event_ctx_t *ctx) { te_ctx_is_alloc(te_ctx_t *ctx) {
return ctx->is_alloc; return ctx->is_alloc;
} }
JEMALLOC_ALWAYS_INLINE uint64_t JEMALLOC_ALWAYS_INLINE uint64_t
event_ctx_current_bytes_get(event_ctx_t *ctx) { te_ctx_current_bytes_get(te_ctx_t *ctx) {
return *ctx->current; return *ctx->current;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
event_ctx_current_bytes_set(event_ctx_t *ctx, uint64_t v) { te_ctx_current_bytes_set(te_ctx_t *ctx, uint64_t v) {
*ctx->current = v; *ctx->current = v;
} }
JEMALLOC_ALWAYS_INLINE uint64_t JEMALLOC_ALWAYS_INLINE uint64_t
event_ctx_last_event_get(event_ctx_t *ctx) { te_ctx_last_event_get(te_ctx_t *ctx) {
return *ctx->last_event; return *ctx->last_event;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
event_ctx_last_event_set(event_ctx_t *ctx, uint64_t v) { te_ctx_last_event_set(te_ctx_t *ctx, uint64_t v) {
*ctx->last_event = v; *ctx->last_event = v;
} }
/* Below 3 for next_event_fast. */ /* Below 3 for next_event_fast. */
JEMALLOC_ALWAYS_INLINE uint64_t JEMALLOC_ALWAYS_INLINE uint64_t
event_ctx_next_event_fast_get(event_ctx_t *ctx) { te_ctx_next_event_fast_get(te_ctx_t *ctx) {
uint64_t v = *ctx->next_event_fast; uint64_t v = *ctx->next_event_fast;
assert(v <= THREAD_NEXT_EVENT_FAST_MAX); assert(v <= TE_NEXT_EVENT_FAST_MAX);
return v; return v;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
event_ctx_next_event_fast_set(event_ctx_t *ctx, uint64_t v) { te_ctx_next_event_fast_set(te_ctx_t *ctx, uint64_t v) {
assert(v <= THREAD_NEXT_EVENT_FAST_MAX); assert(v <= TE_NEXT_EVENT_FAST_MAX);
*ctx->next_event_fast = v; *ctx->next_event_fast = v;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
thread_next_event_fast_set_non_nominal(tsd_t *tsd) { te_next_event_fast_set_non_nominal(tsd_t *tsd) {
/* /*
* Set the fast thresholds to zero when tsd is non-nominal. Use the * Set the fast thresholds to zero when tsd is non-nominal. Use the
* unsafe getter as this may get called during tsd init and clean up. * unsafe getter as this may get called during tsd init and clean up.
@ -175,14 +171,14 @@ thread_next_event_fast_set_non_nominal(tsd_t *tsd) {
/* For next_event. Setter also updates the fast threshold. */ /* For next_event. Setter also updates the fast threshold. */
JEMALLOC_ALWAYS_INLINE uint64_t JEMALLOC_ALWAYS_INLINE uint64_t
event_ctx_next_event_get(event_ctx_t *ctx) { te_ctx_next_event_get(te_ctx_t *ctx) {
return *ctx->next_event; return *ctx->next_event;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
event_ctx_next_event_set(tsd_t *tsd, event_ctx_t *ctx, uint64_t v) { te_ctx_next_event_set(tsd_t *tsd, te_ctx_t *ctx, uint64_t v) {
*ctx->next_event = v; *ctx->next_event = v;
thread_event_recompute_fast_threshold(tsd); te_recompute_fast_threshold(tsd);
} }
/* /*
@ -190,22 +186,22 @@ event_ctx_next_event_set(tsd_t *tsd, event_ctx_t *ctx, uint64_t v) {
* a consistent state, which forms the invariants before and after each round * a consistent state, which forms the invariants before and after each round
* of thread event handling that we can rely on and need to promise. * of thread event handling that we can rely on and need to promise.
* The invariants are only temporarily violated in the middle of: * The invariants are only temporarily violated in the middle of:
* (a) thread_event() if an event is triggered (the thread_event_trigger() call * (a) event_advance() if an event is triggered (the te_event_trigger() call
* at the end will restore the invariants), * at the end will restore the invariants),
* (b) thread_##event##_event_update() (the thread_event_update() call at the * (b) te_##event##_event_update() (the te_event_update() call at the
* end will restore the invariants), or * end will restore the invariants), or
* (c) thread_alloc_event_rollback() if the rollback falls below the last_event * (c) te_alloc_rollback() if the rollback falls below the last_event
* (the thread_event_update() call at the end will restore the invariants). * (the te_event_update() call at the end will restore the invariants).
*/ */
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
thread_event_assert_invariants(tsd_t *tsd) { te_assert_invariants(tsd_t *tsd) {
if (config_debug) { if (config_debug) {
thread_event_assert_invariants_debug(tsd); te_assert_invariants_debug(tsd);
} }
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
event_ctx_get(tsd_t *tsd, event_ctx_t *ctx, bool is_alloc) { te_ctx_get(tsd_t *tsd, te_ctx_t *ctx, bool is_alloc) {
ctx->is_alloc = is_alloc; ctx->is_alloc = is_alloc;
if (is_alloc) { if (is_alloc) {
ctx->current = tsd_thread_allocatedp_get(tsd); ctx->current = tsd_thread_allocatedp_get(tsd);
@ -223,51 +219,51 @@ event_ctx_get(tsd_t *tsd, event_ctx_t *ctx, bool is_alloc) {
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
thread_event_advance(tsd_t *tsd, size_t usize, bool is_alloc) { te_event_advance(tsd_t *tsd, size_t usize, bool is_alloc) {
thread_event_assert_invariants(tsd); te_assert_invariants(tsd);
event_ctx_t ctx; te_ctx_t ctx;
event_ctx_get(tsd, &ctx, is_alloc); te_ctx_get(tsd, &ctx, is_alloc);
uint64_t bytes_before = event_ctx_current_bytes_get(&ctx); uint64_t bytes_before = te_ctx_current_bytes_get(&ctx);
event_ctx_current_bytes_set(&ctx, bytes_before + usize); te_ctx_current_bytes_set(&ctx, bytes_before + usize);
/* The subtraction is intentionally susceptible to underflow. */ /* The subtraction is intentionally susceptible to underflow. */
if (likely(usize < event_ctx_next_event_get(&ctx) - bytes_before)) { if (likely(usize < te_ctx_next_event_get(&ctx) - bytes_before)) {
thread_event_assert_invariants(tsd); te_assert_invariants(tsd);
} else { } else {
thread_event_trigger(tsd, &ctx, false); te_event_trigger(tsd, &ctx, false);
} }
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
thread_dalloc_event(tsd_t *tsd, size_t usize) { thread_dalloc_event(tsd_t *tsd, size_t usize) {
thread_event_advance(tsd, usize, false); te_event_advance(tsd, usize, false);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
thread_alloc_event(tsd_t *tsd, size_t usize) { thread_alloc_event(tsd_t *tsd, size_t usize) {
thread_event_advance(tsd, usize, true); te_event_advance(tsd, usize, true);
} }
#define E(event, condition, is_alloc) \ #define E(event, condition, is_alloc) \
JEMALLOC_ALWAYS_INLINE void \ JEMALLOC_ALWAYS_INLINE void \
thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) { \ te_##event##_event_update(tsd_t *tsd, uint64_t event_wait) { \
thread_event_assert_invariants(tsd); \ te_assert_invariants(tsd); \
assert(condition); \ assert(condition); \
assert(tsd_nominal(tsd)); \ assert(tsd_nominal(tsd)); \
assert(tsd_reentrancy_level_get(tsd) == 0); \ assert(tsd_reentrancy_level_get(tsd) == 0); \
assert(event_wait > 0U); \ assert(event_wait > 0U); \
if (THREAD_EVENT_MIN_START_WAIT > 1U && \ if (TE_MIN_START_WAIT > 1U && \
unlikely(event_wait < THREAD_EVENT_MIN_START_WAIT)) { \ unlikely(event_wait < TE_MIN_START_WAIT)) { \
event_wait = THREAD_EVENT_MIN_START_WAIT; \ event_wait = TE_MIN_START_WAIT; \
} \ } \
if (THREAD_EVENT_MAX_START_WAIT < UINT64_MAX && \ if (TE_MAX_START_WAIT < UINT64_MAX && \
unlikely(event_wait > THREAD_EVENT_MAX_START_WAIT)) { \ unlikely(event_wait > TE_MAX_START_WAIT)) { \
event_wait = THREAD_EVENT_MAX_START_WAIT; \ event_wait = TE_MAX_START_WAIT; \
} \ } \
event##_event_wait_set(tsd, event_wait); \ event##_event_wait_set(tsd, event_wait); \
thread_event_update(tsd, is_alloc); \ te_event_update(tsd, is_alloc); \
} }
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS

18
include/jemalloc/internal/tsd.h
View File

@ -104,10 +104,10 @@ typedef void (*test_callback_t)(int *);
MALLOC_TEST_TSD MALLOC_TEST_TSD
/* /*
* THREAD_EVENT_MIN_START_WAIT should not exceed the minimal allocation usize. * TE_MIN_START_WAIT should not exceed the minimal allocation usize.
*/ */
#define THREAD_EVENT_MIN_START_WAIT ((uint64_t)1U) #define TE_MIN_START_WAIT ((uint64_t)1U)
#define THREAD_EVENT_MAX_START_WAIT UINT64_MAX #define TE_MAX_START_WAIT UINT64_MAX
#define TSD_INITIALIZER { \ #define TSD_INITIALIZER { \
/* state */ ATOMIC_INIT(tsd_state_uninitialized), \ /* state */ ATOMIC_INIT(tsd_state_uninitialized), \
@ -121,14 +121,14 @@ typedef void (*test_callback_t)(int *);
/* thread_deallocated_next_event_fast */ 0, \ /* thread_deallocated_next_event_fast */ 0, \
/* rtree_ctx */ RTREE_CTX_ZERO_INITIALIZER, \ /* rtree_ctx */ RTREE_CTX_ZERO_INITIALIZER, \
/* thread_allocated_last_event */ 0, \ /* thread_allocated_last_event */ 0, \
/* thread_allocated_next_event */ THREAD_EVENT_MIN_START_WAIT, \ /* thread_allocated_next_event */ TE_MIN_START_WAIT, \
/* thread_deallocated_last_event */ 0, \ /* thread_deallocated_last_event */ 0, \
/* thread_deallocated_next_event */ THREAD_EVENT_MIN_START_WAIT, \ /* thread_deallocated_next_event */ TE_MIN_START_WAIT, \
/* tcache_gc_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ /* tcache_gc_event_wait */ TE_MIN_START_WAIT, \
/* tcache_gc_dalloc_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ /* tcache_gc_dalloc_event_wait */ TE_MIN_START_WAIT, \
/* prof_sample_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ /* prof_sample_event_wait */ TE_MIN_START_WAIT, \
/* prof_sample_last_event */ 0, \ /* prof_sample_last_event */ 0, \
/* stats_interval_event_wait */ THREAD_EVENT_MIN_START_WAIT, \ /* stats_interval_event_wait */ TE_MIN_START_WAIT, \
/* stats_interval_last_event */ 0, \ /* stats_interval_last_event */ 0, \
/* prof_tdata */ NULL, \ /* prof_tdata */ NULL, \
/* prng_state */ 0, \ /* prng_state */ 0, \

17
src/jemalloc.c
View File

@ -2180,7 +2180,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
} }
if (unlikely(allocation == NULL)) { if (unlikely(allocation == NULL)) {
thread_alloc_event_rollback(tsd, usize); te_alloc_rollback(tsd, usize);
prof_alloc_rollback(tsd, tctx, true); prof_alloc_rollback(tsd, tctx, true);
goto label_oom; goto label_oom;
} }
@ -2190,7 +2190,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize, allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
ind); ind);
if (unlikely(allocation == NULL)) { if (unlikely(allocation == NULL)) {
thread_alloc_event_rollback(tsd, usize); te_alloc_rollback(tsd, usize);
goto label_oom; goto label_oom;
} }
} }
@ -2386,15 +2386,14 @@ je_malloc(size_t size) {
* it's not always needed in the core allocation logic. * it's not always needed in the core allocation logic.
*/ */
size_t usize; size_t usize;
sz_size2index_usize_fastpath(size, &ind, &usize); sz_size2index_usize_fastpath(size, &ind, &usize);
/* Fast path relies on size being a bin. */ /* Fast path relies on size being a bin. */
assert(ind < SC_NBINS); assert(ind < SC_NBINS);
assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS) && assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS) &&
(size <= SC_SMALL_MAXCLASS)); (size <= SC_SMALL_MAXCLASS));
uint64_t allocated = thread_allocated_malloc_fastpath(tsd); uint64_t allocated, threshold;
uint64_t threshold = thread_allocated_next_event_malloc_fastpath(tsd); te_malloc_fastpath_ctx(tsd, &allocated, &threshold);
uint64_t allocated_after = allocated + usize; uint64_t allocated_after = allocated + usize;
/* /*
* The ind and usize might be uninitialized (or partially) before * The ind and usize might be uninitialized (or partially) before
@ -2729,7 +2728,7 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
szind = sz_size2index_lookup(size); szind = sz_size2index_lookup(size);
} }
uint64_t deallocated, threshold; uint64_t deallocated, threshold;
thread_event_free_fastpath_ctx(tsd, &deallocated, &threshold, size_hint); te_free_fastpath_ctx(tsd, &deallocated, &threshold, size_hint);
size_t usize = sz_index2size(szind); size_t usize = sz_index2size(szind);
uint64_t deallocated_after = deallocated + usize; uint64_t deallocated_after = deallocated + usize;
@ -3161,7 +3160,7 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
zero, tcache, arena, &alloc_ctx, &hook_args); zero, tcache, arena, &alloc_ctx, &hook_args);
if (unlikely(p == NULL)) { if (unlikely(p == NULL)) {
thread_alloc_event_rollback(tsd, usize); te_alloc_rollback(tsd, usize);
goto label_oom; goto label_oom;
} }
} else { } else {
@ -3362,7 +3361,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
extra, alignment, zero); extra, alignment, zero);
} }
if (usize <= usize_max) { if (usize <= usize_max) {
thread_alloc_event_rollback(tsd, usize_max - usize); te_alloc_rollback(tsd, usize_max - usize);
} else { } else {
/* /*
* For downsizing request, usize_max can be less than usize. * For downsizing request, usize_max can be less than usize.
@ -3460,7 +3459,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
assert(iealloc(tsd_tsdn(tsd), ptr) == old_edata); assert(iealloc(tsd_tsdn(tsd), ptr) == old_edata);
if (unlikely(usize == old_usize)) { if (unlikely(usize == old_usize)) {
thread_alloc_event_rollback(tsd, usize); te_alloc_rollback(tsd, usize);
goto label_not_resized; goto label_not_resized;
} }
thread_dalloc_event(tsd, old_usize); thread_dalloc_event(tsd, old_usize);

5
src/prof.c
View File

@ -444,8 +444,7 @@ prof_sample_threshold_update(tsd_t *tsd) {
} }
if (lg_prof_sample == 0) { if (lg_prof_sample == 0) {
thread_prof_sample_event_update(tsd, te_prof_sample_event_update(tsd, TE_MIN_START_WAIT);
THREAD_EVENT_MIN_START_WAIT);
return; return;
} }
@ -472,7 +471,7 @@ prof_sample_threshold_update(tsd_t *tsd) {
uint64_t bytes_until_sample = (uint64_t)(log(u) / uint64_t bytes_until_sample = (uint64_t)(log(u) /
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
+ (uint64_t)1U; + (uint64_t)1U;
thread_prof_sample_event_update(tsd, bytes_until_sample); te_prof_sample_event_update(tsd, bytes_until_sample);
#endif #endif
} }

217
src/thread_event.c
View File

@ -6,46 +6,45 @@
/* TSD event init function signatures. */ /* TSD event init function signatures. */
#define E(event, condition_unused, is_alloc_event_unused) \ #define E(event, condition_unused, is_alloc_event_unused) \
static void tsd_thread_##event##_event_init(tsd_t *tsd); static void te_tsd_##event##_event_init(tsd_t *tsd);
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
/* Event handler function signatures. */ /* Event handler function signatures. */
#define E(event, condition_unused, is_alloc_event_unused) \ #define E(event, condition_unused, is_alloc_event_unused) \
static void thread_##event##_event_handler(tsd_t *tsd); static void te_##event##_event_handler(tsd_t *tsd);
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
/* (Re)Init functions. */ /* (Re)Init functions. */
static void static void
tsd_thread_tcache_gc_event_init(tsd_t *tsd) { te_tsd_tcache_gc_event_init(tsd_t *tsd) {
assert(TCACHE_GC_INCR_BYTES > 0); assert(TCACHE_GC_INCR_BYTES > 0);
thread_tcache_gc_event_update(tsd, TCACHE_GC_INCR_BYTES); te_tcache_gc_event_update(tsd, TCACHE_GC_INCR_BYTES);
} }
static void static void
tsd_thread_tcache_gc_dalloc_event_init(tsd_t *tsd) { te_tsd_tcache_gc_dalloc_event_init(tsd_t *tsd) {
assert(TCACHE_GC_INCR_BYTES > 0); assert(TCACHE_GC_INCR_BYTES > 0);
thread_tcache_gc_dalloc_event_update(tsd, TCACHE_GC_INCR_BYTES); te_tcache_gc_dalloc_event_update(tsd, TCACHE_GC_INCR_BYTES);
} }
static void static void
tsd_thread_prof_sample_event_init(tsd_t *tsd) { te_tsd_prof_sample_event_init(tsd_t *tsd) {
assert(config_prof && opt_prof); assert(config_prof && opt_prof);
prof_sample_threshold_update(tsd); prof_sample_threshold_update(tsd);
} }
static void static void
tsd_thread_stats_interval_event_init(tsd_t *tsd) { te_tsd_stats_interval_event_init(tsd_t *tsd) {
assert(opt_stats_interval >= 0); assert(opt_stats_interval >= 0);
uint64_t interval = stats_interval_accum_batch_size(); uint64_t interval = stats_interval_accum_batch_size();
thread_stats_interval_event_update(tsd, interval); te_stats_interval_event_update(tsd, interval);
} }
/* Handler functions. */ /* Handler functions. */
static void static void
tcache_gc_event(tsd_t *tsd) { tcache_gc_event(tsd_t *tsd) {
assert(TCACHE_GC_INCR_BYTES > 0); assert(TCACHE_GC_INCR_BYTES > 0);
@ -56,21 +55,21 @@ tcache_gc_event(tsd_t *tsd) {
} }
static void static void
thread_tcache_gc_event_handler(tsd_t *tsd) { te_tcache_gc_event_handler(tsd_t *tsd) {
assert(tcache_gc_event_wait_get(tsd) == 0U); assert(tcache_gc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_event_init(tsd); te_tsd_tcache_gc_event_init(tsd);
tcache_gc_event(tsd); tcache_gc_event(tsd);
} }
static void static void
thread_tcache_gc_dalloc_event_handler(tsd_t *tsd) { te_tcache_gc_dalloc_event_handler(tsd_t *tsd) {
assert(tcache_gc_dalloc_event_wait_get(tsd) == 0U); assert(tcache_gc_dalloc_event_wait_get(tsd) == 0U);
tsd_thread_tcache_gc_dalloc_event_init(tsd); te_tsd_tcache_gc_dalloc_event_init(tsd);
tcache_gc_event(tsd); tcache_gc_event(tsd);
} }
static void static void
thread_prof_sample_event_handler(tsd_t *tsd) { te_prof_sample_event_handler(tsd_t *tsd) {
assert(config_prof && opt_prof); assert(config_prof && opt_prof);
assert(prof_sample_event_wait_get(tsd) == 0U); assert(prof_sample_event_wait_get(tsd) == 0U);
uint64_t last_event = thread_allocated_last_event_get(tsd); uint64_t last_event = thread_allocated_last_event_get(tsd);
@ -87,13 +86,13 @@ thread_prof_sample_event_handler(tsd_t *tsd) {
* prof_active is turned on later, the counting for sampling * prof_active is turned on later, the counting for sampling
* can immediately resume as normal. * can immediately resume as normal.
*/ */
thread_prof_sample_event_update(tsd, te_prof_sample_event_update(tsd,
(uint64_t)(1 << lg_prof_sample)); (uint64_t)(1 << lg_prof_sample));
} }
} }
static void static void
thread_stats_interval_event_handler(tsd_t *tsd) { te_stats_interval_event_handler(tsd_t *tsd) {
assert(opt_stats_interval >= 0); assert(opt_stats_interval >= 0);
assert(stats_interval_event_wait_get(tsd) == 0U); assert(stats_interval_event_wait_get(tsd) == 0U);
uint64_t last_event = thread_allocated_last_event_get(tsd); uint64_t last_event = thread_allocated_last_event_get(tsd);
@ -103,12 +102,12 @@ thread_stats_interval_event_handler(tsd_t *tsd) {
if (stats_interval_accum(tsd, last_event - last_stats_event)) { if (stats_interval_accum(tsd, last_event - last_stats_event)) {
je_malloc_stats_print(NULL, NULL, opt_stats_interval_opts); je_malloc_stats_print(NULL, NULL, opt_stats_interval_opts);
} }
tsd_thread_stats_interval_event_init(tsd); te_tsd_stats_interval_event_init(tsd);
} }
/* Per event facilities done. */ /* Per event facilities done. */
static bool static bool
event_ctx_has_active_events(event_ctx_t *ctx) { te_ctx_has_active_events(te_ctx_t *ctx) {
assert(config_debug); assert(config_debug);
#define E(event, condition, alloc_event) \ #define E(event, condition, alloc_event) \
if (condition && alloc_event == ctx->is_alloc) { \ if (condition && alloc_event == ctx->is_alloc) { \
@ -120,13 +119,13 @@ event_ctx_has_active_events(event_ctx_t *ctx) {
} }
static uint64_t static uint64_t
thread_next_event_compute(tsd_t *tsd, bool is_alloc) { te_next_event_compute(tsd_t *tsd, bool is_alloc) {
uint64_t wait = THREAD_EVENT_MAX_START_WAIT; uint64_t wait = TE_MAX_START_WAIT;
#define E(event, condition, alloc_event) \ #define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition) { \ if (is_alloc == alloc_event && condition) { \
uint64_t event_wait = \ uint64_t event_wait = \
event##_event_wait_get(tsd); \ event##_event_wait_get(tsd); \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \ assert(event_wait <= TE_MAX_START_WAIT); \
if (event_wait > 0U && event_wait < wait) { \ if (event_wait > 0U && event_wait < wait) { \
wait = event_wait; \ wait = event_wait; \
} \ } \
@ -134,20 +133,19 @@ thread_next_event_compute(tsd_t *tsd, bool is_alloc) {
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
assert(wait <= THREAD_EVENT_MAX_START_WAIT); assert(wait <= TE_MAX_START_WAIT);
return wait; return wait;
} }
static void static void
thread_event_assert_invariants_impl(tsd_t *tsd, event_ctx_t *ctx) { te_assert_invariants_impl(tsd_t *tsd, te_ctx_t *ctx) {
uint64_t current_bytes = event_ctx_current_bytes_get(ctx); uint64_t current_bytes = te_ctx_current_bytes_get(ctx);
uint64_t last_event = event_ctx_last_event_get(ctx); uint64_t last_event = te_ctx_last_event_get(ctx);
uint64_t next_event = event_ctx_next_event_get(ctx); uint64_t next_event = te_ctx_next_event_get(ctx);
uint64_t next_event_fast = event_ctx_next_event_fast_get(ctx); uint64_t next_event_fast = te_ctx_next_event_fast_get(ctx);
assert(last_event != next_event); assert(last_event != next_event);
if (next_event > THREAD_NEXT_EVENT_FAST_MAX || if (next_event > TE_NEXT_EVENT_FAST_MAX || !tsd_fast(tsd)) {
!tsd_fast(tsd)) {
assert(next_event_fast == 0U); assert(next_event_fast == 0U);
} else { } else {
assert(next_event_fast == next_event); assert(next_event_fast == next_event);
@ -158,27 +156,26 @@ thread_event_assert_invariants_impl(tsd_t *tsd, event_ctx_t *ctx) {
/* The subtraction is intentionally susceptible to underflow. */ /* The subtraction is intentionally susceptible to underflow. */
assert(current_bytes - last_event < interval); assert(current_bytes - last_event < interval);
uint64_t min_wait = thread_next_event_compute(tsd, uint64_t min_wait = te_next_event_compute(tsd, te_ctx_is_alloc(ctx));
event_ctx_is_alloc(ctx));
/* /*
* next_event should have been pushed up only except when no event is * next_event should have been pushed up only except when no event is
* on and the TSD is just initialized. The last_event == 0U guard * on and the TSD is just initialized. The last_event == 0U guard
* below is stronger than needed, but having an exactly accurate guard * below is stronger than needed, but having an exactly accurate guard
* is more complicated to implement. * is more complicated to implement.
*/ */
assert((!event_ctx_has_active_events(ctx) && last_event == 0U) || assert((!te_ctx_has_active_events(ctx) && last_event == 0U) ||
interval == min_wait || interval == min_wait ||
(interval < min_wait && interval == THREAD_EVENT_MAX_INTERVAL)); (interval < min_wait && interval == TE_MAX_INTERVAL));
} }
void void
thread_event_assert_invariants_debug(tsd_t *tsd) { te_assert_invariants_debug(tsd_t *tsd) {
event_ctx_t ctx; te_ctx_t ctx;
event_ctx_get(tsd, &ctx, true); te_ctx_get(tsd, &ctx, true);
thread_event_assert_invariants_impl(tsd, &ctx); te_assert_invariants_impl(tsd, &ctx);
event_ctx_get(tsd, &ctx, false); te_ctx_get(tsd, &ctx, false);
thread_event_assert_invariants_impl(tsd, &ctx); te_assert_invariants_impl(tsd, &ctx);
} }
/* /*
@ -229,66 +226,65 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
* it down the slow path earlier). * it down the slow path earlier).
*/ */
static void static void
event_ctx_next_event_fast_update(event_ctx_t *ctx) { te_ctx_next_event_fast_update(te_ctx_t *ctx) {
uint64_t next_event = event_ctx_next_event_get(ctx); uint64_t next_event = te_ctx_next_event_get(ctx);
uint64_t next_event_fast = (next_event <= uint64_t next_event_fast = (next_event <= TE_NEXT_EVENT_FAST_MAX) ?
THREAD_NEXT_EVENT_FAST_MAX) ? next_event : 0U; next_event : 0U;
event_ctx_next_event_fast_set(ctx, next_event_fast); te_ctx_next_event_fast_set(ctx, next_event_fast);
} }
void void
thread_event_recompute_fast_threshold(tsd_t *tsd) { te_recompute_fast_threshold(tsd_t *tsd) {
if (tsd_state_get(tsd) != tsd_state_nominal) { if (tsd_state_get(tsd) != tsd_state_nominal) {
/* Check first because this is also called on purgatory. */ /* Check first because this is also called on purgatory. */
thread_next_event_fast_set_non_nominal(tsd); te_next_event_fast_set_non_nominal(tsd);
return; return;
} }
event_ctx_t ctx; te_ctx_t ctx;
event_ctx_get(tsd, &ctx, true); te_ctx_get(tsd, &ctx, true);
event_ctx_next_event_fast_update(&ctx); te_ctx_next_event_fast_update(&ctx);
event_ctx_get(tsd, &ctx, false); te_ctx_get(tsd, &ctx, false);
event_ctx_next_event_fast_update(&ctx); te_ctx_next_event_fast_update(&ctx);
atomic_fence(ATOMIC_SEQ_CST); atomic_fence(ATOMIC_SEQ_CST);
if (tsd_state_get(tsd) != tsd_state_nominal) { if (tsd_state_get(tsd) != tsd_state_nominal) {
thread_next_event_fast_set_non_nominal(tsd); te_next_event_fast_set_non_nominal(tsd);
} }
} }
static void static void
thread_event_adjust_thresholds_helper(tsd_t *tsd, event_ctx_t *ctx, te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx,
uint64_t wait) { uint64_t wait) {
assert(wait <= THREAD_EVENT_MAX_START_WAIT); assert(wait <= TE_MAX_START_WAIT);
uint64_t next_event = event_ctx_last_event_get(ctx) + (wait <= uint64_t next_event = te_ctx_last_event_get(ctx) + (wait <=
THREAD_EVENT_MAX_INTERVAL ? wait : THREAD_EVENT_MAX_INTERVAL); TE_MAX_INTERVAL ? wait : TE_MAX_INTERVAL);
event_ctx_next_event_set(tsd, ctx, next_event); te_ctx_next_event_set(tsd, ctx, next_event);
} }
static uint64_t static uint64_t
thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes, te_batch_accum(tsd_t *tsd, uint64_t accumbytes, bool is_alloc,
bool is_alloc, bool allow_event_trigger) { bool allow_event_trigger) {
uint64_t wait = THREAD_EVENT_MAX_START_WAIT; uint64_t wait = TE_MAX_START_WAIT;
#define E(event, condition, alloc_event) \ #define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition) { \ if (is_alloc == alloc_event && condition) { \
uint64_t event_wait = event##_event_wait_get(tsd); \ uint64_t event_wait = event##_event_wait_get(tsd); \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \ assert(event_wait <= TE_MAX_START_WAIT); \
if (event_wait > accumbytes) { \ if (event_wait > accumbytes) { \
event_wait -= accumbytes; \ event_wait -= accumbytes; \
} else { \ } else { \
event_wait = 0U; \ event_wait = 0U; \
if (!allow_event_trigger) { \ if (!allow_event_trigger) { \
event_wait = \ event_wait = TE_MIN_START_WAIT; \
THREAD_EVENT_MIN_START_WAIT; \
} \ } \
} \ } \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \ assert(event_wait <= TE_MAX_START_WAIT); \
event##_event_wait_set(tsd, event_wait); \ event##_event_wait_set(tsd, event_wait); \
/* \ /* \
* If there is a single event, then the remaining wait \ * If there is a single event, then the remaining wait \
* time may become zero, and we rely on either the \ * time may become zero, and we rely on either the \
* event handler or a thread_event_update() call later \ * event handler or a te_event_update() call later \
* to properly set next_event; if there are multiple \ * to properly set next_event; if there are multiple \
* events, then here we can get the minimum remaining \ * events, then here we can get the minimum remaining \
* wait time to the next already set event. \ * wait time to the next already set event. \
@ -301,72 +297,64 @@ thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
assert(wait <= THREAD_EVENT_MAX_START_WAIT); assert(wait <= TE_MAX_START_WAIT);
return wait; return wait;
} }
void void
thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event) { te_event_trigger(tsd_t *tsd, te_ctx_t *ctx, bool delay_event) {
/* usize has already been added to thread_allocated. */ /* usize has already been added to thread_allocated. */
uint64_t bytes_after = event_ctx_current_bytes_get(ctx); uint64_t bytes_after = te_ctx_current_bytes_get(ctx);
/* The subtraction is intentionally susceptible to underflow. */ /* The subtraction is intentionally susceptible to underflow. */
uint64_t accumbytes = bytes_after - event_ctx_last_event_get(ctx); uint64_t accumbytes = bytes_after - te_ctx_last_event_get(ctx);
/* Make sure that accumbytes cannot overflow uint64_t. */ te_ctx_last_event_set(ctx, bytes_after);
assert(THREAD_EVENT_MAX_INTERVAL <= UINT64_MAX - SC_LARGE_MAXCLASS + 1);
event_ctx_last_event_set(ctx, bytes_after);
bool allow_event_trigger = !delay_event && tsd_nominal(tsd) && bool allow_event_trigger = !delay_event && tsd_nominal(tsd) &&
tsd_reentrancy_level_get(tsd) == 0; tsd_reentrancy_level_get(tsd) == 0;
bool is_alloc = ctx->is_alloc; bool is_alloc = ctx->is_alloc;
uint64_t wait = thread_event_trigger_batch_update(tsd, accumbytes, uint64_t wait = te_batch_accum(tsd, accumbytes, is_alloc,
is_alloc, allow_event_trigger); allow_event_trigger);
thread_event_adjust_thresholds_helper(tsd, ctx, wait); te_adjust_thresholds_helper(tsd, ctx, wait);
thread_event_assert_invariants(tsd);
te_assert_invariants(tsd);
#define E(event, condition, alloc_event) \ #define E(event, condition, alloc_event) \
if (is_alloc == alloc_event && condition && \ if (is_alloc == alloc_event && condition && \
event##_event_wait_get(tsd) == 0U) { \ event##_event_wait_get(tsd) == 0U) { \
assert(allow_event_trigger); \ assert(allow_event_trigger); \
thread_##event##_event_handler(tsd); \ te_##event##_event_handler(tsd); \
} }
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
te_assert_invariants(tsd);
thread_event_assert_invariants(tsd);
} }
void void
thread_alloc_event_rollback(tsd_t *tsd, size_t diff) { te_alloc_rollback(tsd_t *tsd, size_t diff) {
thread_event_assert_invariants(tsd); te_assert_invariants(tsd);
if (diff == 0U) { if (diff == 0U) {
return; return;
} }
/* Rollback happens only on alloc events. */ /* Rollback happens only on alloc events. */
event_ctx_t ctx; te_ctx_t ctx;
event_ctx_get(tsd, &ctx, true); te_ctx_get(tsd, &ctx, true);
uint64_t thread_allocated = event_ctx_current_bytes_get(&ctx); uint64_t thread_allocated = te_ctx_current_bytes_get(&ctx);
/* The subtraction is intentionally susceptible to underflow. */ /* The subtraction is intentionally susceptible to underflow. */
uint64_t thread_allocated_rollback = thread_allocated - diff; uint64_t thread_allocated_rollback = thread_allocated - diff;
event_ctx_current_bytes_set(&ctx, thread_allocated_rollback); te_ctx_current_bytes_set(&ctx, thread_allocated_rollback);
uint64_t last_event = event_ctx_last_event_get(&ctx); uint64_t last_event = te_ctx_last_event_get(&ctx);
/* Both subtractions are intentionally susceptible to underflow. */ /* Both subtractions are intentionally susceptible to underflow. */
if (thread_allocated_rollback - last_event <= if (thread_allocated_rollback - last_event <=
thread_allocated - last_event) { thread_allocated - last_event) {
thread_event_assert_invariants(tsd); te_assert_invariants(tsd);
return; return;
} }
event_ctx_last_event_set(&ctx, thread_allocated_rollback); te_ctx_last_event_set(&ctx, thread_allocated_rollback);
/* The subtraction is intentionally susceptible to underflow. */ /* The subtraction is intentionally susceptible to underflow. */
uint64_t wait_diff = last_event - thread_allocated_rollback; uint64_t wait_diff = last_event - thread_allocated_rollback;
assert(wait_diff <= diff); assert(wait_diff <= diff);
@ -374,49 +362,48 @@ thread_alloc_event_rollback(tsd_t *tsd, size_t diff) {
#define E(event, condition, alloc_event) \ #define E(event, condition, alloc_event) \
if (alloc_event == true && condition) { \ if (alloc_event == true && condition) { \
uint64_t event_wait = event##_event_wait_get(tsd); \ uint64_t event_wait = event##_event_wait_get(tsd); \
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \ assert(event_wait <= TE_MAX_START_WAIT); \
if (event_wait > 0U) { \ if (event_wait > 0U) { \
if (wait_diff > \ if (wait_diff > TE_MAX_START_WAIT - event_wait) {\
THREAD_EVENT_MAX_START_WAIT - event_wait) { \ event_wait = TE_MAX_START_WAIT; \
event_wait = \
THREAD_EVENT_MAX_START_WAIT; \
} else { \ } else { \
event_wait += wait_diff; \ event_wait += wait_diff; \
} \ } \
assert(event_wait <= \ assert(event_wait <= TE_MAX_START_WAIT); \
THREAD_EVENT_MAX_START_WAIT); \
event##_event_wait_set(tsd, event_wait); \ event##_event_wait_set(tsd, event_wait); \
} \ } \
} }
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
te_event_update(tsd, true);
thread_event_update(tsd, true);
} }
void void
thread_event_update(tsd_t *tsd, bool is_alloc) { te_event_update(tsd_t *tsd, bool is_alloc) {
event_ctx_t ctx; te_ctx_t ctx;
event_ctx_get(tsd, &ctx, is_alloc); te_ctx_get(tsd, &ctx, is_alloc);
uint64_t wait = thread_next_event_compute(tsd, is_alloc); uint64_t wait = te_next_event_compute(tsd, is_alloc);
thread_event_adjust_thresholds_helper(tsd, &ctx, wait); te_adjust_thresholds_helper(tsd, &ctx, wait);
uint64_t last_event = event_ctx_last_event_get(&ctx); uint64_t last_event = te_ctx_last_event_get(&ctx);
/* Both subtractions are intentionally susceptible to underflow. */ /* Both subtractions are intentionally susceptible to underflow. */
if (event_ctx_current_bytes_get(&ctx) - last_event >= if (te_ctx_current_bytes_get(&ctx) - last_event >=
event_ctx_next_event_get(&ctx) - last_event) { te_ctx_next_event_get(&ctx) - last_event) {
thread_event_trigger(tsd, &ctx, true); te_event_trigger(tsd, &ctx, true);
} else { } else {
thread_event_assert_invariants(tsd); te_assert_invariants(tsd);
} }
} }
void tsd_thread_event_init(tsd_t *tsd) { void tsd_te_init(tsd_t *tsd) {
/* Make sure no overflow for the bytes accumulated on event_trigger. */
assert(TE_MAX_INTERVAL <= UINT64_MAX - SC_LARGE_MAXCLASS + 1);
#define E(event, condition, is_alloc_event_unused) \ #define E(event, condition, is_alloc_event_unused) \
if (condition) { \ if (condition) { \
tsd_thread_##event##_event_init(tsd); \ te_tsd_##event##_event_init(tsd); \
} }
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS

10
src/tsd.c
View File

@ -117,9 +117,9 @@ tsd_force_recompute(tsdn_t *tsdn) {
<= tsd_state_nominal_max); <= tsd_state_nominal_max);
tsd_atomic_store(&remote_tsd->state, tsd_atomic_store(&remote_tsd->state,
tsd_state_nominal_recompute, ATOMIC_RELAXED); tsd_state_nominal_recompute, ATOMIC_RELAXED);
/* See comments in thread_event_recompute_fast_threshold(). */ /* See comments in te_recompute_fast_threshold(). */
atomic_fence(ATOMIC_SEQ_CST); atomic_fence(ATOMIC_SEQ_CST);
thread_next_event_fast_set_non_nominal(remote_tsd); te_next_event_fast_set_non_nominal(remote_tsd);
} }
malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock); malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock);
} }
@ -179,7 +179,7 @@ tsd_slow_update(tsd_t *tsd) {
ATOMIC_ACQUIRE); ATOMIC_ACQUIRE);
} while (old_state == tsd_state_nominal_recompute); } while (old_state == tsd_state_nominal_recompute);
thread_event_recompute_fast_threshold(tsd); te_recompute_fast_threshold(tsd);
} }
void void
@ -218,7 +218,7 @@ tsd_state_set(tsd_t *tsd, uint8_t new_state) {
tsd_slow_update(tsd); tsd_slow_update(tsd);
} }
} }
thread_event_recompute_fast_threshold(tsd); te_recompute_fast_threshold(tsd);
} }
static bool static bool
@ -240,7 +240,7 @@ tsd_data_init(tsd_t *tsd) {
(uint64_t)(uintptr_t)tsd; (uint64_t)(uintptr_t)tsd;
/* event_init may use the prng state above. */ /* event_init may use the prng state above. */
tsd_thread_event_init(tsd); tsd_te_init(tsd);
return tsd_tcache_enabled_data_init(tsd); return tsd_tcache_enabled_data_init(tsd);
} }

32
test/unit/thread_event.c
View File

@ -2,17 +2,15 @@
TEST_BEGIN(test_next_event_fast_roll_back) { TEST_BEGIN(test_next_event_fast_roll_back) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
event_ctx_t ctx; te_ctx_t ctx;
event_ctx_get(tsd, &ctx, true); te_ctx_get(tsd, &ctx, true);
event_ctx_last_event_set(&ctx, 0); te_ctx_last_event_set(&ctx, 0);
event_ctx_current_bytes_set(&ctx, te_ctx_current_bytes_set(&ctx, TE_NEXT_EVENT_FAST_MAX - 8U);
THREAD_NEXT_EVENT_FAST_MAX - 8U); te_ctx_next_event_set(tsd, &ctx, TE_NEXT_EVENT_FAST_MAX);
event_ctx_next_event_set(tsd, &ctx,
THREAD_NEXT_EVENT_FAST_MAX);
#define E(event, condition, is_alloc) \ #define E(event, condition, is_alloc) \
if (is_alloc && condition) { \ if (is_alloc && condition) { \
event##_event_wait_set(tsd, THREAD_NEXT_EVENT_FAST_MAX);\ event##_event_wait_set(tsd, TE_NEXT_EVENT_FAST_MAX); \
} }
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
@ -25,18 +23,16 @@ TEST_END
TEST_BEGIN(test_next_event_fast_resume) { TEST_BEGIN(test_next_event_fast_resume) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
event_ctx_t ctx; te_ctx_t ctx;
event_ctx_get(tsd, &ctx, true); te_ctx_get(tsd, &ctx, true);
event_ctx_last_event_set(&ctx, 0); te_ctx_last_event_set(&ctx, 0);
event_ctx_current_bytes_set(&ctx, te_ctx_current_bytes_set(&ctx, TE_NEXT_EVENT_FAST_MAX + 8U);
THREAD_NEXT_EVENT_FAST_MAX + 8U); te_ctx_next_event_set(tsd, &ctx, TE_NEXT_EVENT_FAST_MAX + 16U);
event_ctx_next_event_set(tsd, &ctx,
THREAD_NEXT_EVENT_FAST_MAX + 16U);
#define E(event, condition, is_alloc) \ #define E(event, condition, is_alloc) \
if (is_alloc && condition) { \ if (is_alloc && condition) { \
event##_event_wait_set(tsd, \ event##_event_wait_set(tsd, \
THREAD_NEXT_EVENT_FAST_MAX + 16U); \ TE_NEXT_EVENT_FAST_MAX + 16U); \
} }
ITERATE_OVER_ALL_EVENTS ITERATE_OVER_ALL_EVENTS
#undef E #undef E
@ -48,11 +44,11 @@ TEST_END
TEST_BEGIN(test_event_rollback) { TEST_BEGIN(test_event_rollback) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
const uint64_t diff = THREAD_EVENT_MAX_INTERVAL >> 2; const uint64_t diff = TE_MAX_INTERVAL >> 2;
size_t count = 10; size_t count = 10;
uint64_t thread_allocated = thread_allocated_get(tsd); uint64_t thread_allocated = thread_allocated_get(tsd);
while (count-- != 0) { while (count-- != 0) {
thread_alloc_event_rollback(tsd, diff); te_alloc_rollback(tsd, diff);
uint64_t thread_allocated_after = thread_allocated_get(tsd); uint64_t thread_allocated_after = thread_allocated_get(tsd);
assert_u64_eq(thread_allocated - thread_allocated_after, diff, assert_u64_eq(thread_allocated - thread_allocated_after, diff,
"thread event counters are not properly rolled back"); "thread event counters are not properly rolled back");