Build a general purpose thread event handler
This commit is contained in:
parent
6924f83cb2
commit
152c0ef954
10
Makefile.in
10
Makefile.in
@ -123,11 +123,12 @@ C_SRCS := $(srcroot)src/jemalloc.c \
|
|||||||
$(srcroot)src/prof_log.c \
|
$(srcroot)src/prof_log.c \
|
||||||
$(srcroot)src/rtree.c \
|
$(srcroot)src/rtree.c \
|
||||||
$(srcroot)src/safety_check.c \
|
$(srcroot)src/safety_check.c \
|
||||||
$(srcroot)src/stats.c \
|
|
||||||
$(srcroot)src/sc.c \
|
$(srcroot)src/sc.c \
|
||||||
|
$(srcroot)src/stats.c \
|
||||||
$(srcroot)src/sz.c \
|
$(srcroot)src/sz.c \
|
||||||
$(srcroot)src/tcache.c \
|
$(srcroot)src/tcache.c \
|
||||||
$(srcroot)src/test_hooks.c \
|
$(srcroot)src/test_hooks.c \
|
||||||
|
$(srcroot)src/thread_event.c \
|
||||||
$(srcroot)src/ticker.c \
|
$(srcroot)src/ticker.c \
|
||||||
$(srcroot)src/tsd.c \
|
$(srcroot)src/tsd.c \
|
||||||
$(srcroot)src/witness.c
|
$(srcroot)src/witness.c
|
||||||
@ -176,9 +177,9 @@ TESTS_UNIT := \
|
|||||||
$(srcroot)test/unit/background_thread.c \
|
$(srcroot)test/unit/background_thread.c \
|
||||||
$(srcroot)test/unit/background_thread_enable.c \
|
$(srcroot)test/unit/background_thread_enable.c \
|
||||||
$(srcroot)test/unit/base.c \
|
$(srcroot)test/unit/base.c \
|
||||||
|
$(srcroot)test/unit/binshard.c \
|
||||||
$(srcroot)test/unit/bitmap.c \
|
$(srcroot)test/unit/bitmap.c \
|
||||||
$(srcroot)test/unit/bit_util.c \
|
$(srcroot)test/unit/bit_util.c \
|
||||||
$(srcroot)test/unit/binshard.c \
|
|
||||||
$(srcroot)test/unit/buf_writer.c \
|
$(srcroot)test/unit/buf_writer.c \
|
||||||
$(srcroot)test/unit/cache_bin.c \
|
$(srcroot)test/unit/cache_bin.c \
|
||||||
$(srcroot)test/unit/ckh.c \
|
$(srcroot)test/unit/ckh.c \
|
||||||
@ -200,6 +201,7 @@ TESTS_UNIT := \
|
|||||||
$(srcroot)test/unit/math.c \
|
$(srcroot)test/unit/math.c \
|
||||||
$(srcroot)test/unit/mq.c \
|
$(srcroot)test/unit/mq.c \
|
||||||
$(srcroot)test/unit/mtx.c \
|
$(srcroot)test/unit/mtx.c \
|
||||||
|
$(srcroot)test/unit/nstime.c \
|
||||||
$(srcroot)test/unit/pack.c \
|
$(srcroot)test/unit/pack.c \
|
||||||
$(srcroot)test/unit/pages.c \
|
$(srcroot)test/unit/pages.c \
|
||||||
$(srcroot)test/unit/ph.c \
|
$(srcroot)test/unit/ph.c \
|
||||||
@ -218,9 +220,9 @@ TESTS_UNIT := \
|
|||||||
$(srcroot)test/unit/retained.c \
|
$(srcroot)test/unit/retained.c \
|
||||||
$(srcroot)test/unit/rtree.c \
|
$(srcroot)test/unit/rtree.c \
|
||||||
$(srcroot)test/unit/safety_check.c \
|
$(srcroot)test/unit/safety_check.c \
|
||||||
|
$(srcroot)test/unit/sc.c \
|
||||||
$(srcroot)test/unit/seq.c \
|
$(srcroot)test/unit/seq.c \
|
||||||
$(srcroot)test/unit/SFMT.c \
|
$(srcroot)test/unit/SFMT.c \
|
||||||
$(srcroot)test/unit/sc.c \
|
|
||||||
$(srcroot)test/unit/size_classes.c \
|
$(srcroot)test/unit/size_classes.c \
|
||||||
$(srcroot)test/unit/slab.c \
|
$(srcroot)test/unit/slab.c \
|
||||||
$(srcroot)test/unit/smoothstep.c \
|
$(srcroot)test/unit/smoothstep.c \
|
||||||
@ -228,8 +230,8 @@ TESTS_UNIT := \
|
|||||||
$(srcroot)test/unit/stats.c \
|
$(srcroot)test/unit/stats.c \
|
||||||
$(srcroot)test/unit/stats_print.c \
|
$(srcroot)test/unit/stats_print.c \
|
||||||
$(srcroot)test/unit/test_hooks.c \
|
$(srcroot)test/unit/test_hooks.c \
|
||||||
|
$(srcroot)test/unit/thread_event.c \
|
||||||
$(srcroot)test/unit/ticker.c \
|
$(srcroot)test/unit/ticker.c \
|
||||||
$(srcroot)test/unit/nstime.c \
|
|
||||||
$(srcroot)test/unit/tsd.c \
|
$(srcroot)test/unit/tsd.c \
|
||||||
$(srcroot)test/unit/witness.c \
|
$(srcroot)test/unit/witness.c \
|
||||||
$(srcroot)test/unit/zero.c \
|
$(srcroot)test/unit/zero.c \
|
||||||
|
@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
#include "jemalloc/internal/safety_check.h"
|
#include "jemalloc/internal/safety_check.h"
|
||||||
#include "jemalloc/internal/sz.h"
|
#include "jemalloc/internal/sz.h"
|
||||||
|
#include "jemalloc/internal/thread_event.h"
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE bool
|
JEMALLOC_ALWAYS_INLINE bool
|
||||||
prof_gdump_get_unlocked(void) {
|
prof_gdump_get_unlocked(void) {
|
||||||
@ -79,24 +80,6 @@ prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, nstime_t t) {
|
|||||||
arena_prof_alloc_time_set(tsdn, ptr, t);
|
arena_prof_alloc_time_set(tsdn, ptr, t);
|
||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE bool
|
|
||||||
prof_sample_check(tsd_t *tsd, size_t usize, bool update) {
|
|
||||||
ssize_t check = update ? 0 : usize;
|
|
||||||
|
|
||||||
int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd);
|
|
||||||
if (update) {
|
|
||||||
bytes_until_sample -= usize;
|
|
||||||
if (tsd_nominal(tsd)) {
|
|
||||||
tsd_bytes_until_sample_set(tsd, bytes_until_sample);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (likely(bytes_until_sample >= check)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE bool
|
JEMALLOC_ALWAYS_INLINE bool
|
||||||
prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
|
prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
|
||||||
prof_tdata_t **tdata_out) {
|
prof_tdata_t **tdata_out) {
|
||||||
@ -105,7 +88,7 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
|
|||||||
cassert(config_prof);
|
cassert(config_prof);
|
||||||
|
|
||||||
/* Fastpath: no need to load tdata */
|
/* Fastpath: no need to load tdata */
|
||||||
if (likely(prof_sample_check(tsd, usize, update))) {
|
if (likely(prof_sample_event_wait_get(tsd) > 0)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -127,13 +110,40 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
if (!booted) {
|
||||||
* If this was the first creation of tdata, then
|
/*
|
||||||
* prof_tdata_get() reset bytes_until_sample, so decrement and
|
* If this was the first creation of tdata, then it means that
|
||||||
* check it again
|
* the previous thread_event() relied on the wrong prof_sample
|
||||||
*/
|
* wait time, and that it should have relied on the new
|
||||||
if (!booted && prof_sample_check(tsd, usize, update)) {
|
* prof_sample wait time just set by prof_tdata_get(), so we
|
||||||
return true;
|
* now manually check again.
|
||||||
|
*
|
||||||
|
* If the check fails, then even though we relied on the wrong
|
||||||
|
* prof_sample wait time, we're now actually in perfect shape,
|
||||||
|
* in the sense that we can pretend that we have used the right
|
||||||
|
* prof_sample wait time.
|
||||||
|
*
|
||||||
|
* If the check succeeds, then we are now in a tougher
|
||||||
|
* situation, in the sense that we cannot pretend that we have
|
||||||
|
* used the right prof_sample wait time. A straightforward
|
||||||
|
* solution would be to fully roll back thread_event(), set the
|
||||||
|
* right prof_sample wait time, and then redo thread_event().
|
||||||
|
* A simpler way, which is implemented below, is to just set a
|
||||||
|
* new prof_sample wait time that is usize less, and do nothing
|
||||||
|
* else. Strictly speaking, the thread event handler may end
|
||||||
|
* up in a wrong state, since it has still recorded an event
|
||||||
|
* whereas in reality there may be no event. However, the
|
||||||
|
* difference in the wait time offsets the wrongly recorded
|
||||||
|
* event, so that, functionally, the countdown to the next
|
||||||
|
* event will behave exactly as if we have used the right
|
||||||
|
* prof_sample wait time in the first place.
|
||||||
|
*/
|
||||||
|
uint64_t wait = prof_sample_event_wait_get(tsd);
|
||||||
|
assert(wait > 0);
|
||||||
|
if (usize < wait) {
|
||||||
|
thread_prof_sample_event_update(tsd, wait - usize);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Compute new sample threshold. */
|
/* Compute new sample threshold. */
|
||||||
|
139
include/jemalloc/internal/thread_event.h
Normal file
139
include/jemalloc/internal/thread_event.h
Normal file
@ -0,0 +1,139 @@
|
|||||||
|
#ifndef JEMALLOC_INTERNAL_THREAD_EVENT_H
|
||||||
|
#define JEMALLOC_INTERNAL_THREAD_EVENT_H
|
||||||
|
|
||||||
|
#include "jemalloc/internal/tsd.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Maximum threshold on thread_allocated_next_event_fast, so that there is no
|
||||||
|
* need to check overflow in malloc fast path. (The allocation size in malloc
|
||||||
|
* fast path never exceeds SC_LOOKUP_MAXCLASS.)
|
||||||
|
*/
|
||||||
|
#define THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX \
|
||||||
|
(UINT64_MAX - SC_LOOKUP_MAXCLASS + 1U)
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The max interval helps make sure that malloc stays on the fast path in the
|
||||||
|
* common case, i.e. thread_allocated < thread_allocated_next_event_fast.
|
||||||
|
* When thread_allocated is within an event's distance to
|
||||||
|
* THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX above, thread_allocated_next_event_fast
|
||||||
|
* is wrapped around and we fall back to the medium-fast path. The max interval
|
||||||
|
* makes sure that we're not staying on the fallback case for too long, even if
|
||||||
|
* there's no active event or if all active events have long wait times.
|
||||||
|
*/
|
||||||
|
#define THREAD_EVENT_MAX_INTERVAL ((uint64_t)(4U << 20))
|
||||||
|
|
||||||
|
void thread_event_assert_invariants_debug(tsd_t *tsd);
|
||||||
|
void thread_event_trigger(tsd_t *tsd, bool delay_event);
|
||||||
|
void thread_event_rollback(tsd_t *tsd, size_t diff);
|
||||||
|
void thread_event_update(tsd_t *tsd);
|
||||||
|
void thread_event_boot();
|
||||||
|
|
||||||
|
/*
|
||||||
|
* List of all events, in the following format:
|
||||||
|
* E(event, (condition))
|
||||||
|
*/
|
||||||
|
#define ITERATE_OVER_ALL_EVENTS \
|
||||||
|
E(prof_sample, (config_prof && opt_prof))
|
||||||
|
|
||||||
|
#define E(event, condition) \
|
||||||
|
C(event##_event_wait)
|
||||||
|
|
||||||
|
/* List of all thread event counters. */
|
||||||
|
#define ITERATE_OVER_ALL_COUNTERS \
|
||||||
|
C(thread_allocated) \
|
||||||
|
C(thread_allocated_next_event_fast) \
|
||||||
|
C(thread_allocated_last_event) \
|
||||||
|
C(thread_allocated_next_event) \
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
|
||||||
|
/* Getters directly wrap TSD getters. */
|
||||||
|
#define C(counter) \
|
||||||
|
JEMALLOC_ALWAYS_INLINE uint64_t \
|
||||||
|
counter##_get(tsd_t *tsd) { \
|
||||||
|
return tsd_##counter##_get(tsd); \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_COUNTERS
|
||||||
|
#undef C
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Setters call the TSD pointer getters rather than the TSD setters, so that
|
||||||
|
* the counters can be modified even when TSD state is reincarnated or
|
||||||
|
* minimal_initialized: if an event is triggered in such cases, we will
|
||||||
|
* temporarily delay the event and let it be immediately triggered at the next
|
||||||
|
* allocation call.
|
||||||
|
*/
|
||||||
|
#define C(counter) \
|
||||||
|
JEMALLOC_ALWAYS_INLINE void \
|
||||||
|
counter##_set(tsd_t *tsd, uint64_t v) { \
|
||||||
|
*tsd_##counter##p_get(tsd) = v; \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_COUNTERS
|
||||||
|
#undef C
|
||||||
|
|
||||||
|
/*
|
||||||
|
* For generating _event_wait getter / setter functions for each individual
|
||||||
|
* event.
|
||||||
|
*/
|
||||||
|
#undef E
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The function checks in debug mode whether the thread event counters are in
|
||||||
|
* a consistent state, which forms the invariants before and after each round
|
||||||
|
* of thread event handling that we can rely on and need to promise.
|
||||||
|
* The invariants are only temporarily violated in the middle of:
|
||||||
|
* (a) thread_event() if an event is triggered (the thread_event_trigger() call
|
||||||
|
* at the end will restore the invariants),
|
||||||
|
* (b) thread_##event##_event_update() (the thread_event_update() call at the
|
||||||
|
* end will restore the invariants), or
|
||||||
|
* (c) thread_event_rollback() if the rollback falls below the last_event (the
|
||||||
|
* thread_event_update() call at the end will restore the invariants).
|
||||||
|
*/
|
||||||
|
JEMALLOC_ALWAYS_INLINE void
|
||||||
|
thread_event_assert_invariants(tsd_t *tsd) {
|
||||||
|
if (config_debug) {
|
||||||
|
thread_event_assert_invariants_debug(tsd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
JEMALLOC_ALWAYS_INLINE void
|
||||||
|
thread_event(tsd_t *tsd, size_t usize) {
|
||||||
|
thread_event_assert_invariants(tsd);
|
||||||
|
|
||||||
|
uint64_t thread_allocated_before = thread_allocated_get(tsd);
|
||||||
|
thread_allocated_set(tsd, thread_allocated_before + usize);
|
||||||
|
|
||||||
|
/* The subtraction is intentionally susceptible to underflow. */
|
||||||
|
if (likely(usize < thread_allocated_next_event_get(tsd) -
|
||||||
|
thread_allocated_before)) {
|
||||||
|
thread_event_assert_invariants(tsd);
|
||||||
|
} else {
|
||||||
|
thread_event_trigger(tsd, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#define E(event, condition) \
|
||||||
|
JEMALLOC_ALWAYS_INLINE void \
|
||||||
|
thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) { \
|
||||||
|
thread_event_assert_invariants(tsd); \
|
||||||
|
assert(condition); \
|
||||||
|
assert(tsd_nominal(tsd)); \
|
||||||
|
assert(tsd_reentrancy_level_get(tsd) == 0); \
|
||||||
|
assert(event_wait > 0U); \
|
||||||
|
if (THREAD_EVENT_MIN_START_WAIT > 1U && \
|
||||||
|
unlikely(event_wait < THREAD_EVENT_MIN_START_WAIT)) { \
|
||||||
|
event_wait = THREAD_EVENT_MIN_START_WAIT; \
|
||||||
|
} \
|
||||||
|
if (THREAD_EVENT_MAX_START_WAIT < UINT64_MAX && \
|
||||||
|
unlikely(event_wait > THREAD_EVENT_MAX_START_WAIT)) { \
|
||||||
|
event_wait = THREAD_EVENT_MAX_START_WAIT; \
|
||||||
|
} \
|
||||||
|
event##_event_wait_set(tsd, event_wait); \
|
||||||
|
thread_event_update(tsd); \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
#undef E
|
||||||
|
|
||||||
|
#endif /* JEMALLOC_INTERNAL_THREAD_EVENT_H */
|
@ -15,39 +15,45 @@
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
* Thread-Specific-Data layout
|
* Thread-Specific-Data layout
|
||||||
* --- data accessed on tcache fast path: state, rtree_ctx, stats, prof ---
|
* --- data accessed on tcache fast path: state, rtree_ctx, stats ---
|
||||||
* s: state
|
* s: state
|
||||||
* e: tcache_enabled
|
* e: tcache_enabled
|
||||||
* m: thread_allocated
|
* m: thread_allocated
|
||||||
|
* k: thread_allocated_next_event_fast
|
||||||
* f: thread_deallocated
|
* f: thread_deallocated
|
||||||
* b: bytes_until_sample (config_prof)
|
|
||||||
* p: prof_tdata (config_prof)
|
|
||||||
* c: rtree_ctx (rtree cache accessed on deallocation)
|
* c: rtree_ctx (rtree cache accessed on deallocation)
|
||||||
* t: tcache
|
* t: tcache
|
||||||
* --- data not accessed on tcache fast path: arena-related fields ---
|
* --- data not accessed on tcache fast path: arena-related fields ---
|
||||||
* d: arenas_tdata_bypass
|
* d: arenas_tdata_bypass
|
||||||
* r: reentrancy_level
|
* r: reentrancy_level
|
||||||
* x: narenas_tdata
|
* x: narenas_tdata
|
||||||
|
* l: thread_allocated_last_event
|
||||||
|
* j: thread_allocated_next_event
|
||||||
|
* w: prof_sample_event_wait (config_prof)
|
||||||
|
* p: prof_tdata (config_prof)
|
||||||
* v: offset_state
|
* v: offset_state
|
||||||
* i: iarena
|
* i: iarena
|
||||||
* a: arena
|
* a: arena
|
||||||
* o: arenas_tdata
|
* o: arenas_tdata
|
||||||
|
* b: binshards
|
||||||
* Loading TSD data is on the critical path of basically all malloc operations.
|
* Loading TSD data is on the critical path of basically all malloc operations.
|
||||||
* In particular, tcache and rtree_ctx rely on hot CPU cache to be effective.
|
* In particular, tcache and rtree_ctx rely on hot CPU cache to be effective.
|
||||||
* Use a compact layout to reduce cache footprint.
|
* Use a compact layout to reduce cache footprint.
|
||||||
* +--- 64-bit and 64B cacheline; 1B each letter; First byte on the left. ---+
|
* +--- 64-bit and 64B cacheline; 1B each letter; First byte on the left. ---+
|
||||||
* |---------------------------- 1st cacheline ----------------------------|
|
* |---------------------------- 1st cacheline ----------------------------|
|
||||||
* | sedrxxxx vvvvvvvv mmmmmmmm ffffffff bbbbbbbb pppppppp [c * 16 .......] |
|
* | sedrxxxx mmmmmmmm kkkkkkkk ffffffff [c * 32 ........ ........ .......] |
|
||||||
* |---------------------------- 2nd cacheline ----------------------------|
|
* |---------------------------- 2nd cacheline ----------------------------|
|
||||||
* | [c * 64 ........ ........ ........ ........ ........ ........ .......] |
|
* | [c * 64 ........ ........ ........ ........ ........ ........ .......] |
|
||||||
* |---------------------------- 3nd cacheline ----------------------------|
|
* |---------------------------- 3nd cacheline ----------------------------|
|
||||||
* | [c * 48 ........ ........ ........ ........ .......] iiiiiiii aaaaaaaa |
|
* | [c * 32 ........ ........ .......] llllllll jjjjjjjj wwwwwwww pppppppp |
|
||||||
* +---------------------------- 4th cacheline ----------------------------+
|
* +---------------------------- 4th cacheline ----------------------------+
|
||||||
* | oooooooo [t...... ........ ........ ........ ........ ........ ........ |
|
* | vvvvvvvv iiiiiiii aaaaaaaa oooooooo [b...... ........ ........ ........ |
|
||||||
|
* +---------------------------- 5th cacheline ----------------------------+
|
||||||
|
* | ..b][t.. ........ ........ ........ ........ ........ ........ ........ |
|
||||||
* +-------------------------------------------------------------------------+
|
* +-------------------------------------------------------------------------+
|
||||||
* Note: the entire tcache is embedded into TSD and spans multiple cachelines.
|
* Note: the entire tcache is embedded into TSD and spans multiple cachelines.
|
||||||
*
|
*
|
||||||
* The last 3 members (i, a and o) before tcache isn't really needed on tcache
|
* The elements after rtree_ctx and before tcache aren't really needed on tcache
|
||||||
* fast path. However we have a number of unused tcache bins and witnesses
|
* fast path. However we have a number of unused tcache bins and witnesses
|
||||||
* (never touched unless config_debug) at the end of tcache, so we place them
|
* (never touched unless config_debug) at the end of tcache, so we place them
|
||||||
* there to avoid breaking the cachelines and possibly paging in an extra page.
|
* there to avoid breaking the cachelines and possibly paging in an extra page.
|
||||||
@ -64,18 +70,21 @@ typedef void (*test_callback_t)(int *);
|
|||||||
# define MALLOC_TEST_TSD_INITIALIZER
|
# define MALLOC_TEST_TSD_INITIALIZER
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* O(name, type, nullable type */
|
/* O(name, type, nullable type) */
|
||||||
#define MALLOC_TSD \
|
#define MALLOC_TSD \
|
||||||
O(tcache_enabled, bool, bool) \
|
O(tcache_enabled, bool, bool) \
|
||||||
O(arenas_tdata_bypass, bool, bool) \
|
O(arenas_tdata_bypass, bool, bool) \
|
||||||
O(reentrancy_level, int8_t, int8_t) \
|
O(reentrancy_level, int8_t, int8_t) \
|
||||||
O(narenas_tdata, uint32_t, uint32_t) \
|
O(narenas_tdata, uint32_t, uint32_t) \
|
||||||
O(offset_state, uint64_t, uint64_t) \
|
|
||||||
O(thread_allocated, uint64_t, uint64_t) \
|
O(thread_allocated, uint64_t, uint64_t) \
|
||||||
|
O(thread_allocated_next_event_fast, uint64_t, uint64_t) \
|
||||||
O(thread_deallocated, uint64_t, uint64_t) \
|
O(thread_deallocated, uint64_t, uint64_t) \
|
||||||
O(bytes_until_sample, int64_t, int64_t) \
|
|
||||||
O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \
|
|
||||||
O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \
|
O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \
|
||||||
|
O(thread_allocated_last_event, uint64_t, uint64_t) \
|
||||||
|
O(thread_allocated_next_event, uint64_t, uint64_t) \
|
||||||
|
O(prof_sample_event_wait, uint64_t, uint64_t) \
|
||||||
|
O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \
|
||||||
|
O(offset_state, uint64_t, uint64_t) \
|
||||||
O(iarena, arena_t *, arena_t *) \
|
O(iarena, arena_t *, arena_t *) \
|
||||||
O(arena, arena_t *, arena_t *) \
|
O(arena, arena_t *, arena_t *) \
|
||||||
O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\
|
O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\
|
||||||
@ -84,25 +93,34 @@ typedef void (*test_callback_t)(int *);
|
|||||||
O(witness_tsd, witness_tsd_t, witness_tsdn_t) \
|
O(witness_tsd, witness_tsd_t, witness_tsdn_t) \
|
||||||
MALLOC_TEST_TSD
|
MALLOC_TEST_TSD
|
||||||
|
|
||||||
|
/*
|
||||||
|
* THREAD_EVENT_MIN_START_WAIT should not exceed the minimal allocation usize.
|
||||||
|
*/
|
||||||
|
#define THREAD_EVENT_MIN_START_WAIT ((uint64_t)1U)
|
||||||
|
#define THREAD_EVENT_MAX_START_WAIT UINT64_MAX
|
||||||
|
|
||||||
#define TSD_INITIALIZER { \
|
#define TSD_INITIALIZER { \
|
||||||
ATOMIC_INIT(tsd_state_uninitialized), \
|
/* state */ ATOMIC_INIT(tsd_state_uninitialized), \
|
||||||
TCACHE_ENABLED_ZERO_INITIALIZER, \
|
/* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \
|
||||||
false, \
|
/* arenas_tdata_bypass */ false, \
|
||||||
0, \
|
/* reentrancy_level */ 0, \
|
||||||
0, \
|
/* narenas_tdata */ 0, \
|
||||||
0, \
|
/* thread_allocated */ 0, \
|
||||||
0, \
|
/* thread_allocated_next_event_fast */ THREAD_EVENT_MIN_START_WAIT, \
|
||||||
0, \
|
/* thread_deallocated */ 0, \
|
||||||
0, \
|
/* rtree_ctx */ RTREE_CTX_ZERO_INITIALIZER, \
|
||||||
NULL, \
|
/* thread_allocated_last_event */ 0, \
|
||||||
RTREE_CTX_ZERO_INITIALIZER, \
|
/* thread_allocated_next_event */ THREAD_EVENT_MIN_START_WAIT, \
|
||||||
NULL, \
|
/* prof_sample_event_wait */ THREAD_EVENT_MIN_START_WAIT, \
|
||||||
NULL, \
|
/* prof_tdata */ NULL, \
|
||||||
NULL, \
|
/* offset_state */ 0, \
|
||||||
TSD_BINSHARDS_ZERO_INITIALIZER, \
|
/* iarena */ NULL, \
|
||||||
TCACHE_ZERO_INITIALIZER, \
|
/* arena */ NULL, \
|
||||||
WITNESS_TSD_INITIALIZER \
|
/* arenas_tdata */ NULL, \
|
||||||
MALLOC_TEST_TSD_INITIALIZER \
|
/* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \
|
||||||
|
/* tcache */ TCACHE_ZERO_INITIALIZER, \
|
||||||
|
/* witness */ WITNESS_TSD_INITIALIZER \
|
||||||
|
/* test data */ MALLOC_TEST_TSD_INITIALIZER \
|
||||||
}
|
}
|
||||||
|
|
||||||
void *malloc_tsd_malloc(size_t size);
|
void *malloc_tsd_malloc(size_t size);
|
||||||
|
@ -63,15 +63,16 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\prof_data.c" />
|
<ClCompile Include="..\..\..\..\src\prof_data.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\prof_log.c" />
|
<ClCompile Include="..\..\..\..\src\prof_log.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\rtree.c" />
|
<ClCompile Include="..\..\..\..\src\rtree.c" />
|
||||||
|
<ClCompile Include="..\..\..\..\src\safety_check.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\sc.c" />
|
<ClCompile Include="..\..\..\..\src\sc.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\stats.c" />
|
<ClCompile Include="..\..\..\..\src\stats.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\sz.c" />
|
<ClCompile Include="..\..\..\..\src\sz.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\tcache.c" />
|
<ClCompile Include="..\..\..\..\src\tcache.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\test_hooks.c" />
|
<ClCompile Include="..\..\..\..\src\test_hooks.c" />
|
||||||
|
<ClCompile Include="..\..\..\..\src\thread_event.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\ticker.c" />
|
<ClCompile Include="..\..\..\..\src\ticker.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\tsd.c" />
|
<ClCompile Include="..\..\..\..\src\tsd.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\witness.c" />
|
<ClCompile Include="..\..\..\..\src\witness.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\safety_check.c" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<PropertyGroup Label="Globals">
|
<PropertyGroup Label="Globals">
|
||||||
<ProjectGuid>{8D6BB292-9E1C-413D-9F98-4864BDC1514A}</ProjectGuid>
|
<ProjectGuid>{8D6BB292-9E1C-413D-9F98-4864BDC1514A}</ProjectGuid>
|
||||||
|
@ -16,6 +16,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\base.c">
|
<ClCompile Include="..\..\..\..\src\base.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\bin.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\bitmap.c">
|
<ClCompile Include="..\..\..\..\src\bitmap.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -25,6 +28,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\ctl.c">
|
<ClCompile Include="..\..\..\..\src\ctl.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\div.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\extent.c">
|
<ClCompile Include="..\..\..\..\src\extent.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -46,6 +52,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\large.c">
|
<ClCompile Include="..\..\..\..\src\large.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\log.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\malloc_io.c">
|
<ClCompile Include="..\..\..\..\src\malloc_io.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -76,6 +85,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\rtree.c">
|
<ClCompile Include="..\..\..\..\src\rtree.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\safety_check.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\sc.c">
|
<ClCompile Include="..\..\..\..\src\sc.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -91,6 +103,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\test_hooks.c">
|
<ClCompile Include="..\..\..\..\src\test_hooks.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\thread_event.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\ticker.c">
|
<ClCompile Include="..\..\..\..\src\ticker.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -100,17 +115,5 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\witness.c">
|
<ClCompile Include="..\..\..\..\src\witness.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\log.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
<ClCompile Include="..\..\..\..\src\bin.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
<ClCompile Include="..\..\..\..\src\div.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
<ClCompile Include="..\..\..\..\src\safety_check.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
|
@ -63,15 +63,16 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\prof_data.c" />
|
<ClCompile Include="..\..\..\..\src\prof_data.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\prof_log.c" />
|
<ClCompile Include="..\..\..\..\src\prof_log.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\rtree.c" />
|
<ClCompile Include="..\..\..\..\src\rtree.c" />
|
||||||
|
<ClCompile Include="..\..\..\..\src\safety_check.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\sc.c" />
|
<ClCompile Include="..\..\..\..\src\sc.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\stats.c" />
|
<ClCompile Include="..\..\..\..\src\stats.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\sz.c" />
|
<ClCompile Include="..\..\..\..\src\sz.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\tcache.c" />
|
<ClCompile Include="..\..\..\..\src\tcache.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\test_hooks.c" />
|
<ClCompile Include="..\..\..\..\src\test_hooks.c" />
|
||||||
|
<ClCompile Include="..\..\..\..\src\thread_event.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\ticker.c" />
|
<ClCompile Include="..\..\..\..\src\ticker.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\tsd.c" />
|
<ClCompile Include="..\..\..\..\src\tsd.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\witness.c" />
|
<ClCompile Include="..\..\..\..\src\witness.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\safety_check.c" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<PropertyGroup Label="Globals">
|
<PropertyGroup Label="Globals">
|
||||||
<ProjectGuid>{8D6BB292-9E1C-413D-9F98-4864BDC1514A}</ProjectGuid>
|
<ProjectGuid>{8D6BB292-9E1C-413D-9F98-4864BDC1514A}</ProjectGuid>
|
||||||
|
@ -16,6 +16,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\base.c">
|
<ClCompile Include="..\..\..\..\src\base.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\bin.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\bitmap.c">
|
<ClCompile Include="..\..\..\..\src\bitmap.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -25,6 +28,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\ctl.c">
|
<ClCompile Include="..\..\..\..\src\ctl.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\div.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\extent.c">
|
<ClCompile Include="..\..\..\..\src\extent.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -46,6 +52,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\large.c">
|
<ClCompile Include="..\..\..\..\src\large.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\log.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\malloc_io.c">
|
<ClCompile Include="..\..\..\..\src\malloc_io.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -76,6 +85,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\rtree.c">
|
<ClCompile Include="..\..\..\..\src\rtree.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\safety_check.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\sc.c">
|
<ClCompile Include="..\..\..\..\src\sc.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -88,6 +100,12 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\tcache.c">
|
<ClCompile Include="..\..\..\..\src\tcache.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\test_hooks.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\thread_event.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\ticker.c">
|
<ClCompile Include="..\..\..\..\src\ticker.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
@ -97,20 +115,5 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\witness.c">
|
<ClCompile Include="..\..\..\..\src\witness.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\log.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
<ClCompile Include="..\..\..\..\src\bin.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
<ClCompile Include="..\..\..\..\src\div.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
<ClCompile Include="..\..\..\..\src\test_hooks.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
<ClCompile Include="..\..\..\..\src\safety_check.c">
|
|
||||||
<Filter>Source Files</Filter>
|
|
||||||
</ClCompile>
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
</Project>
|
</Project>
|
||||||
|
@ -18,6 +18,7 @@
|
|||||||
#include "jemalloc/internal/spin.h"
|
#include "jemalloc/internal/spin.h"
|
||||||
#include "jemalloc/internal/sz.h"
|
#include "jemalloc/internal/sz.h"
|
||||||
#include "jemalloc/internal/ticker.h"
|
#include "jemalloc/internal/ticker.h"
|
||||||
|
#include "jemalloc/internal/thread_event.h"
|
||||||
#include "jemalloc/internal/util.h"
|
#include "jemalloc/internal/util.h"
|
||||||
|
|
||||||
/******************************************************************************/
|
/******************************************************************************/
|
||||||
@ -1530,6 +1531,7 @@ malloc_init_hard_a0_locked() {
|
|||||||
prof_boot0();
|
prof_boot0();
|
||||||
}
|
}
|
||||||
malloc_conf_init(&sc_data, bin_shard_sizes);
|
malloc_conf_init(&sc_data, bin_shard_sizes);
|
||||||
|
thread_event_boot();
|
||||||
sz_boot(&sc_data);
|
sz_boot(&sc_data);
|
||||||
bin_info_boot(&sc_data, bin_shard_sizes);
|
bin_info_boot(&sc_data, bin_shard_sizes);
|
||||||
|
|
||||||
@ -2128,6 +2130,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
|
|||||||
dopts->arena_ind = 0;
|
dopts->arena_ind = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
thread_event(tsd, usize);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* If dopts->alignment > 0, then ind is still 0, but usize was computed
|
* If dopts->alignment > 0, then ind is still 0, but usize was computed
|
||||||
* in the previous if statement. Down the positive alignment path,
|
* in the previous if statement. Down the positive alignment path,
|
||||||
@ -2136,20 +2140,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
|
|||||||
|
|
||||||
/* If profiling is on, get our profiling context. */
|
/* If profiling is on, get our profiling context. */
|
||||||
if (config_prof && opt_prof) {
|
if (config_prof && opt_prof) {
|
||||||
/*
|
|
||||||
* The fast path modifies bytes_until_sample regardless of
|
|
||||||
* prof_active. We reset it to be the sample interval, so that
|
|
||||||
* there won't be excessive routings to the slow path, and that
|
|
||||||
* when prof_active is turned on later, the counting for
|
|
||||||
* sampling can immediately resume as normal (though the very
|
|
||||||
* first sampling interval is not randomized).
|
|
||||||
*/
|
|
||||||
if (unlikely(tsd_bytes_until_sample_get(tsd) < 0) &&
|
|
||||||
!prof_active_get_unlocked()) {
|
|
||||||
tsd_bytes_until_sample_set(tsd,
|
|
||||||
(ssize_t)(1 << lg_prof_sample));
|
|
||||||
}
|
|
||||||
|
|
||||||
prof_tctx_t *tctx = prof_alloc_prep(
|
prof_tctx_t *tctx = prof_alloc_prep(
|
||||||
tsd, usize, prof_active_get_unlocked(), true);
|
tsd, usize, prof_active_get_unlocked(), true);
|
||||||
|
|
||||||
@ -2167,24 +2157,17 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (unlikely(allocation == NULL)) {
|
if (unlikely(allocation == NULL)) {
|
||||||
|
thread_event_rollback(tsd, usize);
|
||||||
prof_alloc_rollback(tsd, tctx, true);
|
prof_alloc_rollback(tsd, tctx, true);
|
||||||
goto label_oom;
|
goto label_oom;
|
||||||
}
|
}
|
||||||
prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx);
|
prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx);
|
||||||
} else {
|
} else {
|
||||||
assert(!opt_prof);
|
assert(!opt_prof);
|
||||||
/*
|
|
||||||
* The fast path modifies bytes_until_sample regardless of
|
|
||||||
* opt_prof. We reset it to a huge value here, so as to
|
|
||||||
* minimize the triggering for slow path.
|
|
||||||
*/
|
|
||||||
if (config_prof &&
|
|
||||||
unlikely(tsd_bytes_until_sample_get(tsd) < 0)) {
|
|
||||||
tsd_bytes_until_sample_set(tsd, SSIZE_MAX);
|
|
||||||
}
|
|
||||||
allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
|
allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
|
||||||
ind);
|
ind);
|
||||||
if (unlikely(allocation == NULL)) {
|
if (unlikely(allocation == NULL)) {
|
||||||
|
thread_event_rollback(tsd, usize);
|
||||||
goto label_oom;
|
goto label_oom;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -2197,7 +2180,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
|
|||||||
|| ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
|
|| ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
|
||||||
|
|
||||||
assert(usize == isalloc(tsd_tsdn(tsd), allocation));
|
assert(usize == isalloc(tsd_tsdn(tsd), allocation));
|
||||||
*tsd_thread_allocatedp_get(tsd) += usize;
|
|
||||||
|
|
||||||
if (sopts->slow) {
|
if (sopts->slow) {
|
||||||
UTRACE(0, size, allocation);
|
UTRACE(0, size, allocation);
|
||||||
@ -2373,7 +2355,12 @@ je_malloc(size_t size) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
szind_t ind = sz_size2index_lookup(size);
|
szind_t ind = sz_size2index_lookup(size);
|
||||||
/* usize is always needed to increment thread_allocated. */
|
/*
|
||||||
|
* The thread_allocated counter in tsd serves as a general purpose
|
||||||
|
* accumulator for bytes of allocation to trigger different types of
|
||||||
|
* events. usize is always needed to advance thread_allocated, though
|
||||||
|
* it's not always needed in the core allocation logic.
|
||||||
|
*/
|
||||||
size_t usize = sz_index2size(ind);
|
size_t usize = sz_index2size(ind);
|
||||||
/*
|
/*
|
||||||
* Fast path relies on size being a bin.
|
* Fast path relies on size being a bin.
|
||||||
@ -2382,19 +2369,12 @@ je_malloc(size_t size) {
|
|||||||
assert(ind < SC_NBINS);
|
assert(ind < SC_NBINS);
|
||||||
assert(size <= SC_SMALL_MAXCLASS);
|
assert(size <= SC_SMALL_MAXCLASS);
|
||||||
|
|
||||||
if (config_prof) {
|
uint64_t thread_allocated_after = thread_allocated_get(tsd) + usize;
|
||||||
int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd);
|
assert(thread_allocated_next_event_fast_get(tsd) <=
|
||||||
bytes_until_sample -= usize;
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
|
||||||
tsd_bytes_until_sample_set(tsd, bytes_until_sample);
|
if (unlikely(thread_allocated_after >=
|
||||||
|
thread_allocated_next_event_fast_get(tsd))) {
|
||||||
if (unlikely(bytes_until_sample < 0)) {
|
return malloc_default(size);
|
||||||
/*
|
|
||||||
* Avoid a prof_active check on the fastpath.
|
|
||||||
* If prof_active is false, bytes_until_sample will be
|
|
||||||
* reset in slow path.
|
|
||||||
*/
|
|
||||||
return malloc_default(size);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
cache_bin_t *bin = tcache_small_bin_get(tcache, ind);
|
cache_bin_t *bin = tcache_small_bin_get(tcache, ind);
|
||||||
@ -2402,7 +2382,7 @@ je_malloc(size_t size) {
|
|||||||
void *ret = cache_bin_alloc_easy_reduced(bin, &tcache_success);
|
void *ret = cache_bin_alloc_easy_reduced(bin, &tcache_success);
|
||||||
|
|
||||||
if (tcache_success) {
|
if (tcache_success) {
|
||||||
*tsd_thread_allocatedp_get(tsd) += usize;
|
thread_allocated_set(tsd, thread_allocated_after);
|
||||||
if (config_stats) {
|
if (config_stats) {
|
||||||
bin->tstats.nrequests++;
|
bin->tstats.nrequests++;
|
||||||
}
|
}
|
||||||
@ -3116,9 +3096,11 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
|
|||||||
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
|
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
|
||||||
goto label_oom;
|
goto label_oom;
|
||||||
}
|
}
|
||||||
|
thread_event(tsd, usize);
|
||||||
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
|
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
|
||||||
zero, tcache, arena, &alloc_ctx, &hook_args);
|
zero, tcache, arena, &alloc_ctx, &hook_args);
|
||||||
if (unlikely(p == NULL)) {
|
if (unlikely(p == NULL)) {
|
||||||
|
thread_event_rollback(tsd, usize);
|
||||||
goto label_oom;
|
goto label_oom;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@ -3128,10 +3110,10 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
|
|||||||
goto label_oom;
|
goto label_oom;
|
||||||
}
|
}
|
||||||
usize = isalloc(tsd_tsdn(tsd), p);
|
usize = isalloc(tsd_tsdn(tsd), p);
|
||||||
|
thread_event(tsd, usize);
|
||||||
}
|
}
|
||||||
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
|
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
|
||||||
|
|
||||||
*tsd_thread_allocatedp_get(tsd) += usize;
|
|
||||||
*tsd_thread_deallocatedp_get(tsd) += old_usize;
|
*tsd_thread_deallocatedp_get(tsd) += old_usize;
|
||||||
|
|
||||||
UTRACE(ptr, size, p);
|
UTRACE(ptr, size, p);
|
||||||
@ -3307,6 +3289,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
|
|||||||
usize_max = SC_LARGE_MAXCLASS;
|
usize_max = SC_LARGE_MAXCLASS;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
thread_event(tsd, usize_max);
|
||||||
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
|
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
|
||||||
|
|
||||||
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
|
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
|
||||||
@ -3316,6 +3299,18 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
|
|||||||
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
|
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
|
||||||
extra, alignment, zero);
|
extra, alignment, zero);
|
||||||
}
|
}
|
||||||
|
if (usize <= usize_max) {
|
||||||
|
thread_event_rollback(tsd, usize_max - usize);
|
||||||
|
} else {
|
||||||
|
/*
|
||||||
|
* For downsizing request, usize_max can be less than usize.
|
||||||
|
* We here further increase thread event counters so as to
|
||||||
|
* record the true usize, and then when the execution goes back
|
||||||
|
* to xallocx(), the entire usize will be rolled back if it's
|
||||||
|
* equal to the old usize.
|
||||||
|
*/
|
||||||
|
thread_event(tsd, usize - usize_max);
|
||||||
|
}
|
||||||
if (usize == old_usize) {
|
if (usize == old_usize) {
|
||||||
prof_alloc_rollback(tsd, tctx, false);
|
prof_alloc_rollback(tsd, tctx, false);
|
||||||
return usize;
|
return usize;
|
||||||
@ -3373,12 +3368,13 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
|
|||||||
} else {
|
} else {
|
||||||
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
|
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
|
||||||
extra, alignment, zero);
|
extra, alignment, zero);
|
||||||
|
thread_event(tsd, usize);
|
||||||
}
|
}
|
||||||
if (unlikely(usize == old_usize)) {
|
if (unlikely(usize == old_usize)) {
|
||||||
|
thread_event_rollback(tsd, usize);
|
||||||
goto label_not_resized;
|
goto label_not_resized;
|
||||||
}
|
}
|
||||||
|
|
||||||
*tsd_thread_allocatedp_get(tsd) += usize;
|
|
||||||
*tsd_thread_deallocatedp_get(tsd) += old_usize;
|
*tsd_thread_deallocatedp_get(tsd) += old_usize;
|
||||||
|
|
||||||
label_not_resized:
|
label_not_resized:
|
||||||
|
21
src/prof.c
21
src/prof.c
@ -5,6 +5,7 @@
|
|||||||
#include "jemalloc/internal/ctl.h"
|
#include "jemalloc/internal/ctl.h"
|
||||||
#include "jemalloc/internal/assert.h"
|
#include "jemalloc/internal/assert.h"
|
||||||
#include "jemalloc/internal/mutex.h"
|
#include "jemalloc/internal/mutex.h"
|
||||||
|
#include "jemalloc/internal/thread_event.h"
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* This file implements the profiling "APIs" needed by other parts of jemalloc,
|
* This file implements the profiling "APIs" needed by other parts of jemalloc,
|
||||||
@ -471,8 +472,11 @@ prof_sample_threshold_update(prof_tdata_t *tdata) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
tsd_t *tsd = tsd_fetch();
|
||||||
|
|
||||||
if (lg_prof_sample == 0) {
|
if (lg_prof_sample == 0) {
|
||||||
tsd_bytes_until_sample_set(tsd_fetch(), 0);
|
thread_prof_sample_event_update(tsd,
|
||||||
|
THREAD_EVENT_MIN_START_WAIT);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -480,11 +484,11 @@ prof_sample_threshold_update(prof_tdata_t *tdata) {
|
|||||||
* Compute sample interval as a geometrically distributed random
|
* Compute sample interval as a geometrically distributed random
|
||||||
* variable with mean (2^lg_prof_sample).
|
* variable with mean (2^lg_prof_sample).
|
||||||
*
|
*
|
||||||
* __ __
|
* __ __
|
||||||
* | log(u) | 1
|
* | log(u) | 1
|
||||||
* tdata->bytes_until_sample = | -------- |, where p = ---------------
|
* bytes_until_sample = | -------- |, where p = ---------------
|
||||||
* | log(1-p) | lg_prof_sample
|
* | log(1-p) | lg_prof_sample
|
||||||
* 2
|
* 2
|
||||||
*
|
*
|
||||||
* For more information on the math, see:
|
* For more information on the math, see:
|
||||||
*
|
*
|
||||||
@ -499,10 +503,7 @@ prof_sample_threshold_update(prof_tdata_t *tdata) {
|
|||||||
uint64_t bytes_until_sample = (uint64_t)(log(u) /
|
uint64_t bytes_until_sample = (uint64_t)(log(u) /
|
||||||
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
|
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))
|
||||||
+ (uint64_t)1U;
|
+ (uint64_t)1U;
|
||||||
if (bytes_until_sample > SSIZE_MAX) {
|
thread_prof_sample_event_update(tsd, bytes_until_sample);
|
||||||
bytes_until_sample = SSIZE_MAX;
|
|
||||||
}
|
|
||||||
tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample);
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
255
src/thread_event.c
Normal file
255
src/thread_event.c
Normal file
@ -0,0 +1,255 @@
|
|||||||
|
#define JEMALLOC_THREAD_EVENT_C_
|
||||||
|
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||||
|
#include "jemalloc/internal/jemalloc_internal_includes.h"
|
||||||
|
|
||||||
|
#include "jemalloc/internal/thread_event.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
* There's no lock for thread_event_active because write is only done in
|
||||||
|
* malloc_init(), where init_lock there serves as the guard, and ever since
|
||||||
|
* then thread_event_active becomes read only.
|
||||||
|
*/
|
||||||
|
static bool thread_event_active = false;
|
||||||
|
|
||||||
|
/* Event handler function signatures. */
|
||||||
|
#define E(event, condition) \
|
||||||
|
static void thread_##event##_event_handler(tsd_t *tsd);
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
#undef E
|
||||||
|
|
||||||
|
static uint64_t
|
||||||
|
thread_allocated_next_event_compute(tsd_t *tsd) {
|
||||||
|
uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
|
||||||
|
bool no_event_on = true;
|
||||||
|
|
||||||
|
#define E(event, condition) \
|
||||||
|
if (condition) { \
|
||||||
|
no_event_on = false; \
|
||||||
|
uint64_t event_wait = \
|
||||||
|
event##_event_wait_get(tsd); \
|
||||||
|
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
|
||||||
|
if (event_wait > 0U && event_wait < wait) { \
|
||||||
|
wait = event_wait; \
|
||||||
|
} \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
#undef E
|
||||||
|
|
||||||
|
assert(no_event_on == !thread_event_active);
|
||||||
|
assert(wait <= THREAD_EVENT_MAX_START_WAIT);
|
||||||
|
return wait;
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
thread_event_assert_invariants_debug(tsd_t *tsd) {
|
||||||
|
uint64_t thread_allocated = thread_allocated_get(tsd);
|
||||||
|
uint64_t last_event = thread_allocated_last_event_get(tsd);
|
||||||
|
uint64_t next_event = thread_allocated_next_event_get(tsd);
|
||||||
|
uint64_t next_event_fast = thread_allocated_next_event_fast_get(tsd);
|
||||||
|
|
||||||
|
assert(last_event != next_event);
|
||||||
|
if (next_event <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX) {
|
||||||
|
assert(next_event_fast == next_event);
|
||||||
|
} else {
|
||||||
|
assert(next_event_fast == 0U);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The subtraction is intentionally susceptible to underflow. */
|
||||||
|
uint64_t interval = next_event - last_event;
|
||||||
|
|
||||||
|
/* The subtraction is intentionally susceptible to underflow. */
|
||||||
|
assert(thread_allocated - last_event < interval);
|
||||||
|
|
||||||
|
uint64_t min_wait = thread_allocated_next_event_compute(tsd);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* next_event should have been pushed up only except when no event is
|
||||||
|
* on and the TSD is just initialized. The last_event == 0U guard
|
||||||
|
* below is stronger than needed, but having an exactly accurate guard
|
||||||
|
* is more complicated to implement.
|
||||||
|
*/
|
||||||
|
assert((!thread_event_active && last_event == 0U) ||
|
||||||
|
interval == min_wait ||
|
||||||
|
(interval < min_wait && interval == THREAD_EVENT_MAX_INTERVAL));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
thread_event_adjust_thresholds_helper(tsd_t *tsd, uint64_t wait) {
|
||||||
|
assert(wait <= THREAD_EVENT_MAX_START_WAIT);
|
||||||
|
uint64_t next_event = thread_allocated_last_event_get(tsd) + (wait <=
|
||||||
|
THREAD_EVENT_MAX_INTERVAL ? wait : THREAD_EVENT_MAX_INTERVAL);
|
||||||
|
thread_allocated_next_event_set(tsd, next_event);
|
||||||
|
uint64_t next_event_fast = (next_event <=
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX) ? next_event : 0U;
|
||||||
|
thread_allocated_next_event_fast_set(tsd, next_event_fast);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
thread_prof_sample_event_handler(tsd_t *tsd) {
|
||||||
|
assert(config_prof && opt_prof);
|
||||||
|
assert(prof_sample_event_wait_get(tsd) == 0U);
|
||||||
|
if (!prof_active_get_unlocked()) {
|
||||||
|
/*
|
||||||
|
* If prof_active is off, we reset prof_sample_event_wait to be
|
||||||
|
* the sample interval when it drops to 0, so that there won't
|
||||||
|
* be excessive routings to the slow path, and that when
|
||||||
|
* prof_active is turned on later, the counting for sampling
|
||||||
|
* can immediately resume as normal.
|
||||||
|
*/
|
||||||
|
thread_prof_sample_event_update(tsd,
|
||||||
|
(uint64_t)(1 << lg_prof_sample));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint64_t
|
||||||
|
thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
|
||||||
|
bool allow_event_trigger) {
|
||||||
|
uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
|
||||||
|
|
||||||
|
#define E(event, condition) \
|
||||||
|
if (condition) { \
|
||||||
|
uint64_t event_wait = event##_event_wait_get(tsd); \
|
||||||
|
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
|
||||||
|
if (event_wait > accumbytes) { \
|
||||||
|
event_wait -= accumbytes; \
|
||||||
|
} else { \
|
||||||
|
event_wait = 0U; \
|
||||||
|
if (!allow_event_trigger) { \
|
||||||
|
event_wait = \
|
||||||
|
THREAD_EVENT_MIN_START_WAIT; \
|
||||||
|
} \
|
||||||
|
} \
|
||||||
|
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
|
||||||
|
event##_event_wait_set(tsd, event_wait); \
|
||||||
|
/* \
|
||||||
|
* If there is a single event, then the remaining wait \
|
||||||
|
* time may become zero, and we rely on either the \
|
||||||
|
* event handler or a thread_event_update() call later \
|
||||||
|
* to properly set next_event; if there are multiple \
|
||||||
|
* events, then here we can get the minimum remaining \
|
||||||
|
* wait time to the next already set event. \
|
||||||
|
*/ \
|
||||||
|
if (event_wait > 0U && event_wait < wait) { \
|
||||||
|
wait = event_wait; \
|
||||||
|
} \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
#undef E
|
||||||
|
|
||||||
|
assert(wait <= THREAD_EVENT_MAX_START_WAIT);
|
||||||
|
return wait;
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
thread_event_trigger(tsd_t *tsd, bool delay_event) {
|
||||||
|
/* usize has already been added to thread_allocated. */
|
||||||
|
uint64_t thread_allocated_after = thread_allocated_get(tsd);
|
||||||
|
|
||||||
|
/* The subtraction is intentionally susceptible to underflow. */
|
||||||
|
uint64_t accumbytes = thread_allocated_after -
|
||||||
|
thread_allocated_last_event_get(tsd);
|
||||||
|
|
||||||
|
/* Make sure that accumbytes cannot overflow uint64_t. */
|
||||||
|
cassert(THREAD_EVENT_MAX_INTERVAL <=
|
||||||
|
UINT64_MAX - SC_LARGE_MAXCLASS + 1);
|
||||||
|
|
||||||
|
thread_allocated_last_event_set(tsd, thread_allocated_after);
|
||||||
|
bool allow_event_trigger = !delay_event && tsd_nominal(tsd) &&
|
||||||
|
tsd_reentrancy_level_get(tsd) == 0;
|
||||||
|
uint64_t wait = thread_event_trigger_batch_update(tsd, accumbytes,
|
||||||
|
allow_event_trigger);
|
||||||
|
thread_event_adjust_thresholds_helper(tsd, wait);
|
||||||
|
|
||||||
|
thread_event_assert_invariants(tsd);
|
||||||
|
|
||||||
|
#define E(event, condition) \
|
||||||
|
if (condition && event##_event_wait_get(tsd) == 0U) { \
|
||||||
|
assert(allow_event_trigger); \
|
||||||
|
thread_##event##_event_handler(tsd); \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
#undef E
|
||||||
|
|
||||||
|
thread_event_assert_invariants(tsd);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
thread_event_rollback(tsd_t *tsd, size_t diff) {
|
||||||
|
thread_event_assert_invariants(tsd);
|
||||||
|
|
||||||
|
if (diff == 0U) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t thread_allocated = thread_allocated_get(tsd);
|
||||||
|
/* The subtraction is intentionally susceptible to underflow. */
|
||||||
|
uint64_t thread_allocated_rollback = thread_allocated - diff;
|
||||||
|
thread_allocated_set(tsd, thread_allocated_rollback);
|
||||||
|
|
||||||
|
uint64_t last_event = thread_allocated_last_event_get(tsd);
|
||||||
|
/* Both subtractions are intentionally susceptible to underflow. */
|
||||||
|
if (thread_allocated_rollback - last_event <=
|
||||||
|
thread_allocated - last_event) {
|
||||||
|
thread_event_assert_invariants(tsd);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
thread_allocated_last_event_set(tsd, thread_allocated_rollback);
|
||||||
|
|
||||||
|
/* The subtraction is intentionally susceptible to underflow. */
|
||||||
|
uint64_t wait_diff = last_event - thread_allocated_rollback;
|
||||||
|
assert(wait_diff <= diff);
|
||||||
|
|
||||||
|
#define E(event, condition) \
|
||||||
|
if (condition) { \
|
||||||
|
uint64_t event_wait = event##_event_wait_get(tsd); \
|
||||||
|
assert(event_wait <= THREAD_EVENT_MAX_START_WAIT); \
|
||||||
|
if (event_wait > 0U) { \
|
||||||
|
if (wait_diff > \
|
||||||
|
THREAD_EVENT_MAX_START_WAIT - event_wait) { \
|
||||||
|
event_wait = \
|
||||||
|
THREAD_EVENT_MAX_START_WAIT; \
|
||||||
|
} else { \
|
||||||
|
event_wait += wait_diff; \
|
||||||
|
} \
|
||||||
|
assert(event_wait <= \
|
||||||
|
THREAD_EVENT_MAX_START_WAIT); \
|
||||||
|
event##_event_wait_set(tsd, event_wait); \
|
||||||
|
} \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
#undef E
|
||||||
|
|
||||||
|
thread_event_update(tsd);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
thread_event_update(tsd_t *tsd) {
|
||||||
|
uint64_t wait = thread_allocated_next_event_compute(tsd);
|
||||||
|
thread_event_adjust_thresholds_helper(tsd, wait);
|
||||||
|
|
||||||
|
uint64_t last_event = thread_allocated_last_event_get(tsd);
|
||||||
|
|
||||||
|
/* Both subtractions are intentionally susceptible to underflow. */
|
||||||
|
if (thread_allocated_get(tsd) - last_event >=
|
||||||
|
thread_allocated_next_event_get(tsd) - last_event) {
|
||||||
|
thread_event_trigger(tsd, true);
|
||||||
|
} else {
|
||||||
|
thread_event_assert_invariants(tsd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void thread_event_boot() {
|
||||||
|
#define E(event, condition) \
|
||||||
|
if (condition) { \
|
||||||
|
thread_event_active = true; \
|
||||||
|
}
|
||||||
|
|
||||||
|
ITERATE_OVER_ALL_EVENTS
|
||||||
|
#undef E
|
||||||
|
}
|
57
test/unit/thread_event.c
Normal file
57
test/unit/thread_event.c
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
#include "test/jemalloc_test.h"
|
||||||
|
|
||||||
|
TEST_BEGIN(test_next_event_fast_roll_back) {
|
||||||
|
tsd_t *tsd = tsd_fetch();
|
||||||
|
thread_allocated_last_event_set(tsd, 0);
|
||||||
|
thread_allocated_set(tsd,
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX - 8U);
|
||||||
|
thread_allocated_next_event_set(tsd,
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
|
||||||
|
thread_allocated_next_event_fast_set(tsd,
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
|
||||||
|
prof_sample_event_wait_set(tsd,
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
|
||||||
|
void *p = malloc(16U);
|
||||||
|
assert_ptr_not_null(p, "malloc() failed");
|
||||||
|
free(p);
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
|
TEST_BEGIN(test_next_event_fast_resume) {
|
||||||
|
tsd_t *tsd = tsd_fetch();
|
||||||
|
thread_allocated_last_event_set(tsd, 0);
|
||||||
|
thread_allocated_set(tsd,
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 8U);
|
||||||
|
thread_allocated_next_event_set(tsd,
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U);
|
||||||
|
thread_allocated_next_event_fast_set(tsd, 0);
|
||||||
|
prof_sample_event_wait_set(tsd,
|
||||||
|
THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U);
|
||||||
|
void *p = malloc(SC_LOOKUP_MAXCLASS);
|
||||||
|
assert_ptr_not_null(p, "malloc() failed");
|
||||||
|
free(p);
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
|
TEST_BEGIN(test_event_rollback) {
|
||||||
|
tsd_t *tsd = tsd_fetch();
|
||||||
|
const uint64_t diff = THREAD_EVENT_MAX_INTERVAL >> 2;
|
||||||
|
size_t count = 10;
|
||||||
|
uint64_t thread_allocated = thread_allocated_get(tsd);
|
||||||
|
while (count-- != 0) {
|
||||||
|
thread_event_rollback(tsd, diff);
|
||||||
|
uint64_t thread_allocated_after = thread_allocated_get(tsd);
|
||||||
|
assert_u64_eq(thread_allocated - thread_allocated_after, diff,
|
||||||
|
"thread event counters are not properly rolled back");
|
||||||
|
thread_allocated = thread_allocated_after;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TEST_END
|
||||||
|
|
||||||
|
int
|
||||||
|
main(void) {
|
||||||
|
return test(
|
||||||
|
test_next_event_fast_roll_back,
|
||||||
|
test_next_event_fast_resume,
|
||||||
|
test_event_rollback);
|
||||||
|
}
|
5
test/unit/thread_event.sh
Normal file
5
test/unit/thread_event.sh
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
if [ "x${enable_prof}" = "x1" ] ; then
|
||||||
|
export MALLOC_CONF="prof:true,lg_prof_sample:0"
|
||||||
|
fi
|
Loading…
Reference in New Issue
Block a user