project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integrate auto tcache into TSD.

Browse Source 
The embedded tcache is initialized upon tsd initialization.  The avail arrays
for the tbins will be allocated / deallocated accordingly during init / cleanup.

With this change, the pointer to the auto tcache will always be available, as
long as we have access to the TSD.  tcache_available() (called in tcache_get())
is provided to check if we should use tcache.
This commit is contained in:
Qi Wang 2017-03-27 21:50:38 -07:00 committed by Qi Wang
parent eeabdd2466
commit fde3e20cc0
16 changed files with 300 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/jemalloc/internal/arena_inlines_a.h
View File

@ -57,15 +57,13 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
/* Set new arena/tcache associations. */ /* Set new arena/tcache associations. */
arena_migrate(tsd, oldind, newind); arena_migrate(tsd, oldind, newind);
if (config_tcache) { tcache_t *tcache = tcache_get(tsd);
tcache_t *tcache = tsd_tcache_get(tsd); if (config_tcache && tcache) {
if (tcache) {
tcache_arena_reassociate(tsd_tsdn(tsd), tcache, tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
newarena); newarena);
} }
} }
} }
}
#endif /* (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) */ #endif /* (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) */

62
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -505,6 +505,7 @@ void arenas_tdata_cleanup(tsd_t *tsd);
void jemalloc_prefork(void); void jemalloc_prefork(void);
void jemalloc_postfork_parent(void); void jemalloc_postfork_parent(void);
void jemalloc_postfork_child(void); void jemalloc_postfork_child(void);
bool malloc_initialized(void);
#include "jemalloc/internal/nstime_externs.h" #include "jemalloc/internal/nstime_externs.h"
#include "jemalloc/internal/ckh_externs.h" #include "jemalloc/internal/ckh_externs.h"
@ -559,6 +560,8 @@ arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind,
bool refresh_if_missing); bool refresh_if_missing);
arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing);
ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind);
bool tcache_available(tsd_t *tsd);
tcache_t *tcache_get(tsd_t *tsd);
malloc_cpuid_t malloc_getcpu(void); malloc_cpuid_t malloc_getcpu(void);
unsigned percpu_arena_choose(void); unsigned percpu_arena_choose(void);
unsigned percpu_arena_ind_limit(void); unsigned percpu_arena_ind_limit(void);
@ -929,6 +932,38 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) {
} }
return &tdata->decay_ticker; return &tdata->decay_ticker;
} }
JEMALLOC_ALWAYS_INLINE bool
tcache_available(tsd_t *tsd) {
cassert(config_tcache);
/*
* Thread specific auto tcache might be unavailable if: 1) during tcache
* initialization, or 2) disabled through thread.tcache.enabled mallctl
* or config options. This check covers all cases.
*/
if (likely(tsd_tcache_enabled_get(tsd) == tcache_enabled_true)) {
/* Associated arena == null implies tcache init in progress. */
if (tsd_tcachep_get(tsd)->arena != NULL) {
assert(tsd_tcachep_get(tsd)->tbins[0].avail != NULL);
}
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get(tsd_t *tsd) {
if (!config_tcache) {
return NULL;
}
if (!tcache_available(tsd)) {
return NULL;
}
return tsd_tcachep_get(tsd);
}
#endif #endif
#include "jemalloc/internal/rtree_inlines.h" #include "jemalloc/internal/rtree_inlines.h"
@ -959,9 +994,24 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd); ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
if (unlikely(ret == NULL)) { if (unlikely(ret == NULL)) {
ret = arena_choose_hard(tsd, internal); ret = arena_choose_hard(tsd, internal);
assert(ret);
if (config_tcache && tcache_available(tsd)) {
tcache_t *tcache = tcache_get(tsd);
if (tcache->arena != NULL) {
/* See comments in tcache_data_init().*/
assert(tcache->arena ==
arena_get(tsd_tsdn(tsd), 0, false));
if (tcache->arena != ret) {
tcache_arena_reassociate(tsd_tsdn(tsd),
tcache, ret);
}
} else {
tcache_arena_associate(tsd_tsdn(tsd), tcache,
ret);
}
}
} }
assert(ret != NULL);
/* /*
* Note that for percpu arena, if the current arena is outside of the * Note that for percpu arena, if the current arena is outside of the
* auto percpu arena range, (i.e. thread is assigned to a manually * auto percpu arena range, (i.e. thread is assigned to a manually
@ -1069,8 +1119,8 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) {
return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd, true), return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false,
false, NULL, slow_path); NULL, slow_path);
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
@ -1102,7 +1152,7 @@ ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) {
return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero,
tcache_get(tsd, true), false, NULL); tcache_get(tsd), false, NULL);
} }
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
@ -1127,7 +1177,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal,
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
idalloc(tsd_t *tsd, void *ptr) { idalloc(tsd_t *tsd, void *ptr) {
idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true); idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), false, true);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
@ -1199,7 +1249,7 @@ JEMALLOC_ALWAYS_INLINE void *
iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
bool zero) { bool zero) {
return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero,
tcache_get(tsd, true), NULL); tcache_get(tsd), NULL);
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool

8
include/jemalloc/internal/private_symbols.txt
View File

@ -274,6 +274,7 @@ lg_floor
lg_prof_sample lg_prof_sample
malloc_cprintf malloc_cprintf
malloc_getcpu malloc_getcpu
malloc_initialized
malloc_mutex_prof_data_reset malloc_mutex_prof_data_reset
malloc_mutex_assert_not_owner malloc_mutex_assert_not_owner
malloc_mutex_assert_owner malloc_mutex_assert_owner
@ -293,7 +294,6 @@ malloc_tsd_boot1
malloc_tsd_cleanup_register malloc_tsd_cleanup_register
malloc_tsd_dalloc malloc_tsd_dalloc
malloc_tsd_malloc malloc_tsd_malloc
malloc_tsd_no_cleanup
malloc_vcprintf malloc_vcprintf
malloc_vsnprintf malloc_vsnprintf
malloc_write malloc_write
@ -475,22 +475,23 @@ tcache_alloc_easy
tcache_alloc_large tcache_alloc_large
tcache_alloc_small tcache_alloc_small
tcache_alloc_small_hard tcache_alloc_small_hard
tcache_arena_associate
tcache_arena_reassociate tcache_arena_reassociate
tcache_bin_flush_large tcache_bin_flush_large
tcache_bin_flush_small tcache_bin_flush_small
tcache_bin_info tcache_bin_info
tcache_boot tcache_boot
tcache_cleanup tcache_cleanup
tcache_create tcache_create_explicit
tcache_dalloc_large tcache_dalloc_large
tcache_dalloc_small tcache_dalloc_small
tcache_data_init
tcache_enabled_get tcache_enabled_get
tcache_enabled_set tcache_enabled_set
tcache_event tcache_event
tcache_event_hard tcache_event_hard
tcache_flush tcache_flush
tcache_get tcache_get
tcache_get_hard
tcache_maxclass tcache_maxclass
tcache_prefork tcache_prefork
tcache_postfork_child tcache_postfork_child
@ -521,7 +522,6 @@ tsd_booted
tsd_booted_get tsd_booted_get
tsd_cleanup tsd_cleanup
tsd_cleanup_wrapper tsd_cleanup_wrapper
tsd_data_init
tsd_fetch tsd_fetch
tsd_fetch_impl tsd_fetch_impl
tsd_get tsd_get

7
include/jemalloc/internal/rtree_inlines.h
View File

@ -321,13 +321,18 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
assert(!dependent || !init_missing); assert(!dependent || !init_missing);
uintptr_t leafkey = rtree_leafkey(key); uintptr_t leafkey = rtree_leafkey(key);
assert(leafkey != RTREE_LEAFKEY_INVALID);
#define RTREE_CACHE_CHECK(i) do { \ #define RTREE_CACHE_CHECK(i) do { \
if (likely(rtree_ctx->cache[i].leafkey == leafkey)) { \ if (likely(rtree_ctx->cache[i].leafkey == leafkey)) { \
rtree_leaf_elm_t *leaf = rtree_ctx->cache[i].leaf; \ rtree_leaf_elm_t *leaf = rtree_ctx->cache[i].leaf; \
assert(leaf != NULL); \ assert(leaf != NULL); \
if (i > 0) { \ if (i > 0) { \
/* Bubble up by one. */ \ /* Bubble up by one. */ \
rtree_ctx->cache[i] = rtree_ctx->cache[i - 1]; \ rtree_ctx->cache[i].leafkey = \
rtree_ctx->cache[i - 1].leafkey; \
rtree_ctx->cache[i].leaf = \
rtree_ctx->cache[i - 1].leaf; \
rtree_ctx->cache[i - 1].leafkey = leafkey; \ rtree_ctx->cache[i - 1].leafkey = leafkey; \
rtree_ctx->cache[i - 1].leaf = leaf; \ rtree_ctx->cache[i - 1].leaf = leaf; \
} \ } \

7
include/jemalloc/internal/tcache_externs.h
View File

@ -35,16 +35,19 @@ void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
unsigned rem, tcache_t *tcache); unsigned rem, tcache_t *tcache);
void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
arena_t *arena); arena_t *arena);
tcache_t *tcache_get_hard(tsd_t *tsd); tcache_t *tcache_create_explicit(tsd_t *tsd);
tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena);
void tcache_cleanup(tsd_t *tsd); void tcache_cleanup(tsd_t *tsd);
void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
bool tcaches_create(tsd_t *tsd, unsigned *r_ind); bool tcaches_create(tsd_t *tsd, unsigned *r_ind);
void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_flush(tsd_t *tsd, unsigned ind);
void tcaches_destroy(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind);
bool tcache_boot(tsdn_t *tsdn); bool tcache_boot(tsdn_t *tsdn);
void tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
void tcache_prefork(tsdn_t *tsdn); void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn); void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn); void tcache_postfork_child(tsdn_t *tsdn);
void tcache_flush(void);
bool tsd_tcache_data_init(tsd_t *tsd);
bool tsd_tcache_enabled_data_init(tsd_t *tsd);
#endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */ #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

65
include/jemalloc/internal/tcache_inlines.h
View File

@ -4,9 +4,9 @@
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
void tcache_event(tsd_t *tsd, tcache_t *tcache); void tcache_event(tsd_t *tsd, tcache_t *tcache);
void tcache_flush(void); void tcache_flush(void);
bool tcache_enabled_get(void); bool tcache_enabled_get(tsd_t *tsd);
tcache_t *tcache_get(tsd_t *tsd, bool create); tcache_t *tcache_get(tsd_t *tsd);
void tcache_enabled_set(bool enabled); void tcache_enabled_set(tsd_t *tsd, bool enabled);
void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success); void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
size_t size, szind_t ind, bool zero, bool slow_path); size_t size, szind_t ind, bool zero, bool slow_path);
@ -20,68 +20,32 @@ tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
#endif #endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
JEMALLOC_INLINE void
tcache_flush(void) {
tsd_t *tsd;
cassert(config_tcache);
tsd = tsd_fetch();
tcache_cleanup(tsd);
}
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
tcache_enabled_get(void) { tcache_enabled_get(tsd_t *tsd) {
tsd_t *tsd;
tcache_enabled_t tcache_enabled; tcache_enabled_t tcache_enabled;
cassert(config_tcache); cassert(config_tcache);
tsd = tsd_fetch();
tcache_enabled = tsd_tcache_enabled_get(tsd); tcache_enabled = tsd_tcache_enabled_get(tsd);
if (tcache_enabled == tcache_enabled_default) { assert(tcache_enabled != tcache_enabled_default);
tcache_enabled = (tcache_enabled_t)opt_tcache;
tsd_tcache_enabled_set(tsd, tcache_enabled);
}
return (bool)tcache_enabled; return (bool)tcache_enabled;
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
tcache_enabled_set(bool enabled) { tcache_enabled_set(tsd_t *tsd, bool enabled) {
tsd_t *tsd;
tcache_enabled_t tcache_enabled;
cassert(config_tcache); cassert(config_tcache);
tsd = tsd_fetch(); tcache_enabled_t old = tsd_tcache_enabled_get(tsd);
tcache_enabled = (tcache_enabled_t)enabled; if ((old != tcache_enabled_true) && enabled) {
tsd_tcache_enabled_set(tsd, tcache_enabled); tsd_tcache_data_init(tsd);
} else if ((old == tcache_enabled_true) && !enabled) {
if (!enabled) {
tcache_cleanup(tsd); tcache_cleanup(tsd);
} }
} /* Commit the state last. Above calls check current state. */
tcache_enabled_t tcache_enabled = (tcache_enabled_t)enabled;
JEMALLOC_ALWAYS_INLINE tcache_t * tsd_tcache_enabled_set(tsd, tcache_enabled);
tcache_get(tsd_t *tsd, bool create) {
tcache_t *tcache;
if (!config_tcache) {
return NULL;
}
tcache = tsd_tcache_get(tsd);
if (!create) {
return tcache;
}
if (unlikely(tcache == NULL) && tsd_nominal(tsd)) {
tcache = tcache_get_hard(tsd);
tsd_tcache_set(tsd, tcache);
}
return tcache;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
@ -300,8 +264,7 @@ JEMALLOC_ALWAYS_INLINE tcache_t *
tcaches_get(tsd_t *tsd, unsigned ind) { tcaches_get(tsd_t *tsd, unsigned ind) {
tcaches_t *elm = &tcaches[ind]; tcaches_t *elm = &tcaches[ind];
if (unlikely(elm->tcache == NULL)) { if (unlikely(elm->tcache == NULL)) {
elm->tcache = tcache_create(tsd_tsdn(tsd), arena_choose(tsd, elm->tcache = tcache_create_explicit(tsd);
NULL));
} }
return elm->tcache; return elm->tcache;
} }

6
include/jemalloc/internal/tcache_structs.h
View File

@ -36,13 +36,17 @@ struct tcache_s {
ticker_t gc_ticker; /* Drives incremental GC. */ ticker_t gc_ticker; /* Drives incremental GC. */
szind_t next_gc_bin; /* Next bin to GC. */ szind_t next_gc_bin; /* Next bin to GC. */
arena_t *arena; /* Associated arena. */ arena_t *arena; /* Associated arena. */
tcache_bin_t tbins[1]; /* Dynamically sized. */
/* /*
* The pointer stacks associated with tbins follow as a contiguous * The pointer stacks associated with tbins follow as a contiguous
* array. During tcache initialization, the avail pointer in each * array. During tcache initialization, the avail pointer in each
* element of tbins is initialized to point to the proper offset within * element of tbins is initialized to point to the proper offset within
* this array. * this array.
*/ */
#ifdef JEMALLOC_TCACHE
tcache_bin_t tbins[NSIZES];
#else
tcache_bin_t tbins[0];
#endif
}; };
/* Linkage for list of available (previously used) explicit tcache IDs. */ /* Linkage for list of available (previously used) explicit tcache IDs. */

3
include/jemalloc/internal/tcache_types.h
View File

@ -47,4 +47,7 @@ typedef struct tcaches_s tcaches_t;
#define TCACHE_GC_INCR \ #define TCACHE_GC_INCR \
((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1))
/* Used in TSD static initializer only. Real init in tcache_data_init(). */
#define TCACHE_ZERO_INITIALIZER {{NULL}}
#endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */

3
include/jemalloc/internal/tsd_externs.h
View File

@ -3,7 +3,6 @@
void *malloc_tsd_malloc(size_t size); void *malloc_tsd_malloc(size_t size);
void malloc_tsd_dalloc(void *wrapper); void malloc_tsd_dalloc(void *wrapper);
void malloc_tsd_no_cleanup(void *arg);
void malloc_tsd_cleanup_register(bool (*f)(void)); void malloc_tsd_cleanup_register(bool (*f)(void));
tsd_t *malloc_tsd_boot0(void); tsd_t *malloc_tsd_boot0(void);
void malloc_tsd_boot1(void); void malloc_tsd_boot1(void);
@ -13,7 +12,7 @@ void *tsd_init_check_recursion(tsd_init_head_t *head,
tsd_init_block_t *block); tsd_init_block_t *block);
void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
#endif #endif
void tsd_cleanup(void *arg);
bool tsd_data_init(void *arg); bool tsd_data_init(void *arg);
void tsd_cleanup(void *arg);
#endif /* JEMALLOC_INTERNAL_TSD_EXTERNS_H */ #endif /* JEMALLOC_INTERNAL_TSD_EXTERNS_H */

7
include/jemalloc/internal/tsd_structs.h
View File

@ -16,7 +16,7 @@ struct tsd_init_head_s {
#define MALLOC_TSD \ #define MALLOC_TSD \
/* O(name, type, [gs]et, init, cleanup) */ \ /* O(name, type, [gs]et, init, cleanup) */ \
O(tcache, tcache_t *, yes, no, yes) \ O(tcache, tcache_t, yes, no, yes) \
O(thread_allocated, uint64_t, yes, no, no) \ O(thread_allocated, uint64_t, yes, no, no) \
O(thread_deallocated, uint64_t, yes, no, no) \ O(thread_deallocated, uint64_t, yes, no, no) \
O(prof_tdata, prof_tdata_t *, yes, no, yes) \ O(prof_tdata, prof_tdata_t *, yes, no, yes) \
@ -26,7 +26,7 @@ struct tsd_init_head_s {
O(narenas_tdata, unsigned, yes, no, no) \ O(narenas_tdata, unsigned, yes, no, no) \
O(arenas_tdata_bypass, bool, no, no, no) \ O(arenas_tdata_bypass, bool, no, no, no) \
O(tcache_enabled, tcache_enabled_t, \ O(tcache_enabled, tcache_enabled_t, \
yes, no, no) \ yes, yes, no) \
O(rtree_ctx, rtree_ctx_t, no, yes, no) \ O(rtree_ctx, rtree_ctx_t, no, yes, no) \
O(witnesses, witness_list_t, no, no, yes) \ O(witnesses, witness_list_t, no, no, yes) \
O(rtree_leaf_elm_witnesses, rtree_leaf_elm_witness_tsd_t, \ O(rtree_leaf_elm_witnesses, rtree_leaf_elm_witness_tsd_t, \
@ -35,7 +35,7 @@ struct tsd_init_head_s {
#define TSD_INITIALIZER { \ #define TSD_INITIALIZER { \
tsd_state_uninitialized, \ tsd_state_uninitialized, \
NULL, \ TCACHE_ZERO_INITIALIZER, \
0, \ 0, \
0, \ 0, \
NULL, \ NULL, \
@ -69,6 +69,7 @@ struct tsdn_s {
}; };
static const tsd_t tsd_initializer = TSD_INITIALIZER; static const tsd_t tsd_initializer = TSD_INITIALIZER;
UNUSED static const void *malloc_tsd_no_cleanup = (void *)0;
malloc_tsd_types(, tsd_t) malloc_tsd_types(, tsd_t)

12
include/jemalloc/internal/tsd_types.h
View File

@ -357,8 +357,10 @@ a_name##tsd_boot1(void) { \
" TSD for "#a_name"\n"); \ " TSD for "#a_name"\n"); \
abort(); \ abort(); \
} \ } \
memcpy(wrapper, &a_name##tsd_boot_wrapper, \ a_name##tsd_boot_wrapper.initialized = false; \
sizeof(a_name##tsd_wrapper_t)); \ a_cleanup(&a_name##tsd_boot_wrapper.val); \
wrapper->initialized = false; \
wrapper->val = a_initializer; \
a_name##tsd_wrapper_set(wrapper); \ a_name##tsd_wrapper_set(wrapper); \
} \ } \
a_attr bool \ a_attr bool \
@ -487,8 +489,10 @@ a_name##tsd_boot1(void) { \
" TSD for "#a_name"\n"); \ " TSD for "#a_name"\n"); \
abort(); \ abort(); \
} \ } \
memcpy(wrapper, &a_name##tsd_boot_wrapper, \ a_name##tsd_boot_wrapper.initialized = false; \
sizeof(a_name##tsd_wrapper_t)); \ a_cleanup(&a_name##tsd_boot_wrapper.val); \
wrapper->initialized = false; \
wrapper->val = a_initializer; \
a_name##tsd_wrapper_set(wrapper); \ a_name##tsd_wrapper_set(wrapper); \
} \ } \
a_attr bool \ a_attr bool \

6
src/ctl.c
View File

@ -1532,7 +1532,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
/* Set new arena/tcache associations. */ /* Set new arena/tcache associations. */
arena_migrate(tsd, oldind, newind); arena_migrate(tsd, oldind, newind);
if (config_tcache) { if (config_tcache) {
tcache_t *tcache = tsd_tcache_get(tsd); tcache_t *tcache = tsd_tcachep_get(tsd);
if (tcache != NULL) { if (tcache != NULL) {
tcache_arena_reassociate(tsd_tsdn(tsd), tcache, tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
newarena); newarena);
@ -1564,13 +1564,13 @@ thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
return ENOENT; return ENOENT;
} }
oldval = tcache_enabled_get(); oldval = tcache_enabled_get(tsd);
if (newp != NULL) { if (newp != NULL) {
if (newlen != sizeof(bool)) { if (newlen != sizeof(bool)) {
ret = EINVAL; ret = EINVAL;
goto label_return; goto label_return;
} }
tcache_enabled_set(*(bool *)newp); tcache_enabled_set(tsd, *(bool *)newp);
} }
READ(oldval, bool); READ(oldval, bool);

16
src/jemalloc.c
View File

@ -275,7 +275,7 @@ static bool malloc_init_hard(void);
* Begin miscellaneous support functions. * Begin miscellaneous support functions.
*/ */
JEMALLOC_ALWAYS_INLINE_C bool bool
malloc_initialized(void) { malloc_initialized(void) {
return (malloc_init_state == malloc_init_initialized); return (malloc_init_state == malloc_init_initialized);
} }
@ -1536,7 +1536,7 @@ imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd,
/* Fill in the tcache. */ /* Fill in the tcache. */
if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) { if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) {
tcache = tcache_get(tsd, true); tcache = tcache_get(tsd);
} else if (dopts->tcache_ind == TCACHE_IND_NONE) { } else if (dopts->tcache_ind == TCACHE_IND_NONE) {
tcache = NULL; tcache = NULL;
} else { } else {
@ -2056,7 +2056,7 @@ je_realloc(void *ptr, size_t size) {
/* realloc(ptr, 0) is equivalent to free(ptr). */ /* realloc(ptr, 0) is equivalent to free(ptr). */
UTRACE(ptr, 0, 0); UTRACE(ptr, 0, 0);
tsd = tsd_fetch(); tsd = tsd_fetch();
ifree(tsd, ptr, tcache_get(tsd, false), true); ifree(tsd, ptr, tcache_get(tsd), true);
return NULL; return NULL;
} }
size = 1; size = 1;
@ -2113,9 +2113,9 @@ je_free(void *ptr) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
if (likely(!malloc_slow)) { if (likely(!malloc_slow)) {
ifree(tsd, ptr, tcache_get(tsd, false), false); ifree(tsd, ptr, tcache_get(tsd), false);
} else { } else {
ifree(tsd, ptr, tcache_get(tsd, false), true); ifree(tsd, ptr, tcache_get(tsd), true);
} }
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
} }
@ -2393,7 +2393,7 @@ je_rallocx(void *ptr, size_t size, int flags) {
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
} }
} else { } else {
tcache = tcache_get(tsd, true); tcache = tcache_get(tsd);
} }
old_usize = isalloc(tsd_tsdn(tsd), ptr); old_usize = isalloc(tsd_tsdn(tsd), ptr);
@ -2605,7 +2605,7 @@ je_dallocx(void *ptr, int flags) {
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
} }
} else { } else {
tcache = tcache_get(tsd, false); tcache = tcache_get(tsd);
} }
UTRACE(ptr, 0, 0); UTRACE(ptr, 0, 0);
@ -2652,7 +2652,7 @@ je_sdallocx(void *ptr, size_t size, int flags) {
tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags));
} }
} else { } else {
tcache = tcache_get(tsd, false); tcache = tcache_get(tsd);
} }
UTRACE(ptr, 0, 0); UTRACE(ptr, 0, 0);

217
src/tcache.c
View File

@ -4,7 +4,13 @@
/******************************************************************************/ /******************************************************************************/
/* Data. */ /* Data. */
bool opt_tcache = true; bool opt_tcache =
#ifdef JEMALLOC_TCACHE
true
#else
false
#endif
;
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
tcache_bin_info_t *tcache_bin_info; tcache_bin_info_t *tcache_bin_info;
@ -78,6 +84,7 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
tcache_bin_t *tbin, szind_t binind, bool *tcache_success) { tcache_bin_t *tbin, szind_t binind, bool *tcache_success) {
void *ret; void *ret;
assert(tcache->arena);
arena_tcache_fill_small(tsdn, arena, tbin, binind, config_prof ? arena_tcache_fill_small(tsdn, arena, tbin, binind, config_prof ?
tcache->prof_accumbytes : 0); tcache->prof_accumbytes : 0);
if (config_prof) { if (config_prof) {
@ -271,9 +278,11 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
} }
} }
static void void
tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
assert(tcache->arena == NULL);
tcache->arena = arena; tcache->arena = arena;
if (config_stats) { if (config_stats) {
/* Link into list of extant tcaches. */ /* Link into list of extant tcaches. */
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
@ -286,6 +295,7 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
static void static void
tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
arena_t *arena = tcache->arena; arena_t *arena = tcache->arena;
assert(arena);
if (config_stats) { if (config_stats) {
/* Unlink from list of extant tcaches. */ /* Unlink from list of extant tcaches. */
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
@ -304,6 +314,7 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
tcache_stats_merge(tsdn, tcache, arena); tcache_stats_merge(tsdn, tcache, arena);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
} }
tcache->arena = NULL;
} }
void void
@ -312,49 +323,32 @@ tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
tcache_arena_associate(tsdn, tcache, arena); tcache_arena_associate(tsdn, tcache, arena);
} }
tcache_t * bool
tcache_get_hard(tsd_t *tsd) { tsd_tcache_enabled_data_init(tsd_t *tsd) {
arena_t *arena; /* Called upon tsd initialization. */
tsd_tcache_enabled_set(tsd, (tcache_enabled_t)opt_tcache);
if (!tcache_enabled_get()) { if (opt_tcache) {
if (tsd_nominal(tsd)) { /* Trigger tcache init. */
tcache_enabled_set(false); /* Memoize. */ tsd_tcache_data_init(tsd);
}
return NULL;
}
arena = arena_choose(tsd, NULL);
if (unlikely(arena == NULL)) {
return NULL;
}
return tcache_create(tsd_tsdn(tsd), arena);
} }
tcache_t * return false;
tcache_create(tsdn_t *tsdn, arena_t *arena) {
tcache_t *tcache;
size_t size, stack_offset;
unsigned i;
size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins);
/* Naturally align the pointer stacks. */
size = PTR_CEILING(size);
stack_offset = size;
size += stack_nelms * sizeof(void *);
/* Avoid false cacheline sharing. */
size = sa2u(size, CACHELINE);
tcache = ipallocztm(tsdn, size, CACHELINE, true, NULL, true,
arena_get(TSDN_NULL, 0, true));
if (tcache == NULL) {
return NULL;
} }
tcache_arena_associate(tsdn, tcache, arena); /* Initialize auto tcache (embedded in TSD). */
static void
tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
memset(&tcache->link, 0, sizeof(ql_elm(tcache_t)));
tcache->prof_accumbytes = 0;
tcache->next_gc_bin = 0;
tcache->arena = NULL;
ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR); ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
size_t stack_offset = 0;
assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
for (i = 0; i < nhbins; i++) { memset(tcache->tbins, 0, sizeof(tcache_bin_t) * nhbins);
for (unsigned i = 0; i < nhbins; i++) {
tcache->tbins[i].lg_fill_div = 1; tcache->tbins[i].lg_fill_div = 1;
stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
/* /*
@ -362,15 +356,90 @@ tcache_create(tsdn_t *tsdn, arena_t *arena) {
* access the slots toward higher addresses (for the benefit of * access the slots toward higher addresses (for the benefit of
* prefetch). * prefetch).
*/ */
tcache->tbins[i].avail = (void **)((uintptr_t)tcache + tcache->tbins[i].avail = (void **)((uintptr_t)avail_stack +
(uintptr_t)stack_offset); (uintptr_t)stack_offset);
} }
assert(stack_offset == stack_nelms * sizeof(void *));
}
/* Initialize auto tcache (embedded in TSD). */
bool
tsd_tcache_data_init(tsd_t *tsd) {
if (!config_tcache) {
return false;
}
tcache_t *tcache = &tsd->tcache;
assert(tcache->tbins[0].avail == NULL);
size_t size = stack_nelms * sizeof(void *);
/* Avoid false cacheline sharing. */
size = sa2u(size, CACHELINE);
/* Manually initialize rcache as we may need it for allocation. */
tsd_rtree_ctx_data_init(tsd);
void *avail_array = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true,
NULL, true, arena_get(TSDN_NULL, 0, true));
if (avail_array == NULL) {
return true;
}
tcache_init(tsd, tcache, avail_array);
/*
* Initialization is a bit tricky here. After malloc init is done, all
* threads can rely on arena_choose and associate tcache accordingly.
* However, the thread that does actual malloc bootstrapping relies on
* functional tsd, and it can only rely on a0. In that case, we
* associate its tcache to a0 temporarily, and later on
* arena_choose_hard() will re-associate properly.
*/
tcache->arena = NULL;
arena_t *arena;
if (!malloc_initialized()) {
/* If in initialization, assign to a0. */
arena = arena_get(tsd_tsdn(tsd), 0, false);
tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
} else {
arena = arena_choose(tsd, NULL);
/* This may happen if thread.tcache.enabled is used. */
if (tcache->arena == NULL) {
tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
}
}
assert(arena == tcache->arena);
return false;
}
/* Created manual tcache for tcache.create mallctl. */
tcache_t *
tcache_create_explicit(tsd_t *tsd) {
tcache_t *tcache;
size_t size, stack_offset;
size = sizeof(tcache_t);
/* Naturally align the pointer stacks. */
size = PTR_CEILING(size);
stack_offset = size;
size += stack_nelms * sizeof(void *);
/* Avoid false cacheline sharing. */
size = sa2u(size, CACHELINE);
tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true,
arena_get(TSDN_NULL, 0, true));
if (tcache == NULL) {
return NULL;
}
tcache_init(tsd, tcache,
(void *)((uintptr_t)tcache + (uintptr_t)stack_offset));
tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL));
return tcache; return tcache;
} }
static void static void
tcache_destroy(tsd_t *tsd, tcache_t *tcache) { tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
unsigned i; unsigned i;
for (i = 0; i < NBINS; i++) { for (i = 0; i < NBINS; i++) {
@ -381,7 +450,6 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) {
assert(tbin->tstats.nrequests == 0); assert(tbin->tstats.nrequests == 0);
} }
} }
for (; i < nhbins; i++) { for (; i < nhbins; i++) {
tcache_bin_t *tbin = &tcache->tbins[i]; tcache_bin_t *tbin = &tcache->tbins[i];
tcache_bin_flush_large(tsd, tbin, i, 0, tcache); tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
@ -391,32 +459,60 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) {
} }
} }
/* arena_t *arena = tcache->arena;
* Get arena after flushing -- when using percpu arena, the associated if (config_prof && arena && tcache->prof_accumbytes > 0 &&
* arena could change during flush.
*/
arena_t *arena = arena_choose(tsd, NULL);
tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
if (config_prof && tcache->prof_accumbytes > 0 &&
arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes)) { arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes)) {
prof_idump(tsd_tsdn(tsd)); prof_idump(tsd_tsdn(tsd));
} }
idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true);
} }
void void
tcache_cleanup(tsd_t *tsd) { tcache_flush(void) {
tcache_t *tcache; tsd_t *tsd;
cassert(config_tcache);
tsd = tsd_fetch();
tcache_flush_cache(tsd, tsd_tcachep_get(tsd));
}
static void
tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
tcache_flush_cache(tsd, tcache);
tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
if (tsd_tcache) {
/* Release the avail array for the TSD embedded auto tcache. */
void *avail_array = (void *)((uintptr_t)tcache->tbins[0].avail -
(uintptr_t)tcache_bin_info[0].ncached_max * sizeof(void *));
idalloctm(tsd_tsdn(tsd), avail_array, NULL, true, true);
} else {
/* Release both the tcache struct and avail array. */
idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true);
}
}
/* For auto tcache (embedded in TSD) only. */
void
tcache_cleanup(tsd_t *tsd) {
if (!config_tcache) { if (!config_tcache) {
return; return;
} }
if ((tcache = tsd_tcache_get(tsd)) != NULL) { tcache_t *tcache = tsd_tcachep_get(tsd);
tcache_destroy(tsd, tcache); if (!tcache_available(tsd)) {
tsd_tcache_set(tsd, NULL); assert(tsd_tcache_enabled_get(tsd) == tcache_enabled_false);
if (config_debug) {
assert(tcache->tbins[0].avail == NULL);
}
return;
}
assert(tsd_tcache_enabled_get(tsd) == tcache_enabled_true);
assert(tcache->tbins[0].avail != NULL);
tcache_destroy(tsd, tcache, true);
if (config_debug) {
tcache->tbins[0].avail = NULL;
} }
} }
@ -481,12 +577,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) {
goto label_return; goto label_return;
} }
arena_t *arena = arena_ichoose(tsd, NULL); tcache_t *tcache = tcache_create_explicit(tsd);
if (unlikely(arena == NULL)) {
err = true;
goto label_return;
}
tcache_t *tcache = tcache_create(tsd_tsdn(tsd), arena);
if (tcache == NULL) { if (tcache == NULL) {
err = true; err = true;
goto label_return; goto label_return;
@ -531,7 +622,7 @@ tcaches_flush(tsd_t *tsd, unsigned ind) {
tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]);
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcache != NULL) { if (tcache != NULL) {
tcache_destroy(tsd, tcache); tcache_destroy(tsd, tcache, false);
} }
} }
@ -544,7 +635,7 @@ tcaches_destroy(tsd_t *tsd, unsigned ind) {
tcaches_avail = elm; tcaches_avail = elm;
malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcache != NULL) { if (tcache != NULL) {
tcache_destroy(tsd, tcache); tcache_destroy(tsd, tcache, false);
} }
} }

5
src/tsd.c
View File

@ -21,11 +21,6 @@ malloc_tsd_dalloc(void *wrapper) {
a0dalloc(wrapper); a0dalloc(wrapper);
} }
void
malloc_tsd_no_cleanup(void *arg) {
not_reached();
}
#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
#ifndef _WIN32 #ifndef _WIN32
JEMALLOC_EXPORT JEMALLOC_EXPORT

6
test/unit/tsd.c
View File

@ -5,6 +5,7 @@
typedef unsigned int data_t; typedef unsigned int data_t;
static bool data_cleanup_executed; static bool data_cleanup_executed;
static bool data_test_started;
malloc_tsd_types(data_, data_t) malloc_tsd_types(data_, data_t)
malloc_tsd_protos(, data_, data_t) malloc_tsd_protos(, data_, data_t)
@ -13,6 +14,9 @@ void
data_cleanup(void *arg) { data_cleanup(void *arg) {
data_t *data = (data_t *)arg; data_t *data = (data_t *)arg;
if (!data_test_started) {
return;
}
if (!data_cleanup_executed) { if (!data_cleanup_executed) {
assert_x_eq(*data, THREAD_DATA, assert_x_eq(*data, THREAD_DATA,
"Argument passed into cleanup function should match tsd " "Argument passed into cleanup function should match tsd "
@ -135,7 +139,9 @@ main(void) {
malloc_printf("Initialization error"); malloc_printf("Initialization error");
return test_status_fail; return test_status_fail;
} }
data_test_started = false;
data_tsd_boot(); data_tsd_boot();
data_test_started = true;
return test( return test(
test_tsd_main_thread, test_tsd_main_thread,