diff --git a/include/jemalloc/internal/arena_inlines_a.h b/include/jemalloc/internal/arena_inlines_a.h index e1c47652..cf92342b 100644 --- a/include/jemalloc/internal/arena_inlines_a.h +++ b/include/jemalloc/internal/arena_inlines_a.h @@ -57,12 +57,10 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) { /* Set new arena/tcache associations. */ arena_migrate(tsd, oldind, newind); - if (config_tcache) { - tcache_t *tcache = tsd_tcache_get(tsd); - if (tcache) { - tcache_arena_reassociate(tsd_tsdn(tsd), tcache, - newarena); - } + tcache_t *tcache = tcache_get(tsd); + if (config_tcache && tcache) { + tcache_arena_reassociate(tsd_tsdn(tsd), tcache, + newarena); } } } diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 04f91c07..449a4ab8 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -488,23 +488,24 @@ extern size_t const index2size_tab[NSIZES]; */ extern uint8_t const size2index_tab[]; -void *a0malloc(size_t size); -void a0dalloc(void *ptr); -void *bootstrap_malloc(size_t size); -void *bootstrap_calloc(size_t num, size_t size); -void bootstrap_free(void *ptr); -void arena_set(unsigned ind, arena_t *arena); -unsigned narenas_total_get(void); -arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind); -arena_t *arena_choose_hard(tsd_t *tsd, bool internal); -void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); -void iarena_cleanup(tsd_t *tsd); -void arena_cleanup(tsd_t *tsd); -void arenas_tdata_cleanup(tsd_t *tsd); -void jemalloc_prefork(void); -void jemalloc_postfork_parent(void); -void jemalloc_postfork_child(void); +void *a0malloc(size_t size); +void a0dalloc(void *ptr); +void *bootstrap_malloc(size_t size); +void *bootstrap_calloc(size_t num, size_t size); +void bootstrap_free(void *ptr); +void arena_set(unsigned ind, arena_t *arena); +unsigned narenas_total_get(void); +arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); +arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind); +arena_t *arena_choose_hard(tsd_t *tsd, bool internal); +void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); +void iarena_cleanup(tsd_t *tsd); +void arena_cleanup(tsd_t *tsd); +void arenas_tdata_cleanup(tsd_t *tsd); +void jemalloc_prefork(void); +void jemalloc_postfork_parent(void); +void jemalloc_postfork_child(void); +bool malloc_initialized(void); #include "jemalloc/internal/nstime_externs.h" #include "jemalloc/internal/ckh_externs.h" @@ -559,6 +560,8 @@ arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing); arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); +bool tcache_available(tsd_t *tsd); +tcache_t *tcache_get(tsd_t *tsd); malloc_cpuid_t malloc_getcpu(void); unsigned percpu_arena_choose(void); unsigned percpu_arena_ind_limit(void); @@ -929,6 +932,38 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) { } return &tdata->decay_ticker; } + +JEMALLOC_ALWAYS_INLINE bool +tcache_available(tsd_t *tsd) { + cassert(config_tcache); + + /* + * Thread specific auto tcache might be unavailable if: 1) during tcache + * initialization, or 2) disabled through thread.tcache.enabled mallctl + * or config options. This check covers all cases. + */ + if (likely(tsd_tcache_enabled_get(tsd) == tcache_enabled_true)) { + /* Associated arena == null implies tcache init in progress. */ + if (tsd_tcachep_get(tsd)->arena != NULL) { + assert(tsd_tcachep_get(tsd)->tbins[0].avail != NULL); + } + return true; + } + + return false; +} + +JEMALLOC_ALWAYS_INLINE tcache_t * +tcache_get(tsd_t *tsd) { + if (!config_tcache) { + return NULL; + } + if (!tcache_available(tsd)) { + return NULL; + } + + return tsd_tcachep_get(tsd); +} #endif #include "jemalloc/internal/rtree_inlines.h" @@ -959,9 +994,24 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd); if (unlikely(ret == NULL)) { ret = arena_choose_hard(tsd, internal); + assert(ret); + if (config_tcache && tcache_available(tsd)) { + tcache_t *tcache = tcache_get(tsd); + if (tcache->arena != NULL) { + /* See comments in tcache_data_init().*/ + assert(tcache->arena == + arena_get(tsd_tsdn(tsd), 0, false)); + if (tcache->arena != ret) { + tcache_arena_reassociate(tsd_tsdn(tsd), + tcache, ret); + } + } else { + tcache_arena_associate(tsd_tsdn(tsd), tcache, + ret); + } + } } - assert(ret != NULL); /* * Note that for percpu arena, if the current arena is outside of the * auto percpu arena range, (i.e. thread is assigned to a manually @@ -1069,8 +1119,8 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, JEMALLOC_ALWAYS_INLINE void * ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { - return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd, true), - false, NULL, slow_path); + return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false, + NULL, slow_path); } JEMALLOC_ALWAYS_INLINE void * @@ -1102,7 +1152,7 @@ ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, JEMALLOC_ALWAYS_INLINE void * ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, - tcache_get(tsd, true), false, NULL); + tcache_get(tsd), false, NULL); } JEMALLOC_ALWAYS_INLINE size_t @@ -1127,7 +1177,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal, JEMALLOC_ALWAYS_INLINE void idalloc(tsd_t *tsd, void *ptr) { - idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true); + idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), false, true); } JEMALLOC_ALWAYS_INLINE void @@ -1199,7 +1249,7 @@ JEMALLOC_ALWAYS_INLINE void * iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero) { return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, - tcache_get(tsd, true), NULL); + tcache_get(tsd), NULL); } JEMALLOC_ALWAYS_INLINE bool diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 1cced603..e2bb0592 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -274,6 +274,7 @@ lg_floor lg_prof_sample malloc_cprintf malloc_getcpu +malloc_initialized malloc_mutex_prof_data_reset malloc_mutex_assert_not_owner malloc_mutex_assert_owner @@ -293,7 +294,6 @@ malloc_tsd_boot1 malloc_tsd_cleanup_register malloc_tsd_dalloc malloc_tsd_malloc -malloc_tsd_no_cleanup malloc_vcprintf malloc_vsnprintf malloc_write @@ -475,22 +475,23 @@ tcache_alloc_easy tcache_alloc_large tcache_alloc_small tcache_alloc_small_hard +tcache_arena_associate tcache_arena_reassociate tcache_bin_flush_large tcache_bin_flush_small tcache_bin_info tcache_boot tcache_cleanup -tcache_create +tcache_create_explicit tcache_dalloc_large tcache_dalloc_small +tcache_data_init tcache_enabled_get tcache_enabled_set tcache_event tcache_event_hard tcache_flush tcache_get -tcache_get_hard tcache_maxclass tcache_prefork tcache_postfork_child @@ -521,7 +522,6 @@ tsd_booted tsd_booted_get tsd_cleanup tsd_cleanup_wrapper -tsd_data_init tsd_fetch tsd_fetch_impl tsd_get diff --git a/include/jemalloc/internal/rtree_inlines.h b/include/jemalloc/internal/rtree_inlines.h index bebe49e0..ce03c578 100644 --- a/include/jemalloc/internal/rtree_inlines.h +++ b/include/jemalloc/internal/rtree_inlines.h @@ -321,13 +321,18 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, assert(!dependent || !init_missing); uintptr_t leafkey = rtree_leafkey(key); + assert(leafkey != RTREE_LEAFKEY_INVALID); + #define RTREE_CACHE_CHECK(i) do { \ if (likely(rtree_ctx->cache[i].leafkey == leafkey)) { \ rtree_leaf_elm_t *leaf = rtree_ctx->cache[i].leaf; \ assert(leaf != NULL); \ if (i > 0) { \ /* Bubble up by one. */ \ - rtree_ctx->cache[i] = rtree_ctx->cache[i - 1]; \ + rtree_ctx->cache[i].leafkey = \ + rtree_ctx->cache[i - 1].leafkey; \ + rtree_ctx->cache[i].leaf = \ + rtree_ctx->cache[i - 1].leaf; \ rtree_ctx->cache[i - 1].leafkey = leafkey; \ rtree_ctx->cache[i - 1].leaf = leaf; \ } \ diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 83643033..75ff3214 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -35,16 +35,19 @@ void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); -tcache_t *tcache_get_hard(tsd_t *tsd); -tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena); +tcache_t *tcache_create_explicit(tsd_t *tsd); void tcache_cleanup(tsd_t *tsd); void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); bool tcaches_create(tsd_t *tsd, unsigned *r_ind); void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind); bool tcache_boot(tsdn_t *tsdn); +void tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); void tcache_prefork(tsdn_t *tsdn); void tcache_postfork_parent(tsdn_t *tsdn); void tcache_postfork_child(tsdn_t *tsdn); +void tcache_flush(void); +bool tsd_tcache_data_init(tsd_t *tsd); +bool tsd_tcache_enabled_data_init(tsd_t *tsd); #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */ diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index fd7e1764..c3660963 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -4,9 +4,9 @@ #ifndef JEMALLOC_ENABLE_INLINE void tcache_event(tsd_t *tsd, tcache_t *tcache); void tcache_flush(void); -bool tcache_enabled_get(void); -tcache_t *tcache_get(tsd_t *tsd, bool create); -void tcache_enabled_set(bool enabled); +bool tcache_enabled_get(tsd_t *tsd); +tcache_t *tcache_get(tsd_t *tsd); +void tcache_enabled_set(tsd_t *tsd, bool enabled); void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success); void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, szind_t ind, bool zero, bool slow_path); @@ -20,68 +20,32 @@ tcache_t *tcaches_get(tsd_t *tsd, unsigned ind); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) -JEMALLOC_INLINE void -tcache_flush(void) { - tsd_t *tsd; - - cassert(config_tcache); - - tsd = tsd_fetch(); - tcache_cleanup(tsd); -} - JEMALLOC_INLINE bool -tcache_enabled_get(void) { - tsd_t *tsd; +tcache_enabled_get(tsd_t *tsd) { tcache_enabled_t tcache_enabled; cassert(config_tcache); - tsd = tsd_fetch(); tcache_enabled = tsd_tcache_enabled_get(tsd); - if (tcache_enabled == tcache_enabled_default) { - tcache_enabled = (tcache_enabled_t)opt_tcache; - tsd_tcache_enabled_set(tsd, tcache_enabled); - } + assert(tcache_enabled != tcache_enabled_default); return (bool)tcache_enabled; } JEMALLOC_INLINE void -tcache_enabled_set(bool enabled) { - tsd_t *tsd; - tcache_enabled_t tcache_enabled; - +tcache_enabled_set(tsd_t *tsd, bool enabled) { cassert(config_tcache); - tsd = tsd_fetch(); + tcache_enabled_t old = tsd_tcache_enabled_get(tsd); - tcache_enabled = (tcache_enabled_t)enabled; - tsd_tcache_enabled_set(tsd, tcache_enabled); - - if (!enabled) { + if ((old != tcache_enabled_true) && enabled) { + tsd_tcache_data_init(tsd); + } else if ((old == tcache_enabled_true) && !enabled) { tcache_cleanup(tsd); } -} - -JEMALLOC_ALWAYS_INLINE tcache_t * -tcache_get(tsd_t *tsd, bool create) { - tcache_t *tcache; - - if (!config_tcache) { - return NULL; - } - - tcache = tsd_tcache_get(tsd); - if (!create) { - return tcache; - } - if (unlikely(tcache == NULL) && tsd_nominal(tsd)) { - tcache = tcache_get_hard(tsd); - tsd_tcache_set(tsd, tcache); - } - - return tcache; + /* Commit the state last. Above calls check current state. */ + tcache_enabled_t tcache_enabled = (tcache_enabled_t)enabled; + tsd_tcache_enabled_set(tsd, tcache_enabled); } JEMALLOC_ALWAYS_INLINE void @@ -300,8 +264,7 @@ JEMALLOC_ALWAYS_INLINE tcache_t * tcaches_get(tsd_t *tsd, unsigned ind) { tcaches_t *elm = &tcaches[ind]; if (unlikely(elm->tcache == NULL)) { - elm->tcache = tcache_create(tsd_tsdn(tsd), arena_choose(tsd, - NULL)); + elm->tcache = tcache_create_explicit(tsd); } return elm->tcache; } diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index a9b70312..c9c05cd2 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -36,13 +36,17 @@ struct tcache_s { ticker_t gc_ticker; /* Drives incremental GC. */ szind_t next_gc_bin; /* Next bin to GC. */ arena_t *arena; /* Associated arena. */ - tcache_bin_t tbins[1]; /* Dynamically sized. */ /* * The pointer stacks associated with tbins follow as a contiguous * array. During tcache initialization, the avail pointer in each * element of tbins is initialized to point to the proper offset within * this array. */ +#ifdef JEMALLOC_TCACHE + tcache_bin_t tbins[NSIZES]; +#else + tcache_bin_t tbins[0]; +#endif }; /* Linkage for list of available (previously used) explicit tcache IDs. */ diff --git a/include/jemalloc/internal/tcache_types.h b/include/jemalloc/internal/tcache_types.h index 2d396bf6..8624ac2f 100644 --- a/include/jemalloc/internal/tcache_types.h +++ b/include/jemalloc/internal/tcache_types.h @@ -47,4 +47,7 @@ typedef struct tcaches_s tcaches_t; #define TCACHE_GC_INCR \ ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) +/* Used in TSD static initializer only. Real init in tcache_data_init(). */ +#define TCACHE_ZERO_INITIALIZER {{NULL}} + #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ diff --git a/include/jemalloc/internal/tsd_externs.h b/include/jemalloc/internal/tsd_externs.h index 9b88a567..d15fd591 100644 --- a/include/jemalloc/internal/tsd_externs.h +++ b/include/jemalloc/internal/tsd_externs.h @@ -3,7 +3,6 @@ void *malloc_tsd_malloc(size_t size); void malloc_tsd_dalloc(void *wrapper); -void malloc_tsd_no_cleanup(void *arg); void malloc_tsd_cleanup_register(bool (*f)(void)); tsd_t *malloc_tsd_boot0(void); void malloc_tsd_boot1(void); @@ -13,7 +12,7 @@ void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block); void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); #endif -void tsd_cleanup(void *arg); -bool tsd_data_init(void *arg); +bool tsd_data_init(void *arg); +void tsd_cleanup(void *arg); #endif /* JEMALLOC_INTERNAL_TSD_EXTERNS_H */ diff --git a/include/jemalloc/internal/tsd_structs.h b/include/jemalloc/internal/tsd_structs.h index b4ac09fd..d399563c 100644 --- a/include/jemalloc/internal/tsd_structs.h +++ b/include/jemalloc/internal/tsd_structs.h @@ -16,7 +16,7 @@ struct tsd_init_head_s { #define MALLOC_TSD \ /* O(name, type, [gs]et, init, cleanup) */ \ - O(tcache, tcache_t *, yes, no, yes) \ + O(tcache, tcache_t, yes, no, yes) \ O(thread_allocated, uint64_t, yes, no, no) \ O(thread_deallocated, uint64_t, yes, no, no) \ O(prof_tdata, prof_tdata_t *, yes, no, yes) \ @@ -26,7 +26,7 @@ struct tsd_init_head_s { O(narenas_tdata, unsigned, yes, no, no) \ O(arenas_tdata_bypass, bool, no, no, no) \ O(tcache_enabled, tcache_enabled_t, \ - yes, no, no) \ + yes, yes, no) \ O(rtree_ctx, rtree_ctx_t, no, yes, no) \ O(witnesses, witness_list_t, no, no, yes) \ O(rtree_leaf_elm_witnesses, rtree_leaf_elm_witness_tsd_t, \ @@ -35,7 +35,7 @@ struct tsd_init_head_s { #define TSD_INITIALIZER { \ tsd_state_uninitialized, \ - NULL, \ + TCACHE_ZERO_INITIALIZER, \ 0, \ 0, \ NULL, \ @@ -69,6 +69,7 @@ struct tsdn_s { }; static const tsd_t tsd_initializer = TSD_INITIALIZER; +UNUSED static const void *malloc_tsd_no_cleanup = (void *)0; malloc_tsd_types(, tsd_t) diff --git a/include/jemalloc/internal/tsd_types.h b/include/jemalloc/internal/tsd_types.h index 195b6493..29c6378a 100644 --- a/include/jemalloc/internal/tsd_types.h +++ b/include/jemalloc/internal/tsd_types.h @@ -357,8 +357,10 @@ a_name##tsd_boot1(void) { \ " TSD for "#a_name"\n"); \ abort(); \ } \ - memcpy(wrapper, &a_name##tsd_boot_wrapper, \ - sizeof(a_name##tsd_wrapper_t)); \ + a_name##tsd_boot_wrapper.initialized = false; \ + a_cleanup(&a_name##tsd_boot_wrapper.val); \ + wrapper->initialized = false; \ + wrapper->val = a_initializer; \ a_name##tsd_wrapper_set(wrapper); \ } \ a_attr bool \ @@ -487,8 +489,10 @@ a_name##tsd_boot1(void) { \ " TSD for "#a_name"\n"); \ abort(); \ } \ - memcpy(wrapper, &a_name##tsd_boot_wrapper, \ - sizeof(a_name##tsd_wrapper_t)); \ + a_name##tsd_boot_wrapper.initialized = false; \ + a_cleanup(&a_name##tsd_boot_wrapper.val); \ + wrapper->initialized = false; \ + wrapper->val = a_initializer; \ a_name##tsd_wrapper_set(wrapper); \ } \ a_attr bool \ diff --git a/src/ctl.c b/src/ctl.c index 36f5634d..a59a741f 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1532,7 +1532,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, /* Set new arena/tcache associations. */ arena_migrate(tsd, oldind, newind); if (config_tcache) { - tcache_t *tcache = tsd_tcache_get(tsd); + tcache_t *tcache = tsd_tcachep_get(tsd); if (tcache != NULL) { tcache_arena_reassociate(tsd_tsdn(tsd), tcache, newarena); @@ -1564,13 +1564,13 @@ thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, return ENOENT; } - oldval = tcache_enabled_get(); + oldval = tcache_enabled_get(tsd); if (newp != NULL) { if (newlen != sizeof(bool)) { ret = EINVAL; goto label_return; } - tcache_enabled_set(*(bool *)newp); + tcache_enabled_set(tsd, *(bool *)newp); } READ(oldval, bool); diff --git a/src/jemalloc.c b/src/jemalloc.c index 94ae030c..9d66f7f6 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -275,7 +275,7 @@ static bool malloc_init_hard(void); * Begin miscellaneous support functions. */ -JEMALLOC_ALWAYS_INLINE_C bool +bool malloc_initialized(void) { return (malloc_init_state == malloc_init_initialized); } @@ -1536,7 +1536,7 @@ imalloc_no_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, /* Fill in the tcache. */ if (dopts->tcache_ind == TCACHE_IND_AUTOMATIC) { - tcache = tcache_get(tsd, true); + tcache = tcache_get(tsd); } else if (dopts->tcache_ind == TCACHE_IND_NONE) { tcache = NULL; } else { @@ -2056,7 +2056,7 @@ je_realloc(void *ptr, size_t size) { /* realloc(ptr, 0) is equivalent to free(ptr). */ UTRACE(ptr, 0, 0); tsd = tsd_fetch(); - ifree(tsd, ptr, tcache_get(tsd, false), true); + ifree(tsd, ptr, tcache_get(tsd), true); return NULL; } size = 1; @@ -2113,9 +2113,9 @@ je_free(void *ptr) { tsd_t *tsd = tsd_fetch(); witness_assert_lockless(tsd_tsdn(tsd)); if (likely(!malloc_slow)) { - ifree(tsd, ptr, tcache_get(tsd, false), false); + ifree(tsd, ptr, tcache_get(tsd), false); } else { - ifree(tsd, ptr, tcache_get(tsd, false), true); + ifree(tsd, ptr, tcache_get(tsd), true); } witness_assert_lockless(tsd_tsdn(tsd)); } @@ -2393,7 +2393,7 @@ je_rallocx(void *ptr, size_t size, int flags) { tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); } } else { - tcache = tcache_get(tsd, true); + tcache = tcache_get(tsd); } old_usize = isalloc(tsd_tsdn(tsd), ptr); @@ -2605,7 +2605,7 @@ je_dallocx(void *ptr, int flags) { tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); } } else { - tcache = tcache_get(tsd, false); + tcache = tcache_get(tsd); } UTRACE(ptr, 0, 0); @@ -2652,7 +2652,7 @@ je_sdallocx(void *ptr, size_t size, int flags) { tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); } } else { - tcache = tcache_get(tsd, false); + tcache = tcache_get(tsd); } UTRACE(ptr, 0, 0); diff --git a/src/tcache.c b/src/tcache.c index 6057c890..aa2917b2 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -4,7 +4,13 @@ /******************************************************************************/ /* Data. */ -bool opt_tcache = true; +bool opt_tcache = +#ifdef JEMALLOC_TCACHE + true +#else + false +#endif + ; ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; tcache_bin_info_t *tcache_bin_info; @@ -78,6 +84,7 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, tcache_bin_t *tbin, szind_t binind, bool *tcache_success) { void *ret; + assert(tcache->arena); arena_tcache_fill_small(tsdn, arena, tbin, binind, config_prof ? tcache->prof_accumbytes : 0); if (config_prof) { @@ -271,9 +278,11 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, } } -static void +void tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { + assert(tcache->arena == NULL); tcache->arena = arena; + if (config_stats) { /* Link into list of extant tcaches. */ malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); @@ -286,6 +295,7 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { static void tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { arena_t *arena = tcache->arena; + assert(arena); if (config_stats) { /* Unlink from list of extant tcaches. */ malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); @@ -304,6 +314,7 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { tcache_stats_merge(tsdn, tcache, arena); malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } + tcache->arena = NULL; } void @@ -312,49 +323,32 @@ tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { tcache_arena_associate(tsdn, tcache, arena); } -tcache_t * -tcache_get_hard(tsd_t *tsd) { - arena_t *arena; +bool +tsd_tcache_enabled_data_init(tsd_t *tsd) { + /* Called upon tsd initialization. */ + tsd_tcache_enabled_set(tsd, (tcache_enabled_t)opt_tcache); + if (opt_tcache) { + /* Trigger tcache init. */ + tsd_tcache_data_init(tsd); + } - if (!tcache_enabled_get()) { - if (tsd_nominal(tsd)) { - tcache_enabled_set(false); /* Memoize. */ - } - return NULL; - } - arena = arena_choose(tsd, NULL); - if (unlikely(arena == NULL)) { - return NULL; - } - return tcache_create(tsd_tsdn(tsd), arena); + return false; } -tcache_t * -tcache_create(tsdn_t *tsdn, arena_t *arena) { - tcache_t *tcache; - size_t size, stack_offset; - unsigned i; - - size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins); - /* Naturally align the pointer stacks. */ - size = PTR_CEILING(size); - stack_offset = size; - size += stack_nelms * sizeof(void *); - /* Avoid false cacheline sharing. */ - size = sa2u(size, CACHELINE); - - tcache = ipallocztm(tsdn, size, CACHELINE, true, NULL, true, - arena_get(TSDN_NULL, 0, true)); - if (tcache == NULL) { - return NULL; - } - - tcache_arena_associate(tsdn, tcache, arena); +/* Initialize auto tcache (embedded in TSD). */ +static void +tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { + memset(&tcache->link, 0, sizeof(ql_elm(tcache_t))); + tcache->prof_accumbytes = 0; + tcache->next_gc_bin = 0; + tcache->arena = NULL; ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR); + size_t stack_offset = 0; assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); - for (i = 0; i < nhbins; i++) { + memset(tcache->tbins, 0, sizeof(tcache_bin_t) * nhbins); + for (unsigned i = 0; i < nhbins; i++) { tcache->tbins[i].lg_fill_div = 1; stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); /* @@ -362,15 +356,90 @@ tcache_create(tsdn_t *tsdn, arena_t *arena) { * access the slots toward higher addresses (for the benefit of * prefetch). */ - tcache->tbins[i].avail = (void **)((uintptr_t)tcache + + tcache->tbins[i].avail = (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset); } + assert(stack_offset == stack_nelms * sizeof(void *)); +} + +/* Initialize auto tcache (embedded in TSD). */ +bool +tsd_tcache_data_init(tsd_t *tsd) { + if (!config_tcache) { + return false; + } + + tcache_t *tcache = &tsd->tcache; + assert(tcache->tbins[0].avail == NULL); + size_t size = stack_nelms * sizeof(void *); + /* Avoid false cacheline sharing. */ + size = sa2u(size, CACHELINE); + + /* Manually initialize rcache as we may need it for allocation. */ + tsd_rtree_ctx_data_init(tsd); + + void *avail_array = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, + NULL, true, arena_get(TSDN_NULL, 0, true)); + if (avail_array == NULL) { + return true; + } + + tcache_init(tsd, tcache, avail_array); + /* + * Initialization is a bit tricky here. After malloc init is done, all + * threads can rely on arena_choose and associate tcache accordingly. + * However, the thread that does actual malloc bootstrapping relies on + * functional tsd, and it can only rely on a0. In that case, we + * associate its tcache to a0 temporarily, and later on + * arena_choose_hard() will re-associate properly. + */ + tcache->arena = NULL; + arena_t *arena; + if (!malloc_initialized()) { + /* If in initialization, assign to a0. */ + arena = arena_get(tsd_tsdn(tsd), 0, false); + tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); + } else { + arena = arena_choose(tsd, NULL); + /* This may happen if thread.tcache.enabled is used. */ + if (tcache->arena == NULL) { + tcache_arena_associate(tsd_tsdn(tsd), tcache, arena); + } + } + assert(arena == tcache->arena); + + return false; +} + +/* Created manual tcache for tcache.create mallctl. */ +tcache_t * +tcache_create_explicit(tsd_t *tsd) { + tcache_t *tcache; + size_t size, stack_offset; + + size = sizeof(tcache_t); + /* Naturally align the pointer stacks. */ + size = PTR_CEILING(size); + stack_offset = size; + size += stack_nelms * sizeof(void *); + /* Avoid false cacheline sharing. */ + size = sa2u(size, CACHELINE); + + tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true, + arena_get(TSDN_NULL, 0, true)); + if (tcache == NULL) { + return NULL; + } + + tcache_init(tsd, tcache, + (void *)((uintptr_t)tcache + (uintptr_t)stack_offset)); + tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL)); return tcache; } static void -tcache_destroy(tsd_t *tsd, tcache_t *tcache) { +tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { unsigned i; for (i = 0; i < NBINS; i++) { @@ -381,7 +450,6 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) { assert(tbin->tstats.nrequests == 0); } } - for (; i < nhbins; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; tcache_bin_flush_large(tsd, tbin, i, 0, tcache); @@ -391,32 +459,60 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) { } } - /* - * Get arena after flushing -- when using percpu arena, the associated - * arena could change during flush. - */ - arena_t *arena = arena_choose(tsd, NULL); - tcache_arena_dissociate(tsd_tsdn(tsd), tcache); - - if (config_prof && tcache->prof_accumbytes > 0 && + arena_t *arena = tcache->arena; + if (config_prof && arena && tcache->prof_accumbytes > 0 && arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes)) { prof_idump(tsd_tsdn(tsd)); } - - idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true); } void -tcache_cleanup(tsd_t *tsd) { - tcache_t *tcache; +tcache_flush(void) { + tsd_t *tsd; + cassert(config_tcache); + + tsd = tsd_fetch(); + tcache_flush_cache(tsd, tsd_tcachep_get(tsd)); +} + +static void +tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { + tcache_flush_cache(tsd, tcache); + tcache_arena_dissociate(tsd_tsdn(tsd), tcache); + + if (tsd_tcache) { + /* Release the avail array for the TSD embedded auto tcache. */ + void *avail_array = (void *)((uintptr_t)tcache->tbins[0].avail - + (uintptr_t)tcache_bin_info[0].ncached_max * sizeof(void *)); + idalloctm(tsd_tsdn(tsd), avail_array, NULL, true, true); + } else { + /* Release both the tcache struct and avail array. */ + idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true); + } +} + +/* For auto tcache (embedded in TSD) only. */ +void +tcache_cleanup(tsd_t *tsd) { if (!config_tcache) { return; } - if ((tcache = tsd_tcache_get(tsd)) != NULL) { - tcache_destroy(tsd, tcache); - tsd_tcache_set(tsd, NULL); + tcache_t *tcache = tsd_tcachep_get(tsd); + if (!tcache_available(tsd)) { + assert(tsd_tcache_enabled_get(tsd) == tcache_enabled_false); + if (config_debug) { + assert(tcache->tbins[0].avail == NULL); + } + return; + } + assert(tsd_tcache_enabled_get(tsd) == tcache_enabled_true); + assert(tcache->tbins[0].avail != NULL); + + tcache_destroy(tsd, tcache, true); + if (config_debug) { + tcache->tbins[0].avail = NULL; } } @@ -481,12 +577,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) { goto label_return; } - arena_t *arena = arena_ichoose(tsd, NULL); - if (unlikely(arena == NULL)) { - err = true; - goto label_return; - } - tcache_t *tcache = tcache_create(tsd_tsdn(tsd), arena); + tcache_t *tcache = tcache_create_explicit(tsd); if (tcache == NULL) { err = true; goto label_return; @@ -531,7 +622,7 @@ tcaches_flush(tsd_t *tsd, unsigned ind) { tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); if (tcache != NULL) { - tcache_destroy(tsd, tcache); + tcache_destroy(tsd, tcache, false); } } @@ -544,7 +635,7 @@ tcaches_destroy(tsd_t *tsd, unsigned ind) { tcaches_avail = elm; malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); if (tcache != NULL) { - tcache_destroy(tsd, tcache); + tcache_destroy(tsd, tcache, false); } } diff --git a/src/tsd.c b/src/tsd.c index 86502116..8b54770e 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -21,11 +21,6 @@ malloc_tsd_dalloc(void *wrapper) { a0dalloc(wrapper); } -void -malloc_tsd_no_cleanup(void *arg) { - not_reached(); -} - #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) #ifndef _WIN32 JEMALLOC_EXPORT diff --git a/test/unit/tsd.c b/test/unit/tsd.c index e033bb76..5bfcdf49 100644 --- a/test/unit/tsd.c +++ b/test/unit/tsd.c @@ -5,6 +5,7 @@ typedef unsigned int data_t; static bool data_cleanup_executed; +static bool data_test_started; malloc_tsd_types(data_, data_t) malloc_tsd_protos(, data_, data_t) @@ -13,6 +14,9 @@ void data_cleanup(void *arg) { data_t *data = (data_t *)arg; + if (!data_test_started) { + return; + } if (!data_cleanup_executed) { assert_x_eq(*data, THREAD_DATA, "Argument passed into cleanup function should match tsd " @@ -135,7 +139,9 @@ main(void) { malloc_printf("Initialization error"); return test_status_fail; } + data_test_started = false; data_tsd_boot(); + data_test_started = true; return test( test_tsd_main_thread,