From d1e11d48d4c706e17ef3508e2ddb910f109b779f Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Thu, 21 Jun 2018 13:02:49 -0700 Subject: [PATCH] Move tsd link and in_hook after tcache. This can lead to better cache utilization down the common paths where we don't touch the link. --- include/jemalloc/internal/tcache_structs.h | 9 +++++++++ include/jemalloc/internal/tsd.h | 7 ------- src/hook.c | 6 +++--- src/tsd.c | 10 +++++----- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index 07b73870..b3cd4e5f 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -6,6 +6,10 @@ #include "jemalloc/internal/cache_bin.h" #include "jemalloc/internal/ticker.h" +/* Various uses of this struct need it to be a named type. */ +typedef struct tsd_s tsd_t; +typedef ql_elm(tsd_t) tsd_link_t; + struct tcache_s { /* * To minimize our cache-footprint, we put the frequently accessed data @@ -29,6 +33,11 @@ struct tcache_s { */ /* Lets us track all the tcaches in an arena. */ ql_elm(tcache_t) link; + + /* Logically scoped to tsd, but put here for cache layout reasons. */ + ql_elm(tsd_t) tsd_link; + bool in_hook; + /* * The descriptor lets the arena find our cache bins without seeing the * tcache definition. This enables arenas to aggregate stats across diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 3097ce06..e5e82f42 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -59,14 +59,10 @@ typedef void (*test_callback_t)(int *); # define MALLOC_TEST_TSD_INITIALIZER #endif -/* Various uses of this struct need it to be a named type. */ -typedef ql_elm(tsd_t) tsd_link_t; - /* O(name, type, nullable type */ #define MALLOC_TSD \ O(tcache_enabled, bool, bool) \ O(arenas_tdata_bypass, bool, bool) \ - O(in_hook, bool, bool) \ O(reentrancy_level, int8_t, int8_t) \ O(narenas_tdata, uint32_t, uint32_t) \ O(offset_state, uint64_t, uint64_t) \ @@ -77,7 +73,6 @@ typedef ql_elm(tsd_t) tsd_link_t; O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ - O(link, tsd_link_t, tsd_link_t) \ O(tcache, tcache_t, tcache_t) \ O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ MALLOC_TEST_TSD @@ -86,7 +81,6 @@ typedef ql_elm(tsd_t) tsd_link_t; ATOMIC_INIT(tsd_state_uninitialized), \ TCACHE_ENABLED_ZERO_INITIALIZER, \ false, \ - false, \ 0, \ 0, \ 0, \ @@ -97,7 +91,6 @@ typedef ql_elm(tsd_t) tsd_link_t; NULL, \ NULL, \ NULL, \ - {NULL}, \ TCACHE_ZERO_INITIALIZER, \ WITNESS_TSD_INITIALIZER \ MALLOC_TEST_TSD_INITIALIZER \ diff --git a/src/hook.c b/src/hook.c index f66d4239..9ac703cf 100644 --- a/src/hook.c +++ b/src/hook.c @@ -130,9 +130,9 @@ hook_reentrantp() { */ static bool in_hook_global = true; tsdn_t *tsdn = tsdn_fetch(); - bool *in_hook = tsdn_in_hookp_get(tsdn); - if (in_hook != NULL) { - return in_hook; + tcache_t *tcache = tsdn_tcachep_get(tsdn); + if (tcache != NULL) { + return &tcache->in_hook; } return &in_hook_global; } diff --git a/src/tsd.c b/src/tsd.c index 91a964ac..4eceee79 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -68,7 +68,7 @@ tsd_in_nominal_list(tsd_t *tsd) { * out of it here. */ malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); - ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(link)) { + ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { if (tsd == tsd_list) { found = true; break; @@ -82,9 +82,9 @@ static void tsd_add_nominal(tsd_t *tsd) { assert(!tsd_in_nominal_list(tsd)); assert(tsd_state_get(tsd) <= tsd_state_nominal_max); - ql_elm_new(tsd, TSD_MANGLE(link)); + ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link); malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(link)); + ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); } @@ -93,7 +93,7 @@ tsd_remove_nominal(tsd_t *tsd) { assert(tsd_in_nominal_list(tsd)); assert(tsd_state_get(tsd) <= tsd_state_nominal_max); malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(link)); + ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); } @@ -106,7 +106,7 @@ tsd_force_recompute(tsdn_t *tsdn) { atomic_fence(ATOMIC_RELEASE); malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); tsd_t *remote_tsd; - ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(link)) { + ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { assert(atomic_load_u8(&remote_tsd->state, ATOMIC_RELAXED) <= tsd_state_nominal_max); atomic_store_u8(&remote_tsd->state, tsd_state_nominal_recompute,