From 40e7aed59ea1ec8edbeabee71c288afdc2316d72 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 7 Apr 2020 16:59:57 -0700 Subject: [PATCH] TSD: Move in some of the tcache fields. We had put these in the tcache for cache optimization reasons. After the previous diff, these no longer apply. --- include/jemalloc/internal/tcache_structs.h | 7 ------- include/jemalloc/internal/tsd.h | 10 ++++++++-- src/hook.c | 6 +++--- src/tsd.c | 10 +++++----- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index 48dbf0fe..783b2dfd 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -7,9 +7,6 @@ #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/tsd_types.h" -/* Various uses of this struct need it to be a named type. */ -typedef ql_elm(tsd_t) tsd_link_t; - struct tcache_s { /* * To minimize our cache-footprint, we put the frequently accessed data @@ -30,10 +27,6 @@ struct tcache_s { /* Lets us track all the tcaches in an arena. */ ql_elm(tcache_t) link; - /* Logically scoped to tsd, but put here for cache layout reasons. */ - ql_elm(tsd_t) tsd_link; - bool in_hook; - /* * The descriptor lets the arena find our cache bins without seeing the * tcache definition. This enables arenas to aggregate stats across diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 7e08f6b2..66f27173 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -51,6 +51,8 @@ typedef void (*test_callback_t)(int *); # define MALLOC_TEST_TSD_INITIALIZER #endif +typedef ql_elm(tsd_t) tsd_link_t; + /* O(name, type, nullable type) */ #define TSD_DATA_SLOW \ O(tcache_enabled, bool, bool) \ @@ -72,7 +74,9 @@ typedef void (*test_callback_t)(int *); O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ - O(binshards, tsd_binshards_t, tsd_binshards_t) + O(binshards, tsd_binshards_t, tsd_binshards_t)\ + O(tsd_link, tsd_link_t, tsd_link_t) \ + O(in_hook, bool, bool) #define TSD_DATA_SLOW_INITIALIZER \ /* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \ @@ -94,7 +98,9 @@ typedef void (*test_callback_t)(int *); /* iarena */ NULL, \ /* arena */ NULL, \ /* arenas_tdata */ NULL, \ - /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, + /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ + /* tsd_link */ {NULL}, \ + /* in_hook */ false, /* O(name, type, nullable type) */ #define TSD_DATA_FAST \ diff --git a/src/hook.c b/src/hook.c index 9ac703cf..493edbbe 100644 --- a/src/hook.c +++ b/src/hook.c @@ -130,9 +130,9 @@ hook_reentrantp() { */ static bool in_hook_global = true; tsdn_t *tsdn = tsdn_fetch(); - tcache_t *tcache = tsdn_tcachep_get(tsdn); - if (tcache != NULL) { - return &tcache->in_hook; + bool *in_hook = tsdn_in_hookp_get(tsdn); + if (in_hook!= NULL) { + return in_hook; } return &in_hook_global; } diff --git a/src/tsd.c b/src/tsd.c index 38196c80..c07a4bff 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -74,7 +74,7 @@ tsd_in_nominal_list(tsd_t *tsd) { * out of it here. */ malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); - ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { if (tsd == tsd_list) { found = true; break; @@ -88,9 +88,9 @@ static void tsd_add_nominal(tsd_t *tsd) { assert(!tsd_in_nominal_list(tsd)); assert(tsd_state_get(tsd) <= tsd_state_nominal_max); - ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link); + ql_elm_new(tsd, TSD_MANGLE(tsd_link)); malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tsd_link)); malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); } @@ -99,7 +99,7 @@ tsd_remove_nominal(tsd_t *tsd) { assert(tsd_in_nominal_list(tsd)); assert(tsd_state_get(tsd) <= tsd_state_nominal_max); malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tsd_link)); malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); } @@ -112,7 +112,7 @@ tsd_force_recompute(tsdn_t *tsdn) { atomic_fence(ATOMIC_RELEASE); malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); tsd_t *remote_tsd; - ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tsd_link)) { assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED) <= tsd_state_nominal_max); tsd_atomic_store(&remote_tsd->state,