diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 318de11c..13e6eb52 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -116,18 +116,22 @@ arena_prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, JEMALLOC_ALWAYS_INLINE void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { - tsd_t *tsd; - ticker_t *decay_ticker; - if (unlikely(tsdn_null(tsdn))) { return; } - tsd = tsdn_tsd(tsdn); - decay_ticker = decay_ticker_get(tsd, arena_ind_get(arena)); - if (unlikely(decay_ticker == NULL)) { - return; - } - if (unlikely(ticker_ticks(decay_ticker, nticks))) { + tsd_t *tsd = tsdn_tsd(tsdn); + /* + * We use the ticker_geom_t to avoid having per-arena state in the tsd. + * Instead of having a countdown-until-decay timer running for every + * arena in every thread, we flip a coin once per tick, whose + * probability of coming up heads is 1/nticks; this is effectively the + * operation of the ticker_geom_t. Each arena has the same chance of a + * coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can + * use a single ticker for all of them. + */ + ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd); + uint64_t *prng_state = tsd_prng_statep_get(tsd); + if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks))) { arena_decay(tsdn, arena, false, false); } } diff --git a/include/jemalloc/internal/arena_structs.h b/include/jemalloc/internal/arena_structs.h index 4aff63c9..ad76a79a 100644 --- a/include/jemalloc/internal/arena_structs.h +++ b/include/jemalloc/internal/arena_structs.h @@ -98,9 +98,4 @@ struct arena_s { bin_t bins[0]; }; -/* Used in conjunction with tsd for fast arena-related context lookup. */ -struct arena_tdata_s { - ticker_t decay_ticker; -}; - #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H */ diff --git a/include/jemalloc/internal/arena_types.h b/include/jemalloc/internal/arena_types.h index b13d8a05..e0f8218d 100644 --- a/include/jemalloc/internal/arena_types.h +++ b/include/jemalloc/internal/arena_types.h @@ -7,11 +7,10 @@ #define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) #define MUZZY_DECAY_MS_DEFAULT (0) /* Number of event ticks between time checks. */ -#define DECAY_NTICKS_PER_UPDATE 1000 +#define ARENA_DECAY_NTICKS_PER_UPDATE 1000 typedef struct arena_decay_s arena_decay_t; typedef struct arena_s arena_t; -typedef struct arena_tdata_s arena_tdata_t; typedef enum { percpu_arena_mode_names_base = 0, /* Used for options processing. */ diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index 40591b99..c78db06e 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -56,12 +56,10 @@ void bootstrap_free(void *ptr); void arena_set(unsigned ind, arena_t *arena); unsigned narenas_total_get(void); arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind); arena_t *arena_choose_hard(tsd_t *tsd, bool internal); void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); void iarena_cleanup(tsd_t *tsd); void arena_cleanup(tsd_t *tsd); -void arenas_tdata_cleanup(tsd_t *tsd); size_t batch_alloc(void **ptrs, size_t num, size_t size, int flags); void jemalloc_prefork(void); void jemalloc_postfork_parent(void); diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h index 25e5b50e..24e42d38 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -56,31 +56,6 @@ percpu_arena_ind_limit(percpu_arena_mode_t mode) { } } -static inline arena_tdata_t * -arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) { - arena_tdata_t *tdata; - arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); - - if (unlikely(arenas_tdata == NULL)) { - /* arenas_tdata hasn't been initialized yet. */ - return arena_tdata_get_hard(tsd, ind); - } - if (unlikely(ind >= tsd_narenas_tdata_get(tsd))) { - /* - * ind is invalid, cache is old (too small), or tdata to be - * initialized. - */ - return (refresh_if_missing ? arena_tdata_get_hard(tsd, ind) : - NULL); - } - - tdata = &arenas_tdata[ind]; - if (likely(tdata != NULL) || !refresh_if_missing) { - return tdata; - } - return arena_tdata_get_hard(tsd, ind); -} - static inline arena_t * arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) { arena_t *ret; @@ -97,17 +72,6 @@ arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) { return ret; } -static inline ticker_t * -decay_ticker_get(tsd_t *tsd, unsigned ind) { - arena_tdata_t *tdata; - - tdata = arena_tdata_get(tsd, ind, true); - if (unlikely(tdata == NULL)) { - return NULL; - } - return &tdata->decay_ticker; -} - JEMALLOC_ALWAYS_INLINE bool tcache_available(tsd_t *tsd) { /* diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 60764199..d22fdc94 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -58,9 +58,7 @@ typedef ql_elm(tsd_t) tsd_link_t; /* O(name, type, nullable type) */ #define TSD_DATA_SLOW \ O(tcache_enabled, bool, bool) \ - O(arenas_tdata_bypass, bool, bool) \ O(reentrancy_level, int8_t, int8_t) \ - O(narenas_tdata, uint32_t, uint32_t) \ O(thread_allocated_last_event, uint64_t, uint64_t) \ O(thread_allocated_next_event, uint64_t, uint64_t) \ O(thread_deallocated_last_event, uint64_t, uint64_t) \ @@ -77,7 +75,7 @@ typedef ql_elm(tsd_t) tsd_link_t; O(prng_state, uint64_t, uint64_t) \ O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ - O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ + O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ O(sec_shard, uint8_t, uint8_t) \ O(binshards, tsd_binshards_t, tsd_binshards_t)\ O(tsd_link, tsd_link_t, tsd_link_t) \ @@ -90,9 +88,7 @@ typedef ql_elm(tsd_t) tsd_link_t; #define TSD_DATA_SLOW_INITIALIZER \ /* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \ - /* arenas_tdata_bypass */ false, \ /* reentrancy_level */ 0, \ - /* narenas_tdata */ 0, \ /* thread_allocated_last_event */ 0, \ /* thread_allocated_next_event */ 0, \ /* thread_deallocated_last_event */ 0, \ @@ -109,7 +105,8 @@ typedef ql_elm(tsd_t) tsd_link_t; /* prng_state */ 0, \ /* iarena */ NULL, \ /* arena */ NULL, \ - /* arenas_tdata */ NULL, \ + /* arena_decay_ticker */ \ + TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \ /* sec_shard */ (uint8_t)-1, \ /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ /* tsd_link */ {NULL}, \ diff --git a/src/jemalloc.c b/src/jemalloc.c index f7c3963d..ca8a7deb 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -493,82 +493,6 @@ arena_unbind(tsd_t *tsd, unsigned ind, bool internal) { } } -arena_tdata_t * -arena_tdata_get_hard(tsd_t *tsd, unsigned ind) { - arena_tdata_t *tdata, *arenas_tdata_old; - arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); - unsigned narenas_tdata_old, i; - unsigned narenas_tdata = tsd_narenas_tdata_get(tsd); - unsigned narenas_actual = narenas_total_get(); - - /* - * Dissociate old tdata array (and set up for deallocation upon return) - * if it's too small. - */ - if (arenas_tdata != NULL && narenas_tdata < narenas_actual) { - arenas_tdata_old = arenas_tdata; - narenas_tdata_old = narenas_tdata; - arenas_tdata = NULL; - narenas_tdata = 0; - tsd_arenas_tdata_set(tsd, arenas_tdata); - tsd_narenas_tdata_set(tsd, narenas_tdata); - } else { - arenas_tdata_old = NULL; - narenas_tdata_old = 0; - } - - /* Allocate tdata array if it's missing. */ - if (arenas_tdata == NULL) { - bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd); - narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1; - - if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) { - *arenas_tdata_bypassp = true; - arenas_tdata = (arena_tdata_t *)a0malloc( - sizeof(arena_tdata_t) * narenas_tdata); - *arenas_tdata_bypassp = false; - } - if (arenas_tdata == NULL) { - tdata = NULL; - goto label_return; - } - assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp); - tsd_arenas_tdata_set(tsd, arenas_tdata); - tsd_narenas_tdata_set(tsd, narenas_tdata); - } - - /* - * Copy to tdata array. It's possible that the actual number of arenas - * has increased since narenas_total_get() was called above, but that - * causes no correctness issues unless two threads concurrently execute - * the arenas.create mallctl, which we trust mallctl synchronization to - * prevent. - */ - - /* Copy/initialize tickers. */ - for (i = 0; i < narenas_actual; i++) { - if (i < narenas_tdata_old) { - ticker_copy(&arenas_tdata[i].decay_ticker, - &arenas_tdata_old[i].decay_ticker); - } else { - ticker_init(&arenas_tdata[i].decay_ticker, - DECAY_NTICKS_PER_UPDATE); - } - } - if (narenas_tdata > narenas_actual) { - memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t) - * (narenas_tdata - narenas_actual)); - } - - /* Read the refreshed tdata array. */ - tdata = &arenas_tdata[ind]; -label_return: - if (arenas_tdata_old != NULL) { - a0dalloc(arenas_tdata_old); - } - return tdata; -} - /* Slow path, called only by arena_choose(). */ arena_t * arena_choose_hard(tsd_t *tsd, bool internal) { @@ -705,20 +629,6 @@ arena_cleanup(tsd_t *tsd) { } } -void -arenas_tdata_cleanup(tsd_t *tsd) { - arena_tdata_t *arenas_tdata; - - /* Prevent tsd->arenas_tdata from being (re)created. */ - *tsd_arenas_tdata_bypassp_get(tsd) = true; - - arenas_tdata = tsd_arenas_tdata_get(tsd); - if (arenas_tdata != NULL) { - tsd_arenas_tdata_set(tsd, NULL); - a0dalloc(arenas_tdata); - } -} - static void stats_print_atexit(void) { if (config_stats) { diff --git a/src/tsd.c b/src/tsd.c index 0dd4036b..6820eb62 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -251,8 +251,6 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) { assert(!tsd_in_nominal_list(tsd)); assert(*tsd_arenap_get_unsafe(tsd) == NULL); assert(*tsd_iarenap_get_unsafe(tsd) == NULL); - assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true); - assert(*tsd_arenas_tdatap_get_unsafe(tsd) == NULL); assert(*tsd_tcache_enabledp_get_unsafe(tsd) == false); assert(*tsd_prof_tdatap_get_unsafe(tsd) == NULL); } @@ -267,7 +265,6 @@ tsd_data_init_nocleanup(tsd_t *tsd) { * We set up tsd in a way that no cleanup is needed. */ rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); - *tsd_arenas_tdata_bypassp_get(tsd) = true; *tsd_tcache_enabledp_get_unsafe(tsd) = false; *tsd_reentrancy_levelp_get(tsd) = 1; tsd_prng_state_init(tsd); @@ -375,7 +372,6 @@ tsd_do_data_cleanup(tsd_t *tsd) { prof_tdata_cleanup(tsd); iarena_cleanup(tsd); arena_cleanup(tsd); - arenas_tdata_cleanup(tsd); tcache_cleanup(tsd); witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd)); *tsd_reentrancy_levelp_get(tsd) = 1; @@ -439,7 +435,6 @@ malloc_tsd_boot0(void) { return NULL; } tsd = tsd_fetch(); - *tsd_arenas_tdata_bypassp_get(tsd) = true; return tsd; } @@ -449,7 +444,6 @@ malloc_tsd_boot1(void) { tsd_t *tsd = tsd_fetch(); /* malloc_slow has been set properly. Update tsd_slow. */ tsd_slow_update(tsd); - *tsd_arenas_tdata_bypassp_get(tsd) = false; } #ifdef _WIN32 diff --git a/test/unit/arena_decay.c b/test/unit/arena_decay.c index a2661682..cea39e09 100644 --- a/test/unit/arena_decay.c +++ b/test/unit/arena_decay.c @@ -187,7 +187,7 @@ TEST_BEGIN(test_decay_ticks) { test_skip_if(check_background_thread_enabled()); test_skip_if(opt_hpa); - ticker_t *decay_ticker; + ticker_geom_t *decay_ticker; unsigned tick0, tick1, arena_ind; size_t sz, large0; void *p; @@ -205,7 +205,7 @@ TEST_BEGIN(test_decay_ticks) { expect_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, &sz_arena_ind, (void *)&arena_ind, sizeof(arena_ind)), 0, "Unexpected mallctl() failure"); - decay_ticker = decay_ticker_get(tsd_fetch(), arena_ind); + decay_ticker = tsd_arena_decay_tickerp_get(tsd_fetch()); expect_ptr_not_null(decay_ticker, "Unexpected failure getting decay ticker"); @@ -216,60 +216,60 @@ TEST_BEGIN(test_decay_ticks) { */ /* malloc(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = malloc(large0); expect_ptr_not_null(p, "Unexpected malloc() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()"); /* free(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); free(p); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during free()"); /* calloc(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = calloc(1, large0); expect_ptr_not_null(p, "Unexpected calloc() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()"); free(p); /* posix_memalign(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); expect_d_eq(posix_memalign(&p, sizeof(size_t), large0), 0, "Unexpected posix_memalign() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during posix_memalign()"); free(p); /* aligned_alloc(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = aligned_alloc(sizeof(size_t), large0); expect_ptr_not_null(p, "Unexpected aligned_alloc() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during aligned_alloc()"); free(p); /* realloc(). */ /* Allocate. */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = realloc(NULL, large0); expect_ptr_not_null(p, "Unexpected realloc() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); /* Reallocate. */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = realloc(p, large0); expect_ptr_not_null(p, "Unexpected realloc() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); /* Deallocate. */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); realloc(p, 0); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()"); /* @@ -286,41 +286,41 @@ TEST_BEGIN(test_decay_ticks) { sz = allocx_sizes[i]; /* mallocx(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = mallocx(sz, MALLOCX_TCACHE_NONE); expect_ptr_not_null(p, "Unexpected mallocx() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during mallocx() (sz=%zu)", sz); /* rallocx(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = rallocx(p, sz, MALLOCX_TCACHE_NONE); expect_ptr_not_null(p, "Unexpected rallocx() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during rallocx() (sz=%zu)", sz); /* xallocx(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); xallocx(p, sz, 0, MALLOCX_TCACHE_NONE); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during xallocx() (sz=%zu)", sz); /* dallocx(). */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); dallocx(p, MALLOCX_TCACHE_NONE); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during dallocx() (sz=%zu)", sz); /* sdallocx(). */ p = mallocx(sz, MALLOCX_TCACHE_NONE); expect_ptr_not_null(p, "Unexpected mallocx() failure"); - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); sdallocx(p, sz, MALLOCX_TCACHE_NONE); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during sdallocx() " "(sz=%zu)", sz); @@ -349,31 +349,24 @@ TEST_BEGIN(test_decay_ticks) { sz = tcache_sizes[i]; /* tcache fill. */ - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); p = mallocx(sz, MALLOCX_TCACHE(tcache_ind)); expect_ptr_not_null(p, "Unexpected mallocx() failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); expect_u32_ne(tick1, tick0, "Expected ticker to tick during tcache fill " "(sz=%zu)", sz); /* tcache flush. */ dallocx(p, MALLOCX_TCACHE(tcache_ind)); - tick0 = ticker_read(decay_ticker); + tick0 = ticker_geom_read(decay_ticker); expect_d_eq(mallctl("tcache.flush", NULL, NULL, (void *)&tcache_ind, sizeof(unsigned)), 0, "Unexpected mallctl failure"); - tick1 = ticker_read(decay_ticker); + tick1 = ticker_geom_read(decay_ticker); /* Will only tick if it's in tcache. */ - if (sz <= tcache_max) { - expect_u32_ne(tick1, tick0, - "Expected ticker to tick during tcache " - "flush (sz=%zu)", sz); - } else { - expect_u32_eq(tick1, tick0, - "Unexpected ticker tick during tcache " - "flush (sz=%zu)", sz); - } + expect_u32_ne(tick1, tick0, + "Expected ticker to tick during tcache flush (sz=%zu)", sz); } } TEST_END @@ -401,7 +394,7 @@ decay_ticker_helper(unsigned arena_ind, int flags, bool dirty, ssize_t dt, void *p = do_mallocx(1, flags); uint64_t dirty_npurge1, muzzy_npurge1; do { - for (unsigned i = 0; i < DECAY_NTICKS_PER_UPDATE / 2; + for (unsigned i = 0; i < ARENA_DECAY_NTICKS_PER_UPDATE / 2; i++) { void *q = do_mallocx(1, flags); dallocx(q, flags);