Use ticker_geom_t for arena tcache decay.

include/jemalloc/internal/arena_inlines_b.h

@@ -116,18 +116,22 @@ arena_prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx,
 
 JEMALLOC_ALWAYS_INLINE void
 arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
-	tsd_t *tsd;
-	ticker_t *decay_ticker;
-
 	if (unlikely(tsdn_null(tsdn))) {
 		return;
 	}
-	tsd = tsdn_tsd(tsdn);
+	tsd_t *tsd = tsdn_tsd(tsdn);
-	decay_ticker = decay_ticker_get(tsd, arena_ind_get(arena));
+	/*
-	if (unlikely(decay_ticker == NULL)) {
+	 * We use the ticker_geom_t to avoid having per-arena state in the tsd.
-		return;
+	 * Instead of having a countdown-until-decay timer running for every
-	}
+	 * arena in every thread, we flip a coin once per tick, whose
-	if (unlikely(ticker_ticks(decay_ticker, nticks))) {
+	 * probability of coming up heads is 1/nticks; this is effectively the
+	 * operation of the ticker_geom_t.  Each arena has the same chance of a
+	 * coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can
+	 * use a single ticker for all of them.
+	 */
+	ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
+	uint64_t *prng_state = tsd_prng_statep_get(tsd);
+	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks))) {
 		arena_decay(tsdn, arena, false, false);
 	}
 }

include/jemalloc/internal/arena_structs.h

@@ -98,9 +98,4 @@ struct arena_s {
 	bin_t			bins[0];
 };
 
-/* Used in conjunction with tsd for fast arena-related context lookup. */
-struct arena_tdata_s {
-	ticker_t		decay_ticker;
-};
-
 #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H */

include/jemalloc/internal/arena_types.h

@@ -7,11 +7,10 @@
 #define DIRTY_DECAY_MS_DEFAULT	ZD(10 * 1000)
 #define MUZZY_DECAY_MS_DEFAULT	(0)
 /* Number of event ticks between time checks. */
-#define DECAY_NTICKS_PER_UPDATE	1000
+#define ARENA_DECAY_NTICKS_PER_UPDATE	1000
 
 typedef struct arena_decay_s arena_decay_t;
 typedef struct arena_s arena_t;
-typedef struct arena_tdata_s arena_tdata_t;
 
 typedef enum {
 	percpu_arena_mode_names_base   = 0, /* Used for options processing. */

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -56,12 +56,10 @@ void bootstrap_free(void *ptr);
 void arena_set(unsigned ind, arena_t *arena);
 unsigned narenas_total_get(void);
 arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
-arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind);
 arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
 void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
 void iarena_cleanup(tsd_t *tsd);
 void arena_cleanup(tsd_t *tsd);
-void arenas_tdata_cleanup(tsd_t *tsd);
 size_t batch_alloc(void **ptrs, size_t num, size_t size, int flags);
 void jemalloc_prefork(void);
 void jemalloc_postfork_parent(void);

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -56,31 +56,6 @@ percpu_arena_ind_limit(percpu_arena_mode_t mode) {
 	}
 }
 
-static inline arena_tdata_t *
-arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) {
-	arena_tdata_t *tdata;
-	arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
-
-	if (unlikely(arenas_tdata == NULL)) {
-		/* arenas_tdata hasn't been initialized yet. */
-		return arena_tdata_get_hard(tsd, ind);
-	}
-	if (unlikely(ind >= tsd_narenas_tdata_get(tsd))) {
-		/*
-		 * ind is invalid, cache is old (too small), or tdata to be
-		 * initialized.
-		 */
-		return (refresh_if_missing ? arena_tdata_get_hard(tsd, ind) :
-		    NULL);
-	}
-
-	tdata = &arenas_tdata[ind];
-	if (likely(tdata != NULL) || !refresh_if_missing) {
-		return tdata;
-	}
-	return arena_tdata_get_hard(tsd, ind);
-}
-
 static inline arena_t *
 arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) {
 	arena_t *ret;
@@ -97,17 +72,6 @@ arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) {
 	return ret;
 }
 
-static inline ticker_t *
-decay_ticker_get(tsd_t *tsd, unsigned ind) {
-	arena_tdata_t *tdata;
-
-	tdata = arena_tdata_get(tsd, ind, true);
-	if (unlikely(tdata == NULL)) {
-		return NULL;
-	}
-	return &tdata->decay_ticker;
-}
-
 JEMALLOC_ALWAYS_INLINE bool
 tcache_available(tsd_t *tsd) {
 	/*

include/jemalloc/internal/tsd.h

@@ -58,9 +58,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
 /*  O(name,			type,			nullable type) */
 #define TSD_DATA_SLOW							\
     O(tcache_enabled,		bool,			bool)		\
-    O(arenas_tdata_bypass,	bool,			bool)		\
     O(reentrancy_level,		int8_t,			int8_t)		\
-    O(narenas_tdata,		uint32_t,		uint32_t)	\
     O(thread_allocated_last_event,	uint64_t,	uint64_t)	\
     O(thread_allocated_next_event,	uint64_t,	uint64_t)	\
     O(thread_deallocated_last_event,	uint64_t,	uint64_t)	\
@@ -77,7 +75,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
     O(prng_state,		uint64_t,		uint64_t)	\
     O(iarena,			arena_t *,		arena_t *)	\
     O(arena,			arena_t *,		arena_t *)	\
-    O(arenas_tdata,		arena_tdata_t *,	arena_tdata_t *)\
+    O(arena_decay_ticker,	ticker_geom_t,		ticker_geom_t)	\
     O(sec_shard,		uint8_t,		uint8_t)	\
     O(binshards,		tsd_binshards_t,	tsd_binshards_t)\
     O(tsd_link,			tsd_link_t,		tsd_link_t)	\
@@ -90,9 +88,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
 
 #define TSD_DATA_SLOW_INITIALIZER					\
     /* tcache_enabled */	TCACHE_ENABLED_ZERO_INITIALIZER,	\
-    /* arenas_tdata_bypass */	false,					\
     /* reentrancy_level */	0,					\
-    /* narenas_tdata */		0,					\
     /* thread_allocated_last_event */	0,				\
     /* thread_allocated_next_event */	0,				\
     /* thread_deallocated_last_event */	0,				\
@@ -109,7 +105,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
     /* prng_state */		0,					\
     /* iarena */		NULL,					\
     /* arena */			NULL,					\
-    /* arenas_tdata */		NULL,					\
+    /* arena_decay_ticker */						\
+	TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),		\
     /* sec_shard */		(uint8_t)-1,				\
     /* binshards */		TSD_BINSHARDS_ZERO_INITIALIZER,		\
     /* tsd_link */		{NULL},					\

src/jemalloc.c

@@ -493,82 +493,6 @@ arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
 	}
 }
 
-arena_tdata_t *
-arena_tdata_get_hard(tsd_t *tsd, unsigned ind) {
-	arena_tdata_t *tdata, *arenas_tdata_old;
-	arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
-	unsigned narenas_tdata_old, i;
-	unsigned narenas_tdata = tsd_narenas_tdata_get(tsd);
-	unsigned narenas_actual = narenas_total_get();
-
-	/*
-	 * Dissociate old tdata array (and set up for deallocation upon return)
-	 * if it's too small.
-	 */
-	if (arenas_tdata != NULL && narenas_tdata < narenas_actual) {
-		arenas_tdata_old = arenas_tdata;
-		narenas_tdata_old = narenas_tdata;
-		arenas_tdata = NULL;
-		narenas_tdata = 0;
-		tsd_arenas_tdata_set(tsd, arenas_tdata);
-		tsd_narenas_tdata_set(tsd, narenas_tdata);
-	} else {
-		arenas_tdata_old = NULL;
-		narenas_tdata_old = 0;
-	}
-
-	/* Allocate tdata array if it's missing. */
-	if (arenas_tdata == NULL) {
-		bool *arenas_tdata_bypassp = tsd_arenas_tdata_bypassp_get(tsd);
-		narenas_tdata = (ind < narenas_actual) ? narenas_actual : ind+1;
-
-		if (tsd_nominal(tsd) && !*arenas_tdata_bypassp) {
-			*arenas_tdata_bypassp = true;
-			arenas_tdata = (arena_tdata_t *)a0malloc(
-			    sizeof(arena_tdata_t) * narenas_tdata);
-			*arenas_tdata_bypassp = false;
-		}
-		if (arenas_tdata == NULL) {
-			tdata = NULL;
-			goto label_return;
-		}
-		assert(tsd_nominal(tsd) && !*arenas_tdata_bypassp);
-		tsd_arenas_tdata_set(tsd, arenas_tdata);
-		tsd_narenas_tdata_set(tsd, narenas_tdata);
-	}
-
-	/*
-	 * Copy to tdata array.  It's possible that the actual number of arenas
-	 * has increased since narenas_total_get() was called above, but that
-	 * causes no correctness issues unless two threads concurrently execute
-	 * the arenas.create mallctl, which we trust mallctl synchronization to
-	 * prevent.
-	 */
-
-	/* Copy/initialize tickers. */
-	for (i = 0; i < narenas_actual; i++) {
-		if (i < narenas_tdata_old) {
-			ticker_copy(&arenas_tdata[i].decay_ticker,
-			    &arenas_tdata_old[i].decay_ticker);
-		} else {
-			ticker_init(&arenas_tdata[i].decay_ticker,
-			    DECAY_NTICKS_PER_UPDATE);
-		}
-	}
-	if (narenas_tdata > narenas_actual) {
-		memset(&arenas_tdata[narenas_actual], 0, sizeof(arena_tdata_t)
-		    * (narenas_tdata - narenas_actual));
-	}
-
-	/* Read the refreshed tdata array. */
-	tdata = &arenas_tdata[ind];
-label_return:
-	if (arenas_tdata_old != NULL) {
-		a0dalloc(arenas_tdata_old);
-	}
-	return tdata;
-}
-
 /* Slow path, called only by arena_choose(). */
 arena_t *
 arena_choose_hard(tsd_t *tsd, bool internal) {
@@ -705,20 +629,6 @@ arena_cleanup(tsd_t *tsd) {
 	}
 }
 
-void
-arenas_tdata_cleanup(tsd_t *tsd) {
-	arena_tdata_t *arenas_tdata;
-
-	/* Prevent tsd->arenas_tdata from being (re)created. */
-	*tsd_arenas_tdata_bypassp_get(tsd) = true;
-
-	arenas_tdata = tsd_arenas_tdata_get(tsd);
-	if (arenas_tdata != NULL) {
-		tsd_arenas_tdata_set(tsd, NULL);
-		a0dalloc(arenas_tdata);
-	}
-}
-
 static void
 stats_print_atexit(void) {
 	if (config_stats) {

src/tsd.c

@@ -251,8 +251,6 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) {
 	assert(!tsd_in_nominal_list(tsd));
 	assert(*tsd_arenap_get_unsafe(tsd) == NULL);
 	assert(*tsd_iarenap_get_unsafe(tsd) == NULL);
-	assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true);
-	assert(*tsd_arenas_tdatap_get_unsafe(tsd) == NULL);
 	assert(*tsd_tcache_enabledp_get_unsafe(tsd) == false);
 	assert(*tsd_prof_tdatap_get_unsafe(tsd) == NULL);
 }
@@ -267,7 +265,6 @@ tsd_data_init_nocleanup(tsd_t *tsd) {
 	 * We set up tsd in a way that no cleanup is needed.
 	 */
 	rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd));
-	*tsd_arenas_tdata_bypassp_get(tsd) = true;
 	*tsd_tcache_enabledp_get_unsafe(tsd) = false;
 	*tsd_reentrancy_levelp_get(tsd) = 1;
 	tsd_prng_state_init(tsd);
@@ -375,7 +372,6 @@ tsd_do_data_cleanup(tsd_t *tsd) {
 	prof_tdata_cleanup(tsd);
 	iarena_cleanup(tsd);
 	arena_cleanup(tsd);
-	arenas_tdata_cleanup(tsd);
 	tcache_cleanup(tsd);
 	witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd));
 	*tsd_reentrancy_levelp_get(tsd) = 1;
@@ -439,7 +435,6 @@ malloc_tsd_boot0(void) {
 		return NULL;
 	}
 	tsd = tsd_fetch();
-	*tsd_arenas_tdata_bypassp_get(tsd) = true;
 	return tsd;
 }
 
@@ -449,7 +444,6 @@ malloc_tsd_boot1(void) {
 	tsd_t *tsd = tsd_fetch();
 	/* malloc_slow has been set properly.  Update tsd_slow. */
 	tsd_slow_update(tsd);
-	*tsd_arenas_tdata_bypassp_get(tsd) = false;
 }
 
 #ifdef _WIN32

test/unit/arena_decay.c

@@ -187,7 +187,7 @@ TEST_BEGIN(test_decay_ticks) {
 	test_skip_if(check_background_thread_enabled());
 	test_skip_if(opt_hpa);
 
-	ticker_t *decay_ticker;
+	ticker_geom_t *decay_ticker;
 	unsigned tick0, tick1, arena_ind;
 	size_t sz, large0;
 	void *p;
@@ -205,7 +205,7 @@ TEST_BEGIN(test_decay_ticks) {
 	expect_d_eq(mallctl("thread.arena", (void *)&old_arena_ind,
 	    &sz_arena_ind, (void *)&arena_ind, sizeof(arena_ind)), 0,
 	    "Unexpected mallctl() failure");
-	decay_ticker = decay_ticker_get(tsd_fetch(), arena_ind);
+	decay_ticker = tsd_arena_decay_tickerp_get(tsd_fetch());
 	expect_ptr_not_null(decay_ticker,
 	    "Unexpected failure getting decay ticker");
 
@@ -216,60 +216,60 @@ TEST_BEGIN(test_decay_ticks) {
 	 */
 
 	/* malloc(). */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	p = malloc(large0);
 	expect_ptr_not_null(p, "Unexpected malloc() failure");
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()");
 	/* free(). */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	free(p);
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0, "Expected ticker to tick during free()");
 
 	/* calloc(). */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	p = calloc(1, large0);
 	expect_ptr_not_null(p, "Unexpected calloc() failure");
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()");
 	free(p);
 
 	/* posix_memalign(). */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	expect_d_eq(posix_memalign(&p, sizeof(size_t), large0), 0,
 	    "Unexpected posix_memalign() failure");
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0,
 	    "Expected ticker to tick during posix_memalign()");
 	free(p);
 
 	/* aligned_alloc(). */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	p = aligned_alloc(sizeof(size_t), large0);
 	expect_ptr_not_null(p, "Unexpected aligned_alloc() failure");
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0,
 	    "Expected ticker to tick during aligned_alloc()");
 	free(p);
 
 	/* realloc(). */
 	/* Allocate. */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	p = realloc(NULL, large0);
 	expect_ptr_not_null(p, "Unexpected realloc() failure");
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
 	/* Reallocate. */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	p = realloc(p, large0);
 	expect_ptr_not_null(p, "Unexpected realloc() failure");
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
 	/* Deallocate. */
-	tick0 = ticker_read(decay_ticker);
+	tick0 = ticker_geom_read(decay_ticker);
 	realloc(p, 0);
-	tick1 = ticker_read(decay_ticker);
+	tick1 = ticker_geom_read(decay_ticker);
 	expect_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
 
 	/*
@@ -286,41 +286,41 @@ TEST_BEGIN(test_decay_ticks) {
 			sz = allocx_sizes[i];
 
 			/* mallocx(). */
-			tick0 = ticker_read(decay_ticker);
+			tick0 = ticker_geom_read(decay_ticker);
 			p = mallocx(sz, MALLOCX_TCACHE_NONE);
 			expect_ptr_not_null(p, "Unexpected mallocx() failure");
-			tick1 = ticker_read(decay_ticker);
+			tick1 = ticker_geom_read(decay_ticker);
 			expect_u32_ne(tick1, tick0,
 			    "Expected ticker to tick during mallocx() (sz=%zu)",
 			    sz);
 			/* rallocx(). */
-			tick0 = ticker_read(decay_ticker);
+			tick0 = ticker_geom_read(decay_ticker);
 			p = rallocx(p, sz, MALLOCX_TCACHE_NONE);
 			expect_ptr_not_null(p, "Unexpected rallocx() failure");
-			tick1 = ticker_read(decay_ticker);
+			tick1 = ticker_geom_read(decay_ticker);
 			expect_u32_ne(tick1, tick0,
 			    "Expected ticker to tick during rallocx() (sz=%zu)",
 			    sz);
 			/* xallocx(). */
-			tick0 = ticker_read(decay_ticker);
+			tick0 = ticker_geom_read(decay_ticker);
 			xallocx(p, sz, 0, MALLOCX_TCACHE_NONE);
-			tick1 = ticker_read(decay_ticker);
+			tick1 = ticker_geom_read(decay_ticker);
 			expect_u32_ne(tick1, tick0,
 			    "Expected ticker to tick during xallocx() (sz=%zu)",
 			    sz);
 			/* dallocx(). */
-			tick0 = ticker_read(decay_ticker);
+			tick0 = ticker_geom_read(decay_ticker);
 			dallocx(p, MALLOCX_TCACHE_NONE);
-			tick1 = ticker_read(decay_ticker);
+			tick1 = ticker_geom_read(decay_ticker);
 			expect_u32_ne(tick1, tick0,
 			    "Expected ticker to tick during dallocx() (sz=%zu)",
 			    sz);
 			/* sdallocx(). */
 			p = mallocx(sz, MALLOCX_TCACHE_NONE);
 			expect_ptr_not_null(p, "Unexpected mallocx() failure");
-			tick0 = ticker_read(decay_ticker);
+			tick0 = ticker_geom_read(decay_ticker);
 			sdallocx(p, sz, MALLOCX_TCACHE_NONE);
-			tick1 = ticker_read(decay_ticker);
+			tick1 = ticker_geom_read(decay_ticker);
 			expect_u32_ne(tick1, tick0,
 			    "Expected ticker to tick during sdallocx() "
 			    "(sz=%zu)", sz);
@@ -349,31 +349,24 @@ TEST_BEGIN(test_decay_ticks) {
 		sz = tcache_sizes[i];
 
 		/* tcache fill. */
-		tick0 = ticker_read(decay_ticker);
+		tick0 = ticker_geom_read(decay_ticker);
 		p = mallocx(sz, MALLOCX_TCACHE(tcache_ind));
 		expect_ptr_not_null(p, "Unexpected mallocx() failure");
-		tick1 = ticker_read(decay_ticker);
+		tick1 = ticker_geom_read(decay_ticker);
 		expect_u32_ne(tick1, tick0,
 		    "Expected ticker to tick during tcache fill "
 		    "(sz=%zu)", sz);
 		/* tcache flush. */
 		dallocx(p, MALLOCX_TCACHE(tcache_ind));
-		tick0 = ticker_read(decay_ticker);
+		tick0 = ticker_geom_read(decay_ticker);
 		expect_d_eq(mallctl("tcache.flush", NULL, NULL,
 		    (void *)&tcache_ind, sizeof(unsigned)), 0,
 		    "Unexpected mallctl failure");
-		tick1 = ticker_read(decay_ticker);
+		tick1 = ticker_geom_read(decay_ticker);
 
 		/* Will only tick if it's in tcache. */
-		if (sz <= tcache_max) {
 		expect_u32_ne(tick1, tick0,
-			    "Expected ticker to tick during tcache "
+		    "Expected ticker to tick during tcache flush (sz=%zu)", sz);
-			    "flush (sz=%zu)", sz);
-		} else {
-			expect_u32_eq(tick1, tick0,
-			    "Unexpected ticker tick during tcache "
-			    "flush (sz=%zu)", sz);
-		}
 	}
 }
 TEST_END
@@ -401,7 +394,7 @@ decay_ticker_helper(unsigned arena_ind, int flags, bool dirty, ssize_t dt,
 	void *p = do_mallocx(1, flags);
 	uint64_t dirty_npurge1, muzzy_npurge1;
 	do {
-		for (unsigned i = 0; i < DECAY_NTICKS_PER_UPDATE / 2;
+		for (unsigned i = 0; i < ARENA_DECAY_NTICKS_PER_UPDATE / 2;
 		    i++) {
 			void *q = do_mallocx(1, flags);
 			dallocx(q, flags);