Store associated arena in tcache.

This fixes tcache_flush for manual tcaches, which wasn't able to find
the correct arena it associated with. Also changed the decay test to
cover this case (by using manually created arenas).

include/jemalloc/internal/tcache_externs.h

@@ -34,7 +34,7 @@ void	tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
 void	tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
     unsigned rem, tcache_t *tcache);
 void	tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache,
-    arena_t *oldarena, arena_t *newarena);
+    arena_t *arena);
 tcache_t *tcache_get_hard(tsd_t *tsd);
 tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena);
 void	tcache_cleanup(tsd_t *tsd);

include/jemalloc/internal/tcache_structs.h

@@ -35,6 +35,7 @@ struct tcache_s {
 	uint64_t	prof_accumbytes;/* Cleared after arena_prof_accum(). */
 	ticker_t	gc_ticker;	/* Drives incremental GC. */
 	szind_t		next_gc_bin;	/* Next bin to GC. */
+	arena_t		*arena;		/* Associated arena. */
 	tcache_bin_t	tbins[1];	/* Dynamically sized. */
 	/*
 	 * The pointer stacks associated with tbins follow as a contiguous

src/ctl.c

@@ -1342,7 +1342,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 			tcache_t *tcache = tsd_tcache_get(tsd);
 			if (tcache != NULL) {
 				tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
-				    oldarena, newarena);
+				    newarena);
 			}
 		}
 	}

src/jemalloc.c

@@ -446,6 +446,7 @@ arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) {
 	arena_nthreads_dec(oldarena, false);
 	arena_nthreads_inc(newarena, false);
 	tsd_arena_set(tsd, newarena);
+	tsd_iarena_set(tsd, newarena);
 }
 
 static void

src/tcache.c

@@ -99,7 +99,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
 	assert(binind < NBINS);
 	assert(rem <= tbin->ncached);
 
-	arena = arena_choose(tsd, NULL);
+	arena = tcache->arena;
 	assert(arena != NULL);
 	for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
 		/* Lock the arena bin associated with the first object. */
@@ -175,7 +175,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
 	assert(binind < nhbins);
 	assert(rem <= tbin->ncached);
 
-	arena_t *arena = arena_choose(tsd, NULL);
+	arena_t *arena = tcache->arena;
 	assert(arena != NULL);
 	unsigned nflush = tbin->ncached - rem;
 	while (nflush > 0) {
@@ -259,6 +259,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
 
 static void
 tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
+	tcache->arena = arena;
 	if (config_stats) {
 		/* Link into list of extant tcaches. */
 		malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
@@ -269,7 +270,8 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
 }
 
 static void
-tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
+tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
+	arena_t *arena = tcache->arena;
 	if (config_stats) {
 		/* Unlink from list of extant tcaches. */
 		malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
@@ -291,10 +293,9 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
 }
 
 void
-tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *oldarena,
+tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
-    arena_t *newarena) {
+	tcache_arena_dissociate(tsdn, tcache);
-	tcache_arena_dissociate(tsdn, tcache, oldarena);
+	tcache_arena_associate(tsdn, tcache, arena);
-	tcache_arena_associate(tsdn, tcache, newarena);
 }
 
 tcache_t *
@@ -360,7 +361,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) {
 	unsigned i;
 
 	arena = arena_choose(tsd, NULL);
-	tcache_arena_dissociate(tsd_tsdn(tsd), tcache, arena);
+	tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
 
 	for (i = 0; i < NBINS; i++) {
 		tcache_bin_t *tbin = &tcache->tbins[i];

test/unit/decay.c

@@ -123,18 +123,28 @@ generate_dirty(unsigned arena_ind, size_t size) {
 
 TEST_BEGIN(test_decay_ticks) {
 	ticker_t *decay_ticker;
-	unsigned tick0, tick1;
+	unsigned tick0, tick1, arena_ind;
 	size_t sz, large0;
 	void *p;
 
-	decay_ticker = decay_ticker_get(tsd_fetch(), 0);
-	assert_ptr_not_null(decay_ticker,
-	    "Unexpected failure getting decay ticker");
-
 	sz = sizeof(size_t);
 	assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large0, &sz, NULL,
 	    0), 0, "Unexpected mallctl failure");
 
+	int err;
+	/* Set up a manually managed arena for test. */
+	arena_ind = do_arena_create(0);
+
+	/* Migrate to the new arena, and get the ticker. */
+	unsigned old_arena_ind;
+	size_t sz_arena_ind = sizeof(old_arena_ind);
+	err = mallctl("thread.arena", (void *)&old_arena_ind, &sz_arena_ind,
+		      (void *)&arena_ind, sizeof(arena_ind));
+	assert_d_eq(err, 0, "Unexpected mallctl() failure");
+	decay_ticker = decay_ticker_get(tsd_fetch(), arena_ind);
+	assert_ptr_not_null(decay_ticker,
+	    "Unexpected failure getting decay ticker");
+
 	/*
 	 * Test the standard APIs using a large size class, since we can't
 	 * control tcache interactions for small size classes (except by
@@ -263,6 +273,12 @@ TEST_BEGIN(test_decay_ticks) {
 		tcache_sizes[0] = large0;
 		tcache_sizes[1] = 1;
 
+		size_t tcache_max, sz_tcache_max;
+		sz_tcache_max = sizeof(tcache_max);
+		err = mallctl("arenas.tcache_max", (void *)&tcache_max,
+		    &sz_tcache_max, NULL, 0);
+		assert_d_eq(err, 0, "Unexpected mallctl() failure");
+
 		sz = sizeof(unsigned);
 		assert_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz,
 		    NULL, 0), 0, "Unexpected mallctl failure");
@@ -285,9 +301,17 @@ TEST_BEGIN(test_decay_ticks) {
 			    (void *)&tcache_ind, sizeof(unsigned)), 0,
 			    "Unexpected mallctl failure");
 			tick1 = ticker_read(decay_ticker);
-			assert_u32_ne(tick1, tick0,
+
-			    "Expected ticker to tick during tcache flush "
+			/* Will only tick if it's in tcache. */
-			    "(sz=%zu)", sz);
+			if (sz <= tcache_max) {
+				assert_u32_ne(tick1, tick0,
+				    "Expected ticker to tick during tcache "
+				    "flush (sz=%zu)", sz);
+			} else {
+				assert_u32_eq(tick1, tick0,
+				    "Unexpected ticker tick during tcache "
+				    "flush (sz=%zu)", sz);
+			}
 		}
 	}
 }