Tcache: make slot sizing configurable.

include/jemalloc/internal/tcache_externs.h

@@ -1,9 +1,13 @@
 #ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
 #define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
 
-extern bool	opt_tcache;
+extern bool opt_tcache;
-extern ssize_t	opt_lg_tcache_max;
+extern ssize_t opt_lg_tcache_max;
 extern ssize_t	opt_lg_tcache_nslots_mul;
+extern unsigned opt_tcache_nslots_small_min;
+extern unsigned opt_tcache_nslots_small_max;
+extern unsigned opt_tcache_nslots_large;
+extern ssize_t opt_lg_tcache_shift;
 
 /*
  * Number of tcache bins.  There are SC_NBINS small-object bins, plus 0 or more

include/jemalloc/internal/tcache_types.h

@@ -17,23 +17,6 @@ typedef struct tcaches_s tcaches_t;
 #define TCACHE_STATE_PURGATORY		((tcache_t *)(uintptr_t)3)
 #define TCACHE_STATE_MAX		TCACHE_STATE_PURGATORY
 
-/*
- * Absolute minimum number of cache slots for each small bin.
- */
-#define TCACHE_NSLOTS_SMALL_MIN		20
-
-/*
- * Absolute maximum number of cache slots for each small bin in the thread
- * cache.  This is an additional constraint beyond that imposed as: twice the
- * number of regions per slab for this size class.
- *
- * This constant must be an even number.
- */
-#define TCACHE_NSLOTS_SMALL_MAX		200
-
-/* Number of cache slots for large size classes. */
-#define TCACHE_NSLOTS_LARGE		20
-
 /* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
 #define LG_TCACHE_MAXCLASS_DEFAULT	15
 

src/jemalloc.c

@@ -1379,6 +1379,16 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			 */
 			CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul,
 			    "lg_tcache_nslots_mul", -16, 16)
+			/* Ditto with values past 2048. */
+			CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min,
+			    "tcache_nslots_small_min", 1, 2048,
+			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
+			CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max,
+			    "tcache_nslots_small_max", 1, 2048,
+			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
+			CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large,
+			    "tcache_nslots_large", 1, 2048,
+			    CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
 
 			/*
 			 * The runtime option of oversize_threshold remains

src/tcache.c

@@ -10,8 +10,13 @@
 /******************************************************************************/
 /* Data. */
 
-bool	opt_tcache = true;
+bool opt_tcache = true;
-ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
+ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
+
+/* Reasonable defaults for min and max values. */
+unsigned opt_tcache_nslots_small_min = 20;
+unsigned opt_tcache_nslots_small_max = 200;
+unsigned opt_tcache_nslots_large = 20;
 
 /*
  * We attempt to make the number of slots in a tcache bin for a given size class
@@ -19,7 +24,7 @@ ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
  * the multiplier is 1/2 (i.e. we set the maximum number of objects in the
  * tcache to half the number of objects in a slab).
  * This is bounded by some other constraints as well, like the fact that it
- * must be even, must be less than TCACHE_NSLOTS_SMALL_MAX, etc..
+ * must be even, must be less than opt_tcache_nslots_small_max, etc..
  */
 ssize_t	opt_lg_tcache_nslots_mul = -1;
 
@@ -485,7 +490,6 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
 	tcache_slow->arena = NULL;
 	tcache_slow->dyn_alloc = mem;
 
-	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
 	memset(tcache->bins, 0, sizeof(cache_bin_t) * nhbins);
 
 	size_t cur_offset = 0;
@@ -792,10 +796,37 @@ static unsigned
 tcache_ncached_max_compute(szind_t szind) {
 	if (szind >= SC_NBINS) {
 		assert(szind < nhbins);
-		return TCACHE_NSLOTS_LARGE;
+		return opt_tcache_nslots_large;
 	}
 	unsigned slab_nregs = bin_infos[szind].nregs;
 
+	/* We may modify these values; start with the opt versions. */
+	unsigned nslots_small_min = opt_tcache_nslots_small_min;
+	unsigned nslots_small_max = opt_tcache_nslots_small_max;
+
+	/*
+	 * Clamp values to meet our constraints -- even, nonzero, min < max, and
+	 * suitable for a cache bin size.
+	 */
+	if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) {
+		nslots_small_max = CACHE_BIN_NCACHED_MAX;
+	}
+	if (nslots_small_min % 2 != 0) {
+		nslots_small_min++;
+	}
+	if (nslots_small_max % 2 != 0) {
+		nslots_small_max--;
+	}
+	if (nslots_small_min < 2) {
+		nslots_small_min = 2;
+	}
+	if (nslots_small_max < 2) {
+		nslots_small_max = 2;
+	}
+	if (nslots_small_min > nslots_small_max) {
+		nslots_small_min = nslots_small_max;
+	}
+
 	unsigned candidate;
 	if (opt_lg_tcache_nslots_mul < 0) {
 		candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul);
@@ -810,12 +841,12 @@ tcache_ncached_max_compute(szind_t szind) {
 		 */
 		++candidate;
 	}
-	if (candidate <= TCACHE_NSLOTS_SMALL_MIN) {
+	if (candidate <= nslots_small_min) {
-		return TCACHE_NSLOTS_SMALL_MIN;
+		return nslots_small_min;
-	} else if (candidate <= TCACHE_NSLOTS_SMALL_MAX) {
+	} else if (candidate <= nslots_small_max) {
 		return candidate;
 	} else {
-		return TCACHE_NSLOTS_SMALL_MAX;
+		return nslots_small_max;
 	}
 }
 

test/unit/cache_bin.c

@@ -53,12 +53,13 @@ do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
 }
 
 TEST_BEGIN(test_cache_bin) {
+	const int ncached_max = 100;
 	bool success;
 	void *ptr;
 
 	cache_bin_t bin;
 	cache_bin_info_t info;
-	cache_bin_info_init(&info, TCACHE_NSLOTS_SMALL_MAX);
+	cache_bin_info_init(&info, ncached_max);
 
 	size_t size;
 	size_t alignment;
@@ -74,7 +75,7 @@ TEST_BEGIN(test_cache_bin) {
 	assert_zu_eq(cur_offset, size, "Should use all requested memory");
 
 	/* Initialize to empty; should then have 0 elements. */
-	cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(&info);
+	expect_d_eq(ncached_max, cache_bin_info_ncached_max(&info), "");
 	expect_true(cache_bin_ncached_get(&bin, &info) == 0, "");
 	expect_true(cache_bin_low_water_get(&bin, &info) == 0, "");