diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 67fdc00c..e043ef49 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -1,9 +1,13 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H #define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H -extern bool opt_tcache; -extern ssize_t opt_lg_tcache_max; +extern bool opt_tcache; +extern ssize_t opt_lg_tcache_max; extern ssize_t opt_lg_tcache_nslots_mul; +extern unsigned opt_tcache_nslots_small_min; +extern unsigned opt_tcache_nslots_small_max; +extern unsigned opt_tcache_nslots_large; +extern ssize_t opt_lg_tcache_shift; /* * Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more diff --git a/include/jemalloc/internal/tcache_types.h b/include/jemalloc/internal/tcache_types.h index cba86f43..34a0599c 100644 --- a/include/jemalloc/internal/tcache_types.h +++ b/include/jemalloc/internal/tcache_types.h @@ -17,23 +17,6 @@ typedef struct tcaches_s tcaches_t; #define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3) #define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY -/* - * Absolute minimum number of cache slots for each small bin. - */ -#define TCACHE_NSLOTS_SMALL_MIN 20 - -/* - * Absolute maximum number of cache slots for each small bin in the thread - * cache. This is an additional constraint beyond that imposed as: twice the - * number of regions per slab for this size class. - * - * This constant must be an even number. - */ -#define TCACHE_NSLOTS_SMALL_MAX 200 - -/* Number of cache slots for large size classes. */ -#define TCACHE_NSLOTS_LARGE 20 - /* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */ #define LG_TCACHE_MAXCLASS_DEFAULT 15 diff --git a/src/jemalloc.c b/src/jemalloc.c index fbec733e..4f911e22 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1379,6 +1379,16 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], */ CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul, "lg_tcache_nslots_mul", -16, 16) + /* Ditto with values past 2048. */ + CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min, + "tcache_nslots_small_min", 1, 2048, + CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max, + "tcache_nslots_small_max", 1, 2048, + CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large, + "tcache_nslots_large", 1, 2048, + CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) /* * The runtime option of oversize_threshold remains diff --git a/src/tcache.c b/src/tcache.c index a18d91de..9586556e 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -10,8 +10,13 @@ /******************************************************************************/ /* Data. */ -bool opt_tcache = true; -ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; +bool opt_tcache = true; +ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; + +/* Reasonable defaults for min and max values. */ +unsigned opt_tcache_nslots_small_min = 20; +unsigned opt_tcache_nslots_small_max = 200; +unsigned opt_tcache_nslots_large = 20; /* * We attempt to make the number of slots in a tcache bin for a given size class @@ -19,7 +24,7 @@ ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; * the multiplier is 1/2 (i.e. we set the maximum number of objects in the * tcache to half the number of objects in a slab). * This is bounded by some other constraints as well, like the fact that it - * must be even, must be less than TCACHE_NSLOTS_SMALL_MAX, etc.. + * must be even, must be less than opt_tcache_nslots_small_max, etc.. */ ssize_t opt_lg_tcache_nslots_mul = -1; @@ -485,7 +490,6 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, tcache_slow->arena = NULL; tcache_slow->dyn_alloc = mem; - assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); memset(tcache->bins, 0, sizeof(cache_bin_t) * nhbins); size_t cur_offset = 0; @@ -792,10 +796,37 @@ static unsigned tcache_ncached_max_compute(szind_t szind) { if (szind >= SC_NBINS) { assert(szind < nhbins); - return TCACHE_NSLOTS_LARGE; + return opt_tcache_nslots_large; } unsigned slab_nregs = bin_infos[szind].nregs; + /* We may modify these values; start with the opt versions. */ + unsigned nslots_small_min = opt_tcache_nslots_small_min; + unsigned nslots_small_max = opt_tcache_nslots_small_max; + + /* + * Clamp values to meet our constraints -- even, nonzero, min < max, and + * suitable for a cache bin size. + */ + if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) { + nslots_small_max = CACHE_BIN_NCACHED_MAX; + } + if (nslots_small_min % 2 != 0) { + nslots_small_min++; + } + if (nslots_small_max % 2 != 0) { + nslots_small_max--; + } + if (nslots_small_min < 2) { + nslots_small_min = 2; + } + if (nslots_small_max < 2) { + nslots_small_max = 2; + } + if (nslots_small_min > nslots_small_max) { + nslots_small_min = nslots_small_max; + } + unsigned candidate; if (opt_lg_tcache_nslots_mul < 0) { candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul); @@ -810,12 +841,12 @@ tcache_ncached_max_compute(szind_t szind) { */ ++candidate; } - if (candidate <= TCACHE_NSLOTS_SMALL_MIN) { - return TCACHE_NSLOTS_SMALL_MIN; - } else if (candidate <= TCACHE_NSLOTS_SMALL_MAX) { + if (candidate <= nslots_small_min) { + return nslots_small_min; + } else if (candidate <= nslots_small_max) { return candidate; } else { - return TCACHE_NSLOTS_SMALL_MAX; + return nslots_small_max; } } diff --git a/test/unit/cache_bin.c b/test/unit/cache_bin.c index cbd8ce02..43fe8c6c 100644 --- a/test/unit/cache_bin.c +++ b/test/unit/cache_bin.c @@ -53,12 +53,13 @@ do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs, } TEST_BEGIN(test_cache_bin) { + const int ncached_max = 100; bool success; void *ptr; cache_bin_t bin; cache_bin_info_t info; - cache_bin_info_init(&info, TCACHE_NSLOTS_SMALL_MAX); + cache_bin_info_init(&info, ncached_max); size_t size; size_t alignment; @@ -74,7 +75,7 @@ TEST_BEGIN(test_cache_bin) { assert_zu_eq(cur_offset, size, "Should use all requested memory"); /* Initialize to empty; should then have 0 elements. */ - cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(&info); + expect_d_eq(ncached_max, cache_bin_info_ncached_max(&info), ""); expect_true(cache_bin_ncached_get(&bin, &info) == 0, ""); expect_true(cache_bin_low_water_get(&bin, &info) == 0, "");