Tcache: make slot sizing configurable.
This commit is contained in:
parent
b58dea8d1b
commit
181093173d
@ -1,9 +1,13 @@
|
|||||||
#ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
|
#ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
|
||||||
#define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
|
#define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
|
||||||
|
|
||||||
extern bool opt_tcache;
|
extern bool opt_tcache;
|
||||||
extern ssize_t opt_lg_tcache_max;
|
extern ssize_t opt_lg_tcache_max;
|
||||||
extern ssize_t opt_lg_tcache_nslots_mul;
|
extern ssize_t opt_lg_tcache_nslots_mul;
|
||||||
|
extern unsigned opt_tcache_nslots_small_min;
|
||||||
|
extern unsigned opt_tcache_nslots_small_max;
|
||||||
|
extern unsigned opt_tcache_nslots_large;
|
||||||
|
extern ssize_t opt_lg_tcache_shift;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more
|
* Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more
|
||||||
|
@ -17,23 +17,6 @@ typedef struct tcaches_s tcaches_t;
|
|||||||
#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
|
#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
|
||||||
#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
|
#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
|
||||||
|
|
||||||
/*
|
|
||||||
* Absolute minimum number of cache slots for each small bin.
|
|
||||||
*/
|
|
||||||
#define TCACHE_NSLOTS_SMALL_MIN 20
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Absolute maximum number of cache slots for each small bin in the thread
|
|
||||||
* cache. This is an additional constraint beyond that imposed as: twice the
|
|
||||||
* number of regions per slab for this size class.
|
|
||||||
*
|
|
||||||
* This constant must be an even number.
|
|
||||||
*/
|
|
||||||
#define TCACHE_NSLOTS_SMALL_MAX 200
|
|
||||||
|
|
||||||
/* Number of cache slots for large size classes. */
|
|
||||||
#define TCACHE_NSLOTS_LARGE 20
|
|
||||||
|
|
||||||
/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
|
/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */
|
||||||
#define LG_TCACHE_MAXCLASS_DEFAULT 15
|
#define LG_TCACHE_MAXCLASS_DEFAULT 15
|
||||||
|
|
||||||
|
@ -1379,6 +1379,16 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
|
|||||||
*/
|
*/
|
||||||
CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul,
|
CONF_HANDLE_SSIZE_T(opt_lg_tcache_nslots_mul,
|
||||||
"lg_tcache_nslots_mul", -16, 16)
|
"lg_tcache_nslots_mul", -16, 16)
|
||||||
|
/* Ditto with values past 2048. */
|
||||||
|
CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_min,
|
||||||
|
"tcache_nslots_small_min", 1, 2048,
|
||||||
|
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
|
||||||
|
CONF_HANDLE_UNSIGNED(opt_tcache_nslots_small_max,
|
||||||
|
"tcache_nslots_small_max", 1, 2048,
|
||||||
|
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
|
||||||
|
CONF_HANDLE_UNSIGNED(opt_tcache_nslots_large,
|
||||||
|
"tcache_nslots_large", 1, 2048,
|
||||||
|
CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* The runtime option of oversize_threshold remains
|
* The runtime option of oversize_threshold remains
|
||||||
|
49
src/tcache.c
49
src/tcache.c
@ -10,8 +10,13 @@
|
|||||||
/******************************************************************************/
|
/******************************************************************************/
|
||||||
/* Data. */
|
/* Data. */
|
||||||
|
|
||||||
bool opt_tcache = true;
|
bool opt_tcache = true;
|
||||||
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
|
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
|
||||||
|
|
||||||
|
/* Reasonable defaults for min and max values. */
|
||||||
|
unsigned opt_tcache_nslots_small_min = 20;
|
||||||
|
unsigned opt_tcache_nslots_small_max = 200;
|
||||||
|
unsigned opt_tcache_nslots_large = 20;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* We attempt to make the number of slots in a tcache bin for a given size class
|
* We attempt to make the number of slots in a tcache bin for a given size class
|
||||||
@ -19,7 +24,7 @@ ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
|
|||||||
* the multiplier is 1/2 (i.e. we set the maximum number of objects in the
|
* the multiplier is 1/2 (i.e. we set the maximum number of objects in the
|
||||||
* tcache to half the number of objects in a slab).
|
* tcache to half the number of objects in a slab).
|
||||||
* This is bounded by some other constraints as well, like the fact that it
|
* This is bounded by some other constraints as well, like the fact that it
|
||||||
* must be even, must be less than TCACHE_NSLOTS_SMALL_MAX, etc..
|
* must be even, must be less than opt_tcache_nslots_small_max, etc..
|
||||||
*/
|
*/
|
||||||
ssize_t opt_lg_tcache_nslots_mul = -1;
|
ssize_t opt_lg_tcache_nslots_mul = -1;
|
||||||
|
|
||||||
@ -485,7 +490,6 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
|
|||||||
tcache_slow->arena = NULL;
|
tcache_slow->arena = NULL;
|
||||||
tcache_slow->dyn_alloc = mem;
|
tcache_slow->dyn_alloc = mem;
|
||||||
|
|
||||||
assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
|
|
||||||
memset(tcache->bins, 0, sizeof(cache_bin_t) * nhbins);
|
memset(tcache->bins, 0, sizeof(cache_bin_t) * nhbins);
|
||||||
|
|
||||||
size_t cur_offset = 0;
|
size_t cur_offset = 0;
|
||||||
@ -792,10 +796,37 @@ static unsigned
|
|||||||
tcache_ncached_max_compute(szind_t szind) {
|
tcache_ncached_max_compute(szind_t szind) {
|
||||||
if (szind >= SC_NBINS) {
|
if (szind >= SC_NBINS) {
|
||||||
assert(szind < nhbins);
|
assert(szind < nhbins);
|
||||||
return TCACHE_NSLOTS_LARGE;
|
return opt_tcache_nslots_large;
|
||||||
}
|
}
|
||||||
unsigned slab_nregs = bin_infos[szind].nregs;
|
unsigned slab_nregs = bin_infos[szind].nregs;
|
||||||
|
|
||||||
|
/* We may modify these values; start with the opt versions. */
|
||||||
|
unsigned nslots_small_min = opt_tcache_nslots_small_min;
|
||||||
|
unsigned nslots_small_max = opt_tcache_nslots_small_max;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Clamp values to meet our constraints -- even, nonzero, min < max, and
|
||||||
|
* suitable for a cache bin size.
|
||||||
|
*/
|
||||||
|
if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) {
|
||||||
|
nslots_small_max = CACHE_BIN_NCACHED_MAX;
|
||||||
|
}
|
||||||
|
if (nslots_small_min % 2 != 0) {
|
||||||
|
nslots_small_min++;
|
||||||
|
}
|
||||||
|
if (nslots_small_max % 2 != 0) {
|
||||||
|
nslots_small_max--;
|
||||||
|
}
|
||||||
|
if (nslots_small_min < 2) {
|
||||||
|
nslots_small_min = 2;
|
||||||
|
}
|
||||||
|
if (nslots_small_max < 2) {
|
||||||
|
nslots_small_max = 2;
|
||||||
|
}
|
||||||
|
if (nslots_small_min > nslots_small_max) {
|
||||||
|
nslots_small_min = nslots_small_max;
|
||||||
|
}
|
||||||
|
|
||||||
unsigned candidate;
|
unsigned candidate;
|
||||||
if (opt_lg_tcache_nslots_mul < 0) {
|
if (opt_lg_tcache_nslots_mul < 0) {
|
||||||
candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul);
|
candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul);
|
||||||
@ -810,12 +841,12 @@ tcache_ncached_max_compute(szind_t szind) {
|
|||||||
*/
|
*/
|
||||||
++candidate;
|
++candidate;
|
||||||
}
|
}
|
||||||
if (candidate <= TCACHE_NSLOTS_SMALL_MIN) {
|
if (candidate <= nslots_small_min) {
|
||||||
return TCACHE_NSLOTS_SMALL_MIN;
|
return nslots_small_min;
|
||||||
} else if (candidate <= TCACHE_NSLOTS_SMALL_MAX) {
|
} else if (candidate <= nslots_small_max) {
|
||||||
return candidate;
|
return candidate;
|
||||||
} else {
|
} else {
|
||||||
return TCACHE_NSLOTS_SMALL_MAX;
|
return nslots_small_max;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -53,12 +53,13 @@ do_flush_test(cache_bin_t *bin, cache_bin_info_t *info, void **ptrs,
|
|||||||
}
|
}
|
||||||
|
|
||||||
TEST_BEGIN(test_cache_bin) {
|
TEST_BEGIN(test_cache_bin) {
|
||||||
|
const int ncached_max = 100;
|
||||||
bool success;
|
bool success;
|
||||||
void *ptr;
|
void *ptr;
|
||||||
|
|
||||||
cache_bin_t bin;
|
cache_bin_t bin;
|
||||||
cache_bin_info_t info;
|
cache_bin_info_t info;
|
||||||
cache_bin_info_init(&info, TCACHE_NSLOTS_SMALL_MAX);
|
cache_bin_info_init(&info, ncached_max);
|
||||||
|
|
||||||
size_t size;
|
size_t size;
|
||||||
size_t alignment;
|
size_t alignment;
|
||||||
@ -74,7 +75,7 @@ TEST_BEGIN(test_cache_bin) {
|
|||||||
assert_zu_eq(cur_offset, size, "Should use all requested memory");
|
assert_zu_eq(cur_offset, size, "Should use all requested memory");
|
||||||
|
|
||||||
/* Initialize to empty; should then have 0 elements. */
|
/* Initialize to empty; should then have 0 elements. */
|
||||||
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(&info);
|
expect_d_eq(ncached_max, cache_bin_info_ncached_max(&info), "");
|
||||||
expect_true(cache_bin_ncached_get(&bin, &info) == 0, "");
|
expect_true(cache_bin_ncached_get(&bin, &info) == 0, "");
|
||||||
expect_true(cache_bin_low_water_get(&bin, &info) == 0, "");
|
expect_true(cache_bin_low_water_get(&bin, &info) == 0, "");
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user