Reduce size of small_size2bin lookup table.
Convert all direct small_size2bin[...] accesses to SMALL_SIZE2BIN(...) macro calls, and use a couple of cheap math operations to allow compacting the table by 4X or 8X, on 32- and 64-bit systems, respectively.
This commit is contained in:
parent
ff7450727f
commit
41ade967c2
@ -19,6 +19,7 @@
|
|||||||
#ifdef JEMALLOC_TINY
|
#ifdef JEMALLOC_TINY
|
||||||
/* Smallest size class to support. */
|
/* Smallest size class to support. */
|
||||||
# define LG_TINY_MIN LG_SIZEOF_PTR
|
# define LG_TINY_MIN LG_SIZEOF_PTR
|
||||||
|
# define TINY_MIN (1U << LG_TINY_MIN)
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -389,7 +390,13 @@ struct arena_s {
|
|||||||
extern size_t opt_lg_qspace_max;
|
extern size_t opt_lg_qspace_max;
|
||||||
extern size_t opt_lg_cspace_max;
|
extern size_t opt_lg_cspace_max;
|
||||||
extern ssize_t opt_lg_dirty_mult;
|
extern ssize_t opt_lg_dirty_mult;
|
||||||
|
/*
|
||||||
|
* small_size2bin is a compact lookup table that rounds request sizes up to
|
||||||
|
* size classes. In order to reduce cache footprint, the table is compressed,
|
||||||
|
* and all accesses are via the SMALL_SIZE2BIN macro.
|
||||||
|
*/
|
||||||
extern uint8_t const *small_size2bin;
|
extern uint8_t const *small_size2bin;
|
||||||
|
#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN])
|
||||||
|
|
||||||
/* Various bin-related settings. */
|
/* Various bin-related settings. */
|
||||||
#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */
|
#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */
|
||||||
|
@ -402,7 +402,7 @@ s2u(size_t size)
|
|||||||
{
|
{
|
||||||
|
|
||||||
if (size <= small_maxclass)
|
if (size <= small_maxclass)
|
||||||
return (arenas[0]->bins[small_size2bin[size]].reg_size);
|
return (arenas[0]->bins[SMALL_SIZE2BIN(size)].reg_size);
|
||||||
if (size <= arena_maxclass)
|
if (size <= arena_maxclass)
|
||||||
return (PAGE_CEILING(size));
|
return (PAGE_CEILING(size));
|
||||||
return (CHUNK_CEILING(size));
|
return (CHUNK_CEILING(size));
|
||||||
@ -448,7 +448,7 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p)
|
|||||||
if (usize <= arena_maxclass && alignment <= PAGE_SIZE) {
|
if (usize <= arena_maxclass && alignment <= PAGE_SIZE) {
|
||||||
if (usize <= small_maxclass) {
|
if (usize <= small_maxclass) {
|
||||||
return
|
return
|
||||||
(arenas[0]->bins[small_size2bin[usize]].reg_size);
|
(arenas[0]->bins[SMALL_SIZE2BIN(usize)].reg_size);
|
||||||
}
|
}
|
||||||
return (PAGE_CEILING(usize));
|
return (PAGE_CEILING(usize));
|
||||||
} else {
|
} else {
|
||||||
|
@ -223,7 +223,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
|
|||||||
size_t binind;
|
size_t binind;
|
||||||
tcache_bin_t *tbin;
|
tcache_bin_t *tbin;
|
||||||
|
|
||||||
binind = small_size2bin[size];
|
binind = SMALL_SIZE2BIN(size);
|
||||||
assert(binind < nbins);
|
assert(binind < nbins);
|
||||||
tbin = &tcache->tbins[binind];
|
tbin = &tcache->tbins[binind];
|
||||||
ret = tcache_alloc_easy(tbin);
|
ret = tcache_alloc_easy(tbin);
|
||||||
|
@ -25,26 +25,27 @@ size_t mspace_mask;
|
|||||||
|
|
||||||
/*
|
/*
|
||||||
* const_small_size2bin is a static constant lookup table that in the common
|
* const_small_size2bin is a static constant lookup table that in the common
|
||||||
* case can be used as-is for small_size2bin. For dynamically linked programs,
|
* case can be used as-is for small_size2bin.
|
||||||
* this avoids a page of memory overhead per process.
|
|
||||||
*/
|
*/
|
||||||
#define S2B_1(i) i,
|
#if (LG_TINY_MIN == 2)
|
||||||
#define S2B_2(i) S2B_1(i) S2B_1(i)
|
#define S2B_4(i) i,
|
||||||
#define S2B_4(i) S2B_2(i) S2B_2(i)
|
|
||||||
#define S2B_8(i) S2B_4(i) S2B_4(i)
|
#define S2B_8(i) S2B_4(i) S2B_4(i)
|
||||||
|
#elif (LG_TINY_MIN == 3)
|
||||||
|
#define S2B_8(i) i,
|
||||||
|
#else
|
||||||
|
# error "Unsupported LG_TINY_MIN"
|
||||||
|
#endif
|
||||||
#define S2B_16(i) S2B_8(i) S2B_8(i)
|
#define S2B_16(i) S2B_8(i) S2B_8(i)
|
||||||
#define S2B_32(i) S2B_16(i) S2B_16(i)
|
#define S2B_32(i) S2B_16(i) S2B_16(i)
|
||||||
#define S2B_64(i) S2B_32(i) S2B_32(i)
|
#define S2B_64(i) S2B_32(i) S2B_32(i)
|
||||||
#define S2B_128(i) S2B_64(i) S2B_64(i)
|
#define S2B_128(i) S2B_64(i) S2B_64(i)
|
||||||
#define S2B_256(i) S2B_128(i) S2B_128(i)
|
#define S2B_256(i) S2B_128(i) S2B_128(i)
|
||||||
/*
|
/*
|
||||||
* The number of elements in const_small_size2bin is dependent on page size
|
* The number of elements in const_small_size2bin is dependent on the
|
||||||
* and on the definition for SUBPAGE. If SUBPAGE changes, the '- 255' must also
|
* definition for SUBPAGE.
|
||||||
* change, along with the addition/removal of static lookup table element
|
|
||||||
* definitions.
|
|
||||||
*/
|
*/
|
||||||
static const uint8_t const_small_size2bin[STATIC_PAGE_SIZE - 255] = {
|
static JEMALLOC_ATTR(aligned(CACHELINE))
|
||||||
S2B_1(0xffU) /* 0 */
|
const uint8_t const_small_size2bin[] = {
|
||||||
#if (LG_QUANTUM == 4)
|
#if (LG_QUANTUM == 4)
|
||||||
/* 16-byte quantum **********************/
|
/* 16-byte quantum **********************/
|
||||||
# ifdef JEMALLOC_TINY
|
# ifdef JEMALLOC_TINY
|
||||||
@ -1475,7 +1476,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
|
|||||||
arena_run_t *run;
|
arena_run_t *run;
|
||||||
size_t binind;
|
size_t binind;
|
||||||
|
|
||||||
binind = small_size2bin[size];
|
binind = SMALL_SIZE2BIN(size);
|
||||||
assert(binind < nbins);
|
assert(binind < nbins);
|
||||||
bin = &arena->bins[binind];
|
bin = &arena->bins[binind];
|
||||||
size = bin->reg_size;
|
size = bin->reg_size;
|
||||||
@ -1713,7 +1714,7 @@ arena_prof_promoted(const void *ptr, size_t size)
|
|||||||
|
|
||||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||||
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
|
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
|
||||||
binind = small_size2bin[size];
|
binind = SMALL_SIZE2BIN(size);
|
||||||
assert(binind < nbins);
|
assert(binind < nbins);
|
||||||
chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits &
|
chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits &
|
||||||
~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT);
|
~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT);
|
||||||
@ -2166,11 +2167,11 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
|
|||||||
*/
|
*/
|
||||||
if (oldsize <= arena_maxclass) {
|
if (oldsize <= arena_maxclass) {
|
||||||
if (oldsize <= small_maxclass) {
|
if (oldsize <= small_maxclass) {
|
||||||
assert(choose_arena()->bins[small_size2bin[
|
assert(choose_arena()->bins[SMALL_SIZE2BIN(
|
||||||
oldsize]].reg_size == oldsize);
|
oldsize)].reg_size == oldsize);
|
||||||
if ((size + extra <= small_maxclass &&
|
if ((size + extra <= small_maxclass &&
|
||||||
small_size2bin[size + extra] ==
|
SMALL_SIZE2BIN(size + extra) ==
|
||||||
small_size2bin[oldsize]) || (size <= oldsize &&
|
SMALL_SIZE2BIN(oldsize)) || (size <= oldsize &&
|
||||||
size + extra >= oldsize)) {
|
size + extra >= oldsize)) {
|
||||||
#ifdef JEMALLOC_FILL
|
#ifdef JEMALLOC_FILL
|
||||||
if (opt_junk && size < oldsize) {
|
if (opt_junk && size < oldsize) {
|
||||||
@ -2371,40 +2372,39 @@ small_size2bin_validate(void)
|
|||||||
{
|
{
|
||||||
size_t i, size, binind;
|
size_t i, size, binind;
|
||||||
|
|
||||||
assert(small_size2bin[0] == 0xffU);
|
|
||||||
i = 1;
|
i = 1;
|
||||||
# ifdef JEMALLOC_TINY
|
# ifdef JEMALLOC_TINY
|
||||||
/* Tiny. */
|
/* Tiny. */
|
||||||
for (; i < (1U << LG_TINY_MIN); i++) {
|
for (; i < (1U << LG_TINY_MIN); i++) {
|
||||||
size = pow2_ceil(1U << LG_TINY_MIN);
|
size = pow2_ceil(1U << LG_TINY_MIN);
|
||||||
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
||||||
assert(small_size2bin[i] == binind);
|
assert(SMALL_SIZE2BIN(i) == binind);
|
||||||
}
|
}
|
||||||
for (; i < qspace_min; i++) {
|
for (; i < qspace_min; i++) {
|
||||||
size = pow2_ceil(i);
|
size = pow2_ceil(i);
|
||||||
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
||||||
assert(small_size2bin[i] == binind);
|
assert(SMALL_SIZE2BIN(i) == binind);
|
||||||
}
|
}
|
||||||
# endif
|
# endif
|
||||||
/* Quantum-spaced. */
|
/* Quantum-spaced. */
|
||||||
for (; i <= qspace_max; i++) {
|
for (; i <= qspace_max; i++) {
|
||||||
size = QUANTUM_CEILING(i);
|
size = QUANTUM_CEILING(i);
|
||||||
binind = ntbins + (size >> LG_QUANTUM) - 1;
|
binind = ntbins + (size >> LG_QUANTUM) - 1;
|
||||||
assert(small_size2bin[i] == binind);
|
assert(SMALL_SIZE2BIN(i) == binind);
|
||||||
}
|
}
|
||||||
/* Cacheline-spaced. */
|
/* Cacheline-spaced. */
|
||||||
for (; i <= cspace_max; i++) {
|
for (; i <= cspace_max; i++) {
|
||||||
size = CACHELINE_CEILING(i);
|
size = CACHELINE_CEILING(i);
|
||||||
binind = ntbins + nqbins + ((size - cspace_min) >>
|
binind = ntbins + nqbins + ((size - cspace_min) >>
|
||||||
LG_CACHELINE);
|
LG_CACHELINE);
|
||||||
assert(small_size2bin[i] == binind);
|
assert(SMALL_SIZE2BIN(i) == binind);
|
||||||
}
|
}
|
||||||
/* Sub-page. */
|
/* Sub-page. */
|
||||||
for (; i <= sspace_max; i++) {
|
for (; i <= sspace_max; i++) {
|
||||||
size = SUBPAGE_CEILING(i);
|
size = SUBPAGE_CEILING(i);
|
||||||
binind = ntbins + nqbins + ncbins + ((size - sspace_min)
|
binind = ntbins + nqbins + ncbins + ((size - sspace_min)
|
||||||
>> LG_SUBPAGE);
|
>> LG_SUBPAGE);
|
||||||
assert(small_size2bin[i] == binind);
|
assert(SMALL_SIZE2BIN(i) == binind);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@ -2415,12 +2415,12 @@ small_size2bin_init(void)
|
|||||||
|
|
||||||
if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT
|
if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT
|
||||||
|| opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT
|
|| opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT
|
||||||
|| sizeof(const_small_size2bin) != small_maxclass + 1)
|
|| (sizeof(const_small_size2bin) != ((small_maxclass-1) >>
|
||||||
|
LG_TINY_MIN) + 1))
|
||||||
return (small_size2bin_init_hard());
|
return (small_size2bin_init_hard());
|
||||||
|
|
||||||
small_size2bin = const_small_size2bin;
|
small_size2bin = const_small_size2bin;
|
||||||
#ifdef JEMALLOC_DEBUG
|
#ifdef JEMALLOC_DEBUG
|
||||||
assert(sizeof(const_small_size2bin) == small_maxclass + 1);
|
|
||||||
small_size2bin_validate();
|
small_size2bin_validate();
|
||||||
#endif
|
#endif
|
||||||
return (false);
|
return (false);
|
||||||
@ -2431,49 +2431,52 @@ small_size2bin_init_hard(void)
|
|||||||
{
|
{
|
||||||
size_t i, size, binind;
|
size_t i, size, binind;
|
||||||
uint8_t *custom_small_size2bin;
|
uint8_t *custom_small_size2bin;
|
||||||
|
#define CUSTOM_SMALL_SIZE2BIN(s) \
|
||||||
|
custom_small_size2bin[(s-1) >> LG_TINY_MIN]
|
||||||
|
|
||||||
assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT
|
assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT
|
||||||
|| opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT
|
|| opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT
|
||||||
|| sizeof(const_small_size2bin) != small_maxclass + 1);
|
|| (sizeof(const_small_size2bin) != ((small_maxclass-1) >>
|
||||||
|
LG_TINY_MIN) + 1));
|
||||||
|
|
||||||
custom_small_size2bin = (uint8_t *)base_alloc(small_maxclass + 1);
|
custom_small_size2bin = (uint8_t *)
|
||||||
|
base_alloc(small_maxclass >> LG_TINY_MIN);
|
||||||
if (custom_small_size2bin == NULL)
|
if (custom_small_size2bin == NULL)
|
||||||
return (true);
|
return (true);
|
||||||
|
|
||||||
custom_small_size2bin[0] = 0xffU;
|
|
||||||
i = 1;
|
i = 1;
|
||||||
#ifdef JEMALLOC_TINY
|
#ifdef JEMALLOC_TINY
|
||||||
/* Tiny. */
|
/* Tiny. */
|
||||||
for (; i < (1U << LG_TINY_MIN); i++) {
|
for (; i < (1U << LG_TINY_MIN); i += TINY_MIN) {
|
||||||
size = pow2_ceil(1U << LG_TINY_MIN);
|
size = pow2_ceil(1U << LG_TINY_MIN);
|
||||||
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
||||||
custom_small_size2bin[i] = binind;
|
CUSTOM_SMALL_SIZE2BIN(i) = binind;
|
||||||
}
|
}
|
||||||
for (; i < qspace_min; i++) {
|
for (; i < qspace_min; i += TINY_MIN) {
|
||||||
size = pow2_ceil(i);
|
size = pow2_ceil(i);
|
||||||
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
|
||||||
custom_small_size2bin[i] = binind;
|
CUSTOM_SMALL_SIZE2BIN(i) = binind;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
/* Quantum-spaced. */
|
/* Quantum-spaced. */
|
||||||
for (; i <= qspace_max; i++) {
|
for (; i <= qspace_max; i += TINY_MIN) {
|
||||||
size = QUANTUM_CEILING(i);
|
size = QUANTUM_CEILING(i);
|
||||||
binind = ntbins + (size >> LG_QUANTUM) - 1;
|
binind = ntbins + (size >> LG_QUANTUM) - 1;
|
||||||
custom_small_size2bin[i] = binind;
|
CUSTOM_SMALL_SIZE2BIN(i) = binind;
|
||||||
}
|
}
|
||||||
/* Cacheline-spaced. */
|
/* Cacheline-spaced. */
|
||||||
for (; i <= cspace_max; i++) {
|
for (; i <= cspace_max; i += TINY_MIN) {
|
||||||
size = CACHELINE_CEILING(i);
|
size = CACHELINE_CEILING(i);
|
||||||
binind = ntbins + nqbins + ((size - cspace_min) >>
|
binind = ntbins + nqbins + ((size - cspace_min) >>
|
||||||
LG_CACHELINE);
|
LG_CACHELINE);
|
||||||
custom_small_size2bin[i] = binind;
|
CUSTOM_SMALL_SIZE2BIN(i) = binind;
|
||||||
}
|
}
|
||||||
/* Sub-page. */
|
/* Sub-page. */
|
||||||
for (; i <= sspace_max; i++) {
|
for (; i <= sspace_max; i += TINY_MIN) {
|
||||||
size = SUBPAGE_CEILING(i);
|
size = SUBPAGE_CEILING(i);
|
||||||
binind = ntbins + nqbins + ncbins + ((size - sspace_min) >>
|
binind = ntbins + nqbins + ncbins + ((size - sspace_min) >>
|
||||||
LG_SUBPAGE);
|
LG_SUBPAGE);
|
||||||
custom_small_size2bin[i] = binind;
|
CUSTOM_SMALL_SIZE2BIN(i) = binind;
|
||||||
}
|
}
|
||||||
|
|
||||||
small_size2bin = custom_small_size2bin;
|
small_size2bin = custom_small_size2bin;
|
||||||
@ -2481,6 +2484,7 @@ small_size2bin_init_hard(void)
|
|||||||
small_size2bin_validate();
|
small_size2bin_validate();
|
||||||
#endif
|
#endif
|
||||||
return (false);
|
return (false);
|
||||||
|
#undef CUSTOM_SMALL_SIZE2BIN
|
||||||
}
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
|
Loading…
Reference in New Issue
Block a user