project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce size of small_size2bin lookup table.

Browse Source 
Convert all direct small_size2bin[...] accesses to SMALL_SIZE2BIN(...)
macro calls, and use a couple of cheap math operations to allow
compacting the table by 4X or 8X, on 32- and 64-bit systems,
respectively.
This commit is contained in:
Jason Evans 2011-03-06 22:56:36 -08:00
parent ff7450727f
commit 41ade967c2
4 changed files with 52 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
jemalloc/include/jemalloc/internal/arena.h
View File

@ -19,6 +19,7 @@
#ifdef JEMALLOC_TINY #ifdef JEMALLOC_TINY
/* Smallest size class to support. */ /* Smallest size class to support. */
# define LG_TINY_MIN LG_SIZEOF_PTR # define LG_TINY_MIN LG_SIZEOF_PTR
# define TINY_MIN (1U << LG_TINY_MIN)
#endif #endif
/* /*
@ -389,7 +390,13 @@ struct arena_s {
extern size_t opt_lg_qspace_max; extern size_t opt_lg_qspace_max;
extern size_t opt_lg_cspace_max; extern size_t opt_lg_cspace_max;
extern ssize_t opt_lg_dirty_mult; extern ssize_t opt_lg_dirty_mult;
/*
* small_size2bin is a compact lookup table that rounds request sizes up to
* size classes. In order to reduce cache footprint, the table is compressed,
* and all accesses are via the SMALL_SIZE2BIN macro.
*/
extern uint8_t const *small_size2bin; extern uint8_t const *small_size2bin;
#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN])
/* Various bin-related settings. */ /* Various bin-related settings. */
#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */ #ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */

4
jemalloc/include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -402,7 +402,7 @@ s2u(size_t size)
{ {
if (size <= small_maxclass) if (size <= small_maxclass)
return (arenas[0]->bins[small_size2bin[size]].reg_size); return (arenas[0]->bins[SMALL_SIZE2BIN(size)].reg_size);
if (size <= arena_maxclass) if (size <= arena_maxclass)
return (PAGE_CEILING(size)); return (PAGE_CEILING(size));
return (CHUNK_CEILING(size)); return (CHUNK_CEILING(size));
@ -448,7 +448,7 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p)
if (usize <= arena_maxclass && alignment <= PAGE_SIZE) { if (usize <= arena_maxclass && alignment <= PAGE_SIZE) {
if (usize <= small_maxclass) { if (usize <= small_maxclass) {
return return
(arenas[0]->bins[small_size2bin[usize]].reg_size); (arenas[0]->bins[SMALL_SIZE2BIN(usize)].reg_size);
} }
return (PAGE_CEILING(usize)); return (PAGE_CEILING(usize));
} else { } else {

2
jemalloc/include/jemalloc/internal/tcache.h
View File

@ -223,7 +223,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
size_t binind; size_t binind;
tcache_bin_t *tbin; tcache_bin_t *tbin;
binind = small_size2bin[size]; binind = SMALL_SIZE2BIN(size);
assert(binind < nbins); assert(binind < nbins);
tbin = &tcache->tbins[binind]; tbin = &tcache->tbins[binind];
ret = tcache_alloc_easy(tbin); ret = tcache_alloc_easy(tbin);

80
jemalloc/src/arena.c
View File

@ -25,26 +25,27 @@ size_t mspace_mask;
/* /*
* const_small_size2bin is a static constant lookup table that in the common * const_small_size2bin is a static constant lookup table that in the common
* case can be used as-is for small_size2bin. For dynamically linked programs, * case can be used as-is for small_size2bin.
* this avoids a page of memory overhead per process.
*/ */
#define S2B_1(i) i, #if (LG_TINY_MIN == 2)
#define S2B_2(i) S2B_1(i) S2B_1(i) #define S2B_4(i) i,
#define S2B_4(i) S2B_2(i) S2B_2(i)
#define S2B_8(i) S2B_4(i) S2B_4(i) #define S2B_8(i) S2B_4(i) S2B_4(i)
#elif (LG_TINY_MIN == 3)
#define S2B_8(i) i,
#else
# error "Unsupported LG_TINY_MIN"
#endif
#define S2B_16(i) S2B_8(i) S2B_8(i) #define S2B_16(i) S2B_8(i) S2B_8(i)
#define S2B_32(i) S2B_16(i) S2B_16(i) #define S2B_32(i) S2B_16(i) S2B_16(i)
#define S2B_64(i) S2B_32(i) S2B_32(i) #define S2B_64(i) S2B_32(i) S2B_32(i)
#define S2B_128(i) S2B_64(i) S2B_64(i) #define S2B_128(i) S2B_64(i) S2B_64(i)
#define S2B_256(i) S2B_128(i) S2B_128(i) #define S2B_256(i) S2B_128(i) S2B_128(i)
/* /*
* The number of elements in const_small_size2bin is dependent on page size * The number of elements in const_small_size2bin is dependent on the
* and on the definition for SUBPAGE. If SUBPAGE changes, the '- 255' must also * definition for SUBPAGE.
* change, along with the addition/removal of static lookup table element
* definitions.
*/ */
static const uint8_t const_small_size2bin[STATIC_PAGE_SIZE - 255] = { static JEMALLOC_ATTR(aligned(CACHELINE))
S2B_1(0xffU) /* 0 */ const uint8_t const_small_size2bin[] = {
#if (LG_QUANTUM == 4) #if (LG_QUANTUM == 4)
/* 16-byte quantum **********************/ /* 16-byte quantum **********************/
# ifdef JEMALLOC_TINY # ifdef JEMALLOC_TINY
@ -1475,7 +1476,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
arena_run_t *run; arena_run_t *run;
size_t binind; size_t binind;
binind = small_size2bin[size]; binind = SMALL_SIZE2BIN(size);
assert(binind < nbins); assert(binind < nbins);
bin = &arena->bins[binind]; bin = &arena->bins[binind];
size = bin->reg_size; size = bin->reg_size;
@ -1713,7 +1714,7 @@ arena_prof_promoted(const void *ptr, size_t size)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
binind = small_size2bin[size]; binind = SMALL_SIZE2BIN(size);
assert(binind < nbins); assert(binind < nbins);
chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits &
~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT);
@ -2166,11 +2167,11 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
*/ */
if (oldsize <= arena_maxclass) { if (oldsize <= arena_maxclass) {
if (oldsize <= small_maxclass) { if (oldsize <= small_maxclass) {
assert(choose_arena()->bins[small_size2bin[ assert(choose_arena()->bins[SMALL_SIZE2BIN(
oldsize]].reg_size == oldsize); oldsize)].reg_size == oldsize);
if ((size + extra <= small_maxclass && if ((size + extra <= small_maxclass &&
small_size2bin[size + extra] == SMALL_SIZE2BIN(size + extra) ==
small_size2bin[oldsize]) || (size <= oldsize && SMALL_SIZE2BIN(oldsize)) || (size <= oldsize &&
size + extra >= oldsize)) { size + extra >= oldsize)) {
#ifdef JEMALLOC_FILL #ifdef JEMALLOC_FILL
if (opt_junk && size < oldsize) { if (opt_junk && size < oldsize) {
@ -2371,40 +2372,39 @@ small_size2bin_validate(void)
{ {
size_t i, size, binind; size_t i, size, binind;
assert(small_size2bin[0] == 0xffU);
i = 1; i = 1;
# ifdef JEMALLOC_TINY # ifdef JEMALLOC_TINY
/* Tiny. */ /* Tiny. */
for (; i < (1U << LG_TINY_MIN); i++) { for (; i < (1U << LG_TINY_MIN); i++) {
size = pow2_ceil(1U << LG_TINY_MIN); size = pow2_ceil(1U << LG_TINY_MIN);
binind = ffs((int)(size >> (LG_TINY_MIN + 1))); binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
assert(small_size2bin[i] == binind); assert(SMALL_SIZE2BIN(i) == binind);
} }
for (; i < qspace_min; i++) { for (; i < qspace_min; i++) {
size = pow2_ceil(i); size = pow2_ceil(i);
binind = ffs((int)(size >> (LG_TINY_MIN + 1))); binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
assert(small_size2bin[i] == binind); assert(SMALL_SIZE2BIN(i) == binind);
} }
# endif # endif
/* Quantum-spaced. */ /* Quantum-spaced. */
for (; i <= qspace_max; i++) { for (; i <= qspace_max; i++) {
size = QUANTUM_CEILING(i); size = QUANTUM_CEILING(i);
binind = ntbins + (size >> LG_QUANTUM) - 1; binind = ntbins + (size >> LG_QUANTUM) - 1;
assert(small_size2bin[i] == binind); assert(SMALL_SIZE2BIN(i) == binind);
} }
/* Cacheline-spaced. */ /* Cacheline-spaced. */
for (; i <= cspace_max; i++) { for (; i <= cspace_max; i++) {
size = CACHELINE_CEILING(i); size = CACHELINE_CEILING(i);
binind = ntbins + nqbins + ((size - cspace_min) >> binind = ntbins + nqbins + ((size - cspace_min) >>
LG_CACHELINE); LG_CACHELINE);
assert(small_size2bin[i] == binind); assert(SMALL_SIZE2BIN(i) == binind);
} }
/* Sub-page. */ /* Sub-page. */
for (; i <= sspace_max; i++) { for (; i <= sspace_max; i++) {
size = SUBPAGE_CEILING(i); size = SUBPAGE_CEILING(i);
binind = ntbins + nqbins + ncbins + ((size - sspace_min) binind = ntbins + nqbins + ncbins + ((size - sspace_min)
>> LG_SUBPAGE); >> LG_SUBPAGE);
assert(small_size2bin[i] == binind); assert(SMALL_SIZE2BIN(i) == binind);
} }
} }
#endif #endif
@ -2415,12 +2415,12 @@ small_size2bin_init(void)
if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT
|| opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT
|| sizeof(const_small_size2bin) != small_maxclass + 1) || (sizeof(const_small_size2bin) != ((small_maxclass-1) >>
LG_TINY_MIN) + 1))
return (small_size2bin_init_hard()); return (small_size2bin_init_hard());
small_size2bin = const_small_size2bin; small_size2bin = const_small_size2bin;
#ifdef JEMALLOC_DEBUG #ifdef JEMALLOC_DEBUG
assert(sizeof(const_small_size2bin) == small_maxclass + 1);
small_size2bin_validate(); small_size2bin_validate();
#endif #endif
return (false); return (false);
@ -2431,49 +2431,52 @@ small_size2bin_init_hard(void)
{ {
size_t i, size, binind; size_t i, size, binind;
uint8_t *custom_small_size2bin; uint8_t *custom_small_size2bin;
#define CUSTOM_SMALL_SIZE2BIN(s) \
custom_small_size2bin[(s-1) >> LG_TINY_MIN]
assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT
|| opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT
|| sizeof(const_small_size2bin) != small_maxclass + 1); || (sizeof(const_small_size2bin) != ((small_maxclass-1) >>
LG_TINY_MIN) + 1));
custom_small_size2bin = (uint8_t *)base_alloc(small_maxclass + 1); custom_small_size2bin = (uint8_t *)
base_alloc(small_maxclass >> LG_TINY_MIN);
if (custom_small_size2bin == NULL) if (custom_small_size2bin == NULL)
return (true); return (true);
custom_small_size2bin[0] = 0xffU;
i = 1; i = 1;
#ifdef JEMALLOC_TINY #ifdef JEMALLOC_TINY
/* Tiny. */ /* Tiny. */
for (; i < (1U << LG_TINY_MIN); i++) { for (; i < (1U << LG_TINY_MIN); i += TINY_MIN) {
size = pow2_ceil(1U << LG_TINY_MIN); size = pow2_ceil(1U << LG_TINY_MIN);
binind = ffs((int)(size >> (LG_TINY_MIN + 1))); binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
custom_small_size2bin[i] = binind; CUSTOM_SMALL_SIZE2BIN(i) = binind;
} }
for (; i < qspace_min; i++) { for (; i < qspace_min; i += TINY_MIN) {
size = pow2_ceil(i); size = pow2_ceil(i);
binind = ffs((int)(size >> (LG_TINY_MIN + 1))); binind = ffs((int)(size >> (LG_TINY_MIN + 1)));
custom_small_size2bin[i] = binind; CUSTOM_SMALL_SIZE2BIN(i) = binind;
} }
#endif #endif
/* Quantum-spaced. */ /* Quantum-spaced. */
for (; i <= qspace_max; i++) { for (; i <= qspace_max; i += TINY_MIN) {
size = QUANTUM_CEILING(i); size = QUANTUM_CEILING(i);
binind = ntbins + (size >> LG_QUANTUM) - 1; binind = ntbins + (size >> LG_QUANTUM) - 1;
custom_small_size2bin[i] = binind; CUSTOM_SMALL_SIZE2BIN(i) = binind;
} }
/* Cacheline-spaced. */ /* Cacheline-spaced. */
for (; i <= cspace_max; i++) { for (; i <= cspace_max; i += TINY_MIN) {
size = CACHELINE_CEILING(i); size = CACHELINE_CEILING(i);
binind = ntbins + nqbins + ((size - cspace_min) >> binind = ntbins + nqbins + ((size - cspace_min) >>
LG_CACHELINE); LG_CACHELINE);
custom_small_size2bin[i] = binind; CUSTOM_SMALL_SIZE2BIN(i) = binind;
} }
/* Sub-page. */ /* Sub-page. */
for (; i <= sspace_max; i++) { for (; i <= sspace_max; i += TINY_MIN) {
size = SUBPAGE_CEILING(i); size = SUBPAGE_CEILING(i);
binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> binind = ntbins + nqbins + ncbins + ((size - sspace_min) >>
LG_SUBPAGE); LG_SUBPAGE);
custom_small_size2bin[i] = binind; CUSTOM_SMALL_SIZE2BIN(i) = binind;
} }
small_size2bin = custom_small_size2bin; small_size2bin = custom_small_size2bin;
@ -2481,6 +2484,7 @@ small_size2bin_init_hard(void)
small_size2bin_validate(); small_size2bin_validate();
#endif #endif
return (false); return (false);
#undef CUSTOM_SMALL_SIZE2BIN
} }
bool bool