project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add configure options.

Browse Source 
Add:
  --with-lg-page
  --with-lg-page-sizes
  --with-lg-size-class-group
  --with-lg-quantum

Get rid of STATIC_PAGE_SHIFT, in favor of directly setting LG_PAGE.

Fix various edge conditions exposed by the configure options.
This commit is contained in:
Jason Evans
2014-10-09 17:54:06 -07:00
parent b123ddc760
commit fc0b3b7383
16 changed files with 278 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/arena.c
View File

@@ -623,7 +623,7 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero)
arena_chunk_t *chunk;
arena_run_t *run;
assert(size <= arena_maxclass);
assert(size <= arena_maxrun);
assert((size & PAGE_MASK) == 0);
/* Search the arena's chunks for the lowest best fit. */
@@ -673,7 +673,7 @@ arena_run_alloc_small(arena_t *arena, size_t size, index_t binind)
arena_chunk_t *chunk;
arena_run_t *run;
assert(size <= arena_maxclass);
assert(size <= arena_maxrun);
assert((size & PAGE_MASK) == 0);
assert(binind != BININD_INVALID);
@@ -1728,9 +1728,9 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
arena_bin_runs_insert(bin, run);
}
void
arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_chunk_map_bits_t *bitselm)
static void
arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_chunk_map_bits_t *bitselm, bool junked)
{
size_t pageind, rpages_ind;
arena_run_t *run;
@@ -1749,7 +1749,7 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
if (config_fill || config_stats)
size = bin_info->reg_size;
if (config_fill && unlikely(opt_junk))
if (!junked && config_fill && unlikely(opt_junk))
arena_dalloc_junk_small(ptr, bin_info);
arena_run_reg_dalloc(run, ptr);
@@ -1765,6 +1765,14 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
}
}
void
arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_chunk_map_bits_t *bitselm)
{
arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true);
}
void
arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t pageind, arena_chunk_map_bits_t *bitselm)
@@ -1777,7 +1785,7 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
run = &arena_miscelm_get(chunk, rpages_ind)->run;
bin = run->bin;
malloc_mutex_lock(&bin->lock);
arena_dalloc_bin_locked(arena, chunk, ptr, bitselm);
arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false);
malloc_mutex_unlock(&bin->lock);
}
@@ -1800,7 +1808,7 @@ arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
#undef arena_dalloc_junk_large
#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl)
#endif
static void
void
arena_dalloc_junk_large(void *ptr, size_t usize)
{
@@ -1815,7 +1823,8 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large =
#endif
void
arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr)
arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk,
void *ptr, bool junked)
{
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind);
@@ -1824,7 +1833,8 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr)
if (config_fill || config_stats) {
size_t usize = arena_mapbits_large_size_get(chunk, pageind);
arena_dalloc_junk_large(ptr, usize);
if (!junked)
arena_dalloc_junk_large(ptr, usize);
if (config_stats) {
index_t index = size2index(usize) - NBINS;
@@ -1838,12 +1848,20 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr)
arena_run_dalloc(arena, run, true, false);
}
void
arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk,
void *ptr)
{
arena_dalloc_large_locked_impl(arena, chunk, ptr, true);
}
void
arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
{
malloc_mutex_lock(&arena->lock);
arena_dalloc_large_locked(arena, chunk, ptr);
arena_dalloc_large_locked_impl(arena, chunk, ptr, false);
malloc_mutex_unlock(&arena->lock);
}
@@ -2398,6 +2416,7 @@ arena_boot(void)
sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias);
arena_maxrun = chunksize - (map_bias << LG_PAGE);
assert(arena_maxrun > 0);
arena_maxclass = index2size(size2index(chunksize)-1);
if (arena_maxclass > arena_maxrun) {
/*
@@ -2407,6 +2426,7 @@ arena_boot(void)
*/
arena_maxclass = arena_maxrun;
}
assert(arena_maxclass > 0);
nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS);
bin_info_init();

52
src/huge.c
View File

@@ -13,7 +13,7 @@ static malloc_mutex_t huge_mtx;
static extent_tree_t huge;
void *
huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero)
huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, bool try_tcache)
{
size_t usize;
@@ -23,12 +23,12 @@ huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero)
return (NULL);
}
return (huge_palloc(tsd, arena, usize, chunksize, zero));
return (huge_palloc(tsd, arena, usize, chunksize, zero, try_tcache));
}
void *
huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
bool zero)
bool zero, bool try_tcache)
{
void *ret;
size_t csize;
@@ -42,7 +42,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
/* Allocate an extent node with which to track the chunk. */
node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
CACHELINE, false, tsd != NULL, NULL);
CACHELINE, false, try_tcache, NULL);
if (node == NULL)
return (NULL);
@@ -58,7 +58,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
}
ret = arena_chunk_alloc_huge(arena, NULL, csize, alignment, &is_zeroed);
if (ret == NULL) {
idalloct(tsd, node, tsd != NULL);
idalloct(tsd, node, try_tcache);
return (NULL);
}
@@ -122,6 +122,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
expand_addr = ptr + CHUNK_CEILING(oldsize);
expand_size = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
assert(expand_size > 0);
malloc_mutex_lock(&huge_mtx);
@@ -223,13 +224,8 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
return (false);
}
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
return (false);
}
/* Shrink the allocation in-place. */
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize)) {
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)) {
extent_node_t *node, key;
void *excess_addr;
size_t excess_size;
@@ -251,7 +247,10 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
/* Zap the excess chunks. */
huge_dalloc_junk(ptr + usize, oldsize - usize);
arena_chunk_dalloc_huge(node->arena, excess_addr, excess_size);
if (excess_size > 0) {
arena_chunk_dalloc_huge(node->arena, excess_addr,
excess_size);
}
return (false);
}
@@ -269,7 +268,8 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
void *
huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc)
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
bool try_tcache_dalloc)
{
void *ret;
size_t copysize;
@@ -283,19 +283,25 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
* different size class. In that case, fall back to allocating new
* space and copying.
*/
if (alignment > chunksize)
ret = huge_palloc(tsd, arena, size + extra, alignment, zero);
else
ret = huge_malloc(tsd, arena, size + extra, zero);
if (alignment > chunksize) {
ret = huge_palloc(tsd, arena, size + extra, alignment, zero,
try_tcache_alloc);
} else {
ret = huge_malloc(tsd, arena, size + extra, zero,
try_tcache_alloc);
}
if (ret == NULL) {
if (extra == 0)
return (NULL);
/* Try again, this time without extra. */
if (alignment > chunksize)
ret = huge_palloc(tsd, arena, size, alignment, zero);
else
ret = huge_malloc(tsd, arena, size, zero);
if (alignment > chunksize) {
ret = huge_palloc(tsd, arena, size, alignment, zero,
try_tcache_alloc);
} else {
ret = huge_malloc(tsd, arena, size, zero,
try_tcache_alloc);
}
if (ret == NULL)
return (NULL);
@@ -312,7 +318,7 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
}
void
huge_dalloc(tsd_t *tsd, void *ptr)
huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache)
{
extent_node_t *node, key;
@@ -330,7 +336,7 @@ huge_dalloc(tsd_t *tsd, void *ptr)
huge_dalloc_junk(node->addr, node->size);
arena_chunk_dalloc_huge(node->arena, node->addr,
CHUNK_CEILING(node->size));
idalloct(tsd, node, tsd != NULL);
idalloct(tsd, node, try_tcache);
}
size_t

50
src/jemalloc.c
View File

@@ -67,6 +67,8 @@ const uint8_t size2index_tab[] = {
#define S2B_7(i) S2B_6(i) S2B_6(i)
#define S2B_8(i) S2B_7(i) S2B_7(i)
#define S2B_9(i) S2B_8(i) S2B_8(i)
#define S2B_10(i) S2B_9(i) S2B_9(i)
#define S2B_11(i) S2B_10(i) S2B_10(i)
#define S2B_no(i)
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
S2B_##lg_delta_lookup(index)
@@ -78,6 +80,8 @@ const uint8_t size2index_tab[] = {
#undef S2B_7
#undef S2B_8
#undef S2B_9
#undef S2B_10
#undef S2B_11
#undef S2B_no
#undef SC
};
@@ -199,6 +203,7 @@ static void *
a0alloc(size_t size, bool zero)
{
void *ret;
tsd_t *tsd;
if (unlikely(malloc_init()))
return (NULL);
@@ -206,10 +211,11 @@ a0alloc(size_t size, bool zero)
if (size == 0)
size = 1;
tsd = tsd_fetch();
if (size <= arena_maxclass)
ret = arena_malloc(NULL, a0get(), size, zero, false);
ret = arena_malloc(tsd, a0get(), size, zero, false);
else
ret = huge_malloc(NULL, a0get(), size, zero);
ret = huge_malloc(tsd, a0get(), size, zero, false);
return (ret);
}
@@ -231,16 +237,18 @@ a0calloc(size_t num, size_t size)
void
a0free(void *ptr)
{
tsd_t *tsd;
arena_chunk_t *chunk;
if (ptr == NULL)
return;
tsd = tsd_fetch();
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
arena_dalloc(NULL, chunk, ptr, false);
arena_dalloc(tsd, chunk, ptr, false);
else
huge_dalloc(NULL, ptr);
huge_dalloc(tsd, ptr, false);
}
/* Create a new arena and insert it into the arenas array at index ind. */
@@ -817,15 +825,15 @@ malloc_conf_init(void)
"Invalid conf value", \
k, klen, v, vlen); \
} else if (clip) { \
if (min != 0 && um < min) \
o = min; \
else if (um > max) \
o = max; \
if ((min) != 0 && um < (min)) \
o = (min); \
else if (um > (max)) \
o = (max); \
else \
o = um; \
} else { \
if ((min != 0 && um < min) || \
um > max) { \
if (((min) != 0 && um < (min)) \
|| um > (max)) { \
malloc_conf_error( \
"Out-of-range " \
"conf value", \
@@ -847,8 +855,8 @@ malloc_conf_init(void)
malloc_conf_error( \
"Invalid conf value", \
k, klen, v, vlen); \
} else if (l < (ssize_t)min || l > \
(ssize_t)max) { \
} else if (l < (ssize_t)(min) || l > \
(ssize_t)(max)) { \
malloc_conf_error( \
"Out-of-range conf value", \
k, klen, v, vlen); \
@@ -868,15 +876,16 @@ malloc_conf_init(void)
CONF_HANDLE_BOOL(opt_abort, "abort", true)
/*
* Chunks always require at least one header page, plus
* one data page in the absence of redzones, or three
* pages in the presence of redzones. In order to
* simplify options processing, fix the limit based on
* config_fill.
* Chunks always require at least one header page,
* as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and
* possibly an additional page in the presence of
* redzones. In order to simplify options processing,
* use a conservative bound that accommodates all these
* constraints.
*/
CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
(config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1,
true)
LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1),
(sizeof(size_t) << 3) - 1, true)
if (strncmp("dss", k, klen) == 0) {
int i;
bool match = false;
@@ -2088,8 +2097,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
// XX Dangerous arenas read.
arena = arenas[arena_ind];
arena = arena_get(tsd, arena_ind, true, true);
} else
arena = NULL;

53
src/tcache.c
View File

@@ -117,8 +117,8 @@ tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem,
(uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_bits_t *bitselm =
arena_bitselm_get(chunk, pageind);
arena_dalloc_bin_locked(arena, chunk, ptr,
bitselm);
arena_dalloc_bin_junked_locked(arena, chunk,
ptr, bitselm);
} else {
/*
* This object was allocated via a different
@@ -193,9 +193,10 @@ tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem,
ptr = tbin->avail[i];
assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk->arena == arena)
arena_dalloc_large_locked(arena, chunk, ptr);
else {
if (chunk->arena == arena) {
arena_dalloc_large_junked_locked(arena, chunk,
ptr);
} else {
/*
* This object was allocated via a different
* arena than the one that is currently locked.
@@ -279,11 +280,11 @@ tcache_get_hard(tsd_t *tsd)
arena = arena_choose(tsd, NULL);
if (unlikely(arena == NULL))
return (NULL);
return (tcache_create(arena));
return (tcache_create(tsd, arena));
}
tcache_t *
tcache_create(arena_t *arena)
tcache_create(tsd_t *tsd, arena_t *arena)
{
tcache_t *tcache;
size_t size, stack_offset;
@@ -294,23 +295,10 @@ tcache_create(arena_t *arena)
size = PTR_CEILING(size);
stack_offset = size;
size += stack_nelms * sizeof(void *);
/*
* Round up to the nearest multiple of the cacheline size, in order to
* avoid the possibility of false cacheline sharing.
*
* That this works relies on the same logic as in ipalloc(), but we
* cannot directly call ipalloc() here due to tcache bootstrapping
* issues.
*/
size = (size + CACHELINE_MASK) & (-CACHELINE);
if (size <= SMALL_MAXCLASS)
tcache = (tcache_t *)arena_malloc_small(arena, size, true);
else if (size <= tcache_maxclass)
tcache = (tcache_t *)arena_malloc_large(arena, size, true);
else
tcache = (tcache_t *)icalloct(NULL, size, false, arena);
/* Avoid false cacheline sharing. */
size = sa2u(size, CACHELINE);
tcache = ipalloct(tsd, size, CACHELINE, true, false, arena);
if (tcache == NULL)
return (NULL);
@@ -331,7 +319,6 @@ static void
tcache_destroy(tsd_t *tsd, tcache_t *tcache)
{
unsigned i;
size_t tcache_size;
tcache_arena_dissociate(tcache);
@@ -366,23 +353,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
arena_prof_accum(tcache->arena, tcache->prof_accumbytes))
prof_idump();
tcache_size = arena_salloc(tcache, false);
if (tcache_size <= SMALL_MAXCLASS) {
arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
arena_t *arena = chunk->arena;
size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >>
LG_PAGE;
arena_chunk_map_bits_t *bitselm = arena_bitselm_get(chunk,
pageind);
arena_dalloc_bin(arena, chunk, tcache, pageind, bitselm);
} else if (tcache_size <= tcache_maxclass) {
arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
arena_t *arena = chunk->arena;
arena_dalloc_large(arena, chunk, tcache);
} else
idalloct(tsd, tcache, false);
idalloct(tsd, tcache, false);
}
void