project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix huge_palloc() regression.

Browse Source 
Split arena_choose() into arena_[i]choose() and use arena_ichoose() for
arena lookup during internal allocation.  This fixes huge_palloc() so
that it always succeeds during extent node allocation.

This regression was introduced by
66cd953514 (Do not allocate metadata via
non-auto arenas, nor tcaches.).
This commit is contained in:
Jason Evans 2016-05-03 15:00:42 -07:00
parent 21cda0dc42
commit 90827a3f3e
9 changed files with 42 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -550,7 +550,9 @@ size_t s2u_compute(size_t size);
size_t s2u_lookup(size_t size);
size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment);
arena_t *arena_choose(tsd_t *tsd, arena_t *arena, bool internal);
arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal);
arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena);
arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind,
bool refresh_if_missing);
arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing);
@ -788,7 +790,7 @@ sa2u(size_t size, size_t alignment)
/* Choose an arena based on a per-thread value. */
JEMALLOC_INLINE arena_t *
arena_choose(tsd_t *tsd, arena_t *arena, bool internal)
arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal)
{
arena_t *ret;
@ -802,6 +804,24 @@ arena_choose(tsd_t *tsd, arena_t *arena, bool internal)
return (ret);
}
JEMALLOC_INLINE arena_t *
arena_choose(tsd_t *tsd, arena_t *arena)
{
return (arena_choose_impl(tsd, arena, false));
}
JEMALLOC_INLINE arena_t *
arena_ichoose(tsd_t *tsd, arena_t *arena)
{
assert(tsd != NULL || arena != NULL);
if (tsd != NULL)
return (arena_choose_impl(tsd, NULL, true));
return (arena);
}
JEMALLOC_INLINE arena_tdata_t *
arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing)
{

2
include/jemalloc/internal/private_symbols.txt
View File

@ -10,6 +10,7 @@ arena_bitselm_get_mutable
arena_boot
arena_choose
arena_choose_hard
arena_choose_impl
arena_chunk_alloc_huge
arena_chunk_cache_maybe_insert
arena_chunk_cache_maybe_remove
@ -35,6 +36,7 @@ arena_decay_time_set
arena_dss_prec_get
arena_dss_prec_set
arena_get
arena_ichoose
arena_init
arena_lg_dirty_mult_default_get
arena_lg_dirty_mult_default_set

7
include/jemalloc/internal/tcache.h
View File

@ -293,7 +293,7 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) {
bool tcache_hard_success;
arena = arena_choose(tsd, arena, false);
arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL))
return (NULL);
@ -354,7 +354,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
* Only allocate one large object at a time, because it's quite
* expensive to create one and not use it.
*/
arena = arena_choose(tsd, arena, false);
arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL))
return (NULL);
@ -460,8 +460,7 @@ tcaches_get(tsd_t *tsd, unsigned ind)
{
tcaches_t *elm = &tcaches[ind];
if (unlikely(elm->tcache == NULL)) {
elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL,
false));
elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL));
}
return (elm->tcache);
}

4
src/arena.c
View File

@ -2649,7 +2649,7 @@ arena_malloc_hard(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind,
bool zero)
{
arena = arena_choose(tsd, arena, false);
arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL))
return (NULL);
@ -2674,7 +2674,7 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
assert(usize == PAGE_CEILING(usize));
arena = arena_choose(tsd, arena, false);
arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL))
return (NULL);

6
src/ckh.c
View File

@ -271,7 +271,7 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh)
goto label_return;
}
tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL,
true, arena_choose(tsd, NULL, true));
true, arena_ichoose(tsd, NULL));
if (tab == NULL) {
ret = true;
goto label_return;
@ -315,7 +315,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
return;
tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true,
arena_choose(tsd, NULL, true));
arena_ichoose(tsd, NULL));
if (tab == NULL) {
/*
* An OOM error isn't worth propagating, since it doesn't
@ -392,7 +392,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
goto label_return;
}
ckh->tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true,
arena_choose(tsd, NULL, true));
arena_ichoose(tsd, NULL));
if (ckh->tab == NULL) {
ret = true;
goto label_return;

2
src/ctl.c
View File

@ -1306,7 +1306,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
arena_t *oldarena;
unsigned newind, oldind;
oldarena = arena_choose(tsd, NULL, false);
oldarena = arena_choose(tsd, NULL);
if (oldarena == NULL)
return (EAGAIN);

5
src/huge.c
View File

@ -56,8 +56,9 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
assert(ausize >= chunksize);
/* Allocate an extent node with which to track the chunk. */
assert(tsd != NULL || arena != NULL);
node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
CACHELINE, false, NULL, true, arena_choose(tsd, NULL, true));
CACHELINE, false, NULL, true, arena_ichoose(tsd, arena));
if (node == NULL)
return (NULL);
@ -66,7 +67,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
* it is possible to make correct junk/zero fill decisions below.
*/
is_zeroed = zero;
arena = arena_choose(tsd, arena, false);
arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsd, arena,
usize, alignment, &is_zeroed)) == NULL) {
idalloctm(tsd, node, NULL, true, true);

2
src/prof.c
View File

@ -795,7 +795,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt)
/* Link a prof_tctx_t into gctx for this thread. */
ret.v = iallocztm(tsd, sizeof(prof_tctx_t),
size2index(sizeof(prof_tctx_t)), false, NULL, true,
arena_choose(tsd, NULL, true), true);
arena_ichoose(tsd, NULL), true);
if (ret.p == NULL) {
if (new_gctx)
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);

10
src/tcache.c
View File

@ -97,7 +97,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
assert(binind < NBINS);
assert(rem <= tbin->ncached);
arena = arena_choose(tsd, NULL, false);
arena = arena_choose(tsd, NULL);
assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena bin associated with the first object. */
@ -179,7 +179,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
assert(binind < nhbins);
assert(rem <= tbin->ncached);
arena = arena_choose(tsd, NULL, false);
arena = arena_choose(tsd, NULL);
assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena associated with the first object. */
@ -307,7 +307,7 @@ tcache_get_hard(tsd_t *tsd)
tcache_enabled_set(false); /* Memoize. */
return (NULL);
}
arena = arena_choose(tsd, NULL, false);
arena = arena_choose(tsd, NULL);
if (unlikely(arena == NULL))
return (NULL);
return (tcache_create(tsd, arena));
@ -359,7 +359,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
arena_t *arena;
unsigned i;
arena = arena_choose(tsd, NULL, false);
arena = arena_choose(tsd, NULL);
tcache_arena_dissociate(tsd, tcache, arena);
for (i = 0; i < NBINS; i++) {
@ -459,7 +459,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
return (true);
arena = arena_choose(tsd, NULL, true);
arena = arena_ichoose(tsd, NULL);
if (unlikely(arena == NULL))
return (true);
tcache = tcache_create(tsd, arena);