Fix huge_palloc() regression.

Split arena_choose() into arena_[i]choose() and use arena_ichoose() for
arena lookup during internal allocation.  This fixes huge_palloc() so
that it always succeeds during extent node allocation.

This regression was introduced by
66cd953514 (Do not allocate metadata via
non-auto arenas, nor tcaches.).
This commit is contained in:
Jason Evans 2016-05-03 15:00:42 -07:00
parent 21cda0dc42
commit 90827a3f3e
9 changed files with 42 additions and 20 deletions

View File

@ -550,7 +550,9 @@ size_t s2u_compute(size_t size);
size_t s2u_lookup(size_t size); size_t s2u_lookup(size_t size);
size_t s2u(size_t size); size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment); size_t sa2u(size_t size, size_t alignment);
arena_t *arena_choose(tsd_t *tsd, arena_t *arena, bool internal); arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal);
arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena);
arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind,
bool refresh_if_missing); bool refresh_if_missing);
arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing); arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing);
@ -788,7 +790,7 @@ sa2u(size_t size, size_t alignment)
/* Choose an arena based on a per-thread value. */ /* Choose an arena based on a per-thread value. */
JEMALLOC_INLINE arena_t * JEMALLOC_INLINE arena_t *
arena_choose(tsd_t *tsd, arena_t *arena, bool internal) arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal)
{ {
arena_t *ret; arena_t *ret;
@ -802,6 +804,24 @@ arena_choose(tsd_t *tsd, arena_t *arena, bool internal)
return (ret); return (ret);
} }
JEMALLOC_INLINE arena_t *
arena_choose(tsd_t *tsd, arena_t *arena)
{
return (arena_choose_impl(tsd, arena, false));
}
JEMALLOC_INLINE arena_t *
arena_ichoose(tsd_t *tsd, arena_t *arena)
{
assert(tsd != NULL || arena != NULL);
if (tsd != NULL)
return (arena_choose_impl(tsd, NULL, true));
return (arena);
}
JEMALLOC_INLINE arena_tdata_t * JEMALLOC_INLINE arena_tdata_t *
arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing)
{ {

View File

@ -10,6 +10,7 @@ arena_bitselm_get_mutable
arena_boot arena_boot
arena_choose arena_choose
arena_choose_hard arena_choose_hard
arena_choose_impl
arena_chunk_alloc_huge arena_chunk_alloc_huge
arena_chunk_cache_maybe_insert arena_chunk_cache_maybe_insert
arena_chunk_cache_maybe_remove arena_chunk_cache_maybe_remove
@ -35,6 +36,7 @@ arena_decay_time_set
arena_dss_prec_get arena_dss_prec_get
arena_dss_prec_set arena_dss_prec_set
arena_get arena_get
arena_ichoose
arena_init arena_init
arena_lg_dirty_mult_default_get arena_lg_dirty_mult_default_get
arena_lg_dirty_mult_default_set arena_lg_dirty_mult_default_set

View File

@ -293,7 +293,7 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
assert(tcache_success == (ret != NULL)); assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) { if (unlikely(!tcache_success)) {
bool tcache_hard_success; bool tcache_hard_success;
arena = arena_choose(tsd, arena, false); arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (NULL); return (NULL);
@ -354,7 +354,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
* Only allocate one large object at a time, because it's quite * Only allocate one large object at a time, because it's quite
* expensive to create one and not use it. * expensive to create one and not use it.
*/ */
arena = arena_choose(tsd, arena, false); arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (NULL); return (NULL);
@ -460,8 +460,7 @@ tcaches_get(tsd_t *tsd, unsigned ind)
{ {
tcaches_t *elm = &tcaches[ind]; tcaches_t *elm = &tcaches[ind];
if (unlikely(elm->tcache == NULL)) { if (unlikely(elm->tcache == NULL)) {
elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL, elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL));
false));
} }
return (elm->tcache); return (elm->tcache);
} }

View File

@ -2649,7 +2649,7 @@ arena_malloc_hard(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind,
bool zero) bool zero)
{ {
arena = arena_choose(tsd, arena, false); arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (NULL); return (NULL);
@ -2674,7 +2674,7 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
assert(usize == PAGE_CEILING(usize)); assert(usize == PAGE_CEILING(usize));
arena = arena_choose(tsd, arena, false); arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (NULL); return (NULL);

View File

@ -271,7 +271,7 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh)
goto label_return; goto label_return;
} }
tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL,
true, arena_choose(tsd, NULL, true)); true, arena_ichoose(tsd, NULL));
if (tab == NULL) { if (tab == NULL) {
ret = true; ret = true;
goto label_return; goto label_return;
@ -315,7 +315,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh)
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
return; return;
tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true, tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true,
arena_choose(tsd, NULL, true)); arena_ichoose(tsd, NULL));
if (tab == NULL) { if (tab == NULL) {
/* /*
* An OOM error isn't worth propagating, since it doesn't * An OOM error isn't worth propagating, since it doesn't
@ -392,7 +392,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
goto label_return; goto label_return;
} }
ckh->tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true, ckh->tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true,
arena_choose(tsd, NULL, true)); arena_ichoose(tsd, NULL));
if (ckh->tab == NULL) { if (ckh->tab == NULL) {
ret = true; ret = true;
goto label_return; goto label_return;

View File

@ -1306,7 +1306,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
arena_t *oldarena; arena_t *oldarena;
unsigned newind, oldind; unsigned newind, oldind;
oldarena = arena_choose(tsd, NULL, false); oldarena = arena_choose(tsd, NULL);
if (oldarena == NULL) if (oldarena == NULL)
return (EAGAIN); return (EAGAIN);

View File

@ -56,8 +56,9 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
assert(ausize >= chunksize); assert(ausize >= chunksize);
/* Allocate an extent node with which to track the chunk. */ /* Allocate an extent node with which to track the chunk. */
assert(tsd != NULL || arena != NULL);
node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)),
CACHELINE, false, NULL, true, arena_choose(tsd, NULL, true)); CACHELINE, false, NULL, true, arena_ichoose(tsd, arena));
if (node == NULL) if (node == NULL)
return (NULL); return (NULL);
@ -66,7 +67,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
* it is possible to make correct junk/zero fill decisions below. * it is possible to make correct junk/zero fill decisions below.
*/ */
is_zeroed = zero; is_zeroed = zero;
arena = arena_choose(tsd, arena, false); arena = arena_choose(tsd, arena);
if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsd, arena, if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsd, arena,
usize, alignment, &is_zeroed)) == NULL) { usize, alignment, &is_zeroed)) == NULL) {
idalloctm(tsd, node, NULL, true, true); idalloctm(tsd, node, NULL, true, true);

View File

@ -795,7 +795,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt)
/* Link a prof_tctx_t into gctx for this thread. */ /* Link a prof_tctx_t into gctx for this thread. */
ret.v = iallocztm(tsd, sizeof(prof_tctx_t), ret.v = iallocztm(tsd, sizeof(prof_tctx_t),
size2index(sizeof(prof_tctx_t)), false, NULL, true, size2index(sizeof(prof_tctx_t)), false, NULL, true,
arena_choose(tsd, NULL, true), true); arena_ichoose(tsd, NULL), true);
if (ret.p == NULL) { if (ret.p == NULL) {
if (new_gctx) if (new_gctx)
prof_gctx_try_destroy(tsd, tdata, gctx, tdata); prof_gctx_try_destroy(tsd, tdata, gctx, tdata);

View File

@ -97,7 +97,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
assert(binind < NBINS); assert(binind < NBINS);
assert(rem <= tbin->ncached); assert(rem <= tbin->ncached);
arena = arena_choose(tsd, NULL, false); arena = arena_choose(tsd, NULL);
assert(arena != NULL); assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena bin associated with the first object. */ /* Lock the arena bin associated with the first object. */
@ -179,7 +179,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
assert(binind < nhbins); assert(binind < nhbins);
assert(rem <= tbin->ncached); assert(rem <= tbin->ncached);
arena = arena_choose(tsd, NULL, false); arena = arena_choose(tsd, NULL);
assert(arena != NULL); assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena associated with the first object. */ /* Lock the arena associated with the first object. */
@ -307,7 +307,7 @@ tcache_get_hard(tsd_t *tsd)
tcache_enabled_set(false); /* Memoize. */ tcache_enabled_set(false); /* Memoize. */
return (NULL); return (NULL);
} }
arena = arena_choose(tsd, NULL, false); arena = arena_choose(tsd, NULL);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (NULL); return (NULL);
return (tcache_create(tsd, arena)); return (tcache_create(tsd, arena));
@ -359,7 +359,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
arena_t *arena; arena_t *arena;
unsigned i; unsigned i;
arena = arena_choose(tsd, NULL, false); arena = arena_choose(tsd, NULL);
tcache_arena_dissociate(tsd, tcache, arena); tcache_arena_dissociate(tsd, tcache, arena);
for (i = 0; i < NBINS; i++) { for (i = 0; i < NBINS; i++) {
@ -459,7 +459,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
return (true); return (true);
arena = arena_choose(tsd, NULL, true); arena = arena_ichoose(tsd, NULL);
if (unlikely(arena == NULL)) if (unlikely(arena == NULL))
return (true); return (true);
tcache = tcache_create(tsd, arena); tcache = tcache_create(tsd, arena);