diff --git a/include/jemalloc/internal/arena_structs.h b/include/jemalloc/internal/arena_structs.h index dc4e3268..ed16337b 100644 --- a/include/jemalloc/internal/arena_structs.h +++ b/include/jemalloc/internal/arena_structs.h @@ -163,9 +163,6 @@ struct arena_s { arena_decay_t decay_dirty; /* dirty --> muzzy */ arena_decay_t decay_muzzy; /* muzzy --> retained */ - /* The grow info for the retained ecache. */ - ecache_grow_t ecache_grow; - /* * bins is used to store heaps of free regions. * diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 890f7b16..3b1a7659 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -30,6 +30,9 @@ struct pa_shard_s { /* The source of edata_t objects. */ edata_cache_t edata_cache; + /* The grow info for the retained ecache. */ + ecache_grow_t ecache_grow; + pa_shard_stats_t *stats; }; diff --git a/src/arena.c b/src/arena.c index d03bc72d..ced01d73 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1935,14 +1935,14 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, } } - malloc_mutex_lock(tsd_tsdn(tsd), &arena->ecache_grow.mtx); + malloc_mutex_lock(tsd_tsdn(tsd), &arena->pa_shard.ecache_grow.mtx); if (old_limit != NULL) { - *old_limit = sz_pind2sz(arena->ecache_grow.limit); + *old_limit = sz_pind2sz(arena->pa_shard.ecache_grow.limit); } if (new_limit != NULL) { - arena->ecache_grow.limit = new_ind; + arena->pa_shard.ecache_grow.limit = new_ind; } - malloc_mutex_unlock(tsd_tsdn(tsd), &arena->ecache_grow.mtx); + malloc_mutex_unlock(tsd_tsdn(tsd), &arena->pa_shard.ecache_grow.mtx); return false; } @@ -2042,10 +2042,6 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { goto label_error; } - if (ecache_grow_init(tsdn, &arena->ecache_grow)) { - goto label_error; - } - /* Initialize bins. */ uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t); atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); @@ -2176,7 +2172,7 @@ arena_prefork1(tsdn_t *tsdn, arena_t *arena) { void arena_prefork2(tsdn_t *tsdn, arena_t *arena) { - ecache_grow_prefork(tsdn, &arena->ecache_grow); + ecache_grow_prefork(tsdn, &arena->pa_shard.ecache_grow); } void @@ -2226,7 +2222,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_dirty); ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_muzzy); ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_retained); - ecache_grow_postfork_parent(tsdn, &arena->ecache_grow); + ecache_grow_postfork_parent(tsdn, &arena->pa_shard.ecache_grow); malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx); malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx); if (config_stats) { @@ -2272,7 +2268,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { ecache_postfork_child(tsdn, &arena->pa_shard.ecache_dirty); ecache_postfork_child(tsdn, &arena->pa_shard.ecache_muzzy); ecache_postfork_child(tsdn, &arena->pa_shard.ecache_retained); - ecache_grow_postfork_child(tsdn, &arena->ecache_grow); + ecache_grow_postfork_child(tsdn, &arena->pa_shard.ecache_grow); malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx); malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx); if (config_stats) { diff --git a/src/extent.c b/src/extent.c index 1b7f00fb..7c00525a 100644 --- a/src/extent.c +++ b/src/extent.c @@ -619,7 +619,7 @@ static edata_t * extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, size_t size, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { - malloc_mutex_assert_owner(tsdn, &arena->ecache_grow.mtx); + malloc_mutex_assert_owner(tsdn, &arena->pa_shard.ecache_grow.mtx); assert(!*zero || !slab); size_t alloc_size_min = size + PAGE_CEILING(alignment) - PAGE; @@ -632,15 +632,17 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, * satisfy this request. */ pszind_t egn_skip = 0; - size_t alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip); + size_t alloc_size = sz_pind2sz( + arena->pa_shard.ecache_grow.next + egn_skip); while (alloc_size < alloc_size_min) { egn_skip++; - if (arena->ecache_grow.next + egn_skip >= + if (arena->pa_shard.ecache_grow.next + egn_skip >= sz_psz2ind(SC_LARGE_MAXCLASS)) { /* Outside legal range. */ goto label_err; } - alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip); + alloc_size = sz_pind2sz( + arena->pa_shard.ecache_grow.next + egn_skip); } edata_t *edata = edata_cache_get(tsdn, &arena->pa_shard.edata_cache); @@ -735,14 +737,15 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, * Increment extent_grow_next if doing so wouldn't exceed the allowed * range. */ - if (arena->ecache_grow.next + egn_skip + 1 <= - arena->ecache_grow.limit) { - arena->ecache_grow.next += egn_skip + 1; + if (arena->pa_shard.ecache_grow.next + egn_skip + 1 <= + arena->pa_shard.ecache_grow.limit) { + arena->pa_shard.ecache_grow.next += egn_skip + 1; } else { - arena->ecache_grow.next = arena->ecache_grow.limit; + arena->pa_shard.ecache_grow.next + = arena->pa_shard.ecache_grow.limit; } /* All opportunities for failure are past. */ - malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); + malloc_mutex_unlock(tsdn, &arena->pa_shard.ecache_grow.mtx); if (config_prof) { /* Adjust gdump stats now that extent is final size. */ @@ -760,7 +763,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, return edata; label_err: - malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); + malloc_mutex_unlock(tsdn, &arena->pa_shard.ecache_grow.mtx); return NULL; } @@ -771,13 +774,13 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, assert(size != 0); assert(alignment != 0); - malloc_mutex_lock(tsdn, &arena->ecache_grow.mtx); + malloc_mutex_lock(tsdn, &arena->pa_shard.ecache_grow.mtx); edata_t *edata = extent_recycle(tsdn, arena, ehooks, &arena->pa_shard.ecache_retained, new_addr, size, alignment, slab, szind, zero, commit, true); if (edata != NULL) { - malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); + malloc_mutex_unlock(tsdn, &arena->pa_shard.ecache_grow.mtx); if (config_prof) { extent_gdump_add(tsdn, edata); } @@ -786,9 +789,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, alignment, slab, szind, zero, commit); /* extent_grow_retained() always releases extent_grow_mtx. */ } else { - malloc_mutex_unlock(tsdn, &arena->ecache_grow.mtx); + malloc_mutex_unlock(tsdn, &arena->pa_shard.ecache_grow.mtx); } - malloc_mutex_assert_not_owner(tsdn, &arena->ecache_grow.mtx); + malloc_mutex_assert_not_owner(tsdn, &arena->pa_shard.ecache_grow.mtx); return edata; } diff --git a/src/pa.c b/src/pa.c index 516ae1d3..5063d48f 100644 --- a/src/pa.c +++ b/src/pa.c @@ -38,6 +38,10 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, base_t *base, unsigned ind, return true; } + if (ecache_grow_init(tsdn, &shard->ecache_grow)) { + return true; + } + shard->stats = stats; memset(shard->stats, 0, sizeof(*shard->stats)); diff --git a/test/unit/retained.c b/test/unit/retained.c index 9ac83ef6..cf3de1ea 100644 --- a/test/unit/retained.c +++ b/test/unit/retained.c @@ -142,7 +142,7 @@ TEST_BEGIN(test_retained) { size_t usable = 0; size_t fragmented = 0; for (pszind_t pind = sz_psz2ind(HUGEPAGE); pind < - arena->ecache_grow.next; pind++) { + arena->pa_shard.ecache_grow.next; pind++) { size_t psz = sz_pind2sz(pind); size_t psz_fragmented = psz % esz; size_t psz_usable = psz - psz_fragmented;