project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

PA: Move in the arena edata_cache.

Browse Source
This commit is contained in:
David Goldblatt 2020-03-08 11:41:19 -07:00 committed by David Goldblatt
parent 8433ad84ea
commit 688fb3eb89
8 changed files with 48 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/jemalloc/internal/arena_structs.h
View File

@ -166,9 +166,6 @@ struct arena_s {
/* The grow info for the retained ecache. */ /* The grow info for the retained ecache. */
ecache_grow_t ecache_grow; ecache_grow_t ecache_grow;
/* The source of edata_t objects. */
edata_cache_t edata_cache;
/* /*
* bins is used to store heaps of free regions. * bins is used to store heaps of free regions.
* *

5
include/jemalloc/internal/pa.h
View File

@ -17,9 +17,12 @@ struct pa_shard_s {
ecache_t ecache_dirty; ecache_t ecache_dirty;
ecache_t ecache_muzzy; ecache_t ecache_muzzy;
ecache_t ecache_retained; ecache_t ecache_retained;
/* The source of edata_t objects. */
edata_cache_t edata_cache;
}; };
/* Returns true on error. */ /* Returns true on error. */
bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, unsigned ind); bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, base_t *base, unsigned ind);
#endif /* JEMALLOC_INTERNAL_PA_H */ #endif /* JEMALLOC_INTERNAL_PA_H */

16
src/arena.c
View File

@ -101,7 +101,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
ecache_npages_get(&arena->pa_shard.ecache_retained) << LG_PAGE); ecache_npages_get(&arena->pa_shard.ecache_retained) << LG_PAGE);
atomic_store_zu(&astats->edata_avail, atomic_store_zu(&astats->edata_avail,
atomic_load_zu(&arena->edata_cache.count, ATOMIC_RELAXED), atomic_load_zu(&arena->pa_shard.edata_cache.count, ATOMIC_RELAXED),
ATOMIC_RELAXED); ATOMIC_RELAXED);
arena_stats_accum_u64(&astats->decay_dirty.npurge, arena_stats_accum_u64(&astats->decay_dirty.npurge,
@ -228,7 +228,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
/* Gather per arena mutex profiling data. */ /* Gather per arena mutex profiling data. */
READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large); READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
READ_ARENA_MUTEX_PROF_DATA(edata_cache.mtx, READ_ARENA_MUTEX_PROF_DATA(pa_shard.edata_cache.mtx,
arena_prof_mutex_extent_avail) arena_prof_mutex_extent_avail)
READ_ARENA_MUTEX_PROF_DATA(pa_shard.ecache_dirty.mtx, READ_ARENA_MUTEX_PROF_DATA(pa_shard.ecache_dirty.mtx,
arena_prof_mutex_extents_dirty) arena_prof_mutex_extents_dirty)
@ -2027,7 +2027,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
goto label_error; goto label_error;
} }
if (pa_shard_init(tsdn, &arena->pa_shard, ind)) { if (pa_shard_init(tsdn, &arena->pa_shard, base, ind)) {
goto label_error; goto label_error;
} }
@ -2044,10 +2044,6 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
goto label_error; goto label_error;
} }
if (edata_cache_init(&arena->edata_cache, base)) {
goto label_error;
}
/* Initialize bins. */ /* Initialize bins. */
uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t); uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t);
atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE);
@ -2190,7 +2186,7 @@ arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
void void
arena_prefork4(tsdn_t *tsdn, arena_t *arena) { arena_prefork4(tsdn_t *tsdn, arena_t *arena) {
edata_cache_prefork(tsdn, &arena->edata_cache); edata_cache_prefork(tsdn, &arena->pa_shard.edata_cache);
} }
void void
@ -2224,7 +2220,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
} }
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
base_postfork_parent(tsdn, arena->base); base_postfork_parent(tsdn, arena->base);
edata_cache_postfork_parent(tsdn, &arena->edata_cache); edata_cache_postfork_parent(tsdn, &arena->pa_shard.edata_cache);
ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_dirty); ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_dirty);
ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_muzzy); ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_muzzy);
ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_retained); ecache_postfork_parent(tsdn, &arena->pa_shard.ecache_retained);
@ -2270,7 +2266,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
} }
malloc_mutex_postfork_child(tsdn, &arena->large_mtx); malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
base_postfork_child(tsdn, arena->base); base_postfork_child(tsdn, arena->base);
edata_cache_postfork_child(tsdn, &arena->edata_cache); edata_cache_postfork_child(tsdn, &arena->pa_shard.edata_cache);
ecache_postfork_child(tsdn, &arena->pa_shard.ecache_dirty); ecache_postfork_child(tsdn, &arena->pa_shard.ecache_dirty);
ecache_postfork_child(tsdn, &arena->pa_shard.ecache_muzzy); ecache_postfork_child(tsdn, &arena->pa_shard.ecache_muzzy);
ecache_postfork_child(tsdn, &arena->pa_shard.ecache_retained); ecache_postfork_child(tsdn, &arena->pa_shard.ecache_retained);

2
src/ctl.c
View File

@ -3072,7 +3072,7 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
continue; continue;
} }
MUTEX_PROF_RESET(arena->large_mtx); MUTEX_PROF_RESET(arena->large_mtx);
MUTEX_PROF_RESET(arena->edata_cache.mtx); MUTEX_PROF_RESET(arena->pa_shard.edata_cache.mtx);
MUTEX_PROF_RESET(arena->pa_shard.ecache_dirty.mtx); MUTEX_PROF_RESET(arena->pa_shard.ecache_dirty.mtx);
MUTEX_PROF_RESET(arena->pa_shard.ecache_muzzy.mtx); MUTEX_PROF_RESET(arena->pa_shard.ecache_muzzy.mtx);
MUTEX_PROF_RESET(arena->pa_shard.ecache_retained.mtx); MUTEX_PROF_RESET(arena->pa_shard.ecache_retained.mtx);

42
src/extent.c
View File

@ -157,8 +157,8 @@ ecache_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
break; break;
} }
/* Try to coalesce. */ /* Try to coalesce. */
if (extent_try_delayed_coalesce(tsdn, &arena->edata_cache, if (extent_try_delayed_coalesce(tsdn,
ehooks, ecache, edata)) { &arena->pa_shard.edata_cache, ehooks, ecache, edata)) {
break; break;
} }
/* /*
@ -212,7 +212,7 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_size_get(edata), growing_retained); edata_size_get(edata), growing_retained);
} }
} }
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
} }
static void static void
@ -462,9 +462,9 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
/* Split the lead. */ /* Split the lead. */
if (leadsize != 0) { if (leadsize != 0) {
*lead = *edata; *lead = *edata;
*edata = extent_split_impl(tsdn, &arena->edata_cache, ehooks, *edata = extent_split_impl(tsdn, &arena->pa_shard.edata_cache,
*lead, leadsize, SC_NSIZES, false, size + trailsize, szind, ehooks, *lead, leadsize, SC_NSIZES, false, size + trailsize,
slab, growing_retained); szind, slab, growing_retained);
if (*edata == NULL) { if (*edata == NULL) {
*to_leak = *lead; *to_leak = *lead;
*lead = NULL; *lead = NULL;
@ -474,9 +474,9 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
/* Split the trail. */ /* Split the trail. */
if (trailsize != 0) { if (trailsize != 0) {
*trail = extent_split_impl(tsdn, &arena->edata_cache, ehooks, *trail = extent_split_impl(tsdn, &arena->pa_shard.edata_cache,
*edata, size, szind, slab, trailsize, SC_NSIZES, false, ehooks, *edata, size, szind, slab, trailsize, SC_NSIZES,
growing_retained); false, growing_retained);
if (*trail == NULL) { if (*trail == NULL) {
*to_leak = *edata; *to_leak = *edata;
*to_salvage = *lead; *to_salvage = *lead;
@ -643,7 +643,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip); alloc_size = sz_pind2sz(arena->ecache_grow.next + egn_skip);
} }
edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache); edata_t *edata = edata_cache_get(tsdn, &arena->pa_shard.edata_cache);
if (edata == NULL) { if (edata == NULL) {
goto label_err; goto label_err;
} }
@ -654,7 +654,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
&committed); &committed);
if (ptr == NULL) { if (ptr == NULL) {
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
goto label_err; goto label_err;
} }
@ -663,7 +663,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
committed, true, EXTENT_IS_HEAD); committed, true, EXTENT_IS_HEAD);
if (extent_register_no_gdump_add(tsdn, edata)) { if (extent_register_no_gdump_add(tsdn, edata)) {
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
goto label_err; goto label_err;
} }
@ -800,7 +800,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache); edata_t *edata = edata_cache_get(tsdn, &arena->pa_shard.edata_cache);
if (edata == NULL) { if (edata == NULL) {
return NULL; return NULL;
} }
@ -808,14 +808,14 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *addr = ehooks_alloc(tsdn, ehooks, new_addr, size, palignment, void *addr = ehooks_alloc(tsdn, ehooks, new_addr, size, palignment,
zero, commit); zero, commit);
if (addr == NULL) { if (addr == NULL) {
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
return NULL; return NULL;
} }
edata_init(edata, arena_ind_get(arena), addr, size, slab, szind, edata_init(edata, arena_ind_get(arena), addr, size, slab, szind,
arena_extent_sn_next(arena), extent_state_active, *zero, *commit, arena_extent_sn_next(arena), extent_state_active, *zero, *commit,
true, EXTENT_NOT_HEAD); true, EXTENT_NOT_HEAD);
if (extent_register(tsdn, edata)) { if (extent_register(tsdn, edata)) {
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
return NULL; return NULL;
} }
@ -971,8 +971,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
emap_assert_mapped(tsdn, &emap_global, edata); emap_assert_mapped(tsdn, &emap_global, edata);
if (!ecache->delay_coalesce) { if (!ecache->delay_coalesce) {
edata = extent_try_coalesce(tsdn, &arena->edata_cache, ehooks, edata = extent_try_coalesce(tsdn, &arena->pa_shard.edata_cache,
ecache, edata, NULL, growing_retained); ehooks, ecache, edata, NULL, growing_retained);
} else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) { } else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) {
assert(ecache == &arena->pa_shard.ecache_dirty); assert(ecache == &arena->pa_shard.ecache_dirty);
/* Always coalesce large extents eagerly. */ /* Always coalesce large extents eagerly. */
@ -980,7 +980,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, ecache_t *ecache,
do { do {
assert(edata_state_get(edata) == extent_state_active); assert(edata_state_get(edata) == extent_state_active);
edata = extent_try_coalesce_large(tsdn, edata = extent_try_coalesce_large(tsdn,
&arena->edata_cache, ehooks, ecache, edata, &arena->pa_shard.edata_cache, ehooks, ecache, edata,
&coalesced, growing_retained); &coalesced, growing_retained);
} while (coalesced); } while (coalesced);
if (edata_size_get(edata) >= oversize_threshold && if (edata_size_get(edata) >= oversize_threshold &&
@ -1004,7 +1004,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
if (extent_register(tsdn, edata)) { if (extent_register(tsdn, edata)) {
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
return; return;
} }
extent_dalloc_wrapper(tsdn, arena, ehooks, edata); extent_dalloc_wrapper(tsdn, arena, ehooks, edata);
@ -1027,7 +1027,7 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_size_get(edata), edata_committed_get(edata)); edata_size_get(edata), edata_committed_get(edata));
if (!err) { if (!err) {
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
} }
return err; return err;
@ -1097,7 +1097,7 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
ehooks_destroy(tsdn, ehooks, edata_base_get(edata), ehooks_destroy(tsdn, ehooks, edata_base_get(edata),
edata_size_get(edata), edata_committed_get(edata)); edata_size_get(edata), edata_committed_get(edata));
edata_cache_put(tsdn, &arena->edata_cache, edata); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, edata);
} }
static bool static bool

6
src/extent_dss.c
View File

@ -123,7 +123,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
return NULL; return NULL;
} }
gap = edata_cache_get(tsdn, &arena->edata_cache); gap = edata_cache_get(tsdn, &arena->pa_shard.edata_cache);
if (gap == NULL) { if (gap == NULL) {
return NULL; return NULL;
} }
@ -189,7 +189,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
extent_dalloc_gap(tsdn, arena, gap); extent_dalloc_gap(tsdn, arena, gap);
} else { } else {
edata_cache_put(tsdn, edata_cache_put(tsdn,
&arena->edata_cache, gap); &arena->pa_shard.edata_cache, gap);
} }
if (!*commit) { if (!*commit) {
*commit = pages_decommit(ret, size); *commit = pages_decommit(ret, size);
@ -225,7 +225,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
} }
label_oom: label_oom:
extent_dss_extending_finish(); extent_dss_extending_finish();
edata_cache_put(tsdn, &arena->edata_cache, gap); edata_cache_put(tsdn, &arena->pa_shard.edata_cache, gap);
return NULL; return NULL;
} }

11
src/large.c
View File

@ -81,9 +81,10 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
/* Split excess pages. */ /* Split excess pages. */
if (diff != 0) { if (diff != 0) {
edata_t *trail = extent_split_wrapper(tsdn, &arena->edata_cache, edata_t *trail = extent_split_wrapper(tsdn,
ehooks, edata, usize + sz_large_pad, sz_size2index(usize), &arena->pa_shard.edata_cache, ehooks, edata,
false, diff, SC_NSIZES, false); usize + sz_large_pad, sz_size2index(usize), false, diff,
SC_NSIZES, false);
if (trail == NULL) { if (trail == NULL) {
return true; return true;
} }
@ -140,8 +141,8 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
} }
} }
if (extent_merge_wrapper(tsdn, ehooks, &arena->edata_cache, edata, if (extent_merge_wrapper(tsdn, ehooks, &arena->pa_shard.edata_cache,
trail)) { edata, trail)) {
extent_dalloc_wrapper(tsdn, arena, ehooks, trail); extent_dalloc_wrapper(tsdn, arena, ehooks, trail);
return true; return true;
} }

8
src/pa.c
View File

@ -2,7 +2,9 @@
#include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/jemalloc_internal_includes.h"
bool bool
pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, unsigned ind) { pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, base_t *base, unsigned ind) {
/* This will change eventually, but for now it should hold. */
assert(base_ind_get(base) == ind);
/* /*
* Delay coalescing for dirty extents despite the disruptive effect on * Delay coalescing for dirty extents despite the disruptive effect on
* memory layout for best-fit extent allocation, since cached extents * memory layout for best-fit extent allocation, since cached extents
@ -31,5 +33,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, unsigned ind) {
ind, /* delay_coalesce */ false)) { ind, /* delay_coalesce */ false)) {
return true; return true;
} }
if (edata_cache_init(&shard->edata_cache, base)) {
return true;
}
return false; return false;
} }