project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Protect extents_dirty access with extents_mtx.

Browse Source 
This fixes race conditions during purging.
This commit is contained in:
Jason Evans 2016-09-22 11:57:28 -07:00
parent bc49157d21
commit f6d01ff4b7
5 changed files with 112 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/jemalloc/internal/arena.h
View File

@ -176,12 +176,6 @@ struct arena_s {
*/ */
size_t ndirty; size_t ndirty;
/*
* Ring sentinel used to track unused dirty memory. Dirty memory is
* managed as an LRU of cached extents.
*/
extent_t extents_dirty;
/* /*
* Approximate time in seconds from the creation of a set of unused * Approximate time in seconds from the creation of a set of unused
* dirty pages until an equivalent set of unused dirty pages is purged * dirty pages until an equivalent set of unused dirty pages is purged
@ -240,7 +234,12 @@ struct arena_s {
*/ */
extent_heap_t extents_cached[NPSIZES]; extent_heap_t extents_cached[NPSIZES];
extent_heap_t extents_retained[NPSIZES]; extent_heap_t extents_retained[NPSIZES];
/* Protects extents_cached and extents_retained. */ /*
* Ring sentinel used to track unused dirty memory. Dirty memory is
* managed as an LRU of cached extents.
*/
extent_t extents_dirty;
/* Protects extents_{cached,retained,dirty}. */
malloc_mutex_t extents_mtx; malloc_mutex_t extents_mtx;
/* User-configurable extent hook functions. */ /* User-configurable extent hook functions. */
@ -287,10 +286,10 @@ extent_t *arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
size_t alignment, bool *zero); size_t alignment, bool *zero);
void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent); extent_hooks_t **r_extent_hooks, extent_t *extent);
void arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent, void arena_extent_cache_maybe_insert(tsdn_t *tsdn, arena_t *arena,
bool cache); extent_t *extent, bool cache);
void arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent, void arena_extent_cache_maybe_remove(tsdn_t *tsdn, arena_t *arena,
bool cache); extent_t *extent, bool cache);
extent_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
size_t usize, size_t alignment, bool *zero); size_t usize, size_t alignment, bool *zero);
void arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, void arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena,

3
include/jemalloc/internal/extent.h
View File

@ -99,6 +99,9 @@ size_t extent_size_quantize_ceil(size_t size);
ph_proto(, extent_heap_, extent_heap_t, extent_t) ph_proto(, extent_heap_, extent_heap_t, extent_t)
extent_t *extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab);
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab); size_t alignment, bool *zero, bool slab);

1
include/jemalloc/internal/private_symbols.txt
View File

@ -145,6 +145,7 @@ extent_addr_randomize
extent_addr_set extent_addr_set
extent_alloc extent_alloc
extent_alloc_cache extent_alloc_cache
extent_alloc_cache_locked
extent_alloc_dss extent_alloc_dss
extent_alloc_mmap extent_alloc_mmap
extent_alloc_wrapper extent_alloc_wrapper

29
src/arena.c
View File

@ -101,9 +101,12 @@ arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
} }
void void
arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache) arena_extent_cache_maybe_insert(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool cache)
{ {
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
if (cache) { if (cache) {
extent_ring_insert(&arena->extents_dirty, extent); extent_ring_insert(&arena->extents_dirty, extent);
arena->ndirty += arena_extent_dirty_npages(extent); arena->ndirty += arena_extent_dirty_npages(extent);
@ -111,9 +114,12 @@ arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache)
} }
void void
arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent, bool dirty) arena_extent_cache_maybe_remove(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool dirty)
{ {
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
if (dirty) { if (dirty) {
extent_ring_remove(extent); extent_ring_remove(extent);
assert(arena->ndirty >= arena_extent_dirty_npages(extent)); assert(arena->ndirty >= arena_extent_dirty_npages(extent));
@ -727,6 +733,8 @@ arena_dirty_count(tsdn_t *tsdn, arena_t *arena)
extent_t *extent; extent_t *extent;
size_t ndirty = 0; size_t ndirty = 0;
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
for (extent = qr_next(&arena->extents_dirty, qr_link); extent != for (extent = qr_next(&arena->extents_dirty, qr_link); extent !=
&arena->extents_dirty; extent = qr_next(extent, qr_link)) &arena->extents_dirty; extent = qr_next(extent, qr_link))
ndirty += extent_size_get(extent) >> LG_PAGE; ndirty += extent_size_get(extent) >> LG_PAGE;
@ -741,6 +749,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_t *extent, *next; extent_t *extent, *next;
size_t nstashed = 0; size_t nstashed = 0;
malloc_mutex_lock(tsdn, &arena->extents_mtx);
/* Stash extents according to ndirty_limit. */ /* Stash extents according to ndirty_limit. */
for (extent = qr_next(&arena->extents_dirty, qr_link); extent != for (extent = qr_next(&arena->extents_dirty, qr_link); extent !=
&arena->extents_dirty; extent = next) { &arena->extents_dirty; extent = next) {
@ -756,9 +766,9 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
next = qr_next(extent, qr_link); next = qr_next(extent, qr_link);
/* Allocate. */ /* Allocate. */
zero = false; zero = false;
textent = arena_extent_cache_alloc_locked(tsdn, arena, textent = extent_alloc_cache_locked(tsdn, arena, r_extent_hooks,
r_extent_hooks, extent_base_get(extent), extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), 0, CACHELINE, &zero, false); CACHELINE, &zero, false);
assert(textent == extent); assert(textent == extent);
assert(zero == extent_zeroed_get(extent)); assert(zero == extent_zeroed_get(extent));
extent_ring_remove(extent); extent_ring_remove(extent);
@ -770,6 +780,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
break; break;
} }
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (nstashed); return (nstashed);
} }
@ -1788,9 +1799,6 @@ arena_new(tsdn_t *tsdn, unsigned ind)
arena->nactive = 0; arena->nactive = 0;
arena->ndirty = 0; arena->ndirty = 0;
extent_init(&arena->extents_dirty, arena, NULL, 0, 0, false, false,
false, false);
if (opt_purge == purge_mode_decay) if (opt_purge == purge_mode_decay)
arena_decay_init(arena, arena_decay_time_default_get()); arena_decay_init(arena, arena_decay_time_default_get());
@ -1804,12 +1812,15 @@ arena_new(tsdn_t *tsdn, unsigned ind)
extent_heap_new(&arena->extents_retained[i]); extent_heap_new(&arena->extents_retained[i]);
} }
arena->extent_hooks = (extent_hooks_t *)&extent_hooks_default; extent_init(&arena->extents_dirty, arena, NULL, 0, 0, false, false,
false, false);
if (malloc_mutex_init(&arena->extents_mtx, "arena_extents", if (malloc_mutex_init(&arena->extents_mtx, "arena_extents",
WITNESS_RANK_ARENA_EXTENTS)) WITNESS_RANK_ARENA_EXTENTS))
return (NULL); return (NULL);
arena->extent_hooks = (extent_hooks_t *)&extent_hooks_default;
ql_new(&arena->extent_cache); ql_new(&arena->extent_cache);
if (malloc_mutex_init(&arena->extent_cache_mtx, "arena_extent_cache", if (malloc_mutex_init(&arena->extent_cache_mtx, "arena_extent_cache",
WITNESS_RANK_ARENA_EXTENT_CACHE)) WITNESS_RANK_ARENA_EXTENT_CACHE))

116
src/extent.c
View File

@ -191,18 +191,26 @@ extent_ad_comp(const extent_t *a, const extent_t *b)
ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_ad_comp) ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_ad_comp)
static void static void
extent_heaps_insert(extent_heap_t extent_heaps[NPSIZES], extent_t *extent) extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES],
extent_t *extent)
{ {
size_t psz = extent_size_quantize_floor(extent_size_get(extent)); size_t psz = extent_size_quantize_floor(extent_size_get(extent));
pszind_t pind = psz2ind(psz); pszind_t pind = psz2ind(psz);
malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx);
extent_heap_insert(&extent_heaps[pind], extent); extent_heap_insert(&extent_heaps[pind], extent);
} }
static void static void
extent_heaps_remove(extent_heap_t extent_heaps[NPSIZES], extent_t *extent) extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES],
extent_t *extent)
{ {
size_t psz = extent_size_quantize_floor(extent_size_get(extent)); size_t psz = extent_size_quantize_floor(extent_size_get(extent));
pszind_t pind = psz2ind(psz); pszind_t pind = psz2ind(psz);
malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx);
extent_heap_remove(&extent_heaps[pind], extent); extent_heap_remove(&extent_heaps[pind], extent);
} }
@ -381,9 +389,9 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
static extent_t * static extent_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr, extent_heap_t extent_heaps[NPSIZES], bool locked, bool cache,
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
bool slab) bool *commit, bool slab)
{ {
extent_t *extent; extent_t *extent;
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
@ -398,7 +406,8 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
/* Beware size_t wrap-around. */ /* Beware size_t wrap-around. */
if (alloc_size < usize) if (alloc_size < usize)
return (NULL); return (NULL);
malloc_mutex_lock(tsdn, &arena->extents_mtx); if (!locked)
malloc_mutex_lock(tsdn, &arena->extents_mtx);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
if (new_addr != NULL) { if (new_addr != NULL) {
rtree_elm_t *elm; rtree_elm_t *elm;
@ -419,11 +428,12 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent = extent_first_best_fit(arena, extent_heaps, alloc_size); extent = extent_first_best_fit(arena, extent_heaps, alloc_size);
if (extent == NULL || (new_addr != NULL && extent_size_get(extent) < if (extent == NULL || (new_addr != NULL && extent_size_get(extent) <
size)) { size)) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx); if (!locked)
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (NULL); return (NULL);
} }
extent_heaps_remove(extent_heaps, extent); extent_heaps_remove(tsdn, extent_heaps, extent);
arena_extent_cache_maybe_remove(arena, extent, cache); arena_extent_cache_maybe_remove(tsdn, arena, extent, cache);
leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent), leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent),
PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent); PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent);
@ -444,11 +454,12 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
if (extent == NULL) { if (extent == NULL) {
extent_deregister(tsdn, lead); extent_deregister(tsdn, lead);
extent_leak(tsdn, arena, r_extent_hooks, cache, lead); extent_leak(tsdn, arena, r_extent_hooks, cache, lead);
malloc_mutex_unlock(tsdn, &arena->extents_mtx); if (!locked)
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (NULL); return (NULL);
} }
extent_heaps_insert(extent_heaps, lead); extent_heaps_insert(tsdn, extent_heaps, lead);
arena_extent_cache_maybe_insert(arena, lead, cache); arena_extent_cache_maybe_insert(tsdn, arena, lead, cache);
} }
/* Split the trail. */ /* Split the trail. */
@ -459,11 +470,12 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_deregister(tsdn, extent); extent_deregister(tsdn, extent);
extent_leak(tsdn, arena, r_extent_hooks, cache, extent_leak(tsdn, arena, r_extent_hooks, cache,
extent); extent);
malloc_mutex_unlock(tsdn, &arena->extents_mtx); if (!locked)
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (NULL); return (NULL);
} }
extent_heaps_insert(extent_heaps, trail); extent_heaps_insert(tsdn, extent_heaps, trail);
arena_extent_cache_maybe_insert(arena, trail, cache); arena_extent_cache_maybe_insert(tsdn, arena, trail, cache);
} else if (leadsize == 0) { } else if (leadsize == 0) {
/* /*
* Splitting causes usize to be set as a side effect, but no * Splitting causes usize to be set as a side effect, but no
@ -474,7 +486,8 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
if (!extent_committed_get(extent) && extent_commit_wrapper(tsdn, arena, if (!extent_committed_get(extent) && extent_commit_wrapper(tsdn, arena,
r_extent_hooks, extent, 0, extent_size_get(extent))) { r_extent_hooks, extent, 0, extent_size_get(extent))) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx); if (!locked)
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
extent_record(tsdn, arena, r_extent_hooks, extent_heaps, cache, extent_record(tsdn, arena, r_extent_hooks, extent_heaps, cache,
extent); extent);
return (NULL); return (NULL);
@ -488,7 +501,8 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_interior_register(tsdn, rtree_ctx, extent); extent_interior_register(tsdn, rtree_ctx, extent);
} }
malloc_mutex_unlock(tsdn, &arena->extents_mtx); if (!locked)
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
if (*zero) { if (*zero) {
if (!extent_zeroed_get(extent)) { if (!extent_zeroed_get(extent)) {
@ -540,27 +554,51 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
return (NULL); return (NULL);
} }
extent_t * static extent_t *
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_alloc_cache_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, bool locked, void *new_addr, size_t usize,
size_t alignment, bool *zero, bool slab) size_t pad, size_t alignment, bool *zero, bool slab)
{ {
extent_t *extent; extent_t *extent;
bool commit; bool commit;
assert(usize + pad != 0); assert(usize + pad != 0);
assert(alignment != 0); assert(alignment != 0);
if (locked)
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
commit = true; commit = true;
extent = extent_recycle(tsdn, arena, r_extent_hooks, extent = extent_recycle(tsdn, arena, r_extent_hooks,
arena->extents_cached, true, new_addr, usize, pad, alignment, zero, arena->extents_cached, locked, true, new_addr, usize, pad,
&commit, slab); alignment, zero, &commit, slab);
if (extent == NULL) if (extent == NULL)
return (NULL); return (NULL);
assert(commit); assert(commit);
return (extent); return (extent);
} }
extent_t *
extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab)
{
malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, true,
new_addr, usize, pad, alignment, zero, slab));
}
extent_t *
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab)
{
return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, false,
new_addr, usize, pad, alignment, zero, slab));
}
static void * static void *
extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit) size_t size, size_t alignment, bool *zero, bool *commit)
@ -607,8 +645,8 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
assert(alignment != 0); assert(alignment != 0);
extent = extent_recycle(tsdn, arena, r_extent_hooks, extent = extent_recycle(tsdn, arena, r_extent_hooks,
arena->extents_retained, false, new_addr, usize, pad, alignment, arena->extents_retained, false, false, new_addr, usize, pad,
zero, commit, slab); alignment, zero, commit, slab);
if (extent != NULL && config_stats) { if (extent != NULL && config_stats) {
size_t size = usize + pad; size_t size = usize + pad;
arena->stats.retained -= size; arena->stats.retained -= size;
@ -697,22 +735,24 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
if (!extent_can_coalesce(a, b)) if (!extent_can_coalesce(a, b))
return; return;
extent_heaps_remove(extent_heaps, a); extent_heaps_remove(tsdn, extent_heaps, a);
extent_heaps_remove(extent_heaps, b); extent_heaps_remove(tsdn, extent_heaps, b);
arena_extent_cache_maybe_remove(extent_arena_get(a), a, cache); arena_extent_cache_maybe_remove(tsdn, extent_arena_get(a), a, cache);
arena_extent_cache_maybe_remove(extent_arena_get(b), b, cache); arena_extent_cache_maybe_remove(tsdn, extent_arena_get(b), b, cache);
if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) { if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) {
extent_heaps_insert(extent_heaps, a); extent_heaps_insert(tsdn, extent_heaps, a);
extent_heaps_insert(extent_heaps, b); extent_heaps_insert(tsdn, extent_heaps, b);
arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache); arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a,
arena_extent_cache_maybe_insert(extent_arena_get(b), b, cache); cache);
arena_extent_cache_maybe_insert(tsdn, extent_arena_get(b), b,
cache);
return; return;
} }
extent_heaps_insert(extent_heaps, a); extent_heaps_insert(tsdn, extent_heaps, a);
arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache); arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a, cache);
} }
static void static void
@ -737,8 +777,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
} }
assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent); assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent);
extent_heaps_insert(extent_heaps, extent); extent_heaps_insert(tsdn, extent_heaps, extent);
arena_extent_cache_maybe_insert(arena, extent, cache); arena_extent_cache_maybe_insert(tsdn, arena, extent, cache);
/* Try to coalesce forward. */ /* Try to coalesce forward. */
next = rtree_read(tsdn, &extents_rtree, rtree_ctx, next = rtree_read(tsdn, &extents_rtree, rtree_ctx,
@ -1021,7 +1061,7 @@ extent_merge_default_impl(tsdn_t *tsdn, void *addr_a, void *addr_b)
return (true); return (true);
if (have_dss && extent_in_dss(tsdn, addr_a) != extent_in_dss(tsdn, if (have_dss && extent_in_dss(tsdn, addr_a) != extent_in_dss(tsdn,
addr_b)) addr_b))
return (true); return (true);
return (false); return (false);
} }