From 6c944708222b3f1843ad224c88ffdafa02da1bb8 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 23 May 2016 14:56:35 -0700 Subject: [PATCH] Refactor chunk_dalloc_{cache,wrapper}() to take extent arguments. Rename arena_extent_[d]alloc() to extent_[d]alloc(). Move all chunk [de]registration responsibility into chunk.c. --- include/jemalloc/internal/arena.h | 7 +- include/jemalloc/internal/chunk.h | 8 +- include/jemalloc/internal/extent.h | 3 + include/jemalloc/internal/private_symbols.txt | 7 +- src/arena.c | 106 +++----------- src/chunk.c | 133 +++++++++--------- src/chunk_dss.c | 19 ++- src/chunk_mmap.c | 1 - src/extent.c | 26 ++++ src/huge.c | 37 ++--- 10 files changed, 148 insertions(+), 199 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 187b6256..4e5e3029 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -478,17 +478,14 @@ extent_t *arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, bool *zero); void arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed); + chunk_hooks_t *chunk_hooks, extent_t *extent); void arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache); void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent, bool cache); -extent_t *arena_extent_alloc(tsdn_t *tsdn, arena_t *arena); -void arena_extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent); extent_t *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero); -void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, - size_t usize); +void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent); void arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t oldsize); void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 78cc4c2d..cef9fe03 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -52,9 +52,6 @@ chunk_hooks_t chunk_hooks_get(tsdn_t *tsdn, arena_t *arena); chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena, const chunk_hooks_t *chunk_hooks); -bool chunk_register(tsdn_t *tsdn, const extent_t *extent); -void chunk_deregister(tsdn_t *tsdn, const extent_t *extent); -void chunk_reregister(tsdn_t *tsdn, const extent_t *extent); extent_t *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, bool *zero); @@ -62,10 +59,9 @@ extent_t *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit); void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed); + chunk_hooks_t *chunk_hooks, extent_t *extent); void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool zeroed, - bool committed); + chunk_hooks_t *chunk_hooks, extent_t *extent); bool chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset, size_t length); diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 82da8004..cfc908d8 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -62,6 +62,9 @@ typedef ph(extent_t) extent_heap_t; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS +extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); +void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent); + #ifdef JEMALLOC_JET typedef size_t (extent_size_quantize_t)(size_t); extern extent_size_quantize_t *extent_size_quantize_floor; diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 34a6816f..8998aed9 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -37,8 +37,6 @@ arena_decay_time_get arena_decay_time_set arena_dss_prec_get arena_dss_prec_set -arena_extent_alloc -arena_extent_dalloc arena_get arena_ichoose arena_init @@ -166,7 +164,6 @@ chunk_dalloc_cache chunk_dalloc_mmap chunk_dalloc_wrapper chunk_decommit_wrapper -chunk_deregister chunk_dss_boot chunk_dss_postfork_child chunk_dss_postfork_parent @@ -184,8 +181,6 @@ chunk_postfork_child chunk_postfork_parent chunk_prefork chunk_purge_wrapper -chunk_register -chunk_reregister chunk_split_wrapper chunks_rtree chunksize @@ -214,10 +209,12 @@ extent_active_get extent_active_set extent_addr_get extent_addr_set +extent_alloc extent_arena_get extent_arena_set extent_committed_get extent_committed_set +extent_dalloc extent_dirty_insert extent_dirty_remove extent_init diff --git a/src/arena.c b/src/arena.c index a610ec15..9eb08979 100644 --- a/src/arena.c +++ b/src/arena.c @@ -249,23 +249,22 @@ arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena, static void arena_chunk_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed) + chunk_hooks_t *chunk_hooks, extent_t *extent) { malloc_mutex_assert_owner(tsdn, &arena->lock); - chunk_dalloc_cache(tsdn, arena, chunk_hooks, chunk, size, committed); + chunk_dalloc_cache(tsdn, arena, chunk_hooks, extent); arena_maybe_purge(tsdn, arena); } void arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed) + chunk_hooks_t *chunk_hooks, extent_t *extent) { malloc_mutex_lock(tsdn, &arena->lock); - arena_chunk_cache_dalloc_locked(tsdn, arena, chunk_hooks, chunk, size, - committed); + arena_chunk_cache_dalloc_locked(tsdn, arena, chunk_hooks, extent); malloc_mutex_unlock(tsdn, &arena->lock); } @@ -582,33 +581,14 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, if (chunk_commit_wrapper(tsdn, arena, chunk_hooks, extent_addr_get(extent), extent_size_get(extent), 0, map_bias << LG_PAGE)) { - chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, - extent_addr_get(extent), extent_size_get(extent), - extent_zeroed_get(extent), - extent_committed_get(extent)); + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, extent); extent = NULL; } } - if (extent != NULL) { + if (extent != NULL) extent_slab_set(extent, true); - if (chunk_register(tsdn, extent)) { - if (!*commit) { - /* Undo commit of header. */ - chunk_decommit_wrapper(tsdn, arena, chunk_hooks, - extent_addr_get(extent), - extent_size_get(extent), 0, map_bias << - LG_PAGE); - } - chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, - extent_addr_get(extent), extent_size_get(extent), - extent_zeroed_get(extent), - extent_committed_get(extent)); - extent = NULL; - } - } - malloc_mutex_lock(tsdn, &arena->lock); return (extent); @@ -625,13 +605,6 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, chunksize, chunksize, zero); if (extent != NULL) { extent_slab_set(extent, true); - - if (chunk_register(tsdn, extent)) { - arena_chunk_cache_dalloc_locked(tsdn, arena, - &chunk_hooks, extent_addr_get(extent), - extent_size_get(extent), true); - return (NULL); - } *commit = true; } if (extent == NULL) { @@ -722,14 +695,13 @@ arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena) static void arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { - bool committed; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; - chunk_deregister(tsdn, extent); - - committed = (arena_mapbits_decommitted_get((arena_chunk_t *) - extent_addr_get(extent), map_bias) == 0); - if (!committed) { + extent_committed_set(extent, + (arena_mapbits_decommitted_get((arena_chunk_t *) + extent_addr_get(extent), map_bias) == 0)); + extent_slab_set(extent, false); + if (!extent_committed_get(extent)) { /* * Decommit the header. Mark the chunk as decommitted even if * header decommit fails, since treating a partially committed @@ -741,15 +713,12 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, extent_t *extent) map_bias << LG_PAGE); } - arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, - extent_addr_get(extent), extent_size_get(extent), committed); - if (config_stats) { arena->stats.mapped -= extent_size_get(extent); arena->stats.metadata_mapped -= (map_bias << LG_PAGE); } - arena_extent_dalloc(tsdn, arena, extent); + arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, extent); } static void @@ -852,32 +821,6 @@ arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize) arena_huge_malloc_stats_update(arena, usize); } -extent_t * -arena_extent_alloc(tsdn_t *tsdn, arena_t *arena) -{ - extent_t *extent; - - malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); - extent = ql_last(&arena->extent_cache, ql_link); - if (extent == NULL) { - malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); - return (base_alloc(tsdn, sizeof(extent_t))); - } - ql_tail_remove(&arena->extent_cache, extent_t, ql_link); - malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); - return (extent); -} - -void -arena_extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) -{ - - malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); - ql_elm_new(extent, ql_link); - ql_tail_insert(&arena->extent_cache, extent, ql_link); - malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); -} - static extent_t * arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero, @@ -931,21 +874,21 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize, } void -arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize) +arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; - size_t csize; - csize = CHUNK_CEILING(usize); malloc_mutex_lock(tsdn, &arena->lock); if (config_stats) { - arena_huge_dalloc_stats_update(arena, usize); - arena->stats.mapped -= usize; + arena_huge_dalloc_stats_update(arena, extent_size_get(extent)); + arena->stats.mapped -= extent_size_get(extent); } - arena_nactive_sub(arena, usize >> LG_PAGE); + arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE); - arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, chunk, csize, - true); + if ((extent_size_get(extent) & chunksize_mask) != 0) + extent_size_set(extent, CHUNK_CEILING(extent_size_get(extent))); + + arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, extent); malloc_mutex_unlock(tsdn, &arena->lock); } @@ -1656,15 +1599,10 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, rdelm_next = qr_next(rdelm, rd_link); if (rdelm == &chunkselm->rd) { extent_t *chunkselm_next = qr_next(chunkselm, cc_link); - void *addr = extent_addr_get(chunkselm); - size_t size = extent_size_get(chunkselm); - bool zeroed = extent_zeroed_get(chunkselm); - bool committed = extent_committed_get(chunkselm); extent_dirty_remove(chunkselm); - arena_extent_dalloc(tsdn, arena, chunkselm); + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, + chunkselm); chunkselm = chunkselm_next; - chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr, - size, zeroed, committed); } else { extent_t *extent = iealloc(tsdn, rdelm); arena_chunk_t *chunk = diff --git a/src/chunk.c b/src/chunk.c index 4efba4a7..d0763212 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -51,7 +51,7 @@ const chunk_hooks_t chunk_hooks_default = { static void chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent_heap_t extent_heaps[NPSIZES], bool cache, - void *chunk, size_t size, bool zeroed, bool committed); + extent_t *extent); /******************************************************************************/ @@ -203,7 +203,7 @@ extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b) rtree_elm_release(tsdn, &chunks_rtree, elm_b); } -bool +static bool chunk_register(tsdn_t *tsdn, const extent_t *extent) { rtree_elm_t *elm_a, *elm_b; @@ -232,7 +232,7 @@ chunk_register(tsdn_t *tsdn, const extent_t *extent) return (false); } -void +static void chunk_deregister(tsdn_t *tsdn, const extent_t *extent) { rtree_elm_t *elm_a, *elm_b; @@ -249,15 +249,6 @@ chunk_deregister(tsdn_t *tsdn, const extent_t *extent) } } -void -chunk_reregister(tsdn_t *tsdn, const extent_t *extent) -{ - bool err; - - err = chunk_register(tsdn, extent); - assert(!err); -} - /* * Do first-best-fit chunk selection, i.e. select the lowest chunk that best * fits. @@ -282,7 +273,7 @@ chunk_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES], static void chunk_leak(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, bool cache, - void *addr, size_t size) + extent_t *extent) { /* @@ -290,9 +281,11 @@ chunk_leak(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, bool cache, * that this is only a virtual memory leak. */ if (cache) { - chunk_purge_wrapper(tsdn, arena, chunk_hooks, addr, size, 0, - size); + chunk_purge_wrapper(tsdn, arena, chunk_hooks, + extent_addr_get(extent), extent_size_get(extent), 0, + extent_size_get(extent)); } + extent_dalloc(tsdn, arena, extent); } static extent_t * @@ -351,9 +344,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent = chunk_split_wrapper(tsdn, arena, chunk_hooks, lead, leadsize, size + trailsize); if (extent == NULL) { - chunk_leak(tsdn, arena, chunk_hooks, cache, - extent_addr_get(lead), extent_size_get(lead)); - arena_extent_dalloc(tsdn, arena, lead); + chunk_leak(tsdn, arena, chunk_hooks, cache, lead); malloc_mutex_unlock(tsdn, &arena->chunks_mtx); return (NULL); } @@ -366,9 +357,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent_t *trail = chunk_split_wrapper(tsdn, arena, chunk_hooks, extent, size, trailsize); if (trail == NULL) { - chunk_leak(tsdn, arena, chunk_hooks, cache, - extent_addr_get(extent), extent_size_get(extent)); - arena_extent_dalloc(tsdn, arena, extent); + chunk_leak(tsdn, arena, chunk_hooks, cache, extent); malloc_mutex_unlock(tsdn, &arena->chunks_mtx); return (NULL); } @@ -381,9 +370,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) { malloc_mutex_unlock(tsdn, &arena->chunks_mtx); chunk_record(tsdn, arena, chunk_hooks, extent_heaps, cache, - extent_addr_get(extent), extent_size_get(extent), - extent_zeroed_get(extent), extent_committed_get(extent)); - arena_extent_dalloc(tsdn, arena, extent); + extent); return (NULL); } @@ -529,7 +516,7 @@ chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, if (extent == NULL) { void *chunk; - extent = arena_extent_alloc(tsdn, arena); + extent = extent_alloc(tsdn, arena); if (extent == NULL) return (NULL); chunk = chunk_hooks->alloc(new_addr, size, alignment, @@ -540,6 +527,11 @@ chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, false); } + if (chunk_register(tsdn, extent)) { + chunk_leak(tsdn, arena, chunk_hooks, false, extent); + return (NULL); + } + return (extent); } @@ -590,29 +582,21 @@ chunk_try_coalesce(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, static void chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - extent_heap_t extent_heaps[NPSIZES], bool cache, void *chunk, size_t size, - bool zeroed, bool committed) + extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent) { - extent_t *extent, *prev, *next; + extent_t *prev, *next; - assert(!cache || !zeroed); + assert(!cache || !extent_zeroed_get(extent)); malloc_mutex_lock(tsdn, &arena->chunks_mtx); chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks); - /* Create/initialize/insert extent. */ - extent = arena_extent_alloc(tsdn, arena); - if (extent == NULL) { - chunk_leak(tsdn, arena, chunk_hooks, cache, chunk, size); - goto label_return; - } - extent_init(extent, arena, chunk, size, false, !cache && zeroed, - committed, false); - if (chunk_register(tsdn, extent)) { - arena_extent_dalloc(tsdn, arena, extent); - chunk_leak(tsdn, arena, chunk_hooks, cache, chunk, size); - goto label_return; - } + assert((extent_size_get(extent) & chunksize_mask) == 0); + extent_active_set(extent, false); + extent_zeroed_set(extent, !cache && extent_zeroed_get(extent)); + extent_slab_set(extent, false); + + assert(chunk_lookup(tsdn, extent_addr_get(extent), true) == extent); extent_heaps_insert(extent_heaps, extent); arena_chunk_cache_maybe_insert(arena, extent, cache); @@ -632,22 +616,24 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent_heaps, cache); } -label_return: malloc_mutex_unlock(tsdn, &arena->chunks_mtx); } void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - void *chunk, size_t size, bool committed) + extent_t *extent) { - assert(chunk != NULL); - assert(CHUNK_ADDR2BASE(chunk) == chunk); - assert(size != 0); - assert((size & chunksize_mask) == 0); + assert(extent_addr_get(extent) != NULL); + assert(CHUNK_ADDR2BASE(extent_addr_get(extent)) == + extent_addr_get(extent)); + assert(extent_size_get(extent) != 0); + assert((extent_size_get(extent) & chunksize_mask) == 0); + + extent_zeroed_set(extent, false); chunk_record(tsdn, arena, chunk_hooks, arena->chunks_cached, true, - chunk, size, false, committed); + extent); } static bool @@ -662,30 +648,40 @@ chunk_dalloc_default(void *chunk, size_t size, bool committed, void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - void *chunk, size_t size, bool zeroed, bool committed) + extent_t *extent) { - assert(chunk != NULL); - assert(CHUNK_ADDR2BASE(chunk) == chunk); - assert(size != 0); - assert((size & chunksize_mask) == 0); + assert(extent_addr_get(extent) != NULL); + assert(CHUNK_ADDR2BASE(extent_addr_get(extent)) == + extent_addr_get(extent)); + assert(extent_size_get(extent) != 0); + assert((extent_size_get(extent) & chunksize_mask) == 0); chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); /* Try to deallocate. */ - if (!chunk_hooks->dalloc(chunk, size, committed, arena->ind)) + if (!chunk_hooks->dalloc(extent_addr_get(extent), + extent_size_get(extent), extent_committed_get(extent), + arena->ind)) { + chunk_deregister(tsdn, extent); + extent_dalloc(tsdn, arena, extent); return; - /* Try to decommit; purge if that fails. */ - if (committed) { - committed = chunk_hooks->decommit(chunk, size, 0, size, - arena->ind); } - zeroed = !committed || !chunk_hooks->purge(chunk, size, 0, size, - arena->ind); - chunk_record(tsdn, arena, chunk_hooks, arena->chunks_retained, false, - chunk, size, zeroed, committed); + /* Try to decommit; purge if that fails. */ + if (extent_committed_get(extent)) { + extent_committed_set(extent, + chunk_hooks->decommit(extent_addr_get(extent), + extent_size_get(extent), 0, extent_size_get(extent), + arena->ind)); + } + extent_zeroed_set(extent, !extent_committed_get(extent) || + !chunk_hooks->purge(extent_addr_get(extent), + extent_size_get(extent), 0, extent_size_get(extent), arena->ind)); if (config_stats) - arena->stats.retained += size; + arena->stats.retained += extent_size_get(extent); + + chunk_record(tsdn, arena, chunk_hooks, arena->chunks_retained, false, + extent); } static bool @@ -771,7 +767,7 @@ chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); - trail = arena_extent_alloc(tsdn, arena); + trail = extent_alloc(tsdn, arena); if (trail == NULL) goto label_error_a; @@ -814,7 +810,7 @@ label_error_d: label_error_c: extent_rtree_release(tsdn, lead_elm_a, lead_elm_b); label_error_b: - arena_extent_dalloc(tsdn, arena, trail); + extent_dalloc(tsdn, arena, trail); label_error_a: return (NULL); } @@ -841,6 +837,9 @@ chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, { rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; + assert((extent_size_get(a) & chunksize_mask) == 0); + assert((extent_size_get(b) & chunksize_mask) == 0); + chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); if (chunk_hooks->merge(extent_addr_get(a), extent_size_get(a), extent_addr_get(b), extent_size_get(b), extent_committed_get(a), @@ -871,7 +870,7 @@ chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a); extent_rtree_release(tsdn, a_elm_a, b_elm_b); - arena_extent_dalloc(tsdn, extent_arena_get(b), b); + extent_dalloc(tsdn, extent_arena_get(b), b); return (false); } diff --git a/src/chunk_dss.c b/src/chunk_dss.c index d42aeb0b..251f17b8 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -89,7 +89,8 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, * malloc. */ do { - void *ret, *cpad, *dss_next; + void *ret, *cpad_addr, *dss_next; + extent_t *cpad; size_t gap_size, cpad_size; intptr_t incr; /* Avoid an unnecessary system call. */ @@ -114,10 +115,19 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, * necessary to satisfy alignment. This space can be * recycled for later use. */ - cpad = (void *)((uintptr_t)dss_max + gap_size); + cpad_addr = (void *)((uintptr_t)dss_max + gap_size); ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, alignment); - cpad_size = (uintptr_t)ret - (uintptr_t)cpad; + cpad_size = (uintptr_t)ret - (uintptr_t)cpad_addr; + if (cpad_size != 0) { + cpad = extent_alloc(tsdn, arena); + if (cpad == NULL) { + malloc_mutex_unlock(tsdn, &dss_mtx); + return (NULL); + } + extent_init(cpad, arena, cpad_addr, cpad_size, + false, false, true, false); + } dss_next = (void *)((uintptr_t)ret + size); if ((uintptr_t)ret < (uintptr_t)dss_max || (uintptr_t)dss_next < (uintptr_t)dss_max) { @@ -135,8 +145,7 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; chunk_dalloc_wrapper(tsdn, arena, - &chunk_hooks, cpad, cpad_size, - false, true); + &chunk_hooks, cpad); } if (*zero) memset(ret, 0, size); diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c index f95ae756..13708027 100644 --- a/src/chunk_mmap.c +++ b/src/chunk_mmap.c @@ -73,6 +73,5 @@ chunk_dalloc_mmap(void *chunk, size_t size) if (config_munmap) pages_unmap(chunk, size); - return (!config_munmap); } diff --git a/src/extent.c b/src/extent.c index 4757f750..d7f3b6cc 100644 --- a/src/extent.c +++ b/src/extent.c @@ -3,6 +3,32 @@ /******************************************************************************/ +extent_t * +extent_alloc(tsdn_t *tsdn, arena_t *arena) +{ + extent_t *extent; + + malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); + extent = ql_last(&arena->extent_cache, ql_link); + if (extent == NULL) { + malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); + return (base_alloc(tsdn, sizeof(extent_t))); + } + ql_tail_remove(&arena->extent_cache, extent_t, ql_link); + malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); + return (extent); +} + +void +extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) +{ + + malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); + ql_elm_new(extent, ql_link); + ql_tail_insert(&arena->extent_cache, extent, ql_link); + malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); +} + #ifdef JEMALLOC_JET #undef extent_size_quantize_floor #define extent_size_quantize_floor JEMALLOC_N(n_extent_size_quantize_floor) diff --git a/src/huge.c b/src/huge.c index fe4c6e00..880e4333 100644 --- a/src/huge.c +++ b/src/huge.c @@ -43,13 +43,6 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, if (usize < extent_size_get(extent)) extent_size_set(extent, usize); - if (chunk_register(tsdn, extent)) { - arena_chunk_dalloc_huge(tsdn, arena, extent_addr_get(extent), - usize); - arena_extent_dalloc(tsdn, arena, extent); - return (NULL); - } - /* Insert extent into huge. */ malloc_mutex_lock(tsdn, &arena->huge_mtx); ql_elm_new(extent, ql_link); @@ -57,10 +50,14 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, malloc_mutex_unlock(tsdn, &arena->huge_mtx); if (zero || (config_fill && unlikely(opt_zero))) { - if (!is_zeroed) - memset(extent_addr_get(extent), 0, usize); - } else if (config_fill && unlikely(opt_junk_alloc)) - memset(extent_addr_get(extent), JEMALLOC_ALLOC_JUNK, usize); + if (!is_zeroed) { + memset(extent_addr_get(extent), 0, + extent_size_get(extent)); + } + } else if (config_fill && unlikely(opt_junk_alloc)) { + memset(extent_addr_get(extent), JEMALLOC_ALLOC_JUNK, + extent_size_get(extent)); + } arena_decay_tick(tsdn, arena); return (extent_addr_get(extent)); @@ -126,11 +123,9 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, size_t usize_min, /* Update the size of the huge allocation. */ assert(extent_size_get(extent) != usize); - chunk_deregister(tsdn, extent); malloc_mutex_lock(tsdn, &arena->huge_mtx); extent_size_set(extent, usize); malloc_mutex_unlock(tsdn, &arena->huge_mtx); - chunk_reregister(tsdn, extent); /* Update zeroed. */ extent_zeroed_set(extent, post_zeroed); @@ -174,11 +169,7 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) extent_size_get(trail)); } - arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, - extent_addr_get(trail), extent_size_get(trail), - extent_committed_get(trail)); - - arena_extent_dalloc(tsdn, arena, trail); + arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, trail); } /* Optionally fill trailing subchunk. */ @@ -233,10 +224,7 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, } if (chunk_merge_wrapper(tsdn, arena, &chunk_hooks, extent, trail)) { - arena_extent_dalloc(tsdn, arena, trail); - chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, - extent_addr_get(trail), extent_size_get(trail), - extent_zeroed_get(trail), extent_committed_get(trail)); + chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, trail); return (true); } @@ -362,16 +350,13 @@ huge_dalloc(tsdn_t *tsdn, extent_t *extent) arena_t *arena; arena = extent_arena_get(extent); - chunk_deregister(tsdn, extent); malloc_mutex_lock(tsdn, &arena->huge_mtx); ql_remove(&arena->huge, extent, ql_link); malloc_mutex_unlock(tsdn, &arena->huge_mtx); huge_dalloc_junk(tsdn, extent_addr_get(extent), extent_size_get(extent)); - arena_chunk_dalloc_huge(tsdn, extent_arena_get(extent), - extent_addr_get(extent), extent_size_get(extent)); - arena_extent_dalloc(tsdn, arena, extent); + arena_chunk_dalloc_huge(tsdn, extent_arena_get(extent), extent); arena_decay_tick(tsdn, arena); }