diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 6e155206..d7944c1c 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -105,6 +105,7 @@ extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab); +void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent); void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index d2c882d3..a489e14a 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -158,6 +158,7 @@ extent_committed_get extent_committed_set extent_dalloc extent_dalloc_cache +extent_dalloc_gap extent_dalloc_mmap extent_dalloc_wrapper extent_decommit_wrapper diff --git a/src/extent.c b/src/extent.c index 0ea10fb9..838cb73b 100644 --- a/src/extent.c +++ b/src/extent.c @@ -732,6 +732,18 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, malloc_mutex_unlock(tsdn, &arena->extents_mtx); } +void +extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) +{ + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; + + if (extent_register(tsdn, extent)) { + extent_leak(tsdn, arena, &extent_hooks, false, extent); + return; + } + extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); +} + void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) diff --git a/src/extent_dss.c b/src/extent_dss.c index 9cc1d8f7..9c5cd25a 100644 --- a/src/extent_dss.c +++ b/src/extent_dss.c @@ -70,7 +70,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { void *ret; - extent_t *pad; + extent_t *gap; cassert(have_dss); assert(size > 0); @@ -83,8 +83,8 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, if ((intptr_t)size < 0) return (NULL); - pad = extent_alloc(tsdn, arena); - if (pad == NULL) + gap = extent_alloc(tsdn, arena); + if (gap == NULL) return (NULL); malloc_mutex_lock(tsdn, &dss_mtx); @@ -95,8 +95,8 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, * malloc. */ while (true) { - void *pad_addr, *dss_next; - size_t pad_size; + void *gap_addr, *dss_next; + size_t gap_size; intptr_t incr; /* Avoid an unnecessary system call. */ @@ -111,23 +111,23 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, break; /* - * Compute how much pad space (if any) is necessary to + * Compute how much gap space (if any) is necessary to * satisfy alignment. This space can be recycled for * later use. */ - pad_addr = (void *)((uintptr_t)dss_max); - ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, - alignment); - pad_size = (uintptr_t)ret - (uintptr_t)pad_addr; - if (pad_size != 0) { - extent_init(pad, arena, pad_addr, pad_size, - pad_size, false, false, true, false); + gap_addr = (void *)(PAGE_CEILING((uintptr_t)dss_max)); + ret = (void *)ALIGNMENT_CEILING((uintptr_t)gap_addr, + PAGE_CEILING(alignment)); + gap_size = (uintptr_t)ret - (uintptr_t)gap_addr; + if (gap_size != 0) { + extent_init(gap, arena, gap_addr, gap_size, + gap_size, false, false, true, false); } dss_next = (void *)((uintptr_t)ret + size); if ((uintptr_t)ret < (uintptr_t)dss_max || (uintptr_t)dss_next < (uintptr_t)dss_max) break; /* Wrap-around. */ - incr = pad_size + size; + incr = gap_size + size; dss_prev = extent_dss_sbrk(incr); if (dss_prev == (void *)-1) break; @@ -135,13 +135,10 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, /* Success. */ dss_max = dss_next; malloc_mutex_unlock(tsdn, &dss_mtx); - if (pad_size != 0) { - extent_hooks_t *extent_hooks = - EXTENT_HOOKS_INITIALIZER; - extent_dalloc_wrapper(tsdn, arena, - &extent_hooks, pad); - } else - extent_dalloc(tsdn, arena, pad); + if (gap_size != 0) + extent_dalloc_gap(tsdn, arena, gap); + else + extent_dalloc(tsdn, arena, gap); if (*zero) memset(ret, 0, size); if (!*commit) @@ -152,7 +149,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, } /* OOM. */ malloc_mutex_unlock(tsdn, &dss_mtx); - extent_dalloc(tsdn, arena, pad); + extent_dalloc(tsdn, arena, gap); return (NULL); }