diff --git a/include/jemalloc/internal/edata.h b/include/jemalloc/internal/edata.h index 990c3256..86f5ac57 100644 --- a/include/jemalloc/internal/edata.h +++ b/include/jemalloc/internal/edata.h @@ -467,20 +467,12 @@ edata_prof_alloc_time_set(edata_t *edata, nstime_t *t) { static inline bool edata_is_head_get(edata_t *edata) { - if (maps_coalesce) { - not_reached(); - } - return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) >> EDATA_BITS_IS_HEAD_SHIFT); } static inline void edata_is_head_set(edata_t *edata, bool is_head) { - if (maps_coalesce) { - not_reached(); - } - edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) | ((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT); } @@ -502,9 +494,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size, edata_committed_set(edata, committed); edata_dumpable_set(edata, dumpable); ql_elm_new(edata, ql_link); - if (!maps_coalesce) { - edata_is_head_set(edata, is_head == EXTENT_IS_HEAD); - } + edata_is_head_set(edata, is_head == EXTENT_IS_HEAD); if (config_prof) { edata_prof_tctx_set(edata, NULL); } diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index 711a534b..6f4f950c 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -1,16 +1,21 @@ #ifndef JEMALLOC_INTERNAL_EHOOKS_H #define JEMALLOC_INTERNAL_EHOOKS_H +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/extent_mmap.h" + /* * This module is the internal interface to the extent hooks (both * user-specified and external). Eventually, this will give us the flexibility * to use multiple different versions of user-visible extent-hook APIs under a * single user interface. + * + * Current API expansions (not available to anyone but the default hooks yet): + * - Head state tracking. Hooks can decide whether or not to merge two + * extents based on whether or not one of them is the head (i.e. was + * allocated on its own). The later extent loses its "head" status. */ -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/extent_mmap.h" - extern const extent_hooks_t ehooks_default_extent_hooks; typedef struct ehooks_s ehooks_t; @@ -43,7 +48,8 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length); bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length); #endif bool ehooks_default_split_impl(); -bool ehooks_default_merge_impl(void *addr_a, void *addr_b); +bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, bool head_a, + void *addr_b, bool head_b); void ehooks_default_zero_impl(void *addr, size_t size); /* @@ -314,10 +320,12 @@ ehooks_split(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, static inline bool ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a, - void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { + bool head_a, void *addr_b, size_t size_b, bool head_b, bool committed, + unsigned arena_ind) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); if (extent_hooks == &ehooks_default_extent_hooks) { - return ehooks_default_merge_impl(addr_a, addr_b); + return ehooks_default_merge_impl(tsdn, addr_a, head_a, addr_b, + head_b); } else if (extent_hooks->merge == NULL) { return true; } else { diff --git a/include/jemalloc/internal/extent2.h b/include/jemalloc/internal/extent2.h index 629474ee..08443366 100644 --- a/include/jemalloc/internal/extent2.h +++ b/include/jemalloc/internal/extent2.h @@ -54,7 +54,6 @@ edata_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, size_t size_b, szind_t szind_b, bool slab_b); bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a, edata_t *b); -bool extent_head_no_merge(edata_t *a, edata_t *b); bool extent_boot(void); diff --git a/src/ehooks.c b/src/ehooks.c index a62586b9..51b1514a 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -183,8 +183,51 @@ ehooks_default_split(extent_hooks_t *extent_hooks, void *addr, size_t size, return ehooks_default_split_impl(); } +static inline bool +ehooks_same_sn(tsdn_t *tsdn, void *addr_a, void *addr_b) { + edata_t *a = iealloc(tsdn, addr_a); + edata_t *b = iealloc(tsdn, addr_b); + return edata_sn_comp(a, b) == 0; +} + +/* + * Returns true if the given extents can't be merged because of their head bit + * settings. Assumes the second extent has the higher address. + */ +static bool +ehooks_no_merge_heads(tsdn_t *tsdn, void *addr_a, bool head_a, void *addr_b, + bool head_b) { + /* + * When coalesce is not always allowed (Windows), only merge extents + * from the same VirtualAlloc region under opt.retain (in which case + * MEM_DECOMMIT is utilized for purging). + */ + if (maps_coalesce) { + return false; + } + if (!opt_retain) { + return true; + } + /* If b is a head extent, disallow the cross-region merge. */ + if (head_b) { + /* + * Additionally, sn should not overflow with retain; sanity + * check that different regions have unique sn. + */ + assert(!ehooks_same_sn(tsdn, addr_a, addr_b)); + return true; + } + assert(ehooks_same_sn(tsdn, addr_a, addr_b)); + + return false; +} + bool -ehooks_default_merge_impl(void *addr_a, void *addr_b) { +ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, bool head_a, void *addr_b, + bool head_b) { + if (ehooks_no_merge_heads(tsdn, addr_a, head_a, addr_b, head_b)) { + return true; + } if (!maps_coalesce && !opt_retain) { return true; } @@ -198,15 +241,12 @@ ehooks_default_merge_impl(void *addr_a, void *addr_b) { static bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { - if (!maps_coalesce) { - tsdn_t *tsdn = tsdn_fetch(); - edata_t *a = iealloc(tsdn, addr_a); - edata_t *b = iealloc(tsdn, addr_b); - if (extent_head_no_merge(a, b)) { - return true; - } - } - return ehooks_default_merge_impl(addr_a, addr_b); + tsdn_t *tsdn = tsdn_fetch(); + edata_t *a = iealloc(tsdn, addr_a); + bool head_a = edata_is_head_get(a); + edata_t *b = iealloc(tsdn, addr_b); + bool head_b = edata_is_head_get(b); + return ehooks_default_merge_impl(tsdn, addr_a, head_a, addr_b, head_b); } void diff --git a/src/extent2.c b/src/extent2.c index 148c3283..21f9cdbd 100644 --- a/src/extent2.c +++ b/src/extent2.c @@ -1493,38 +1493,6 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, slab_a, size_b, szind_b, slab_b, false); } -/* - * Returns true if the given extents can't be merged because of their head bit - * settings. Assumes the second extent has the higher address. - */ -bool -extent_head_no_merge(edata_t *a, edata_t *b) { - assert(edata_base_get(a) < edata_base_get(b)); - /* - * When coalesce is not always allowed (Windows), only merge extents - * from the same VirtualAlloc region under opt.retain (in which case - * MEM_DECOMMIT is utilized for purging). - */ - if (maps_coalesce) { - return false; - } - if (!opt_retain) { - return true; - } - /* If b is a head extent, disallow the cross-region merge. */ - if (edata_is_head_get(b)) { - /* - * Additionally, sn should not overflow with retain; sanity - * check that different regions have unique sn. - */ - assert(edata_sn_comp(a, b) != 0); - return true; - } - assert(edata_sn_comp(a, b) == 0); - - return false; -} - static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a, edata_t *b, bool growing_retained) { @@ -1532,13 +1500,10 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a, WITNESS_RANK_CORE, growing_retained ? 1 : 0); assert(edata_base_get(a) < edata_base_get(b)); - if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) { - return true; - } - bool err = ehooks_merge(tsdn, ehooks, edata_base_get(a), - edata_size_get(a), edata_base_get(b), edata_size_get(b), - edata_committed_get(a), arena_ind_get(arena)); + edata_size_get(a), edata_is_head_get(a), edata_base_get(b), + edata_size_get(b), edata_is_head_get(b), edata_committed_get(a), + arena_ind_get(arena)); if (err) { return true;