diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index 97c3f442..734cd181 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -43,6 +43,7 @@ bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length); #endif bool ehooks_default_split_impl(); bool ehooks_default_merge_impl(void *addr_a, void *addr_b); +void ehooks_default_zero_impl(void *addr, size_t size); /* * We don't officially support reentrancy from wtihin the extent hooks. But @@ -261,4 +262,21 @@ ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a, } } +static inline void +ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, + unsigned arena_ind) { + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + if (extent_hooks == &ehooks_default_extent_hooks) { + ehooks_default_zero_impl(addr, size); + } else { + /* + * It would be correct to try using the user-provided purge + * hooks (since they are required to have zeroed the extent if + * they indicate success), but we don't necessarily know their + * cost. We'll be conservative and use memset. + */ + memset(addr, 0, size); + } +} + #endif /* JEMALLOC_INTERNAL_EHOOKS_H */ diff --git a/src/ehooks.c b/src/ehooks.c index d7d1613f..25aef1c3 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -209,6 +209,23 @@ ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, return ehooks_default_merge_impl(addr_a, addr_b); } +void +ehooks_default_zero_impl(void *addr, size_t size) { + /* + * By default, we try to zero out memory using OS-provided demand-zeroed + * pages. If the user has specifically requested hugepages, though, we + * don't want to purge in the middle of a hugepage (which would break it + * up), so we act conservatively and use memset. + */ + bool needs_memset = true; + if (opt_thp != thp_mode_always) { + needs_memset = pages_purge_forced(addr, size); + } + if (needs_memset) { + memset(addr, 0, size); + } +} + const extent_hooks_t ehooks_default_extent_hooks = { ehooks_default_alloc, ehooks_default_dalloc, diff --git a/src/extent2.c b/src/extent2.c index 0b097160..55f72dff 100644 --- a/src/extent2.c +++ b/src/extent2.c @@ -758,17 +758,6 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, unreachable(); } -static bool -extent_need_manual_zero(arena_t *arena) { - /* - * Need to manually zero the extent on repopulating if either; 1) non - * default extent hooks installed (in which case the purge semantics may - * change); or 2) transparent huge pages enabled. - */ - return (!ehooks_are_default(arena_get_ehooks(arena)) || - (opt_thp == thp_mode_always)); -} - /* * Tries to satisfy the given allocation request by reusing one of the extents * in the given eset_t. @@ -807,9 +796,6 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, growing_retained); return NULL; } - if (!extent_need_manual_zero(arena)) { - extent_zeroed_set(extent, true); - } } if (extent_committed_get(extent)) { @@ -832,11 +818,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset, void *addr = extent_base_get(extent); if (!extent_zeroed_get(extent)) { size_t size = extent_size_get(extent); - if (extent_need_manual_zero(arena) || - pages_purge_forced(addr, size)) { - memset(addr, 0, size); - } - } else if (config_debug) { + ehooks_zero(tsdn, ehooks, addr, size, + arena_ind_get(arena)); + } + if (config_debug) { size_t *p = (size_t *)(uintptr_t)addr; /* Check the first page only. */ for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { @@ -960,8 +945,14 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, &arena->eset_retained, extent, true); goto label_err; } - if (!extent_need_manual_zero(arena)) { - extent_zeroed_set(extent, true); + /* A successful commit should return zeroed memory. */ + if (config_debug) { + void *addr = extent_addr_get(extent); + size_t *p = (size_t *)(uintptr_t)addr; + /* Check the first page only. */ + for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { + assert(p[i] == 0); + } } } @@ -996,10 +987,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, if (*zero && !extent_zeroed_get(extent)) { void *addr = extent_base_get(extent); size_t size = extent_size_get(extent); - if (extent_need_manual_zero(arena) || - pages_purge_forced(addr, size)) { - memset(addr, 0, size); - } + ehooks_zero(tsdn, ehooks, addr, size, arena_ind_get(arena)); } return extent;