diff --git a/include/jemalloc/internal/emap.h b/include/jemalloc/internal/emap.h index 87ece63d..847af327 100644 --- a/include/jemalloc/internal/emap.h +++ b/include/jemalloc/internal/emap.h @@ -208,6 +208,7 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) { assert(edata_committed_get(inner) == edata_committed_get(outer)); assert(edata_state_get(inner) == extent_state_active); assert(edata_state_get(outer) == extent_state_merging); + assert(!edata_guarded_get(inner) && !edata_guarded_get(outer)); assert(edata_base_get(inner) == edata_past_get(outer) || edata_base_get(outer) == edata_past_get(inner)); } diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 73059ad2..1660f45f 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -127,6 +127,7 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents, return false; } } + assert(!edata_guarded_get(edata) && !edata_guarded_get(neighbor)); return true; } diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h index 7eaaf894..01c4e6af 100644 --- a/include/jemalloc/internal/pac.h +++ b/include/jemalloc/internal/pac.h @@ -99,6 +99,9 @@ struct pac_s { exp_grow_t exp_grow; malloc_mutex_t grow_mtx; + /* Special allocator for guarded frequently reused extents. */ + san_bump_alloc_t sba; + /* How large extents should be before getting auto-purged. */ atomic_zu_t oversize_threshold; diff --git a/include/jemalloc/internal/san_bump.h b/include/jemalloc/internal/san_bump.h index 9c6c224f..8ec4a710 100644 --- a/include/jemalloc/internal/san_bump.h +++ b/include/jemalloc/internal/san_bump.h @@ -5,7 +5,9 @@ #include "jemalloc/internal/exp_grow.h" #include "jemalloc/internal/mutex.h" -extern const size_t SBA_RETAINED_ALLOC_SIZE; +#define SBA_RETAINED_ALLOC_SIZE ((size_t)4 << 20) + +extern bool opt_retain; typedef struct ehooks_s ehooks_t; typedef struct pac_s pac_t; @@ -17,8 +19,31 @@ struct san_bump_alloc_s { edata_t *curr_reg; }; -bool -san_bump_alloc_init(san_bump_alloc_t* sba); +static inline bool +san_bump_enabled() { + /* + * We enable san_bump allocator only when it's possible to break up a + * mapping and unmap a part of it (maps_coalesce). This is needed to + * ensure the arena destruction process can destroy all retained guarded + * extents one by one and to unmap a trailing part of a retained guarded + * region when it's too small to fit a pending allocation. + * opt_retain is required, because this allocator retains a large + * virtual memory mapping and returns smaller parts of it. + */ + return maps_coalesce && opt_retain; +} + +static inline bool +san_bump_alloc_init(san_bump_alloc_t* sba) { + bool err = malloc_mutex_init(&sba->mtx, "sanitizer_bump_allocator", + WITNESS_RANK_SAN_BUMP_ALLOC, malloc_mutex_rank_exclusive); + if (err) { + return true; + } + sba->curr_reg = NULL; + + return false; +} edata_t * san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac, ehooks_t *ehooks, diff --git a/src/arena.c b/src/arena.c index 19e4e85a..121832a7 100644 --- a/src/arena.c +++ b/src/arena.c @@ -328,8 +328,8 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, szind_t szind = sz_size2index(usize); size_t esize = usize + sz_large_pad; - bool guarded = san_large_extent_decide_guard(tsdn, arena_get_ehooks(arena), - esize, alignment); + bool guarded = san_large_extent_decide_guard(tsdn, + arena_get_ehooks(arena), esize, alignment); edata_t *edata = pa_alloc(tsdn, &arena->pa_shard, esize, alignment, /* slab */ false, szind, zero, guarded, &deferred_work_generated); assert(deferred_work_generated == false); @@ -829,7 +829,8 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - bool guarded = san_slab_extent_decide_guard(tsdn, arena_get_ehooks(arena)); + bool guarded = san_slab_extent_decide_guard(tsdn, + arena_get_ehooks(arena)); edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size, /* alignment */ PAGE, /* slab */ true, /* szind */ binind, /* zero */ false, guarded, &deferred_work_generated); diff --git a/src/emap.c b/src/emap.c index e37fea38..9cc95a72 100644 --- a/src/emap.c +++ b/src/emap.c @@ -44,6 +44,7 @@ emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata, bool expanding) { witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE); + assert(!edata_guarded_get(edata)); assert(!expanding || forward); assert(!edata_state_in_transition(expected_state)); assert(expected_state == extent_state_dirty || diff --git a/src/extent.c b/src/extent.c index 13d688d1..6fabcc7d 100644 --- a/src/extent.c +++ b/src/extent.c @@ -87,6 +87,7 @@ ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata = extent_recycle(tsdn, pac, ehooks, ecache, expand_edata, size, alignment, zero, &commit, false, guarded); assert(edata == NULL || edata_pai_get(edata) == EXTENT_PAI_PAC); + assert(edata == NULL || edata_guarded_get(edata) == guarded); return edata; } @@ -179,7 +180,7 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, goto label_return; } eset_remove(eset, edata); - if (!ecache->delay_coalesce) { + if (!ecache->delay_coalesce || edata_guarded_get(edata)) { break; } /* Try to coalesce. */ @@ -399,11 +400,6 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } } } else { - /* - * If split and merge are not allowed (Windows w/o retain), try - * exact fit only. - */ - bool exact_only = (!maps_coalesce && !opt_retain) || guarded; /* * A large extent might be broken up from its original size to * some small size to satisfy a small request. When that small @@ -415,7 +411,18 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, */ unsigned lg_max_fit = ecache->delay_coalesce ? (unsigned)opt_lg_extent_max_active_fit : SC_PTR_BITS; - edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit); + + /* + * If split and merge are not allowed (Windows w/o retain), try + * exact fit only. + * + * For simplicity purposes, splitting guarded extents is not + * supported. Hence, we do only exact fit for guarded + * allocations. + */ + bool exact_only = (!maps_coalesce && !opt_retain) || guarded; + edata = eset_fit(eset, size, alignment, exact_only, + lg_max_fit); } if (edata == NULL) { return NULL; @@ -474,6 +481,7 @@ extent_split_interior(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, /* Split the lead. */ if (leadsize != 0) { + assert(!edata_guarded_get(*edata)); *lead = *edata; *edata = extent_split_impl(tsdn, pac, ehooks, *lead, leadsize, size + trailsize, /* holding_core_locks*/ true); @@ -486,6 +494,7 @@ extent_split_interior(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, /* Split the trail. */ if (trailsize != 0) { + assert(!edata_guarded_get(*edata)); *trail = extent_split_impl(tsdn, pac, ehooks, *edata, size, trailsize, /* holding_core_locks */ true); if (*trail == NULL) { @@ -510,6 +519,7 @@ static edata_t * extent_recycle_split(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, edata_t *edata, bool growing_retained) { + assert(!edata_guarded_get(edata) || size == edata_size_get(edata)); malloc_mutex_assert_owner(tsdn, &ecache->mtx); edata_t *lead; @@ -576,8 +586,10 @@ extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); assert(!guarded || expand_edata == NULL); + assert(!guarded || alignment <= PAGE); malloc_mutex_lock(tsdn, &ecache->mtx); + edata_t *edata = extent_recycle_extract(tsdn, pac, ehooks, ecache, expand_edata, size, alignment, guarded); if (edata == NULL) { @@ -746,7 +758,6 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size = edata_size_get(edata); ehooks_zero(tsdn, ehooks, addr, size); } - return edata; label_err: malloc_mutex_unlock(tsdn, &pac->grow_mtx); @@ -801,6 +812,7 @@ extent_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, static edata_t * extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata, bool *coalesced) { + assert(!edata_guarded_get(edata)); /* * We avoid checking / locking inactive neighbors for large size * classes, since they are eagerly coalesced on deallocation which can @@ -907,7 +919,7 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, goto label_skip_coalesce; } if (!ecache->delay_coalesce) { - edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, edata, + edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, edata, NULL); } else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) { assert(ecache == &pac->ecache_dirty); @@ -1014,7 +1026,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, /* Avoid calling the default extent_dalloc unless have to. */ if (!ehooks_dalloc_will_fail(ehooks)) { - /* Restore guard pages for dalloc / unmap. */ + /* Remove guard pages for dalloc / unmap. */ if (edata_guarded_get(edata)) { assert(ehooks_are_default(ehooks)); san_unguard_pages_two_sided(tsdn, ehooks, edata, diff --git a/src/pac.c b/src/pac.c index e1f60025..c6d9f146 100644 --- a/src/pac.c +++ b/src/pac.c @@ -81,6 +81,9 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, if (decay_init(&pac->decay_muzzy, cur_time, muzzy_decay_ms)) { return true; } + if (san_bump_alloc_init(&pac->sba)) { + return true; + } pac->base = base; pac->emap = emap; @@ -132,18 +135,24 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, static edata_t * pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, - size_t alignment, bool zero) { + size_t alignment, bool zero, bool frequent_reuse) { assert(alignment <= PAGE); - size_t size_with_guards = size + SAN_PAGE_GUARDS_SIZE; - /* Alloc a non-guarded extent first.*/ - edata_t *edata = pac_alloc_real(tsdn, pac, ehooks, size_with_guards, - /* alignment */ PAGE, zero, /* guarded */ false); - if (edata != NULL) { - /* Add guards around it. */ - assert(edata_size_get(edata) == size_with_guards); - san_guard_pages(tsdn, ehooks, edata, pac->emap, true, true, - true); + edata_t *edata; + if (san_bump_enabled() && frequent_reuse) { + edata = san_bump_alloc(tsdn, &pac->sba, pac, ehooks, size, + zero); + } else { + size_t size_with_guards = san_two_side_guarded_sz(size); + /* Alloc a non-guarded extent first.*/ + edata = pac_alloc_real(tsdn, pac, ehooks, size_with_guards, + /* alignment */ PAGE, zero, /* guarded */ false); + if (edata != NULL) { + /* Add guards around it. */ + assert(edata_size_get(edata) == size_with_guards); + san_guard_pages_two_sided(tsdn, ehooks, edata, + pac->emap, true); + } } assert(edata == NULL || (edata_guarded_get(edata) && edata_size_get(edata) == size)); @@ -158,12 +167,21 @@ pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, pac_t *pac = (pac_t *)self; ehooks_t *ehooks = pac_ehooks_get(pac); - edata_t *edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment, - zero, guarded); + edata_t *edata = NULL; + /* + * The condition is an optimization - not frequently reused guarded + * allocations are never put in the ecache. pac_alloc_real also + * doesn't grow retained for guarded allocations. So pac_alloc_real + * for such allocations would always return NULL. + * */ + if (!guarded || frequent_reuse) { + edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment, + zero, guarded); + } if (edata == NULL && guarded) { /* No cached guarded extents; creating a new one. */ edata = pac_alloc_new_guarded(tsdn, pac, ehooks, size, - alignment, zero); + alignment, zero, frequent_reuse); } return edata; @@ -189,8 +207,8 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, } if (trail == NULL) { trail = ecache_alloc_grow(tsdn, pac, ehooks, - &pac->ecache_retained, edata, expand_amount, PAGE, - zero, /* guarded */ false); + &pac->ecache_retained, edata, expand_amount, PAGE, zero, + /* guarded */ false); mapped_add = expand_amount; } if (trail == NULL) { diff --git a/src/san_bump.c b/src/san_bump.c index 6098bd95..1a94e55d 100644 --- a/src/san_bump.c +++ b/src/san_bump.c @@ -7,28 +7,14 @@ #include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/edata_cache.h" -const size_t SBA_RETAINED_ALLOC_SIZE = 1024 * 1024 * 4; /* 4 MB */ - static bool san_bump_grow_locked(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac, ehooks_t *ehooks, size_t size); -bool -san_bump_alloc_init(san_bump_alloc_t* sba) { - bool err = malloc_mutex_init(&sba->mtx, "sanitizer_bump_allocator", - WITNESS_RANK_SAN_BUMP_ALLOC, malloc_mutex_rank_exclusive); - if (err) { - return true; - } - sba->curr_reg = NULL; - - return false; -} - edata_t * san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac, ehooks_t *ehooks, size_t size, bool zero) { - assert(maps_coalesce && opt_retain); + assert(san_bump_enabled()); edata_t* to_destroy; size_t guarded_size = san_one_side_guarded_sz(size); diff --git a/test/unit/san.c b/test/unit/san.c index 0daa282b..5b98f52e 100644 --- a/test/unit/san.c +++ b/test/unit/san.c @@ -13,6 +13,11 @@ verify_extent_guarded(tsdn_t *tsdn, void *ptr) { #define MAX_SMALL_ALLOCATIONS 4096 void *small_alloc[MAX_SMALL_ALLOCATIONS]; +/* + * This test allocates page sized slabs and checks that every two slabs have + * at least one page in between them. That page is supposed to be the guard + * page. + */ TEST_BEGIN(test_guarded_small) { test_skip_if(opt_prof); @@ -21,7 +26,8 @@ TEST_BEGIN(test_guarded_small) { VARIABLE_ARRAY(uintptr_t, pages, npages); /* Allocate to get sanitized pointers. */ - size_t sz = PAGE / 8; + size_t slab_sz = PAGE; + size_t sz = slab_sz / 8; unsigned n_alloc = 0; while (n_alloc < MAX_SMALL_ALLOCATIONS) { void *ptr = malloc(sz); @@ -50,8 +56,9 @@ TEST_BEGIN(test_guarded_small) { for (unsigned j = i + 1; j < npages; j++) { uintptr_t ptr_diff = pages[i] > pages[j] ? pages[i] - pages[j] : pages[j] - pages[i]; - expect_zu_gt((size_t)ptr_diff, 2 * PAGE, - "Pages should not be next to each other."); + expect_zu_ge((size_t)ptr_diff, slab_sz + PAGE, + "There should be at least one pages between " + "guarded slabs"); } } @@ -76,20 +83,15 @@ TEST_BEGIN(test_guarded_large) { } /* Verify the pages are not continuous, i.e. separated by guards. */ - uintptr_t min_diff = (uintptr_t)-1; for (unsigned i = 0; i < nlarge; i++) { for (unsigned j = i + 1; j < nlarge; j++) { uintptr_t ptr_diff = large[i] > large[j] ? large[i] - large[j] : large[j] - large[i]; expect_zu_ge((size_t)ptr_diff, large_sz + 2 * PAGE, - "Pages should not be next to each other."); - if (ptr_diff < min_diff) { - min_diff = ptr_diff; - } + "There should be at least two pages between " + " guarded large allocations"); } } - expect_zu_ge((size_t)min_diff, large_sz + 2 * PAGE, - "Pages should not be next to each other."); for (unsigned i = 0; i < nlarge; i++) { free((void *)large[i]);