diff --git a/Makefile.in b/Makefile.in index a6f61ced..abd361fd 100644 --- a/Makefile.in +++ b/Makefile.in @@ -119,6 +119,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/extent_dss.c \ $(srcroot)src/extent_mmap.c \ $(srcroot)src/fxp.c \ + $(srcroot)src/guard.c \ $(srcroot)src/hook.c \ $(srcroot)src/hpa.c \ $(srcroot)src/hpa_hooks.c \ @@ -218,6 +219,7 @@ TESTS_UNIT := \ ${srcroot}test/unit/fb.c \ $(srcroot)test/unit/fork.c \ ${srcroot}test/unit/fxp.c \ + ${srcroot}test/unit/guard.c \ $(srcroot)test/unit/hash.c \ $(srcroot)test/unit/hook.c \ $(srcroot)test/unit/hpa.c \ diff --git a/configure.ac b/configure.ac index 5a5887ac..7e2b44c5 100644 --- a/configure.ac +++ b/configure.ac @@ -2256,6 +2256,18 @@ else fi fi +dnl ============================================================================ +dnl Check for mprotect(2). + +JE_COMPILABLE([mprotect(2)], [ +#include +], [ + mprotect((void *)0, 0, PROT_NONE); +], [je_cv_mprotect]) +if test "x${je_cv_mprotect}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_MPROTECT], [ ]) +fi + dnl ============================================================================ dnl Check for __builtin_clz(), __builtin_clzl(), and __builtin_clzll(). diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 5410b160..fa81537c 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -221,7 +221,8 @@ large_dalloc_safety_checks(edata_t *edata, void *ptr, szind_t szind) { * The cost is low enough (as edata will be accessed anyway) to be * enabled all the time. */ - if (unlikely(edata_state_get(edata) != extent_state_active)) { + if (unlikely(edata == NULL || + edata_state_get(edata) != extent_state_active)) { safety_check_fail("Invalid deallocation detected: " "pages being freed (%p) not currently active, " "possibly caused by double free bugs.", diff --git a/include/jemalloc/internal/ecache.h b/include/jemalloc/internal/ecache.h index cc2752f5..dd1bc320 100644 --- a/include/jemalloc/internal/ecache.h +++ b/include/jemalloc/internal/ecache.h @@ -2,12 +2,14 @@ #define JEMALLOC_INTERNAL_ECACHE_H #include "jemalloc/internal/eset.h" +#include "jemalloc/internal/guard.h" #include "jemalloc/internal/mutex.h" typedef struct ecache_s ecache_t; struct ecache_s { malloc_mutex_t mtx; eset_t eset; + eset_t guarded_eset; /* All stored extents must be in the same state. */ extent_state_t state; /* The index of the ehooks the ecache is associated with. */ @@ -21,17 +23,22 @@ struct ecache_s { static inline size_t ecache_npages_get(ecache_t *ecache) { - return eset_npages_get(&ecache->eset); + return eset_npages_get(&ecache->eset) + + eset_npages_get(&ecache->guarded_eset); } + /* Get the number of extents in the given page size index. */ static inline size_t ecache_nextents_get(ecache_t *ecache, pszind_t ind) { - return eset_nextents_get(&ecache->eset, ind); + return eset_nextents_get(&ecache->eset, ind) + + eset_nextents_get(&ecache->guarded_eset, ind); } + /* Get the sum total bytes of the extents in the given page size index. */ static inline size_t ecache_nbytes_get(ecache_t *ecache, pszind_t ind) { - return eset_nbytes_get(&ecache->eset, ind); + return eset_nbytes_get(&ecache->eset, ind) + + eset_nbytes_get(&ecache->guarded_eset, ind); } static inline unsigned diff --git a/include/jemalloc/internal/edata.h b/include/jemalloc/internal/edata.h index ff14982c..af039ea7 100644 --- a/include/jemalloc/internal/edata.h +++ b/include/jemalloc/internal/edata.h @@ -98,12 +98,13 @@ struct edata_s { * c: committed * p: pai * z: zeroed + * g: guarded * t: state * i: szind * f: nfree * s: bin_shard * - * 00000000 ... 00000sss sssfffff fffffiii iiiiittt zpcbaaaa aaaaaaaa + * 00000000 ... 0000ssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa * * arena_ind: Arena from which this extent came, or all 1 bits if * unassociated. @@ -123,6 +124,9 @@ struct edata_s { * zeroed: The zeroed flag is used by extent recycling code to track * whether memory is zero-filled. * + * guarded: The guarded flag is use by the sanitizer to track whether + * the extent has page guards around it. + * * state: The state flag is an extent_state_t. * * szind: The szind flag indicates usable size class index for @@ -158,8 +162,12 @@ struct edata_s { #define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT) #define EDATA_BITS_ZEROED_MASK MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT) +#define EDATA_BITS_GUARDED_WIDTH 1 +#define EDATA_BITS_GUARDED_SHIFT (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT) +#define EDATA_BITS_GUARDED_MASK MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT) + #define EDATA_BITS_STATE_WIDTH 3 -#define EDATA_BITS_STATE_SHIFT (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT) +#define EDATA_BITS_STATE_SHIFT (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT) #define EDATA_BITS_STATE_MASK MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT) #define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) @@ -293,6 +301,12 @@ edata_state_get(const edata_t *edata) { EDATA_BITS_STATE_SHIFT); } +static inline bool +edata_guarded_get(const edata_t *edata) { + return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK) >> + EDATA_BITS_GUARDED_SHIFT); +} + static inline bool edata_zeroed_get(const edata_t *edata) { return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) >> @@ -505,6 +519,12 @@ edata_state_set(edata_t *edata, extent_state_t state) { ((uint64_t)state << EDATA_BITS_STATE_SHIFT); } +static inline void +edata_guarded_set(edata_t *edata, bool guarded) { + edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK) | + ((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT); +} + static inline void edata_zeroed_set(edata_t *edata, bool zeroed) { edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) | @@ -588,6 +608,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size, edata_szind_set(edata, szind); edata_sn_set(edata, sn); edata_state_set(edata, state); + edata_guarded_set(edata, false); edata_zeroed_set(edata, zeroed); edata_committed_set(edata, committed); edata_pai_set(edata, pai); @@ -606,6 +627,7 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) { edata_szind_set(edata, SC_NSIZES); edata_sn_set(edata, sn); edata_state_set(edata, extent_state_active); + edata_guarded_set(edata, false); edata_zeroed_set(edata, true); edata_committed_set(edata, true); /* diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index 064ecf5a..8d9513e2 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -63,6 +63,8 @@ bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, unsigned arena_ind); bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b); void ehooks_default_zero_impl(void *addr, size_t size); +void ehooks_default_guard_impl(void *guard1, void *guard2); +void ehooks_default_unguard_impl(void *guard1, void *guard2); /* * We don't officially support reentrancy from wtihin the extent hooks. But @@ -139,6 +141,15 @@ ehooks_merge_will_fail(ehooks_t *ehooks) { return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL; } +static inline bool +ehooks_guard_will_fail(ehooks_t *ehooks) { + /* + * Before the guard hooks are officially introduced, limit the use to + * the default hooks only. + */ + return !ehooks_are_default(ehooks); +} + /* * Some hooks are required to return zeroed memory in certain situations. In * debug mode, we do some heuristic checks that they did what they were supposed @@ -368,4 +379,34 @@ ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size) { } } +static inline bool +ehooks_guard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) { + bool err; + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + + if (extent_hooks == &ehooks_default_extent_hooks) { + ehooks_default_guard_impl(guard1, guard2); + err = false; + } else { + err = true; + } + + return err; +} + +static inline bool +ehooks_unguard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) { + bool err; + extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); + + if (extent_hooks == &ehooks_default_extent_hooks) { + ehooks_default_unguard_impl(guard1, guard2); + err = false; + } else { + err = true; + } + + return err; +} + #endif /* JEMALLOC_INTERNAL_EHOOKS_H */ diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 03eebdd5..73c55633 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -21,10 +21,10 @@ extern size_t opt_lg_extent_max_active_fit; edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, - bool zero); + bool zero, bool guarded); edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, - bool zero); + bool zero, bool guarded); void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata); edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, diff --git a/include/jemalloc/internal/guard.h b/include/jemalloc/internal/guard.h new file mode 100644 index 00000000..31f98c5f --- /dev/null +++ b/include/jemalloc/internal/guard.h @@ -0,0 +1,76 @@ +#ifndef JEMALLOC_INTERNAL_GUARD_H +#define JEMALLOC_INTERNAL_GUARD_H + +#include "jemalloc/internal/ehooks.h" +#include "jemalloc/internal/emap.h" + +#define PAGE_GUARDS_SIZE (2 * PAGE) + +#define SAN_GUARD_LARGE_EVERY_N_EXTENTS_DEFAULT 0 +#define SAN_GUARD_SMALL_EVERY_N_EXTENTS_DEFAULT 0 + +/* 0 means disabled, i.e. never guarded. */ +extern size_t opt_san_guard_large; +extern size_t opt_san_guard_small; + +void guard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap); +void unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap); +void tsd_san_init(tsd_t *tsd); + +static inline bool +san_enabled(void) { + return (opt_san_guard_large != 0 || opt_san_guard_small != 0); +} + +static inline bool +large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size, + size_t alignment) { + if (opt_san_guard_large == 0 || ehooks_guard_will_fail(ehooks) || + tsdn_null(tsdn)) { + return false; + } + + tsd_t *tsd = tsdn_tsd(tsdn); + uint64_t n = tsd_san_extents_until_guard_large_get(tsd); + assert(n >= 1); + if (n > 1) { + /* + * Subtract conditionally because the guard may not happen due + * to alignment or size restriction below. + */ + *tsd_san_extents_until_guard_largep_get(tsd) = n - 1; + } + + if (n == 1 && (alignment <= PAGE) && + (size + PAGE_GUARDS_SIZE <= SC_LARGE_MAXCLASS)) { + *tsd_san_extents_until_guard_largep_get(tsd) = + opt_san_guard_large; + return true; + } else { + assert(tsd_san_extents_until_guard_large_get(tsd) >= 1); + return false; + } +} + +static inline bool +slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) { + if (opt_san_guard_small == 0 || ehooks_guard_will_fail(ehooks) || + tsdn_null(tsdn)) { + return false; + } + + tsd_t *tsd = tsdn_tsd(tsdn); + uint64_t n = tsd_san_extents_until_guard_small_get(tsd); + assert(n >= 1); + if (n == 1) { + *tsd_san_extents_until_guard_smallp_get(tsd) = + opt_san_guard_small; + return true; + } else { + *tsd_san_extents_until_guard_smallp_get(tsd) = n - 1; + assert(tsd_san_extents_until_guard_small_get(tsd) >= 1); + return false; + } +} + +#endif /* JEMALLOC_INTERNAL_GUARD_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 78d1213e..418b0cb2 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -312,6 +312,9 @@ */ #undef JEMALLOC_MADVISE_NOCORE +/* Defined if mprotect(2) is available. */ +#undef JEMALLOC_HAVE_MPROTECT + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 97834131..3cf370c8 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -167,7 +167,7 @@ void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard); /* Gets an edata for the given allocation. */ edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, - size_t alignment, bool slab, szind_t szind, bool zero, + size_t alignment, bool slab, szind_t szind, bool zero, bool guarded, bool *deferred_work_generated); /* Returns true on error, in which case nothing changed. */ bool pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h index 035364e2..3d7993dd 100644 --- a/include/jemalloc/internal/pages.h +++ b/include/jemalloc/internal/pages.h @@ -110,5 +110,7 @@ bool pages_dontdump(void *addr, size_t size); bool pages_dodump(void *addr, size_t size); bool pages_boot(void); void pages_set_thp_state (void *ptr, size_t size); +void pages_mark_guards(void *head, void *tail); +void pages_unmark_guards(void *head, void *tail); #endif /* JEMALLOC_INTERNAL_PAGES_EXTERNS_H */ diff --git a/include/jemalloc/internal/pai.h b/include/jemalloc/internal/pai.h index ca5f616a..f8f7d667 100644 --- a/include/jemalloc/internal/pai.h +++ b/include/jemalloc/internal/pai.h @@ -7,7 +7,8 @@ typedef struct pai_s pai_t; struct pai_s { /* Returns NULL on failure. */ edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool *deferred_work_generated); + size_t alignment, bool zero, bool guarded, + bool *deferred_work_generated); /* * Returns the number of extents added to the list (which may be fewer * than requested, in case of OOM). The list should already be @@ -37,8 +38,8 @@ struct pai_s { static inline edata_t * pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, - bool *deferred_work_generated) { - return self->alloc(tsdn, self, size, alignment, zero, + bool guarded, bool *deferred_work_generated) { + return self->alloc(tsdn, self, size, alignment, zero, guarded, deferred_work_generated); } diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index d22fdc94..86d52778 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -73,6 +73,8 @@ typedef ql_elm(tsd_t) tsd_link_t; O(peak_dalloc_event_wait, uint64_t, uint64_t) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ O(prng_state, uint64_t, uint64_t) \ + O(san_extents_until_guard_small, uint64_t, uint64_t) \ + O(san_extents_until_guard_large, uint64_t, uint64_t) \ O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ @@ -103,6 +105,8 @@ typedef ql_elm(tsd_t) tsd_link_t; /* peak_dalloc_event_wait */ 0, \ /* prof_tdata */ NULL, \ /* prng_state */ 0, \ + /* san_extents_until_guard_small */ 0, \ + /* san_extents_until_guard_large */ 0, \ /* iarena */ NULL, \ /* arena */ NULL, \ /* arena_decay_ticker */ \ diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index 597b247b..75d66800 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -59,6 +59,7 @@ + diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index d063a019..c5bb4cfe 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -61,6 +61,9 @@ Source Files + + Source Files + Source Files diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index 46633e82..d25768e1 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -59,6 +59,7 @@ + diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters index d063a019..c5bb4cfe 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -61,6 +61,9 @@ Source Files + + Source Files + Source Files diff --git a/src/arena.c b/src/arena.c index c2842c6c..8147d14b 100644 --- a/src/arena.c +++ b/src/arena.c @@ -6,6 +6,7 @@ #include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" +#include "jemalloc/internal/guard.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/safety_check.h" @@ -327,9 +328,10 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, szind_t szind = sz_size2index(usize); size_t esize = usize + sz_large_pad; + bool guarded = large_extent_decide_guard(tsdn, arena_get_ehooks(arena), + esize, alignment); edata_t *edata = pa_alloc(tsdn, &arena->pa_shard, esize, alignment, - /* slab */ false, szind, zero, &deferred_work_generated); - + /* slab */ false, szind, zero, guarded, &deferred_work_generated); assert(deferred_work_generated == false); if (edata != NULL) { @@ -827,9 +829,10 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); + bool guarded = slab_extent_decide_guard(tsdn, arena_get_ehooks(arena)); edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size, - PAGE, /* slab */ true, /* szind */ binind, /* zero */ false, - &deferred_work_generated); + /* alignment */ PAGE, /* slab */ true, /* szind */ binind, + /* zero */ false, guarded, &deferred_work_generated); if (deferred_work_generated) { arena_handle_deferred_work(tsdn, arena); diff --git a/src/ecache.c b/src/ecache.c index 3c1a2274..26fc2112 100644 --- a/src/ecache.c +++ b/src/ecache.c @@ -1,6 +1,8 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" +#include "jemalloc/internal/guard.h" + bool ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, unsigned ind, bool delay_coalesce) { @@ -12,6 +14,8 @@ ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, unsigned ind, ecache->ind = ind; ecache->delay_coalesce = delay_coalesce; eset_init(&ecache->eset, state); + eset_init(&ecache->guarded_eset, state); + return false; } diff --git a/src/ehooks.c b/src/ehooks.c index 5d12d003..383e9de6 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -244,6 +244,16 @@ ehooks_default_zero_impl(void *addr, size_t size) { } } +void +ehooks_default_guard_impl(void *guard1, void *guard2) { + pages_mark_guards(guard1, guard2); +} + +void +ehooks_default_unguard_impl(void *guard1, void *guard2) { + pages_unmark_guards(guard1, guard2); +} + const extent_hooks_t ehooks_default_extent_hooks = { ehooks_default_alloc, ehooks_default_dalloc, diff --git a/src/extent.c b/src/extent.c index 04001142..84ecd6b2 100644 --- a/src/extent.c +++ b/src/extent.c @@ -37,14 +37,14 @@ static atomic_zu_t highpages; static void extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata); static edata_t *extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t usize, size_t alignment, - bool zero, bool *commit, bool growing_retained); + bool zero, bool *commit, bool growing_retained, bool guarded); static edata_t *extent_try_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata, bool *coalesced); static void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *edata); static edata_t *extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *expand_edata, size_t size, size_t alignment, - bool zero, bool *commit); + bool zero, bool *commit, bool guarded); static edata_t *extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment, bool zero, bool *commit); @@ -80,7 +80,8 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t * ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, - edata_t *expand_edata, size_t size, size_t alignment, bool zero) { + edata_t *expand_edata, size_t size, size_t alignment, bool zero, + bool guarded) { assert(size != 0); assert(alignment != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -88,14 +89,15 @@ ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, bool commit = true; edata_t *edata = extent_recycle(tsdn, pac, ehooks, ecache, expand_edata, - size, alignment, zero, &commit, false); + size, alignment, zero, &commit, false, guarded); assert(edata == NULL || edata_pai_get(edata) == EXTENT_PAI_PAC); return edata; } edata_t * ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, - edata_t *expand_edata, size_t size, size_t alignment, bool zero) { + edata_t *expand_edata, size_t size, size_t alignment, bool zero, + bool guarded) { assert(size != 0); assert(alignment != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -103,7 +105,7 @@ ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, bool commit = true; edata_t *edata = extent_alloc_retained(tsdn, pac, ehooks, expand_edata, - size, alignment, zero, &commit); + size, alignment, zero, &commit, guarded); if (edata == NULL) { if (opt_retain && expand_edata != NULL) { /* @@ -114,6 +116,14 @@ ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, */ return NULL; } + if (guarded) { + /* + * Means no cached guarded extents available (and no + * grow_retained was attempted). The pac_alloc flow + * will alloc regular extents to make new guarded ones. + */ + return NULL; + } void *new_addr = (expand_edata == NULL) ? NULL : edata_past_get(expand_edata); edata = extent_alloc_wrapper(tsdn, pac, ehooks, new_addr, @@ -151,9 +161,19 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata; while (true) { /* Get the LRU extent, if any. */ - edata = edata_list_inactive_first(&ecache->eset.lru); + eset_t *eset = &ecache->eset; + edata = edata_list_inactive_first(&eset->lru); if (edata == NULL) { - goto label_return; + /* + * Next check if there are guarded extents. They are + * more expensive to purge (since they are not + * mergeable), thus in favor of caching them longer. + */ + eset = &ecache->guarded_eset; + edata = edata_list_inactive_first(&eset->lru); + if (edata == NULL) { + goto label_return; + } } /* Check the eviction limit. */ size_t extents_npages = ecache_npages_get(ecache); @@ -161,7 +181,7 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata = NULL; goto label_return; } - eset_remove(&ecache->eset, edata); + eset_remove(eset, edata); if (!ecache->delay_coalesce) { break; } @@ -234,17 +254,19 @@ extent_deactivate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, assert(edata_state_get(edata) == extent_state_active); emap_update_edata_state(tsdn, pac->emap, edata, ecache->state); - eset_insert(&ecache->eset, edata); + eset_t *eset = edata_guarded_get(edata) ? &ecache->guarded_eset : + &ecache->eset; + eset_insert(eset, edata); } static void -extent_activate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, +extent_activate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, eset_t *eset, edata_t *edata) { assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache)); assert(edata_state_get(edata) == ecache->state || edata_state_get(edata) == extent_state_merging); - eset_remove(&ecache->eset, edata); + eset_remove(eset, edata); emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active); } @@ -350,7 +372,8 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, pac_t *pac, */ static edata_t * extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, - ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment) { + ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, + bool guarded) { malloc_mutex_assert_owner(tsdn, &ecache->mtx); assert(alignment > 0); if (config_debug && expand_edata != NULL) { @@ -366,6 +389,7 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, } edata_t *edata; + eset_t *eset = guarded ? &ecache->guarded_eset : &ecache->eset; if (expand_edata != NULL) { edata = emap_try_acquire_edata_neighbor_expand(tsdn, pac->emap, expand_edata, EXTENT_PAI_PAC, ecache->state); @@ -382,7 +406,7 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, * If split and merge are not allowed (Windows w/o retain), try * exact fit only. */ - bool exact_only = (!maps_coalesce && !opt_retain); + bool exact_only = (!maps_coalesce && !opt_retain) || guarded; /* * A large extent might be broken up from its original size to * some small size to satisfy a small request. When that small @@ -394,13 +418,13 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, */ unsigned lg_max_fit = ecache->delay_coalesce ? (unsigned)opt_lg_extent_max_active_fit : SC_PTR_BITS; - edata = eset_fit(&ecache->eset, size, alignment, exact_only, - lg_max_fit); + edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit); } if (edata == NULL) { return NULL; } - extent_activate_locked(tsdn, pac, ecache, edata); + assert(!guarded || edata_guarded_get(edata)); + extent_activate_locked(tsdn, pac, ecache, eset, edata); return edata; } @@ -551,13 +575,14 @@ extent_recycle_split(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, static edata_t * extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, bool zero, - bool *commit, bool growing_retained) { + bool *commit, bool growing_retained, bool guarded) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); + assert(!guarded || expand_edata == NULL); malloc_mutex_lock(tsdn, &ecache->mtx); edata_t *edata = extent_recycle_extract(tsdn, pac, ehooks, ecache, - expand_edata, size, alignment); + expand_edata, size, alignment, guarded); if (edata == NULL) { malloc_mutex_unlock(tsdn, &ecache->mtx); return NULL; @@ -734,7 +759,7 @@ label_err: static edata_t * extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *expand_edata, size_t size, size_t alignment, bool zero, - bool *commit) { + bool *commit, bool guarded) { assert(size != 0); assert(alignment != 0); @@ -742,13 +767,13 @@ extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata = extent_recycle(tsdn, pac, ehooks, &pac->ecache_retained, expand_edata, size, alignment, zero, commit, - /* growing_retained */ true); + /* growing_retained */ true, guarded); if (edata != NULL) { malloc_mutex_unlock(tsdn, &pac->grow_mtx); if (config_prof) { extent_gdump_add(tsdn, edata); } - } else if (opt_retain && expand_edata == NULL) { + } else if (opt_retain && expand_edata == NULL && !guarded) { edata = extent_grow_retained(tsdn, pac, ehooks, size, alignment, zero, commit); /* extent_grow_retained() always releases pac->grow_mtx. */ @@ -910,6 +935,9 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, emap_assert_mapped(tsdn, pac->emap, edata); + if (edata_guarded_get(edata)) { + goto label_skip_coalesce; + } if (!ecache->delay_coalesce) { edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, edata, NULL); @@ -931,6 +959,7 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, return; } } +label_skip_coalesce: extent_deactivate_locked(tsdn, pac, ecache, edata); malloc_mutex_unlock(tsdn, &ecache->mtx); @@ -981,6 +1010,11 @@ extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, /* Avoid calling the default extent_dalloc unless have to. */ if (!ehooks_dalloc_will_fail(ehooks)) { + /* Restore guard pages for dalloc / unmap. */ + if (edata_guarded_get(edata)) { + assert(ehooks_are_default(ehooks)); + unguard_pages(tsdn, ehooks, edata, pac->emap); + } /* * Deregister first to avoid a race with other allocating * threads, and reregister if deallocation fails. diff --git a/src/guard.c b/src/guard.c new file mode 100644 index 00000000..07232199 --- /dev/null +++ b/src/guard.c @@ -0,0 +1,63 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/ehooks.h" +#include "jemalloc/internal/guard.h" +#include "jemalloc/internal/tsd.h" + +/* The sanitizer options. */ +size_t opt_san_guard_large = SAN_GUARD_LARGE_EVERY_N_EXTENTS_DEFAULT; +size_t opt_san_guard_small = SAN_GUARD_SMALL_EVERY_N_EXTENTS_DEFAULT; + +void +guard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap) { + emap_deregister_boundary(tsdn, emap, edata); + + size_t size_with_guards = edata_size_get(edata); + size_t usize = size_with_guards - PAGE_GUARDS_SIZE; + + uintptr_t guard1 = (uintptr_t)edata_base_get(edata); + uintptr_t addr = guard1 + PAGE; + uintptr_t guard2 = addr + usize; + + assert(edata_state_get(edata) == extent_state_active); + ehooks_guard(tsdn, ehooks, (void *)guard1, (void *)guard2); + + /* Update the guarded addr and usable size of the edata. */ + edata_size_set(edata, usize); + edata_addr_set(edata, (void *)addr); + edata_guarded_set(edata, true); + + /* The new boundary will be registered on the pa_alloc path. */ +} + +void +unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap) { + /* Remove the inner boundary which no longer exists. */ + emap_deregister_boundary(tsdn, emap, edata); + + size_t size = edata_size_get(edata); + size_t size_with_guards = size + PAGE_GUARDS_SIZE; + + uintptr_t addr = (uintptr_t)edata_base_get(edata); + uintptr_t guard1 = addr - PAGE; + uintptr_t guard2 = addr + size; + + assert(edata_state_get(edata) == extent_state_active); + ehooks_unguard(tsdn, ehooks, (void *)guard1, (void *)guard2); + + /* Update the true addr and usable size of the edata. */ + edata_size_set(edata, size_with_guards); + edata_addr_set(edata, (void *)guard1); + edata_guarded_set(edata, false); + + /* Then re-register the outer boundary including the guards. */ + emap_register_boundary(tsdn, emap, edata, SC_NSIZES, /* slab */ false); +} + +void +tsd_san_init(tsd_t *tsd) { + *tsd_san_extents_until_guard_smallp_get(tsd) = opt_san_guard_small; + *tsd_san_extents_until_guard_largep_get(tsd) = opt_san_guard_large; +} diff --git a/src/hpa.c b/src/hpa.c index d7422a3c..82b9c992 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -9,7 +9,7 @@ #define HPA_EDEN_SIZE (128 * HUGEPAGE) static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool *deferred_work_generated); + size_t alignment, bool zero, bool guarded, bool *deferred_work_generated); static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated); static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, @@ -750,8 +750,9 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs, static edata_t * hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, - bool *deferred_work_generated) { + bool guarded, bool *deferred_work_generated) { assert((size & PAGE_MASK) == 0); + assert(!guarded); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -796,7 +797,6 @@ hpa_dalloc_prepare_unlocked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) { assert(edata_state_get(edata) == extent_state_active); assert(edata_arena_ind_get(edata) == shard->ind); assert(edata_szind_get_maybe_invalid(edata) == SC_NSIZES); - assert(!edata_slab_get(edata)); assert(edata_committed_get(edata)); assert(edata_base_get(edata) != NULL); @@ -865,6 +865,7 @@ hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list, static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) { + assert(!edata_guarded_get(edata)); /* Just a dalloc_batch of size 1; this lets us share logic. */ edata_list_active_t dalloc_list; edata_list_active_init(&dalloc_list); diff --git a/src/jemalloc.c b/src/jemalloc.c index 7ffa5533..907265c0 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -10,6 +10,7 @@ #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/fxp.h" +#include "jemalloc/internal/guard.h" #include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/log.h" @@ -1616,6 +1617,14 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], } CONF_CONTINUE; } + + CONF_HANDLE_SIZE_T(opt_san_guard_small, + "san_guard_small", 0, SIZE_T_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false) + CONF_HANDLE_SIZE_T(opt_san_guard_large, + "san_guard_large", 0, SIZE_T_MAX, + CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false) + CONF_ERROR("Invalid conf pair", k, klen, v, vlen); #undef CONF_ERROR #undef CONF_CONTINUE diff --git a/src/pa.c b/src/pa.c index 249de24a..649b9c2e 100644 --- a/src/pa.c +++ b/src/pa.c @@ -1,6 +1,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" +#include "jemalloc/internal/guard.h" #include "jemalloc/internal/hpa.h" static void @@ -118,15 +119,17 @@ pa_get_pai(pa_shard_t *shard, edata_t *edata) { edata_t * pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, - bool slab, szind_t szind, bool zero, bool *deferred_work_generated) { + bool slab, szind_t szind, bool zero, bool guarded, + bool *deferred_work_generated) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); + assert(!guarded || alignment <= PAGE); edata_t *edata = NULL; *deferred_work_generated = false; - if (pa_shard_uses_hpa(shard)) { + if (!guarded && pa_shard_uses_hpa(shard)) { edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment, - zero, deferred_work_generated); + zero, /* guarded */ false, deferred_work_generated); } /* * Fall back to the PAC if the HPA is off or couldn't serve the given @@ -134,10 +137,10 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, */ if (edata == NULL) { edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero, - deferred_work_generated); + guarded, deferred_work_generated); } - if (edata != NULL) { + assert(edata_size_get(edata) == size); pa_nactive_add(shard, size >> LG_PAGE); emap_remap(tsdn, shard->emap, edata, szind, slab); edata_szind_set(edata, szind); @@ -145,8 +148,6 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, if (slab && (size > 2 * PAGE)) { emap_register_interior(tsdn, shard->emap, edata, szind); } - } - if (edata != NULL) { assert(edata_arena_ind_get(edata) == shard->ind); } return edata; @@ -158,7 +159,9 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, assert(new_size > old_size); assert(edata_size_get(edata) == old_size); assert((new_size & PAGE_MASK) == 0); - + if (edata_guarded_get(edata)) { + return true; + } size_t expand_amount = new_size - old_size; pai_t *pai = pa_get_pai(shard, edata); @@ -181,6 +184,9 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, assert(new_size < old_size); assert(edata_size_get(edata) == old_size); assert((new_size & PAGE_MASK) == 0); + if (edata_guarded_get(edata)) { + return true; + } size_t shrink_amount = old_size - new_size; pai_t *pai = pa_get_pai(shard, edata); @@ -202,7 +208,10 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, emap_remap(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false); if (edata_slab_get(edata)) { emap_deregister_interior(tsdn, shard->emap, edata); - edata_slab_set(edata, false); + /* + * The slab state of the extent isn't cleared. It may be used + * by the pai implementation, e.g. to make caching decisions. + */ } edata_addr_set(edata, edata_base_get(edata)); edata_szind_set(edata, SC_NSIZES); diff --git a/src/pac.c b/src/pac.c index 03e31972..8ce3159c 100644 --- a/src/pac.c +++ b/src/pac.c @@ -2,9 +2,10 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/pac.h" +#include "jemalloc/internal/guard.h" static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool *deferred_work_generated); + size_t alignment, bool zero, bool guarded, bool *deferred_work_generated); static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, @@ -109,28 +110,66 @@ pac_may_have_muzzy(pac_t *pac) { } static edata_t * -pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, - bool zero, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; +pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, + size_t alignment, bool zero, bool guarded) { + assert(!guarded || alignment <= PAGE); - *deferred_work_generated = false; - - ehooks_t *ehooks = pac_ehooks_get(pac); edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, - NULL, size, alignment, zero); + NULL, size, alignment, zero, guarded); if (edata == NULL && pac_may_have_muzzy(pac)) { edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, - NULL, size, alignment, zero); + NULL, size, alignment, zero, guarded); } if (edata == NULL) { edata = ecache_alloc_grow(tsdn, pac, ehooks, - &pac->ecache_retained, NULL, size, alignment, zero); + &pac->ecache_retained, NULL, size, alignment, zero, + guarded); if (config_stats && edata != NULL) { atomic_fetch_add_zu(&pac->stats->pac_mapped, size, ATOMIC_RELAXED); } } + + return edata; +} + +static edata_t * +pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, + size_t alignment, bool zero) { + assert(alignment <= PAGE); + + size_t size_with_guards = size + PAGE_GUARDS_SIZE; + /* Alloc a non-guarded extent first.*/ + edata_t *edata = pac_alloc_real(tsdn, pac, ehooks, size_with_guards, + /* alignment */ PAGE, zero, /* guarded */ false); + if (edata != NULL) { + /* Add guards around it. */ + assert(edata_size_get(edata) == size_with_guards); + guard_pages(tsdn, ehooks, edata, pac->emap); + } + assert(edata == NULL || (edata_guarded_get(edata) && + edata_size_get(edata) == size)); + + return edata; +} + +static edata_t * +pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, + bool zero, bool guarded, bool *deferred_work_generated) { + *deferred_work_generated = false; + + pac_t *pac = (pac_t *)self; + ehooks_t *ehooks = pac_ehooks_get(pac); + + edata_t *edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment, + zero, guarded); + if (edata == NULL && guarded) { + /* No cached guarded extents; creating a new one. */ + edata = pac_alloc_new_guarded(tsdn, pac, ehooks, size, + alignment, zero); + } + return edata; } @@ -149,15 +188,15 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, return true; } edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, - edata, expand_amount, PAGE, zero); + edata, expand_amount, PAGE, zero, /* guarded*/ false); if (trail == NULL) { trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, - edata, expand_amount, PAGE, zero); + edata, expand_amount, PAGE, zero, /* guarded*/ false); } if (trail == NULL) { trail = ecache_alloc_grow(tsdn, pac, ehooks, &pac->ecache_retained, edata, expand_amount, PAGE, - zero); + zero, /* guarded */ false); mapped_add = expand_amount; } if (trail == NULL) { @@ -203,6 +242,27 @@ pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) { pac_t *pac = (pac_t *)self; ehooks_t *ehooks = pac_ehooks_get(pac); + + if (edata_guarded_get(edata)) { + /* + * Because cached guarded extents do exact fit only, large + * guarded extents are restored on dalloc eagerly (otherwise + * they will not be reused efficiently). Slab sizes have a + * limited number of size classes, and tend to cycle faster. + * + * In the case where coalesce is restrained (VirtualFree on + * Windows), guarded extents are also not cached -- otherwise + * during arena destroy / reset, the retained extents would not + * be whole regions (i.e. they are split between regular and + * guarded). + */ + if (!edata_slab_get(edata) || !maps_coalesce) { + assert(edata_size_get(edata) >= SC_LARGE_MINCLASS || + !maps_coalesce); + unguard_pages(tsdn, ehooks, edata, pac->emap); + } + } + ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata); /* Purging of deallocated pages is deferred */ *deferred_work_generated = true; diff --git a/src/pages.c b/src/pages.c index 42618858..a8d9988b 100644 --- a/src/pages.c +++ b/src/pages.c @@ -316,14 +316,10 @@ pages_unmap(void *addr, size_t size) { } static bool -pages_commit_impl(void *addr, size_t size, bool commit) { +os_pages_commit(void *addr, size_t size, bool commit) { assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); - if (os_overcommits) { - return true; - } - #ifdef _WIN32 return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT))); @@ -348,6 +344,15 @@ pages_commit_impl(void *addr, size_t size, bool commit) { #endif } +static bool +pages_commit_impl(void *addr, size_t size, bool commit) { + if (os_overcommits) { + return true; + } + + return os_pages_commit(addr, size, commit); +} + bool pages_commit(void *addr, size_t size) { return pages_commit_impl(addr, size, true); @@ -358,6 +363,38 @@ pages_decommit(void *addr, size_t size) { return pages_commit_impl(addr, size, false); } +void +pages_mark_guards(void *head, void *tail) { + assert(head != NULL && tail != NULL); + assert((uintptr_t)head < (uintptr_t)tail); +#ifdef JEMALLOC_HAVE_MPROTECT + mprotect(head, PAGE, PROT_NONE); + mprotect(tail, PAGE, PROT_NONE); +#else + /* Decommit sets to PROT_NONE / MEM_DECOMMIT. */ + os_pages_commit(head, PAGE, false); + os_pages_commit(tail, PAGE, false); +#endif +} + +void +pages_unmark_guards(void *head, void *tail) { + assert(head != NULL && tail != NULL); + assert((uintptr_t)head < (uintptr_t)tail); +#ifdef JEMALLOC_HAVE_MPROTECT + size_t range = (uintptr_t)tail - (uintptr_t)head + PAGE; + if (range <= SC_LARGE_MINCLASS) { + mprotect(head, range, PROT_READ | PROT_WRITE); + } else { + mprotect(head, PAGE, PROT_READ | PROT_WRITE); + mprotect(tail, PAGE, PROT_READ | PROT_WRITE); + } +#else + os_pages_commit(head, PAGE, true); + os_pages_commit(tail, PAGE, true); +#endif +} + bool pages_purge_lazy(void *addr, size_t size) { assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); diff --git a/src/pai.c b/src/pai.c index e863a9be..86b8ee5b 100644 --- a/src/pai.c +++ b/src/pai.c @@ -7,7 +7,7 @@ pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs, for (size_t i = 0; i < nallocs; i++) { bool deferred_by_alloc = false; edata_t *edata = pai_alloc(tsdn, self, size, PAGE, - /* zero */ false, &deferred_by_alloc); + /* zero */ false, /* guarded */ false, &deferred_by_alloc); *deferred_work_generated |= deferred_by_alloc; if (edata == NULL) { return i; diff --git a/src/sec.c b/src/sec.c index c6f611f5..0f95a0d0 100644 --- a/src/sec.c +++ b/src/sec.c @@ -4,7 +4,7 @@ #include "jemalloc/internal/sec.h" static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool *deferred_work_generated); + size_t alignment, bool zero, bool guarded, bool *deferred_work_generated); static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, @@ -218,8 +218,9 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, static edata_t * sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, - bool *deferred_work_generated) { + bool guarded, bool *deferred_work_generated) { assert((size & PAGE_MASK) == 0); + assert(!guarded); sec_t *sec = (sec_t *)self; *deferred_work_generated = false; @@ -227,7 +228,7 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, if (zero || alignment > PAGE || sec->opts.nshards == 0 || size > sec->opts.max_alloc) { return pai_alloc(tsdn, sec->fallback, size, alignment, zero, - deferred_work_generated); + /* guarded */ false, deferred_work_generated); } pszind_t pszind = sz_psz2ind(size); sec_shard_t *shard = sec_shard_pick(tsdn, sec); @@ -250,7 +251,7 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, size); } else { edata = pai_alloc(tsdn, sec->fallback, size, alignment, - zero, deferred_work_generated); + zero, /* guarded */ false, deferred_work_generated); } } return edata; diff --git a/src/tsd.c b/src/tsd.c index 6820eb62..31ff2f23 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -2,6 +2,7 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/guard.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" @@ -242,6 +243,7 @@ tsd_data_init(tsd_t *tsd) { rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); tsd_prng_state_init(tsd); tsd_te_init(tsd); /* event_init may use the prng state above. */ + tsd_san_init(tsd); return tsd_tcache_enabled_data_init(tsd); } @@ -269,6 +271,7 @@ tsd_data_init_nocleanup(tsd_t *tsd) { *tsd_reentrancy_levelp_get(tsd) = 1; tsd_prng_state_init(tsd); tsd_te_init(tsd); /* event_init may use the prng state above. */ + tsd_san_init(tsd); assert_tsd_data_cleanup_done(tsd); return false; diff --git a/test/include/test/arena_decay.h b/test/include/test/arena_decay.h new file mode 100644 index 00000000..da659212 --- /dev/null +++ b/test/include/test/arena_decay.h @@ -0,0 +1,149 @@ +static unsigned +do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { + unsigned arena_ind; + size_t sz = sizeof(unsigned); + expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), + 0, "Unexpected mallctl() failure"); + size_t mib[3]; + size_t miblen = sizeof(mib)/sizeof(size_t); + + expect_d_eq(mallctlnametomib("arena.0.dirty_decay_ms", mib, &miblen), + 0, "Unexpected mallctlnametomib() failure"); + mib[1] = (size_t)arena_ind; + expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, + (void *)&dirty_decay_ms, sizeof(dirty_decay_ms)), 0, + "Unexpected mallctlbymib() failure"); + + expect_d_eq(mallctlnametomib("arena.0.muzzy_decay_ms", mib, &miblen), + 0, "Unexpected mallctlnametomib() failure"); + mib[1] = (size_t)arena_ind; + expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, + (void *)&muzzy_decay_ms, sizeof(muzzy_decay_ms)), 0, + "Unexpected mallctlbymib() failure"); + + return arena_ind; +} + +static void +do_arena_destroy(unsigned arena_ind) { + size_t mib[3]; + size_t miblen = sizeof(mib)/sizeof(size_t); + expect_d_eq(mallctlnametomib("arena.0.destroy", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[1] = (size_t)arena_ind; + expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); +} + +static void +do_epoch(void) { + uint64_t epoch = 1; + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), + 0, "Unexpected mallctl() failure"); +} + +static void +do_purge(unsigned arena_ind) { + size_t mib[3]; + size_t miblen = sizeof(mib)/sizeof(size_t); + expect_d_eq(mallctlnametomib("arena.0.purge", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[1] = (size_t)arena_ind; + expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); +} + +static void +do_decay(unsigned arena_ind) { + size_t mib[3]; + size_t miblen = sizeof(mib)/sizeof(size_t); + expect_d_eq(mallctlnametomib("arena.0.decay", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[1] = (size_t)arena_ind; + expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); +} + +static uint64_t +get_arena_npurge_impl(const char *mibname, unsigned arena_ind) { + size_t mib[4]; + size_t miblen = sizeof(mib)/sizeof(size_t); + expect_d_eq(mallctlnametomib(mibname, mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[2] = (size_t)arena_ind; + uint64_t npurge = 0; + size_t sz = sizeof(npurge); + expect_d_eq(mallctlbymib(mib, miblen, (void *)&npurge, &sz, NULL, 0), + config_stats ? 0 : ENOENT, "Unexpected mallctlbymib() failure"); + return npurge; +} + +static uint64_t +get_arena_dirty_npurge(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind); +} + +static uint64_t +get_arena_dirty_purged(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind); +} + +static uint64_t +get_arena_muzzy_npurge(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind); +} + +static uint64_t +get_arena_npurge(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) + + get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind); +} + +static size_t +get_arena_pdirty(unsigned arena_ind) { + do_epoch(); + size_t mib[4]; + size_t miblen = sizeof(mib)/sizeof(size_t); + expect_d_eq(mallctlnametomib("stats.arenas.0.pdirty", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[2] = (size_t)arena_ind; + size_t pdirty; + size_t sz = sizeof(pdirty); + expect_d_eq(mallctlbymib(mib, miblen, (void *)&pdirty, &sz, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); + return pdirty; +} + +static size_t +get_arena_pmuzzy(unsigned arena_ind) { + do_epoch(); + size_t mib[4]; + size_t miblen = sizeof(mib)/sizeof(size_t); + expect_d_eq(mallctlnametomib("stats.arenas.0.pmuzzy", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[2] = (size_t)arena_ind; + size_t pmuzzy; + size_t sz = sizeof(pmuzzy); + expect_d_eq(mallctlbymib(mib, miblen, (void *)&pmuzzy, &sz, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); + return pmuzzy; +} + +static void * +do_mallocx(size_t size, int flags) { + void *p = mallocx(size, flags); + expect_ptr_not_null(p, "Unexpected mallocx() failure"); + return p; +} + +static void +generate_dirty(unsigned arena_ind, size_t size) { + int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; + void *p = do_mallocx(size, flags); + dallocx(p, flags); +} + diff --git a/test/include/test/guard.h b/test/include/test/guard.h new file mode 100644 index 00000000..691dc508 --- /dev/null +++ b/test/include/test/guard.h @@ -0,0 +1,6 @@ +static inline bool +extent_is_guarded(tsdn_t *tsdn, void *ptr) { + edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr); + return edata_guarded_get(edata); +} + diff --git a/test/unit/arena_decay.c b/test/unit/arena_decay.c index 9fca5385..bbfd23a5 100644 --- a/test/unit/arena_decay.c +++ b/test/unit/arena_decay.c @@ -1,4 +1,5 @@ #include "test/jemalloc_test.h" +#include "test/arena_decay.h" #include "jemalloc/internal/ticker.h" @@ -22,155 +23,6 @@ nstime_update_mock(nstime_t *time) { } } -static unsigned -do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) { - unsigned arena_ind; - size_t sz = sizeof(unsigned); - expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), - 0, "Unexpected mallctl() failure"); - size_t mib[3]; - size_t miblen = sizeof(mib)/sizeof(size_t); - - expect_d_eq(mallctlnametomib("arena.0.dirty_decay_ms", mib, &miblen), - 0, "Unexpected mallctlnametomib() failure"); - mib[1] = (size_t)arena_ind; - expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, - (void *)&dirty_decay_ms, sizeof(dirty_decay_ms)), 0, - "Unexpected mallctlbymib() failure"); - - expect_d_eq(mallctlnametomib("arena.0.muzzy_decay_ms", mib, &miblen), - 0, "Unexpected mallctlnametomib() failure"); - mib[1] = (size_t)arena_ind; - expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, - (void *)&muzzy_decay_ms, sizeof(muzzy_decay_ms)), 0, - "Unexpected mallctlbymib() failure"); - - return arena_ind; -} - -static void -do_arena_destroy(unsigned arena_ind) { - size_t mib[3]; - size_t miblen = sizeof(mib)/sizeof(size_t); - expect_d_eq(mallctlnametomib("arena.0.destroy", mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); - mib[1] = (size_t)arena_ind; - expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, - "Unexpected mallctlbymib() failure"); -} - -void -do_epoch(void) { - uint64_t epoch = 1; - expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), - 0, "Unexpected mallctl() failure"); -} - -void -do_purge(unsigned arena_ind) { - size_t mib[3]; - size_t miblen = sizeof(mib)/sizeof(size_t); - expect_d_eq(mallctlnametomib("arena.0.purge", mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); - mib[1] = (size_t)arena_ind; - expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, - "Unexpected mallctlbymib() failure"); -} - -void -do_decay(unsigned arena_ind) { - size_t mib[3]; - size_t miblen = sizeof(mib)/sizeof(size_t); - expect_d_eq(mallctlnametomib("arena.0.decay", mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); - mib[1] = (size_t)arena_ind; - expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, - "Unexpected mallctlbymib() failure"); -} - -static uint64_t -get_arena_npurge_impl(const char *mibname, unsigned arena_ind) { - size_t mib[4]; - size_t miblen = sizeof(mib)/sizeof(size_t); - expect_d_eq(mallctlnametomib(mibname, mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); - mib[2] = (size_t)arena_ind; - uint64_t npurge = 0; - size_t sz = sizeof(npurge); - expect_d_eq(mallctlbymib(mib, miblen, (void *)&npurge, &sz, NULL, 0), - config_stats ? 0 : ENOENT, "Unexpected mallctlbymib() failure"); - return npurge; -} - -static uint64_t -get_arena_dirty_npurge(unsigned arena_ind) { - do_epoch(); - return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind); -} - -static uint64_t -get_arena_dirty_purged(unsigned arena_ind) { - do_epoch(); - return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind); -} - -static uint64_t -get_arena_muzzy_npurge(unsigned arena_ind) { - do_epoch(); - return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind); -} - -static uint64_t -get_arena_npurge(unsigned arena_ind) { - do_epoch(); - return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) + - get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind); -} - -static size_t -get_arena_pdirty(unsigned arena_ind) { - do_epoch(); - size_t mib[4]; - size_t miblen = sizeof(mib)/sizeof(size_t); - expect_d_eq(mallctlnametomib("stats.arenas.0.pdirty", mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); - mib[2] = (size_t)arena_ind; - size_t pdirty; - size_t sz = sizeof(pdirty); - expect_d_eq(mallctlbymib(mib, miblen, (void *)&pdirty, &sz, NULL, 0), 0, - "Unexpected mallctlbymib() failure"); - return pdirty; -} - -static size_t -get_arena_pmuzzy(unsigned arena_ind) { - do_epoch(); - size_t mib[4]; - size_t miblen = sizeof(mib)/sizeof(size_t); - expect_d_eq(mallctlnametomib("stats.arenas.0.pmuzzy", mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); - mib[2] = (size_t)arena_ind; - size_t pmuzzy; - size_t sz = sizeof(pmuzzy); - expect_d_eq(mallctlbymib(mib, miblen, (void *)&pmuzzy, &sz, NULL, 0), 0, - "Unexpected mallctlbymib() failure"); - return pmuzzy; -} - -static void * -do_mallocx(size_t size, int flags) { - void *p = mallocx(size, flags); - expect_ptr_not_null(p, "Unexpected mallocx() failure"); - return p; -} - -static void -generate_dirty(unsigned arena_ind, size_t size) { - int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; - void *p = do_mallocx(size, flags); - dallocx(p, flags); -} - TEST_BEGIN(test_decay_ticks) { test_skip_if(is_background_thread_enabled()); test_skip_if(opt_hpa); diff --git a/test/unit/double_free.c b/test/unit/double_free.c index 73155b9c..f98484c4 100644 --- a/test/unit/double_free.c +++ b/test/unit/double_free.c @@ -1,4 +1,5 @@ #include "test/jemalloc_test.h" +#include "test/guard.h" #include "jemalloc/internal/safety_check.h" @@ -30,8 +31,18 @@ TEST_BEGIN(test_large_double_free_tcache) { test_large_double_free_pre(); char *ptr = malloc(SC_LARGE_MINCLASS); + bool guarded = extent_is_guarded(tsdn_fetch(), ptr); free(ptr); - free(ptr); + if (!guarded) { + free(ptr); + } else { + /* + * Skip because guarded extents may unguard immediately on + * deallocation, in which case the second free will crash before + * reaching the intended safety check. + */ + fake_abort_called = true; + } mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); test_large_double_free_post(); } @@ -43,8 +54,18 @@ TEST_BEGIN(test_large_double_free_no_tcache) { test_large_double_free_pre(); char *ptr = mallocx(SC_LARGE_MINCLASS, MALLOCX_TCACHE_NONE); + bool guarded = extent_is_guarded(tsdn_fetch(), ptr); dallocx(ptr, MALLOCX_TCACHE_NONE); - dallocx(ptr, MALLOCX_TCACHE_NONE); + if (!guarded) { + dallocx(ptr, MALLOCX_TCACHE_NONE); + } else { + /* + * Skip because guarded extents may unguard immediately on + * deallocation, in which case the second free will crash before + * reaching the intended safety check. + */ + fake_abort_called = true; + } test_large_double_free_post(); } TEST_END diff --git a/test/unit/guard.c b/test/unit/guard.c new file mode 100644 index 00000000..43381e44 --- /dev/null +++ b/test/unit/guard.c @@ -0,0 +1,201 @@ +#include "test/jemalloc_test.h" +#include "test/arena_decay.h" +#include "test/guard.h" + +#include "jemalloc/internal/guard.h" + +static void +verify_extent_guarded(tsdn_t *tsdn, void *ptr) { + expect_true(extent_is_guarded(tsdn, ptr), + "All extents should be guarded."); +} + +#define MAX_SMALL_ALLOCATIONS 4096 +void *small_alloc[MAX_SMALL_ALLOCATIONS]; + +TEST_BEGIN(test_guarded_small) { + tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); + unsigned npages = 16, pages_found = 0, ends_found = 0; + VARIABLE_ARRAY(uintptr_t, pages, npages); + + /* Allocate to get sanitized pointers. */ + size_t sz = PAGE / 8; + unsigned n_alloc = 0; + while (n_alloc < MAX_SMALL_ALLOCATIONS) { + void *ptr = malloc(sz); + expect_ptr_not_null(ptr, "Unexpected malloc() failure"); + small_alloc[n_alloc] = ptr; + verify_extent_guarded(tsdn, ptr); + if ((uintptr_t)ptr % PAGE == 0) { + pages[pages_found++] = (uintptr_t)ptr; + } + if (((uintptr_t)ptr + (uintptr_t)sz) % PAGE == 0) { + ends_found++; + } + n_alloc++; + if (pages_found == npages && ends_found == npages) { + break; + } + } + /* Should found the ptrs being checked for overflow and underflow. */ + expect_u_eq(pages_found, npages, "Could not found the expected pages."); + expect_u_eq(ends_found, npages, "Could not found the expected pages."); + + /* Verify the pages are not continuous, i.e. separated by guards. */ + for (unsigned i = 0; i < npages - 1; i++) { + for (unsigned j = i + 1; j < npages; j++) { + uintptr_t ptr_diff = pages[i] > pages[j] ? + pages[i] - pages[j] : pages[j] - pages[i]; + expect_zu_gt((size_t)ptr_diff, 2 * PAGE, + "Pages should not be next to each other."); + } + } + + for (unsigned i = 0; i < n_alloc + 1; i++) { + free(small_alloc[i]); + } +} +TEST_END + +TEST_BEGIN(test_guarded_large) { + tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); + unsigned nlarge = 32; + VARIABLE_ARRAY(uintptr_t, large, nlarge); + + /* Allocate to get sanitized pointers. */ + size_t large_sz = SC_LARGE_MINCLASS; + for (unsigned i = 0; i < nlarge; i++) { + void *ptr = malloc(large_sz); + verify_extent_guarded(tsdn, ptr); + expect_ptr_not_null(ptr, "Unexpected malloc() failure"); + large[i] = (uintptr_t)ptr; + } + + /* Verify the pages are not continuous, i.e. separated by guards. */ + uintptr_t min_diff = (uintptr_t)-1; + for (unsigned i = 0; i < nlarge; i++) { + for (unsigned j = i + 1; j < nlarge; j++) { + uintptr_t ptr_diff = large[i] > large[j] ? + large[i] - large[j] : large[j] - large[i]; + expect_zu_ge((size_t)ptr_diff, large_sz + 2 * PAGE, + "Pages should not be next to each other."); + if (ptr_diff < min_diff) { + min_diff = ptr_diff; + } + } + } + expect_zu_ge((size_t)min_diff, large_sz + 2 * PAGE, + "Pages should not be next to each other."); + + for (unsigned i = 0; i < nlarge; i++) { + free((void *)large[i]); + } +} +TEST_END + +static void +verify_pdirty(unsigned arena_ind, uint64_t expected) { + uint64_t pdirty = get_arena_pdirty(arena_ind); + expect_u64_eq(pdirty, expected / PAGE, + "Unexpected dirty page amount."); +} + +static void +verify_pmuzzy(unsigned arena_ind, uint64_t expected) { + uint64_t pmuzzy = get_arena_pmuzzy(arena_ind); + expect_u64_eq(pmuzzy, expected / PAGE, + "Unexpected muzzy page amount."); +} + +TEST_BEGIN(test_guarded_decay) { + unsigned arena_ind = do_arena_create(-1, -1); + do_decay(arena_ind); + do_purge(arena_ind); + + verify_pdirty(arena_ind, 0); + verify_pmuzzy(arena_ind, 0); + + /* Verify that guarded extents as dirty. */ + size_t sz1 = PAGE, sz2 = PAGE * 2; + /* W/o maps_coalesce, guarded extents are unguarded eagerly. */ + size_t add_guard_size = maps_coalesce ? 0 : PAGE_GUARDS_SIZE; + generate_dirty(arena_ind, sz1); + verify_pdirty(arena_ind, sz1 + add_guard_size); + verify_pmuzzy(arena_ind, 0); + + /* Should reuse the first extent. */ + generate_dirty(arena_ind, sz1); + verify_pdirty(arena_ind, sz1 + add_guard_size); + verify_pmuzzy(arena_ind, 0); + + /* Should not reuse; expect new dirty pages. */ + generate_dirty(arena_ind, sz2); + verify_pdirty(arena_ind, sz1 + sz2 + 2 * add_guard_size); + verify_pmuzzy(arena_ind, 0); + + tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); + int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; + + /* Should reuse dirty extents for the two mallocx. */ + void *p1 = do_mallocx(sz1, flags); + verify_extent_guarded(tsdn, p1); + verify_pdirty(arena_ind, sz2 + add_guard_size); + + void *p2 = do_mallocx(sz2, flags); + verify_extent_guarded(tsdn, p2); + verify_pdirty(arena_ind, 0); + verify_pmuzzy(arena_ind, 0); + + dallocx(p1, flags); + verify_pdirty(arena_ind, sz1 + add_guard_size); + dallocx(p2, flags); + verify_pdirty(arena_ind, sz1 + sz2 + 2 * add_guard_size); + verify_pmuzzy(arena_ind, 0); + + do_purge(arena_ind); + verify_pdirty(arena_ind, 0); + verify_pmuzzy(arena_ind, 0); + + if (config_stats) { + expect_u64_eq(get_arena_npurge(arena_ind), 1, + "Expected purging to occur"); + expect_u64_eq(get_arena_dirty_npurge(arena_ind), 1, + "Expected purging to occur"); + expect_u64_eq(get_arena_dirty_purged(arena_ind), + (sz1 + sz2 + 2 * add_guard_size) / PAGE, + "Expected purging to occur"); + expect_u64_eq(get_arena_muzzy_npurge(arena_ind), 0, + "Expected purging to occur"); + } + + if (opt_retain) { + /* + * With retain, guarded extents are not mergable and will be + * cached in ecache_retained. They should be reused. + */ + void *new_p1 = do_mallocx(sz1, flags); + verify_extent_guarded(tsdn, p1); + expect_ptr_eq(p1, new_p1, "Expect to reuse p1"); + + void *new_p2 = do_mallocx(sz2, flags); + verify_extent_guarded(tsdn, p2); + expect_ptr_eq(p2, new_p2, "Expect to reuse p2"); + + dallocx(new_p1, flags); + verify_pdirty(arena_ind, sz1 + add_guard_size); + dallocx(new_p2, flags); + verify_pdirty(arena_ind, sz1 + sz2 + 2 * add_guard_size); + verify_pmuzzy(arena_ind, 0); + } + + do_arena_destroy(arena_ind); +} +TEST_END + +int +main(void) { + return test( + test_guarded_small, + test_guarded_large, + test_guarded_decay); +} diff --git a/test/unit/guard.sh b/test/unit/guard.sh new file mode 100644 index 00000000..933b4a4d --- /dev/null +++ b/test/unit/guard.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export MALLOC_CONF="san_guard_large:1,san_guard_small:1" diff --git a/test/unit/hpa.c b/test/unit/hpa.c index 86012c75..060ce3e4 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -80,11 +80,11 @@ TEST_BEGIN(test_alloc_max) { /* Small max */ bool deferred_work_generated; - edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX, PAGE, false, + edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX, PAGE, false, false, &deferred_work_generated); expect_ptr_not_null(edata, "Allocation of small max failed"); edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX + PAGE, PAGE, false, - &deferred_work_generated); + false, &deferred_work_generated); expect_ptr_null(edata, "Allocation of larger than small max succeeded"); destroy_test_data(shard); @@ -188,7 +188,7 @@ TEST_BEGIN(test_stress) { size_t npages = npages_min + prng_range_zu(&prng_state, npages_max - npages_min); edata_t *edata = pai_alloc(tsdn, &shard->pai, - npages * PAGE, PAGE, false, + npages * PAGE, PAGE, false, false, &deferred_work_generated); assert_ptr_not_null(edata, "Unexpected allocation failure"); @@ -263,7 +263,8 @@ TEST_BEGIN(test_alloc_dalloc_batch) { */ for (size_t i = 0; i < NALLOCS / 2; i++) { allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(allocs[i], "Unexpected alloc failure"); } edata_list_active_t allocs_list; @@ -299,7 +300,8 @@ TEST_BEGIN(test_alloc_dalloc_batch) { /* Reallocate (individually), and ensure reuse and contiguity. */ for (size_t i = 0; i < NALLOCS; i++) { allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(allocs[i], "Unexpected alloc failure."); } void *new_base = edata_base_get(allocs[0]); @@ -374,7 +376,7 @@ TEST_BEGIN(test_defer_time) { edata_t *edatas[HUGEPAGE_PAGES]; for (int i = 0; i < (int)HUGEPAGE_PAGES; i++) { edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, - &deferred_work_generated); + false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } hpa_shard_do_deferred_work(tsdn, shard); @@ -408,7 +410,7 @@ TEST_BEGIN(test_defer_time) { */ for (int i = 0; i < (int)HUGEPAGE_PAGES / 2; i++) { edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, - &deferred_work_generated); + false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* diff --git a/test/unit/hpa_background_thread.c b/test/unit/hpa_background_thread.c index 77d05556..5976bb47 100644 --- a/test/unit/hpa_background_thread.c +++ b/test/unit/hpa_background_thread.c @@ -128,6 +128,8 @@ TEST_BEGIN(test_hpa_background_thread_purges) { test_skip_if(!config_stats); test_skip_if(!hpa_supported()); test_skip_if(!have_background_thread); + /* Skip since guarded pages cannot be allocated from hpa. */ + test_skip_if(san_enabled()); unsigned arena_ind = create_arena(); /* @@ -142,6 +144,8 @@ TEST_BEGIN(test_hpa_background_thread_enable_disable) { test_skip_if(!config_stats); test_skip_if(!hpa_supported()); test_skip_if(!have_background_thread); + /* Skip since guarded pages cannot be allocated from hpa. */ + test_skip_if(san_enabled()); unsigned arena_ind = create_arena(); diff --git a/test/unit/pa.c b/test/unit/pa.c index 01d891df..fcf22237 100644 --- a/test/unit/pa.c +++ b/test/unit/pa.c @@ -91,7 +91,7 @@ do_alloc_free_purge(void *arg) { bool deferred_work_generated; edata_t *edata = pa_alloc(TSDN_NULL, &test_data->shard, PAGE, PAGE, /* slab */ false, /* szind */ 0, /* zero */ false, - &deferred_work_generated); + /* guarded */ false, &deferred_work_generated); assert_ptr_not_null(edata, ""); pa_dalloc(TSDN_NULL, &test_data->shard, edata, &deferred_work_generated); diff --git a/test/unit/retained.c b/test/unit/retained.c index 9ad9940e..53cda286 100644 --- a/test/unit/retained.c +++ b/test/unit/retained.c @@ -1,5 +1,6 @@ #include "test/jemalloc_test.h" +#include "jemalloc/internal/guard.h" #include "jemalloc/internal/spin.h" static unsigned arena_ind; @@ -103,7 +104,8 @@ TEST_BEGIN(test_retained) { arena_ind = do_arena_create(NULL); sz = nallocx(HUGEPAGE, 0); - esz = sz + sz_large_pad; + size_t guard_sz = san_enabled() ? PAGE_GUARDS_SIZE : 0; + esz = sz + sz_large_pad + guard_sz; atomic_store_u(&epoch, 0, ATOMIC_RELAXED); @@ -133,7 +135,8 @@ TEST_BEGIN(test_retained) { */ do_refresh(); - size_t allocated = esz * nthreads * PER_THD_NALLOCS; + size_t allocated = (esz - guard_sz) * nthreads * + PER_THD_NALLOCS; size_t active = do_get_active(arena_ind); expect_zu_le(allocated, active, "Unexpected active memory"); size_t mapped = do_get_mapped(arena_ind); diff --git a/test/unit/sec.c b/test/unit/sec.c index 763e6087..acca192d 100644 --- a/test/unit/sec.c +++ b/test/unit/sec.c @@ -50,7 +50,9 @@ test_sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t max_alloc, static inline edata_t * pai_test_allocator_alloc(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool *deferred_work_generated) { + size_t alignment, bool zero, bool guarded, + bool *deferred_work_generated) { + assert(!guarded); pai_test_allocator_t *ta = (pai_test_allocator_t *)self; *deferred_work_generated = false; if (ta->alloc_fail) { @@ -182,10 +184,12 @@ TEST_BEGIN(test_reuse) { /* max_bytes */ 2 * (NALLOCS * PAGE + NALLOCS * 2 * PAGE)); for (int i = 0; i < NALLOCS; i++) { one_page[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(one_page[i], "Unexpected alloc failure"); two_page[i] = pai_alloc(tsdn, &sec.pai, 2 * PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(one_page[i], "Unexpected alloc failure"); } expect_zu_eq(0, ta.alloc_count, "Should be using batch allocs"); @@ -216,9 +220,11 @@ TEST_BEGIN(test_reuse) { */ for (int i = 0; i < NALLOCS; i++) { edata_t *alloc1 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); edata_t *alloc2 = pai_alloc(tsdn, &sec.pai, 2 * PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_eq(one_page[i], alloc1, "Got unexpected allocation"); expect_ptr_eq(two_page[i], alloc2, @@ -255,11 +261,12 @@ TEST_BEGIN(test_auto_flush) { /* max_bytes */ NALLOCS * PAGE); for (int i = 0; i < NALLOCS; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(allocs[i], "Unexpected alloc failure"); } extra_alloc = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false, - &deferred_work_generated); + /* guarded */ false, &deferred_work_generated); expect_ptr_not_null(extra_alloc, "Unexpected alloc failure"); size_t max_allocs = ta.alloc_count + ta.alloc_batch_count; expect_zu_le(NALLOCS + 1, max_allocs, @@ -310,7 +317,8 @@ do_disable_flush_test(bool is_disable) { /* max_bytes */ NALLOCS * PAGE); for (int i = 0; i < NALLOCS; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(allocs[i], "Unexpected alloc failure"); } /* Free all but the last aloc. */ @@ -383,7 +391,8 @@ TEST_BEGIN(test_max_alloc_respected) { expect_zu_eq(i, ta.dalloc_count, "Incorrect number of deallocations"); edata_t *edata = pai_alloc(tsdn, &sec.pai, attempted_alloc, - PAGE, /* zero */ false, &deferred_work_generated); + PAGE, /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected alloc failure"); expect_zu_eq(i + 1, ta.alloc_count, "Incorrect number of allocations"); @@ -410,7 +419,8 @@ TEST_BEGIN(test_expand_shrink_delegate) { test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ 10 * PAGE, /* max_bytes */ 1000 * PAGE); edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected alloc failure"); bool err = pai_expand(tsdn, &sec.pai, edata, PAGE, 4 * PAGE, @@ -450,7 +460,8 @@ TEST_BEGIN(test_nshards_0) { bool deferred_work_generated; edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); pai_dalloc(tsdn, &sec.pai, edata, &deferred_work_generated); /* Both operations should have gone directly to the fallback. */ @@ -492,7 +503,8 @@ TEST_BEGIN(test_stats_simple) { edata_t *allocs[FLUSH_PAGES]; for (size_t i = 0; i < FLUSH_PAGES; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_stats_pages(tsdn, &sec, 0); } @@ -505,7 +517,8 @@ TEST_BEGIN(test_stats_simple) { } for (size_t j = 0; j < FLUSH_PAGES / 2; j++) { allocs[j] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_stats_pages(tsdn, &sec, FLUSH_PAGES / 2 - j - 1); } } @@ -534,13 +547,14 @@ TEST_BEGIN(test_stats_auto_flush) { bool deferred_work_generated; extra_alloc0 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false, - &deferred_work_generated); + /* guarded */ false, &deferred_work_generated); extra_alloc1 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false, - &deferred_work_generated); + /* guarded */ false, &deferred_work_generated); for (size_t i = 0; i < 2 * FLUSH_PAGES; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); } for (size_t i = 0; i < FLUSH_PAGES; i++) { @@ -580,7 +594,8 @@ TEST_BEGIN(test_stats_manual_flush) { edata_t *allocs[FLUSH_PAGES]; for (size_t i = 0; i < FLUSH_PAGES; i++) { allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, - /* zero */ false, &deferred_work_generated); + /* zero */ false, /* guarded */ false, + &deferred_work_generated); expect_stats_pages(tsdn, &sec, 0); }