Implement guard pages.

Adding guarded extents, which are regular extents surrounded by guard pages
(mprotected).  To reduce syscalls, small guarded extents are cached as a
separate eset in ecache, and decay through the dirty / muzzy / retained pipeline
as usual.
This commit is contained in:
Qi Wang 2021-04-26 14:22:25 -07:00 committed by Qi Wang
parent 7bb05e04be
commit deb8e62a83
41 changed files with 920 additions and 251 deletions

View File

@ -119,6 +119,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/extent_dss.c \ $(srcroot)src/extent_dss.c \
$(srcroot)src/extent_mmap.c \ $(srcroot)src/extent_mmap.c \
$(srcroot)src/fxp.c \ $(srcroot)src/fxp.c \
$(srcroot)src/guard.c \
$(srcroot)src/hook.c \ $(srcroot)src/hook.c \
$(srcroot)src/hpa.c \ $(srcroot)src/hpa.c \
$(srcroot)src/hpa_hooks.c \ $(srcroot)src/hpa_hooks.c \
@ -218,6 +219,7 @@ TESTS_UNIT := \
${srcroot}test/unit/fb.c \ ${srcroot}test/unit/fb.c \
$(srcroot)test/unit/fork.c \ $(srcroot)test/unit/fork.c \
${srcroot}test/unit/fxp.c \ ${srcroot}test/unit/fxp.c \
${srcroot}test/unit/guard.c \
$(srcroot)test/unit/hash.c \ $(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hook.c \ $(srcroot)test/unit/hook.c \
$(srcroot)test/unit/hpa.c \ $(srcroot)test/unit/hpa.c \

View File

@ -2256,6 +2256,18 @@ else
fi fi
fi fi
dnl ============================================================================
dnl Check for mprotect(2).
JE_COMPILABLE([mprotect(2)], [
#include <sys/mman.h>
], [
mprotect((void *)0, 0, PROT_NONE);
], [je_cv_mprotect])
if test "x${je_cv_mprotect}" = "xyes" ; then
AC_DEFINE([JEMALLOC_HAVE_MPROTECT], [ ])
fi
dnl ============================================================================ dnl ============================================================================
dnl Check for __builtin_clz(), __builtin_clzl(), and __builtin_clzll(). dnl Check for __builtin_clz(), __builtin_clzl(), and __builtin_clzll().

View File

@ -221,7 +221,8 @@ large_dalloc_safety_checks(edata_t *edata, void *ptr, szind_t szind) {
* The cost is low enough (as edata will be accessed anyway) to be * The cost is low enough (as edata will be accessed anyway) to be
* enabled all the time. * enabled all the time.
*/ */
if (unlikely(edata_state_get(edata) != extent_state_active)) { if (unlikely(edata == NULL ||
edata_state_get(edata) != extent_state_active)) {
safety_check_fail("Invalid deallocation detected: " safety_check_fail("Invalid deallocation detected: "
"pages being freed (%p) not currently active, " "pages being freed (%p) not currently active, "
"possibly caused by double free bugs.", "possibly caused by double free bugs.",

View File

@ -2,12 +2,14 @@
#define JEMALLOC_INTERNAL_ECACHE_H #define JEMALLOC_INTERNAL_ECACHE_H
#include "jemalloc/internal/eset.h" #include "jemalloc/internal/eset.h"
#include "jemalloc/internal/guard.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
typedef struct ecache_s ecache_t; typedef struct ecache_s ecache_t;
struct ecache_s { struct ecache_s {
malloc_mutex_t mtx; malloc_mutex_t mtx;
eset_t eset; eset_t eset;
eset_t guarded_eset;
/* All stored extents must be in the same state. */ /* All stored extents must be in the same state. */
extent_state_t state; extent_state_t state;
/* The index of the ehooks the ecache is associated with. */ /* The index of the ehooks the ecache is associated with. */
@ -21,17 +23,22 @@ struct ecache_s {
static inline size_t static inline size_t
ecache_npages_get(ecache_t *ecache) { ecache_npages_get(ecache_t *ecache) {
return eset_npages_get(&ecache->eset); return eset_npages_get(&ecache->eset) +
eset_npages_get(&ecache->guarded_eset);
} }
/* Get the number of extents in the given page size index. */ /* Get the number of extents in the given page size index. */
static inline size_t static inline size_t
ecache_nextents_get(ecache_t *ecache, pszind_t ind) { ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
return eset_nextents_get(&ecache->eset, ind); return eset_nextents_get(&ecache->eset, ind) +
eset_nextents_get(&ecache->guarded_eset, ind);
} }
/* Get the sum total bytes of the extents in the given page size index. */ /* Get the sum total bytes of the extents in the given page size index. */
static inline size_t static inline size_t
ecache_nbytes_get(ecache_t *ecache, pszind_t ind) { ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
return eset_nbytes_get(&ecache->eset, ind); return eset_nbytes_get(&ecache->eset, ind) +
eset_nbytes_get(&ecache->guarded_eset, ind);
} }
static inline unsigned static inline unsigned

View File

@ -98,12 +98,13 @@ struct edata_s {
* c: committed * c: committed
* p: pai * p: pai
* z: zeroed * z: zeroed
* g: guarded
* t: state * t: state
* i: szind * i: szind
* f: nfree * f: nfree
* s: bin_shard * s: bin_shard
* *
* 00000000 ... 00000sss sssfffff fffffiii iiiiittt zpcbaaaa aaaaaaaa * 00000000 ... 0000ssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
* *
* arena_ind: Arena from which this extent came, or all 1 bits if * arena_ind: Arena from which this extent came, or all 1 bits if
* unassociated. * unassociated.
@ -123,6 +124,9 @@ struct edata_s {
* zeroed: The zeroed flag is used by extent recycling code to track * zeroed: The zeroed flag is used by extent recycling code to track
* whether memory is zero-filled. * whether memory is zero-filled.
* *
* guarded: The guarded flag is use by the sanitizer to track whether
* the extent has page guards around it.
*
* state: The state flag is an extent_state_t. * state: The state flag is an extent_state_t.
* *
* szind: The szind flag indicates usable size class index for * szind: The szind flag indicates usable size class index for
@ -158,8 +162,12 @@ struct edata_s {
#define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT) #define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT)
#define EDATA_BITS_ZEROED_MASK MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT) #define EDATA_BITS_ZEROED_MASK MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT)
#define EDATA_BITS_GUARDED_WIDTH 1
#define EDATA_BITS_GUARDED_SHIFT (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT)
#define EDATA_BITS_GUARDED_MASK MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT)
#define EDATA_BITS_STATE_WIDTH 3 #define EDATA_BITS_STATE_WIDTH 3
#define EDATA_BITS_STATE_SHIFT (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT) #define EDATA_BITS_STATE_SHIFT (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT)
#define EDATA_BITS_STATE_MASK MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT) #define EDATA_BITS_STATE_MASK MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT)
#define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) #define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES)
@ -293,6 +301,12 @@ edata_state_get(const edata_t *edata) {
EDATA_BITS_STATE_SHIFT); EDATA_BITS_STATE_SHIFT);
} }
static inline bool
edata_guarded_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK) >>
EDATA_BITS_GUARDED_SHIFT);
}
static inline bool static inline bool
edata_zeroed_get(const edata_t *edata) { edata_zeroed_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) >> return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) >>
@ -505,6 +519,12 @@ edata_state_set(edata_t *edata, extent_state_t state) {
((uint64_t)state << EDATA_BITS_STATE_SHIFT); ((uint64_t)state << EDATA_BITS_STATE_SHIFT);
} }
static inline void
edata_guarded_set(edata_t *edata, bool guarded) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK) |
((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT);
}
static inline void static inline void
edata_zeroed_set(edata_t *edata, bool zeroed) { edata_zeroed_set(edata_t *edata, bool zeroed) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) |
@ -588,6 +608,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
edata_szind_set(edata, szind); edata_szind_set(edata, szind);
edata_sn_set(edata, sn); edata_sn_set(edata, sn);
edata_state_set(edata, state); edata_state_set(edata, state);
edata_guarded_set(edata, false);
edata_zeroed_set(edata, zeroed); edata_zeroed_set(edata, zeroed);
edata_committed_set(edata, committed); edata_committed_set(edata, committed);
edata_pai_set(edata, pai); edata_pai_set(edata, pai);
@ -606,6 +627,7 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) {
edata_szind_set(edata, SC_NSIZES); edata_szind_set(edata, SC_NSIZES);
edata_sn_set(edata, sn); edata_sn_set(edata, sn);
edata_state_set(edata, extent_state_active); edata_state_set(edata, extent_state_active);
edata_guarded_set(edata, false);
edata_zeroed_set(edata, true); edata_zeroed_set(edata, true);
edata_committed_set(edata, true); edata_committed_set(edata, true);
/* /*

View File

@ -63,6 +63,8 @@ bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a,
unsigned arena_ind); unsigned arena_ind);
bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b); bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b);
void ehooks_default_zero_impl(void *addr, size_t size); void ehooks_default_zero_impl(void *addr, size_t size);
void ehooks_default_guard_impl(void *guard1, void *guard2);
void ehooks_default_unguard_impl(void *guard1, void *guard2);
/* /*
* We don't officially support reentrancy from wtihin the extent hooks. But * We don't officially support reentrancy from wtihin the extent hooks. But
@ -139,6 +141,15 @@ ehooks_merge_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL; return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
} }
static inline bool
ehooks_guard_will_fail(ehooks_t *ehooks) {
/*
* Before the guard hooks are officially introduced, limit the use to
* the default hooks only.
*/
return !ehooks_are_default(ehooks);
}
/* /*
* Some hooks are required to return zeroed memory in certain situations. In * Some hooks are required to return zeroed memory in certain situations. In
* debug mode, we do some heuristic checks that they did what they were supposed * debug mode, we do some heuristic checks that they did what they were supposed
@ -368,4 +379,34 @@ ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size) {
} }
} }
static inline bool
ehooks_guard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) {
bool err;
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks == &ehooks_default_extent_hooks) {
ehooks_default_guard_impl(guard1, guard2);
err = false;
} else {
err = true;
}
return err;
}
static inline bool
ehooks_unguard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) {
bool err;
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks == &ehooks_default_extent_hooks) {
ehooks_default_unguard_impl(guard1, guard2);
err = false;
} else {
err = true;
}
return err;
}
#endif /* JEMALLOC_INTERNAL_EHOOKS_H */ #endif /* JEMALLOC_INTERNAL_EHOOKS_H */

View File

@ -21,10 +21,10 @@ extern size_t opt_lg_extent_max_active_fit;
edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
bool zero); bool zero, bool guarded);
edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
bool zero); bool zero, bool guarded);
void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *edata); ecache_t *ecache, edata_t *edata);
edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,

View File

@ -0,0 +1,76 @@
#ifndef JEMALLOC_INTERNAL_GUARD_H
#define JEMALLOC_INTERNAL_GUARD_H
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/emap.h"
#define PAGE_GUARDS_SIZE (2 * PAGE)
#define SAN_GUARD_LARGE_EVERY_N_EXTENTS_DEFAULT 0
#define SAN_GUARD_SMALL_EVERY_N_EXTENTS_DEFAULT 0
/* 0 means disabled, i.e. never guarded. */
extern size_t opt_san_guard_large;
extern size_t opt_san_guard_small;
void guard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap);
void unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap);
void tsd_san_init(tsd_t *tsd);
static inline bool
san_enabled(void) {
return (opt_san_guard_large != 0 || opt_san_guard_small != 0);
}
static inline bool
large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size,
size_t alignment) {
if (opt_san_guard_large == 0 || ehooks_guard_will_fail(ehooks) ||
tsdn_null(tsdn)) {
return false;
}
tsd_t *tsd = tsdn_tsd(tsdn);
uint64_t n = tsd_san_extents_until_guard_large_get(tsd);
assert(n >= 1);
if (n > 1) {
/*
* Subtract conditionally because the guard may not happen due
* to alignment or size restriction below.
*/
*tsd_san_extents_until_guard_largep_get(tsd) = n - 1;
}
if (n == 1 && (alignment <= PAGE) &&
(size + PAGE_GUARDS_SIZE <= SC_LARGE_MAXCLASS)) {
*tsd_san_extents_until_guard_largep_get(tsd) =
opt_san_guard_large;
return true;
} else {
assert(tsd_san_extents_until_guard_large_get(tsd) >= 1);
return false;
}
}
static inline bool
slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) {
if (opt_san_guard_small == 0 || ehooks_guard_will_fail(ehooks) ||
tsdn_null(tsdn)) {
return false;
}
tsd_t *tsd = tsdn_tsd(tsdn);
uint64_t n = tsd_san_extents_until_guard_small_get(tsd);
assert(n >= 1);
if (n == 1) {
*tsd_san_extents_until_guard_smallp_get(tsd) =
opt_san_guard_small;
return true;
} else {
*tsd_san_extents_until_guard_smallp_get(tsd) = n - 1;
assert(tsd_san_extents_until_guard_small_get(tsd) >= 1);
return false;
}
}
#endif /* JEMALLOC_INTERNAL_GUARD_H */

View File

@ -312,6 +312,9 @@
*/ */
#undef JEMALLOC_MADVISE_NOCORE #undef JEMALLOC_MADVISE_NOCORE
/* Defined if mprotect(2) is available. */
#undef JEMALLOC_HAVE_MPROTECT
/* /*
* Defined if transparent huge pages (THPs) are supported via the * Defined if transparent huge pages (THPs) are supported via the
* MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled.

View File

@ -167,7 +167,7 @@ void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
/* Gets an edata for the given allocation. */ /* Gets an edata for the given allocation. */
edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
size_t alignment, bool slab, szind_t szind, bool zero, size_t alignment, bool slab, szind_t szind, bool zero, bool guarded,
bool *deferred_work_generated); bool *deferred_work_generated);
/* Returns true on error, in which case nothing changed. */ /* Returns true on error, in which case nothing changed. */
bool pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, bool pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,

View File

@ -110,5 +110,7 @@ bool pages_dontdump(void *addr, size_t size);
bool pages_dodump(void *addr, size_t size); bool pages_dodump(void *addr, size_t size);
bool pages_boot(void); bool pages_boot(void);
void pages_set_thp_state (void *ptr, size_t size); void pages_set_thp_state (void *ptr, size_t size);
void pages_mark_guards(void *head, void *tail);
void pages_unmark_guards(void *head, void *tail);
#endif /* JEMALLOC_INTERNAL_PAGES_EXTERNS_H */ #endif /* JEMALLOC_INTERNAL_PAGES_EXTERNS_H */

View File

@ -7,7 +7,8 @@ typedef struct pai_s pai_t;
struct pai_s { struct pai_s {
/* Returns NULL on failure. */ /* Returns NULL on failure. */
edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size, edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool *deferred_work_generated); size_t alignment, bool zero, bool guarded,
bool *deferred_work_generated);
/* /*
* Returns the number of extents added to the list (which may be fewer * Returns the number of extents added to the list (which may be fewer
* than requested, in case of OOM). The list should already be * than requested, in case of OOM). The list should already be
@ -37,8 +38,8 @@ struct pai_s {
static inline edata_t * static inline edata_t *
pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool *deferred_work_generated) { bool guarded, bool *deferred_work_generated) {
return self->alloc(tsdn, self, size, alignment, zero, return self->alloc(tsdn, self, size, alignment, zero, guarded,
deferred_work_generated); deferred_work_generated);
} }

View File

@ -73,6 +73,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
O(peak_dalloc_event_wait, uint64_t, uint64_t) \ O(peak_dalloc_event_wait, uint64_t, uint64_t) \
O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \
O(prng_state, uint64_t, uint64_t) \ O(prng_state, uint64_t, uint64_t) \
O(san_extents_until_guard_small, uint64_t, uint64_t) \
O(san_extents_until_guard_large, uint64_t, uint64_t) \
O(iarena, arena_t *, arena_t *) \ O(iarena, arena_t *, arena_t *) \
O(arena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \
O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \
@ -103,6 +105,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
/* peak_dalloc_event_wait */ 0, \ /* peak_dalloc_event_wait */ 0, \
/* prof_tdata */ NULL, \ /* prof_tdata */ NULL, \
/* prng_state */ 0, \ /* prng_state */ 0, \
/* san_extents_until_guard_small */ 0, \
/* san_extents_until_guard_large */ 0, \
/* iarena */ NULL, \ /* iarena */ NULL, \
/* arena */ NULL, \ /* arena */ NULL, \
/* arena_decay_ticker */ \ /* arena_decay_ticker */ \

View File

@ -59,6 +59,7 @@
<ClCompile Include="..\..\..\..\src\extent_dss.c" /> <ClCompile Include="..\..\..\..\src\extent_dss.c" />
<ClCompile Include="..\..\..\..\src\extent_mmap.c" /> <ClCompile Include="..\..\..\..\src\extent_mmap.c" />
<ClCompile Include="..\..\..\..\src\fxp.c" /> <ClCompile Include="..\..\..\..\src\fxp.c" />
<ClCompile Include="..\..\..\..\src\guard.c" />
<ClCompile Include="..\..\..\..\src\hook.c" /> <ClCompile Include="..\..\..\..\src\hook.c" />
<ClCompile Include="..\..\..\..\src\hpa.c" /> <ClCompile Include="..\..\..\..\src\hpa.c" />
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" /> <ClCompile Include="..\..\..\..\src\hpa_hooks.c" />

View File

@ -61,6 +61,9 @@
<ClCompile Include="..\..\..\..\src\fxp.c"> <ClCompile Include="..\..\..\..\src\fxp.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\..\..\..\src\guard.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\hook.c"> <ClCompile Include="..\..\..\..\src\hook.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>

View File

@ -59,6 +59,7 @@
<ClCompile Include="..\..\..\..\src\extent_dss.c" /> <ClCompile Include="..\..\..\..\src\extent_dss.c" />
<ClCompile Include="..\..\..\..\src\extent_mmap.c" /> <ClCompile Include="..\..\..\..\src\extent_mmap.c" />
<ClCompile Include="..\..\..\..\src\fxp.c" /> <ClCompile Include="..\..\..\..\src\fxp.c" />
<ClCompile Include="..\..\..\..\src\guard.c" />
<ClCompile Include="..\..\..\..\src\hook.c" /> <ClCompile Include="..\..\..\..\src\hook.c" />
<ClCompile Include="..\..\..\..\src\hpa.c" /> <ClCompile Include="..\..\..\..\src\hpa.c" />
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" /> <ClCompile Include="..\..\..\..\src\hpa_hooks.c" />

View File

@ -61,6 +61,9 @@
<ClCompile Include="..\..\..\..\src\fxp.c"> <ClCompile Include="..\..\..\..\src\fxp.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="..\..\..\..\src\guard.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\hook.c"> <ClCompile Include="..\..\..\..\src\hook.c">
<Filter>Source Files</Filter> <Filter>Source Files</Filter>
</ClCompile> </ClCompile>

View File

@ -6,6 +6,7 @@
#include "jemalloc/internal/ehooks.h" #include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/guard.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/safety_check.h"
@ -327,9 +328,10 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
szind_t szind = sz_size2index(usize); szind_t szind = sz_size2index(usize);
size_t esize = usize + sz_large_pad; size_t esize = usize + sz_large_pad;
bool guarded = large_extent_decide_guard(tsdn, arena_get_ehooks(arena),
esize, alignment);
edata_t *edata = pa_alloc(tsdn, &arena->pa_shard, esize, alignment, edata_t *edata = pa_alloc(tsdn, &arena->pa_shard, esize, alignment,
/* slab */ false, szind, zero, &deferred_work_generated); /* slab */ false, szind, zero, guarded, &deferred_work_generated);
assert(deferred_work_generated == false); assert(deferred_work_generated == false);
if (edata != NULL) { if (edata != NULL) {
@ -827,9 +829,10 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
bool guarded = slab_extent_decide_guard(tsdn, arena_get_ehooks(arena));
edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size, edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size,
PAGE, /* slab */ true, /* szind */ binind, /* zero */ false, /* alignment */ PAGE, /* slab */ true, /* szind */ binind,
&deferred_work_generated); /* zero */ false, guarded, &deferred_work_generated);
if (deferred_work_generated) { if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena); arena_handle_deferred_work(tsdn, arena);

View File

@ -1,6 +1,8 @@
#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/guard.h"
bool bool
ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, unsigned ind, ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, unsigned ind,
bool delay_coalesce) { bool delay_coalesce) {
@ -12,6 +14,8 @@ ecache_init(tsdn_t *tsdn, ecache_t *ecache, extent_state_t state, unsigned ind,
ecache->ind = ind; ecache->ind = ind;
ecache->delay_coalesce = delay_coalesce; ecache->delay_coalesce = delay_coalesce;
eset_init(&ecache->eset, state); eset_init(&ecache->eset, state);
eset_init(&ecache->guarded_eset, state);
return false; return false;
} }

View File

@ -244,6 +244,16 @@ ehooks_default_zero_impl(void *addr, size_t size) {
} }
} }
void
ehooks_default_guard_impl(void *guard1, void *guard2) {
pages_mark_guards(guard1, guard2);
}
void
ehooks_default_unguard_impl(void *guard1, void *guard2) {
pages_unmark_guards(guard1, guard2);
}
const extent_hooks_t ehooks_default_extent_hooks = { const extent_hooks_t ehooks_default_extent_hooks = {
ehooks_default_alloc, ehooks_default_alloc,
ehooks_default_dalloc, ehooks_default_dalloc,

View File

@ -37,14 +37,14 @@ static atomic_zu_t highpages;
static void extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata); static void extent_deregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata);
static edata_t *extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, static edata_t *extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *expand_edata, size_t usize, size_t alignment, ecache_t *ecache, edata_t *expand_edata, size_t usize, size_t alignment,
bool zero, bool *commit, bool growing_retained); bool zero, bool *commit, bool growing_retained, bool guarded);
static edata_t *extent_try_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, static edata_t *extent_try_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *edata, bool *coalesced); ecache_t *ecache, edata_t *edata, bool *coalesced);
static void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, static void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *edata); ecache_t *ecache, edata_t *edata);
static edata_t *extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, static edata_t *extent_alloc_retained(tsdn_t *tsdn, pac_t *pac,
ehooks_t *ehooks, edata_t *expand_edata, size_t size, size_t alignment, ehooks_t *ehooks, edata_t *expand_edata, size_t size, size_t alignment,
bool zero, bool *commit); bool zero, bool *commit, bool guarded);
static edata_t *extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, static edata_t *extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
void *new_addr, size_t size, size_t alignment, bool zero, bool *commit); void *new_addr, size_t size, size_t alignment, bool zero, bool *commit);
@ -80,7 +80,8 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
edata_t * edata_t *
ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
edata_t *expand_edata, size_t size, size_t alignment, bool zero) { edata_t *expand_edata, size_t size, size_t alignment, bool zero,
bool guarded) {
assert(size != 0); assert(size != 0);
assert(alignment != 0); assert(alignment != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
@ -88,14 +89,15 @@ ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
bool commit = true; bool commit = true;
edata_t *edata = extent_recycle(tsdn, pac, ehooks, ecache, expand_edata, edata_t *edata = extent_recycle(tsdn, pac, ehooks, ecache, expand_edata,
size, alignment, zero, &commit, false); size, alignment, zero, &commit, false, guarded);
assert(edata == NULL || edata_pai_get(edata) == EXTENT_PAI_PAC); assert(edata == NULL || edata_pai_get(edata) == EXTENT_PAI_PAC);
return edata; return edata;
} }
edata_t * edata_t *
ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
edata_t *expand_edata, size_t size, size_t alignment, bool zero) { edata_t *expand_edata, size_t size, size_t alignment, bool zero,
bool guarded) {
assert(size != 0); assert(size != 0);
assert(alignment != 0); assert(alignment != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
@ -103,7 +105,7 @@ ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
bool commit = true; bool commit = true;
edata_t *edata = extent_alloc_retained(tsdn, pac, ehooks, expand_edata, edata_t *edata = extent_alloc_retained(tsdn, pac, ehooks, expand_edata,
size, alignment, zero, &commit); size, alignment, zero, &commit, guarded);
if (edata == NULL) { if (edata == NULL) {
if (opt_retain && expand_edata != NULL) { if (opt_retain && expand_edata != NULL) {
/* /*
@ -114,6 +116,14 @@ ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
*/ */
return NULL; return NULL;
} }
if (guarded) {
/*
* Means no cached guarded extents available (and no
* grow_retained was attempted). The pac_alloc flow
* will alloc regular extents to make new guarded ones.
*/
return NULL;
}
void *new_addr = (expand_edata == NULL) ? NULL : void *new_addr = (expand_edata == NULL) ? NULL :
edata_past_get(expand_edata); edata_past_get(expand_edata);
edata = extent_alloc_wrapper(tsdn, pac, ehooks, new_addr, edata = extent_alloc_wrapper(tsdn, pac, ehooks, new_addr,
@ -151,17 +161,27 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
edata_t *edata; edata_t *edata;
while (true) { while (true) {
/* Get the LRU extent, if any. */ /* Get the LRU extent, if any. */
edata = edata_list_inactive_first(&ecache->eset.lru); eset_t *eset = &ecache->eset;
edata = edata_list_inactive_first(&eset->lru);
if (edata == NULL) {
/*
* Next check if there are guarded extents. They are
* more expensive to purge (since they are not
* mergeable), thus in favor of caching them longer.
*/
eset = &ecache->guarded_eset;
edata = edata_list_inactive_first(&eset->lru);
if (edata == NULL) { if (edata == NULL) {
goto label_return; goto label_return;
} }
}
/* Check the eviction limit. */ /* Check the eviction limit. */
size_t extents_npages = ecache_npages_get(ecache); size_t extents_npages = ecache_npages_get(ecache);
if (extents_npages <= npages_min) { if (extents_npages <= npages_min) {
edata = NULL; edata = NULL;
goto label_return; goto label_return;
} }
eset_remove(&ecache->eset, edata); eset_remove(eset, edata);
if (!ecache->delay_coalesce) { if (!ecache->delay_coalesce) {
break; break;
} }
@ -234,17 +254,19 @@ extent_deactivate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
assert(edata_state_get(edata) == extent_state_active); assert(edata_state_get(edata) == extent_state_active);
emap_update_edata_state(tsdn, pac->emap, edata, ecache->state); emap_update_edata_state(tsdn, pac->emap, edata, ecache->state);
eset_insert(&ecache->eset, edata); eset_t *eset = edata_guarded_get(edata) ? &ecache->guarded_eset :
&ecache->eset;
eset_insert(eset, edata);
} }
static void static void
extent_activate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, extent_activate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache, eset_t *eset,
edata_t *edata) { edata_t *edata) {
assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache)); assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache));
assert(edata_state_get(edata) == ecache->state || assert(edata_state_get(edata) == ecache->state ||
edata_state_get(edata) == extent_state_merging); edata_state_get(edata) == extent_state_merging);
eset_remove(&ecache->eset, edata); eset_remove(eset, edata);
emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active); emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active);
} }
@ -350,7 +372,8 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, pac_t *pac,
*/ */
static edata_t * static edata_t *
extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment) { ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
bool guarded) {
malloc_mutex_assert_owner(tsdn, &ecache->mtx); malloc_mutex_assert_owner(tsdn, &ecache->mtx);
assert(alignment > 0); assert(alignment > 0);
if (config_debug && expand_edata != NULL) { if (config_debug && expand_edata != NULL) {
@ -366,6 +389,7 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
} }
edata_t *edata; edata_t *edata;
eset_t *eset = guarded ? &ecache->guarded_eset : &ecache->eset;
if (expand_edata != NULL) { if (expand_edata != NULL) {
edata = emap_try_acquire_edata_neighbor_expand(tsdn, pac->emap, edata = emap_try_acquire_edata_neighbor_expand(tsdn, pac->emap,
expand_edata, EXTENT_PAI_PAC, ecache->state); expand_edata, EXTENT_PAI_PAC, ecache->state);
@ -382,7 +406,7 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
* If split and merge are not allowed (Windows w/o retain), try * If split and merge are not allowed (Windows w/o retain), try
* exact fit only. * exact fit only.
*/ */
bool exact_only = (!maps_coalesce && !opt_retain); bool exact_only = (!maps_coalesce && !opt_retain) || guarded;
/* /*
* A large extent might be broken up from its original size to * A large extent might be broken up from its original size to
* some small size to satisfy a small request. When that small * some small size to satisfy a small request. When that small
@ -394,13 +418,13 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
*/ */
unsigned lg_max_fit = ecache->delay_coalesce unsigned lg_max_fit = ecache->delay_coalesce
? (unsigned)opt_lg_extent_max_active_fit : SC_PTR_BITS; ? (unsigned)opt_lg_extent_max_active_fit : SC_PTR_BITS;
edata = eset_fit(&ecache->eset, size, alignment, exact_only, edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit);
lg_max_fit);
} }
if (edata == NULL) { if (edata == NULL) {
return NULL; return NULL;
} }
extent_activate_locked(tsdn, pac, ecache, edata); assert(!guarded || edata_guarded_get(edata));
extent_activate_locked(tsdn, pac, ecache, eset, edata);
return edata; return edata;
} }
@ -551,13 +575,14 @@ extent_recycle_split(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
static edata_t * static edata_t *
extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, extent_recycle(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
edata_t *expand_edata, size_t size, size_t alignment, bool zero, edata_t *expand_edata, size_t size, size_t alignment, bool zero,
bool *commit, bool growing_retained) { bool *commit, bool growing_retained, bool guarded) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0); WITNESS_RANK_CORE, growing_retained ? 1 : 0);
assert(!guarded || expand_edata == NULL);
malloc_mutex_lock(tsdn, &ecache->mtx); malloc_mutex_lock(tsdn, &ecache->mtx);
edata_t *edata = extent_recycle_extract(tsdn, pac, ehooks, ecache, edata_t *edata = extent_recycle_extract(tsdn, pac, ehooks, ecache,
expand_edata, size, alignment); expand_edata, size, alignment, guarded);
if (edata == NULL) { if (edata == NULL) {
malloc_mutex_unlock(tsdn, &ecache->mtx); malloc_mutex_unlock(tsdn, &ecache->mtx);
return NULL; return NULL;
@ -734,7 +759,7 @@ label_err:
static edata_t * static edata_t *
extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
edata_t *expand_edata, size_t size, size_t alignment, bool zero, edata_t *expand_edata, size_t size, size_t alignment, bool zero,
bool *commit) { bool *commit, bool guarded) {
assert(size != 0); assert(size != 0);
assert(alignment != 0); assert(alignment != 0);
@ -742,13 +767,13 @@ extent_alloc_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
edata_t *edata = extent_recycle(tsdn, pac, ehooks, edata_t *edata = extent_recycle(tsdn, pac, ehooks,
&pac->ecache_retained, expand_edata, size, alignment, zero, commit, &pac->ecache_retained, expand_edata, size, alignment, zero, commit,
/* growing_retained */ true); /* growing_retained */ true, guarded);
if (edata != NULL) { if (edata != NULL) {
malloc_mutex_unlock(tsdn, &pac->grow_mtx); malloc_mutex_unlock(tsdn, &pac->grow_mtx);
if (config_prof) { if (config_prof) {
extent_gdump_add(tsdn, edata); extent_gdump_add(tsdn, edata);
} }
} else if (opt_retain && expand_edata == NULL) { } else if (opt_retain && expand_edata == NULL && !guarded) {
edata = extent_grow_retained(tsdn, pac, ehooks, size, edata = extent_grow_retained(tsdn, pac, ehooks, size,
alignment, zero, commit); alignment, zero, commit);
/* extent_grow_retained() always releases pac->grow_mtx. */ /* extent_grow_retained() always releases pac->grow_mtx. */
@ -910,6 +935,9 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
emap_assert_mapped(tsdn, pac->emap, edata); emap_assert_mapped(tsdn, pac->emap, edata);
if (edata_guarded_get(edata)) {
goto label_skip_coalesce;
}
if (!ecache->delay_coalesce) { if (!ecache->delay_coalesce) {
edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, edata, edata = extent_try_coalesce(tsdn, pac, ehooks, ecache, edata,
NULL); NULL);
@ -931,6 +959,7 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
return; return;
} }
} }
label_skip_coalesce:
extent_deactivate_locked(tsdn, pac, ecache, edata); extent_deactivate_locked(tsdn, pac, ecache, edata);
malloc_mutex_unlock(tsdn, &ecache->mtx); malloc_mutex_unlock(tsdn, &ecache->mtx);
@ -981,6 +1010,11 @@ extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
/* Avoid calling the default extent_dalloc unless have to. */ /* Avoid calling the default extent_dalloc unless have to. */
if (!ehooks_dalloc_will_fail(ehooks)) { if (!ehooks_dalloc_will_fail(ehooks)) {
/* Restore guard pages for dalloc / unmap. */
if (edata_guarded_get(edata)) {
assert(ehooks_are_default(ehooks));
unguard_pages(tsdn, ehooks, edata, pac->emap);
}
/* /*
* Deregister first to avoid a race with other allocating * Deregister first to avoid a race with other allocating
* threads, and reregister if deallocation fails. * threads, and reregister if deallocation fails.

63
src/guard.c Normal file
View File

@ -0,0 +1,63 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/guard.h"
#include "jemalloc/internal/tsd.h"
/* The sanitizer options. */
size_t opt_san_guard_large = SAN_GUARD_LARGE_EVERY_N_EXTENTS_DEFAULT;
size_t opt_san_guard_small = SAN_GUARD_SMALL_EVERY_N_EXTENTS_DEFAULT;
void
guard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap) {
emap_deregister_boundary(tsdn, emap, edata);
size_t size_with_guards = edata_size_get(edata);
size_t usize = size_with_guards - PAGE_GUARDS_SIZE;
uintptr_t guard1 = (uintptr_t)edata_base_get(edata);
uintptr_t addr = guard1 + PAGE;
uintptr_t guard2 = addr + usize;
assert(edata_state_get(edata) == extent_state_active);
ehooks_guard(tsdn, ehooks, (void *)guard1, (void *)guard2);
/* Update the guarded addr and usable size of the edata. */
edata_size_set(edata, usize);
edata_addr_set(edata, (void *)addr);
edata_guarded_set(edata, true);
/* The new boundary will be registered on the pa_alloc path. */
}
void
unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap) {
/* Remove the inner boundary which no longer exists. */
emap_deregister_boundary(tsdn, emap, edata);
size_t size = edata_size_get(edata);
size_t size_with_guards = size + PAGE_GUARDS_SIZE;
uintptr_t addr = (uintptr_t)edata_base_get(edata);
uintptr_t guard1 = addr - PAGE;
uintptr_t guard2 = addr + size;
assert(edata_state_get(edata) == extent_state_active);
ehooks_unguard(tsdn, ehooks, (void *)guard1, (void *)guard2);
/* Update the true addr and usable size of the edata. */
edata_size_set(edata, size_with_guards);
edata_addr_set(edata, (void *)guard1);
edata_guarded_set(edata, false);
/* Then re-register the outer boundary including the guards. */
emap_register_boundary(tsdn, emap, edata, SC_NSIZES, /* slab */ false);
}
void
tsd_san_init(tsd_t *tsd) {
*tsd_san_extents_until_guard_smallp_get(tsd) = opt_san_guard_small;
*tsd_san_extents_until_guard_largep_get(tsd) = opt_san_guard_large;
}

View File

@ -9,7 +9,7 @@
#define HPA_EDEN_SIZE (128 * HUGEPAGE) #define HPA_EDEN_SIZE (128 * HUGEPAGE)
static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool *deferred_work_generated); size_t alignment, bool zero, bool guarded, bool *deferred_work_generated);
static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated); size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated);
static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
@ -750,8 +750,9 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
static edata_t * static edata_t *
hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool *deferred_work_generated) { bool guarded, bool *deferred_work_generated) {
assert((size & PAGE_MASK) == 0); assert((size & PAGE_MASK) == 0);
assert(!guarded);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
@ -796,7 +797,6 @@ hpa_dalloc_prepare_unlocked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
assert(edata_state_get(edata) == extent_state_active); assert(edata_state_get(edata) == extent_state_active);
assert(edata_arena_ind_get(edata) == shard->ind); assert(edata_arena_ind_get(edata) == shard->ind);
assert(edata_szind_get_maybe_invalid(edata) == SC_NSIZES); assert(edata_szind_get_maybe_invalid(edata) == SC_NSIZES);
assert(!edata_slab_get(edata));
assert(edata_committed_get(edata)); assert(edata_committed_get(edata));
assert(edata_base_get(edata) != NULL); assert(edata_base_get(edata) != NULL);
@ -865,6 +865,7 @@ hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
static void static void
hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated) { bool *deferred_work_generated) {
assert(!edata_guarded_get(edata));
/* Just a dalloc_batch of size 1; this lets us share logic. */ /* Just a dalloc_batch of size 1; this lets us share logic. */
edata_list_active_t dalloc_list; edata_list_active_t dalloc_list;
edata_list_active_init(&dalloc_list); edata_list_active_init(&dalloc_list);

View File

@ -10,6 +10,7 @@
#include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/fxp.h" #include "jemalloc/internal/fxp.h"
#include "jemalloc/internal/guard.h"
#include "jemalloc/internal/hook.h" #include "jemalloc/internal/hook.h"
#include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/log.h" #include "jemalloc/internal/log.h"
@ -1616,6 +1617,14 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
} }
CONF_CONTINUE; CONF_CONTINUE;
} }
CONF_HANDLE_SIZE_T(opt_san_guard_small,
"san_guard_small", 0, SIZE_T_MAX,
CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
CONF_HANDLE_SIZE_T(opt_san_guard_large,
"san_guard_large", 0, SIZE_T_MAX,
CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false)
CONF_ERROR("Invalid conf pair", k, klen, v, vlen); CONF_ERROR("Invalid conf pair", k, klen, v, vlen);
#undef CONF_ERROR #undef CONF_ERROR
#undef CONF_CONTINUE #undef CONF_CONTINUE

View File

@ -1,6 +1,7 @@
#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/guard.h"
#include "jemalloc/internal/hpa.h" #include "jemalloc/internal/hpa.h"
static void static void
@ -118,15 +119,17 @@ pa_get_pai(pa_shard_t *shard, edata_t *edata) {
edata_t * edata_t *
pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
bool slab, szind_t szind, bool zero, bool *deferred_work_generated) { bool slab, szind_t szind, bool zero, bool guarded,
bool *deferred_work_generated) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0); WITNESS_RANK_CORE, 0);
assert(!guarded || alignment <= PAGE);
edata_t *edata = NULL; edata_t *edata = NULL;
*deferred_work_generated = false; *deferred_work_generated = false;
if (pa_shard_uses_hpa(shard)) { if (!guarded && pa_shard_uses_hpa(shard)) {
edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment, edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment,
zero, deferred_work_generated); zero, /* guarded */ false, deferred_work_generated);
} }
/* /*
* Fall back to the PAC if the HPA is off or couldn't serve the given * Fall back to the PAC if the HPA is off or couldn't serve the given
@ -134,10 +137,10 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
*/ */
if (edata == NULL) { if (edata == NULL) {
edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero, edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero,
deferred_work_generated); guarded, deferred_work_generated);
} }
if (edata != NULL) { if (edata != NULL) {
assert(edata_size_get(edata) == size);
pa_nactive_add(shard, size >> LG_PAGE); pa_nactive_add(shard, size >> LG_PAGE);
emap_remap(tsdn, shard->emap, edata, szind, slab); emap_remap(tsdn, shard->emap, edata, szind, slab);
edata_szind_set(edata, szind); edata_szind_set(edata, szind);
@ -145,8 +148,6 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
if (slab && (size > 2 * PAGE)) { if (slab && (size > 2 * PAGE)) {
emap_register_interior(tsdn, shard->emap, edata, szind); emap_register_interior(tsdn, shard->emap, edata, szind);
} }
}
if (edata != NULL) {
assert(edata_arena_ind_get(edata) == shard->ind); assert(edata_arena_ind_get(edata) == shard->ind);
} }
return edata; return edata;
@ -158,7 +159,9 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
assert(new_size > old_size); assert(new_size > old_size);
assert(edata_size_get(edata) == old_size); assert(edata_size_get(edata) == old_size);
assert((new_size & PAGE_MASK) == 0); assert((new_size & PAGE_MASK) == 0);
if (edata_guarded_get(edata)) {
return true;
}
size_t expand_amount = new_size - old_size; size_t expand_amount = new_size - old_size;
pai_t *pai = pa_get_pai(shard, edata); pai_t *pai = pa_get_pai(shard, edata);
@ -181,6 +184,9 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
assert(new_size < old_size); assert(new_size < old_size);
assert(edata_size_get(edata) == old_size); assert(edata_size_get(edata) == old_size);
assert((new_size & PAGE_MASK) == 0); assert((new_size & PAGE_MASK) == 0);
if (edata_guarded_get(edata)) {
return true;
}
size_t shrink_amount = old_size - new_size; size_t shrink_amount = old_size - new_size;
pai_t *pai = pa_get_pai(shard, edata); pai_t *pai = pa_get_pai(shard, edata);
@ -202,7 +208,10 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
emap_remap(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false); emap_remap(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false);
if (edata_slab_get(edata)) { if (edata_slab_get(edata)) {
emap_deregister_interior(tsdn, shard->emap, edata); emap_deregister_interior(tsdn, shard->emap, edata);
edata_slab_set(edata, false); /*
* The slab state of the extent isn't cleared. It may be used
* by the pai implementation, e.g. to make caching decisions.
*/
} }
edata_addr_set(edata, edata_base_get(edata)); edata_addr_set(edata, edata_base_get(edata));
edata_szind_set(edata, SC_NSIZES); edata_szind_set(edata, SC_NSIZES);

View File

@ -2,9 +2,10 @@
#include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/pac.h" #include "jemalloc/internal/pac.h"
#include "jemalloc/internal/guard.h"
static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool *deferred_work_generated); size_t alignment, bool zero, bool guarded, bool *deferred_work_generated);
static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
@ -109,28 +110,66 @@ pac_may_have_muzzy(pac_t *pac) {
} }
static edata_t * static edata_t *
pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
bool zero, bool *deferred_work_generated) { size_t alignment, bool zero, bool guarded) {
pac_t *pac = (pac_t *)self; assert(!guarded || alignment <= PAGE);
*deferred_work_generated = false;
ehooks_t *ehooks = pac_ehooks_get(pac);
edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
NULL, size, alignment, zero); NULL, size, alignment, zero, guarded);
if (edata == NULL && pac_may_have_muzzy(pac)) { if (edata == NULL && pac_may_have_muzzy(pac)) {
edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
NULL, size, alignment, zero); NULL, size, alignment, zero, guarded);
} }
if (edata == NULL) { if (edata == NULL) {
edata = ecache_alloc_grow(tsdn, pac, ehooks, edata = ecache_alloc_grow(tsdn, pac, ehooks,
&pac->ecache_retained, NULL, size, alignment, zero); &pac->ecache_retained, NULL, size, alignment, zero,
guarded);
if (config_stats && edata != NULL) { if (config_stats && edata != NULL) {
atomic_fetch_add_zu(&pac->stats->pac_mapped, size, atomic_fetch_add_zu(&pac->stats->pac_mapped, size,
ATOMIC_RELAXED); ATOMIC_RELAXED);
} }
} }
return edata;
}
static edata_t *
pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
size_t alignment, bool zero) {
assert(alignment <= PAGE);
size_t size_with_guards = size + PAGE_GUARDS_SIZE;
/* Alloc a non-guarded extent first.*/
edata_t *edata = pac_alloc_real(tsdn, pac, ehooks, size_with_guards,
/* alignment */ PAGE, zero, /* guarded */ false);
if (edata != NULL) {
/* Add guards around it. */
assert(edata_size_get(edata) == size_with_guards);
guard_pages(tsdn, ehooks, edata, pac->emap);
}
assert(edata == NULL || (edata_guarded_get(edata) &&
edata_size_get(edata) == size));
return edata;
}
static edata_t *
pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
bool zero, bool guarded, bool *deferred_work_generated) {
*deferred_work_generated = false;
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
edata_t *edata = pac_alloc_real(tsdn, pac, ehooks, size, alignment,
zero, guarded);
if (edata == NULL && guarded) {
/* No cached guarded extents; creating a new one. */
edata = pac_alloc_new_guarded(tsdn, pac, ehooks, size,
alignment, zero);
}
return edata; return edata;
} }
@ -149,15 +188,15 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
return true; return true;
} }
edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
edata, expand_amount, PAGE, zero); edata, expand_amount, PAGE, zero, /* guarded*/ false);
if (trail == NULL) { if (trail == NULL) {
trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy, trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
edata, expand_amount, PAGE, zero); edata, expand_amount, PAGE, zero, /* guarded*/ false);
} }
if (trail == NULL) { if (trail == NULL) {
trail = ecache_alloc_grow(tsdn, pac, ehooks, trail = ecache_alloc_grow(tsdn, pac, ehooks,
&pac->ecache_retained, edata, expand_amount, PAGE, &pac->ecache_retained, edata, expand_amount, PAGE,
zero); zero, /* guarded */ false);
mapped_add = expand_amount; mapped_add = expand_amount;
} }
if (trail == NULL) { if (trail == NULL) {
@ -203,6 +242,27 @@ pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated) { bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self; pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac); ehooks_t *ehooks = pac_ehooks_get(pac);
if (edata_guarded_get(edata)) {
/*
* Because cached guarded extents do exact fit only, large
* guarded extents are restored on dalloc eagerly (otherwise
* they will not be reused efficiently). Slab sizes have a
* limited number of size classes, and tend to cycle faster.
*
* In the case where coalesce is restrained (VirtualFree on
* Windows), guarded extents are also not cached -- otherwise
* during arena destroy / reset, the retained extents would not
* be whole regions (i.e. they are split between regular and
* guarded).
*/
if (!edata_slab_get(edata) || !maps_coalesce) {
assert(edata_size_get(edata) >= SC_LARGE_MINCLASS ||
!maps_coalesce);
unguard_pages(tsdn, ehooks, edata, pac->emap);
}
}
ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata); ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
/* Purging of deallocated pages is deferred */ /* Purging of deallocated pages is deferred */
*deferred_work_generated = true; *deferred_work_generated = true;

View File

@ -316,14 +316,10 @@ pages_unmap(void *addr, size_t size) {
} }
static bool static bool
pages_commit_impl(void *addr, size_t size, bool commit) { os_pages_commit(void *addr, size_t size, bool commit) {
assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size); assert(PAGE_CEILING(size) == size);
if (os_overcommits) {
return true;
}
#ifdef _WIN32 #ifdef _WIN32
return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT))); PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
@ -348,6 +344,15 @@ pages_commit_impl(void *addr, size_t size, bool commit) {
#endif #endif
} }
static bool
pages_commit_impl(void *addr, size_t size, bool commit) {
if (os_overcommits) {
return true;
}
return os_pages_commit(addr, size, commit);
}
bool bool
pages_commit(void *addr, size_t size) { pages_commit(void *addr, size_t size) {
return pages_commit_impl(addr, size, true); return pages_commit_impl(addr, size, true);
@ -358,6 +363,38 @@ pages_decommit(void *addr, size_t size) {
return pages_commit_impl(addr, size, false); return pages_commit_impl(addr, size, false);
} }
void
pages_mark_guards(void *head, void *tail) {
assert(head != NULL && tail != NULL);
assert((uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
mprotect(head, PAGE, PROT_NONE);
mprotect(tail, PAGE, PROT_NONE);
#else
/* Decommit sets to PROT_NONE / MEM_DECOMMIT. */
os_pages_commit(head, PAGE, false);
os_pages_commit(tail, PAGE, false);
#endif
}
void
pages_unmark_guards(void *head, void *tail) {
assert(head != NULL && tail != NULL);
assert((uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
size_t range = (uintptr_t)tail - (uintptr_t)head + PAGE;
if (range <= SC_LARGE_MINCLASS) {
mprotect(head, range, PROT_READ | PROT_WRITE);
} else {
mprotect(head, PAGE, PROT_READ | PROT_WRITE);
mprotect(tail, PAGE, PROT_READ | PROT_WRITE);
}
#else
os_pages_commit(head, PAGE, true);
os_pages_commit(tail, PAGE, true);
#endif
}
bool bool
pages_purge_lazy(void *addr, size_t size) { pages_purge_lazy(void *addr, size_t size) {
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);

View File

@ -7,7 +7,7 @@ pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
for (size_t i = 0; i < nallocs; i++) { for (size_t i = 0; i < nallocs; i++) {
bool deferred_by_alloc = false; bool deferred_by_alloc = false;
edata_t *edata = pai_alloc(tsdn, self, size, PAGE, edata_t *edata = pai_alloc(tsdn, self, size, PAGE,
/* zero */ false, &deferred_by_alloc); /* zero */ false, /* guarded */ false, &deferred_by_alloc);
*deferred_work_generated |= deferred_by_alloc; *deferred_work_generated |= deferred_by_alloc;
if (edata == NULL) { if (edata == NULL) {
return i; return i;

View File

@ -4,7 +4,7 @@
#include "jemalloc/internal/sec.h" #include "jemalloc/internal/sec.h"
static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool *deferred_work_generated); size_t alignment, bool zero, bool guarded, bool *deferred_work_generated);
static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
@ -218,8 +218,9 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
static edata_t * static edata_t *
sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool *deferred_work_generated) { bool guarded, bool *deferred_work_generated) {
assert((size & PAGE_MASK) == 0); assert((size & PAGE_MASK) == 0);
assert(!guarded);
sec_t *sec = (sec_t *)self; sec_t *sec = (sec_t *)self;
*deferred_work_generated = false; *deferred_work_generated = false;
@ -227,7 +228,7 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
if (zero || alignment > PAGE || sec->opts.nshards == 0 if (zero || alignment > PAGE || sec->opts.nshards == 0
|| size > sec->opts.max_alloc) { || size > sec->opts.max_alloc) {
return pai_alloc(tsdn, sec->fallback, size, alignment, zero, return pai_alloc(tsdn, sec->fallback, size, alignment, zero,
deferred_work_generated); /* guarded */ false, deferred_work_generated);
} }
pszind_t pszind = sz_psz2ind(size); pszind_t pszind = sz_psz2ind(size);
sec_shard_t *shard = sec_shard_pick(tsdn, sec); sec_shard_t *shard = sec_shard_pick(tsdn, sec);
@ -250,7 +251,7 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
size); size);
} else { } else {
edata = pai_alloc(tsdn, sec->fallback, size, alignment, edata = pai_alloc(tsdn, sec->fallback, size, alignment,
zero, deferred_work_generated); zero, /* guarded */ false, deferred_work_generated);
} }
} }
return edata; return edata;

View File

@ -2,6 +2,7 @@
#include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h" #include "jemalloc/internal/assert.h"
#include "jemalloc/internal/guard.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
@ -242,6 +243,7 @@ tsd_data_init(tsd_t *tsd) {
rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd));
tsd_prng_state_init(tsd); tsd_prng_state_init(tsd);
tsd_te_init(tsd); /* event_init may use the prng state above. */ tsd_te_init(tsd); /* event_init may use the prng state above. */
tsd_san_init(tsd);
return tsd_tcache_enabled_data_init(tsd); return tsd_tcache_enabled_data_init(tsd);
} }
@ -269,6 +271,7 @@ tsd_data_init_nocleanup(tsd_t *tsd) {
*tsd_reentrancy_levelp_get(tsd) = 1; *tsd_reentrancy_levelp_get(tsd) = 1;
tsd_prng_state_init(tsd); tsd_prng_state_init(tsd);
tsd_te_init(tsd); /* event_init may use the prng state above. */ tsd_te_init(tsd); /* event_init may use the prng state above. */
tsd_san_init(tsd);
assert_tsd_data_cleanup_done(tsd); assert_tsd_data_cleanup_done(tsd);
return false; return false;

View File

@ -0,0 +1,149 @@
static unsigned
do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
unsigned arena_ind;
size_t sz = sizeof(unsigned);
expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
0, "Unexpected mallctl() failure");
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.dirty_decay_ms", mib, &miblen),
0, "Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL,
(void *)&dirty_decay_ms, sizeof(dirty_decay_ms)), 0,
"Unexpected mallctlbymib() failure");
expect_d_eq(mallctlnametomib("arena.0.muzzy_decay_ms", mib, &miblen),
0, "Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL,
(void *)&muzzy_decay_ms, sizeof(muzzy_decay_ms)), 0,
"Unexpected mallctlbymib() failure");
return arena_ind;
}
static void
do_arena_destroy(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.destroy", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
}
static void
do_epoch(void) {
uint64_t epoch = 1;
expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
0, "Unexpected mallctl() failure");
}
static void
do_purge(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.purge", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
}
static void
do_decay(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.decay", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
}
static uint64_t
get_arena_npurge_impl(const char *mibname, unsigned arena_ind) {
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib(mibname, mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind;
uint64_t npurge = 0;
size_t sz = sizeof(npurge);
expect_d_eq(mallctlbymib(mib, miblen, (void *)&npurge, &sz, NULL, 0),
config_stats ? 0 : ENOENT, "Unexpected mallctlbymib() failure");
return npurge;
}
static uint64_t
get_arena_dirty_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind);
}
static uint64_t
get_arena_dirty_purged(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind);
}
static uint64_t
get_arena_muzzy_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static uint64_t
get_arena_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) +
get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static size_t
get_arena_pdirty(unsigned arena_ind) {
do_epoch();
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("stats.arenas.0.pdirty", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind;
size_t pdirty;
size_t sz = sizeof(pdirty);
expect_d_eq(mallctlbymib(mib, miblen, (void *)&pdirty, &sz, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
return pdirty;
}
static size_t
get_arena_pmuzzy(unsigned arena_ind) {
do_epoch();
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("stats.arenas.0.pmuzzy", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind;
size_t pmuzzy;
size_t sz = sizeof(pmuzzy);
expect_d_eq(mallctlbymib(mib, miblen, (void *)&pmuzzy, &sz, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
return pmuzzy;
}
static void *
do_mallocx(size_t size, int flags) {
void *p = mallocx(size, flags);
expect_ptr_not_null(p, "Unexpected mallocx() failure");
return p;
}
static void
generate_dirty(unsigned arena_ind, size_t size) {
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
void *p = do_mallocx(size, flags);
dallocx(p, flags);
}

View File

@ -0,0 +1,6 @@
static inline bool
extent_is_guarded(tsdn_t *tsdn, void *ptr) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
return edata_guarded_get(edata);
}

View File

@ -1,4 +1,5 @@
#include "test/jemalloc_test.h" #include "test/jemalloc_test.h"
#include "test/arena_decay.h"
#include "jemalloc/internal/ticker.h" #include "jemalloc/internal/ticker.h"
@ -22,155 +23,6 @@ nstime_update_mock(nstime_t *time) {
} }
} }
static unsigned
do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
unsigned arena_ind;
size_t sz = sizeof(unsigned);
expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
0, "Unexpected mallctl() failure");
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.dirty_decay_ms", mib, &miblen),
0, "Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL,
(void *)&dirty_decay_ms, sizeof(dirty_decay_ms)), 0,
"Unexpected mallctlbymib() failure");
expect_d_eq(mallctlnametomib("arena.0.muzzy_decay_ms", mib, &miblen),
0, "Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL,
(void *)&muzzy_decay_ms, sizeof(muzzy_decay_ms)), 0,
"Unexpected mallctlbymib() failure");
return arena_ind;
}
static void
do_arena_destroy(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.destroy", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
}
void
do_epoch(void) {
uint64_t epoch = 1;
expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
0, "Unexpected mallctl() failure");
}
void
do_purge(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.purge", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
}
void
do_decay(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("arena.0.decay", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
}
static uint64_t
get_arena_npurge_impl(const char *mibname, unsigned arena_ind) {
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib(mibname, mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind;
uint64_t npurge = 0;
size_t sz = sizeof(npurge);
expect_d_eq(mallctlbymib(mib, miblen, (void *)&npurge, &sz, NULL, 0),
config_stats ? 0 : ENOENT, "Unexpected mallctlbymib() failure");
return npurge;
}
static uint64_t
get_arena_dirty_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind);
}
static uint64_t
get_arena_dirty_purged(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind);
}
static uint64_t
get_arena_muzzy_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static uint64_t
get_arena_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) +
get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static size_t
get_arena_pdirty(unsigned arena_ind) {
do_epoch();
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("stats.arenas.0.pdirty", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind;
size_t pdirty;
size_t sz = sizeof(pdirty);
expect_d_eq(mallctlbymib(mib, miblen, (void *)&pdirty, &sz, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
return pdirty;
}
static size_t
get_arena_pmuzzy(unsigned arena_ind) {
do_epoch();
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
expect_d_eq(mallctlnametomib("stats.arenas.0.pmuzzy", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind;
size_t pmuzzy;
size_t sz = sizeof(pmuzzy);
expect_d_eq(mallctlbymib(mib, miblen, (void *)&pmuzzy, &sz, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
return pmuzzy;
}
static void *
do_mallocx(size_t size, int flags) {
void *p = mallocx(size, flags);
expect_ptr_not_null(p, "Unexpected mallocx() failure");
return p;
}
static void
generate_dirty(unsigned arena_ind, size_t size) {
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
void *p = do_mallocx(size, flags);
dallocx(p, flags);
}
TEST_BEGIN(test_decay_ticks) { TEST_BEGIN(test_decay_ticks) {
test_skip_if(is_background_thread_enabled()); test_skip_if(is_background_thread_enabled());
test_skip_if(opt_hpa); test_skip_if(opt_hpa);

View File

@ -1,4 +1,5 @@
#include "test/jemalloc_test.h" #include "test/jemalloc_test.h"
#include "test/guard.h"
#include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/safety_check.h"
@ -30,8 +31,18 @@ TEST_BEGIN(test_large_double_free_tcache) {
test_large_double_free_pre(); test_large_double_free_pre();
char *ptr = malloc(SC_LARGE_MINCLASS); char *ptr = malloc(SC_LARGE_MINCLASS);
bool guarded = extent_is_guarded(tsdn_fetch(), ptr);
free(ptr); free(ptr);
if (!guarded) {
free(ptr); free(ptr);
} else {
/*
* Skip because guarded extents may unguard immediately on
* deallocation, in which case the second free will crash before
* reaching the intended safety check.
*/
fake_abort_called = true;
}
mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); mallctl("thread.tcache.flush", NULL, NULL, NULL, 0);
test_large_double_free_post(); test_large_double_free_post();
} }
@ -43,8 +54,18 @@ TEST_BEGIN(test_large_double_free_no_tcache) {
test_large_double_free_pre(); test_large_double_free_pre();
char *ptr = mallocx(SC_LARGE_MINCLASS, MALLOCX_TCACHE_NONE); char *ptr = mallocx(SC_LARGE_MINCLASS, MALLOCX_TCACHE_NONE);
bool guarded = extent_is_guarded(tsdn_fetch(), ptr);
dallocx(ptr, MALLOCX_TCACHE_NONE); dallocx(ptr, MALLOCX_TCACHE_NONE);
if (!guarded) {
dallocx(ptr, MALLOCX_TCACHE_NONE); dallocx(ptr, MALLOCX_TCACHE_NONE);
} else {
/*
* Skip because guarded extents may unguard immediately on
* deallocation, in which case the second free will crash before
* reaching the intended safety check.
*/
fake_abort_called = true;
}
test_large_double_free_post(); test_large_double_free_post();
} }
TEST_END TEST_END

201
test/unit/guard.c Normal file
View File

@ -0,0 +1,201 @@
#include "test/jemalloc_test.h"
#include "test/arena_decay.h"
#include "test/guard.h"
#include "jemalloc/internal/guard.h"
static void
verify_extent_guarded(tsdn_t *tsdn, void *ptr) {
expect_true(extent_is_guarded(tsdn, ptr),
"All extents should be guarded.");
}
#define MAX_SMALL_ALLOCATIONS 4096
void *small_alloc[MAX_SMALL_ALLOCATIONS];
TEST_BEGIN(test_guarded_small) {
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
unsigned npages = 16, pages_found = 0, ends_found = 0;
VARIABLE_ARRAY(uintptr_t, pages, npages);
/* Allocate to get sanitized pointers. */
size_t sz = PAGE / 8;
unsigned n_alloc = 0;
while (n_alloc < MAX_SMALL_ALLOCATIONS) {
void *ptr = malloc(sz);
expect_ptr_not_null(ptr, "Unexpected malloc() failure");
small_alloc[n_alloc] = ptr;
verify_extent_guarded(tsdn, ptr);
if ((uintptr_t)ptr % PAGE == 0) {
pages[pages_found++] = (uintptr_t)ptr;
}
if (((uintptr_t)ptr + (uintptr_t)sz) % PAGE == 0) {
ends_found++;
}
n_alloc++;
if (pages_found == npages && ends_found == npages) {
break;
}
}
/* Should found the ptrs being checked for overflow and underflow. */
expect_u_eq(pages_found, npages, "Could not found the expected pages.");
expect_u_eq(ends_found, npages, "Could not found the expected pages.");
/* Verify the pages are not continuous, i.e. separated by guards. */
for (unsigned i = 0; i < npages - 1; i++) {
for (unsigned j = i + 1; j < npages; j++) {
uintptr_t ptr_diff = pages[i] > pages[j] ?
pages[i] - pages[j] : pages[j] - pages[i];
expect_zu_gt((size_t)ptr_diff, 2 * PAGE,
"Pages should not be next to each other.");
}
}
for (unsigned i = 0; i < n_alloc + 1; i++) {
free(small_alloc[i]);
}
}
TEST_END
TEST_BEGIN(test_guarded_large) {
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
unsigned nlarge = 32;
VARIABLE_ARRAY(uintptr_t, large, nlarge);
/* Allocate to get sanitized pointers. */
size_t large_sz = SC_LARGE_MINCLASS;
for (unsigned i = 0; i < nlarge; i++) {
void *ptr = malloc(large_sz);
verify_extent_guarded(tsdn, ptr);
expect_ptr_not_null(ptr, "Unexpected malloc() failure");
large[i] = (uintptr_t)ptr;
}
/* Verify the pages are not continuous, i.e. separated by guards. */
uintptr_t min_diff = (uintptr_t)-1;
for (unsigned i = 0; i < nlarge; i++) {
for (unsigned j = i + 1; j < nlarge; j++) {
uintptr_t ptr_diff = large[i] > large[j] ?
large[i] - large[j] : large[j] - large[i];
expect_zu_ge((size_t)ptr_diff, large_sz + 2 * PAGE,
"Pages should not be next to each other.");
if (ptr_diff < min_diff) {
min_diff = ptr_diff;
}
}
}
expect_zu_ge((size_t)min_diff, large_sz + 2 * PAGE,
"Pages should not be next to each other.");
for (unsigned i = 0; i < nlarge; i++) {
free((void *)large[i]);
}
}
TEST_END
static void
verify_pdirty(unsigned arena_ind, uint64_t expected) {
uint64_t pdirty = get_arena_pdirty(arena_ind);
expect_u64_eq(pdirty, expected / PAGE,
"Unexpected dirty page amount.");
}
static void
verify_pmuzzy(unsigned arena_ind, uint64_t expected) {
uint64_t pmuzzy = get_arena_pmuzzy(arena_ind);
expect_u64_eq(pmuzzy, expected / PAGE,
"Unexpected muzzy page amount.");
}
TEST_BEGIN(test_guarded_decay) {
unsigned arena_ind = do_arena_create(-1, -1);
do_decay(arena_ind);
do_purge(arena_ind);
verify_pdirty(arena_ind, 0);
verify_pmuzzy(arena_ind, 0);
/* Verify that guarded extents as dirty. */
size_t sz1 = PAGE, sz2 = PAGE * 2;
/* W/o maps_coalesce, guarded extents are unguarded eagerly. */
size_t add_guard_size = maps_coalesce ? 0 : PAGE_GUARDS_SIZE;
generate_dirty(arena_ind, sz1);
verify_pdirty(arena_ind, sz1 + add_guard_size);
verify_pmuzzy(arena_ind, 0);
/* Should reuse the first extent. */
generate_dirty(arena_ind, sz1);
verify_pdirty(arena_ind, sz1 + add_guard_size);
verify_pmuzzy(arena_ind, 0);
/* Should not reuse; expect new dirty pages. */
generate_dirty(arena_ind, sz2);
verify_pdirty(arena_ind, sz1 + sz2 + 2 * add_guard_size);
verify_pmuzzy(arena_ind, 0);
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
/* Should reuse dirty extents for the two mallocx. */
void *p1 = do_mallocx(sz1, flags);
verify_extent_guarded(tsdn, p1);
verify_pdirty(arena_ind, sz2 + add_guard_size);
void *p2 = do_mallocx(sz2, flags);
verify_extent_guarded(tsdn, p2);
verify_pdirty(arena_ind, 0);
verify_pmuzzy(arena_ind, 0);
dallocx(p1, flags);
verify_pdirty(arena_ind, sz1 + add_guard_size);
dallocx(p2, flags);
verify_pdirty(arena_ind, sz1 + sz2 + 2 * add_guard_size);
verify_pmuzzy(arena_ind, 0);
do_purge(arena_ind);
verify_pdirty(arena_ind, 0);
verify_pmuzzy(arena_ind, 0);
if (config_stats) {
expect_u64_eq(get_arena_npurge(arena_ind), 1,
"Expected purging to occur");
expect_u64_eq(get_arena_dirty_npurge(arena_ind), 1,
"Expected purging to occur");
expect_u64_eq(get_arena_dirty_purged(arena_ind),
(sz1 + sz2 + 2 * add_guard_size) / PAGE,
"Expected purging to occur");
expect_u64_eq(get_arena_muzzy_npurge(arena_ind), 0,
"Expected purging to occur");
}
if (opt_retain) {
/*
* With retain, guarded extents are not mergable and will be
* cached in ecache_retained. They should be reused.
*/
void *new_p1 = do_mallocx(sz1, flags);
verify_extent_guarded(tsdn, p1);
expect_ptr_eq(p1, new_p1, "Expect to reuse p1");
void *new_p2 = do_mallocx(sz2, flags);
verify_extent_guarded(tsdn, p2);
expect_ptr_eq(p2, new_p2, "Expect to reuse p2");
dallocx(new_p1, flags);
verify_pdirty(arena_ind, sz1 + add_guard_size);
dallocx(new_p2, flags);
verify_pdirty(arena_ind, sz1 + sz2 + 2 * add_guard_size);
verify_pmuzzy(arena_ind, 0);
}
do_arena_destroy(arena_ind);
}
TEST_END
int
main(void) {
return test(
test_guarded_small,
test_guarded_large,
test_guarded_decay);
}

3
test/unit/guard.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/sh
export MALLOC_CONF="san_guard_large:1,san_guard_small:1"

View File

@ -80,11 +80,11 @@ TEST_BEGIN(test_alloc_max) {
/* Small max */ /* Small max */
bool deferred_work_generated; bool deferred_work_generated;
edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX, PAGE, false, edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX, PAGE, false, false,
&deferred_work_generated); &deferred_work_generated);
expect_ptr_not_null(edata, "Allocation of small max failed"); expect_ptr_not_null(edata, "Allocation of small max failed");
edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX + PAGE, PAGE, false, edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX + PAGE, PAGE, false,
&deferred_work_generated); false, &deferred_work_generated);
expect_ptr_null(edata, "Allocation of larger than small max succeeded"); expect_ptr_null(edata, "Allocation of larger than small max succeeded");
destroy_test_data(shard); destroy_test_data(shard);
@ -188,7 +188,7 @@ TEST_BEGIN(test_stress) {
size_t npages = npages_min + prng_range_zu(&prng_state, size_t npages = npages_min + prng_range_zu(&prng_state,
npages_max - npages_min); npages_max - npages_min);
edata_t *edata = pai_alloc(tsdn, &shard->pai, edata_t *edata = pai_alloc(tsdn, &shard->pai,
npages * PAGE, PAGE, false, npages * PAGE, PAGE, false, false,
&deferred_work_generated); &deferred_work_generated);
assert_ptr_not_null(edata, assert_ptr_not_null(edata,
"Unexpected allocation failure"); "Unexpected allocation failure");
@ -263,7 +263,8 @@ TEST_BEGIN(test_alloc_dalloc_batch) {
*/ */
for (size_t i = 0; i < NALLOCS / 2; i++) { for (size_t i = 0; i < NALLOCS / 2; i++) {
allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(allocs[i], "Unexpected alloc failure"); expect_ptr_not_null(allocs[i], "Unexpected alloc failure");
} }
edata_list_active_t allocs_list; edata_list_active_t allocs_list;
@ -299,7 +300,8 @@ TEST_BEGIN(test_alloc_dalloc_batch) {
/* Reallocate (individually), and ensure reuse and contiguity. */ /* Reallocate (individually), and ensure reuse and contiguity. */
for (size_t i = 0; i < NALLOCS; i++) { for (size_t i = 0; i < NALLOCS; i++) {
allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(allocs[i], "Unexpected alloc failure."); expect_ptr_not_null(allocs[i], "Unexpected alloc failure.");
} }
void *new_base = edata_base_get(allocs[0]); void *new_base = edata_base_get(allocs[0]);
@ -374,7 +376,7 @@ TEST_BEGIN(test_defer_time) {
edata_t *edatas[HUGEPAGE_PAGES]; edata_t *edatas[HUGEPAGE_PAGES];
for (int i = 0; i < (int)HUGEPAGE_PAGES; i++) { for (int i = 0; i < (int)HUGEPAGE_PAGES; i++) {
edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false,
&deferred_work_generated); false, &deferred_work_generated);
expect_ptr_not_null(edatas[i], "Unexpected null edata"); expect_ptr_not_null(edatas[i], "Unexpected null edata");
} }
hpa_shard_do_deferred_work(tsdn, shard); hpa_shard_do_deferred_work(tsdn, shard);
@ -408,7 +410,7 @@ TEST_BEGIN(test_defer_time) {
*/ */
for (int i = 0; i < (int)HUGEPAGE_PAGES / 2; i++) { for (int i = 0; i < (int)HUGEPAGE_PAGES / 2; i++) {
edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false,
&deferred_work_generated); false, &deferred_work_generated);
expect_ptr_not_null(edatas[i], "Unexpected null edata"); expect_ptr_not_null(edatas[i], "Unexpected null edata");
} }
/* /*

View File

@ -128,6 +128,8 @@ TEST_BEGIN(test_hpa_background_thread_purges) {
test_skip_if(!config_stats); test_skip_if(!config_stats);
test_skip_if(!hpa_supported()); test_skip_if(!hpa_supported());
test_skip_if(!have_background_thread); test_skip_if(!have_background_thread);
/* Skip since guarded pages cannot be allocated from hpa. */
test_skip_if(san_enabled());
unsigned arena_ind = create_arena(); unsigned arena_ind = create_arena();
/* /*
@ -142,6 +144,8 @@ TEST_BEGIN(test_hpa_background_thread_enable_disable) {
test_skip_if(!config_stats); test_skip_if(!config_stats);
test_skip_if(!hpa_supported()); test_skip_if(!hpa_supported());
test_skip_if(!have_background_thread); test_skip_if(!have_background_thread);
/* Skip since guarded pages cannot be allocated from hpa. */
test_skip_if(san_enabled());
unsigned arena_ind = create_arena(); unsigned arena_ind = create_arena();

View File

@ -91,7 +91,7 @@ do_alloc_free_purge(void *arg) {
bool deferred_work_generated; bool deferred_work_generated;
edata_t *edata = pa_alloc(TSDN_NULL, &test_data->shard, PAGE, edata_t *edata = pa_alloc(TSDN_NULL, &test_data->shard, PAGE,
PAGE, /* slab */ false, /* szind */ 0, /* zero */ false, PAGE, /* slab */ false, /* szind */ 0, /* zero */ false,
&deferred_work_generated); /* guarded */ false, &deferred_work_generated);
assert_ptr_not_null(edata, ""); assert_ptr_not_null(edata, "");
pa_dalloc(TSDN_NULL, &test_data->shard, edata, pa_dalloc(TSDN_NULL, &test_data->shard, edata,
&deferred_work_generated); &deferred_work_generated);

View File

@ -1,5 +1,6 @@
#include "test/jemalloc_test.h" #include "test/jemalloc_test.h"
#include "jemalloc/internal/guard.h"
#include "jemalloc/internal/spin.h" #include "jemalloc/internal/spin.h"
static unsigned arena_ind; static unsigned arena_ind;
@ -103,7 +104,8 @@ TEST_BEGIN(test_retained) {
arena_ind = do_arena_create(NULL); arena_ind = do_arena_create(NULL);
sz = nallocx(HUGEPAGE, 0); sz = nallocx(HUGEPAGE, 0);
esz = sz + sz_large_pad; size_t guard_sz = san_enabled() ? PAGE_GUARDS_SIZE : 0;
esz = sz + sz_large_pad + guard_sz;
atomic_store_u(&epoch, 0, ATOMIC_RELAXED); atomic_store_u(&epoch, 0, ATOMIC_RELAXED);
@ -133,7 +135,8 @@ TEST_BEGIN(test_retained) {
*/ */
do_refresh(); do_refresh();
size_t allocated = esz * nthreads * PER_THD_NALLOCS; size_t allocated = (esz - guard_sz) * nthreads *
PER_THD_NALLOCS;
size_t active = do_get_active(arena_ind); size_t active = do_get_active(arena_ind);
expect_zu_le(allocated, active, "Unexpected active memory"); expect_zu_le(allocated, active, "Unexpected active memory");
size_t mapped = do_get_mapped(arena_ind); size_t mapped = do_get_mapped(arena_ind);

View File

@ -50,7 +50,9 @@ test_sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t max_alloc,
static inline edata_t * static inline edata_t *
pai_test_allocator_alloc(tsdn_t *tsdn, pai_t *self, size_t size, pai_test_allocator_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool *deferred_work_generated) { size_t alignment, bool zero, bool guarded,
bool *deferred_work_generated) {
assert(!guarded);
pai_test_allocator_t *ta = (pai_test_allocator_t *)self; pai_test_allocator_t *ta = (pai_test_allocator_t *)self;
*deferred_work_generated = false; *deferred_work_generated = false;
if (ta->alloc_fail) { if (ta->alloc_fail) {
@ -182,10 +184,12 @@ TEST_BEGIN(test_reuse) {
/* max_bytes */ 2 * (NALLOCS * PAGE + NALLOCS * 2 * PAGE)); /* max_bytes */ 2 * (NALLOCS * PAGE + NALLOCS * 2 * PAGE));
for (int i = 0; i < NALLOCS; i++) { for (int i = 0; i < NALLOCS; i++) {
one_page[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, one_page[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(one_page[i], "Unexpected alloc failure"); expect_ptr_not_null(one_page[i], "Unexpected alloc failure");
two_page[i] = pai_alloc(tsdn, &sec.pai, 2 * PAGE, PAGE, two_page[i] = pai_alloc(tsdn, &sec.pai, 2 * PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(one_page[i], "Unexpected alloc failure"); expect_ptr_not_null(one_page[i], "Unexpected alloc failure");
} }
expect_zu_eq(0, ta.alloc_count, "Should be using batch allocs"); expect_zu_eq(0, ta.alloc_count, "Should be using batch allocs");
@ -216,9 +220,11 @@ TEST_BEGIN(test_reuse) {
*/ */
for (int i = 0; i < NALLOCS; i++) { for (int i = 0; i < NALLOCS; i++) {
edata_t *alloc1 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, edata_t *alloc1 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
edata_t *alloc2 = pai_alloc(tsdn, &sec.pai, 2 * PAGE, PAGE, edata_t *alloc2 = pai_alloc(tsdn, &sec.pai, 2 * PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_eq(one_page[i], alloc1, expect_ptr_eq(one_page[i], alloc1,
"Got unexpected allocation"); "Got unexpected allocation");
expect_ptr_eq(two_page[i], alloc2, expect_ptr_eq(two_page[i], alloc2,
@ -255,11 +261,12 @@ TEST_BEGIN(test_auto_flush) {
/* max_bytes */ NALLOCS * PAGE); /* max_bytes */ NALLOCS * PAGE);
for (int i = 0; i < NALLOCS; i++) { for (int i = 0; i < NALLOCS; i++) {
allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(allocs[i], "Unexpected alloc failure"); expect_ptr_not_null(allocs[i], "Unexpected alloc failure");
} }
extra_alloc = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false, extra_alloc = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false,
&deferred_work_generated); /* guarded */ false, &deferred_work_generated);
expect_ptr_not_null(extra_alloc, "Unexpected alloc failure"); expect_ptr_not_null(extra_alloc, "Unexpected alloc failure");
size_t max_allocs = ta.alloc_count + ta.alloc_batch_count; size_t max_allocs = ta.alloc_count + ta.alloc_batch_count;
expect_zu_le(NALLOCS + 1, max_allocs, expect_zu_le(NALLOCS + 1, max_allocs,
@ -310,7 +317,8 @@ do_disable_flush_test(bool is_disable) {
/* max_bytes */ NALLOCS * PAGE); /* max_bytes */ NALLOCS * PAGE);
for (int i = 0; i < NALLOCS; i++) { for (int i = 0; i < NALLOCS; i++) {
allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(allocs[i], "Unexpected alloc failure"); expect_ptr_not_null(allocs[i], "Unexpected alloc failure");
} }
/* Free all but the last aloc. */ /* Free all but the last aloc. */
@ -383,7 +391,8 @@ TEST_BEGIN(test_max_alloc_respected) {
expect_zu_eq(i, ta.dalloc_count, expect_zu_eq(i, ta.dalloc_count,
"Incorrect number of deallocations"); "Incorrect number of deallocations");
edata_t *edata = pai_alloc(tsdn, &sec.pai, attempted_alloc, edata_t *edata = pai_alloc(tsdn, &sec.pai, attempted_alloc,
PAGE, /* zero */ false, &deferred_work_generated); PAGE, /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(edata, "Unexpected alloc failure"); expect_ptr_not_null(edata, "Unexpected alloc failure");
expect_zu_eq(i + 1, ta.alloc_count, expect_zu_eq(i + 1, ta.alloc_count,
"Incorrect number of allocations"); "Incorrect number of allocations");
@ -410,7 +419,8 @@ TEST_BEGIN(test_expand_shrink_delegate) {
test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ 10 * PAGE, test_sec_init(&sec, &ta.pai, /* nshards */ 1, /* max_alloc */ 10 * PAGE,
/* max_bytes */ 1000 * PAGE); /* max_bytes */ 1000 * PAGE);
edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_ptr_not_null(edata, "Unexpected alloc failure"); expect_ptr_not_null(edata, "Unexpected alloc failure");
bool err = pai_expand(tsdn, &sec.pai, edata, PAGE, 4 * PAGE, bool err = pai_expand(tsdn, &sec.pai, edata, PAGE, 4 * PAGE,
@ -450,7 +460,8 @@ TEST_BEGIN(test_nshards_0) {
bool deferred_work_generated; bool deferred_work_generated;
edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, edata_t *edata = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
pai_dalloc(tsdn, &sec.pai, edata, &deferred_work_generated); pai_dalloc(tsdn, &sec.pai, edata, &deferred_work_generated);
/* Both operations should have gone directly to the fallback. */ /* Both operations should have gone directly to the fallback. */
@ -492,7 +503,8 @@ TEST_BEGIN(test_stats_simple) {
edata_t *allocs[FLUSH_PAGES]; edata_t *allocs[FLUSH_PAGES];
for (size_t i = 0; i < FLUSH_PAGES; i++) { for (size_t i = 0; i < FLUSH_PAGES; i++) {
allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_stats_pages(tsdn, &sec, 0); expect_stats_pages(tsdn, &sec, 0);
} }
@ -505,7 +517,8 @@ TEST_BEGIN(test_stats_simple) {
} }
for (size_t j = 0; j < FLUSH_PAGES / 2; j++) { for (size_t j = 0; j < FLUSH_PAGES / 2; j++) {
allocs[j] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, allocs[j] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_stats_pages(tsdn, &sec, FLUSH_PAGES / 2 - j - 1); expect_stats_pages(tsdn, &sec, FLUSH_PAGES / 2 - j - 1);
} }
} }
@ -534,13 +547,14 @@ TEST_BEGIN(test_stats_auto_flush) {
bool deferred_work_generated; bool deferred_work_generated;
extra_alloc0 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false, extra_alloc0 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false,
&deferred_work_generated); /* guarded */ false, &deferred_work_generated);
extra_alloc1 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false, extra_alloc1 = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, /* zero */ false,
&deferred_work_generated); /* guarded */ false, &deferred_work_generated);
for (size_t i = 0; i < 2 * FLUSH_PAGES; i++) { for (size_t i = 0; i < 2 * FLUSH_PAGES; i++) {
allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
} }
for (size_t i = 0; i < FLUSH_PAGES; i++) { for (size_t i = 0; i < FLUSH_PAGES; i++) {
@ -580,7 +594,8 @@ TEST_BEGIN(test_stats_manual_flush) {
edata_t *allocs[FLUSH_PAGES]; edata_t *allocs[FLUSH_PAGES];
for (size_t i = 0; i < FLUSH_PAGES; i++) { for (size_t i = 0; i < FLUSH_PAGES; i++) {
allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE, allocs[i] = pai_alloc(tsdn, &sec.pai, PAGE, PAGE,
/* zero */ false, &deferred_work_generated); /* zero */ false, /* guarded */ false,
&deferred_work_generated);
expect_stats_pages(tsdn, &sec, 0); expect_stats_pages(tsdn, &sec, 0);
} }