Implement guard pages.

Adding guarded extents, which are regular extents surrounded by guard pages
(mprotected).  To reduce syscalls, small guarded extents are cached as a
separate eset in ecache, and decay through the dirty / muzzy / retained pipeline
as usual.

include/jemalloc/internal/arena_inlines_b.h

@@ -221,7 +221,8 @@ large_dalloc_safety_checks(edata_t *edata, void *ptr, szind_t szind) {
 	 * The cost is low enough (as edata will be accessed anyway) to be
 	 * enabled all the time.
 	 */
-	if (unlikely(edata_state_get(edata) != extent_state_active)) {
+	if (unlikely(edata == NULL ||
+	    edata_state_get(edata) != extent_state_active)) {
 		safety_check_fail("Invalid deallocation detected: "
 		    "pages being freed (%p) not currently active, "
 		    "possibly caused by double free bugs.",

include/jemalloc/internal/ecache.h

@@ -2,12 +2,14 @@
 #define JEMALLOC_INTERNAL_ECACHE_H
 
 #include "jemalloc/internal/eset.h"
+#include "jemalloc/internal/guard.h"
 #include "jemalloc/internal/mutex.h"
 
 typedef struct ecache_s ecache_t;
 struct ecache_s {
 	malloc_mutex_t mtx;
 	eset_t eset;
+	eset_t guarded_eset;
 	/* All stored extents must be in the same state. */
 	extent_state_t state;
 	/* The index of the ehooks the ecache is associated with. */
@@ -21,17 +23,22 @@ struct ecache_s {
 
 static inline size_t
 ecache_npages_get(ecache_t *ecache) {
-	return eset_npages_get(&ecache->eset);
+	return eset_npages_get(&ecache->eset) +
+	    eset_npages_get(&ecache->guarded_eset);
 }
+
 /* Get the number of extents in the given page size index. */
 static inline size_t
 ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
-	return eset_nextents_get(&ecache->eset, ind);
+	return eset_nextents_get(&ecache->eset, ind) +
+	    eset_nextents_get(&ecache->guarded_eset, ind);
 }
+
 /* Get the sum total bytes of the extents in the given page size index. */
 static inline size_t
 ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
-	return eset_nbytes_get(&ecache->eset, ind);
+	return eset_nbytes_get(&ecache->eset, ind) +
+	    eset_nbytes_get(&ecache->guarded_eset, ind);
 }
 
 static inline unsigned

include/jemalloc/internal/edata.h

@@ -98,12 +98,13 @@ struct edata_s {
 	 * c: committed
 	 * p: pai
 	 * z: zeroed
+	 * g: guarded
 	 * t: state
 	 * i: szind
 	 * f: nfree
 	 * s: bin_shard
 	 *
-	 * 00000000 ... 00000sss sssfffff fffffiii iiiiittt zpcbaaaa aaaaaaaa
+	 * 00000000 ... 0000ssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
 	 *
 	 * arena_ind: Arena from which this extent came, or all 1 bits if
 	 *            unassociated.
@@ -123,6 +124,9 @@ struct edata_s {
 	 * zeroed: The zeroed flag is used by extent recycling code to track
 	 *         whether memory is zero-filled.
 	 *
+	 * guarded: The guarded flag is use by the sanitizer to track whether
+	 *          the extent has page guards around it.
+	 *
 	 * state: The state flag is an extent_state_t.
 	 *
 	 * szind: The szind flag indicates usable size class index for
@@ -158,8 +162,12 @@ struct edata_s {
 #define EDATA_BITS_ZEROED_SHIFT  (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT)
 #define EDATA_BITS_ZEROED_MASK  MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT)
 
+#define EDATA_BITS_GUARDED_WIDTH  1
+#define EDATA_BITS_GUARDED_SHIFT  (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT)
+#define EDATA_BITS_GUARDED_MASK  MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT)
+
 #define EDATA_BITS_STATE_WIDTH  3
-#define EDATA_BITS_STATE_SHIFT  (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT)
+#define EDATA_BITS_STATE_SHIFT  (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT)
 #define EDATA_BITS_STATE_MASK  MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT)
 
 #define EDATA_BITS_SZIND_WIDTH  LG_CEIL(SC_NSIZES)
@@ -293,6 +301,12 @@ edata_state_get(const edata_t *edata) {
 	    EDATA_BITS_STATE_SHIFT);
 }
 
+static inline bool
+edata_guarded_get(const edata_t *edata) {
+	return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK) >>
+	    EDATA_BITS_GUARDED_SHIFT);
+}
+
 static inline bool
 edata_zeroed_get(const edata_t *edata) {
 	return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) >>
@@ -505,6 +519,12 @@ edata_state_set(edata_t *edata, extent_state_t state) {
 	    ((uint64_t)state << EDATA_BITS_STATE_SHIFT);
 }
 
+static inline void
+edata_guarded_set(edata_t *edata, bool guarded) {
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK) |
+	    ((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT);
+}
+
 static inline void
 edata_zeroed_set(edata_t *edata, bool zeroed) {
 	edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) |
@@ -588,6 +608,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
 	edata_szind_set(edata, szind);
 	edata_sn_set(edata, sn);
 	edata_state_set(edata, state);
+	edata_guarded_set(edata, false);
 	edata_zeroed_set(edata, zeroed);
 	edata_committed_set(edata, committed);
 	edata_pai_set(edata, pai);
@@ -606,6 +627,7 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) {
 	edata_szind_set(edata, SC_NSIZES);
 	edata_sn_set(edata, sn);
 	edata_state_set(edata, extent_state_active);
+	edata_guarded_set(edata, false);
 	edata_zeroed_set(edata, true);
 	edata_committed_set(edata, true);
 	/*

include/jemalloc/internal/ehooks.h

@@ -63,6 +63,8 @@ bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a,
     unsigned arena_ind);
 bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b);
 void ehooks_default_zero_impl(void *addr, size_t size);
+void ehooks_default_guard_impl(void *guard1, void *guard2);
+void ehooks_default_unguard_impl(void *guard1, void *guard2);
 
 /*
  * We don't officially support reentrancy from wtihin the extent hooks.  But
@@ -139,6 +141,15 @@ ehooks_merge_will_fail(ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
 }
 
+static inline bool
+ehooks_guard_will_fail(ehooks_t *ehooks) {
+	/*
+	 * Before the guard hooks are officially introduced, limit the use to
+	 * the default hooks only.
+	 */
+	return !ehooks_are_default(ehooks);
+}
+
 /*
  * Some hooks are required to return zeroed memory in certain situations.  In
  * debug mode, we do some heuristic checks that they did what they were supposed
@@ -368,4 +379,34 @@ ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size) {
 	}
 }
 
+static inline bool
+ehooks_guard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) {
+	bool err;
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+
+	if (extent_hooks == &ehooks_default_extent_hooks) {
+		ehooks_default_guard_impl(guard1, guard2);
+		err = false;
+	} else {
+		err = true;
+	}
+
+	return err;
+}
+
+static inline bool
+ehooks_unguard(tsdn_t *tsdn, ehooks_t *ehooks, void *guard1, void *guard2) {
+	bool err;
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+
+	if (extent_hooks == &ehooks_default_extent_hooks) {
+		ehooks_default_unguard_impl(guard1, guard2);
+		err = false;
+	} else {
+		err = true;
+	}
+
+	return err;
+}
+
 #endif /* JEMALLOC_INTERNAL_EHOOKS_H */

include/jemalloc/internal/extent.h

@@ -21,10 +21,10 @@ extern size_t opt_lg_extent_max_active_fit;
 
 edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
-    bool zero);
+    bool zero, bool guarded);
 edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
-    bool zero);
+    bool zero, bool guarded);
 void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, edata_t *edata);
 edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,

include/jemalloc/internal/guard.h

@@ -0,0 +1,76 @@
+#ifndef JEMALLOC_INTERNAL_GUARD_H
+#define JEMALLOC_INTERNAL_GUARD_H
+
+#include "jemalloc/internal/ehooks.h"
+#include "jemalloc/internal/emap.h"
+
+#define PAGE_GUARDS_SIZE (2 * PAGE)
+
+#define SAN_GUARD_LARGE_EVERY_N_EXTENTS_DEFAULT 0
+#define SAN_GUARD_SMALL_EVERY_N_EXTENTS_DEFAULT 0
+
+/* 0 means disabled, i.e. never guarded. */
+extern size_t opt_san_guard_large;
+extern size_t opt_san_guard_small;
+
+void guard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap);
+void unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap);
+void tsd_san_init(tsd_t *tsd);
+
+static inline bool
+san_enabled(void) {
+	return (opt_san_guard_large != 0 || opt_san_guard_small != 0);
+}
+
+static inline bool
+large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size,
+    size_t alignment) {
+	if (opt_san_guard_large == 0 || ehooks_guard_will_fail(ehooks) ||
+	    tsdn_null(tsdn)) {
+		return false;
+	}
+
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	uint64_t n = tsd_san_extents_until_guard_large_get(tsd);
+	assert(n >= 1);
+	if (n > 1) {
+		/*
+		 * Subtract conditionally because the guard may not happen due
+		 * to alignment or size restriction below.
+		 */
+		*tsd_san_extents_until_guard_largep_get(tsd) = n - 1;
+	}
+
+	if (n == 1 && (alignment <= PAGE) &&
+	    (size + PAGE_GUARDS_SIZE <= SC_LARGE_MAXCLASS)) {
+		*tsd_san_extents_until_guard_largep_get(tsd) =
+		    opt_san_guard_large;
+		return true;
+	} else {
+		assert(tsd_san_extents_until_guard_large_get(tsd) >= 1);
+		return false;
+	}
+}
+
+static inline bool
+slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) {
+	if (opt_san_guard_small == 0 || ehooks_guard_will_fail(ehooks) ||
+	    tsdn_null(tsdn)) {
+		return false;
+	}
+
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	uint64_t n = tsd_san_extents_until_guard_small_get(tsd);
+	assert(n >= 1);
+	if (n == 1) {
+		*tsd_san_extents_until_guard_smallp_get(tsd) =
+		    opt_san_guard_small;
+		return true;
+	} else {
+		*tsd_san_extents_until_guard_smallp_get(tsd) = n - 1;
+		assert(tsd_san_extents_until_guard_small_get(tsd) >= 1);
+		return false;
+	}
+}
+
+#endif /* JEMALLOC_INTERNAL_GUARD_H */

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -312,6 +312,9 @@
  */
 #undef JEMALLOC_MADVISE_NOCORE
 
+/* Defined if mprotect(2) is available. */
+#undef JEMALLOC_HAVE_MPROTECT
+
 /*
  * Defined if transparent huge pages (THPs) are supported via the
  * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled.

include/jemalloc/internal/pa.h

@@ -167,7 +167,7 @@ void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
 
 /* Gets an edata for the given allocation. */
 edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
-    size_t alignment, bool slab, szind_t szind, bool zero,
+    size_t alignment, bool slab, szind_t szind, bool zero, bool guarded,
     bool *deferred_work_generated);
 /* Returns true on error, in which case nothing changed. */
 bool pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,

include/jemalloc/internal/pages.h

@@ -110,5 +110,7 @@ bool pages_dontdump(void *addr, size_t size);
 bool pages_dodump(void *addr, size_t size);
 bool pages_boot(void);
 void pages_set_thp_state (void *ptr, size_t size);
+void pages_mark_guards(void *head, void *tail);
+void pages_unmark_guards(void *head, void *tail);
 
 #endif /* JEMALLOC_INTERNAL_PAGES_EXTERNS_H */

include/jemalloc/internal/pai.h

@@ -7,7 +7,8 @@ typedef struct pai_s pai_t;
 struct pai_s {
 	/* Returns NULL on failure. */
 	edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
-	    size_t alignment, bool zero, bool *deferred_work_generated);
+	    size_t alignment, bool zero, bool guarded,
+	    bool *deferred_work_generated);
 	/*
 	 * Returns the number of extents added to the list (which may be fewer
 	 * than requested, in case of OOM).  The list should already be
@@ -37,8 +38,8 @@ struct pai_s {
 
 static inline edata_t *
 pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
-    bool *deferred_work_generated) {
-	return self->alloc(tsdn, self, size, alignment, zero,
+    bool guarded, bool *deferred_work_generated) {
+	return self->alloc(tsdn, self, size, alignment, zero, guarded,
 	    deferred_work_generated);
 }
 

include/jemalloc/internal/tsd.h

@@ -73,6 +73,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
     O(peak_dalloc_event_wait,	uint64_t,	uint64_t)		\
     O(prof_tdata,		prof_tdata_t *,		prof_tdata_t *)	\
     O(prng_state,		uint64_t,		uint64_t)	\
+    O(san_extents_until_guard_small,	uint64_t,	uint64_t)	\
+    O(san_extents_until_guard_large,	uint64_t,	uint64_t)	\
     O(iarena,			arena_t *,		arena_t *)	\
     O(arena,			arena_t *,		arena_t *)	\
     O(arena_decay_ticker,	ticker_geom_t,		ticker_geom_t)	\
@@ -103,6 +105,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
     /* peak_dalloc_event_wait */	0,				\
     /* prof_tdata */		NULL,					\
     /* prng_state */		0,					\
+    /* san_extents_until_guard_small */	0,				\
+    /* san_extents_until_guard_large */	0,				\
     /* iarena */		NULL,					\
     /* arena */			NULL,					\
     /* arena_decay_ticker */						\