Implement chunk hook support for page run commit/decommit.

Cascade from decommit to purge when purging unused dirty pages, so that it is possible to decommit cleaned memory rather than just purging. For non-Windows debug builds, decommit runs rather than purging them, since this causes access of deallocated runs to segfault. This resolves #251.
2015-08-04 10:49:46 -07:00
parent 5716d97f75
commit 8fadb1a8c2
15 changed files with 561 additions and 283 deletions
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -54,15 +54,16 @@ struct arena_chunk_map_bits_s {
 	 * Run address (or size) and various flags are stored together.  The bit
 	 * layout looks like (assuming 32-bit system):
 	 *
-	 *   ???????? ???????? ????nnnn nnnndula
+	 *   ???????? ???????? ???nnnnn nnndulma
 	 *
 	 * ? : Unallocated: Run address for first/last pages, unset for internal
 	 *                  pages.
 	 *     Small: Run page offset.
-	 *     Large: Run size for first page, unset for trailing pages.
+	 *     Large: Run page count for first page, unset for trailing pages.
 	 * n : binind for small size class, BININD_INVALID for large size class.
 	 * d : dirty?
 	 * u : unzeroed?
+	 * m : decommitted?
 	 * l : large?
 	 * a : allocated?
 	 *
@@ -74,51 +75,58 @@ struct arena_chunk_map_bits_s {
 	 * x : don't care
 	 * - : 0
 	 * + : 1
-	 * [DULA] : bit set
-	 * [dula] : bit unset
+	 * [DUMLA] : bit set
+	 * [dumla] : bit unset
 	 *
 	 *   Unallocated (clean):
-	 *     ssssssss ssssssss ssss++++ ++++du-a
-	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx
-	 *     ssssssss ssssssss ssss++++ ++++dU-a
+	 *     ssssssss ssssssss sss+++++ +++dum-a
+	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx
+	 *     ssssssss ssssssss sss+++++ +++dUm-a
 	 *
 	 *   Unallocated (dirty):
-	 *     ssssssss ssssssss ssss++++ ++++D--a
+	 *     ssssssss ssssssss sss+++++ +++D-m-a
 	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     ssssssss ssssssss ssss++++ ++++D--a
+	 *     ssssssss ssssssss sss+++++ +++D-m-a
 	 *
 	 *   Small:
-	 *     pppppppp pppppppp ppppnnnn nnnnd--A
-	 *     pppppppp pppppppp ppppnnnn nnnn---A
-	 *     pppppppp pppppppp ppppnnnn nnnnd--A
+	 *     pppppppp pppppppp pppnnnnn nnnd---A
+	 *     pppppppp pppppppp pppnnnnn nnn----A
+	 *     pppppppp pppppppp pppnnnnn nnnd---A
 	 *
 	 *   Large:
-	 *     ssssssss ssssssss ssss++++ ++++D-LA
+	 *     ssssssss ssssssss sss+++++ +++D--LA
 	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     -------- -------- ----++++ ++++D-LA
+	 *     -------- -------- ---+++++ +++D--LA
 	 *
 	 *   Large (sampled, size <= LARGE_MINCLASS):
-	 *     ssssssss ssssssss ssssnnnn nnnnD-LA
+	 *     ssssssss ssssssss sssnnnnn nnnD--LA
 	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     -------- -------- ----++++ ++++D-LA
+	 *     -------- -------- ---+++++ +++D--LA
 	 *
 	 *   Large (not sampled, size == LARGE_MINCLASS):
-	 *     ssssssss ssssssss ssss++++ ++++D-LA
+	 *     ssssssss ssssssss sss+++++ +++D--LA
 	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     -------- -------- ----++++ ++++D-LA
+	 *     -------- -------- ---+++++ +++D--LA
 	 */
 	size_t				bits;
-#define	CHUNK_MAP_BININD_SHIFT	4
+#define	CHUNK_MAP_ALLOCATED	((size_t)0x01U)
+#define	CHUNK_MAP_LARGE		((size_t)0x02U)
+#define	CHUNK_MAP_STATE_MASK	((size_t)0x3U)
+
+#define	CHUNK_MAP_DECOMMITTED	((size_t)0x04U)
+#define	CHUNK_MAP_UNZEROED	((size_t)0x08U)
+#define	CHUNK_MAP_DIRTY		((size_t)0x10U)
+#define	CHUNK_MAP_FLAGS_MASK	((size_t)0x1cU)
+
+#define	CHUNK_MAP_BININD_SHIFT	5
 #define	BININD_INVALID		((size_t)0xffU)
-/*     CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */
-#define	CHUNK_MAP_BININD_MASK	((size_t)0xff0U)
+#define	CHUNK_MAP_BININD_MASK	(BININD_INVALID << CHUNK_MAP_BININD_SHIFT)
 #define	CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK
-#define	CHUNK_MAP_FLAGS_MASK	((size_t)0xcU)
-#define	CHUNK_MAP_DIRTY		((size_t)0x8U)
-#define	CHUNK_MAP_UNZEROED	((size_t)0x4U)
-#define	CHUNK_MAP_LARGE		((size_t)0x2U)
-#define	CHUNK_MAP_ALLOCATED	((size_t)0x1U)
-#define	CHUNK_MAP_KEY		CHUNK_MAP_ALLOCATED
+
+#define	CHUNK_MAP_RUNIND_SHIFT	(CHUNK_MAP_BININD_SHIFT + 8)
+#define	CHUNK_MAP_SIZE_SHIFT	(CHUNK_MAP_RUNIND_SHIFT - LG_PAGE)
+#define	CHUNK_MAP_SIZE_MASK						\
+    (~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK))
 };

 struct arena_runs_dirty_link_s {
@@ -518,6 +526,7 @@ size_t	arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind);
 index_t	arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind);
+size_t	arena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind);
 void	arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits);
@@ -650,7 +659,7 @@ arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind)

 	mapbits = arena_mapbits_get(chunk, pageind);
 	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
-	return (mapbits & ~PAGE_MASK);
+	return ((mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT);
 }

 JEMALLOC_ALWAYS_INLINE size_t
@@ -661,7 +670,7 @@ arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind)
 	mapbits = arena_mapbits_get(chunk, pageind);
 	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
 	    (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED));
-	return (mapbits & ~PAGE_MASK);
+	return ((mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT);
 }

 JEMALLOC_ALWAYS_INLINE size_t
@@ -672,7 +681,7 @@ arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind)
 	mapbits = arena_mapbits_get(chunk, pageind);
 	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
 	    CHUNK_MAP_ALLOCATED);
-	return (mapbits >> LG_PAGE);
+	return (mapbits >> CHUNK_MAP_RUNIND_SHIFT);
 }

 JEMALLOC_ALWAYS_INLINE index_t
@@ -693,6 +702,8 @@ arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind)
 	size_t mapbits;

 	mapbits = arena_mapbits_get(chunk, pageind);
+	assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
+	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
 	return (mapbits & CHUNK_MAP_DIRTY);
 }

@@ -702,9 +713,22 @@ arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind)
 	size_t mapbits;

 	mapbits = arena_mapbits_get(chunk, pageind);
+	assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
+	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
 	return (mapbits & CHUNK_MAP_UNZEROED);
 }

+JEMALLOC_ALWAYS_INLINE size_t
+arena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind)
+{
+	size_t mapbits;
+
+	mapbits = arena_mapbits_get(chunk, pageind);
+	assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
+	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
+	return (mapbits & CHUNK_MAP_DECOMMITTED);
+}
+
 JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind)
 {
@@ -736,10 +760,13 @@ arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
 {
 	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);

-	assert(size == PAGE_CEILING(size));
-	assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0);
-	assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags);
-	arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags);
+	assert((size & PAGE_MASK) == 0);
+	assert(((size << CHUNK_MAP_SIZE_SHIFT) & ~CHUNK_MAP_SIZE_MASK) == 0);
+	assert((flags & CHUNK_MAP_FLAGS_MASK) == flags);
+	assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags &
+	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
+	arena_mapbitsp_write(mapbitsp, (size << CHUNK_MAP_SIZE_SHIFT) |
+	    CHUNK_MAP_BININD_INVALID | flags);
 }

 JEMALLOC_ALWAYS_INLINE void
@@ -749,9 +776,11 @@ arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
 	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);
 	size_t mapbits = arena_mapbitsp_read(mapbitsp);

-	assert(size == PAGE_CEILING(size));
+	assert((size & PAGE_MASK) == 0);
+	assert(((size << CHUNK_MAP_SIZE_SHIFT) & ~CHUNK_MAP_SIZE_MASK) == 0);
 	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
-	arena_mapbitsp_write(mapbitsp, size | (mapbits & PAGE_MASK));
+	arena_mapbitsp_write(mapbitsp, (size << CHUNK_MAP_SIZE_SHIFT) | (mapbits
+	    & ~CHUNK_MAP_SIZE_MASK));
 }

 JEMALLOC_ALWAYS_INLINE void
@@ -762,11 +791,13 @@ arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
 	size_t mapbits = arena_mapbitsp_read(mapbitsp);
 	size_t unzeroed;

-	assert(size == PAGE_CEILING(size));
-	assert((flags & CHUNK_MAP_DIRTY) == flags);
+	assert((size & PAGE_MASK) == 0);
+	assert(((size << CHUNK_MAP_SIZE_SHIFT) & ~CHUNK_MAP_SIZE_MASK) == 0);
+	assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_DECOMMITTED)) == flags);
 	unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
-	arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags
-	    | unzeroed | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED);
+	arena_mapbitsp_write(mapbitsp, (size << CHUNK_MAP_SIZE_SHIFT) |
+	    CHUNK_MAP_BININD_INVALID | flags | unzeroed | CHUNK_MAP_LARGE |
+	    CHUNK_MAP_ALLOCATED);
 }

 JEMALLOC_ALWAYS_INLINE void
@@ -795,8 +826,9 @@ arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
 	assert(pageind - runind >= map_bias);
 	assert((flags & CHUNK_MAP_DIRTY) == flags);
 	unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
-	arena_mapbitsp_write(mapbitsp, (runind << LG_PAGE) | (binind <<
-	    CHUNK_MAP_BININD_SHIFT) | flags | unzeroed | CHUNK_MAP_ALLOCATED);
+	arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) |
+	    (binind << CHUNK_MAP_BININD_SHIFT) | flags | unzeroed |
+	    CHUNK_MAP_ALLOCATED);
 }

 JEMALLOC_ALWAYS_INLINE void
--- a/include/jemalloc/internal/chunk.h
+++ b/include/jemalloc/internal/chunk.h
@@ -59,13 +59,13 @@ void	*chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *new_addr, size_t size, size_t alignment, bool *zero,
    bool dalloc_node);
 void	*chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *new_addr, size_t size, size_t alignment, bool *zero);
+    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit);
 void	chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *chunk, size_t size);
 void	chunk_dalloc_arena(arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *chunk, size_t size, bool zeroed);
+    void *chunk, size_t size, bool zeroed, bool committed);
 void	chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *chunk, size_t size);
+    void *chunk, size_t size, bool committed);
 bool	chunk_purge_arena(arena_t *arena, void *chunk, size_t offset,
    size_t length);
 bool	chunk_purge_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
--- a/include/jemalloc/internal/chunk_dss.h
+++ b/include/jemalloc/internal/chunk_dss.h
@@ -24,7 +24,7 @@ extern const char *dss_prec_names[];
 dss_prec_t	chunk_dss_prec_get(void);
 bool	chunk_dss_prec_set(dss_prec_t dss_prec);
 void	*chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size,
-    size_t alignment, bool *zero);
+    size_t alignment, bool *zero, bool *commit);
 bool	chunk_in_dss(void *chunk);
 bool	chunk_dss_boot(void);
 void	chunk_dss_prefork(void);
--- a/include/jemalloc/internal/chunk_mmap.h
+++ b/include/jemalloc/internal/chunk_mmap.h
@@ -9,7 +9,8 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS

-void	*chunk_alloc_mmap(size_t size, size_t alignment, bool *zero);
+void	*chunk_alloc_mmap(size_t size, size_t alignment, bool *zero,
+    bool *commit);
 bool	chunk_dalloc_mmap(void *chunk, size_t size);

 #endif /* JEMALLOC_H_EXTERNS */
--- a/include/jemalloc/internal/extent.h
+++ b/include/jemalloc/internal/extent.h
@@ -18,6 +18,12 @@ struct extent_node_s {
 	/* Total region size. */
 	size_t			en_size;

+	/*
+	 * The zeroed flag is used by chunk recycling code to track whether
+	 * memory is zero-filled.
+	 */
+	bool			en_zeroed;
+
 	/*
 	 * True if physical memory is committed to the extent, whether
 	 * explicitly or implicitly as on a system that overcommits and
@@ -25,12 +31,6 @@ struct extent_node_s {
 	 */
 	bool			en_committed;

-	/*
-	 * The zeroed flag is used by chunk recycling code to track whether
-	 * memory is zero-filled.
-	 */
-	bool			en_zeroed;
-
 	/*
 	 * The achunk flag is used to validate that huge allocation lookups
 	 * don't return arena chunks.
@@ -73,19 +73,19 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
 arena_t	*extent_node_arena_get(const extent_node_t *node);
 void	*extent_node_addr_get(const extent_node_t *node);
 size_t	extent_node_size_get(const extent_node_t *node);
-bool	extent_node_committed_get(const extent_node_t *node);
 bool	extent_node_zeroed_get(const extent_node_t *node);
+bool	extent_node_committed_get(const extent_node_t *node);
 bool	extent_node_achunk_get(const extent_node_t *node);
 prof_tctx_t	*extent_node_prof_tctx_get(const extent_node_t *node);
 void	extent_node_arena_set(extent_node_t *node, arena_t *arena);
 void	extent_node_addr_set(extent_node_t *node, void *addr);
 void	extent_node_size_set(extent_node_t *node, size_t size);
-void	extent_node_committed_set(extent_node_t *node, bool committed);
 void	extent_node_zeroed_set(extent_node_t *node, bool zeroed);
+void	extent_node_committed_set(extent_node_t *node, bool committed);
 void	extent_node_achunk_set(extent_node_t *node, bool achunk);
 void	extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx);
 void	extent_node_init(extent_node_t *node, arena_t *arena, void *addr,
-    size_t size, bool committed, bool zeroed);
+    size_t size, bool zeroed, bool committed);
 void	extent_node_dirty_linkage_init(extent_node_t *node);
 void	extent_node_dirty_insert(extent_node_t *node,
    arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty);
@@ -114,13 +114,6 @@ extent_node_size_get(const extent_node_t *node)
 	return (node->en_size);
 }

-JEMALLOC_INLINE bool
-extent_node_committed_get(const extent_node_t *node)
-{
-
-	return (node->en_committed);
-}
-
 JEMALLOC_INLINE bool
 extent_node_zeroed_get(const extent_node_t *node)
 {
@@ -128,6 +121,13 @@ extent_node_zeroed_get(const extent_node_t *node)
 	return (node->en_zeroed);
 }

+JEMALLOC_INLINE bool
+extent_node_committed_get(const extent_node_t *node)
+{
+
+	return (node->en_committed);
+}
+
 JEMALLOC_INLINE bool
 extent_node_achunk_get(const extent_node_t *node)
 {
@@ -163,13 +163,6 @@ extent_node_size_set(extent_node_t *node, size_t size)
 	node->en_size = size;
 }

-JEMALLOC_INLINE void
-extent_node_committed_set(extent_node_t *node, bool committed)
-{
-
-	node->en_committed = committed;
-}
-
 JEMALLOC_INLINE void
 extent_node_zeroed_set(extent_node_t *node, bool zeroed)
 {
@@ -177,6 +170,13 @@ extent_node_zeroed_set(extent_node_t *node, bool zeroed)
 	node->en_zeroed = zeroed;
 }

+JEMALLOC_INLINE void
+extent_node_committed_set(extent_node_t *node, bool committed)
+{
+
+	node->en_committed = committed;
+}
+
 JEMALLOC_INLINE void
 extent_node_achunk_set(extent_node_t *node, bool achunk)
 {
@@ -193,14 +193,14 @@ extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx)

 JEMALLOC_INLINE void
 extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size,
-    bool committed, bool zeroed)
+    bool zeroed, bool committed)
 {

 	extent_node_arena_set(node, arena);
 	extent_node_addr_set(node, addr);
 	extent_node_size_set(node, size);
-	extent_node_committed_set(node, committed);
 	extent_node_zeroed_set(node, zeroed);
+	extent_node_committed_set(node, committed);
 	extent_node_achunk_set(node, false);
 	if (config_prof)
 		extent_node_prof_tctx_set(node, NULL);
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -39,6 +39,7 @@ arena_malloc_large
 arena_malloc_small
 arena_mapbits_allocated_get
 arena_mapbits_binind_get
+arena_mapbits_decommitted_get
 arena_mapbits_dirty_get
 arena_mapbits_get
 arena_mapbits_large_binind_set
--- a/include/jemalloc/jemalloc_typedefs.h.in
+++ b/include/jemalloc/jemalloc_typedefs.h.in
@@ -1,27 +1,29 @@
 /*
 * void *
 * chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
+ *     bool *commit, unsigned arena_ind);
+ */
+typedef void *(chunk_alloc_t)(void *, size_t, size_t, bool *, bool *, unsigned);
+
+/*
+ * bool
+ * chunk_dalloc(void *chunk, size_t size, bool committed, unsigned arena_ind);
+ */
+typedef bool (chunk_dalloc_t)(void *, size_t, bool, unsigned);
+
+/*
+ * bool
+ * chunk_commit(void *chunk, size_t size, size_t offset, size_t length,
 *     unsigned arena_ind);
 */
-typedef void *(chunk_alloc_t)(void *, size_t, size_t, bool *, unsigned);
+typedef bool (chunk_commit_t)(void *, size_t, size_t, size_t, unsigned);

 /*
 * bool
- * chunk_dalloc(void *chunk, size_t size, unsigned arena_ind);
+ * chunk_decommit(void *chunk, size_t size, size_t offset, size_t length,
+ *     unsigned arena_ind);
 */
-typedef bool (chunk_dalloc_t)(void *, size_t, unsigned);
-
-/*
- * bool
- * chunk_commit(void *chunk, size_t size, unsigned arena_ind);
- */
-typedef bool (chunk_commit_t)(void *, size_t, unsigned);
-
-/*
- * bool
- * chunk_decommit(void *chunk, size_t size, unsigned arena_ind);
- */
-typedef bool (chunk_decommit_t)(void *, size_t, unsigned);
+typedef bool (chunk_decommit_t)(void *, size_t, size_t, size_t, unsigned);

 /*
 * bool