Generalize chunk management hooks.

Add the "arena.<i>.chunk_hooks" mallctl, which replaces and expands on the "arena.<i>.chunk.{alloc,dalloc,purge}" mallctls. The chunk hooks allow control over chunk allocation/deallocation, decommit/commit, purging, and splitting/merging, such that the application can rely on jemalloc's internal chunk caching and retaining functionality, yet implement a variety of chunk management mechanisms and policies. Merge the chunks_[sz]ad_{mmap,dss} red-black trees into chunks_[sz]ad_retained. This slightly reduces how hard jemalloc tries to honor the dss precedence setting; prior to this change the precedence setting was also consulted when recycling chunks. Fix chunk purging. Don't purge chunks in arena_purge_stashed(); instead deallocate them in arena_unstash_purged(), so that the dirty memory linkage remains valid until after the last time it is used. This resolves #176 and #201.
2015-07-28 11:28:19 -04:00
parent d059b9d6a1
commit b49a334a64
20 changed files with 1021 additions and 552 deletions
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -379,23 +379,18 @@ struct arena_s {
 	 * orderings are needed, which is why there are two trees with the same
 	 * contents.
 	 */
-	extent_tree_t		chunks_szad_cache;
-	extent_tree_t		chunks_ad_cache;
-	extent_tree_t		chunks_szad_mmap;
-	extent_tree_t		chunks_ad_mmap;
-	extent_tree_t		chunks_szad_dss;
-	extent_tree_t		chunks_ad_dss;
+	extent_tree_t		chunks_szad_cached;
+	extent_tree_t		chunks_ad_cached;
+	extent_tree_t		chunks_szad_retained;
+	extent_tree_t		chunks_ad_retained;
+
 	malloc_mutex_t		chunks_mtx;
 	/* Cache of nodes that were allocated via base_alloc(). */
 	ql_head(extent_node_t)	node_cache;
 	malloc_mutex_t		node_cache_mtx;

-	/*
-	 * User-configurable chunk allocation/deallocation/purge functions.
-	 */
-	chunk_alloc_t		*chunk_alloc;
-	chunk_dalloc_t		*chunk_dalloc;
-	chunk_purge_t		*chunk_purge;
+	/* User-configurable chunk hook functions. */
+	chunk_hooks_t		chunk_hooks;

 	/* bins is used to store trees of free regions. */
 	arena_bin_t		bins[NBINS];
--- a/include/jemalloc/internal/chunk.h
+++ b/include/jemalloc/internal/chunk.h
@@ -19,6 +19,16 @@
 #define	CHUNK_CEILING(s)						\
 	(((s) + chunksize_mask) & ~chunksize_mask)

+#define	CHUNK_HOOKS_INITIALIZER {					\
+    NULL,								\
+    NULL,								\
+    NULL,								\
+    NULL,								\
+    NULL,								\
+    NULL,								\
+    NULL								\
+}
+
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
@@ -36,30 +46,30 @@ extern size_t		chunksize;
 extern size_t		chunksize_mask; /* (chunksize - 1). */
 extern size_t		chunk_npages;

+extern const chunk_hooks_t	chunk_hooks_default;
+
+chunk_hooks_t	chunk_hooks_get(arena_t *arena);
+chunk_hooks_t	chunk_hooks_set(arena_t *arena,
+    const chunk_hooks_t *chunk_hooks);
+
 bool	chunk_register(const void *chunk, const extent_node_t *node);
 void	chunk_deregister(const void *chunk, const extent_node_t *node);
 void	*chunk_alloc_base(size_t size);
-void	*chunk_alloc_cache(arena_t *arena, void *new_addr, size_t size,
-    size_t alignment, bool *zero, bool dalloc_node);
-void	*chunk_alloc_default(void *new_addr, size_t size, size_t alignment,
-    bool *zero, unsigned arena_ind);
-void	*chunk_alloc_wrapper(arena_t *arena, chunk_alloc_t *chunk_alloc,
+void	*chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks,
+    void *new_addr, size_t size, size_t alignment, bool *zero,
+    bool dalloc_node);
+void	*chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *new_addr, size_t size, size_t alignment, bool *zero);
-void	chunk_record(arena_t *arena, extent_tree_t *chunks_szad,
-    extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size,
-    bool zeroed);
-void	chunk_dalloc_cache(arena_t *arena, void *chunk, size_t size);
-void	chunk_dalloc_arena(arena_t *arena, void *chunk, size_t size,
-    bool zeroed);
-bool	chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind);
-void	chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc,
+void	chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks,
+    void *chunk, size_t size);
+void	chunk_dalloc_arena(arena_t *arena, chunk_hooks_t *chunk_hooks,
+    void *chunk, size_t size, bool zeroed);
+void	chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
    void *chunk, size_t size);
 bool	chunk_purge_arena(arena_t *arena, void *chunk, size_t offset,
    size_t length);
-bool	chunk_purge_default(void *chunk, size_t offset, size_t length,
-    unsigned arena_ind);
-bool	chunk_purge_wrapper(arena_t *arena, chunk_purge_t *chunk_purge,
-    void *chunk, size_t offset, size_t length);
+bool	chunk_purge_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks,
+    void *chunk, size_t size, size_t offset, size_t length);
 bool	chunk_boot(void);
 void	chunk_prefork(void);
 void	chunk_postfork_parent(void);
--- a/include/jemalloc/internal/chunk_mmap.h
+++ b/include/jemalloc/internal/chunk_mmap.h
@@ -9,8 +9,6 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS

-bool	pages_purge(void *addr, size_t length);
-
 void	*chunk_alloc_mmap(size_t size, size_t alignment, bool *zero);
 bool	chunk_dalloc_mmap(void *chunk, size_t size);

--- a/include/jemalloc/internal/extent.h
+++ b/include/jemalloc/internal/extent.h
@@ -18,6 +18,13 @@ struct extent_node_s {
 	/* Total region size. */
 	size_t			en_size;

+	/*
+	 * True if physical memory is committed to the extent, whether
+	 * explicitly or implicitly as on a system that overcommits and
+	 * satisfies physical mamory needs on demand via soft page faults.
+	 */
+	bool			en_committed;
+
 	/*
 	 * The zeroed flag is used by chunk recycling code to track whether
 	 * memory is zero-filled.
@@ -66,17 +73,19 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
 arena_t	*extent_node_arena_get(const extent_node_t *node);
 void	*extent_node_addr_get(const extent_node_t *node);
 size_t	extent_node_size_get(const extent_node_t *node);
+bool	extent_node_committed_get(const extent_node_t *node);
 bool	extent_node_zeroed_get(const extent_node_t *node);
 bool	extent_node_achunk_get(const extent_node_t *node);
 prof_tctx_t	*extent_node_prof_tctx_get(const extent_node_t *node);
 void	extent_node_arena_set(extent_node_t *node, arena_t *arena);
 void	extent_node_addr_set(extent_node_t *node, void *addr);
 void	extent_node_size_set(extent_node_t *node, size_t size);
+void	extent_node_committed_set(extent_node_t *node, bool committed);
 void	extent_node_zeroed_set(extent_node_t *node, bool zeroed);
 void	extent_node_achunk_set(extent_node_t *node, bool achunk);
 void	extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx);
 void	extent_node_init(extent_node_t *node, arena_t *arena, void *addr,
-    size_t size, bool zeroed);
+    size_t size, bool committed, bool zeroed);
 void	extent_node_dirty_linkage_init(extent_node_t *node);
 void	extent_node_dirty_insert(extent_node_t *node,
    arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty);
@@ -105,6 +114,13 @@ extent_node_size_get(const extent_node_t *node)
 	return (node->en_size);
 }

+JEMALLOC_INLINE bool
+extent_node_committed_get(const extent_node_t *node)
+{
+
+	return (node->en_committed);
+}
+
 JEMALLOC_INLINE bool
 extent_node_zeroed_get(const extent_node_t *node)
 {
@@ -147,6 +163,13 @@ extent_node_size_set(extent_node_t *node, size_t size)
 	node->en_size = size;
 }

+JEMALLOC_INLINE void
+extent_node_committed_set(extent_node_t *node, bool committed)
+{
+
+	node->en_committed = committed;
+}
+
 JEMALLOC_INLINE void
 extent_node_zeroed_set(extent_node_t *node, bool zeroed)
 {
@@ -170,12 +193,13 @@ extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx)

 JEMALLOC_INLINE void
 extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size,
-    bool zeroed)
+    bool committed, bool zeroed)
 {

 	extent_node_arena_set(node, arena);
 	extent_node_addr_set(node, addr);
 	extent_node_size_set(node, size);
+	extent_node_committed_set(node, committed);
 	extent_node_zeroed_set(node, zeroed);
 	extent_node_achunk_set(node, false);
 	if (config_prof)
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -367,6 +367,7 @@ typedef unsigned index_t;
 #include "jemalloc/internal/bitmap.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/chunk.h"
 #include "jemalloc/internal/huge.h"
 #include "jemalloc/internal/tcache.h"
@@ -398,6 +399,7 @@ typedef unsigned index_t;
 #undef JEMALLOC_ARENA_STRUCTS_B
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/chunk.h"
 #include "jemalloc/internal/huge.h"
 #include "jemalloc/internal/tcache.h"
@@ -477,6 +479,7 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/chunk.h"
 #include "jemalloc/internal/huge.h"
 #include "jemalloc/internal/tcache.h"
@@ -503,6 +506,7 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/chunk.h"
 #include "jemalloc/internal/huge.h"

--- a/include/jemalloc/internal/pages.h
+++ b/include/jemalloc/internal/pages.h
@@ -0,0 +1,26 @@
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+void	*pages_map(void *addr, size_t size);
+void	pages_unmap(void *addr, size_t size);
+void	*pages_trim(void *addr, size_t alloc_size, size_t leadsize,
+    size_t size);
+bool	pages_commit(void *addr, size_t size);
+bool	pages_decommit(void *addr, size_t size);
+bool	pages_purge(void *addr, size_t size);
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/
+
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -132,14 +132,12 @@ bt_init
 buferror
 chunk_alloc_cache
 chunk_alloc_base
-chunk_alloc_default
 chunk_alloc_dss
 chunk_alloc_mmap
 chunk_alloc_wrapper
 chunk_boot
 chunk_dalloc_arena
 chunk_dalloc_cache
-chunk_dalloc_default
 chunk_dalloc_mmap
 chunk_dalloc_wrapper
 chunk_deregister
@@ -149,6 +147,9 @@ chunk_dss_postfork_parent
 chunk_dss_prec_get
 chunk_dss_prec_set
 chunk_dss_prefork
+chunk_hooks_default
+chunk_hooks_get
+chunk_hooks_set
 chunk_in_dss
 chunk_lookup
 chunk_npages
@@ -156,9 +157,7 @@ chunk_postfork_child
 chunk_postfork_parent
 chunk_prefork
 chunk_purge_arena
-chunk_purge_default
 chunk_purge_wrapper
-chunk_record
 chunk_register
 chunks_rtree
 chunksize
@@ -347,7 +346,12 @@ opt_utrace
 opt_xmalloc
 opt_zero
 p2rz
+pages_commit
+pages_decommit
+pages_map
 pages_purge
+pages_trim
+pages_unmap
 pow2_ceil
 prof_active_get
 prof_active_get_unlocked