Move extent ehook calls into ehooks

include/jemalloc/internal/arena_externs.h

@@ -28,7 +28,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats,
     arena_stats_extents_t *estats);
 void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent);
+    ehooks_t *ehooks, extent_t *extent);
 #ifdef JEMALLOC_JET
 size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
 #endif
@@ -72,7 +72,7 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
     size_t size, size_t alignment, bool zero, tcache_t *tcache,
     hook_ralloc_args_t *hook_args);
 dss_prec_t arena_dss_prec_get(arena_t *arena);
-extent_hooks_t *arena_get_extent_hooks(arena_t *arena);
+ehooks_t *arena_get_ehooks(arena_t *arena);
 extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena,
     extent_hooks_t *extent_hooks);
 bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);

include/jemalloc/internal/arena_inlines_b.h

@@ -14,11 +14,6 @@ arena_get_from_extent(extent_t *extent) {
 	    ATOMIC_RELAXED);
 }
 
-JEMALLOC_ALWAYS_INLINE bool
-arena_has_default_hooks(arena_t *arena) {
-	return (arena_get_extent_hooks(arena) == &extent_hooks_default);
-}
-
 JEMALLOC_ALWAYS_INLINE arena_t *
 arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
 	if (arena != NULL) {
@@ -134,10 +129,10 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
 
 /* Purge a single extent to retained / unmapped directly. */
 JEMALLOC_ALWAYS_INLINE void
-arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t *extent_hooks,
+arena_decay_extent(tsdn_t *tsdn,arena_t *arena, ehooks_t *ehooks,
     extent_t *extent) {
 	size_t extent_size = extent_size_get(extent);
-	extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent);
+	extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
 	if (config_stats) {
 		/* Update stats accordingly. */
 		arena_stats_lock(tsdn, &arena->stats);

include/jemalloc/internal/base_externs.h

@@ -7,7 +7,7 @@ extern const char *metadata_thp_mode_names[];
 base_t *b0get(void);
 base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
 void base_delete(tsdn_t *tsdn, base_t *base);
-extent_hooks_t *base_extent_hooks_get(base_t *base);
+ehooks_t *base_ehooks_get(base_t *base);
 extent_hooks_t *base_extent_hooks_set(base_t *base,
     extent_hooks_t *extent_hooks);
 void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);

include/jemalloc/internal/base_structs.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_BASE_STRUCTS_H
 #define JEMALLOC_INTERNAL_BASE_STRUCTS_H
 
+#include "jemalloc/internal/ehooks.h"
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/mutex.h"
@@ -23,10 +24,9 @@ struct base_s {
 	unsigned ind;
 
 	/*
-	 * User-configurable extent hook functions.  Points to an
-	 * extent_hooks_t.
+	 * User-configurable extent hook functions.
 	 */
-	atomic_p_t extent_hooks;
+	ehooks_t ehooks;
 
 	/* Protects base_alloc() and base_stats_get() operations. */
 	malloc_mutex_t mtx;

include/jemalloc/internal/ehooks.h

@@ -1,4 +1,151 @@
 #ifndef JEMALLOC_INTERNAL_EHOOKS_H
 #define JEMALLOC_INTERNAL_EHOOKS_H
 
+#include "jemalloc/internal/atomic.h"
+
+extern const extent_hooks_t extent_hooks_default;
+
+typedef struct ehooks_s ehooks_t;
+struct ehooks_s {
+	/* Logically an extent_hooks_t *. */
+	atomic_p_t ptr;
+};
+
+void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks);
+
+static inline void
+ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
+	atomic_store_p(&ehooks->ptr, extent_hooks, ATOMIC_RELEASE);
+}
+
+static inline extent_hooks_t *
+ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
+	return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
+}
+
+static inline bool
+ehooks_are_default(ehooks_t *ehooks) {
+	return ehooks_get_extent_hooks_ptr(ehooks) == &extent_hooks_default;
+}
+
+static inline bool
+ehooks_destroy_is_noop(ehooks_t *ehooks) {
+	return ehooks_get_extent_hooks_ptr(ehooks)->destroy == NULL;
+}
+
+static inline bool
+ehooks_purge_lazy_will_fail(ehooks_t *ehooks) {
+	return ehooks_get_extent_hooks_ptr(ehooks)->purge_lazy == NULL;
+}
+
+static inline bool
+ehooks_purge_forced_will_fail(ehooks_t *ehooks) {
+	return ehooks_get_extent_hooks_ptr(ehooks)->purge_forced == NULL;
+}
+
+static inline bool
+ehooks_split_will_fail(ehooks_t *ehooks) {
+	return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
+}
+
+static inline bool
+ehooks_merge_will_fail(ehooks_t *ehooks) {
+	return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
+}
+
+static inline void *
+ehooks_alloc(ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment,
+    bool *zero, bool *commit, unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	return extent_hooks->alloc(extent_hooks, new_addr, size, alignment,
+	    zero, commit, arena_ind);
+}
+
+static inline bool
+ehooks_dalloc(ehooks_t *ehooks, void *addr, size_t size, bool committed,
+    unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->dalloc == NULL) {
+		return true;
+	}
+	return extent_hooks->dalloc(extent_hooks, addr, size, committed,
+	    arena_ind);
+}
+
+static inline void
+ehooks_destroy(ehooks_t *ehooks, void *addr, size_t size, bool committed,
+    unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->destroy == NULL) {
+		return;
+	}
+	extent_hooks->destroy(extent_hooks, addr, size, committed, arena_ind);
+}
+
+static inline bool
+ehooks_commit(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
+    size_t length, unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->commit == NULL) {
+		return true;
+	}
+	return extent_hooks->commit(extent_hooks, addr, size, offset, length,
+	    arena_ind);
+}
+
+static inline bool
+ehooks_decommit(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
+    size_t length, unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->decommit == NULL) {
+		return true;
+	}
+	return extent_hooks->decommit(extent_hooks, addr, size, offset, length,
+	    arena_ind);
+}
+
+static inline bool
+ehooks_purge_lazy(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
+    size_t length, unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->purge_lazy == NULL) {
+		return true;
+	}
+	return extent_hooks->purge_lazy(extent_hooks, addr, size, offset,
+	    length, arena_ind);
+}
+
+static inline bool
+ehooks_purge_forced(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
+    size_t length, unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->purge_forced == NULL) {
+		return true;
+	}
+	return extent_hooks->purge_forced(extent_hooks, addr, size, offset,
+	    length, arena_ind);
+}
+
+static inline bool
+ehooks_split(ehooks_t *ehooks, void *addr, size_t size, size_t size_a,
+    size_t size_b, bool committed, unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->split == NULL) {
+		return true;
+	}
+	return extent_hooks->split(extent_hooks, addr, size, size_a, size_b,
+	    committed, arena_ind);
+}
+
+static inline bool
+ehooks_merge(ehooks_t *ehooks, void *addr_a, size_t size_a, void *addr_b,
+    size_t size_b, bool committed, unsigned arena_ind) {
+	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
+	if (extent_hooks->merge == NULL) {
+		return true;
+	}
+	return extent_hooks->merge(extent_hooks, addr_a, size_a, addr_b, size_b,
+	    committed, arena_ind);
+}
+
 #endif /* JEMALLOC_INTERNAL_EHOOKS_H */

include/jemalloc/internal/extent_externs.h

@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
 #define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
 
+#include "jemalloc/internal/ehooks.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/mutex_pool.h"
 #include "jemalloc/internal/ph.h"
@@ -9,7 +10,6 @@
 extern size_t opt_lg_extent_max_active_fit;
 
 extern rtree_t extents_rtree;
-extern const extent_hooks_t extent_hooks_default;
 extern mutex_pool_t extent_mutex_pool;
 
 extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
@@ -18,39 +18,34 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
 ph_proto(, extent_avail_, extent_tree_t, extent_t)
 ph_proto(, extent_heap_, extent_heap_t, extent_t)
 
-extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, eset_t *eset, void *new_addr,
-    size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
-    bool *zero, bool *commit);
-void extents_dalloc(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, eset_t *eset, extent_t *extent);
-extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, eset_t *eset, size_t npages_min);
-extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad,
-    size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit);
+extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
+    bool slab, szind_t szind, bool *zero, bool *commit);
+void extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    eset_t *eset, extent_t *extent);
+extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    eset_t *eset, size_t npages_min);
+extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
+    szind_t szind, bool *zero, bool *commit);
 void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
-void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent);
-void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent);
-bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
-    size_t length);
-bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
-    size_t length);
-bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
-    size_t length);
-bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
-    size_t length);
-extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
-    szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b);
-bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
-    extent_hooks_t *extent_hooks, extent_t *a, extent_t *b);
+void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *extent);
+void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *extent);
+bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *extent, size_t offset, size_t length);
+bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *extent, size_t offset, size_t length);
+bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *extent, size_t offset, size_t length);
+bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *extent, size_t offset, size_t length);
+extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
+    size_t size_b, szind_t szind_b, bool slab_b);
+bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+    extent_t *a, extent_t *b);
 
 bool extent_boot(void);