project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Modify extent hook functions to take an (extent_t *) argument.

Browse Source 
This facilitates the application accessing its own extent allocator
metadata during hook invocations.

This resolves #259.
This commit is contained in:
Jason Evans 2016-06-03 12:05:53 -07:00
parent 6f29a83924
commit f8f0542194
10 changed files with 312 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
doc/jemalloc.xml.in
View File

@ -1551,7 +1551,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<varlistentry id="arena.i.extent_hooks"> <varlistentry id="arena.i.extent_hooks">
<term> <term>
<mallctl>arena.&lt;i&gt;.extent_hooks</mallctl> <mallctl>arena.&lt;i&gt;.extent_hooks</mallctl>
(<type>extent_hooks_t</type>) (<type>extent_hooks_t *</type>)
<literal>rw</literal> <literal>rw</literal>
</term> </term>
<listitem><para>Get or set the extent management hook functions for <listitem><para>Get or set the extent management hook functions for
@ -1567,7 +1567,8 @@ malloc_conf = "xmalloc:true";]]></programlisting>
allocation.</para> allocation.</para>
<programlisting language="C"><![CDATA[ <programlisting language="C"><![CDATA[
typedef struct { typedef extent_hooks_s extent_hooks_t;
struct extent_hooks_s {
extent_alloc_t *alloc; extent_alloc_t *alloc;
extent_dalloc_t *dalloc; extent_dalloc_t *dalloc;
extent_commit_t *commit; extent_commit_t *commit;
@ -1575,7 +1576,7 @@ typedef struct {
extent_purge_t *purge; extent_purge_t *purge;
extent_split_t *split; extent_split_t *split;
extent_merge_t *merge; extent_merge_t *merge;
} extent_hooks_t;]]></programlisting> };]]></programlisting>
<para>The <type>extent_hooks_t</type> structure comprises function <para>The <type>extent_hooks_t</type> structure comprises function
pointers which are described individually below. jemalloc uses these pointers which are described individually below. jemalloc uses these
functions to manage extent lifetime, which starts off with allocation of functions to manage extent lifetime, which starts off with allocation of
@ -1592,6 +1593,7 @@ typedef struct {
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef void *<function>(extent_alloc_t)</function></funcdef> <funcdef>typedef void *<function>(extent_alloc_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>new_addr</parameter></paramdef> <paramdef>void *<parameter>new_addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef> <paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>alignment</parameter></paramdef> <paramdef>size_t <parameter>alignment</parameter></paramdef>
@ -1627,6 +1629,7 @@ typedef struct {
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_dalloc_t)</function></funcdef> <funcdef>typedef bool <function>(extent_dalloc_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef> <paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef> <paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>bool <parameter>committed</parameter></paramdef> <paramdef>bool <parameter>committed</parameter></paramdef>
@ -1646,6 +1649,7 @@ typedef struct {
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_commit_t)</function></funcdef> <funcdef>typedef bool <function>(extent_commit_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef> <paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef> <paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>offset</parameter></paramdef> <paramdef>size_t <parameter>offset</parameter></paramdef>
@ -1667,6 +1671,7 @@ typedef struct {
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_decommit_t)</function></funcdef> <funcdef>typedef bool <function>(extent_decommit_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef> <paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef> <paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>offset</parameter></paramdef> <paramdef>size_t <parameter>offset</parameter></paramdef>
@ -1688,6 +1693,7 @@ typedef struct {
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_purge_t)</function></funcdef> <funcdef>typedef bool <function>(extent_purge_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef> <paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t<parameter>size</parameter></paramdef> <paramdef>size_t<parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>offset</parameter></paramdef> <paramdef>size_t <parameter>offset</parameter></paramdef>
@ -1707,6 +1713,7 @@ typedef struct {
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_split_t)</function></funcdef> <funcdef>typedef bool <function>(extent_split_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef> <paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef> <paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>size_a</parameter></paramdef> <paramdef>size_t <parameter>size_a</parameter></paramdef>
@ -1728,6 +1735,7 @@ typedef struct {
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_merge_t)</function></funcdef> <funcdef>typedef bool <function>(extent_merge_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr_a</parameter></paramdef> <paramdef>void *<parameter>addr_a</parameter></paramdef>
<paramdef>size_t <parameter>size_a</parameter></paramdef> <paramdef>size_t <parameter>size_a</parameter></paramdef>
<paramdef>void *<parameter>addr_b</parameter></paramdef> <paramdef>void *<parameter>addr_b</parameter></paramdef>

16
include/jemalloc/internal/arena.h
View File

@ -240,11 +240,15 @@ struct arena_s {
*/ */
extent_heap_t extents_cached[NPSIZES]; extent_heap_t extents_cached[NPSIZES];
extent_heap_t extents_retained[NPSIZES]; extent_heap_t extents_retained[NPSIZES];
/* User-configurable extent hook functions. */ /* Protects extents_cached and extents_retained. */
extent_hooks_t extent_hooks;
/* Protects extents_cached, extents_retained, and extent_hooks. */
malloc_mutex_t extents_mtx; malloc_mutex_t extents_mtx;
/* User-configurable extent hook functions. */
union {
extent_hooks_t *extent_hooks;
void *extent_hooks_pun;
};
/* Cache of extent structures that were allocated via base_alloc(). */ /* Cache of extent structures that were allocated via base_alloc(). */
ql_head(extent_t) extent_cache; ql_head(extent_t) extent_cache;
malloc_mutex_t extent_cache_mtx; malloc_mutex_t extent_cache_mtx;
@ -279,10 +283,10 @@ extern ssize_t opt_decay_time;
extern const arena_bin_info_t arena_bin_info[NBINS]; extern const arena_bin_info_t arena_bin_info[NBINS];
extent_t *arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena, extent_t *arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, extent_hooks_t **r_extent_hooks, void *new_addr, size_t size,
bool *zero); size_t alignment, bool *zero);
void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent); extent_hooks_t **r_extent_hooks, extent_t *extent);
void arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent, void arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent,
bool cache); bool cache);
void arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent, void arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent,

33
include/jemalloc/internal/extent.h
View File

@ -3,15 +3,7 @@
typedef struct extent_s extent_t; typedef struct extent_s extent_t;
#define EXTENT_HOOKS_INITIALIZER { \ #define EXTENT_HOOKS_INITIALIZER NULL
NULL, \
NULL, \
NULL, \
NULL, \
NULL, \
NULL, \
NULL \
}
#endif /* JEMALLOC_H_TYPES */ #endif /* JEMALLOC_H_TYPES */
/******************************************************************************/ /******************************************************************************/
@ -93,9 +85,8 @@ extern const extent_hooks_t extent_hooks_default;
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent); void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
extent_hooks_t extent_hooks_get(tsdn_t *tsdn, arena_t *arena); extent_hooks_t *extent_hooks_get(arena_t *arena);
extent_hooks_t extent_hooks_set(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks);
const extent_hooks_t *extent_hooks);
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
typedef size_t (extent_size_quantize_t)(size_t); typedef size_t (extent_size_quantize_t)(size_t);
@ -109,29 +100,29 @@ size_t extent_size_quantize_ceil(size_t size);
ph_proto(, extent_heap_, extent_heap_t, extent_t) ph_proto(, extent_heap_, extent_heap_t, extent_t)
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab); size_t alignment, bool *zero, bool slab);
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab); size_t alignment, bool *zero, bool *commit, bool slab);
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent); extent_hooks_t **r_extent_hooks, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent); extent_hooks_t **r_extent_hooks, extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length); size_t length);
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length); size_t length);
bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length); size_t length);
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
size_t usize_a, size_t size_b, size_t usize_b); size_t usize_a, size_t size_b, size_t usize_b);
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b); extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b);
void extent_prefork(tsdn_t *tsdn); void extent_prefork(tsdn_t *tsdn);
void extent_postfork_parent(tsdn_t *tsdn); void extent_postfork_parent(tsdn_t *tsdn);
void extent_postfork_child(tsdn_t *tsdn); void extent_postfork_child(tsdn_t *tsdn);

28
include/jemalloc/jemalloc_typedefs.h.in
View File

@ -1,53 +1,61 @@
typedef struct extent_hooks_s extent_hooks_t;
/* /*
* void * * void *
* extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, * extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
* bool *commit, unsigned arena_ind); * bool *commit, unsigned arena_ind);
*/ */
typedef void *(extent_alloc_t)(void *, size_t, size_t, bool *, bool *, typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *,
unsigned); bool *, unsigned);
/* /*
* bool * bool
* extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind); * extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind);
*/ */
typedef bool (extent_dalloc_t)(void *, size_t, bool, unsigned); typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool,
unsigned);
/* /*
* bool * bool
* extent_commit(void *addr, size_t size, size_t offset, size_t length, * extent_commit(void *addr, size_t size, size_t offset, size_t length,
* unsigned arena_ind); * unsigned arena_ind);
*/ */
typedef bool (extent_commit_t)(void *, size_t, size_t, size_t, unsigned); typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
unsigned);
/* /*
* bool * bool
* extent_decommit(void *addr, size_t size, size_t offset, size_t length, * extent_decommit(void *addr, size_t size, size_t offset, size_t length,
* unsigned arena_ind); * unsigned arena_ind);
*/ */
typedef bool (extent_decommit_t)(void *, size_t, size_t, size_t, unsigned); typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t,
size_t, unsigned);
/* /*
* bool * bool
* extent_purge(void *addr, size_t size, size_t offset, size_t length, * extent_purge(void *addr, size_t size, size_t offset, size_t length,
* unsigned arena_ind); * unsigned arena_ind);
*/ */
typedef bool (extent_purge_t)(void *, size_t, size_t, size_t, unsigned); typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
unsigned);
/* /*
* bool * bool
* extent_split(void *addr, size_t size, size_t size_a, size_t size_b, * extent_split(void *addr, size_t size, size_t size_a, size_t size_b,
* bool committed, unsigned arena_ind); * bool committed, unsigned arena_ind);
*/ */
typedef bool (extent_split_t)(void *, size_t, size_t, size_t, bool, unsigned); typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
bool, unsigned);
/* /*
* bool * bool
* extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b, * extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
* bool committed, unsigned arena_ind); * bool committed, unsigned arena_ind);
*/ */
typedef bool (extent_merge_t)(void *, size_t, void *, size_t, bool, unsigned); typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t,
bool, unsigned);
typedef struct { struct extent_hooks_s {
extent_alloc_t *alloc; extent_alloc_t *alloc;
extent_dalloc_t *dalloc; extent_dalloc_t *dalloc;
extent_commit_t *commit; extent_commit_t *commit;
@ -55,4 +63,4 @@ typedef struct {
extent_purge_t *purge; extent_purge_t *purge;
extent_split_t *split; extent_split_t *split;
extent_merge_t *merge; extent_merge_t *merge;
} extent_hooks_t; };

48
src/arena.c
View File

@ -54,25 +54,25 @@ arena_extent_dirty_npages(const extent_t *extent)
static extent_t * static extent_t *
arena_extent_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena, arena_extent_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab) size_t alignment, bool *zero, bool slab)
{ {
malloc_mutex_assert_owner(tsdn, &arena->lock); malloc_mutex_assert_owner(tsdn, &arena->lock);
return (extent_alloc_cache(tsdn, arena, extent_hooks, new_addr, usize, return (extent_alloc_cache(tsdn, arena, r_extent_hooks, new_addr, usize,
pad, alignment, zero, slab)); pad, alignment, zero, slab));
} }
extent_t * extent_t *
arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena, arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, extent_hooks_t **r_extent_hooks, void *new_addr, size_t size,
bool *zero) size_t alignment, bool *zero)
{ {
extent_t *extent; extent_t *extent;
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
extent = arena_extent_cache_alloc_locked(tsdn, arena, extent_hooks, extent = arena_extent_cache_alloc_locked(tsdn, arena, r_extent_hooks,
new_addr, size, 0, alignment, zero, false); new_addr, size, 0, alignment, zero, false);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
@ -81,22 +81,22 @@ arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
static void static void
arena_extent_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena, arena_extent_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent) extent_hooks_t **r_extent_hooks, extent_t *extent)
{ {
malloc_mutex_assert_owner(tsdn, &arena->lock); malloc_mutex_assert_owner(tsdn, &arena->lock);
extent_dalloc_cache(tsdn, arena, extent_hooks, extent); extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent);
arena_maybe_purge(tsdn, arena); arena_maybe_purge(tsdn, arena);
} }
void void
arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent) extent_hooks_t **r_extent_hooks, extent_t *extent)
{ {
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
arena_extent_cache_dalloc_locked(tsdn, arena, extent_hooks, extent); arena_extent_cache_dalloc_locked(tsdn, arena, r_extent_hooks, extent);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
} }
@ -321,12 +321,12 @@ arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)
static extent_t * static extent_t *
arena_extent_alloc_large_hard(tsdn_t *tsdn, arena_t *arena, arena_extent_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, size_t usize, size_t alignment, bool *zero) extent_hooks_t **r_extent_hooks, size_t usize, size_t alignment, bool *zero)
{ {
extent_t *extent; extent_t *extent;
bool commit = true; bool commit = true;
extent = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, usize, extent = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, usize,
large_pad, alignment, zero, &commit, false); large_pad, alignment, zero, &commit, false);
if (extent == NULL) { if (extent == NULL) {
/* Revert optimistic stats updates. */ /* Revert optimistic stats updates. */
@ -347,7 +347,7 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero) size_t alignment, bool *zero)
{ {
extent_t *extent; extent_t *extent;
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
@ -373,7 +373,7 @@ void
arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent, arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool locked) bool locked)
{ {
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
if (!locked) if (!locked)
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
@ -735,7 +735,7 @@ arena_dirty_count(tsdn_t *tsdn, arena_t *arena)
} }
static size_t static size_t
arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
size_t ndirty_limit, extent_t *purge_extents_sentinel) size_t ndirty_limit, extent_t *purge_extents_sentinel)
{ {
extent_t *extent, *next; extent_t *extent, *next;
@ -757,7 +757,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
/* Allocate. */ /* Allocate. */
zero = false; zero = false;
textent = arena_extent_cache_alloc_locked(tsdn, arena, textent = arena_extent_cache_alloc_locked(tsdn, arena,
extent_hooks, extent_base_get(extent), r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), 0, CACHELINE, &zero, false); extent_size_get(extent), 0, CACHELINE, &zero, false);
assert(textent == extent); assert(textent == extent);
assert(zero == extent_zeroed_get(extent)); assert(zero == extent_zeroed_get(extent));
@ -774,8 +774,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
} }
static size_t static size_t
arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, arena_purge_stashed(tsdn_t *tsdn, arena_t *arena,
extent_t *purge_extents_sentinel) extent_hooks_t **r_extent_hooks, extent_t *purge_extents_sentinel)
{ {
UNUSED size_t nmadvise; UNUSED size_t nmadvise;
size_t npurged; size_t npurged;
@ -793,7 +793,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
next = qr_next(extent, qr_link); next = qr_next(extent, qr_link);
extent_ring_remove(extent); extent_ring_remove(extent);
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent); extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, extent);
} }
if (config_stats) { if (config_stats) {
@ -816,7 +816,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
static void static void
arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit)
{ {
extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena); extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t npurge, npurged; size_t npurge, npurged;
extent_t purge_extents_sentinel; extent_t purge_extents_sentinel;
@ -866,7 +866,7 @@ arena_purge(tsdn_t *tsdn, arena_t *arena, bool all)
static void static void
arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab)
{ {
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab); arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab);
@ -988,7 +988,7 @@ arena_bin_slabs_full_remove(extent_t *slab)
static extent_t * static extent_t *
arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, const arena_bin_info_t *bin_info) extent_hooks_t **r_extent_hooks, const arena_bin_info_t *bin_info)
{ {
extent_t *slab; extent_t *slab;
bool zero, commit; bool zero, commit;
@ -996,7 +996,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
zero = false; zero = false;
commit = true; commit = true;
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
slab = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL,
bin_info->slab_size, 0, PAGE, &zero, &commit, true); bin_info->slab_size, 0, PAGE, &zero, &commit, true);
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
@ -1009,7 +1009,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
{ {
extent_t *slab; extent_t *slab;
arena_slab_data_t *slab_data; arena_slab_data_t *slab_data;
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
bool zero; bool zero;
zero = false; zero = false;
@ -1796,7 +1796,7 @@ arena_new(tsdn_t *tsdn, unsigned ind)
extent_heap_new(&arena->extents_retained[i]); extent_heap_new(&arena->extents_retained[i]);
} }
arena->extent_hooks = extent_hooks_default; arena->extent_hooks = (extent_hooks_t *)&extent_hooks_default;
if (malloc_mutex_init(&arena->extents_mtx, "arena_extents", if (malloc_mutex_init(&arena->extents_mtx, "arena_extents",
WITNESS_RANK_ARENA_EXTENTS)) WITNESS_RANK_ARENA_EXTENTS))

16
src/ctl.c
View File

@ -1646,15 +1646,15 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
if (arena_ind < narenas_total_get() && (arena = if (arena_ind < narenas_total_get() && (arena =
arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
if (newp != NULL) { if (newp != NULL) {
extent_hooks_t old_extent_hooks, new_extent_hooks; extent_hooks_t *old_extent_hooks, *new_extent_hooks;
WRITE(new_extent_hooks, extent_hooks_t); WRITE(new_extent_hooks, extent_hooks_t *);
old_extent_hooks = extent_hooks_set(tsd_tsdn(tsd), old_extent_hooks = extent_hooks_set(arena,
arena, &new_extent_hooks); new_extent_hooks);
READ(old_extent_hooks, extent_hooks_t); READ(old_extent_hooks, extent_hooks_t *);
} else { } else {
extent_hooks_t old_extent_hooks = extent_hooks_t *old_extent_hooks =
extent_hooks_get(tsd_tsdn(tsd), arena); extent_hooks_get(arena);
READ(old_extent_hooks, extent_hooks_t); READ(old_extent_hooks, extent_hooks_t *);
} }
} else { } else {
ret = EFAULT; ret = EFAULT;

307
src/extent.c
View File

@ -6,20 +6,23 @@
rtree_t extents_rtree; rtree_t extents_rtree;
static void *extent_alloc_default(void *new_addr, size_t size, static void *extent_alloc_default(extent_hooks_t *extent_hooks,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind); void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
static bool extent_dalloc_default(void *addr, size_t size, bool committed, unsigned arena_ind);
static bool extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, bool committed, unsigned arena_ind);
static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_decommit_default(extent_hooks_t *extent_hooks,
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_purge_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t size_a, size_t size_b, bool committed,
unsigned arena_ind);
static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
size_t size_a, void *addr_b, size_t size_b, bool committed,
unsigned arena_ind); unsigned arena_ind);
static bool extent_commit_default(void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind);
static bool extent_decommit_default(void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind);
static bool extent_purge_default(void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind);
static bool extent_split_default(void *addr, size_t size, size_t size_a,
size_t size_b, bool committed, unsigned arena_ind);
static bool extent_merge_default(void *addr_a, size_t size_a, void *addr_b,
size_t size_b, bool committed, unsigned arena_ind);
const extent_hooks_t extent_hooks_default = { const extent_hooks_t extent_hooks_default = {
extent_alloc_default, extent_alloc_default,
@ -42,7 +45,7 @@ static size_t highchunks;
*/ */
static void extent_record(tsdn_t *tsdn, arena_t *arena, static void extent_record(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_heap_t extent_heaps[NPSIZES], extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES],
bool cache, extent_t *extent); bool cache, extent_t *extent);
/******************************************************************************/ /******************************************************************************/
@ -73,89 +76,34 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
} }
static extent_hooks_t extent_hooks_t *
extent_hooks_get_locked(arena_t *arena) extent_hooks_get(arena_t *arena)
{ {
return (arena->extent_hooks); return ((extent_hooks_t *)atomic_read_p(&arena->extent_hooks_pun));
} }
extent_hooks_t extent_hooks_t *
extent_hooks_get(tsdn_t *tsdn, arena_t *arena) extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks)
{ {
extent_hooks_t extent_hooks; extent_hooks_t *old_extent_hooks = extent_hooks_get(arena);
union {
extent_hooks_t **h;
void **v;
} u;
malloc_mutex_lock(tsdn, &arena->extents_mtx); u.h = &arena->extent_hooks;
extent_hooks = extent_hooks_get_locked(arena); atomic_write_p(u.v, extent_hooks);
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (extent_hooks);
}
extent_hooks_t
extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
const extent_hooks_t *extent_hooks)
{
extent_hooks_t old_extent_hooks;
malloc_mutex_lock(tsdn, &arena->extents_mtx);
old_extent_hooks = arena->extent_hooks;
/*
* Copy each field atomically so that it is impossible for readers to
* see partially updated pointers. There are places where readers only
* need one hook function pointer (therefore no need to copy the
* entirety of arena->extent_hooks), and stale reads do not affect
* correctness, so they perform unlocked reads.
*/
#define ATOMIC_COPY_HOOK(n) do { \
union { \
extent_##n##_t **n; \
void **v; \
} u; \
u.n = &arena->extent_hooks.n; \
atomic_write_p(u.v, extent_hooks->n); \
} while (0)
ATOMIC_COPY_HOOK(alloc);
ATOMIC_COPY_HOOK(dalloc);
ATOMIC_COPY_HOOK(commit);
ATOMIC_COPY_HOOK(decommit);
ATOMIC_COPY_HOOK(purge);
ATOMIC_COPY_HOOK(split);
ATOMIC_COPY_HOOK(merge);
#undef ATOMIC_COPY_HOOK
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (old_extent_hooks); return (old_extent_hooks);
} }
static void static void
extent_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena_t *arena, extent_hooks_t **r_extent_hooks)
extent_hooks_t *extent_hooks, bool locked)
{
static const extent_hooks_t uninitialized_hooks =
EXTENT_HOOKS_INITIALIZER;
if (memcmp(extent_hooks, &uninitialized_hooks, sizeof(extent_hooks_t))
== 0) {
*extent_hooks = locked ? extent_hooks_get_locked(arena) :
extent_hooks_get(tsdn, arena);
}
}
static void
extent_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks)
{ {
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, true); if (*r_extent_hooks == EXTENT_HOOKS_INITIALIZER)
} *r_extent_hooks = extent_hooks_get(arena);
static void
extent_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks)
{
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, false);
} }
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
@ -409,7 +357,7 @@ extent_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES],
} }
static void static void
extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
bool cache, extent_t *extent) bool cache, extent_t *extent)
{ {
@ -418,14 +366,14 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
* that this is only a virtual memory leak. * that this is only a virtual memory leak.
*/ */
if (cache) { if (cache) {
extent_purge_wrapper(tsdn, arena, extent_hooks, extent, 0, extent_purge_wrapper(tsdn, arena, r_extent_hooks, extent, 0,
extent_size_get(extent)); extent_size_get(extent));
} }
extent_dalloc(tsdn, arena, extent); extent_dalloc(tsdn, arena, extent);
} }
static extent_t * static extent_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr, extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr,
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
bool slab) bool slab)
@ -444,7 +392,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
if (alloc_size < usize) if (alloc_size < usize)
return (NULL); return (NULL);
malloc_mutex_lock(tsdn, &arena->extents_mtx); malloc_mutex_lock(tsdn, &arena->extents_mtx);
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
if (new_addr != NULL) { if (new_addr != NULL) {
rtree_elm_t *elm; rtree_elm_t *elm;
@ -482,10 +430,11 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
/* Split the lead. */ /* Split the lead. */
if (leadsize != 0) { if (leadsize != 0) {
extent_t *lead = extent; extent_t *lead = extent;
extent = extent_split_wrapper(tsdn, arena, extent_hooks, lead, extent = extent_split_wrapper(tsdn, arena, r_extent_hooks,
leadsize, leadsize, size + trailsize, usize + trailsize); lead, leadsize, leadsize, size + trailsize, usize +
trailsize);
if (extent == NULL) { if (extent == NULL) {
extent_leak(tsdn, arena, extent_hooks, cache, lead); extent_leak(tsdn, arena, r_extent_hooks, cache, lead);
malloc_mutex_unlock(tsdn, &arena->extents_mtx); malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (NULL); return (NULL);
} }
@ -496,9 +445,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
/* Split the trail. */ /* Split the trail. */
if (trailsize != 0) { if (trailsize != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena, extent_t *trail = extent_split_wrapper(tsdn, arena,
extent_hooks, extent, size, usize, trailsize, trailsize); r_extent_hooks, extent, size, usize, trailsize, trailsize);
if (trail == NULL) { if (trail == NULL) {
extent_leak(tsdn, arena, extent_hooks, cache, extent); extent_leak(tsdn, arena, r_extent_hooks, cache,
extent);
malloc_mutex_unlock(tsdn, &arena->extents_mtx); malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (NULL); return (NULL);
} }
@ -513,10 +463,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
} }
if (!extent_committed_get(extent) && if (!extent_committed_get(extent) &&
extent_hooks->commit(extent_base_get(extent), (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) { extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx); malloc_mutex_unlock(tsdn, &arena->extents_mtx);
extent_record(tsdn, arena, extent_hooks, extent_heaps, cache, extent_record(tsdn, arena, r_extent_hooks, extent_heaps, cache,
extent); extent);
return (NULL); return (NULL);
} }
@ -582,9 +532,9 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
} }
extent_t * extent_t *
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
bool slab) size_t alignment, bool *zero, bool slab)
{ {
extent_t *extent; extent_t *extent;
bool commit; bool commit;
@ -593,7 +543,7 @@ extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
assert(alignment != 0); assert(alignment != 0);
commit = true; commit = true;
extent = extent_recycle(tsdn, arena, extent_hooks, extent = extent_recycle(tsdn, arena, r_extent_hooks,
arena->extents_cached, true, new_addr, usize, pad, alignment, zero, arena->extents_cached, true, new_addr, usize, pad, alignment, zero,
&commit, slab); &commit, slab);
if (extent == NULL) if (extent == NULL)
@ -603,13 +553,15 @@ extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
} }
static void * static void *
extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
bool *commit, unsigned arena_ind) size_t alignment, bool *zero, bool *commit, unsigned arena_ind)
{ {
void *ret; void *ret;
tsdn_t *tsdn; tsdn_t *tsdn;
arena_t *arena; arena_t *arena;
assert(extent_hooks == &extent_hooks_default);
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
arena = arena_get(tsdn, arena_ind, false); arena = arena_get(tsdn, arena_ind, false);
/* /*
@ -627,7 +579,7 @@ extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
static extent_t * static extent_t *
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab) size_t alignment, bool *zero, bool *commit, bool slab)
{ {
extent_t *extent; extent_t *extent;
@ -635,7 +587,7 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
assert(usize != 0); assert(usize != 0);
assert(alignment != 0); assert(alignment != 0);
extent = extent_recycle(tsdn, arena, extent_hooks, extent = extent_recycle(tsdn, arena, r_extent_hooks,
arena->extents_retained, false, new_addr, usize, pad, alignment, arena->extents_retained, false, new_addr, usize, pad, alignment,
zero, commit, slab); zero, commit, slab);
if (extent != NULL && config_stats) { if (extent != NULL && config_stats) {
@ -648,7 +600,7 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
static extent_t * static extent_t *
extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab) size_t alignment, bool *zero, bool *commit, bool slab)
{ {
extent_t *extent; extent_t *extent;
@ -659,8 +611,8 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent = extent_alloc(tsdn, arena); extent = extent_alloc(tsdn, arena);
if (extent == NULL) if (extent == NULL)
return (NULL); return (NULL);
addr = extent_hooks->alloc(new_addr, size, alignment, zero, commit, addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, size,
arena->ind); alignment, zero, commit, arena->ind);
if (addr == NULL) { if (addr == NULL) {
extent_dalloc(tsdn, arena, extent); extent_dalloc(tsdn, arena, extent);
return (NULL); return (NULL);
@ -669,7 +621,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
if (pad != 0) if (pad != 0)
extent_addr_randomize(tsdn, extent, alignment); extent_addr_randomize(tsdn, extent, alignment);
if (extent_register(tsdn, extent)) { if (extent_register(tsdn, extent)) {
extent_leak(tsdn, arena, extent_hooks, false, extent); extent_leak(tsdn, arena, r_extent_hooks, false, extent);
return (NULL); return (NULL);
} }
@ -677,18 +629,18 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
} }
extent_t * extent_t *
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
bool *commit, bool slab) size_t alignment, bool *zero, bool *commit, bool slab)
{ {
extent_t *extent; extent_t *extent;
extent_hooks_assure_initialized(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
extent = extent_alloc_retained(tsdn, arena, extent_hooks, new_addr, extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, new_addr,
usize, pad, alignment, zero, commit, slab); usize, pad, alignment, zero, commit, slab);
if (extent == NULL) { if (extent == NULL) {
extent = extent_alloc_wrapper_hard(tsdn, arena, extent_hooks, extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks,
new_addr, usize, pad, alignment, zero, commit, slab); new_addr, usize, pad, alignment, zero, commit, slab);
} }
@ -712,8 +664,9 @@ extent_can_coalesce(const extent_t *a, const extent_t *b)
} }
static void static void
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache) extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b,
extent_heap_t extent_heaps[NPSIZES], bool cache)
{ {
if (!extent_can_coalesce(a, b)) if (!extent_can_coalesce(a, b))
@ -725,7 +678,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
arena_extent_cache_maybe_remove(extent_arena_get(a), a, cache); arena_extent_cache_maybe_remove(extent_arena_get(a), a, cache);
arena_extent_cache_maybe_remove(extent_arena_get(b), b, cache); arena_extent_cache_maybe_remove(extent_arena_get(b), b, cache);
if (extent_merge_wrapper(tsdn, arena, extent_hooks, a, b)) { if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) {
extent_heaps_insert(extent_heaps, a); extent_heaps_insert(extent_heaps, a);
extent_heaps_insert(extent_heaps, b); extent_heaps_insert(extent_heaps, b);
arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache); arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache);
@ -738,7 +691,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
} }
static void static void
extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent) extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent)
{ {
extent_t *prev, *next; extent_t *prev, *next;
@ -748,7 +701,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
assert(!cache || !extent_zeroed_get(extent)); assert(!cache || !extent_zeroed_get(extent));
malloc_mutex_lock(tsdn, &arena->extents_mtx); malloc_mutex_lock(tsdn, &arena->extents_mtx);
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
extent_usize_set(extent, 0); extent_usize_set(extent, 0);
extent_active_set(extent, false); extent_active_set(extent, false);
@ -766,7 +719,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
next = rtree_read(tsdn, &extents_rtree, rtree_ctx, next = rtree_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_past_get(extent), false); (uintptr_t)extent_past_get(extent), false);
if (next != NULL) { if (next != NULL) {
extent_try_coalesce(tsdn, arena, extent_hooks, extent, next, extent_try_coalesce(tsdn, arena, r_extent_hooks, extent, next,
extent_heaps, cache); extent_heaps, cache);
} }
@ -774,7 +727,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
prev = rtree_read(tsdn, &extents_rtree, rtree_ctx, prev = rtree_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_before_get(extent), false); (uintptr_t)extent_before_get(extent), false);
if (prev != NULL) { if (prev != NULL) {
extent_try_coalesce(tsdn, arena, extent_hooks, prev, extent, extent_try_coalesce(tsdn, arena, r_extent_hooks, prev, extent,
extent_heaps, cache); extent_heaps, cache);
} }
@ -782,8 +735,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
} }
void void
extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_t *extent) extent_hooks_t **r_extent_hooks, extent_t *extent)
{ {
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
@ -792,15 +745,17 @@ extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false); extent_zeroed_set(extent, false);
extent_record(tsdn, arena, extent_hooks, arena->extents_cached, true, extent_record(tsdn, arena, r_extent_hooks, arena->extents_cached, true,
extent); extent);
} }
static bool static bool
extent_dalloc_default(void *addr, size_t size, bool committed, extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
unsigned arena_ind) bool committed, unsigned arena_ind)
{ {
assert(extent_hooks == &extent_hooks_default);
if (!have_dss || !extent_in_dss(tsdn_fetch(), addr)) if (!have_dss || !extent_in_dss(tsdn_fetch(), addr))
return (extent_dalloc_mmap(addr, size)); return (extent_dalloc_mmap(addr, size));
return (true); return (true);
@ -808,7 +763,7 @@ extent_dalloc_default(void *addr, size_t size, bool committed,
void void
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent) extent_hooks_t **r_extent_hooks, extent_t *extent)
{ {
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
@ -816,9 +771,9 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
extent_hooks_assure_initialized(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
/* Try to deallocate. */ /* Try to deallocate. */
if (!extent_hooks->dalloc(extent_base_get(extent), if (!(*r_extent_hooks)->dalloc(*r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), extent_committed_get(extent), extent_size_get(extent), extent_committed_get(extent),
arena->ind)) { arena->ind)) {
extent_deregister(tsdn, extent); extent_deregister(tsdn, extent);
@ -828,66 +783,73 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
/* Try to decommit; purge if that fails. */ /* Try to decommit; purge if that fails. */
if (extent_committed_get(extent)) { if (extent_committed_get(extent)) {
extent_committed_set(extent, extent_committed_set(extent,
extent_hooks->decommit(extent_base_get(extent), (*r_extent_hooks)->decommit(*r_extent_hooks,
extent_size_get(extent), 0, extent_size_get(extent), extent_base_get(extent), extent_size_get(extent), 0,
arena->ind)); extent_size_get(extent), arena->ind));
} }
extent_zeroed_set(extent, !extent_committed_get(extent) || extent_zeroed_set(extent, !extent_committed_get(extent) ||
!extent_hooks->purge(extent_base_get(extent), !(*r_extent_hooks)->purge(*r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)); extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
if (config_stats) if (config_stats)
arena->stats.retained += extent_size_get(extent); arena->stats.retained += extent_size_get(extent);
extent_record(tsdn, arena, extent_hooks, arena->extents_retained, false, extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
extent); false, extent);
} }
static bool static bool
extent_commit_default(void *addr, size_t size, size_t offset, size_t length, extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
unsigned arena_ind) size_t offset, size_t length, unsigned arena_ind)
{ {
assert(extent_hooks == &extent_hooks_default);
return (pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), return (pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset),
length)); length));
} }
bool bool
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length) size_t length)
{ {
extent_hooks_assure_initialized(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
return (extent_hooks->commit(extent_base_get(extent), return ((*r_extent_hooks)->commit(*r_extent_hooks,
extent_size_get(extent), offset, length, arena->ind)); extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
} }
static bool static bool
extent_decommit_default(void *addr, size_t size, size_t offset, size_t length, extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
unsigned arena_ind) size_t offset, size_t length, unsigned arena_ind)
{ {
assert(extent_hooks == &extent_hooks_default);
return (pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), return (pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset),
length)); length));
} }
bool bool
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length) size_t length)
{ {
extent_hooks_assure_initialized(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
return (extent_hooks->decommit(extent_base_get(extent), return ((*r_extent_hooks)->decommit(*r_extent_hooks,
extent_size_get(extent), offset, length, arena->ind)); extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
} }
static bool static bool
extent_purge_default(void *addr, size_t size, size_t offset, size_t length, extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
unsigned arena_ind) size_t offset, size_t length, unsigned arena_ind)
{ {
assert(extent_hooks == &extent_hooks_default);
assert(addr != NULL); assert(addr != NULL);
assert((offset & PAGE_MASK) == 0); assert((offset & PAGE_MASK) == 0);
assert(length != 0); assert(length != 0);
@ -898,29 +860,33 @@ extent_purge_default(void *addr, size_t size, size_t offset, size_t length,
} }
bool bool
extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t offset, size_t length) extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{ {
extent_hooks_assure_initialized(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
return (extent_hooks->purge(extent_base_get(extent), return ((*r_extent_hooks)->purge(*r_extent_hooks,
extent_size_get(extent), offset, length, arena->ind)); extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
} }
static bool static bool
extent_split_default(void *addr, size_t size, size_t size_a, size_t size_b, extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind) size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
{ {
assert(extent_hooks == &extent_hooks_default);
if (!maps_coalesce) if (!maps_coalesce)
return (true); return (true);
return (false); return (false);
} }
extent_t * extent_t *
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t size_a, size_t usize_a, size_t size_b, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
size_t usize_b) size_t usize_a, size_t size_b, size_t usize_b)
{ {
extent_t *trail; extent_t *trail;
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
@ -929,7 +895,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
assert(extent_size_get(extent) == size_a + size_b); assert(extent_size_get(extent) == size_a + size_b);
extent_hooks_assure_initialized(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
trail = extent_alloc(tsdn, arena); trail = extent_alloc(tsdn, arena);
if (trail == NULL) if (trail == NULL)
@ -956,8 +922,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
&trail_elm_a, &trail_elm_b)) &trail_elm_a, &trail_elm_b))
goto label_error_c; goto label_error_c;
if (extent_hooks->split(extent_base_get(extent), size_a + size_b, if ((*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent),
size_a, size_b, extent_committed_get(extent), arena->ind)) size_a + size_b, size_a, size_b, extent_committed_get(extent),
arena->ind))
goto label_error_d; goto label_error_d;
extent_size_set(extent, size_a); extent_size_set(extent, size_a);
@ -981,10 +948,12 @@ label_error_a:
} }
static bool static bool
extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b, extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
bool committed, unsigned arena_ind) void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
{ {
assert(extent_hooks == &extent_hooks_default);
if (!maps_coalesce) if (!maps_coalesce)
return (true); return (true);
if (have_dss) { if (have_dss) {
@ -997,17 +966,17 @@ extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
} }
bool bool
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_t *a, extent_t *b) extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b)
{ {
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
extent_hooks_assure_initialized(tsdn, arena, extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
if (extent_hooks->merge(extent_base_get(a), extent_size_get(a), if ((*r_extent_hooks)->merge(*r_extent_hooks, extent_base_get(a),
extent_base_get(b), extent_size_get(b), extent_committed_get(a), extent_size_get(a), extent_base_get(b), extent_size_get(b),
arena->ind)) extent_committed_get(a), arena->ind))
return (true); return (true);
/* /*

2
src/extent_dss.c
View File

@ -136,7 +136,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
dss_max = dss_next; dss_max = dss_next;
malloc_mutex_unlock(tsdn, &dss_mtx); malloc_mutex_unlock(tsdn, &dss_mtx);
if (pad_size != 0) { if (pad_size != 0) {
extent_hooks_t extent_hooks = extent_hooks_t *extent_hooks =
EXTENT_HOOKS_INITIALIZER; EXTENT_HOOKS_INITIALIZER;
extent_dalloc_wrapper(tsdn, arena, extent_dalloc_wrapper(tsdn, arena,
&extent_hooks, pad); &extent_hooks, pad);

4
src/large.c
View File

@ -96,7 +96,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
{ {
arena_t *arena = extent_arena_get(extent); arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent); size_t oldusize = extent_usize_get(extent);
extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena); extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t diff = extent_size_get(extent) - (usize + large_pad); size_t diff = extent_size_get(extent) - (usize + large_pad);
assert(oldusize > usize); assert(oldusize > usize);
@ -129,7 +129,7 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
arena_t *arena = extent_arena_get(extent); arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent); size_t oldusize = extent_usize_get(extent);
bool is_zeroed_trail = false; bool is_zeroed_trail = false;
extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena); extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t trailsize = usize - extent_usize_get(extent); size_t trailsize = usize - extent_usize_get(extent);
extent_t *trail; extent_t *trail;

159
test/integration/extent.c
View File

@ -4,8 +4,8 @@
const char *malloc_conf = "junk:false"; const char *malloc_conf = "junk:false";
#endif #endif
static extent_hooks_t orig_hooks; static extent_hooks_t *orig_hooks;
static extent_hooks_t old_hooks; static extent_hooks_t *old_hooks;
static bool do_dalloc = true; static bool do_dalloc = true;
static bool do_decommit; static bool do_decommit;
@ -24,96 +24,111 @@ static bool did_merge;
# define TRACE_HOOK(fmt, ...) # define TRACE_HOOK(fmt, ...)
#endif #endif
void * static void *
extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
bool *commit, unsigned arena_ind) size_t alignment, bool *zero, bool *commit, unsigned arena_ind)
{ {
TRACE_HOOK("%s(new_addr=%p, size=%zu, alignment=%zu, *zero=%s, " TRACE_HOOK("%s(extent_hooks=%p, new_addr=%p, size=%zu, alignment=%zu, "
"*commit=%s, arena_ind=%u)\n", __func__, new_addr, size, alignment, "*zero=%s, *commit=%s, arena_ind=%u)\n", __func__, extent_hooks,
*zero ? "true" : "false", *commit ? "true" : "false", arena_ind); new_addr, size, alignment, *zero ? "true" : "false", *commit ?
"true" : "false", arena_ind);
assert(extent_hooks->alloc == extent_alloc);
did_alloc = true; did_alloc = true;
return (old_hooks.alloc(new_addr, size, alignment, zero, commit, return (old_hooks->alloc(old_hooks, new_addr, size, alignment, zero,
arena_ind)); commit, arena_ind));
} }
bool static bool
extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind) extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind)
{ {
TRACE_HOOK("%s(addr=%p, size=%zu, committed=%s, arena_ind=%u)\n", TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, committed=%s, "
__func__, addr, size, committed ? "true" : "false", arena_ind); "arena_ind=%u)\n", __func__, extent_hooks, addr, size, committed ?
"true" : "false", arena_ind);
assert(extent_hooks->dalloc == extent_dalloc);
did_dalloc = true; did_dalloc = true;
if (!do_dalloc) if (!do_dalloc)
return (true); return (true);
return (old_hooks.dalloc(addr, size, committed, arena_ind)); return (old_hooks->dalloc(old_hooks, addr, size, committed, arena_ind));
} }
bool static bool
extent_commit(void *addr, size_t size, size_t offset, size_t length, extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size,
unsigned arena_ind) size_t offset, size_t length, unsigned arena_ind)
{ {
bool err; bool err;
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, " TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"arena_ind=%u)\n", __func__, addr, size, offset, length, "length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size,
offset, length, arena_ind);
assert(extent_hooks->commit == extent_commit);
err = old_hooks->commit(old_hooks, addr, size, offset, length,
arena_ind); arena_ind);
err = old_hooks.commit(addr, size, offset, length, arena_ind);
did_commit = !err; did_commit = !err;
return (err); return (err);
} }
bool static bool
extent_decommit(void *addr, size_t size, size_t offset, size_t length, extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
unsigned arena_ind) size_t offset, size_t length, unsigned arena_ind)
{ {
bool err; bool err;
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, " TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"arena_ind=%u)\n", __func__, addr, size, offset, length, "length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size,
arena_ind); offset, length, arena_ind);
assert(extent_hooks->decommit == extent_decommit);
if (!do_decommit) if (!do_decommit)
return (true); return (true);
err = old_hooks.decommit(addr, size, offset, length, arena_ind); err = old_hooks->decommit(old_hooks, addr, size, offset, length,
arena_ind);
did_decommit = !err; did_decommit = !err;
return (err); return (err);
} }
bool static bool
extent_purge(void *addr, size_t size, size_t offset, size_t length, extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
unsigned arena_ind) size_t offset, size_t length, unsigned arena_ind)
{ {
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu " TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"arena_ind=%u)\n", __func__, addr, size, offset, length, "length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
arena_ind); offset, length, arena_ind);
assert(extent_hooks->purge == extent_purge);
did_purge = true; did_purge = true;
return (old_hooks.purge(addr, size, offset, length, arena_ind)); return (old_hooks->purge(old_hooks, addr, size, offset, length,
}
bool
extent_split(void *addr, size_t size, size_t size_a, size_t size_b,
bool committed, unsigned arena_ind)
{
TRACE_HOOK("%s(addr=%p, size=%zu, size_a=%zu, size_b=%zu, "
"committed=%s, arena_ind=%u)\n", __func__, addr, size, size_a,
size_b, committed ? "true" : "false", arena_ind);
did_split = true;
return (old_hooks.split(addr, size, size_a, size_b, committed,
arena_ind)); arena_ind));
} }
bool static bool
extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b, extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind) size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
{ {
TRACE_HOOK("%s(addr_a=%p, size_a=%zu, addr_b=%p size_b=%zu, " TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, size_a=%zu, "
"committed=%s, arena_ind=%u)\n", __func__, addr_a, size_a, addr_b, "size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks,
size_b, committed ? "true" : "false", arena_ind); addr, size, size_a, size_b, committed ? "true" : "false",
arena_ind);
assert(extent_hooks->split == extent_split);
did_split = true;
return (old_hooks->split(old_hooks, addr, size, size_a, size_b,
committed, arena_ind));
}
static bool
extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
{
TRACE_HOOK("%s(extent_hooks=%p, addr_a=%p, size_a=%zu, addr_b=%p "
"size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks,
addr_a, size_a, addr_b, size_b, committed ? "true" : "false",
arena_ind);
assert(extent_hooks->merge == extent_merge);
did_merge = true; did_merge = true;
return (old_hooks.merge(addr_a, size_a, addr_b, size_b, return (old_hooks->merge(old_hooks, addr_a, size_a, addr_b, size_b,
committed, arena_ind)); committed, arena_ind));
} }
@ -125,7 +140,7 @@ TEST_BEGIN(test_extent)
int flags; int flags;
size_t hooks_mib[3], purge_mib[3]; size_t hooks_mib[3], purge_mib[3];
size_t hooks_miblen, purge_miblen; size_t hooks_miblen, purge_miblen;
extent_hooks_t new_hooks = { extent_hooks_t hooks = {
extent_alloc, extent_alloc,
extent_dalloc, extent_dalloc,
extent_commit, extent_commit,
@ -134,6 +149,7 @@ TEST_BEGIN(test_extent)
extent_split, extent_split,
extent_merge extent_merge
}; };
extent_hooks_t *new_hooks = &hooks;
bool xallocx_success_a, xallocx_success_b, xallocx_success_c; bool xallocx_success_a, xallocx_success_b, xallocx_success_c;
sz = sizeof(unsigned); sz = sizeof(unsigned);
@ -146,21 +162,21 @@ TEST_BEGIN(test_extent)
assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib, assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib,
&hooks_miblen), 0, "Unexpected mallctlnametomib() failure"); &hooks_miblen), 0, "Unexpected mallctlnametomib() failure");
hooks_mib[1] = (size_t)arena_ind; hooks_mib[1] = (size_t)arena_ind;
old_size = sizeof(extent_hooks_t); old_size = sizeof(extent_hooks_t *);
new_size = sizeof(extent_hooks_t); new_size = sizeof(extent_hooks_t *);
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size, assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
&new_hooks, new_size), 0, "Unexpected extent_hooks error"); &new_hooks, new_size), 0, "Unexpected extent_hooks error");
orig_hooks = old_hooks; orig_hooks = old_hooks;
assert_ptr_ne(old_hooks.alloc, extent_alloc, "Unexpected alloc error"); assert_ptr_ne(old_hooks->alloc, extent_alloc, "Unexpected alloc error");
assert_ptr_ne(old_hooks.dalloc, extent_dalloc, assert_ptr_ne(old_hooks->dalloc, extent_dalloc,
"Unexpected dalloc error"); "Unexpected dalloc error");
assert_ptr_ne(old_hooks.commit, extent_commit, assert_ptr_ne(old_hooks->commit, extent_commit,
"Unexpected commit error"); "Unexpected commit error");
assert_ptr_ne(old_hooks.decommit, extent_decommit, assert_ptr_ne(old_hooks->decommit, extent_decommit,
"Unexpected decommit error"); "Unexpected decommit error");
assert_ptr_ne(old_hooks.purge, extent_purge, "Unexpected purge error"); assert_ptr_ne(old_hooks->purge, extent_purge, "Unexpected purge error");
assert_ptr_ne(old_hooks.split, extent_split, "Unexpected split error"); assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error");
assert_ptr_ne(old_hooks.merge, extent_merge, "Unexpected merge error"); assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error");
/* Get large size classes. */ /* Get large size classes. */
sz = sizeof(size_t); sz = sizeof(size_t);
@ -228,19 +244,20 @@ TEST_BEGIN(test_extent)
&old_hooks, new_size), 0, "Unexpected extent_hooks error"); &old_hooks, new_size), 0, "Unexpected extent_hooks error");
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size, assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
NULL, 0), 0, "Unexpected extent_hooks error"); NULL, 0), 0, "Unexpected extent_hooks error");
assert_ptr_eq(old_hooks.alloc, orig_hooks.alloc, assert_ptr_eq(old_hooks, orig_hooks, "Unexpected hooks error");
assert_ptr_eq(old_hooks->alloc, orig_hooks->alloc,
"Unexpected alloc error"); "Unexpected alloc error");
assert_ptr_eq(old_hooks.dalloc, orig_hooks.dalloc, assert_ptr_eq(old_hooks->dalloc, orig_hooks->dalloc,
"Unexpected dalloc error"); "Unexpected dalloc error");
assert_ptr_eq(old_hooks.commit, orig_hooks.commit, assert_ptr_eq(old_hooks->commit, orig_hooks->commit,
"Unexpected commit error"); "Unexpected commit error");
assert_ptr_eq(old_hooks.decommit, orig_hooks.decommit, assert_ptr_eq(old_hooks->decommit, orig_hooks->decommit,
"Unexpected decommit error"); "Unexpected decommit error");
assert_ptr_eq(old_hooks.purge, orig_hooks.purge, assert_ptr_eq(old_hooks->purge, orig_hooks->purge,
"Unexpected purge error"); "Unexpected purge error");
assert_ptr_eq(old_hooks.split, orig_hooks.split, assert_ptr_eq(old_hooks->split, orig_hooks->split,
"Unexpected split error"); "Unexpected split error");
assert_ptr_eq(old_hooks.merge, orig_hooks.merge, assert_ptr_eq(old_hooks->merge, orig_hooks->merge,
"Unexpected merge error"); "Unexpected merge error");
} }
TEST_END TEST_END