Modify extent hook functions to take an (extent_t *) argument.

This facilitates the application accessing its own extent allocator
metadata during hook invocations.

This resolves #259.
This commit is contained in:
Jason Evans 2016-06-03 12:05:53 -07:00
parent 6f29a83924
commit f8f0542194
10 changed files with 312 additions and 315 deletions

View File

@ -1551,7 +1551,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<varlistentry id="arena.i.extent_hooks">
<term>
<mallctl>arena.&lt;i&gt;.extent_hooks</mallctl>
(<type>extent_hooks_t</type>)
(<type>extent_hooks_t *</type>)
<literal>rw</literal>
</term>
<listitem><para>Get or set the extent management hook functions for
@ -1567,7 +1567,8 @@ malloc_conf = "xmalloc:true";]]></programlisting>
allocation.</para>
<programlisting language="C"><![CDATA[
typedef struct {
typedef extent_hooks_s extent_hooks_t;
struct extent_hooks_s {
extent_alloc_t *alloc;
extent_dalloc_t *dalloc;
extent_commit_t *commit;
@ -1575,7 +1576,7 @@ typedef struct {
extent_purge_t *purge;
extent_split_t *split;
extent_merge_t *merge;
} extent_hooks_t;]]></programlisting>
};]]></programlisting>
<para>The <type>extent_hooks_t</type> structure comprises function
pointers which are described individually below. jemalloc uses these
functions to manage extent lifetime, which starts off with allocation of
@ -1592,6 +1593,7 @@ typedef struct {
<funcsynopsis><funcprototype>
<funcdef>typedef void *<function>(extent_alloc_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>new_addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>alignment</parameter></paramdef>
@ -1627,6 +1629,7 @@ typedef struct {
<funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_dalloc_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>bool <parameter>committed</parameter></paramdef>
@ -1646,6 +1649,7 @@ typedef struct {
<funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_commit_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>offset</parameter></paramdef>
@ -1667,6 +1671,7 @@ typedef struct {
<funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_decommit_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>offset</parameter></paramdef>
@ -1688,6 +1693,7 @@ typedef struct {
<funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_purge_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t<parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>offset</parameter></paramdef>
@ -1707,6 +1713,7 @@ typedef struct {
<funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_split_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr</parameter></paramdef>
<paramdef>size_t <parameter>size</parameter></paramdef>
<paramdef>size_t <parameter>size_a</parameter></paramdef>
@ -1728,6 +1735,7 @@ typedef struct {
<funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_merge_t)</function></funcdef>
<paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
<paramdef>void *<parameter>addr_a</parameter></paramdef>
<paramdef>size_t <parameter>size_a</parameter></paramdef>
<paramdef>void *<parameter>addr_b</parameter></paramdef>

View File

@ -240,11 +240,15 @@ struct arena_s {
*/
extent_heap_t extents_cached[NPSIZES];
extent_heap_t extents_retained[NPSIZES];
/* User-configurable extent hook functions. */
extent_hooks_t extent_hooks;
/* Protects extents_cached, extents_retained, and extent_hooks. */
/* Protects extents_cached and extents_retained. */
malloc_mutex_t extents_mtx;
/* User-configurable extent hook functions. */
union {
extent_hooks_t *extent_hooks;
void *extent_hooks_pun;
};
/* Cache of extent structures that were allocated via base_alloc(). */
ql_head(extent_t) extent_cache;
malloc_mutex_t extent_cache_mtx;
@ -279,10 +283,10 @@ extern ssize_t opt_decay_time;
extern const arena_bin_info_t arena_bin_info[NBINS];
extent_t *arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero);
extent_hooks_t **r_extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero);
void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent);
extent_hooks_t **r_extent_hooks, extent_t *extent);
void arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent,
bool cache);
void arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent,

View File

@ -3,15 +3,7 @@
typedef struct extent_s extent_t;
#define EXTENT_HOOKS_INITIALIZER { \
NULL, \
NULL, \
NULL, \
NULL, \
NULL, \
NULL, \
NULL \
}
#define EXTENT_HOOKS_INITIALIZER NULL
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
@ -93,9 +85,8 @@ extern const extent_hooks_t extent_hooks_default;
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
extent_hooks_t extent_hooks_get(tsdn_t *tsdn, arena_t *arena);
extent_hooks_t extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
const extent_hooks_t *extent_hooks);
extent_hooks_t *extent_hooks_get(arena_t *arena);
extent_hooks_t *extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks);
#ifdef JEMALLOC_JET
typedef size_t (extent_size_quantize_t)(size_t);
@ -109,29 +100,29 @@ size_t extent_size_quantize_ceil(size_t size);
ph_proto(, extent_heap_, extent_heap_t, extent_t)
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab);
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab);
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent);
extent_hooks_t **r_extent_hooks, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent);
extent_hooks_t **r_extent_hooks, extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
size_t usize_a, size_t size_b, size_t usize_b);
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b);
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b);
void extent_prefork(tsdn_t *tsdn);
void extent_postfork_parent(tsdn_t *tsdn);
void extent_postfork_child(tsdn_t *tsdn);

View File

@ -1,53 +1,61 @@
typedef struct extent_hooks_s extent_hooks_t;
/*
* void *
* extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
* bool *commit, unsigned arena_ind);
*/
typedef void *(extent_alloc_t)(void *, size_t, size_t, bool *, bool *,
unsigned);
typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *,
bool *, unsigned);
/*
* bool
* extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind);
*/
typedef bool (extent_dalloc_t)(void *, size_t, bool, unsigned);
typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool,
unsigned);
/*
* bool
* extent_commit(void *addr, size_t size, size_t offset, size_t length,
* unsigned arena_ind);
*/
typedef bool (extent_commit_t)(void *, size_t, size_t, size_t, unsigned);
typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
unsigned);
/*
* bool
* extent_decommit(void *addr, size_t size, size_t offset, size_t length,
* unsigned arena_ind);
*/
typedef bool (extent_decommit_t)(void *, size_t, size_t, size_t, unsigned);
typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t,
size_t, unsigned);
/*
* bool
* extent_purge(void *addr, size_t size, size_t offset, size_t length,
* unsigned arena_ind);
*/
typedef bool (extent_purge_t)(void *, size_t, size_t, size_t, unsigned);
typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
unsigned);
/*
* bool
* extent_split(void *addr, size_t size, size_t size_a, size_t size_b,
* bool committed, unsigned arena_ind);
*/
typedef bool (extent_split_t)(void *, size_t, size_t, size_t, bool, unsigned);
typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t,
bool, unsigned);
/*
* bool
* extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
* bool committed, unsigned arena_ind);
*/
typedef bool (extent_merge_t)(void *, size_t, void *, size_t, bool, unsigned);
typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t,
bool, unsigned);
typedef struct {
struct extent_hooks_s {
extent_alloc_t *alloc;
extent_dalloc_t *dalloc;
extent_commit_t *commit;
@ -55,4 +63,4 @@ typedef struct {
extent_purge_t *purge;
extent_split_t *split;
extent_merge_t *merge;
} extent_hooks_t;
};

View File

@ -54,25 +54,25 @@ arena_extent_dirty_npages(const extent_t *extent)
static extent_t *
arena_extent_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab)
{
malloc_mutex_assert_owner(tsdn, &arena->lock);
return (extent_alloc_cache(tsdn, arena, extent_hooks, new_addr, usize,
return (extent_alloc_cache(tsdn, arena, r_extent_hooks, new_addr, usize,
pad, alignment, zero, slab));
}
extent_t *
arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero)
extent_hooks_t **r_extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero)
{
extent_t *extent;
malloc_mutex_lock(tsdn, &arena->lock);
extent = arena_extent_cache_alloc_locked(tsdn, arena, extent_hooks,
extent = arena_extent_cache_alloc_locked(tsdn, arena, r_extent_hooks,
new_addr, size, 0, alignment, zero, false);
malloc_mutex_unlock(tsdn, &arena->lock);
@ -81,22 +81,22 @@ arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
static void
arena_extent_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent)
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
malloc_mutex_assert_owner(tsdn, &arena->lock);
extent_dalloc_cache(tsdn, arena, extent_hooks, extent);
extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent);
arena_maybe_purge(tsdn, arena);
}
void
arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent)
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
malloc_mutex_lock(tsdn, &arena->lock);
arena_extent_cache_dalloc_locked(tsdn, arena, extent_hooks, extent);
arena_extent_cache_dalloc_locked(tsdn, arena, r_extent_hooks, extent);
malloc_mutex_unlock(tsdn, &arena->lock);
}
@ -321,12 +321,12 @@ arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)
static extent_t *
arena_extent_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, size_t usize, size_t alignment, bool *zero)
extent_hooks_t **r_extent_hooks, size_t usize, size_t alignment, bool *zero)
{
extent_t *extent;
bool commit = true;
extent = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, usize,
extent = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, usize,
large_pad, alignment, zero, &commit, false);
if (extent == NULL) {
/* Revert optimistic stats updates. */
@ -347,7 +347,7 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero)
{
extent_t *extent;
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER;
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
malloc_mutex_lock(tsdn, &arena->lock);
@ -373,7 +373,7 @@ void
arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool locked)
{
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER;
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
if (!locked)
malloc_mutex_lock(tsdn, &arena->lock);
@ -735,7 +735,7 @@ arena_dirty_count(tsdn_t *tsdn, arena_t *arena)
}
static size_t
arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
size_t ndirty_limit, extent_t *purge_extents_sentinel)
{
extent_t *extent, *next;
@ -757,7 +757,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
/* Allocate. */
zero = false;
textent = arena_extent_cache_alloc_locked(tsdn, arena,
extent_hooks, extent_base_get(extent),
r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), 0, CACHELINE, &zero, false);
assert(textent == extent);
assert(zero == extent_zeroed_get(extent));
@ -774,8 +774,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
}
static size_t
arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *purge_extents_sentinel)
arena_purge_stashed(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *purge_extents_sentinel)
{
UNUSED size_t nmadvise;
size_t npurged;
@ -793,7 +793,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
next = qr_next(extent, qr_link);
extent_ring_remove(extent);
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent);
extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, extent);
}
if (config_stats) {
@ -816,7 +816,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
static void
arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit)
{
extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena);
extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t npurge, npurged;
extent_t purge_extents_sentinel;
@ -866,7 +866,7 @@ arena_purge(tsdn_t *tsdn, arena_t *arena, bool all)
static void
arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab)
{
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER;
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab);
@ -988,7 +988,7 @@ arena_bin_slabs_full_remove(extent_t *slab)
static extent_t *
arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, const arena_bin_info_t *bin_info)
extent_hooks_t **r_extent_hooks, const arena_bin_info_t *bin_info)
{
extent_t *slab;
bool zero, commit;
@ -996,7 +996,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
zero = false;
commit = true;
malloc_mutex_unlock(tsdn, &arena->lock);
slab = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL,
slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL,
bin_info->slab_size, 0, PAGE, &zero, &commit, true);
malloc_mutex_lock(tsdn, &arena->lock);
@ -1009,7 +1009,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
{
extent_t *slab;
arena_slab_data_t *slab_data;
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER;
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
bool zero;
zero = false;
@ -1796,7 +1796,7 @@ arena_new(tsdn_t *tsdn, unsigned ind)
extent_heap_new(&arena->extents_retained[i]);
}
arena->extent_hooks = extent_hooks_default;
arena->extent_hooks = (extent_hooks_t *)&extent_hooks_default;
if (malloc_mutex_init(&arena->extents_mtx, "arena_extents",
WITNESS_RANK_ARENA_EXTENTS))

View File

@ -1646,15 +1646,15 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
if (arena_ind < narenas_total_get() && (arena =
arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
if (newp != NULL) {
extent_hooks_t old_extent_hooks, new_extent_hooks;
WRITE(new_extent_hooks, extent_hooks_t);
old_extent_hooks = extent_hooks_set(tsd_tsdn(tsd),
arena, &new_extent_hooks);
READ(old_extent_hooks, extent_hooks_t);
extent_hooks_t *old_extent_hooks, *new_extent_hooks;
WRITE(new_extent_hooks, extent_hooks_t *);
old_extent_hooks = extent_hooks_set(arena,
new_extent_hooks);
READ(old_extent_hooks, extent_hooks_t *);
} else {
extent_hooks_t old_extent_hooks =
extent_hooks_get(tsd_tsdn(tsd), arena);
READ(old_extent_hooks, extent_hooks_t);
extent_hooks_t *old_extent_hooks =
extent_hooks_get(arena);
READ(old_extent_hooks, extent_hooks_t *);
}
} else {
ret = EFAULT;

View File

@ -6,20 +6,23 @@
rtree_t extents_rtree;
static void *extent_alloc_default(void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
static bool extent_dalloc_default(void *addr, size_t size, bool committed,
static void *extent_alloc_default(extent_hooks_t *extent_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
unsigned arena_ind);
static bool extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, bool committed, unsigned arena_ind);
static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_decommit_default(extent_hooks_t *extent_hooks,
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_purge_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t size_a, size_t size_b, bool committed,
unsigned arena_ind);
static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
size_t size_a, void *addr_b, size_t size_b, bool committed,
unsigned arena_ind);
static bool extent_commit_default(void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind);
static bool extent_decommit_default(void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind);
static bool extent_purge_default(void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind);
static bool extent_split_default(void *addr, size_t size, size_t size_a,
size_t size_b, bool committed, unsigned arena_ind);
static bool extent_merge_default(void *addr_a, size_t size_a, void *addr_b,
size_t size_b, bool committed, unsigned arena_ind);
const extent_hooks_t extent_hooks_default = {
extent_alloc_default,
@ -42,7 +45,7 @@ static size_t highchunks;
*/
static void extent_record(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_heap_t extent_heaps[NPSIZES],
extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES],
bool cache, extent_t *extent);
/******************************************************************************/
@ -73,89 +76,34 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
}
static extent_hooks_t
extent_hooks_get_locked(arena_t *arena)
extent_hooks_t *
extent_hooks_get(arena_t *arena)
{
return (arena->extent_hooks);
return ((extent_hooks_t *)atomic_read_p(&arena->extent_hooks_pun));
}
extent_hooks_t
extent_hooks_get(tsdn_t *tsdn, arena_t *arena)
extent_hooks_t *
extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks)
{
extent_hooks_t extent_hooks;
extent_hooks_t *old_extent_hooks = extent_hooks_get(arena);
union {
extent_hooks_t **h;
void **v;
} u;
malloc_mutex_lock(tsdn, &arena->extents_mtx);
extent_hooks = extent_hooks_get_locked(arena);
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (extent_hooks);
}
extent_hooks_t
extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
const extent_hooks_t *extent_hooks)
{
extent_hooks_t old_extent_hooks;
malloc_mutex_lock(tsdn, &arena->extents_mtx);
old_extent_hooks = arena->extent_hooks;
/*
* Copy each field atomically so that it is impossible for readers to
* see partially updated pointers. There are places where readers only
* need one hook function pointer (therefore no need to copy the
* entirety of arena->extent_hooks), and stale reads do not affect
* correctness, so they perform unlocked reads.
*/
#define ATOMIC_COPY_HOOK(n) do { \
union { \
extent_##n##_t **n; \
void **v; \
} u; \
u.n = &arena->extent_hooks.n; \
atomic_write_p(u.v, extent_hooks->n); \
} while (0)
ATOMIC_COPY_HOOK(alloc);
ATOMIC_COPY_HOOK(dalloc);
ATOMIC_COPY_HOOK(commit);
ATOMIC_COPY_HOOK(decommit);
ATOMIC_COPY_HOOK(purge);
ATOMIC_COPY_HOOK(split);
ATOMIC_COPY_HOOK(merge);
#undef ATOMIC_COPY_HOOK
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
u.h = &arena->extent_hooks;
atomic_write_p(u.v, extent_hooks);
return (old_extent_hooks);
}
static void
extent_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, bool locked)
{
static const extent_hooks_t uninitialized_hooks =
EXTENT_HOOKS_INITIALIZER;
if (memcmp(extent_hooks, &uninitialized_hooks, sizeof(extent_hooks_t))
== 0) {
*extent_hooks = locked ? extent_hooks_get_locked(arena) :
extent_hooks_get(tsdn, arena);
}
}
static void
extent_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks)
extent_hooks_assure_initialized(arena_t *arena, extent_hooks_t **r_extent_hooks)
{
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, true);
}
static void
extent_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks)
{
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, false);
if (*r_extent_hooks == EXTENT_HOOKS_INITIALIZER)
*r_extent_hooks = extent_hooks_get(arena);
}
#ifdef JEMALLOC_JET
@ -409,7 +357,7 @@ extent_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES],
}
static void
extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
bool cache, extent_t *extent)
{
@ -418,14 +366,14 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
* that this is only a virtual memory leak.
*/
if (cache) {
extent_purge_wrapper(tsdn, arena, extent_hooks, extent, 0,
extent_purge_wrapper(tsdn, arena, r_extent_hooks, extent, 0,
extent_size_get(extent));
}
extent_dalloc(tsdn, arena, extent);
}
static extent_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr,
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
bool slab)
@ -444,7 +392,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
if (alloc_size < usize)
return (NULL);
malloc_mutex_lock(tsdn, &arena->extents_mtx);
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
extent_hooks_assure_initialized(arena, r_extent_hooks);
if (new_addr != NULL) {
rtree_elm_t *elm;
@ -482,10 +430,11 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
/* Split the lead. */
if (leadsize != 0) {
extent_t *lead = extent;
extent = extent_split_wrapper(tsdn, arena, extent_hooks, lead,
leadsize, leadsize, size + trailsize, usize + trailsize);
extent = extent_split_wrapper(tsdn, arena, r_extent_hooks,
lead, leadsize, leadsize, size + trailsize, usize +
trailsize);
if (extent == NULL) {
extent_leak(tsdn, arena, extent_hooks, cache, lead);
extent_leak(tsdn, arena, r_extent_hooks, cache, lead);
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (NULL);
}
@ -496,9 +445,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
/* Split the trail. */
if (trailsize != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena,
extent_hooks, extent, size, usize, trailsize, trailsize);
r_extent_hooks, extent, size, usize, trailsize, trailsize);
if (trail == NULL) {
extent_leak(tsdn, arena, extent_hooks, cache, extent);
extent_leak(tsdn, arena, r_extent_hooks, cache,
extent);
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
return (NULL);
}
@ -513,10 +463,10 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
}
if (!extent_committed_get(extent) &&
extent_hooks->commit(extent_base_get(extent),
(*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) {
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
extent_record(tsdn, arena, extent_hooks, extent_heaps, cache,
extent_record(tsdn, arena, r_extent_hooks, extent_heaps, cache,
extent);
return (NULL);
}
@ -582,9 +532,9 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
}
extent_t *
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
bool slab)
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab)
{
extent_t *extent;
bool commit;
@ -593,7 +543,7 @@ extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
assert(alignment != 0);
commit = true;
extent = extent_recycle(tsdn, arena, extent_hooks,
extent = extent_recycle(tsdn, arena, r_extent_hooks,
arena->extents_cached, true, new_addr, usize, pad, alignment, zero,
&commit, slab);
if (extent == NULL)
@ -603,13 +553,15 @@ extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
}
static void *
extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
bool *commit, unsigned arena_ind)
extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind)
{
void *ret;
tsdn_t *tsdn;
arena_t *arena;
assert(extent_hooks == &extent_hooks_default);
tsdn = tsdn_fetch();
arena = arena_get(tsdn, arena_ind, false);
/*
@ -627,7 +579,7 @@ extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
static extent_t *
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
extent_t *extent;
@ -635,7 +587,7 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
assert(usize != 0);
assert(alignment != 0);
extent = extent_recycle(tsdn, arena, extent_hooks,
extent = extent_recycle(tsdn, arena, r_extent_hooks,
arena->extents_retained, false, new_addr, usize, pad, alignment,
zero, commit, slab);
if (extent != NULL && config_stats) {
@ -648,7 +600,7 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
static extent_t *
extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
extent_t *extent;
@ -659,8 +611,8 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent = extent_alloc(tsdn, arena);
if (extent == NULL)
return (NULL);
addr = extent_hooks->alloc(new_addr, size, alignment, zero, commit,
arena->ind);
addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, size,
alignment, zero, commit, arena->ind);
if (addr == NULL) {
extent_dalloc(tsdn, arena, extent);
return (NULL);
@ -669,7 +621,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
if (pad != 0)
extent_addr_randomize(tsdn, extent, alignment);
if (extent_register(tsdn, extent)) {
extent_leak(tsdn, arena, extent_hooks, false, extent);
extent_leak(tsdn, arena, r_extent_hooks, false, extent);
return (NULL);
}
@ -677,18 +629,18 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
}
extent_t *
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
bool *commit, bool slab)
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab)
{
extent_t *extent;
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
extent_hooks_assure_initialized(arena, r_extent_hooks);
extent = extent_alloc_retained(tsdn, arena, extent_hooks, new_addr,
extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, new_addr,
usize, pad, alignment, zero, commit, slab);
if (extent == NULL) {
extent = extent_alloc_wrapper_hard(tsdn, arena, extent_hooks,
extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks,
new_addr, usize, pad, alignment, zero, commit, slab);
}
@ -712,8 +664,9 @@ extent_can_coalesce(const extent_t *a, const extent_t *b)
}
static void
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache)
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b,
extent_heap_t extent_heaps[NPSIZES], bool cache)
{
if (!extent_can_coalesce(a, b))
@ -725,7 +678,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
arena_extent_cache_maybe_remove(extent_arena_get(a), a, cache);
arena_extent_cache_maybe_remove(extent_arena_get(b), b, cache);
if (extent_merge_wrapper(tsdn, arena, extent_hooks, a, b)) {
if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) {
extent_heaps_insert(extent_heaps, a);
extent_heaps_insert(extent_heaps, b);
arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache);
@ -738,7 +691,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
}
static void
extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent)
{
extent_t *prev, *next;
@ -748,7 +701,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
assert(!cache || !extent_zeroed_get(extent));
malloc_mutex_lock(tsdn, &arena->extents_mtx);
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
extent_hooks_assure_initialized(arena, r_extent_hooks);
extent_usize_set(extent, 0);
extent_active_set(extent, false);
@ -766,7 +719,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
next = rtree_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_past_get(extent), false);
if (next != NULL) {
extent_try_coalesce(tsdn, arena, extent_hooks, extent, next,
extent_try_coalesce(tsdn, arena, r_extent_hooks, extent, next,
extent_heaps, cache);
}
@ -774,7 +727,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
prev = rtree_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)extent_before_get(extent), false);
if (prev != NULL) {
extent_try_coalesce(tsdn, arena, extent_hooks, prev, extent,
extent_try_coalesce(tsdn, arena, r_extent_hooks, prev, extent,
extent_heaps, cache);
}
@ -782,8 +735,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
}
void
extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent)
extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
assert(extent_base_get(extent) != NULL);
@ -792,15 +745,17 @@ extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false);
extent_record(tsdn, arena, extent_hooks, arena->extents_cached, true,
extent_record(tsdn, arena, r_extent_hooks, arena->extents_cached, true,
extent);
}
static bool
extent_dalloc_default(void *addr, size_t size, bool committed,
unsigned arena_ind)
extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind)
{
assert(extent_hooks == &extent_hooks_default);
if (!have_dss || !extent_in_dss(tsdn_fetch(), addr))
return (extent_dalloc_mmap(addr, size));
return (true);
@ -808,7 +763,7 @@ extent_dalloc_default(void *addr, size_t size, bool committed,
void
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent)
extent_hooks_t **r_extent_hooks, extent_t *extent)
{
assert(extent_base_get(extent) != NULL);
@ -816,9 +771,9 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_addr_set(extent, extent_base_get(extent));
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
extent_hooks_assure_initialized(arena, r_extent_hooks);
/* Try to deallocate. */
if (!extent_hooks->dalloc(extent_base_get(extent),
if (!(*r_extent_hooks)->dalloc(*r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), extent_committed_get(extent),
arena->ind)) {
extent_deregister(tsdn, extent);
@ -828,66 +783,73 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
/* Try to decommit; purge if that fails. */
if (extent_committed_get(extent)) {
extent_committed_set(extent,
extent_hooks->decommit(extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent),
arena->ind));
(*r_extent_hooks)->decommit(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena->ind));
}
extent_zeroed_set(extent, !extent_committed_get(extent) ||
!extent_hooks->purge(extent_base_get(extent),
!(*r_extent_hooks)->purge(*r_extent_hooks, extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
if (config_stats)
arena->stats.retained += extent_size_get(extent);
extent_record(tsdn, arena, extent_hooks, arena->extents_retained, false,
extent);
extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
false, extent);
}
static bool
extent_commit_default(void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind)
extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
assert(extent_hooks == &extent_hooks_default);
return (pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset),
length));
}
bool
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
return (extent_hooks->commit(extent_base_get(extent),
extent_size_get(extent), offset, length, arena->ind));
extent_hooks_assure_initialized(arena, r_extent_hooks);
return ((*r_extent_hooks)->commit(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
}
static bool
extent_decommit_default(void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind)
extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
assert(extent_hooks == &extent_hooks_default);
return (pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset),
length));
}
bool
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
return (extent_hooks->decommit(extent_base_get(extent),
extent_size_get(extent), offset, length, arena->ind));
extent_hooks_assure_initialized(arena, r_extent_hooks);
return ((*r_extent_hooks)->decommit(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
}
static bool
extent_purge_default(void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind)
extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
assert(extent_hooks == &extent_hooks_default);
assert(addr != NULL);
assert((offset & PAGE_MASK) == 0);
assert(length != 0);
@ -898,29 +860,33 @@ extent_purge_default(void *addr, size_t size, size_t offset, size_t length,
}
bool
extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent, size_t offset, size_t length)
extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length)
{
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
return (extent_hooks->purge(extent_base_get(extent),
extent_size_get(extent), offset, length, arena->ind));
extent_hooks_assure_initialized(arena, r_extent_hooks);
return ((*r_extent_hooks)->purge(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena->ind));
}
static bool
extent_split_default(void *addr, size_t size, size_t size_a, size_t size_b,
bool committed, unsigned arena_ind)
extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
{
assert(extent_hooks == &extent_hooks_default);
if (!maps_coalesce)
return (true);
return (false);
}
extent_t *
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent, size_t size_a, size_t usize_a, size_t size_b,
size_t usize_b)
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
size_t usize_a, size_t size_b, size_t usize_b)
{
extent_t *trail;
rtree_ctx_t rtree_ctx_fallback;
@ -929,7 +895,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
assert(extent_size_get(extent) == size_a + size_b);
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
extent_hooks_assure_initialized(arena, r_extent_hooks);
trail = extent_alloc(tsdn, arena);
if (trail == NULL)
@ -956,8 +922,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
&trail_elm_a, &trail_elm_b))
goto label_error_c;
if (extent_hooks->split(extent_base_get(extent), size_a + size_b,
size_a, size_b, extent_committed_get(extent), arena->ind))
if ((*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent),
size_a + size_b, size_a, size_b, extent_committed_get(extent),
arena->ind))
goto label_error_d;
extent_size_set(extent, size_a);
@ -981,10 +948,12 @@ label_error_a:
}
static bool
extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
bool committed, unsigned arena_ind)
extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
{
assert(extent_hooks == &extent_hooks_default);
if (!maps_coalesce)
return (true);
if (have_dss) {
@ -997,17 +966,17 @@ extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
}
bool
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *a, extent_t *b)
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b)
{
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
if (extent_hooks->merge(extent_base_get(a), extent_size_get(a),
extent_base_get(b), extent_size_get(b), extent_committed_get(a),
arena->ind))
extent_hooks_assure_initialized(arena, r_extent_hooks);
if ((*r_extent_hooks)->merge(*r_extent_hooks, extent_base_get(a),
extent_size_get(a), extent_base_get(b), extent_size_get(b),
extent_committed_get(a), arena->ind))
return (true);
/*

View File

@ -136,7 +136,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
dss_max = dss_next;
malloc_mutex_unlock(tsdn, &dss_mtx);
if (pad_size != 0) {
extent_hooks_t extent_hooks =
extent_hooks_t *extent_hooks =
EXTENT_HOOKS_INITIALIZER;
extent_dalloc_wrapper(tsdn, arena,
&extent_hooks, pad);

View File

@ -96,7 +96,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
{
arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent);
extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena);
extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t diff = extent_size_get(extent) - (usize + large_pad);
assert(oldusize > usize);
@ -129,7 +129,7 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent);
bool is_zeroed_trail = false;
extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena);
extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t trailsize = usize - extent_usize_get(extent);
extent_t *trail;

View File

@ -4,8 +4,8 @@
const char *malloc_conf = "junk:false";
#endif
static extent_hooks_t orig_hooks;
static extent_hooks_t old_hooks;
static extent_hooks_t *orig_hooks;
static extent_hooks_t *old_hooks;
static bool do_dalloc = true;
static bool do_decommit;
@ -24,96 +24,111 @@ static bool did_merge;
# define TRACE_HOOK(fmt, ...)
#endif
void *
extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
bool *commit, unsigned arena_ind)
static void *
extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind)
{
TRACE_HOOK("%s(new_addr=%p, size=%zu, alignment=%zu, *zero=%s, "
"*commit=%s, arena_ind=%u)\n", __func__, new_addr, size, alignment,
*zero ? "true" : "false", *commit ? "true" : "false", arena_ind);
TRACE_HOOK("%s(extent_hooks=%p, new_addr=%p, size=%zu, alignment=%zu, "
"*zero=%s, *commit=%s, arena_ind=%u)\n", __func__, extent_hooks,
new_addr, size, alignment, *zero ? "true" : "false", *commit ?
"true" : "false", arena_ind);
assert(extent_hooks->alloc == extent_alloc);
did_alloc = true;
return (old_hooks.alloc(new_addr, size, alignment, zero, commit,
arena_ind));
return (old_hooks->alloc(old_hooks, new_addr, size, alignment, zero,
commit, arena_ind));
}
bool
extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind)
static bool
extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind)
{
TRACE_HOOK("%s(addr=%p, size=%zu, committed=%s, arena_ind=%u)\n",
__func__, addr, size, committed ? "true" : "false", arena_ind);
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, committed=%s, "
"arena_ind=%u)\n", __func__, extent_hooks, addr, size, committed ?
"true" : "false", arena_ind);
assert(extent_hooks->dalloc == extent_dalloc);
did_dalloc = true;
if (!do_dalloc)
return (true);
return (old_hooks.dalloc(addr, size, committed, arena_ind));
return (old_hooks->dalloc(old_hooks, addr, size, committed, arena_ind));
}
bool
extent_commit(void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind)
static bool
extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
bool err;
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, "
"arena_ind=%u)\n", __func__, addr, size, offset, length,
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size,
offset, length, arena_ind);
assert(extent_hooks->commit == extent_commit);
err = old_hooks->commit(old_hooks, addr, size, offset, length,
arena_ind);
err = old_hooks.commit(addr, size, offset, length, arena_ind);
did_commit = !err;
return (err);
}
bool
extent_decommit(void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind)
static bool
extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
bool err;
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, "
"arena_ind=%u)\n", __func__, addr, size, offset, length,
arena_ind);
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size,
offset, length, arena_ind);
assert(extent_hooks->decommit == extent_decommit);
if (!do_decommit)
return (true);
err = old_hooks.decommit(addr, size, offset, length, arena_ind);
err = old_hooks->decommit(old_hooks, addr, size, offset, length,
arena_ind);
did_decommit = !err;
return (err);
}
bool
extent_purge(void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind)
static bool
extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu "
"arena_ind=%u)\n", __func__, addr, size, offset, length,
arena_ind);
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
offset, length, arena_ind);
assert(extent_hooks->purge == extent_purge);
did_purge = true;
return (old_hooks.purge(addr, size, offset, length, arena_ind));
}
bool
extent_split(void *addr, size_t size, size_t size_a, size_t size_b,
bool committed, unsigned arena_ind)
{
TRACE_HOOK("%s(addr=%p, size=%zu, size_a=%zu, size_b=%zu, "
"committed=%s, arena_ind=%u)\n", __func__, addr, size, size_a,
size_b, committed ? "true" : "false", arena_ind);
did_split = true;
return (old_hooks.split(addr, size, size_a, size_b, committed,
return (old_hooks->purge(old_hooks, addr, size, offset, length,
arena_ind));
}
bool
extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
bool committed, unsigned arena_ind)
static bool
extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
{
TRACE_HOOK("%s(addr_a=%p, size_a=%zu, addr_b=%p size_b=%zu, "
"committed=%s, arena_ind=%u)\n", __func__, addr_a, size_a, addr_b,
size_b, committed ? "true" : "false", arena_ind);
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, size_a=%zu, "
"size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks,
addr, size, size_a, size_b, committed ? "true" : "false",
arena_ind);
assert(extent_hooks->split == extent_split);
did_split = true;
return (old_hooks->split(old_hooks, addr, size, size_a, size_b,
committed, arena_ind));
}
static bool
extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
{
TRACE_HOOK("%s(extent_hooks=%p, addr_a=%p, size_a=%zu, addr_b=%p "
"size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks,
addr_a, size_a, addr_b, size_b, committed ? "true" : "false",
arena_ind);
assert(extent_hooks->merge == extent_merge);
did_merge = true;
return (old_hooks.merge(addr_a, size_a, addr_b, size_b,
return (old_hooks->merge(old_hooks, addr_a, size_a, addr_b, size_b,
committed, arena_ind));
}
@ -125,7 +140,7 @@ TEST_BEGIN(test_extent)
int flags;
size_t hooks_mib[3], purge_mib[3];
size_t hooks_miblen, purge_miblen;
extent_hooks_t new_hooks = {
extent_hooks_t hooks = {
extent_alloc,
extent_dalloc,
extent_commit,
@ -134,6 +149,7 @@ TEST_BEGIN(test_extent)
extent_split,
extent_merge
};
extent_hooks_t *new_hooks = &hooks;
bool xallocx_success_a, xallocx_success_b, xallocx_success_c;
sz = sizeof(unsigned);
@ -146,21 +162,21 @@ TEST_BEGIN(test_extent)
assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib,
&hooks_miblen), 0, "Unexpected mallctlnametomib() failure");
hooks_mib[1] = (size_t)arena_ind;
old_size = sizeof(extent_hooks_t);
new_size = sizeof(extent_hooks_t);
old_size = sizeof(extent_hooks_t *);
new_size = sizeof(extent_hooks_t *);
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
&new_hooks, new_size), 0, "Unexpected extent_hooks error");
orig_hooks = old_hooks;
assert_ptr_ne(old_hooks.alloc, extent_alloc, "Unexpected alloc error");
assert_ptr_ne(old_hooks.dalloc, extent_dalloc,
assert_ptr_ne(old_hooks->alloc, extent_alloc, "Unexpected alloc error");
assert_ptr_ne(old_hooks->dalloc, extent_dalloc,
"Unexpected dalloc error");
assert_ptr_ne(old_hooks.commit, extent_commit,
assert_ptr_ne(old_hooks->commit, extent_commit,
"Unexpected commit error");
assert_ptr_ne(old_hooks.decommit, extent_decommit,
assert_ptr_ne(old_hooks->decommit, extent_decommit,
"Unexpected decommit error");
assert_ptr_ne(old_hooks.purge, extent_purge, "Unexpected purge error");
assert_ptr_ne(old_hooks.split, extent_split, "Unexpected split error");
assert_ptr_ne(old_hooks.merge, extent_merge, "Unexpected merge error");
assert_ptr_ne(old_hooks->purge, extent_purge, "Unexpected purge error");
assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error");
assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error");
/* Get large size classes. */
sz = sizeof(size_t);
@ -228,19 +244,20 @@ TEST_BEGIN(test_extent)
&old_hooks, new_size), 0, "Unexpected extent_hooks error");
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
NULL, 0), 0, "Unexpected extent_hooks error");
assert_ptr_eq(old_hooks.alloc, orig_hooks.alloc,
assert_ptr_eq(old_hooks, orig_hooks, "Unexpected hooks error");
assert_ptr_eq(old_hooks->alloc, orig_hooks->alloc,
"Unexpected alloc error");
assert_ptr_eq(old_hooks.dalloc, orig_hooks.dalloc,
assert_ptr_eq(old_hooks->dalloc, orig_hooks->dalloc,
"Unexpected dalloc error");
assert_ptr_eq(old_hooks.commit, orig_hooks.commit,
assert_ptr_eq(old_hooks->commit, orig_hooks->commit,
"Unexpected commit error");
assert_ptr_eq(old_hooks.decommit, orig_hooks.decommit,
assert_ptr_eq(old_hooks->decommit, orig_hooks->decommit,
"Unexpected decommit error");
assert_ptr_eq(old_hooks.purge, orig_hooks.purge,
assert_ptr_eq(old_hooks->purge, orig_hooks->purge,
"Unexpected purge error");
assert_ptr_eq(old_hooks.split, orig_hooks.split,
assert_ptr_eq(old_hooks->split, orig_hooks->split,
"Unexpected split error");
assert_ptr_eq(old_hooks.merge, orig_hooks.merge,
assert_ptr_eq(old_hooks->merge, orig_hooks->merge,
"Unexpected merge error");
}
TEST_END