project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

s/chunk_hook/extent_hook/g

Browse Source
This commit is contained in:
Jason Evans 2016-05-31 15:03:51 -07:00
parent 7d63fed0fd
commit 9c305c9e5c
12 changed files with 200 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
doc/jemalloc.xml.in
View File

@ -555,7 +555,7 @@ for (i = 0; i < nbins; i++) {
allocations in place, as long as the pre-size and post-size are both large. allocations in place, as long as the pre-size and post-size are both large.
For shrinkage to succeed, the extent allocator must support splitting (see For shrinkage to succeed, the extent allocator must support splitting (see
<link <link
linkend="arena.i.chunk_hooks"><mallctl>arena.&lt;i&gt;.chunk_hooks</mallctl></link>). linkend="arena.i.extent_hooks"><mallctl>arena.&lt;i&gt;.extent_hooks</mallctl></link>).
Growth only succeeds if the trailing memory is currently available, and the Growth only succeeds if the trailing memory is currently available, and the
extent allocator supports merging.</para> extent allocator supports merging.</para>
@ -1548,10 +1548,10 @@ malloc_conf = "xmalloc:true";]]></programlisting>
additional information.</para></listitem> additional information.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arena.i.chunk_hooks"> <varlistentry id="arena.i.extent_hooks">
<term> <term>
<mallctl>arena.&lt;i&gt;.chunk_hooks</mallctl> <mallctl>arena.&lt;i&gt;.extent_hooks</mallctl>
(<type>chunk_hooks_t</type>) (<type>extent_hooks_t</type>)
<literal>rw</literal> <literal>rw</literal>
</term> </term>
<listitem><para>Get or set the chunk management hook functions for arena <listitem><para>Get or set the chunk management hook functions for arena
@ -1561,7 +1561,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
control allocation for arenas created via <link control allocation for arenas created via <link
linkend="arenas.extend"><mallctl>arenas.extend</mallctl></link> such linkend="arenas.extend"><mallctl>arenas.extend</mallctl></link> such
that all chunks originate from an application-supplied chunk allocator that all chunks originate from an application-supplied chunk allocator
(by setting custom chunk hook functions just after arena creation), but (by setting custom extent hook functions just after arena creation), but
the automatically created arenas may have already created chunks prior the automatically created arenas may have already created chunks prior
to the application having an opportunity to take over chunk to the application having an opportunity to take over chunk
allocation.</para> allocation.</para>
@ -1575,8 +1575,8 @@ typedef struct {
chunk_purge_t *purge; chunk_purge_t *purge;
chunk_split_t *split; chunk_split_t *split;
chunk_merge_t *merge; chunk_merge_t *merge;
} chunk_hooks_t;]]></programlisting> } extent_hooks_t;]]></programlisting>
<para>The <type>chunk_hooks_t</type> structure comprises function <para>The <type>extent_hooks_t</type> structure comprises function
pointers which are described individually below. jemalloc uses these pointers which are described individually below. jemalloc uses these
functions to manage chunk lifetime, which starts off with allocation of functions to manage chunk lifetime, which starts off with allocation of
mapped committed memory, in the simplest case followed by deallocation. mapped committed memory, in the simplest case followed by deallocation.
@ -2109,8 +2109,8 @@ typedef struct {
<manvolnum>2</manvolnum></citerefentry>. Retained virtual memory is <manvolnum>2</manvolnum></citerefentry>. Retained virtual memory is
typically untouched, decommitted, or purged, so it has no strongly typically untouched, decommitted, or purged, so it has no strongly
associated physical memory (see <link associated physical memory (see <link
linkend="arena.i.chunk_hooks">chunk hooks</link> for details). Retained linkend="arena.i.extent_hooks">extent hooks</link> for details).
memory is excluded from mapped memory statistics, e.g. <link Retained memory is excluded from mapped memory statistics, e.g. <link
linkend="stats.mapped"><mallctl>stats.mapped</mallctl></link>. linkend="stats.mapped"><mallctl>stats.mapped</mallctl></link>.
</para></listitem> </para></listitem>
</varlistentry> </varlistentry>

8
include/jemalloc/internal/arena.h
View File

@ -246,8 +246,8 @@ struct arena_s {
ql_head(extent_t) extent_cache; ql_head(extent_t) extent_cache;
malloc_mutex_t extent_cache_mtx; malloc_mutex_t extent_cache_mtx;
/* User-configurable chunk hook functions. */ /* User-configurable extent hook functions. */
chunk_hooks_t chunk_hooks; extent_hooks_t extent_hooks;
/* bins is used to store heaps of free regions. */ /* bins is used to store heaps of free regions. */
arena_bin_t bins[NBINS]; arena_bin_t bins[NBINS];
@ -279,10 +279,10 @@ extern ssize_t opt_decay_time;
extern const arena_bin_info_t arena_bin_info[NBINS]; extern const arena_bin_info_t arena_bin_info[NBINS];
extent_t *arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena, extent_t *arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero); bool *zero);
void arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena, void arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent); extent_hooks_t *extent_hooks, extent_t *extent);
void arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent, void arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent,
bool cache); bool cache);
void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent, void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent,

31
include/jemalloc/internal/chunk.h
View File

@ -38,33 +38,36 @@ extern size_t chunksize;
extern size_t chunksize_mask; /* (chunksize - 1). */ extern size_t chunksize_mask; /* (chunksize - 1). */
extern size_t chunk_npages; extern size_t chunk_npages;
extern const chunk_hooks_t chunk_hooks_default; extern const extent_hooks_t extent_hooks_default;
chunk_hooks_t chunk_hooks_get(tsdn_t *tsdn, arena_t *arena); extent_hooks_t extent_hooks_get(tsdn_t *tsdn, arena_t *arena);
chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena, extent_hooks_t extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
const chunk_hooks_t *chunk_hooks); const extent_hooks_t *extent_hooks);
extent_t *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_t *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab); size_t alignment, bool *zero, bool slab);
extent_t *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab); size_t alignment, bool *zero, bool *commit, bool slab);
void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent); extent_hooks_t *extent_hooks, extent_t *extent);
void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent); extent_hooks_t *extent_hooks, extent_t *extent);
bool chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena, bool chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent, size_t offset, size_t length); extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, bool chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent, size_t offset, size_t length); extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent, size_t offset, size_t length); extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length);
extent_t *chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent, size_t size_a, size_t usize_a, extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
size_t size_b, size_t usize_b); size_t usize_a, size_t size_b, size_t usize_b);
bool chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, bool chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *a, extent_t *b); extent_hooks_t *extent_hooks, extent_t *a, extent_t *b);
bool chunk_boot(void); bool chunk_boot(void);
void chunk_prefork(tsdn_t *tsdn); void chunk_prefork(tsdn_t *tsdn);
void chunk_postfork_parent(tsdn_t *tsdn); void chunk_postfork_parent(tsdn_t *tsdn);

6
include/jemalloc/internal/private_symbols.txt
View File

@ -131,9 +131,6 @@ chunk_dss_postfork_parent
chunk_dss_prec_get chunk_dss_prec_get
chunk_dss_prec_set chunk_dss_prec_set
chunk_dss_prefork chunk_dss_prefork
chunk_hooks_default
chunk_hooks_get
chunk_hooks_set
chunk_in_dss chunk_in_dss
chunk_lookup chunk_lookup
chunk_merge_wrapper chunk_merge_wrapper
@ -185,6 +182,9 @@ extent_heap_insert
extent_heap_new extent_heap_new
extent_heap_remove extent_heap_remove
extent_heap_remove_first extent_heap_remove_first
extent_hooks_default
extent_hooks_get
extent_hooks_set
extent_init extent_init
extent_last_get extent_last_get
extent_past_get extent_past_get

2
include/jemalloc/jemalloc_typedefs.h.in
View File

@ -54,4 +54,4 @@ typedef struct {
chunk_purge_t *purge; chunk_purge_t *purge;
chunk_split_t *split; chunk_split_t *split;
chunk_merge_t *merge; chunk_merge_t *merge;
} chunk_hooks_t; } extent_hooks_t;

66
src/arena.c
View File

@ -54,25 +54,25 @@ arena_chunk_dirty_npages(const extent_t *extent)
static extent_t * static extent_t *
arena_chunk_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena, arena_chunk_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab) size_t alignment, bool *zero, bool slab)
{ {
malloc_mutex_assert_owner(tsdn, &arena->lock); malloc_mutex_assert_owner(tsdn, &arena->lock);
return (chunk_alloc_cache(tsdn, arena, chunk_hooks, new_addr, usize, return (chunk_alloc_cache(tsdn, arena, extent_hooks, new_addr, usize,
pad, alignment, zero, slab)); pad, alignment, zero, slab));
} }
extent_t * extent_t *
arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero) bool *zero)
{ {
extent_t *extent; extent_t *extent;
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
extent = arena_chunk_cache_alloc_locked(tsdn, arena, chunk_hooks, extent = arena_chunk_cache_alloc_locked(tsdn, arena, extent_hooks,
new_addr, size, 0, alignment, zero, false); new_addr, size, 0, alignment, zero, false);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
@ -81,22 +81,22 @@ arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
static void static void
arena_chunk_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena, arena_chunk_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent) extent_hooks_t *extent_hooks, extent_t *extent)
{ {
malloc_mutex_assert_owner(tsdn, &arena->lock); malloc_mutex_assert_owner(tsdn, &arena->lock);
chunk_dalloc_cache(tsdn, arena, chunk_hooks, extent); chunk_dalloc_cache(tsdn, arena, extent_hooks, extent);
arena_maybe_purge(tsdn, arena); arena_maybe_purge(tsdn, arena);
} }
void void
arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent) extent_hooks_t *extent_hooks, extent_t *extent)
{ {
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
arena_chunk_cache_dalloc_locked(tsdn, arena, chunk_hooks, extent); arena_chunk_cache_dalloc_locked(tsdn, arena, extent_hooks, extent);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
} }
@ -321,12 +321,12 @@ arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)
static extent_t * static extent_t *
arena_chunk_alloc_large_hard(tsdn_t *tsdn, arena_t *arena, arena_chunk_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero) extent_hooks_t *extent_hooks, size_t usize, size_t alignment, bool *zero)
{ {
extent_t *extent; extent_t *extent;
bool commit = true; bool commit = true;
extent = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, usize, extent = chunk_alloc_wrapper(tsdn, arena, extent_hooks, NULL, usize,
large_pad, alignment, zero, &commit, false); large_pad, alignment, zero, &commit, false);
if (extent == NULL) { if (extent == NULL) {
/* Revert optimistic stats updates. */ /* Revert optimistic stats updates. */
@ -347,7 +347,7 @@ arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero) size_t alignment, bool *zero)
{ {
extent_t *extent; extent_t *extent;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; extent_hooks_t extent_hooks = CHUNK_HOOKS_INITIALIZER;
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
@ -358,12 +358,12 @@ arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
} }
arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE); arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE);
extent = arena_chunk_cache_alloc_locked(tsdn, arena, &chunk_hooks, NULL, extent = arena_chunk_cache_alloc_locked(tsdn, arena, &extent_hooks,
usize, large_pad, alignment, zero, false); NULL, usize, large_pad, alignment, zero, false);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
if (extent == NULL) { if (extent == NULL) {
extent = arena_chunk_alloc_large_hard(tsdn, arena, &chunk_hooks, extent = arena_chunk_alloc_large_hard(tsdn, arena,
usize, alignment, zero); &extent_hooks, usize, alignment, zero);
} }
return (extent); return (extent);
@ -373,7 +373,7 @@ void
arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent, arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool locked) bool locked)
{ {
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; extent_hooks_t extent_hooks = CHUNK_HOOKS_INITIALIZER;
if (!locked) if (!locked)
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
@ -384,7 +384,7 @@ arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
} }
arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE); arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE);
arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, extent); arena_chunk_cache_dalloc_locked(tsdn, arena, &extent_hooks, extent);
if (!locked) if (!locked)
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
} }
@ -735,7 +735,7 @@ arena_dirty_count(tsdn_t *tsdn, arena_t *arena)
} }
static size_t static size_t
arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
size_t ndirty_limit, extent_t *purge_extents_sentinel) size_t ndirty_limit, extent_t *purge_extents_sentinel)
{ {
extent_t *extent, *next; extent_t *extent, *next;
@ -757,7 +757,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
/* Allocate. */ /* Allocate. */
zero = false; zero = false;
textent = arena_chunk_cache_alloc_locked(tsdn, arena, textent = arena_chunk_cache_alloc_locked(tsdn, arena,
chunk_hooks, extent_base_get(extent), extent_hooks, extent_base_get(extent),
extent_size_get(extent), 0, CACHELINE, &zero, false); extent_size_get(extent), 0, CACHELINE, &zero, false);
assert(textent == extent); assert(textent == extent);
assert(zero == extent_zeroed_get(extent)); assert(zero == extent_zeroed_get(extent));
@ -774,7 +774,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
} }
static size_t static size_t
arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *purge_extents_sentinel) extent_t *purge_extents_sentinel)
{ {
UNUSED size_t nmadvise; UNUSED size_t nmadvise;
@ -793,7 +793,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
next = qr_next(extent, qr_link); next = qr_next(extent, qr_link);
extent_ring_remove(extent); extent_ring_remove(extent);
chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, extent); chunk_dalloc_wrapper(tsdn, arena, extent_hooks, extent);
} }
if (config_stats) { if (config_stats) {
@ -816,7 +816,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
static void static void
arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit)
{ {
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena);
size_t npurge, npurged; size_t npurge, npurged;
extent_t purge_extents_sentinel; extent_t purge_extents_sentinel;
@ -836,11 +836,11 @@ arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit)
extent_init(&purge_extents_sentinel, arena, NULL, 0, 0, false, false, extent_init(&purge_extents_sentinel, arena, NULL, 0, 0, false, false,
false, false); false, false);
npurge = arena_stash_dirty(tsdn, arena, &chunk_hooks, ndirty_limit, npurge = arena_stash_dirty(tsdn, arena, &extent_hooks, ndirty_limit,
&purge_extents_sentinel); &purge_extents_sentinel);
if (npurge == 0) if (npurge == 0)
goto label_return; goto label_return;
npurged = arena_purge_stashed(tsdn, arena, &chunk_hooks, npurged = arena_purge_stashed(tsdn, arena, &extent_hooks,
&purge_extents_sentinel); &purge_extents_sentinel);
assert(npurged == npurge); assert(npurged == npurge);
@ -866,10 +866,10 @@ arena_purge(tsdn_t *tsdn, arena_t *arena, bool all)
static void static void
arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab)
{ {
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; extent_hooks_t extent_hooks = CHUNK_HOOKS_INITIALIZER;
arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, slab); arena_chunk_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab);
} }
void void
@ -987,8 +987,8 @@ arena_bin_slabs_full_remove(extent_t *slab)
} }
static extent_t * static extent_t *
arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
const arena_bin_info_t *bin_info) extent_hooks_t *extent_hooks, const arena_bin_info_t *bin_info)
{ {
extent_t *slab; extent_t *slab;
bool zero, commit; bool zero, commit;
@ -996,7 +996,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
zero = false; zero = false;
commit = true; commit = true;
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
slab = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, slab = chunk_alloc_wrapper(tsdn, arena, extent_hooks, NULL,
bin_info->slab_size, 0, PAGE, &zero, &commit, true); bin_info->slab_size, 0, PAGE, &zero, &commit, true);
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
@ -1009,14 +1009,14 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
{ {
extent_t *slab; extent_t *slab;
arena_slab_data_t *slab_data; arena_slab_data_t *slab_data;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; extent_hooks_t extent_hooks = CHUNK_HOOKS_INITIALIZER;
bool zero; bool zero;
zero = false; zero = false;
slab = arena_chunk_cache_alloc_locked(tsdn, arena, &chunk_hooks, NULL, slab = arena_chunk_cache_alloc_locked(tsdn, arena, &extent_hooks, NULL,
bin_info->slab_size, 0, PAGE, &zero, true); bin_info->slab_size, 0, PAGE, &zero, true);
if (slab == NULL) { if (slab == NULL) {
slab = arena_slab_alloc_hard(tsdn, arena, &chunk_hooks, slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks,
bin_info); bin_info);
if (slab == NULL) if (slab == NULL)
return (NULL); return (NULL);
@ -1805,7 +1805,7 @@ arena_new(tsdn_t *tsdn, unsigned ind)
WITNESS_RANK_ARENA_EXTENT_CACHE)) WITNESS_RANK_ARENA_EXTENT_CACHE))
return (NULL); return (NULL);
arena->chunk_hooks = chunk_hooks_default; arena->extent_hooks = extent_hooks_default;
/* Initialize bins. */ /* Initialize bins. */
for (i = 0; i < NBINS; i++) { for (i = 0; i < NBINS; i++) {

175
src/chunk.c
View File

@ -33,7 +33,7 @@ static bool chunk_split_default(void *chunk, size_t size, size_t size_a,
static bool chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, static bool chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b,
size_t size_b, bool committed, unsigned arena_ind); size_t size_b, bool committed, unsigned arena_ind);
const chunk_hooks_t chunk_hooks_default = { const extent_hooks_t extent_hooks_default = {
chunk_alloc_default, chunk_alloc_default,
chunk_dalloc_default, chunk_dalloc_default,
chunk_commit_default, chunk_commit_default,
@ -50,8 +50,8 @@ const chunk_hooks_t chunk_hooks_default = {
*/ */
static void chunk_record(tsdn_t *tsdn, arena_t *arena, static void chunk_record(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_heap_t extent_heaps[NPSIZES], bool cache, extent_hooks_t *extent_hooks, extent_heap_t extent_heaps[NPSIZES],
extent_t *extent); bool cache, extent_t *extent);
/******************************************************************************/ /******************************************************************************/
@ -71,37 +71,38 @@ extent_heaps_remove(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
extent_heap_remove(&extent_heaps[pind], extent); extent_heap_remove(&extent_heaps[pind], extent);
} }
static chunk_hooks_t static extent_hooks_t
chunk_hooks_get_locked(arena_t *arena) extent_hooks_get_locked(arena_t *arena)
{ {
return (arena->chunk_hooks); return (arena->extent_hooks);
} }
chunk_hooks_t extent_hooks_t
chunk_hooks_get(tsdn_t *tsdn, arena_t *arena) extent_hooks_get(tsdn_t *tsdn, arena_t *arena)
{ {
chunk_hooks_t chunk_hooks; extent_hooks_t extent_hooks;
malloc_mutex_lock(tsdn, &arena->chunks_mtx); malloc_mutex_lock(tsdn, &arena->chunks_mtx);
chunk_hooks = chunk_hooks_get_locked(arena); extent_hooks = extent_hooks_get_locked(arena);
malloc_mutex_unlock(tsdn, &arena->chunks_mtx); malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
return (chunk_hooks); return (extent_hooks);
} }
chunk_hooks_t extent_hooks_t
chunk_hooks_set(tsdn_t *tsdn, arena_t *arena, const chunk_hooks_t *chunk_hooks) extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
const extent_hooks_t *extent_hooks)
{ {
chunk_hooks_t old_chunk_hooks; extent_hooks_t old_extent_hooks;
malloc_mutex_lock(tsdn, &arena->chunks_mtx); malloc_mutex_lock(tsdn, &arena->chunks_mtx);
old_chunk_hooks = arena->chunk_hooks; old_extent_hooks = arena->extent_hooks;
/* /*
* Copy each field atomically so that it is impossible for readers to * Copy each field atomically so that it is impossible for readers to
* see partially updated pointers. There are places where readers only * see partially updated pointers. There are places where readers only
* need one hook function pointer (therefore no need to copy the * need one hook function pointer (therefore no need to copy the
* entirety of arena->chunk_hooks), and stale reads do not affect * entirety of arena->extent_hooks), and stale reads do not affect
* correctness, so they perform unlocked reads. * correctness, so they perform unlocked reads.
*/ */
#define ATOMIC_COPY_HOOK(n) do { \ #define ATOMIC_COPY_HOOK(n) do { \
@ -109,8 +110,8 @@ chunk_hooks_set(tsdn_t *tsdn, arena_t *arena, const chunk_hooks_t *chunk_hooks)
chunk_##n##_t **n; \ chunk_##n##_t **n; \
void **v; \ void **v; \
} u; \ } u; \
u.n = &arena->chunk_hooks.n; \ u.n = &arena->extent_hooks.n; \
atomic_write_p(u.v, chunk_hooks->n); \ atomic_write_p(u.v, extent_hooks->n); \
} while (0) } while (0)
ATOMIC_COPY_HOOK(alloc); ATOMIC_COPY_HOOK(alloc);
ATOMIC_COPY_HOOK(dalloc); ATOMIC_COPY_HOOK(dalloc);
@ -122,37 +123,37 @@ chunk_hooks_set(tsdn_t *tsdn, arena_t *arena, const chunk_hooks_t *chunk_hooks)
#undef ATOMIC_COPY_HOOK #undef ATOMIC_COPY_HOOK
malloc_mutex_unlock(tsdn, &arena->chunks_mtx); malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
return (old_chunk_hooks); return (old_extent_hooks);
} }
static void static void
chunk_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, bool locked) extent_hooks_t *extent_hooks, bool locked)
{ {
static const chunk_hooks_t uninitialized_hooks = static const extent_hooks_t uninitialized_hooks =
CHUNK_HOOKS_INITIALIZER; CHUNK_HOOKS_INITIALIZER;
if (memcmp(chunk_hooks, &uninitialized_hooks, sizeof(chunk_hooks_t)) == if (memcmp(extent_hooks, &uninitialized_hooks, sizeof(extent_hooks_t))
0) { == 0) {
*chunk_hooks = locked ? chunk_hooks_get_locked(arena) : *extent_hooks = locked ? extent_hooks_get_locked(arena) :
chunk_hooks_get(tsdn, arena); extent_hooks_get(tsdn, arena);
} }
} }
static void static void
chunk_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks) extent_hooks_t *extent_hooks)
{ {
chunk_hooks_assure_initialized_impl(tsdn, arena, chunk_hooks, true); extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, true);
} }
static void static void
chunk_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks) extent_hooks_t *extent_hooks)
{ {
chunk_hooks_assure_initialized_impl(tsdn, arena, chunk_hooks, false); extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, false);
} }
static bool static bool
@ -297,8 +298,8 @@ chunk_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES],
} }
static void static void
chunk_leak(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, bool cache, chunk_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent) bool cache, extent_t *extent)
{ {
/* /*
@ -306,14 +307,14 @@ chunk_leak(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, bool cache,
* that this is only a virtual memory leak. * that this is only a virtual memory leak.
*/ */
if (cache) { if (cache) {
chunk_purge_wrapper(tsdn, arena, chunk_hooks, extent, 0, chunk_purge_wrapper(tsdn, arena, extent_hooks, extent, 0,
extent_size_get(extent)); extent_size_get(extent));
} }
extent_dalloc(tsdn, arena, extent); extent_dalloc(tsdn, arena, extent);
} }
static extent_t * static extent_t *
chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr, extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr,
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
bool slab) bool slab)
@ -330,7 +331,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
if (alloc_size < usize) if (alloc_size < usize)
return (NULL); return (NULL);
malloc_mutex_lock(tsdn, &arena->chunks_mtx); malloc_mutex_lock(tsdn, &arena->chunks_mtx);
chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
if (new_addr != NULL) { if (new_addr != NULL) {
rtree_elm_t *elm; rtree_elm_t *elm;
@ -368,10 +369,10 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
/* Split the lead. */ /* Split the lead. */
if (leadsize != 0) { if (leadsize != 0) {
extent_t *lead = extent; extent_t *lead = extent;
extent = chunk_split_wrapper(tsdn, arena, chunk_hooks, lead, extent = chunk_split_wrapper(tsdn, arena, extent_hooks, lead,
leadsize, leadsize, size + trailsize, usize + trailsize); leadsize, leadsize, size + trailsize, usize + trailsize);
if (extent == NULL) { if (extent == NULL) {
chunk_leak(tsdn, arena, chunk_hooks, cache, lead); chunk_leak(tsdn, arena, extent_hooks, cache, lead);
malloc_mutex_unlock(tsdn, &arena->chunks_mtx); malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
return (NULL); return (NULL);
} }
@ -381,10 +382,10 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
/* Split the trail. */ /* Split the trail. */
if (trailsize != 0) { if (trailsize != 0) {
extent_t *trail = chunk_split_wrapper(tsdn, arena, chunk_hooks, extent_t *trail = chunk_split_wrapper(tsdn, arena, extent_hooks,
extent, size, usize, trailsize, trailsize); extent, size, usize, trailsize, trailsize);
if (trail == NULL) { if (trail == NULL) {
chunk_leak(tsdn, arena, chunk_hooks, cache, extent); chunk_leak(tsdn, arena, extent_hooks, cache, extent);
malloc_mutex_unlock(tsdn, &arena->chunks_mtx); malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
return (NULL); return (NULL);
} }
@ -399,10 +400,10 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
} }
if (!extent_committed_get(extent) && if (!extent_committed_get(extent) &&
chunk_hooks->commit(extent_base_get(extent), extent_hooks->commit(extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) { extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) {
malloc_mutex_unlock(tsdn, &arena->chunks_mtx); malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
chunk_record(tsdn, arena, chunk_hooks, extent_heaps, cache, chunk_record(tsdn, arena, extent_hooks, extent_heaps, cache,
extent); extent);
return (NULL); return (NULL);
} }
@ -468,7 +469,7 @@ chunk_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
} }
extent_t * extent_t *
chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
bool slab) bool slab)
{ {
@ -479,7 +480,7 @@ chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
assert(alignment != 0); assert(alignment != 0);
commit = true; commit = true;
extent = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_cached, extent = chunk_recycle(tsdn, arena, extent_hooks, arena->chunks_cached,
true, new_addr, usize, pad, alignment, zero, &commit, slab); true, new_addr, usize, pad, alignment, zero, &commit, slab);
if (extent == NULL) if (extent == NULL)
return (NULL); return (NULL);
@ -520,7 +521,7 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
} }
static extent_t * static extent_t *
chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
bool *commit, bool slab) bool *commit, bool slab)
{ {
@ -529,8 +530,9 @@ chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
assert(usize != 0); assert(usize != 0);
assert(alignment != 0); assert(alignment != 0);
extent = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_retained, extent = chunk_recycle(tsdn, arena, extent_hooks,
false, new_addr, usize, pad, alignment, zero, commit, slab); arena->chunks_retained, false, new_addr, usize, pad, alignment,
zero, commit, slab);
if (extent != NULL && config_stats) { if (extent != NULL && config_stats) {
size_t size = usize + pad; size_t size = usize + pad;
arena->stats.retained -= size; arena->stats.retained -= size;
@ -541,7 +543,7 @@ chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
static extent_t * static extent_t *
chunk_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, chunk_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab) size_t alignment, bool *zero, bool *commit, bool slab)
{ {
extent_t *extent; extent_t *extent;
@ -552,7 +554,7 @@ chunk_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent = extent_alloc(tsdn, arena); extent = extent_alloc(tsdn, arena);
if (extent == NULL) if (extent == NULL)
return (NULL); return (NULL);
addr = chunk_hooks->alloc(new_addr, size, alignment, zero, commit, addr = extent_hooks->alloc(new_addr, size, alignment, zero, commit,
arena->ind); arena->ind);
if (addr == NULL) { if (addr == NULL) {
extent_dalloc(tsdn, arena, extent); extent_dalloc(tsdn, arena, extent);
@ -562,7 +564,7 @@ chunk_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
if (pad != 0) if (pad != 0)
extent_addr_randomize(tsdn, extent, alignment); extent_addr_randomize(tsdn, extent, alignment);
if (chunk_register(tsdn, extent)) { if (chunk_register(tsdn, extent)) {
chunk_leak(tsdn, arena, chunk_hooks, false, extent); chunk_leak(tsdn, arena, extent_hooks, false, extent);
return (NULL); return (NULL);
} }
@ -570,18 +572,18 @@ chunk_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
} }
extent_t * extent_t *
chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
bool *commit, bool slab) bool *commit, bool slab)
{ {
extent_t *extent; extent_t *extent;
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
extent = chunk_alloc_retained(tsdn, arena, chunk_hooks, new_addr, usize, extent = chunk_alloc_retained(tsdn, arena, extent_hooks, new_addr,
pad, alignment, zero, commit, slab); usize, pad, alignment, zero, commit, slab);
if (extent == NULL) { if (extent == NULL) {
extent = chunk_alloc_wrapper_hard(tsdn, arena, chunk_hooks, extent = chunk_alloc_wrapper_hard(tsdn, arena, extent_hooks,
new_addr, usize, pad, alignment, zero, commit, slab); new_addr, usize, pad, alignment, zero, commit, slab);
} }
@ -605,7 +607,7 @@ chunk_can_coalesce(const extent_t *a, const extent_t *b)
} }
static void static void
chunk_try_coalesce(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache) extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache)
{ {
@ -618,7 +620,7 @@ chunk_try_coalesce(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
arena_chunk_cache_maybe_remove(extent_arena_get(a), a, cache); arena_chunk_cache_maybe_remove(extent_arena_get(a), a, cache);
arena_chunk_cache_maybe_remove(extent_arena_get(b), b, cache); arena_chunk_cache_maybe_remove(extent_arena_get(b), b, cache);
if (chunk_merge_wrapper(tsdn, arena, chunk_hooks, a, b)) { if (chunk_merge_wrapper(tsdn, arena, extent_hooks, a, b)) {
extent_heaps_insert(extent_heaps, a); extent_heaps_insert(extent_heaps, a);
extent_heaps_insert(extent_heaps, b); extent_heaps_insert(extent_heaps, b);
arena_chunk_cache_maybe_insert(extent_arena_get(a), a, cache); arena_chunk_cache_maybe_insert(extent_arena_get(a), a, cache);
@ -631,7 +633,7 @@ chunk_try_coalesce(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
} }
static void static void
chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent) extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent)
{ {
extent_t *prev, *next; extent_t *prev, *next;
@ -639,7 +641,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
assert(!cache || !extent_zeroed_get(extent)); assert(!cache || !extent_zeroed_get(extent));
malloc_mutex_lock(tsdn, &arena->chunks_mtx); malloc_mutex_lock(tsdn, &arena->chunks_mtx);
chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
extent_usize_set(extent, 0); extent_usize_set(extent, 0);
extent_active_set(extent, false); extent_active_set(extent, false);
@ -657,7 +659,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
next = rtree_read(tsdn, &chunks_rtree, next = rtree_read(tsdn, &chunks_rtree,
(uintptr_t)extent_past_get(extent), false); (uintptr_t)extent_past_get(extent), false);
if (next != NULL) { if (next != NULL) {
chunk_try_coalesce(tsdn, arena, chunk_hooks, extent, next, chunk_try_coalesce(tsdn, arena, extent_hooks, extent, next,
extent_heaps, cache); extent_heaps, cache);
} }
@ -665,7 +667,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
prev = rtree_read(tsdn, &chunks_rtree, prev = rtree_read(tsdn, &chunks_rtree,
(uintptr_t)extent_before_get(extent), false); (uintptr_t)extent_before_get(extent), false);
if (prev != NULL) { if (prev != NULL) {
chunk_try_coalesce(tsdn, arena, chunk_hooks, prev, extent, chunk_try_coalesce(tsdn, arena, extent_hooks, prev, extent,
extent_heaps, cache); extent_heaps, cache);
} }
@ -673,7 +675,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
} }
void void
chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent) extent_t *extent)
{ {
@ -683,7 +685,7 @@ chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false); extent_zeroed_set(extent, false);
chunk_record(tsdn, arena, chunk_hooks, arena->chunks_cached, true, chunk_record(tsdn, arena, extent_hooks, arena->chunks_cached, true,
extent); extent);
} }
@ -698,7 +700,7 @@ chunk_dalloc_default(void *chunk, size_t size, bool committed,
} }
void void
chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent) extent_t *extent)
{ {
@ -707,9 +709,9 @@ chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
/* Try to deallocate. */ /* Try to deallocate. */
if (!chunk_hooks->dalloc(extent_base_get(extent), if (!extent_hooks->dalloc(extent_base_get(extent),
extent_size_get(extent), extent_committed_get(extent), extent_size_get(extent), extent_committed_get(extent),
arena->ind)) { arena->ind)) {
chunk_deregister(tsdn, extent); chunk_deregister(tsdn, extent);
@ -719,18 +721,18 @@ chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
/* Try to decommit; purge if that fails. */ /* Try to decommit; purge if that fails. */
if (extent_committed_get(extent)) { if (extent_committed_get(extent)) {
extent_committed_set(extent, extent_committed_set(extent,
chunk_hooks->decommit(extent_base_get(extent), extent_hooks->decommit(extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), extent_size_get(extent), 0, extent_size_get(extent),
arena->ind)); arena->ind));
} }
extent_zeroed_set(extent, !extent_committed_get(extent) || extent_zeroed_set(extent, !extent_committed_get(extent) ||
!chunk_hooks->purge(extent_base_get(extent), !extent_hooks->purge(extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)); extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
if (config_stats) if (config_stats)
arena->stats.retained += extent_size_get(extent); arena->stats.retained += extent_size_get(extent);
chunk_record(tsdn, arena, chunk_hooks, arena->chunks_retained, false, chunk_record(tsdn, arena, extent_hooks, arena->chunks_retained, false,
extent); extent);
} }
@ -744,12 +746,12 @@ chunk_commit_default(void *chunk, size_t size, size_t offset, size_t length,
} }
bool bool
chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent, size_t offset, size_t length) extent_t *extent, size_t offset, size_t length)
{ {
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
return (chunk_hooks->commit(extent_base_get(extent), return (extent_hooks->commit(extent_base_get(extent),
extent_size_get(extent), offset, length, arena->ind)); extent_size_get(extent), offset, length, arena->ind));
} }
@ -763,12 +765,13 @@ chunk_decommit_default(void *chunk, size_t size, size_t offset, size_t length,
} }
bool bool
chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t offset, size_t length) extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length)
{ {
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
return (chunk_hooks->decommit(extent_base_get(extent), return (extent_hooks->decommit(extent_base_get(extent),
extent_size_get(extent), offset, length, arena->ind)); extent_size_get(extent), offset, length, arena->ind));
} }
@ -787,12 +790,12 @@ chunk_purge_default(void *chunk, size_t size, size_t offset, size_t length,
} }
bool bool
chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent, size_t offset, size_t length) extent_t *extent, size_t offset, size_t length)
{ {
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
return (chunk_hooks->purge(extent_base_get(extent), return (extent_hooks->purge(extent_base_get(extent),
extent_size_get(extent), offset, length, arena->ind)); extent_size_get(extent), offset, length, arena->ind));
} }
@ -807,7 +810,7 @@ chunk_split_default(void *chunk, size_t size, size_t size_a, size_t size_b,
} }
extent_t * extent_t *
chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *extent, size_t size_a, size_t usize_a, size_t size_b, extent_t *extent, size_t size_a, size_t usize_a, size_t size_b,
size_t usize_b) size_t usize_b)
{ {
@ -816,7 +819,7 @@ chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
assert(extent_size_get(extent) == size_a + size_b); assert(extent_size_get(extent) == size_a + size_b);
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
trail = extent_alloc(tsdn, arena); trail = extent_alloc(tsdn, arena);
if (trail == NULL) if (trail == NULL)
@ -843,7 +846,7 @@ chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
&trail_elm_b)) &trail_elm_b))
goto label_error_c; goto label_error_c;
if (chunk_hooks->split(extent_base_get(extent), size_a + size_b, size_a, if (extent_hooks->split(extent_base_get(extent), size_a + size_b, size_a,
size_b, extent_committed_get(extent), arena->ind)) size_b, extent_committed_get(extent), arena->ind))
goto label_error_d; goto label_error_d;
@ -884,13 +887,13 @@ chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
} }
bool bool
chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_t *a, extent_t *b) extent_t *a, extent_t *b)
{ {
rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks); extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
if (chunk_hooks->merge(extent_base_get(a), extent_size_get(a), if (extent_hooks->merge(extent_base_get(a), extent_size_get(a),
extent_base_get(b), extent_size_get(b), extent_committed_get(a), extent_base_get(b), extent_size_get(b), extent_committed_get(a),
arena->ind)) arena->ind))
return (true); return (true);

4
src/chunk_dss.c
View File

@ -136,10 +136,10 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
dss_max = dss_next; dss_max = dss_next;
malloc_mutex_unlock(tsdn, &dss_mtx); malloc_mutex_unlock(tsdn, &dss_mtx);
if (pad_size != 0) { if (pad_size != 0) {
chunk_hooks_t chunk_hooks = extent_hooks_t extent_hooks =
CHUNK_HOOKS_INITIALIZER; CHUNK_HOOKS_INITIALIZER;
chunk_dalloc_wrapper(tsdn, arena, chunk_dalloc_wrapper(tsdn, arena,
&chunk_hooks, pad); &extent_hooks, pad);
} else } else
extent_dalloc(tsdn, arena, pad); extent_dalloc(tsdn, arena, pad);
if (*zero) if (*zero)

22
src/ctl.c
View File

@ -120,7 +120,7 @@ CTL_PROTO(arena_i_reset)
CTL_PROTO(arena_i_dss) CTL_PROTO(arena_i_dss)
CTL_PROTO(arena_i_lg_dirty_mult) CTL_PROTO(arena_i_lg_dirty_mult)
CTL_PROTO(arena_i_decay_time) CTL_PROTO(arena_i_decay_time)
CTL_PROTO(arena_i_chunk_hooks) CTL_PROTO(arena_i_extent_hooks)
INDEX_PROTO(arena_i) INDEX_PROTO(arena_i)
CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_size)
CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_nregs)
@ -287,7 +287,7 @@ static const ctl_named_node_t arena_i_node[] = {
{NAME("dss"), CTL(arena_i_dss)}, {NAME("dss"), CTL(arena_i_dss)},
{NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)}, {NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)},
{NAME("decay_time"), CTL(arena_i_decay_time)}, {NAME("decay_time"), CTL(arena_i_decay_time)},
{NAME("chunk_hooks"), CTL(arena_i_chunk_hooks)} {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}
}; };
static const ctl_named_node_t super_arena_i_node[] = { static const ctl_named_node_t super_arena_i_node[] = {
{NAME(""), CHILD(named, arena_i)} {NAME(""), CHILD(named, arena_i)}
@ -1647,7 +1647,7 @@ label_return:
} }
static int static int
arena_i_chunk_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{ {
int ret; int ret;
@ -1658,15 +1658,15 @@ arena_i_chunk_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
if (arena_ind < narenas_total_get() && (arena = if (arena_ind < narenas_total_get() && (arena =
arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) {
if (newp != NULL) { if (newp != NULL) {
chunk_hooks_t old_chunk_hooks, new_chunk_hooks; extent_hooks_t old_extent_hooks, new_extent_hooks;
WRITE(new_chunk_hooks, chunk_hooks_t); WRITE(new_extent_hooks, extent_hooks_t);
old_chunk_hooks = chunk_hooks_set(tsd_tsdn(tsd), arena, old_extent_hooks = extent_hooks_set(tsd_tsdn(tsd),
&new_chunk_hooks); arena, &new_extent_hooks);
READ(old_chunk_hooks, chunk_hooks_t); READ(old_extent_hooks, extent_hooks_t);
} else { } else {
chunk_hooks_t old_chunk_hooks = extent_hooks_t old_extent_hooks =
chunk_hooks_get(tsd_tsdn(tsd), arena); extent_hooks_get(tsd_tsdn(tsd), arena);
READ(old_chunk_hooks, chunk_hooks_t); READ(old_extent_hooks, extent_hooks_t);
} }
} else { } else {
ret = EFAULT; ret = EFAULT;

18
src/jemalloc.c
View File

@ -207,14 +207,16 @@ static void WINAPI
_init_init_lock(void) _init_init_lock(void)
{ {
/* If another constructor in the same binary is using mallctl to /*
* e.g. setup chunk hooks, it may end up running before this one, * If another constructor in the same binary is using mallctl to e.g.
* and malloc_init_hard will crash trying to lock the uninitialized * set up extent hooks, it may end up running before this one, and
* lock. So we force an initialization of the lock in * malloc_init_hard will crash trying to lock the uninitialized lock. So
* malloc_init_hard as well. We don't try to care about atomicity * we force an initialization of the lock in malloc_init_hard as well.
* of the accessed to the init_lock_initialized boolean, since it * We don't try to care about atomicity of the accessed to the
* really only matters early in the process creation, before any * init_lock_initialized boolean, since it really only matters early in
* separate thread normally starts doing anything. */ * the process creation, before any separate thread normally starts
* doing anything.
*/
if (!init_lock_initialized) if (!init_lock_initialized)
malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT); malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT);
init_lock_initialized = true; init_lock_initialized = true;

19
src/large.c
View File

@ -96,15 +96,16 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
{ {
arena_t *arena = extent_arena_get(extent); arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent); size_t oldusize = extent_usize_get(extent);
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena);
size_t diff = extent_size_get(extent) - (usize + large_pad); size_t diff = extent_size_get(extent) - (usize + large_pad);
assert(oldusize > usize); assert(oldusize > usize);
/* Split excess pages. */ /* Split excess pages. */
if (diff != 0) { if (diff != 0) {
extent_t *trail = chunk_split_wrapper(tsdn, arena, &chunk_hooks, extent_t *trail = chunk_split_wrapper(tsdn, arena,
extent, usize + large_pad, usize, diff, diff); &extent_hooks, extent, usize + large_pad, usize, diff,
diff);
if (trail == NULL) if (trail == NULL)
return (true); return (true);
@ -113,7 +114,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
extent_usize_get(trail)); extent_usize_get(trail));
} }
arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, trail); arena_chunk_cache_dalloc(tsdn, arena, &extent_hooks, trail);
} }
arena_chunk_ralloc_large_shrink(tsdn, arena, extent, oldusize); arena_chunk_ralloc_large_shrink(tsdn, arena, extent, oldusize);
@ -128,22 +129,22 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
arena_t *arena = extent_arena_get(extent); arena_t *arena = extent_arena_get(extent);
size_t oldusize = extent_usize_get(extent); size_t oldusize = extent_usize_get(extent);
bool is_zeroed_trail = false; bool is_zeroed_trail = false;
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); extent_hooks_t extent_hooks = extent_hooks_get(tsdn, arena);
size_t trailsize = usize - extent_usize_get(extent); size_t trailsize = usize - extent_usize_get(extent);
extent_t *trail; extent_t *trail;
if ((trail = arena_chunk_cache_alloc(tsdn, arena, &chunk_hooks, if ((trail = arena_chunk_cache_alloc(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail)) extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail))
== NULL) { == NULL) {
bool commit = true; bool commit = true;
if ((trail = chunk_alloc_wrapper(tsdn, arena, &chunk_hooks, if ((trail = chunk_alloc_wrapper(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, 0, CACHELINE, extent_past_get(extent), trailsize, 0, CACHELINE,
&is_zeroed_trail, &commit, false)) == NULL) &is_zeroed_trail, &commit, false)) == NULL)
return (true); return (true);
} }
if (chunk_merge_wrapper(tsdn, arena, &chunk_hooks, extent, trail)) { if (chunk_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, trail); chunk_dalloc_wrapper(tsdn, arena, &extent_hooks, trail);
return (true); return (true);
} }

22
test/integration/chunk.c
View File

@ -4,8 +4,8 @@
const char *malloc_conf = "junk:false"; const char *malloc_conf = "junk:false";
#endif #endif
static chunk_hooks_t orig_hooks; static extent_hooks_t orig_hooks;
static chunk_hooks_t old_hooks; static extent_hooks_t old_hooks;
static bool do_dalloc = true; static bool do_dalloc = true;
static bool do_decommit; static bool do_decommit;
@ -125,7 +125,7 @@ TEST_BEGIN(test_chunk)
int flags; int flags;
size_t hooks_mib[3], purge_mib[3]; size_t hooks_mib[3], purge_mib[3];
size_t hooks_miblen, purge_miblen; size_t hooks_miblen, purge_miblen;
chunk_hooks_t new_hooks = { extent_hooks_t new_hooks = {
chunk_alloc, chunk_alloc,
chunk_dalloc, chunk_dalloc,
chunk_commit, chunk_commit,
@ -141,15 +141,15 @@ TEST_BEGIN(test_chunk)
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
/* Install custom chunk hooks. */ /* Install custom extent hooks. */
hooks_miblen = sizeof(hooks_mib)/sizeof(size_t); hooks_miblen = sizeof(hooks_mib)/sizeof(size_t);
assert_d_eq(mallctlnametomib("arena.0.chunk_hooks", hooks_mib, assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib,
&hooks_miblen), 0, "Unexpected mallctlnametomib() failure"); &hooks_miblen), 0, "Unexpected mallctlnametomib() failure");
hooks_mib[1] = (size_t)arena_ind; hooks_mib[1] = (size_t)arena_ind;
old_size = sizeof(chunk_hooks_t); old_size = sizeof(extent_hooks_t);
new_size = sizeof(chunk_hooks_t); new_size = sizeof(extent_hooks_t);
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size, assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
&new_hooks, new_size), 0, "Unexpected chunk_hooks error"); &new_hooks, new_size), 0, "Unexpected extent_hooks error");
orig_hooks = old_hooks; orig_hooks = old_hooks;
assert_ptr_ne(old_hooks.alloc, chunk_alloc, "Unexpected alloc error"); assert_ptr_ne(old_hooks.alloc, chunk_alloc, "Unexpected alloc error");
assert_ptr_ne(old_hooks.dalloc, chunk_dalloc, assert_ptr_ne(old_hooks.dalloc, chunk_dalloc,
@ -223,11 +223,11 @@ TEST_BEGIN(test_chunk)
assert_ptr_not_null(p, "Unexpected mallocx() error"); assert_ptr_not_null(p, "Unexpected mallocx() error");
dallocx(p, flags); dallocx(p, flags);
/* Restore chunk hooks. */ /* Restore extent hooks. */
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, NULL, NULL, assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, NULL, NULL,
&old_hooks, new_size), 0, "Unexpected chunk_hooks error"); &old_hooks, new_size), 0, "Unexpected extent_hooks error");
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size, assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
NULL, 0), 0, "Unexpected chunk_hooks error"); NULL, 0), 0, "Unexpected extent_hooks error");
assert_ptr_eq(old_hooks.alloc, orig_hooks.alloc, assert_ptr_eq(old_hooks.alloc, orig_hooks.alloc,
"Unexpected alloc error"); "Unexpected alloc error");
assert_ptr_eq(old_hooks.dalloc, orig_hooks.dalloc, assert_ptr_eq(old_hooks.dalloc, orig_hooks.dalloc,