Rename most remaining *chunk* APIs to *extent*.
This commit is contained in:
Jason Evans
parent
0c4932eb1e
commit
22588dda6e
10
Makefile.in
10
Makefile.in
@ -82,11 +82,11 @@ C_SRCS := $(srcroot)src/jemalloc.c \
|
||||
$(srcroot)src/base.c \
|
||||
$(srcroot)src/bitmap.c \
|
||||
$(srcroot)src/chunk.c \
|
||||
$(srcroot)src/chunk_dss.c \
|
||||
$(srcroot)src/chunk_mmap.c \
|
||||
$(srcroot)src/ckh.c \
|
||||
$(srcroot)src/ctl.c \
|
||||
$(srcroot)src/extent.c \
|
||||
$(srcroot)src/extent_dss.c \
|
||||
$(srcroot)src/extent_mmap.c \
|
||||
$(srcroot)src/hash.c \
|
||||
$(srcroot)src/large.c \
|
||||
$(srcroot)src/mb.c \
|
||||
@ -171,16 +171,16 @@ TESTS_UNIT := \
|
||||
$(srcroot)test/unit/zero.c
|
||||
TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
|
||||
$(srcroot)test/integration/allocated.c \
|
||||
$(srcroot)test/integration/sdallocx.c \
|
||||
$(srcroot)test/integration/extent.c \
|
||||
$(srcroot)test/integration/mallocx.c \
|
||||
$(srcroot)test/integration/MALLOCX_ARENA.c \
|
||||
$(srcroot)test/integration/overflow.c \
|
||||
$(srcroot)test/integration/posix_memalign.c \
|
||||
$(srcroot)test/integration/rallocx.c \
|
||||
$(srcroot)test/integration/sdallocx.c \
|
||||
$(srcroot)test/integration/thread_arena.c \
|
||||
$(srcroot)test/integration/thread_tcache_enabled.c \
|
||||
$(srcroot)test/integration/xallocx.c \
|
||||
$(srcroot)test/integration/chunk.c
|
||||
$(srcroot)test/integration/xallocx.c
|
||||
TESTS_STRESS := $(srcroot)test/stress/microbench.c
|
||||
TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS)
|
||||
|
||||
|
||||
@ -278,22 +278,22 @@ extern ssize_t opt_decay_time;
|
||||
|
||||
extern const arena_bin_info_t arena_bin_info[NBINS];
|
||||
|
||||
extent_t *arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_t *arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment,
|
||||
bool *zero);
|
||||
void arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
|
||||
void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent);
|
||||
void arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent,
|
||||
void arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent,
|
||||
bool cache);
|
||||
void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent,
|
||||
void arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent,
|
||||
bool cache);
|
||||
extent_t *arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena,
|
||||
size_t usize, size_t alignment, bool *zero);
|
||||
void arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
bool locked);
|
||||
void arena_chunk_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
|
||||
void arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_t *extent, bool locked);
|
||||
void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_t *extent, size_t oldsize);
|
||||
void arena_chunk_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
|
||||
void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_t *extent, size_t oldsize);
|
||||
ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena);
|
||||
bool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena,
|
||||
|
||||
@ -26,40 +26,7 @@ extern size_t chunksize;
|
||||
extern size_t chunksize_mask; /* (chunksize - 1). */
|
||||
extern size_t chunk_npages;
|
||||
|
||||
extern const extent_hooks_t extent_hooks_default;
|
||||
|
||||
extent_hooks_t extent_hooks_get(tsdn_t *tsdn, arena_t *arena);
|
||||
extent_hooks_t extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
|
||||
const extent_hooks_t *extent_hooks);
|
||||
|
||||
extent_t *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool slab);
|
||||
extent_t *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool *commit, bool slab);
|
||||
void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent);
|
||||
void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent);
|
||||
bool chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
bool chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
extent_t *chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
|
||||
size_t usize_a, size_t size_b, size_t usize_b);
|
||||
bool chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b);
|
||||
bool chunk_boot(void);
|
||||
void chunk_prefork(tsdn_t *tsdn);
|
||||
void chunk_postfork_parent(tsdn_t *tsdn);
|
||||
void chunk_postfork_child(tsdn_t *tsdn);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
@ -67,6 +34,3 @@ void chunk_postfork_child(tsdn_t *tsdn);
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
#include "jemalloc/internal/chunk_dss.h"
|
||||
#include "jemalloc/internal/chunk_mmap.h"
|
||||
|
||||
@ -38,7 +38,7 @@ struct extent_s {
|
||||
bool e_active;
|
||||
|
||||
/*
|
||||
* The zeroed flag is used by chunk recycling code to track whether
|
||||
* The zeroed flag is used by extent recycling code to track whether
|
||||
* memory is zero-filled.
|
||||
*/
|
||||
bool e_zeroed;
|
||||
@ -88,10 +88,15 @@ typedef ph(extent_t) extent_heap_t;
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
extern rtree_t extents_rtree;
|
||||
extern const extent_hooks_t extent_hooks_default;
|
||||
|
||||
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
|
||||
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
|
||||
|
||||
extent_hooks_t extent_hooks_get(tsdn_t *tsdn, arena_t *arena);
|
||||
extent_hooks_t extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
|
||||
const extent_hooks_t *extent_hooks);
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef size_t (extent_size_quantize_t)(size_t);
|
||||
extern extent_size_quantize_t *extent_size_quantize_floor;
|
||||
@ -103,6 +108,34 @@ size_t extent_size_quantize_ceil(size_t size);
|
||||
|
||||
ph_proto(, extent_heap_, extent_heap_t, extent_t)
|
||||
|
||||
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool slab);
|
||||
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool *commit, bool slab);
|
||||
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent);
|
||||
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent);
|
||||
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
|
||||
size_t usize_a, size_t size_b, size_t usize_b);
|
||||
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b);
|
||||
void extent_prefork(tsdn_t *tsdn);
|
||||
void extent_postfork_parent(tsdn_t *tsdn);
|
||||
void extent_postfork_child(tsdn_t *tsdn);
|
||||
|
||||
bool extent_boot(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
@ -110,7 +143,7 @@ bool extent_boot(void);
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
extent_t *extent_lookup(tsdn_t *tsdn, const void *chunk, bool dependent);
|
||||
extent_t *extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent);
|
||||
arena_t *extent_arena_get(const extent_t *extent);
|
||||
void *extent_base_get(const extent_t *extent);
|
||||
void *extent_addr_get(const extent_t *extent);
|
||||
@ -395,3 +428,5 @@ extent_ring_remove(extent_t *extent)
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
#include "jemalloc/internal/extent_dss.h"
|
||||
#include "jemalloc/internal/extent_mmap.h"
|
||||
|
||||
@ -21,15 +21,15 @@ extern const char *dss_prec_names[];
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
dss_prec_t chunk_dss_prec_get(tsdn_t *tsdn);
|
||||
bool chunk_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec);
|
||||
void *chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr,
|
||||
dss_prec_t extent_dss_prec_get(tsdn_t *tsdn);
|
||||
bool extent_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec);
|
||||
void *extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr,
|
||||
size_t size, size_t alignment, bool *zero, bool *commit);
|
||||
bool chunk_in_dss(tsdn_t *tsdn, void *chunk);
|
||||
bool chunk_dss_boot(void);
|
||||
void chunk_dss_prefork(tsdn_t *tsdn);
|
||||
void chunk_dss_postfork_parent(tsdn_t *tsdn);
|
||||
void chunk_dss_postfork_child(tsdn_t *tsdn);
|
||||
bool extent_in_dss(tsdn_t *tsdn, void *addr);
|
||||
bool extent_dss_boot(void);
|
||||
void extent_dss_prefork(tsdn_t *tsdn);
|
||||
void extent_dss_postfork_parent(tsdn_t *tsdn);
|
||||
void extent_dss_postfork_child(tsdn_t *tsdn);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
@ -9,9 +9,9 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void *chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment,
|
||||
void *extent_alloc_mmap(void *new_addr, size_t size, size_t alignment,
|
||||
bool *zero, bool *commit);
|
||||
bool chunk_dalloc_mmap(void *chunk, size_t size);
|
||||
bool extent_dalloc_mmap(void *addr, size_t size);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
@ -9,14 +9,6 @@ arena_boot
|
||||
arena_choose
|
||||
arena_choose_hard
|
||||
arena_choose_impl
|
||||
arena_chunk_alloc_large
|
||||
arena_chunk_cache_alloc
|
||||
arena_chunk_cache_dalloc
|
||||
arena_chunk_cache_maybe_insert
|
||||
arena_chunk_cache_maybe_remove
|
||||
arena_chunk_dalloc_large
|
||||
arena_chunk_ralloc_large_expand
|
||||
arena_chunk_ralloc_large_shrink
|
||||
arena_cleanup
|
||||
arena_dalloc
|
||||
arena_dalloc_bin_junked_locked
|
||||
@ -31,6 +23,14 @@ arena_decay_time_get
|
||||
arena_decay_time_set
|
||||
arena_dss_prec_get
|
||||
arena_dss_prec_set
|
||||
arena_extent_alloc_large
|
||||
arena_extent_cache_alloc
|
||||
arena_extent_cache_dalloc
|
||||
arena_extent_cache_maybe_insert
|
||||
arena_extent_cache_maybe_remove
|
||||
arena_extent_dalloc_large
|
||||
arena_extent_ralloc_large_expand
|
||||
arena_extent_ralloc_large_shrink
|
||||
arena_get
|
||||
arena_ichoose
|
||||
arena_init
|
||||
@ -115,30 +115,8 @@ bootstrap_free
|
||||
bootstrap_malloc
|
||||
bt_init
|
||||
buferror
|
||||
chunk_alloc_cache
|
||||
chunk_alloc_dss
|
||||
chunk_alloc_mmap
|
||||
chunk_alloc_wrapper
|
||||
chunk_boot
|
||||
chunk_commit_wrapper
|
||||
chunk_dalloc_cache
|
||||
chunk_dalloc_mmap
|
||||
chunk_dalloc_wrapper
|
||||
chunk_decommit_wrapper
|
||||
chunk_dss_boot
|
||||
chunk_dss_postfork_child
|
||||
chunk_dss_postfork_parent
|
||||
chunk_dss_prec_get
|
||||
chunk_dss_prec_set
|
||||
chunk_dss_prefork
|
||||
chunk_in_dss
|
||||
chunk_merge_wrapper
|
||||
chunk_npages
|
||||
chunk_postfork_child
|
||||
chunk_postfork_parent
|
||||
chunk_prefork
|
||||
chunk_purge_wrapper
|
||||
chunk_split_wrapper
|
||||
chunksize
|
||||
chunksize_mask
|
||||
ckh_count
|
||||
@ -167,14 +145,29 @@ extent_addr_get
|
||||
extent_addr_randomize
|
||||
extent_addr_set
|
||||
extent_alloc
|
||||
extent_alloc_cache
|
||||
extent_alloc_dss
|
||||
extent_alloc_mmap
|
||||
extent_alloc_wrapper
|
||||
extent_arena_get
|
||||
extent_arena_set
|
||||
extent_base_get
|
||||
extent_before_get
|
||||
extent_boot
|
||||
extent_commit_wrapper
|
||||
extent_committed_get
|
||||
extent_committed_set
|
||||
extent_dalloc
|
||||
extent_dalloc_cache
|
||||
extent_dalloc_mmap
|
||||
extent_dalloc_wrapper
|
||||
extent_decommit_wrapper
|
||||
extent_dss_boot
|
||||
extent_dss_postfork_child
|
||||
extent_dss_postfork_parent
|
||||
extent_dss_prec_get
|
||||
extent_dss_prec_set
|
||||
extent_dss_prefork
|
||||
extent_heap_empty
|
||||
extent_heap_first
|
||||
extent_heap_insert
|
||||
@ -184,12 +177,18 @@ extent_heap_remove_first
|
||||
extent_hooks_default
|
||||
extent_hooks_get
|
||||
extent_hooks_set
|
||||
extent_in_dss
|
||||
extent_init
|
||||
extent_last_get
|
||||
extent_lookup
|
||||
extent_merge_wrapper
|
||||
extent_past_get
|
||||
extent_postfork_child
|
||||
extent_postfork_parent
|
||||
extent_prefork
|
||||
extent_prof_tctx_get
|
||||
extent_prof_tctx_set
|
||||
extent_purge_wrapper
|
||||
extent_retained_get
|
||||
extent_ring_insert
|
||||
extent_ring_remove
|
||||
@ -201,6 +200,7 @@ extent_slab_data_get
|
||||
extent_slab_data_get_const
|
||||
extent_slab_get
|
||||
extent_slab_set
|
||||
extent_split_wrapper
|
||||
extent_usize_get
|
||||
extent_usize_set
|
||||
extent_zeroed_get
|
||||
|
||||
@ -41,11 +41,11 @@
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\base.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\bitmap.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_dss.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_mmap.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ckh.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ctl.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent_dss.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent_mmap.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal.h" />
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\jemalloc_internal_decls.h" />
|
||||
@ -92,11 +92,11 @@
|
||||
<ClCompile Include="..\..\..\..\src\base.c" />
|
||||
<ClCompile Include="..\..\..\..\src\bitmap.c" />
|
||||
<ClCompile Include="..\..\..\..\src\chunk.c" />
|
||||
<ClCompile Include="..\..\..\..\src\chunk_dss.c" />
|
||||
<ClCompile Include="..\..\..\..\src\chunk_mmap.c" />
|
||||
<ClCompile Include="..\..\..\..\src\ckh.c" />
|
||||
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
||||
<ClCompile Include="..\..\..\..\src\extent.c" />
|
||||
<ClCompile Include="..\..\..\..\src\extent_dss.c" />
|
||||
<ClCompile Include="..\..\..\..\src\extent_mmap.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hash.c" />
|
||||
<ClCompile Include="..\..\..\..\src\jemalloc.c" />
|
||||
<ClCompile Include="..\..\..\..\src\large.c" />
|
||||
|
||||
@ -62,12 +62,6 @@
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_dss.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\chunk_mmap.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\ckh.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
@ -77,6 +71,12 @@
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent_dss.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\extent_mmap.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="..\..\..\..\include\jemalloc\internal\hash.h">
|
||||
<Filter>Header Files\internal</Filter>
|
||||
</ClInclude>
|
||||
@ -187,12 +187,6 @@
|
||||
<ClCompile Include="..\..\..\..\src\chunk.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\chunk_dss.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\chunk_mmap.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\ckh.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
@ -202,6 +196,12 @@
|
||||
<ClCompile Include="..\..\..\..\src\extent.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\extent_dss.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\extent_mmap.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hash.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
|
||||
58
src/arena.c
58
src/arena.c
@ -46,33 +46,33 @@ static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena,
|
||||
/******************************************************************************/
|
||||
|
||||
static size_t
|
||||
arena_chunk_dirty_npages(const extent_t *extent)
|
||||
arena_extent_dirty_npages(const extent_t *extent)
|
||||
{
|
||||
|
||||
return (extent_size_get(extent) >> LG_PAGE);
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
arena_chunk_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_extent_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool slab)
|
||||
{
|
||||
|
||||
malloc_mutex_assert_owner(tsdn, &arena->lock);
|
||||
|
||||
return (chunk_alloc_cache(tsdn, arena, extent_hooks, new_addr, usize,
|
||||
return (extent_alloc_cache(tsdn, arena, extent_hooks, new_addr, usize,
|
||||
pad, alignment, zero, slab));
|
||||
}
|
||||
|
||||
extent_t *
|
||||
arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment,
|
||||
bool *zero)
|
||||
{
|
||||
extent_t *extent;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->lock);
|
||||
extent = arena_chunk_cache_alloc_locked(tsdn, arena, extent_hooks,
|
||||
extent = arena_extent_cache_alloc_locked(tsdn, arena, extent_hooks,
|
||||
new_addr, size, 0, alignment, zero, false);
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
|
||||
@ -80,44 +80,44 @@ arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
|
||||
}
|
||||
|
||||
static void
|
||||
arena_chunk_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_extent_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent)
|
||||
{
|
||||
|
||||
malloc_mutex_assert_owner(tsdn, &arena->lock);
|
||||
|
||||
chunk_dalloc_cache(tsdn, arena, extent_hooks, extent);
|
||||
extent_dalloc_cache(tsdn, arena, extent_hooks, extent);
|
||||
arena_maybe_purge(tsdn, arena);
|
||||
}
|
||||
|
||||
void
|
||||
arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent)
|
||||
{
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->lock);
|
||||
arena_chunk_cache_dalloc_locked(tsdn, arena, extent_hooks, extent);
|
||||
arena_extent_cache_dalloc_locked(tsdn, arena, extent_hooks, extent);
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
}
|
||||
|
||||
void
|
||||
arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache)
|
||||
arena_extent_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache)
|
||||
{
|
||||
|
||||
if (cache) {
|
||||
extent_ring_insert(&arena->extents_dirty, extent);
|
||||
arena->ndirty += arena_chunk_dirty_npages(extent);
|
||||
arena->ndirty += arena_extent_dirty_npages(extent);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent, bool dirty)
|
||||
arena_extent_cache_maybe_remove(arena_t *arena, extent_t *extent, bool dirty)
|
||||
{
|
||||
|
||||
if (dirty) {
|
||||
extent_ring_remove(extent);
|
||||
assert(arena->ndirty >= arena_chunk_dirty_npages(extent));
|
||||
arena->ndirty -= arena_chunk_dirty_npages(extent);
|
||||
assert(arena->ndirty >= arena_extent_dirty_npages(extent));
|
||||
arena->ndirty -= arena_extent_dirty_npages(extent);
|
||||
}
|
||||
}
|
||||
|
||||
@ -320,13 +320,13 @@ arena_large_ralloc_stats_update(arena_t *arena, size_t oldusize, size_t usize)
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
arena_chunk_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_extent_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, size_t usize, size_t alignment, bool *zero)
|
||||
{
|
||||
extent_t *extent;
|
||||
bool commit = true;
|
||||
|
||||
extent = chunk_alloc_wrapper(tsdn, arena, extent_hooks, NULL, usize,
|
||||
extent = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL, usize,
|
||||
large_pad, alignment, zero, &commit, false);
|
||||
if (extent == NULL) {
|
||||
/* Revert optimistic stats updates. */
|
||||
@ -343,7 +343,7 @@ arena_chunk_alloc_large_hard(tsdn_t *tsdn, arena_t *arena,
|
||||
}
|
||||
|
||||
extent_t *
|
||||
arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||
arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||
size_t alignment, bool *zero)
|
||||
{
|
||||
extent_t *extent;
|
||||
@ -358,11 +358,11 @@ arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||
}
|
||||
arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE);
|
||||
|
||||
extent = arena_chunk_cache_alloc_locked(tsdn, arena, &extent_hooks,
|
||||
extent = arena_extent_cache_alloc_locked(tsdn, arena, &extent_hooks,
|
||||
NULL, usize, large_pad, alignment, zero, false);
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
if (extent == NULL) {
|
||||
extent = arena_chunk_alloc_large_hard(tsdn, arena,
|
||||
extent = arena_extent_alloc_large_hard(tsdn, arena,
|
||||
&extent_hooks, usize, alignment, zero);
|
||||
}
|
||||
|
||||
@ -370,7 +370,7 @@ arena_chunk_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||
}
|
||||
|
||||
void
|
||||
arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
bool locked)
|
||||
{
|
||||
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER;
|
||||
@ -384,13 +384,13 @@ arena_chunk_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
}
|
||||
arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE);
|
||||
|
||||
arena_chunk_cache_dalloc_locked(tsdn, arena, &extent_hooks, extent);
|
||||
arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, extent);
|
||||
if (!locked)
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
}
|
||||
|
||||
void
|
||||
arena_chunk_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
size_t oldusize)
|
||||
{
|
||||
size_t usize = extent_usize_get(extent);
|
||||
@ -406,7 +406,7 @@ arena_chunk_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
}
|
||||
|
||||
void
|
||||
arena_chunk_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
size_t oldusize)
|
||||
{
|
||||
size_t usize = extent_usize_get(extent);
|
||||
@ -756,7 +756,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
next = qr_next(extent, qr_link);
|
||||
/* Allocate. */
|
||||
zero = false;
|
||||
textent = arena_chunk_cache_alloc_locked(tsdn, arena,
|
||||
textent = arena_extent_cache_alloc_locked(tsdn, arena,
|
||||
extent_hooks, extent_base_get(extent),
|
||||
extent_size_get(extent), 0, CACHELINE, &zero, false);
|
||||
assert(textent == extent);
|
||||
@ -793,7 +793,7 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
|
||||
next = qr_next(extent, qr_link);
|
||||
extent_ring_remove(extent);
|
||||
chunk_dalloc_wrapper(tsdn, arena, extent_hooks, extent);
|
||||
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent);
|
||||
}
|
||||
|
||||
if (config_stats) {
|
||||
@ -869,7 +869,7 @@ arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab)
|
||||
extent_hooks_t extent_hooks = EXTENT_HOOKS_INITIALIZER;
|
||||
|
||||
arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
|
||||
arena_chunk_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab);
|
||||
arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab);
|
||||
}
|
||||
|
||||
void
|
||||
@ -996,7 +996,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
|
||||
zero = false;
|
||||
commit = true;
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
slab = chunk_alloc_wrapper(tsdn, arena, extent_hooks, NULL,
|
||||
slab = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL,
|
||||
bin_info->slab_size, 0, PAGE, &zero, &commit, true);
|
||||
malloc_mutex_lock(tsdn, &arena->lock);
|
||||
|
||||
@ -1013,7 +1013,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
|
||||
bool zero;
|
||||
|
||||
zero = false;
|
||||
slab = arena_chunk_cache_alloc_locked(tsdn, arena, &extent_hooks, NULL,
|
||||
slab = arena_extent_cache_alloc_locked(tsdn, arena, &extent_hooks, NULL,
|
||||
bin_info->slab_size, 0, PAGE, &zero, true);
|
||||
if (slab == NULL) {
|
||||
slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks,
|
||||
@ -1774,7 +1774,7 @@ arena_new(tsdn_t *tsdn, unsigned ind)
|
||||
(uint64_t)(uintptr_t)arena;
|
||||
}
|
||||
|
||||
arena->dss_prec = chunk_dss_prec_get(tsdn);
|
||||
arena->dss_prec = extent_dss_prec_get(tsdn);
|
||||
|
||||
arena->lg_dirty_mult = arena_lg_dirty_mult_default_get();
|
||||
arena->purging = false;
|
||||
|
||||
@ -38,7 +38,7 @@ base_extent_dalloc(tsdn_t *tsdn, extent_t *extent)
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
base_chunk_alloc(tsdn_t *tsdn, size_t minsize)
|
||||
base_extent_alloc(tsdn_t *tsdn, size_t minsize)
|
||||
{
|
||||
extent_t *extent;
|
||||
size_t csize, nsize;
|
||||
@ -51,13 +51,13 @@ base_chunk_alloc(tsdn_t *tsdn, size_t minsize)
|
||||
nsize = (extent == NULL) ? CACHELINE_CEILING(sizeof(extent_t)) : 0;
|
||||
csize = CHUNK_CEILING(minsize + nsize);
|
||||
/*
|
||||
* Directly call chunk_alloc_mmap() because it's critical to allocate
|
||||
* Directly call extent_alloc_mmap() because it's critical to allocate
|
||||
* untouched demand-zeroed virtual memory.
|
||||
*/
|
||||
{
|
||||
bool zero = true;
|
||||
bool commit = true;
|
||||
addr = chunk_alloc_mmap(NULL, csize, PAGE, &zero, &commit);
|
||||
addr = extent_alloc_mmap(NULL, csize, PAGE, &zero, &commit);
|
||||
}
|
||||
if (addr == NULL) {
|
||||
if (extent != NULL)
|
||||
@ -108,7 +108,7 @@ base_alloc(tsdn_t *tsdn, size_t size)
|
||||
}
|
||||
if (extent == NULL) {
|
||||
/* Try to allocate more space. */
|
||||
extent = base_chunk_alloc(tsdn, csize);
|
||||
extent = base_extent_alloc(tsdn, csize);
|
||||
}
|
||||
if (extent == NULL) {
|
||||
ret = NULL;
|
||||
|
||||
936
src/chunk.c
936
src/chunk.c
@ -7,924 +7,12 @@
|
||||
const char *opt_dss = DSS_DEFAULT;
|
||||
size_t opt_lg_chunk = 0;
|
||||
|
||||
/* Used exclusively for gdump triggering. */
|
||||
static size_t curchunks;
|
||||
static size_t highchunks;
|
||||
|
||||
/* Various chunk-related settings. */
|
||||
size_t chunksize;
|
||||
size_t chunksize_mask; /* (chunksize - 1). */
|
||||
size_t chunk_npages;
|
||||
|
||||
static void *extent_alloc_default(void *new_addr, size_t size,
|
||||
size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
|
||||
static bool extent_dalloc_default(void *addr, size_t size, bool committed,
|
||||
unsigned arena_ind);
|
||||
static bool extent_commit_default(void *addr, size_t size, size_t offset,
|
||||
size_t length, unsigned arena_ind);
|
||||
static bool extent_decommit_default(void *addr, size_t size, size_t offset,
|
||||
size_t length, unsigned arena_ind);
|
||||
static bool extent_purge_default(void *addr, size_t size, size_t offset,
|
||||
size_t length, unsigned arena_ind);
|
||||
static bool extent_split_default(void *addr, size_t size, size_t size_a,
|
||||
size_t size_b, bool committed, unsigned arena_ind);
|
||||
static bool extent_merge_default(void *addr_a, size_t size_a, void *addr_b,
|
||||
size_t size_b, bool committed, unsigned arena_ind);
|
||||
|
||||
const extent_hooks_t extent_hooks_default = {
|
||||
extent_alloc_default,
|
||||
extent_dalloc_default,
|
||||
extent_commit_default,
|
||||
extent_decommit_default,
|
||||
extent_purge_default,
|
||||
extent_split_default,
|
||||
extent_merge_default
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
/*
|
||||
* Function prototypes for static functions that are referenced prior to
|
||||
* definition.
|
||||
*/
|
||||
|
||||
static void chunk_record(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_heap_t extent_heaps[NPSIZES],
|
||||
bool cache, extent_t *extent);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
static void
|
||||
extent_heaps_insert(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
|
||||
{
|
||||
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
|
||||
pszind_t pind = psz2ind(psz);
|
||||
extent_heap_insert(&extent_heaps[pind], extent);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_heaps_remove(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
|
||||
{
|
||||
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
|
||||
pszind_t pind = psz2ind(psz);
|
||||
extent_heap_remove(&extent_heaps[pind], extent);
|
||||
}
|
||||
|
||||
static extent_hooks_t
|
||||
extent_hooks_get_locked(arena_t *arena)
|
||||
{
|
||||
|
||||
return (arena->extent_hooks);
|
||||
}
|
||||
|
||||
extent_hooks_t
|
||||
extent_hooks_get(tsdn_t *tsdn, arena_t *arena)
|
||||
{
|
||||
extent_hooks_t extent_hooks;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
extent_hooks = extent_hooks_get_locked(arena);
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
|
||||
return (extent_hooks);
|
||||
}
|
||||
|
||||
extent_hooks_t
|
||||
extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
|
||||
const extent_hooks_t *extent_hooks)
|
||||
{
|
||||
extent_hooks_t old_extent_hooks;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
old_extent_hooks = arena->extent_hooks;
|
||||
/*
|
||||
* Copy each field atomically so that it is impossible for readers to
|
||||
* see partially updated pointers. There are places where readers only
|
||||
* need one hook function pointer (therefore no need to copy the
|
||||
* entirety of arena->extent_hooks), and stale reads do not affect
|
||||
* correctness, so they perform unlocked reads.
|
||||
*/
|
||||
#define ATOMIC_COPY_HOOK(n) do { \
|
||||
union { \
|
||||
extent_##n##_t **n; \
|
||||
void **v; \
|
||||
} u; \
|
||||
u.n = &arena->extent_hooks.n; \
|
||||
atomic_write_p(u.v, extent_hooks->n); \
|
||||
} while (0)
|
||||
ATOMIC_COPY_HOOK(alloc);
|
||||
ATOMIC_COPY_HOOK(dalloc);
|
||||
ATOMIC_COPY_HOOK(commit);
|
||||
ATOMIC_COPY_HOOK(decommit);
|
||||
ATOMIC_COPY_HOOK(purge);
|
||||
ATOMIC_COPY_HOOK(split);
|
||||
ATOMIC_COPY_HOOK(merge);
|
||||
#undef ATOMIC_COPY_HOOK
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
|
||||
return (old_extent_hooks);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, bool locked)
|
||||
{
|
||||
static const extent_hooks_t uninitialized_hooks =
|
||||
EXTENT_HOOKS_INITIALIZER;
|
||||
|
||||
if (memcmp(extent_hooks, &uninitialized_hooks, sizeof(extent_hooks_t))
|
||||
== 0) {
|
||||
*extent_hooks = locked ? extent_hooks_get_locked(arena) :
|
||||
extent_hooks_get(tsdn, arena);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
extent_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, true);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, false);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_rtree_acquire(tsdn_t *tsdn, const extent_t *extent, bool dependent,
|
||||
bool init_missing, rtree_elm_t **r_elm_a, rtree_elm_t **r_elm_b)
|
||||
{
|
||||
|
||||
*r_elm_a = rtree_elm_acquire(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_base_get(extent), dependent, init_missing);
|
||||
if (!dependent && *r_elm_a == NULL)
|
||||
return (true);
|
||||
assert(*r_elm_a != NULL);
|
||||
|
||||
if (extent_size_get(extent) > PAGE) {
|
||||
*r_elm_b = rtree_elm_acquire(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_last_get(extent), dependent,
|
||||
init_missing);
|
||||
if (!dependent && *r_elm_b == NULL)
|
||||
return (true);
|
||||
assert(*r_elm_b != NULL);
|
||||
} else
|
||||
*r_elm_b = NULL;
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_rtree_write_acquired(tsdn_t *tsdn, rtree_elm_t *elm_a,
|
||||
rtree_elm_t *elm_b, const extent_t *extent)
|
||||
{
|
||||
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, elm_a, extent);
|
||||
if (elm_b != NULL)
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, elm_b, extent);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b)
|
||||
{
|
||||
|
||||
rtree_elm_release(tsdn, &extents_rtree, elm_a);
|
||||
if (elm_b != NULL)
|
||||
rtree_elm_release(tsdn, &extents_rtree, elm_b);
|
||||
}
|
||||
|
||||
static void
|
||||
chunk_interior_register(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
assert(extent_slab_get(extent));
|
||||
|
||||
for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
|
||||
rtree_write(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
|
||||
LG_PAGE), extent);
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
chunk_register(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
rtree_elm_t *elm_a, *elm_b;
|
||||
|
||||
if (extent_rtree_acquire(tsdn, extent, false, true, &elm_a, &elm_b))
|
||||
return (true);
|
||||
extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent);
|
||||
if (extent_slab_get(extent))
|
||||
chunk_interior_register(tsdn, extent);
|
||||
extent_rtree_release(tsdn, elm_a, elm_b);
|
||||
|
||||
if (config_prof && opt_prof && extent_active_get(extent)) {
|
||||
size_t nadd = (extent_size_get(extent) == 0) ? 1 :
|
||||
extent_size_get(extent) / chunksize;
|
||||
size_t cur = atomic_add_z(&curchunks, nadd);
|
||||
size_t high = atomic_read_z(&highchunks);
|
||||
while (cur > high && atomic_cas_z(&highchunks, high, cur)) {
|
||||
/*
|
||||
* Don't refresh cur, because it may have decreased
|
||||
* since this thread lost the highchunks update race.
|
||||
*/
|
||||
high = atomic_read_z(&highchunks);
|
||||
}
|
||||
if (cur > high && prof_gdump_get_unlocked())
|
||||
prof_gdump(tsdn);
|
||||
}
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
static void
|
||||
chunk_interior_deregister(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
assert(extent_slab_get(extent));
|
||||
|
||||
for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
|
||||
rtree_clear(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
|
||||
LG_PAGE));
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
chunk_deregister(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
rtree_elm_t *elm_a, *elm_b;
|
||||
|
||||
extent_rtree_acquire(tsdn, extent, true, false, &elm_a, &elm_b);
|
||||
extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL);
|
||||
if (extent_slab_get(extent))
|
||||
chunk_interior_deregister(tsdn, extent);
|
||||
extent_rtree_release(tsdn, elm_a, elm_b);
|
||||
|
||||
if (config_prof && opt_prof && extent_active_get(extent)) {
|
||||
size_t nsub = (extent_size_get(extent) == 0) ? 1 :
|
||||
extent_size_get(extent) / chunksize;
|
||||
assert(atomic_read_z(&curchunks) >= nsub);
|
||||
atomic_sub_z(&curchunks, nsub);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Do first-best-fit chunk selection, i.e. select the lowest chunk that best
|
||||
* fits.
|
||||
*/
|
||||
static extent_t *
|
||||
chunk_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES],
|
||||
size_t size)
|
||||
{
|
||||
pszind_t pind, i;
|
||||
|
||||
pind = psz2ind(extent_size_quantize_ceil(size));
|
||||
for (i = pind; i < NPSIZES; i++) {
|
||||
extent_t *extent = extent_heap_first(&extent_heaps[i]);
|
||||
if (extent != NULL)
|
||||
return (extent);
|
||||
}
|
||||
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
chunk_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
bool cache, extent_t *extent)
|
||||
{
|
||||
|
||||
/*
|
||||
* Leak chunk after making sure its pages have already been purged, so
|
||||
* that this is only a virtual memory leak.
|
||||
*/
|
||||
if (cache) {
|
||||
chunk_purge_wrapper(tsdn, arena, extent_hooks, extent, 0,
|
||||
extent_size_get(extent));
|
||||
}
|
||||
extent_dalloc(tsdn, arena, extent);
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
chunk_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr,
|
||||
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
|
||||
bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
size_t size, alloc_size, leadsize, trailsize;
|
||||
|
||||
assert(new_addr == NULL || !slab);
|
||||
assert(pad == 0 || !slab);
|
||||
|
||||
size = usize + pad;
|
||||
alloc_size = s2u(size + PAGE_CEILING(alignment) - PAGE);
|
||||
/* Beware size_t wrap-around. */
|
||||
if (alloc_size < usize)
|
||||
return (NULL);
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
|
||||
if (new_addr != NULL) {
|
||||
rtree_elm_t *elm;
|
||||
|
||||
elm = rtree_elm_acquire(tsdn, &extents_rtree,
|
||||
(uintptr_t)new_addr, false, false);
|
||||
if (elm != NULL) {
|
||||
extent = rtree_elm_read_acquired(tsdn, &extents_rtree,
|
||||
elm);
|
||||
if (extent != NULL && (extent_active_get(extent) ||
|
||||
extent_retained_get(extent) == cache))
|
||||
extent = NULL;
|
||||
rtree_elm_release(tsdn, &extents_rtree, elm);
|
||||
} else
|
||||
extent = NULL;
|
||||
} else
|
||||
extent = chunk_first_best_fit(arena, extent_heaps, alloc_size);
|
||||
if (extent == NULL || (new_addr != NULL && extent_size_get(extent) <
|
||||
size)) {
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
extent_heaps_remove(extent_heaps, extent);
|
||||
arena_chunk_cache_maybe_remove(arena, extent, cache);
|
||||
|
||||
leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent),
|
||||
PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent);
|
||||
assert(new_addr == NULL || leadsize == 0);
|
||||
assert(extent_size_get(extent) >= leadsize + size);
|
||||
trailsize = extent_size_get(extent) - leadsize - size;
|
||||
if (extent_zeroed_get(extent))
|
||||
*zero = true;
|
||||
if (extent_committed_get(extent))
|
||||
*commit = true;
|
||||
|
||||
/* Split the lead. */
|
||||
if (leadsize != 0) {
|
||||
extent_t *lead = extent;
|
||||
extent = chunk_split_wrapper(tsdn, arena, extent_hooks, lead,
|
||||
leadsize, leadsize, size + trailsize, usize + trailsize);
|
||||
if (extent == NULL) {
|
||||
chunk_leak(tsdn, arena, extent_hooks, cache, lead);
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
extent_heaps_insert(extent_heaps, lead);
|
||||
arena_chunk_cache_maybe_insert(arena, lead, cache);
|
||||
}
|
||||
|
||||
/* Split the trail. */
|
||||
if (trailsize != 0) {
|
||||
extent_t *trail = chunk_split_wrapper(tsdn, arena, extent_hooks,
|
||||
extent, size, usize, trailsize, trailsize);
|
||||
if (trail == NULL) {
|
||||
chunk_leak(tsdn, arena, extent_hooks, cache, extent);
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
extent_heaps_insert(extent_heaps, trail);
|
||||
arena_chunk_cache_maybe_insert(arena, trail, cache);
|
||||
} else if (leadsize == 0) {
|
||||
/*
|
||||
* Splitting causes usize to be set as a side effect, but no
|
||||
* splitting occurred.
|
||||
*/
|
||||
extent_usize_set(extent, usize);
|
||||
}
|
||||
|
||||
if (!extent_committed_get(extent) &&
|
||||
extent_hooks->commit(extent_base_get(extent),
|
||||
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) {
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
chunk_record(tsdn, arena, extent_hooks, extent_heaps, cache,
|
||||
extent);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
if (pad != 0)
|
||||
extent_addr_randomize(tsdn, extent, alignment);
|
||||
extent_active_set(extent, true);
|
||||
if (slab) {
|
||||
extent_slab_set(extent, slab);
|
||||
chunk_interior_register(tsdn, extent);
|
||||
}
|
||||
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
|
||||
if (*zero) {
|
||||
if (!extent_zeroed_get(extent)) {
|
||||
memset(extent_addr_get(extent), 0,
|
||||
extent_usize_get(extent));
|
||||
} else if (config_debug) {
|
||||
size_t i;
|
||||
size_t *p = (size_t *)(uintptr_t)
|
||||
extent_addr_get(extent);
|
||||
|
||||
for (i = 0; i < usize / sizeof(size_t); i++)
|
||||
assert(p[i] == 0);
|
||||
}
|
||||
}
|
||||
return (extent);
|
||||
}
|
||||
|
||||
/*
|
||||
* If the caller specifies (!*zero), it is still possible to receive zeroed
|
||||
* memory, in which case *zero is toggled to true. arena_chunk_alloc() takes
|
||||
* advantage of this to avoid demanding zeroed chunks, but taking advantage of
|
||||
* them if they are returned.
|
||||
*/
|
||||
static void *
|
||||
chunk_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
assert(size != 0);
|
||||
assert(alignment != 0);
|
||||
|
||||
/* "primary" dss. */
|
||||
if (have_dss && dss_prec == dss_prec_primary && (ret =
|
||||
chunk_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
|
||||
commit)) != NULL)
|
||||
return (ret);
|
||||
/* mmap. */
|
||||
if ((ret = chunk_alloc_mmap(new_addr, size, alignment, zero, commit)) !=
|
||||
NULL)
|
||||
return (ret);
|
||||
/* "secondary" dss. */
|
||||
if (have_dss && dss_prec == dss_prec_secondary && (ret =
|
||||
chunk_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
|
||||
commit)) != NULL)
|
||||
return (ret);
|
||||
|
||||
/* All strategies for allocation failed. */
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
extent_t *
|
||||
chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
|
||||
bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
bool commit;
|
||||
|
||||
assert(usize + pad != 0);
|
||||
assert(alignment != 0);
|
||||
|
||||
commit = true;
|
||||
extent = chunk_recycle(tsdn, arena, extent_hooks, arena->extents_cached,
|
||||
true, new_addr, usize, pad, alignment, zero, &commit, slab);
|
||||
if (extent == NULL)
|
||||
return (NULL);
|
||||
assert(commit);
|
||||
return (extent);
|
||||
}
|
||||
|
||||
static arena_t *
|
||||
chunk_arena_get(tsdn_t *tsdn, unsigned arena_ind)
|
||||
{
|
||||
arena_t *arena;
|
||||
|
||||
arena = arena_get(tsdn, arena_ind, false);
|
||||
/*
|
||||
* The arena we're allocating on behalf of must have been initialized
|
||||
* already.
|
||||
*/
|
||||
assert(arena != NULL);
|
||||
return (arena);
|
||||
}
|
||||
|
||||
static void *
|
||||
extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
bool *commit, unsigned arena_ind)
|
||||
{
|
||||
void *ret;
|
||||
tsdn_t *tsdn;
|
||||
arena_t *arena;
|
||||
|
||||
tsdn = tsdn_fetch();
|
||||
arena = chunk_arena_get(tsdn, arena_ind);
|
||||
ret = chunk_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
|
||||
commit, arena->dss_prec);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
|
||||
bool *commit, bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
|
||||
assert(usize != 0);
|
||||
assert(alignment != 0);
|
||||
|
||||
extent = chunk_recycle(tsdn, arena, extent_hooks,
|
||||
arena->extents_retained, false, new_addr, usize, pad, alignment,
|
||||
zero, commit, slab);
|
||||
if (extent != NULL && config_stats) {
|
||||
size_t size = usize + pad;
|
||||
arena->stats.retained -= size;
|
||||
}
|
||||
|
||||
return (extent);
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
chunk_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool *commit, bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
size_t size;
|
||||
void *addr;
|
||||
|
||||
size = usize + pad;
|
||||
extent = extent_alloc(tsdn, arena);
|
||||
if (extent == NULL)
|
||||
return (NULL);
|
||||
addr = extent_hooks->alloc(new_addr, size, alignment, zero, commit,
|
||||
arena->ind);
|
||||
if (addr == NULL) {
|
||||
extent_dalloc(tsdn, arena, extent);
|
||||
return (NULL);
|
||||
}
|
||||
extent_init(extent, arena, addr, size, usize, true, zero, commit, slab);
|
||||
if (pad != 0)
|
||||
extent_addr_randomize(tsdn, extent, alignment);
|
||||
if (chunk_register(tsdn, extent)) {
|
||||
chunk_leak(tsdn, arena, extent_hooks, false, extent);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
return (extent);
|
||||
}
|
||||
|
||||
extent_t *
|
||||
chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
|
||||
bool *commit, bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
|
||||
extent = chunk_alloc_retained(tsdn, arena, extent_hooks, new_addr,
|
||||
usize, pad, alignment, zero, commit, slab);
|
||||
if (extent == NULL) {
|
||||
extent = chunk_alloc_wrapper_hard(tsdn, arena, extent_hooks,
|
||||
new_addr, usize, pad, alignment, zero, commit, slab);
|
||||
}
|
||||
|
||||
return (extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
chunk_can_coalesce(const extent_t *a, const extent_t *b)
|
||||
{
|
||||
|
||||
if (extent_arena_get(a) != extent_arena_get(b))
|
||||
return (false);
|
||||
if (extent_active_get(a) != extent_active_get(b))
|
||||
return (false);
|
||||
if (extent_committed_get(a) != extent_committed_get(b))
|
||||
return (false);
|
||||
if (extent_retained_get(a) != extent_retained_get(b))
|
||||
return (false);
|
||||
|
||||
return (true);
|
||||
}
|
||||
|
||||
static void
|
||||
chunk_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache)
|
||||
{
|
||||
|
||||
if (!chunk_can_coalesce(a, b))
|
||||
return;
|
||||
|
||||
extent_heaps_remove(extent_heaps, a);
|
||||
extent_heaps_remove(extent_heaps, b);
|
||||
|
||||
arena_chunk_cache_maybe_remove(extent_arena_get(a), a, cache);
|
||||
arena_chunk_cache_maybe_remove(extent_arena_get(b), b, cache);
|
||||
|
||||
if (chunk_merge_wrapper(tsdn, arena, extent_hooks, a, b)) {
|
||||
extent_heaps_insert(extent_heaps, a);
|
||||
extent_heaps_insert(extent_heaps, b);
|
||||
arena_chunk_cache_maybe_insert(extent_arena_get(a), a, cache);
|
||||
arena_chunk_cache_maybe_insert(extent_arena_get(b), b, cache);
|
||||
return;
|
||||
}
|
||||
|
||||
extent_heaps_insert(extent_heaps, a);
|
||||
arena_chunk_cache_maybe_insert(extent_arena_get(a), a, cache);
|
||||
}
|
||||
|
||||
static void
|
||||
chunk_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent)
|
||||
{
|
||||
extent_t *prev, *next;
|
||||
|
||||
assert(!cache || !extent_zeroed_get(extent));
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
|
||||
|
||||
extent_usize_set(extent, 0);
|
||||
extent_active_set(extent, false);
|
||||
extent_zeroed_set(extent, !cache && extent_zeroed_get(extent));
|
||||
if (extent_slab_get(extent)) {
|
||||
chunk_interior_deregister(tsdn, extent);
|
||||
extent_slab_set(extent, false);
|
||||
}
|
||||
|
||||
assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent);
|
||||
extent_heaps_insert(extent_heaps, extent);
|
||||
arena_chunk_cache_maybe_insert(arena, extent, cache);
|
||||
|
||||
/* Try to coalesce forward. */
|
||||
next = rtree_read(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_past_get(extent), false);
|
||||
if (next != NULL) {
|
||||
chunk_try_coalesce(tsdn, arena, extent_hooks, extent, next,
|
||||
extent_heaps, cache);
|
||||
}
|
||||
|
||||
/* Try to coalesce backward. */
|
||||
prev = rtree_read(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_before_get(extent), false);
|
||||
if (prev != NULL) {
|
||||
chunk_try_coalesce(tsdn, arena, extent_hooks, prev, extent,
|
||||
extent_heaps, cache);
|
||||
}
|
||||
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent)
|
||||
{
|
||||
|
||||
assert(extent_base_get(extent) != NULL);
|
||||
assert(extent_size_get(extent) != 0);
|
||||
|
||||
extent_addr_set(extent, extent_base_get(extent));
|
||||
extent_zeroed_set(extent, false);
|
||||
|
||||
chunk_record(tsdn, arena, extent_hooks, arena->extents_cached, true,
|
||||
extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_dalloc_default(void *addr, size_t size, bool committed,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
if (!have_dss || !chunk_in_dss(tsdn_fetch(), addr))
|
||||
return (chunk_dalloc_mmap(addr, size));
|
||||
return (true);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent)
|
||||
{
|
||||
|
||||
assert(extent_base_get(extent) != NULL);
|
||||
assert(extent_size_get(extent) != 0);
|
||||
|
||||
extent_addr_set(extent, extent_base_get(extent));
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
/* Try to deallocate. */
|
||||
if (!extent_hooks->dalloc(extent_base_get(extent),
|
||||
extent_size_get(extent), extent_committed_get(extent),
|
||||
arena->ind)) {
|
||||
chunk_deregister(tsdn, extent);
|
||||
extent_dalloc(tsdn, arena, extent);
|
||||
return;
|
||||
}
|
||||
/* Try to decommit; purge if that fails. */
|
||||
if (extent_committed_get(extent)) {
|
||||
extent_committed_set(extent,
|
||||
extent_hooks->decommit(extent_base_get(extent),
|
||||
extent_size_get(extent), 0, extent_size_get(extent),
|
||||
arena->ind));
|
||||
}
|
||||
extent_zeroed_set(extent, !extent_committed_get(extent) ||
|
||||
!extent_hooks->purge(extent_base_get(extent),
|
||||
extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
|
||||
|
||||
if (config_stats)
|
||||
arena->stats.retained += extent_size_get(extent);
|
||||
|
||||
chunk_record(tsdn, arena, extent_hooks, arena->extents_retained, false,
|
||||
extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_commit_default(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
return (pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
length));
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_commit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent, size_t offset, size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
return (extent_hooks->commit(extent_base_get(extent),
|
||||
extent_size_get(extent), offset, length, arena->ind));
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_decommit_default(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
return (pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
length));
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
return (extent_hooks->decommit(extent_base_get(extent),
|
||||
extent_size_get(extent), offset, length, arena->ind));
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_purge_default(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
assert(addr != NULL);
|
||||
assert((offset & PAGE_MASK) == 0);
|
||||
assert(length != 0);
|
||||
assert((length & PAGE_MASK) == 0);
|
||||
|
||||
return (pages_purge((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
length));
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent, size_t offset, size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
return (extent_hooks->purge(extent_base_get(extent),
|
||||
extent_size_get(extent), offset, length, arena->ind));
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_split_default(void *addr, size_t size, size_t size_a, size_t size_b,
|
||||
bool committed, unsigned arena_ind)
|
||||
{
|
||||
|
||||
if (!maps_coalesce)
|
||||
return (true);
|
||||
return (false);
|
||||
}
|
||||
|
||||
extent_t *
|
||||
chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent, size_t size_a, size_t usize_a, size_t size_b,
|
||||
size_t usize_b)
|
||||
{
|
||||
extent_t *trail;
|
||||
rtree_elm_t *lead_elm_a, *lead_elm_b, *trail_elm_a, *trail_elm_b;
|
||||
|
||||
assert(extent_size_get(extent) == size_a + size_b);
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
|
||||
trail = extent_alloc(tsdn, arena);
|
||||
if (trail == NULL)
|
||||
goto label_error_a;
|
||||
|
||||
{
|
||||
extent_t lead;
|
||||
|
||||
extent_init(&lead, arena, extent_addr_get(extent), size_a,
|
||||
usize_a, extent_active_get(extent),
|
||||
extent_zeroed_get(extent), extent_committed_get(extent),
|
||||
extent_slab_get(extent));
|
||||
|
||||
if (extent_rtree_acquire(tsdn, &lead, false, true, &lead_elm_a,
|
||||
&lead_elm_b))
|
||||
goto label_error_b;
|
||||
}
|
||||
|
||||
extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) +
|
||||
size_a), size_b, usize_b, extent_active_get(extent),
|
||||
extent_zeroed_get(extent), extent_committed_get(extent),
|
||||
extent_slab_get(extent));
|
||||
if (extent_rtree_acquire(tsdn, trail, false, true, &trail_elm_a,
|
||||
&trail_elm_b))
|
||||
goto label_error_c;
|
||||
|
||||
if (extent_hooks->split(extent_base_get(extent), size_a + size_b, size_a,
|
||||
size_b, extent_committed_get(extent), arena->ind))
|
||||
goto label_error_d;
|
||||
|
||||
extent_size_set(extent, size_a);
|
||||
extent_usize_set(extent, usize_a);
|
||||
|
||||
extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent);
|
||||
extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail);
|
||||
|
||||
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
|
||||
extent_rtree_release(tsdn, trail_elm_a, trail_elm_b);
|
||||
|
||||
return (trail);
|
||||
label_error_d:
|
||||
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
|
||||
label_error_c:
|
||||
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
|
||||
label_error_b:
|
||||
extent_dalloc(tsdn, arena, trail);
|
||||
label_error_a:
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
|
||||
bool committed, unsigned arena_ind)
|
||||
{
|
||||
|
||||
if (!maps_coalesce)
|
||||
return (true);
|
||||
if (have_dss) {
|
||||
tsdn_t *tsdn = tsdn_fetch();
|
||||
if (chunk_in_dss(tsdn, addr_a) != chunk_in_dss(tsdn, addr_b))
|
||||
return (true);
|
||||
}
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *a, extent_t *b)
|
||||
{
|
||||
rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
if (extent_hooks->merge(extent_base_get(a), extent_size_get(a),
|
||||
extent_base_get(b), extent_size_get(b), extent_committed_get(a),
|
||||
arena->ind))
|
||||
return (true);
|
||||
|
||||
/*
|
||||
* The rtree writes must happen while all the relevant elements are
|
||||
* owned, so the following code uses decomposed helper functions rather
|
||||
* than chunk_{,de}register() to do things in the right order.
|
||||
*/
|
||||
extent_rtree_acquire(tsdn, a, true, false, &a_elm_a, &a_elm_b);
|
||||
extent_rtree_acquire(tsdn, b, true, false, &b_elm_a, &b_elm_b);
|
||||
|
||||
if (a_elm_b != NULL) {
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, a_elm_b, NULL);
|
||||
rtree_elm_release(tsdn, &extents_rtree, a_elm_b);
|
||||
}
|
||||
if (b_elm_b != NULL) {
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, b_elm_a, NULL);
|
||||
rtree_elm_release(tsdn, &extents_rtree, b_elm_a);
|
||||
} else
|
||||
b_elm_b = b_elm_a;
|
||||
|
||||
extent_size_set(a, extent_size_get(a) + extent_size_get(b));
|
||||
extent_usize_set(a, extent_usize_get(a) + extent_usize_get(b));
|
||||
extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b));
|
||||
|
||||
extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a);
|
||||
extent_rtree_release(tsdn, a_elm_a, b_elm_b);
|
||||
|
||||
extent_dalloc(tsdn, extent_arena_get(b), b);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_boot(void)
|
||||
@ -959,29 +47,5 @@ chunk_boot(void)
|
||||
chunksize_mask = chunksize - 1;
|
||||
chunk_npages = (chunksize >> LG_PAGE);
|
||||
|
||||
if (have_dss && chunk_dss_boot())
|
||||
return (true);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_prefork(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
chunk_dss_prefork(tsdn);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_postfork_parent(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
chunk_dss_postfork_parent(tsdn);
|
||||
}
|
||||
|
||||
void
|
||||
chunk_postfork_child(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
chunk_dss_postfork_child(tsdn);
|
||||
}
|
||||
|
||||
@ -1560,11 +1560,11 @@ arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
|
||||
dss_prec_old = arena_dss_prec_get(tsd_tsdn(tsd), arena);
|
||||
} else {
|
||||
if (dss_prec != dss_prec_limit &&
|
||||
chunk_dss_prec_set(tsd_tsdn(tsd), dss_prec)) {
|
||||
extent_dss_prec_set(tsd_tsdn(tsd), dss_prec)) {
|
||||
ret = EFAULT;
|
||||
goto label_return;
|
||||
}
|
||||
dss_prec_old = chunk_dss_prec_get(tsd_tsdn(tsd));
|
||||
dss_prec_old = extent_dss_prec_get(tsd_tsdn(tsd));
|
||||
}
|
||||
|
||||
dss = dss_prec_names[dss_prec_old];
|
||||
|
||||
929
src/extent.c
929
src/extent.c
@ -6,6 +6,45 @@
|
||||
|
||||
rtree_t extents_rtree;
|
||||
|
||||
static void *extent_alloc_default(void *new_addr, size_t size,
|
||||
size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
|
||||
static bool extent_dalloc_default(void *addr, size_t size, bool committed,
|
||||
unsigned arena_ind);
|
||||
static bool extent_commit_default(void *addr, size_t size, size_t offset,
|
||||
size_t length, unsigned arena_ind);
|
||||
static bool extent_decommit_default(void *addr, size_t size, size_t offset,
|
||||
size_t length, unsigned arena_ind);
|
||||
static bool extent_purge_default(void *addr, size_t size, size_t offset,
|
||||
size_t length, unsigned arena_ind);
|
||||
static bool extent_split_default(void *addr, size_t size, size_t size_a,
|
||||
size_t size_b, bool committed, unsigned arena_ind);
|
||||
static bool extent_merge_default(void *addr_a, size_t size_a, void *addr_b,
|
||||
size_t size_b, bool committed, unsigned arena_ind);
|
||||
|
||||
const extent_hooks_t extent_hooks_default = {
|
||||
extent_alloc_default,
|
||||
extent_dalloc_default,
|
||||
extent_commit_default,
|
||||
extent_decommit_default,
|
||||
extent_purge_default,
|
||||
extent_split_default,
|
||||
extent_merge_default
|
||||
};
|
||||
|
||||
/* Used exclusively for gdump triggering. */
|
||||
static size_t curchunks;
|
||||
static size_t highchunks;
|
||||
|
||||
/******************************************************************************/
|
||||
/*
|
||||
* Function prototypes for static functions that are referenced prior to
|
||||
* definition.
|
||||
*/
|
||||
|
||||
static void extent_record(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_heap_t extent_heaps[NPSIZES],
|
||||
bool cache, extent_t *extent);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
extent_t *
|
||||
@ -34,6 +73,91 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
|
||||
malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
|
||||
}
|
||||
|
||||
static extent_hooks_t
|
||||
extent_hooks_get_locked(arena_t *arena)
|
||||
{
|
||||
|
||||
return (arena->extent_hooks);
|
||||
}
|
||||
|
||||
extent_hooks_t
|
||||
extent_hooks_get(tsdn_t *tsdn, arena_t *arena)
|
||||
{
|
||||
extent_hooks_t extent_hooks;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
extent_hooks = extent_hooks_get_locked(arena);
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
|
||||
return (extent_hooks);
|
||||
}
|
||||
|
||||
extent_hooks_t
|
||||
extent_hooks_set(tsdn_t *tsdn, arena_t *arena,
|
||||
const extent_hooks_t *extent_hooks)
|
||||
{
|
||||
extent_hooks_t old_extent_hooks;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
old_extent_hooks = arena->extent_hooks;
|
||||
/*
|
||||
* Copy each field atomically so that it is impossible for readers to
|
||||
* see partially updated pointers. There are places where readers only
|
||||
* need one hook function pointer (therefore no need to copy the
|
||||
* entirety of arena->extent_hooks), and stale reads do not affect
|
||||
* correctness, so they perform unlocked reads.
|
||||
*/
|
||||
#define ATOMIC_COPY_HOOK(n) do { \
|
||||
union { \
|
||||
extent_##n##_t **n; \
|
||||
void **v; \
|
||||
} u; \
|
||||
u.n = &arena->extent_hooks.n; \
|
||||
atomic_write_p(u.v, extent_hooks->n); \
|
||||
} while (0)
|
||||
ATOMIC_COPY_HOOK(alloc);
|
||||
ATOMIC_COPY_HOOK(dalloc);
|
||||
ATOMIC_COPY_HOOK(commit);
|
||||
ATOMIC_COPY_HOOK(decommit);
|
||||
ATOMIC_COPY_HOOK(purge);
|
||||
ATOMIC_COPY_HOOK(split);
|
||||
ATOMIC_COPY_HOOK(merge);
|
||||
#undef ATOMIC_COPY_HOOK
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
|
||||
return (old_extent_hooks);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_hooks_assure_initialized_impl(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, bool locked)
|
||||
{
|
||||
static const extent_hooks_t uninitialized_hooks =
|
||||
EXTENT_HOOKS_INITIALIZER;
|
||||
|
||||
if (memcmp(extent_hooks, &uninitialized_hooks, sizeof(extent_hooks_t))
|
||||
== 0) {
|
||||
*extent_hooks = locked ? extent_hooks_get_locked(arena) :
|
||||
extent_hooks_get(tsdn, arena);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
extent_hooks_assure_initialized_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, true);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized_impl(tsdn, arena, extent_hooks, false);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef extent_size_quantize_floor
|
||||
#define extent_size_quantize_floor JEMALLOC_N(n_extent_size_quantize_floor)
|
||||
@ -118,6 +242,787 @@ extent_ad_comp(const extent_t *a, const extent_t *b)
|
||||
/* Generate pairing heap functions. */
|
||||
ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_ad_comp)
|
||||
|
||||
static void
|
||||
extent_heaps_insert(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
|
||||
{
|
||||
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
|
||||
pszind_t pind = psz2ind(psz);
|
||||
extent_heap_insert(&extent_heaps[pind], extent);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_heaps_remove(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
|
||||
{
|
||||
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
|
||||
pszind_t pind = psz2ind(psz);
|
||||
extent_heap_remove(&extent_heaps[pind], extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_rtree_acquire(tsdn_t *tsdn, const extent_t *extent, bool dependent,
|
||||
bool init_missing, rtree_elm_t **r_elm_a, rtree_elm_t **r_elm_b)
|
||||
{
|
||||
|
||||
*r_elm_a = rtree_elm_acquire(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_base_get(extent), dependent, init_missing);
|
||||
if (!dependent && *r_elm_a == NULL)
|
||||
return (true);
|
||||
assert(*r_elm_a != NULL);
|
||||
|
||||
if (extent_size_get(extent) > PAGE) {
|
||||
*r_elm_b = rtree_elm_acquire(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_last_get(extent), dependent,
|
||||
init_missing);
|
||||
if (!dependent && *r_elm_b == NULL)
|
||||
return (true);
|
||||
assert(*r_elm_b != NULL);
|
||||
} else
|
||||
*r_elm_b = NULL;
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_rtree_write_acquired(tsdn_t *tsdn, rtree_elm_t *elm_a,
|
||||
rtree_elm_t *elm_b, const extent_t *extent)
|
||||
{
|
||||
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, elm_a, extent);
|
||||
if (elm_b != NULL)
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, elm_b, extent);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b)
|
||||
{
|
||||
|
||||
rtree_elm_release(tsdn, &extents_rtree, elm_a);
|
||||
if (elm_b != NULL)
|
||||
rtree_elm_release(tsdn, &extents_rtree, elm_b);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_interior_register(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
assert(extent_slab_get(extent));
|
||||
|
||||
for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
|
||||
rtree_write(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
|
||||
LG_PAGE), extent);
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_register(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
rtree_elm_t *elm_a, *elm_b;
|
||||
|
||||
if (extent_rtree_acquire(tsdn, extent, false, true, &elm_a, &elm_b))
|
||||
return (true);
|
||||
extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent);
|
||||
if (extent_slab_get(extent))
|
||||
extent_interior_register(tsdn, extent);
|
||||
extent_rtree_release(tsdn, elm_a, elm_b);
|
||||
|
||||
if (config_prof && opt_prof && extent_active_get(extent)) {
|
||||
size_t nadd = (extent_size_get(extent) == 0) ? 1 :
|
||||
extent_size_get(extent) / chunksize;
|
||||
size_t cur = atomic_add_z(&curchunks, nadd);
|
||||
size_t high = atomic_read_z(&highchunks);
|
||||
while (cur > high && atomic_cas_z(&highchunks, high, cur)) {
|
||||
/*
|
||||
* Don't refresh cur, because it may have decreased
|
||||
* since this thread lost the highchunks update race.
|
||||
*/
|
||||
high = atomic_read_z(&highchunks);
|
||||
}
|
||||
if (cur > high && prof_gdump_get_unlocked())
|
||||
prof_gdump(tsdn);
|
||||
}
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_interior_deregister(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
assert(extent_slab_get(extent));
|
||||
|
||||
for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
|
||||
rtree_clear(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
|
||||
LG_PAGE));
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
extent_deregister(tsdn_t *tsdn, const extent_t *extent)
|
||||
{
|
||||
rtree_elm_t *elm_a, *elm_b;
|
||||
|
||||
extent_rtree_acquire(tsdn, extent, true, false, &elm_a, &elm_b);
|
||||
extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL);
|
||||
if (extent_slab_get(extent))
|
||||
extent_interior_deregister(tsdn, extent);
|
||||
extent_rtree_release(tsdn, elm_a, elm_b);
|
||||
|
||||
if (config_prof && opt_prof && extent_active_get(extent)) {
|
||||
size_t nsub = (extent_size_get(extent) == 0) ? 1 :
|
||||
extent_size_get(extent) / chunksize;
|
||||
assert(atomic_read_z(&curchunks) >= nsub);
|
||||
atomic_sub_z(&curchunks, nsub);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Do first-best-fit extent selection, i.e. select the lowest extent that best
|
||||
* fits.
|
||||
*/
|
||||
static extent_t *
|
||||
extent_first_best_fit(arena_t *arena, extent_heap_t extent_heaps[NPSIZES],
|
||||
size_t size)
|
||||
{
|
||||
pszind_t pind, i;
|
||||
|
||||
pind = psz2ind(extent_size_quantize_ceil(size));
|
||||
for (i = pind; i < NPSIZES; i++) {
|
||||
extent_t *extent = extent_heap_first(&extent_heaps[i]);
|
||||
if (extent != NULL)
|
||||
return (extent);
|
||||
}
|
||||
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
bool cache, extent_t *extent)
|
||||
{
|
||||
|
||||
/*
|
||||
* Leak extent after making sure its pages have already been purged, so
|
||||
* that this is only a virtual memory leak.
|
||||
*/
|
||||
if (cache) {
|
||||
extent_purge_wrapper(tsdn, arena, extent_hooks, extent, 0,
|
||||
extent_size_get(extent));
|
||||
}
|
||||
extent_dalloc(tsdn, arena, extent);
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr,
|
||||
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
|
||||
bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
size_t size, alloc_size, leadsize, trailsize;
|
||||
|
||||
assert(new_addr == NULL || !slab);
|
||||
assert(pad == 0 || !slab);
|
||||
|
||||
size = usize + pad;
|
||||
alloc_size = s2u(size + PAGE_CEILING(alignment) - PAGE);
|
||||
/* Beware size_t wrap-around. */
|
||||
if (alloc_size < usize)
|
||||
return (NULL);
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
|
||||
if (new_addr != NULL) {
|
||||
rtree_elm_t *elm;
|
||||
|
||||
elm = rtree_elm_acquire(tsdn, &extents_rtree,
|
||||
(uintptr_t)new_addr, false, false);
|
||||
if (elm != NULL) {
|
||||
extent = rtree_elm_read_acquired(tsdn, &extents_rtree,
|
||||
elm);
|
||||
if (extent != NULL && (extent_active_get(extent) ||
|
||||
extent_retained_get(extent) == cache))
|
||||
extent = NULL;
|
||||
rtree_elm_release(tsdn, &extents_rtree, elm);
|
||||
} else
|
||||
extent = NULL;
|
||||
} else
|
||||
extent = extent_first_best_fit(arena, extent_heaps, alloc_size);
|
||||
if (extent == NULL || (new_addr != NULL && extent_size_get(extent) <
|
||||
size)) {
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
extent_heaps_remove(extent_heaps, extent);
|
||||
arena_extent_cache_maybe_remove(arena, extent, cache);
|
||||
|
||||
leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent),
|
||||
PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent);
|
||||
assert(new_addr == NULL || leadsize == 0);
|
||||
assert(extent_size_get(extent) >= leadsize + size);
|
||||
trailsize = extent_size_get(extent) - leadsize - size;
|
||||
if (extent_zeroed_get(extent))
|
||||
*zero = true;
|
||||
if (extent_committed_get(extent))
|
||||
*commit = true;
|
||||
|
||||
/* Split the lead. */
|
||||
if (leadsize != 0) {
|
||||
extent_t *lead = extent;
|
||||
extent = extent_split_wrapper(tsdn, arena, extent_hooks, lead,
|
||||
leadsize, leadsize, size + trailsize, usize + trailsize);
|
||||
if (extent == NULL) {
|
||||
extent_leak(tsdn, arena, extent_hooks, cache, lead);
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
extent_heaps_insert(extent_heaps, lead);
|
||||
arena_extent_cache_maybe_insert(arena, lead, cache);
|
||||
}
|
||||
|
||||
/* Split the trail. */
|
||||
if (trailsize != 0) {
|
||||
extent_t *trail = extent_split_wrapper(tsdn, arena,
|
||||
extent_hooks, extent, size, usize, trailsize, trailsize);
|
||||
if (trail == NULL) {
|
||||
extent_leak(tsdn, arena, extent_hooks, cache, extent);
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
return (NULL);
|
||||
}
|
||||
extent_heaps_insert(extent_heaps, trail);
|
||||
arena_extent_cache_maybe_insert(arena, trail, cache);
|
||||
} else if (leadsize == 0) {
|
||||
/*
|
||||
* Splitting causes usize to be set as a side effect, but no
|
||||
* splitting occurred.
|
||||
*/
|
||||
extent_usize_set(extent, usize);
|
||||
}
|
||||
|
||||
if (!extent_committed_get(extent) &&
|
||||
extent_hooks->commit(extent_base_get(extent),
|
||||
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) {
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
extent_record(tsdn, arena, extent_hooks, extent_heaps, cache,
|
||||
extent);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
if (pad != 0)
|
||||
extent_addr_randomize(tsdn, extent, alignment);
|
||||
extent_active_set(extent, true);
|
||||
if (slab) {
|
||||
extent_slab_set(extent, slab);
|
||||
extent_interior_register(tsdn, extent);
|
||||
}
|
||||
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
|
||||
if (*zero) {
|
||||
if (!extent_zeroed_get(extent)) {
|
||||
memset(extent_addr_get(extent), 0,
|
||||
extent_usize_get(extent));
|
||||
} else if (config_debug) {
|
||||
size_t i;
|
||||
size_t *p = (size_t *)(uintptr_t)
|
||||
extent_addr_get(extent);
|
||||
|
||||
for (i = 0; i < usize / sizeof(size_t); i++)
|
||||
assert(p[i] == 0);
|
||||
}
|
||||
}
|
||||
return (extent);
|
||||
}
|
||||
|
||||
/*
|
||||
* If the caller specifies (!*zero), it is still possible to receive zeroed
|
||||
* memory, in which case *zero is toggled to true. arena_extent_alloc() takes
|
||||
* advantage of this to avoid demanding zeroed extents, but taking advantage of
|
||||
* them if they are returned.
|
||||
*/
|
||||
static void *
|
||||
extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
assert(size != 0);
|
||||
assert(alignment != 0);
|
||||
|
||||
/* "primary" dss. */
|
||||
if (have_dss && dss_prec == dss_prec_primary && (ret =
|
||||
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
|
||||
commit)) != NULL)
|
||||
return (ret);
|
||||
/* mmap. */
|
||||
if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit))
|
||||
!= NULL)
|
||||
return (ret);
|
||||
/* "secondary" dss. */
|
||||
if (have_dss && dss_prec == dss_prec_secondary && (ret =
|
||||
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
|
||||
commit)) != NULL)
|
||||
return (ret);
|
||||
|
||||
/* All strategies for allocation failed. */
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
extent_t *
|
||||
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
|
||||
bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
bool commit;
|
||||
|
||||
assert(usize + pad != 0);
|
||||
assert(alignment != 0);
|
||||
|
||||
commit = true;
|
||||
extent = extent_recycle(tsdn, arena, extent_hooks,
|
||||
arena->extents_cached, true, new_addr, usize, pad, alignment, zero,
|
||||
&commit, slab);
|
||||
if (extent == NULL)
|
||||
return (NULL);
|
||||
assert(commit);
|
||||
return (extent);
|
||||
}
|
||||
|
||||
static void *
|
||||
extent_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
bool *commit, unsigned arena_ind)
|
||||
{
|
||||
void *ret;
|
||||
tsdn_t *tsdn;
|
||||
arena_t *arena;
|
||||
|
||||
tsdn = tsdn_fetch();
|
||||
arena = arena_get(tsdn, arena_ind, false);
|
||||
/*
|
||||
* The arena we're allocating on behalf of must have been initialized
|
||||
* already.
|
||||
*/
|
||||
assert(arena != NULL);
|
||||
ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
|
||||
commit, arena->dss_prec);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool *commit, bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
|
||||
assert(usize != 0);
|
||||
assert(alignment != 0);
|
||||
|
||||
extent = extent_recycle(tsdn, arena, extent_hooks,
|
||||
arena->extents_retained, false, new_addr, usize, pad, alignment,
|
||||
zero, commit, slab);
|
||||
if (extent != NULL && config_stats) {
|
||||
size_t size = usize + pad;
|
||||
arena->stats.retained -= size;
|
||||
}
|
||||
|
||||
return (extent);
|
||||
}
|
||||
|
||||
static extent_t *
|
||||
extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, void *new_addr, size_t usize, size_t pad,
|
||||
size_t alignment, bool *zero, bool *commit, bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
size_t size;
|
||||
void *addr;
|
||||
|
||||
size = usize + pad;
|
||||
extent = extent_alloc(tsdn, arena);
|
||||
if (extent == NULL)
|
||||
return (NULL);
|
||||
addr = extent_hooks->alloc(new_addr, size, alignment, zero, commit,
|
||||
arena->ind);
|
||||
if (addr == NULL) {
|
||||
extent_dalloc(tsdn, arena, extent);
|
||||
return (NULL);
|
||||
}
|
||||
extent_init(extent, arena, addr, size, usize, true, zero, commit, slab);
|
||||
if (pad != 0)
|
||||
extent_addr_randomize(tsdn, extent, alignment);
|
||||
if (extent_register(tsdn, extent)) {
|
||||
extent_leak(tsdn, arena, extent_hooks, false, extent);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
return (extent);
|
||||
}
|
||||
|
||||
extent_t *
|
||||
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero,
|
||||
bool *commit, bool slab)
|
||||
{
|
||||
extent_t *extent;
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
|
||||
extent = extent_alloc_retained(tsdn, arena, extent_hooks, new_addr,
|
||||
usize, pad, alignment, zero, commit, slab);
|
||||
if (extent == NULL) {
|
||||
extent = extent_alloc_wrapper_hard(tsdn, arena, extent_hooks,
|
||||
new_addr, usize, pad, alignment, zero, commit, slab);
|
||||
}
|
||||
|
||||
return (extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_can_coalesce(const extent_t *a, const extent_t *b)
|
||||
{
|
||||
|
||||
if (extent_arena_get(a) != extent_arena_get(b))
|
||||
return (false);
|
||||
if (extent_active_get(a) != extent_active_get(b))
|
||||
return (false);
|
||||
if (extent_committed_get(a) != extent_committed_get(b))
|
||||
return (false);
|
||||
if (extent_retained_get(a) != extent_retained_get(b))
|
||||
return (false);
|
||||
|
||||
return (true);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *a, extent_t *b, extent_heap_t extent_heaps[NPSIZES], bool cache)
|
||||
{
|
||||
|
||||
if (!extent_can_coalesce(a, b))
|
||||
return;
|
||||
|
||||
extent_heaps_remove(extent_heaps, a);
|
||||
extent_heaps_remove(extent_heaps, b);
|
||||
|
||||
arena_extent_cache_maybe_remove(extent_arena_get(a), a, cache);
|
||||
arena_extent_cache_maybe_remove(extent_arena_get(b), b, cache);
|
||||
|
||||
if (extent_merge_wrapper(tsdn, arena, extent_hooks, a, b)) {
|
||||
extent_heaps_insert(extent_heaps, a);
|
||||
extent_heaps_insert(extent_heaps, b);
|
||||
arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache);
|
||||
arena_extent_cache_maybe_insert(extent_arena_get(b), b, cache);
|
||||
return;
|
||||
}
|
||||
|
||||
extent_heaps_insert(extent_heaps, a);
|
||||
arena_extent_cache_maybe_insert(extent_arena_get(a), a, cache);
|
||||
}
|
||||
|
||||
static void
|
||||
extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent)
|
||||
{
|
||||
extent_t *prev, *next;
|
||||
|
||||
assert(!cache || !extent_zeroed_get(extent));
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->extents_mtx);
|
||||
extent_hooks_assure_initialized_locked(tsdn, arena, extent_hooks);
|
||||
|
||||
extent_usize_set(extent, 0);
|
||||
extent_active_set(extent, false);
|
||||
extent_zeroed_set(extent, !cache && extent_zeroed_get(extent));
|
||||
if (extent_slab_get(extent)) {
|
||||
extent_interior_deregister(tsdn, extent);
|
||||
extent_slab_set(extent, false);
|
||||
}
|
||||
|
||||
assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent);
|
||||
extent_heaps_insert(extent_heaps, extent);
|
||||
arena_extent_cache_maybe_insert(arena, extent, cache);
|
||||
|
||||
/* Try to coalesce forward. */
|
||||
next = rtree_read(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_past_get(extent), false);
|
||||
if (next != NULL) {
|
||||
extent_try_coalesce(tsdn, arena, extent_hooks, extent, next,
|
||||
extent_heaps, cache);
|
||||
}
|
||||
|
||||
/* Try to coalesce backward. */
|
||||
prev = rtree_read(tsdn, &extents_rtree,
|
||||
(uintptr_t)extent_before_get(extent), false);
|
||||
if (prev != NULL) {
|
||||
extent_try_coalesce(tsdn, arena, extent_hooks, prev, extent,
|
||||
extent_heaps, cache);
|
||||
}
|
||||
|
||||
malloc_mutex_unlock(tsdn, &arena->extents_mtx);
|
||||
}
|
||||
|
||||
void
|
||||
extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent)
|
||||
{
|
||||
|
||||
assert(extent_base_get(extent) != NULL);
|
||||
assert(extent_size_get(extent) != 0);
|
||||
|
||||
extent_addr_set(extent, extent_base_get(extent));
|
||||
extent_zeroed_set(extent, false);
|
||||
|
||||
extent_record(tsdn, arena, extent_hooks, arena->extents_cached, true,
|
||||
extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_dalloc_default(void *addr, size_t size, bool committed,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
if (!have_dss || !extent_in_dss(tsdn_fetch(), addr))
|
||||
return (extent_dalloc_mmap(addr, size));
|
||||
return (true);
|
||||
}
|
||||
|
||||
void
|
||||
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent)
|
||||
{
|
||||
|
||||
assert(extent_base_get(extent) != NULL);
|
||||
assert(extent_size_get(extent) != 0);
|
||||
|
||||
extent_addr_set(extent, extent_base_get(extent));
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
/* Try to deallocate. */
|
||||
if (!extent_hooks->dalloc(extent_base_get(extent),
|
||||
extent_size_get(extent), extent_committed_get(extent),
|
||||
arena->ind)) {
|
||||
extent_deregister(tsdn, extent);
|
||||
extent_dalloc(tsdn, arena, extent);
|
||||
return;
|
||||
}
|
||||
/* Try to decommit; purge if that fails. */
|
||||
if (extent_committed_get(extent)) {
|
||||
extent_committed_set(extent,
|
||||
extent_hooks->decommit(extent_base_get(extent),
|
||||
extent_size_get(extent), 0, extent_size_get(extent),
|
||||
arena->ind));
|
||||
}
|
||||
extent_zeroed_set(extent, !extent_committed_get(extent) ||
|
||||
!extent_hooks->purge(extent_base_get(extent),
|
||||
extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
|
||||
|
||||
if (config_stats)
|
||||
arena->stats.retained += extent_size_get(extent);
|
||||
|
||||
extent_record(tsdn, arena, extent_hooks, arena->extents_retained, false,
|
||||
extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_commit_default(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
return (pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
length));
|
||||
}
|
||||
|
||||
bool
|
||||
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
return (extent_hooks->commit(extent_base_get(extent),
|
||||
extent_size_get(extent), offset, length, arena->ind));
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_decommit_default(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
return (pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
length));
|
||||
}
|
||||
|
||||
bool
|
||||
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
return (extent_hooks->decommit(extent_base_get(extent),
|
||||
extent_size_get(extent), offset, length, arena->ind));
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_purge_default(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
assert(addr != NULL);
|
||||
assert((offset & PAGE_MASK) == 0);
|
||||
assert(length != 0);
|
||||
assert((length & PAGE_MASK) == 0);
|
||||
|
||||
return (pages_purge((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
length));
|
||||
}
|
||||
|
||||
bool
|
||||
extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent, size_t offset, size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
return (extent_hooks->purge(extent_base_get(extent),
|
||||
extent_size_get(extent), offset, length, arena->ind));
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_split_default(void *addr, size_t size, size_t size_a, size_t size_b,
|
||||
bool committed, unsigned arena_ind)
|
||||
{
|
||||
|
||||
if (!maps_coalesce)
|
||||
return (true);
|
||||
return (false);
|
||||
}
|
||||
|
||||
extent_t *
|
||||
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *extent, size_t size_a, size_t usize_a, size_t size_b,
|
||||
size_t usize_b)
|
||||
{
|
||||
extent_t *trail;
|
||||
rtree_elm_t *lead_elm_a, *lead_elm_b, *trail_elm_a, *trail_elm_b;
|
||||
|
||||
assert(extent_size_get(extent) == size_a + size_b);
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
|
||||
trail = extent_alloc(tsdn, arena);
|
||||
if (trail == NULL)
|
||||
goto label_error_a;
|
||||
|
||||
{
|
||||
extent_t lead;
|
||||
|
||||
extent_init(&lead, arena, extent_addr_get(extent), size_a,
|
||||
usize_a, extent_active_get(extent),
|
||||
extent_zeroed_get(extent), extent_committed_get(extent),
|
||||
extent_slab_get(extent));
|
||||
|
||||
if (extent_rtree_acquire(tsdn, &lead, false, true, &lead_elm_a,
|
||||
&lead_elm_b))
|
||||
goto label_error_b;
|
||||
}
|
||||
|
||||
extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) +
|
||||
size_a), size_b, usize_b, extent_active_get(extent),
|
||||
extent_zeroed_get(extent), extent_committed_get(extent),
|
||||
extent_slab_get(extent));
|
||||
if (extent_rtree_acquire(tsdn, trail, false, true, &trail_elm_a,
|
||||
&trail_elm_b))
|
||||
goto label_error_c;
|
||||
|
||||
if (extent_hooks->split(extent_base_get(extent), size_a + size_b,
|
||||
size_a, size_b, extent_committed_get(extent), arena->ind))
|
||||
goto label_error_d;
|
||||
|
||||
extent_size_set(extent, size_a);
|
||||
extent_usize_set(extent, usize_a);
|
||||
|
||||
extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent);
|
||||
extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail);
|
||||
|
||||
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
|
||||
extent_rtree_release(tsdn, trail_elm_a, trail_elm_b);
|
||||
|
||||
return (trail);
|
||||
label_error_d:
|
||||
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
|
||||
label_error_c:
|
||||
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
|
||||
label_error_b:
|
||||
extent_dalloc(tsdn, arena, trail);
|
||||
label_error_a:
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_merge_default(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
|
||||
bool committed, unsigned arena_ind)
|
||||
{
|
||||
|
||||
if (!maps_coalesce)
|
||||
return (true);
|
||||
if (have_dss) {
|
||||
tsdn_t *tsdn = tsdn_fetch();
|
||||
if (extent_in_dss(tsdn, addr_a) != extent_in_dss(tsdn, addr_b))
|
||||
return (true);
|
||||
}
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
bool
|
||||
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
|
||||
extent_t *a, extent_t *b)
|
||||
{
|
||||
rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
|
||||
|
||||
extent_hooks_assure_initialized(tsdn, arena, extent_hooks);
|
||||
if (extent_hooks->merge(extent_base_get(a), extent_size_get(a),
|
||||
extent_base_get(b), extent_size_get(b), extent_committed_get(a),
|
||||
arena->ind))
|
||||
return (true);
|
||||
|
||||
/*
|
||||
* The rtree writes must happen while all the relevant elements are
|
||||
* owned, so the following code uses decomposed helper functions rather
|
||||
* than extent_{,de}register() to do things in the right order.
|
||||
*/
|
||||
extent_rtree_acquire(tsdn, a, true, false, &a_elm_a, &a_elm_b);
|
||||
extent_rtree_acquire(tsdn, b, true, false, &b_elm_a, &b_elm_b);
|
||||
|
||||
if (a_elm_b != NULL) {
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, a_elm_b, NULL);
|
||||
rtree_elm_release(tsdn, &extents_rtree, a_elm_b);
|
||||
}
|
||||
if (b_elm_b != NULL) {
|
||||
rtree_elm_write_acquired(tsdn, &extents_rtree, b_elm_a, NULL);
|
||||
rtree_elm_release(tsdn, &extents_rtree, b_elm_a);
|
||||
} else
|
||||
b_elm_b = b_elm_a;
|
||||
|
||||
extent_size_set(a, extent_size_get(a) + extent_size_get(b));
|
||||
extent_usize_set(a, extent_usize_get(a) + extent_usize_get(b));
|
||||
extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b));
|
||||
|
||||
extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a);
|
||||
extent_rtree_release(tsdn, a_elm_a, b_elm_b);
|
||||
|
||||
extent_dalloc(tsdn, extent_arena_get(b), b);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
bool
|
||||
extent_boot(void)
|
||||
{
|
||||
@ -126,5 +1031,29 @@ extent_boot(void)
|
||||
LG_PAGE)))
|
||||
return (true);
|
||||
|
||||
if (have_dss && extent_dss_boot())
|
||||
return (true);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
void
|
||||
extent_prefork(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
extent_dss_prefork(tsdn);
|
||||
}
|
||||
|
||||
void
|
||||
extent_postfork_parent(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
extent_dss_postfork_parent(tsdn);
|
||||
}
|
||||
|
||||
void
|
||||
extent_postfork_child(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
extent_dss_postfork_child(tsdn);
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
#define JEMALLOC_CHUNK_DSS_C_
|
||||
#define JEMALLOC_EXTENT_DSS_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
/******************************************************************************/
|
||||
/* Data. */
|
||||
@ -29,7 +29,7 @@ static void *dss_max;
|
||||
/******************************************************************************/
|
||||
|
||||
static void *
|
||||
chunk_dss_sbrk(intptr_t increment)
|
||||
extent_dss_sbrk(intptr_t increment)
|
||||
{
|
||||
|
||||
#ifdef JEMALLOC_DSS
|
||||
@ -41,7 +41,7 @@ chunk_dss_sbrk(intptr_t increment)
|
||||
}
|
||||
|
||||
dss_prec_t
|
||||
chunk_dss_prec_get(tsdn_t *tsdn)
|
||||
extent_dss_prec_get(tsdn_t *tsdn)
|
||||
{
|
||||
dss_prec_t ret;
|
||||
|
||||
@ -54,7 +54,7 @@ chunk_dss_prec_get(tsdn_t *tsdn)
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec)
|
||||
extent_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec)
|
||||
{
|
||||
|
||||
if (!have_dss)
|
||||
@ -66,7 +66,7 @@ chunk_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec)
|
||||
}
|
||||
|
||||
void *
|
||||
chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
size_t alignment, bool *zero, bool *commit)
|
||||
{
|
||||
void *ret;
|
||||
@ -104,7 +104,7 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
break;
|
||||
|
||||
/* Get the current end of the DSS. */
|
||||
dss_max = chunk_dss_sbrk(0);
|
||||
dss_max = extent_dss_sbrk(0);
|
||||
|
||||
/* Make sure the earlier condition still holds. */
|
||||
if (new_addr != NULL && dss_max != new_addr)
|
||||
@ -128,7 +128,7 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
(uintptr_t)dss_next < (uintptr_t)dss_max)
|
||||
break; /* Wrap-around. */
|
||||
incr = pad_size + size;
|
||||
dss_prev = chunk_dss_sbrk(incr);
|
||||
dss_prev = extent_dss_sbrk(incr);
|
||||
if (dss_prev == (void *)-1)
|
||||
break;
|
||||
if (dss_prev == dss_max) {
|
||||
@ -138,7 +138,7 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
if (pad_size != 0) {
|
||||
extent_hooks_t extent_hooks =
|
||||
EXTENT_HOOKS_INITIALIZER;
|
||||
chunk_dalloc_wrapper(tsdn, arena,
|
||||
extent_dalloc_wrapper(tsdn, arena,
|
||||
&extent_hooks, pad);
|
||||
} else
|
||||
extent_dalloc(tsdn, arena, pad);
|
||||
@ -157,15 +157,15 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_in_dss(tsdn_t *tsdn, void *chunk)
|
||||
extent_in_dss(tsdn_t *tsdn, void *addr)
|
||||
{
|
||||
bool ret;
|
||||
|
||||
cassert(have_dss);
|
||||
|
||||
malloc_mutex_lock(tsdn, &dss_mtx);
|
||||
if ((uintptr_t)chunk >= (uintptr_t)dss_base
|
||||
&& (uintptr_t)chunk < (uintptr_t)dss_max)
|
||||
if ((uintptr_t)addr >= (uintptr_t)dss_base
|
||||
&& (uintptr_t)addr < (uintptr_t)dss_max)
|
||||
ret = true;
|
||||
else
|
||||
ret = false;
|
||||
@ -175,14 +175,14 @@ chunk_in_dss(tsdn_t *tsdn, void *chunk)
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_dss_boot(void)
|
||||
extent_dss_boot(void)
|
||||
{
|
||||
|
||||
cassert(have_dss);
|
||||
|
||||
if (malloc_mutex_init(&dss_mtx, "dss", WITNESS_RANK_DSS))
|
||||
return (true);
|
||||
dss_base = chunk_dss_sbrk(0);
|
||||
dss_base = extent_dss_sbrk(0);
|
||||
dss_prev = dss_base;
|
||||
dss_max = dss_base;
|
||||
|
||||
@ -190,7 +190,7 @@ chunk_dss_boot(void)
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dss_prefork(tsdn_t *tsdn)
|
||||
extent_dss_prefork(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
if (have_dss)
|
||||
@ -198,7 +198,7 @@ chunk_dss_prefork(tsdn_t *tsdn)
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dss_postfork_parent(tsdn_t *tsdn)
|
||||
extent_dss_postfork_parent(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
if (have_dss)
|
||||
@ -206,7 +206,7 @@ chunk_dss_postfork_parent(tsdn_t *tsdn)
|
||||
}
|
||||
|
||||
void
|
||||
chunk_dss_postfork_child(tsdn_t *tsdn)
|
||||
extent_dss_postfork_child(tsdn_t *tsdn)
|
||||
{
|
||||
|
||||
if (have_dss)
|
||||
@ -1,10 +1,10 @@
|
||||
#define JEMALLOC_CHUNK_MMAP_C_
|
||||
#define JEMALLOC_EXTENT_MMAP_C_
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
static void *
|
||||
chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit)
|
||||
extent_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit)
|
||||
{
|
||||
void *ret;
|
||||
size_t alloc_size;
|
||||
@ -30,7 +30,7 @@ chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit)
|
||||
}
|
||||
|
||||
void *
|
||||
chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
bool *commit)
|
||||
{
|
||||
void *ret;
|
||||
@ -58,7 +58,7 @@ chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
|
||||
if (offset != 0) {
|
||||
pages_unmap(ret, size);
|
||||
return (chunk_alloc_mmap_slow(size, alignment, zero, commit));
|
||||
return (extent_alloc_mmap_slow(size, alignment, zero, commit));
|
||||
}
|
||||
|
||||
assert(ret != NULL);
|
||||
@ -67,10 +67,10 @@ chunk_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_dalloc_mmap(void *chunk, size_t size)
|
||||
extent_dalloc_mmap(void *addr, size_t size)
|
||||
{
|
||||
|
||||
if (config_munmap)
|
||||
pages_unmap(chunk, size);
|
||||
pages_unmap(addr, size);
|
||||
return (!config_munmap);
|
||||
}
|
||||
@ -1067,7 +1067,7 @@ malloc_conf_init(void)
|
||||
for (i = 0; i < dss_prec_limit; i++) {
|
||||
if (strncmp(dss_prec_names[i], v, vlen)
|
||||
== 0) {
|
||||
if (chunk_dss_prec_set(NULL,
|
||||
if (extent_dss_prec_set(NULL,
|
||||
i)) {
|
||||
malloc_conf_error(
|
||||
"Error setting dss",
|
||||
@ -2686,7 +2686,7 @@ _malloc_prefork(void)
|
||||
}
|
||||
}
|
||||
base_prefork(tsd_tsdn(tsd));
|
||||
chunk_prefork(tsd_tsdn(tsd));
|
||||
extent_prefork(tsd_tsdn(tsd));
|
||||
for (i = 0; i < narenas; i++) {
|
||||
if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
|
||||
arena_prefork3(tsd_tsdn(tsd), arena);
|
||||
@ -2715,7 +2715,7 @@ _malloc_postfork(void)
|
||||
|
||||
witness_postfork_parent(tsd);
|
||||
/* Release all mutexes, now that fork() has completed. */
|
||||
chunk_postfork_parent(tsd_tsdn(tsd));
|
||||
extent_postfork_parent(tsd_tsdn(tsd));
|
||||
base_postfork_parent(tsd_tsdn(tsd));
|
||||
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
|
||||
arena_t *arena;
|
||||
@ -2740,7 +2740,7 @@ jemalloc_postfork_child(void)
|
||||
|
||||
witness_postfork_child(tsd);
|
||||
/* Release all mutexes, now that fork() has completed. */
|
||||
chunk_postfork_child(tsd_tsdn(tsd));
|
||||
extent_postfork_child(tsd_tsdn(tsd));
|
||||
base_postfork_child(tsd_tsdn(tsd));
|
||||
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
|
||||
arena_t *arena;
|
||||
|
||||
28
src/large.c
28
src/large.c
@ -28,13 +28,13 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
||||
return (NULL);
|
||||
|
||||
/*
|
||||
* Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
|
||||
* Copy zero into is_zeroed and pass the copy to extent_alloc(), so that
|
||||
* it is possible to make correct junk/zero fill decisions below.
|
||||
*/
|
||||
is_zeroed = zero;
|
||||
if (likely(!tsdn_null(tsdn)))
|
||||
arena = arena_choose(tsdn_tsd(tsdn), arena);
|
||||
if (unlikely(arena == NULL) || (extent = arena_chunk_alloc_large(tsdn,
|
||||
if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn,
|
||||
arena, usize, alignment, &is_zeroed)) == NULL)
|
||||
return (NULL);
|
||||
|
||||
@ -82,10 +82,10 @@ large_dalloc_maybe_junk(tsdn_t *tsdn, void *ptr, size_t usize)
|
||||
|
||||
if (config_fill && have_dss && unlikely(opt_junk_free)) {
|
||||
/*
|
||||
* Only bother junk filling if the chunk isn't about to be
|
||||
* Only bother junk filling if the extent isn't about to be
|
||||
* unmapped.
|
||||
*/
|
||||
if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr)))
|
||||
if (!config_munmap || (have_dss && extent_in_dss(tsdn, ptr)))
|
||||
large_dalloc_junk(ptr, usize);
|
||||
memset(ptr, JEMALLOC_FREE_JUNK, usize);
|
||||
}
|
||||
@ -103,7 +103,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
|
||||
|
||||
/* Split excess pages. */
|
||||
if (diff != 0) {
|
||||
extent_t *trail = chunk_split_wrapper(tsdn, arena,
|
||||
extent_t *trail = extent_split_wrapper(tsdn, arena,
|
||||
&extent_hooks, extent, usize + large_pad, usize, diff,
|
||||
diff);
|
||||
if (trail == NULL)
|
||||
@ -114,10 +114,10 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
|
||||
extent_usize_get(trail));
|
||||
}
|
||||
|
||||
arena_chunk_cache_dalloc(tsdn, arena, &extent_hooks, trail);
|
||||
arena_extent_cache_dalloc(tsdn, arena, &extent_hooks, trail);
|
||||
}
|
||||
|
||||
arena_chunk_ralloc_large_shrink(tsdn, arena, extent, oldusize);
|
||||
arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize);
|
||||
|
||||
return (false);
|
||||
}
|
||||
@ -133,18 +133,18 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
|
||||
size_t trailsize = usize - extent_usize_get(extent);
|
||||
extent_t *trail;
|
||||
|
||||
if ((trail = arena_chunk_cache_alloc(tsdn, arena, &extent_hooks,
|
||||
if ((trail = arena_extent_cache_alloc(tsdn, arena, &extent_hooks,
|
||||
extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail))
|
||||
== NULL) {
|
||||
bool commit = true;
|
||||
if ((trail = chunk_alloc_wrapper(tsdn, arena, &extent_hooks,
|
||||
if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,
|
||||
extent_past_get(extent), trailsize, 0, CACHELINE,
|
||||
&is_zeroed_trail, &commit, false)) == NULL)
|
||||
return (true);
|
||||
}
|
||||
|
||||
if (chunk_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
|
||||
chunk_dalloc_wrapper(tsdn, arena, &extent_hooks, trail);
|
||||
if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
|
||||
extent_dalloc_wrapper(tsdn, arena, &extent_hooks, trail);
|
||||
return (true);
|
||||
}
|
||||
|
||||
@ -174,7 +174,7 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
|
||||
JEMALLOC_ALLOC_JUNK, usize - oldusize);
|
||||
}
|
||||
|
||||
arena_chunk_ralloc_large_expand(tsdn, arena, extent, oldusize);
|
||||
arena_extent_ralloc_large_expand(tsdn, arena, extent, oldusize);
|
||||
|
||||
return (false);
|
||||
}
|
||||
@ -209,7 +209,7 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
|
||||
}
|
||||
|
||||
/*
|
||||
* Avoid moving the allocation if the existing chunk size accommodates
|
||||
* Avoid moving the allocation if the existing extent size accommodates
|
||||
* the new size.
|
||||
*/
|
||||
if (extent_usize_get(extent) >= usize_min && extent_usize_get(extent) <=
|
||||
@ -287,7 +287,7 @@ large_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked)
|
||||
large_dalloc_maybe_junk(tsdn, extent_addr_get(extent),
|
||||
extent_usize_get(extent));
|
||||
}
|
||||
arena_chunk_dalloc_large(tsdn, arena, extent, junked_locked);
|
||||
arena_extent_dalloc_large(tsdn, arena, extent, junked_locked);
|
||||
|
||||
if (!junked_locked)
|
||||
arena_decay_tick(tsdn, arena);
|
||||
|
||||
@ -25,7 +25,7 @@ static bool did_merge;
|
||||
#endif
|
||||
|
||||
void *
|
||||
chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
extent_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
bool *commit, unsigned arena_ind)
|
||||
{
|
||||
|
||||
@ -38,86 +38,86 @@ chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero,
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_dalloc(void *chunk, size_t size, bool committed, unsigned arena_ind)
|
||||
extent_dalloc(void *addr, size_t size, bool committed, unsigned arena_ind)
|
||||
{
|
||||
|
||||
TRACE_HOOK("%s(chunk=%p, size=%zu, committed=%s, arena_ind=%u)\n",
|
||||
__func__, chunk, size, committed ? "true" : "false", arena_ind);
|
||||
TRACE_HOOK("%s(addr=%p, size=%zu, committed=%s, arena_ind=%u)\n",
|
||||
__func__, addr, size, committed ? "true" : "false", arena_ind);
|
||||
did_dalloc = true;
|
||||
if (!do_dalloc)
|
||||
return (true);
|
||||
return (old_hooks.dalloc(chunk, size, committed, arena_ind));
|
||||
return (old_hooks.dalloc(addr, size, committed, arena_ind));
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_commit(void *chunk, size_t size, size_t offset, size_t length,
|
||||
extent_commit(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
bool err;
|
||||
|
||||
TRACE_HOOK("%s(chunk=%p, size=%zu, offset=%zu, length=%zu, "
|
||||
"arena_ind=%u)\n", __func__, chunk, size, offset, length,
|
||||
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, "
|
||||
"arena_ind=%u)\n", __func__, addr, size, offset, length,
|
||||
arena_ind);
|
||||
err = old_hooks.commit(chunk, size, offset, length, arena_ind);
|
||||
err = old_hooks.commit(addr, size, offset, length, arena_ind);
|
||||
did_commit = !err;
|
||||
return (err);
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_decommit(void *chunk, size_t size, size_t offset, size_t length,
|
||||
extent_decommit(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
bool err;
|
||||
|
||||
TRACE_HOOK("%s(chunk=%p, size=%zu, offset=%zu, length=%zu, "
|
||||
"arena_ind=%u)\n", __func__, chunk, size, offset, length,
|
||||
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu, "
|
||||
"arena_ind=%u)\n", __func__, addr, size, offset, length,
|
||||
arena_ind);
|
||||
if (!do_decommit)
|
||||
return (true);
|
||||
err = old_hooks.decommit(chunk, size, offset, length, arena_ind);
|
||||
err = old_hooks.decommit(addr, size, offset, length, arena_ind);
|
||||
did_decommit = !err;
|
||||
return (err);
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_purge(void *chunk, size_t size, size_t offset, size_t length,
|
||||
extent_purge(void *addr, size_t size, size_t offset, size_t length,
|
||||
unsigned arena_ind)
|
||||
{
|
||||
|
||||
TRACE_HOOK("%s(chunk=%p, size=%zu, offset=%zu, length=%zu "
|
||||
"arena_ind=%u)\n", __func__, chunk, size, offset, length,
|
||||
TRACE_HOOK("%s(addr=%p, size=%zu, offset=%zu, length=%zu "
|
||||
"arena_ind=%u)\n", __func__, addr, size, offset, length,
|
||||
arena_ind);
|
||||
did_purge = true;
|
||||
return (old_hooks.purge(chunk, size, offset, length, arena_ind));
|
||||
return (old_hooks.purge(addr, size, offset, length, arena_ind));
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_split(void *chunk, size_t size, size_t size_a, size_t size_b,
|
||||
extent_split(void *addr, size_t size, size_t size_a, size_t size_b,
|
||||
bool committed, unsigned arena_ind)
|
||||
{
|
||||
|
||||
TRACE_HOOK("%s(chunk=%p, size=%zu, size_a=%zu, size_b=%zu, "
|
||||
"committed=%s, arena_ind=%u)\n", __func__, chunk, size, size_a,
|
||||
TRACE_HOOK("%s(addr=%p, size=%zu, size_a=%zu, size_b=%zu, "
|
||||
"committed=%s, arena_ind=%u)\n", __func__, addr, size, size_a,
|
||||
size_b, committed ? "true" : "false", arena_ind);
|
||||
did_split = true;
|
||||
return (old_hooks.split(chunk, size, size_a, size_b, committed,
|
||||
return (old_hooks.split(addr, size, size_a, size_b, committed,
|
||||
arena_ind));
|
||||
}
|
||||
|
||||
bool
|
||||
chunk_merge(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
|
||||
extent_merge(void *addr_a, size_t size_a, void *addr_b, size_t size_b,
|
||||
bool committed, unsigned arena_ind)
|
||||
{
|
||||
|
||||
TRACE_HOOK("%s(chunk_a=%p, size_a=%zu, chunk_b=%p size_b=%zu, "
|
||||
"committed=%s, arena_ind=%u)\n", __func__, chunk_a, size_a, chunk_b,
|
||||
TRACE_HOOK("%s(addr_a=%p, size_a=%zu, addr_b=%p size_b=%zu, "
|
||||
"committed=%s, arena_ind=%u)\n", __func__, addr_a, size_a, addr_b,
|
||||
size_b, committed ? "true" : "false", arena_ind);
|
||||
did_merge = true;
|
||||
return (old_hooks.merge(chunk_a, size_a, chunk_b, size_b,
|
||||
return (old_hooks.merge(addr_a, size_a, addr_b, size_b,
|
||||
committed, arena_ind));
|
||||
}
|
||||
|
||||
TEST_BEGIN(test_chunk)
|
||||
TEST_BEGIN(test_extent)
|
||||
{
|
||||
void *p;
|
||||
size_t old_size, new_size, large0, large1, large2, sz;
|
||||
@ -126,13 +126,13 @@ TEST_BEGIN(test_chunk)
|
||||
size_t hooks_mib[3], purge_mib[3];
|
||||
size_t hooks_miblen, purge_miblen;
|
||||
extent_hooks_t new_hooks = {
|
||||
chunk_alloc,
|
||||
chunk_dalloc,
|
||||
chunk_commit,
|
||||
chunk_decommit,
|
||||
chunk_purge,
|
||||
chunk_split,
|
||||
chunk_merge
|
||||
extent_alloc,
|
||||
extent_dalloc,
|
||||
extent_commit,
|
||||
extent_decommit,
|
||||
extent_purge,
|
||||
extent_split,
|
||||
extent_merge
|
||||
};
|
||||
bool xallocx_success_a, xallocx_success_b, xallocx_success_c;
|
||||
|
||||
@ -151,16 +151,16 @@ TEST_BEGIN(test_chunk)
|
||||
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
|
||||
&new_hooks, new_size), 0, "Unexpected extent_hooks error");
|
||||
orig_hooks = old_hooks;
|
||||
assert_ptr_ne(old_hooks.alloc, chunk_alloc, "Unexpected alloc error");
|
||||
assert_ptr_ne(old_hooks.dalloc, chunk_dalloc,
|
||||
assert_ptr_ne(old_hooks.alloc, extent_alloc, "Unexpected alloc error");
|
||||
assert_ptr_ne(old_hooks.dalloc, extent_dalloc,
|
||||
"Unexpected dalloc error");
|
||||
assert_ptr_ne(old_hooks.commit, chunk_commit,
|
||||
assert_ptr_ne(old_hooks.commit, extent_commit,
|
||||
"Unexpected commit error");
|
||||
assert_ptr_ne(old_hooks.decommit, chunk_decommit,
|
||||
assert_ptr_ne(old_hooks.decommit, extent_decommit,
|
||||
"Unexpected decommit error");
|
||||
assert_ptr_ne(old_hooks.purge, chunk_purge, "Unexpected purge error");
|
||||
assert_ptr_ne(old_hooks.split, chunk_split, "Unexpected split error");
|
||||
assert_ptr_ne(old_hooks.merge, chunk_merge, "Unexpected merge error");
|
||||
assert_ptr_ne(old_hooks.purge, extent_purge, "Unexpected purge error");
|
||||
assert_ptr_ne(old_hooks.split, extent_split, "Unexpected split error");
|
||||
assert_ptr_ne(old_hooks.merge, extent_merge, "Unexpected merge error");
|
||||
|
||||
/* Get large size classes. */
|
||||
sz = sizeof(size_t);
|
||||
@ -249,5 +249,5 @@ int
|
||||
main(void)
|
||||
{
|
||||
|
||||
return (test(test_chunk));
|
||||
return (test(test_extent));
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user