Add/use chunk_split_wrapper().

Remove redundant ptr/oldsize args from huge_*().

Refactor huge/chunk/arena code boundaries.
This commit is contained in:
Jason Evans 2016-05-18 21:02:46 -07:00
parent 1ad060584f
commit de0305a7f3
7 changed files with 564 additions and 520 deletions

View File

@ -176,13 +176,6 @@ typedef ph(arena_chunk_map_misc_t) arena_run_heap_t;
#ifdef JEMALLOC_ARENA_STRUCTS_B
/* Arena chunk header. */
struct arena_chunk_s {
/*
* A pointer to the arena that owns the chunk is stored within the
* extent structure. This field as a whole is used by chunks_rtree to
* support both ivsalloc() and core-based debugging.
*/
extent_t extent;
/*
* Map of pages within chunk that keeps track of free/large/small. The
* first map_bias entries are omitted, since the chunk header does not
@ -315,7 +308,7 @@ struct arena_s {
* order to avoid interactions between multiple threads that could make
* a single spare inadequate.
*/
arena_chunk_t *spare;
extent_t *spare;
/* Minimum ratio (log base 2) of nactive:ndirty. */
ssize_t lg_dirty_mult;
@ -481,22 +474,27 @@ typedef size_t (run_quantize_t)(size_t);
extern run_quantize_t *run_quantize_floor;
extern run_quantize_t *run_quantize_ceil;
#endif
extent_t *arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero);
void arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed);
void arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent,
bool cache);
void arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent,
bool cache);
extent_t *arena_extent_alloc(tsdn_t *tsdn, arena_t *arena);
void arena_extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
void *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero);
extent_t *arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena,
size_t usize, size_t alignment, bool *zero);
void arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk,
size_t usize);
void arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena,
void *chunk, size_t oldsize, size_t usize);
extent_t *extent, size_t oldsize);
void arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena,
void *chunk, size_t oldsize, size_t usize);
bool arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t usize);
extent_t *extent, size_t oldsize);
void arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize);
ssize_t arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena);
bool arena_lg_dirty_mult_set(tsdn_t *tsdn, arena_t *arena,
ssize_t lg_dirty_mult);
@ -1193,7 +1191,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
ret = atomic_read_p(&elm->prof_tctx_pun);
}
} else
ret = huge_prof_tctx_get(tsdn, extent, ptr);
ret = huge_prof_tctx_get(tsdn, extent);
return (ret);
}
@ -1230,7 +1228,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
assert(arena_mapbits_large_get(chunk, pageind) == 0);
}
} else
huge_prof_tctx_set(tsdn, extent, ptr, tctx);
huge_prof_tctx_set(tsdn, extent, tctx);
}
JEMALLOC_INLINE void
@ -1258,7 +1256,7 @@ arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
atomic_write_p(&elm->prof_tctx_pun,
(prof_tctx_t *)(uintptr_t)1U);
} else
huge_prof_tctx_reset(tsdn, extent, ptr);
huge_prof_tctx_reset(tsdn, extent);
}
}
@ -1362,7 +1360,7 @@ arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr, bool demote)
ret = index2size(binind);
}
} else
ret = huge_salloc(tsdn, extent, ptr);
ret = huge_salloc(tsdn, extent);
return (ret);
}
@ -1413,7 +1411,7 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
}
}
} else
huge_dalloc(tsdn, extent, ptr);
huge_dalloc(tsdn, extent);
}
JEMALLOC_ALWAYS_INLINE void
@ -1470,7 +1468,7 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
}
}
} else
huge_dalloc(tsdn, extent, ptr);
huge_dalloc(tsdn, extent);
}
# endif /* JEMALLOC_ARENA_INLINE_B */
#endif

View File

@ -55,10 +55,10 @@ chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena,
bool chunk_register(tsdn_t *tsdn, const extent_t *extent);
void chunk_deregister(tsdn_t *tsdn, const extent_t *extent);
void chunk_reregister(tsdn_t *tsdn, const extent_t *extent);
void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_t *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool dalloc_extent);
void *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
bool *zero);
extent_t *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool *commit);
void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
@ -75,6 +75,8 @@ bool chunk_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset,
size_t length);
extent_t *chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent, size_t size_a, size_t size_b);
bool chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *a, extent_t *b);
bool chunk_boot(void);

View File

@ -12,22 +12,19 @@
void *huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero);
bool huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t usize_min, size_t usize_max, bool zero);
void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t oldsize, size_t usize, size_t alignment, bool zero,
tcache_t *tcache);
bool huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero);
void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
#ifdef JEMALLOC_JET
typedef void (huge_dalloc_junk_t)(tsdn_t *, void *, size_t);
extern huge_dalloc_junk_t *huge_dalloc_junk;
#endif
void huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr);
size_t huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr);
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
const void *ptr);
void huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx);
void huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr);
void huge_dalloc(tsdn_t *tsdn, extent_t *extent);
size_t huge_salloc(tsdn_t *tsdn, const extent_t *extent);
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
void huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
void huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/

View File

@ -12,6 +12,8 @@ arena_choose
arena_choose_hard
arena_choose_impl
arena_chunk_alloc_huge
arena_chunk_cache_alloc
arena_chunk_cache_dalloc
arena_chunk_cache_maybe_insert
arena_chunk_cache_maybe_remove
arena_chunk_dalloc_huge
@ -184,6 +186,7 @@ chunk_prefork
chunk_purge_wrapper
chunk_register
chunk_reregister
chunk_split_wrapper
chunks_rtree
chunksize
chunksize_mask

View File

@ -220,6 +220,55 @@ arena_chunk_dirty_npages(const extent_t *extent)
return (extent_size_get(extent) >> LG_PAGE);
}
static extent_t *
arena_chunk_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero)
{
malloc_mutex_assert_owner(tsdn, &arena->lock);
return (chunk_alloc_cache(tsdn, arena, chunk_hooks, new_addr, size,
alignment, zero));
}
extent_t *
arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero)
{
extent_t *extent;
malloc_mutex_lock(tsdn, &arena->lock);
extent = arena_chunk_cache_alloc_locked(tsdn, arena, chunk_hooks,
new_addr, size, alignment, zero);
malloc_mutex_unlock(tsdn, &arena->lock);
return (extent);
}
static void
arena_chunk_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed)
{
malloc_mutex_assert_owner(tsdn, &arena->lock);
chunk_dalloc_cache(tsdn, arena, chunk_hooks, chunk, size, committed);
arena_maybe_purge(tsdn, arena);
}
void
arena_chunk_cache_dalloc(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed)
{
malloc_mutex_lock(tsdn, &arena->lock);
arena_chunk_cache_dalloc_locked(tsdn, arena, chunk_hooks, chunk, size,
committed);
malloc_mutex_unlock(tsdn, &arena->lock);
}
void
arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache)
{
@ -492,112 +541,119 @@ arena_run_split_small(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
return (false);
}
static arena_chunk_t *
static extent_t *
arena_chunk_init_spare(arena_t *arena)
{
arena_chunk_t *chunk;
extent_t *extent;
assert(arena->spare != NULL);
chunk = arena->spare;
extent = arena->spare;
arena->spare = NULL;
assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
arena_maxrun);
assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) ==
arena_maxrun);
assert(arena_mapbits_dirty_get(chunk, map_bias) ==
arena_mapbits_dirty_get(chunk, chunk_npages-1));
assert(arena_mapbits_allocated_get((arena_chunk_t *)
extent_addr_get(extent), map_bias) == 0);
assert(arena_mapbits_allocated_get((arena_chunk_t *)
extent_addr_get(extent), chunk_npages-1) == 0);
assert(arena_mapbits_unallocated_size_get((arena_chunk_t *)
extent_addr_get(extent), map_bias) == arena_maxrun);
assert(arena_mapbits_unallocated_size_get((arena_chunk_t *)
extent_addr_get(extent), chunk_npages-1) == arena_maxrun);
assert(arena_mapbits_dirty_get((arena_chunk_t *)
extent_addr_get(extent), map_bias) ==
arena_mapbits_dirty_get((arena_chunk_t *)extent_addr_get(extent),
chunk_npages-1));
return (chunk);
return (extent);
}
static bool
arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
bool zero)
{
/*
* The extent notion of "committed" doesn't directly apply to arena
* chunks. Arbitrarily mark them as committed. The commit state of
* runs is tracked individually, and upon chunk deallocation the entire
* chunk is in a consistent commit state.
*/
extent_init(&chunk->extent, arena, chunk, chunksize, true, zero, true,
true);
return (chunk_register(tsdn, &chunk->extent));
}
static arena_chunk_t *
static extent_t *
arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, bool *zero, bool *commit)
{
arena_chunk_t *chunk;
extent_t *extent;
malloc_mutex_unlock(tsdn, &arena->lock);
chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks,
NULL, chunksize, chunksize, zero, commit);
if (chunk != NULL && !*commit) {
extent = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, chunksize,
chunksize, zero, commit);
if (extent != NULL && !*commit) {
/* Commit header. */
if (chunk_commit_wrapper(tsdn, arena, chunk_hooks, chunk,
chunksize, 0, map_bias << LG_PAGE)) {
if (chunk_commit_wrapper(tsdn, arena, chunk_hooks,
extent_addr_get(extent), extent_size_get(extent), 0,
map_bias << LG_PAGE)) {
chunk_dalloc_wrapper(tsdn, arena, chunk_hooks,
(void *)chunk, chunksize, *zero, *commit);
chunk = NULL;
extent_addr_get(extent), extent_size_get(extent),
extent_zeroed_get(extent),
extent_committed_get(extent));
extent = NULL;
}
}
if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, *zero)) {
if (!*commit) {
/* Undo commit of header. */
chunk_decommit_wrapper(tsdn, arena, chunk_hooks,
chunk, chunksize, 0, map_bias << LG_PAGE);
if (extent != NULL) {
extent_slab_set(extent, true);
if (chunk_register(tsdn, extent)) {
if (!*commit) {
/* Undo commit of header. */
chunk_decommit_wrapper(tsdn, arena, chunk_hooks,
extent_addr_get(extent),
extent_size_get(extent), 0, map_bias <<
LG_PAGE);
}
chunk_dalloc_wrapper(tsdn, arena, chunk_hooks,
extent_addr_get(extent), extent_size_get(extent),
extent_zeroed_get(extent),
extent_committed_get(extent));
extent = NULL;
}
chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk,
chunksize, *zero, *commit);
chunk = NULL;
}
malloc_mutex_lock(tsdn, &arena->lock);
return (chunk);
return (extent);
}
static arena_chunk_t *
static extent_t *
arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero,
bool *commit)
{
arena_chunk_t *chunk;
extent_t *extent;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize,
chunksize, zero, true);
if (chunk != NULL) {
if (arena_chunk_register(tsdn, arena, chunk, *zero)) {
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk,
chunksize, true);
extent = arena_chunk_cache_alloc_locked(tsdn, arena, &chunk_hooks, NULL,
chunksize, chunksize, zero);
if (extent != NULL) {
extent_slab_set(extent, true);
if (chunk_register(tsdn, extent)) {
arena_chunk_cache_dalloc_locked(tsdn, arena,
&chunk_hooks, extent_addr_get(extent),
extent_size_get(extent), true);
return (NULL);
}
*commit = true;
}
if (chunk == NULL) {
chunk = arena_chunk_alloc_internal_hard(tsdn, arena,
if (extent == NULL) {
extent = arena_chunk_alloc_internal_hard(tsdn, arena,
&chunk_hooks, zero, commit);
if (extent == NULL)
return (NULL);
}
assert(extent_slab_get(extent));
if (config_stats && chunk != NULL) {
arena->stats.mapped += chunksize;
if (config_stats) {
arena->stats.mapped += extent_size_get(extent);
arena->stats.metadata_mapped += (map_bias << LG_PAGE);
}
return (chunk);
return (extent);
}
static arena_chunk_t *
static extent_t *
arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena)
{
arena_chunk_t *chunk;
extent_t *extent;
bool zero, commit;
size_t flag_unzeroed, flag_decommitted, i;
@ -605,8 +661,8 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena)
zero = false;
commit = false;
chunk = arena_chunk_alloc_internal(tsdn, arena, &zero, &commit);
if (chunk == NULL)
extent = arena_chunk_alloc_internal(tsdn, arena, &zero, &commit);
if (extent == NULL)
return (NULL);
/*
@ -616,58 +672,63 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena)
*/
flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED;
flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED;
arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun,
flag_unzeroed | flag_decommitted);
arena_mapbits_unallocated_set((arena_chunk_t *)extent_addr_get(extent),
map_bias, arena_maxrun, flag_unzeroed | flag_decommitted);
/*
* There is no need to initialize the internal page map entries unless
* the chunk is not zeroed.
*/
if (!zero) {
for (i = map_bias+1; i < chunk_npages-1; i++)
arena_mapbits_internal_set(chunk, i, flag_unzeroed);
for (i = map_bias+1; i < chunk_npages-1; i++) {
arena_mapbits_internal_set((arena_chunk_t *)
extent_addr_get(extent), i, flag_unzeroed);
}
} else {
if (config_debug) {
for (i = map_bias+1; i < chunk_npages-1; i++) {
assert(arena_mapbits_unzeroed_get(chunk, i) ==
flag_unzeroed);
assert(arena_mapbits_unzeroed_get(
(arena_chunk_t *)extent_addr_get(extent), i)
== flag_unzeroed);
}
}
}
arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun,
flag_unzeroed);
arena_mapbits_unallocated_set((arena_chunk_t *)extent_addr_get(extent),
chunk_npages-1, arena_maxrun, flag_unzeroed);
return (chunk);
return (extent);
}
static arena_chunk_t *
static extent_t *
arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena)
{
arena_chunk_t *chunk;
extent_t *extent;
if (arena->spare != NULL)
chunk = arena_chunk_init_spare(arena);
extent = arena_chunk_init_spare(arena);
else {
chunk = arena_chunk_init_hard(tsdn, arena);
if (chunk == NULL)
extent = arena_chunk_init_hard(tsdn, arena);
if (extent == NULL)
return (NULL);
}
ql_elm_new(&chunk->extent, ql_link);
ql_tail_insert(&arena->achunks, &chunk->extent, ql_link);
arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias);
ql_elm_new(extent, ql_link);
ql_tail_insert(&arena->achunks, extent, ql_link);
arena_avail_insert(arena, (arena_chunk_t *)extent_addr_get(extent),
map_bias, chunk_npages-map_bias);
return (chunk);
return (extent);
}
static void
arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
{
bool committed;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
chunk_deregister(tsdn, &chunk->extent);
chunk_deregister(tsdn, extent);
committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0);
committed = (arena_mapbits_decommitted_get((arena_chunk_t *)
extent_addr_get(extent), map_bias) == 0);
if (!committed) {
/*
* Decommit the header. Mark the chunk as decommitted even if
@ -675,37 +736,42 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
* chunk as committed has a high potential for causing later
* access of decommitted memory.
*/
chunk_decommit_wrapper(tsdn, arena, &chunk_hooks, chunk,
chunksize, 0, map_bias << LG_PAGE);
chunk_decommit_wrapper(tsdn, arena, &chunk_hooks,
extent_addr_get(extent), extent_size_get(extent), 0,
map_bias << LG_PAGE);
}
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize,
committed);
arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks,
extent_addr_get(extent), extent_size_get(extent), committed);
if (config_stats) {
arena->stats.mapped -= chunksize;
arena->stats.mapped -= extent_size_get(extent);
arena->stats.metadata_mapped -= (map_bias << LG_PAGE);
}
arena_extent_dalloc(tsdn, arena, extent);
}
static void
arena_spare_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *spare)
arena_spare_discard(tsdn_t *tsdn, arena_t *arena, extent_t *spare)
{
assert(arena->spare != spare);
if (arena_mapbits_dirty_get(spare, map_bias) != 0) {
arena_run_dirty_remove(arena, spare, map_bias,
chunk_npages-map_bias);
if (arena_mapbits_dirty_get((arena_chunk_t *)extent_addr_get(spare),
map_bias) != 0) {
arena_run_dirty_remove(arena, (arena_chunk_t *)
extent_addr_get(spare), map_bias, chunk_npages-map_bias);
}
arena_chunk_discard(tsdn, arena, spare);
}
static void
arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
{
arena_chunk_t *spare;
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
extent_t *spare;
assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
@ -721,9 +787,9 @@ arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
/* Remove run from runs_avail, so that the arena does not use it. */
arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias);
ql_remove(&arena->achunks, &chunk->extent, ql_link);
ql_remove(&arena->achunks, extent, ql_link);
spare = arena->spare;
arena->spare = chunk;
arena->spare = extent;
if (spare != NULL)
arena_spare_discard(tsdn, arena, spare);
}
@ -778,19 +844,6 @@ arena_huge_reset_stats_cancel(arena_t *arena, size_t usize)
arena->stats.hstats[index].ndalloc--;
}
static void
arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize)
{
szind_t index = size2index(usize) - nlclasses - NBINS;
cassert(config_stats);
arena->stats.ndalloc_huge--;
arena->stats.allocated_huge += usize;
arena->stats.hstats[index].ndalloc--;
arena->stats.hstats[index].curhchunks++;
}
static void
arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize)
{
@ -799,15 +852,6 @@ arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize)
arena_huge_malloc_stats_update(arena, usize);
}
static void
arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize,
size_t usize)
{
arena_huge_dalloc_stats_update_undo(arena, oldsize);
arena_huge_malloc_stats_update_undo(arena, usize);
}
extent_t *
arena_extent_alloc(tsdn_t *tsdn, arena_t *arena)
{
@ -834,17 +878,17 @@ arena_extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
}
static void *
static extent_t *
arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero,
size_t csize)
{
void *ret;
extent_t *extent;
bool commit = true;
ret = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, csize,
extent = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, csize,
alignment, zero, &commit);
if (ret == NULL) {
if (extent == NULL) {
/* Revert optimistic stats updates. */
malloc_mutex_lock(tsdn, &arena->lock);
if (config_stats) {
@ -855,14 +899,14 @@ arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
malloc_mutex_unlock(tsdn, &arena->lock);
}
return (ret);
return (extent);
}
void *
extent_t *
arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero)
{
void *ret;
extent_t *extent;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
size_t csize = CHUNK_CEILING(usize);
@ -875,15 +919,15 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
}
arena_nactive_add(arena, usize >> LG_PAGE);
ret = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, csize,
alignment, zero, true);
extent = arena_chunk_cache_alloc_locked(tsdn, arena, &chunk_hooks, NULL,
csize, alignment, zero);
malloc_mutex_unlock(tsdn, &arena->lock);
if (ret == NULL) {
ret = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks,
if (extent == NULL) {
extent = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks,
usize, alignment, zero, csize);
}
return (ret);
return (extent);
}
void
@ -900,14 +944,16 @@ arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize)
}
arena_nactive_sub(arena, usize >> LG_PAGE);
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, true);
arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, chunk, csize,
true);
malloc_mutex_unlock(tsdn, &arena->lock);
}
void
arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk,
size_t oldsize, size_t usize)
arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t oldsize)
{
size_t usize = extent_size_get(extent);
assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize));
assert(oldsize != usize);
@ -923,9 +969,10 @@ arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk,
}
void
arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk,
size_t oldsize, size_t usize)
arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t oldsize)
{
size_t usize = extent_size_get(extent);
size_t udiff = oldsize - usize;
size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
@ -936,83 +983,24 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk,
arena->stats.mapped -= cdiff;
}
arena_nactive_sub(arena, udiff >> LG_PAGE);
if (cdiff != 0) {
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
void *nchunk = (void *)((uintptr_t)chunk +
CHUNK_CEILING(usize));
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff,
true);
}
malloc_mutex_unlock(tsdn, &arena->lock);
}
bool
void
arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t usize)
size_t oldsize)
{
bool err;
bool zero = false;
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena);
void *nchunk =
(void *)CHUNK_CEILING((uintptr_t)extent_past_get(extent));
size_t udiff = usize - extent_size_get(extent);
size_t cdiff = CHUNK_CEILING(usize) -
CHUNK_CEILING(extent_size_get(extent));
extent_t *trail;
size_t usize = extent_size_get(extent);
size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
size_t udiff = usize - oldsize;
malloc_mutex_lock(tsdn, &arena->lock);
/* Optimistically update stats. */
if (config_stats) {
arena_huge_ralloc_stats_update(arena, extent_size_get(extent),
usize);
arena_huge_ralloc_stats_update(arena, oldsize, usize);
arena->stats.mapped += cdiff;
}
arena_nactive_add(arena, udiff >> LG_PAGE);
err = (chunk_alloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff,
chunksize, &zero, true) == NULL);
malloc_mutex_unlock(tsdn, &arena->lock);
if (err) {
bool commit = true;
if (chunk_alloc_wrapper(tsdn, arena, &chunk_hooks, nchunk,
cdiff, chunksize, &zero, &commit) == NULL)
goto label_revert;
}
trail = arena_extent_alloc(tsdn, arena);
if (trail == NULL) {
chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, nchunk, cdiff,
zero, true);
goto label_revert;
}
extent_init(trail, arena, nchunk, cdiff, true, zero, true, false);
if (chunk_merge_wrapper(tsdn, arena, &chunk_hooks, extent, trail)) {
arena_extent_dalloc(tsdn, arena, trail);
chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, nchunk, cdiff,
zero, true);
goto label_revert;
}
if (usize < extent_size_get(extent))
extent_size_set(extent, usize);
return (false);
label_revert:
/* Revert optimistic stats updates. */
malloc_mutex_lock(tsdn, &arena->lock);
if (config_stats) {
arena_huge_ralloc_stats_update_undo(arena,
extent_size_get(extent), usize);
arena->stats.mapped -= cdiff;
}
arena_nactive_sub(arena, udiff >> LG_PAGE);
malloc_mutex_unlock(tsdn, &arena->lock);
return (true);
}
/*
@ -1053,8 +1041,8 @@ arena_run_alloc_large_helper(tsdn_t *tsdn, arena_t *arena, size_t size,
static arena_run_t *
arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero)
{
arena_chunk_t *chunk;
arena_run_t *run;
extent_t *extent;
assert(size <= arena_maxrun);
assert(size == PAGE_CEILING(size));
@ -1067,9 +1055,10 @@ arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero)
/*
* No usable runs. Create a new chunk from which to allocate the run.
*/
chunk = arena_chunk_alloc(tsdn, arena);
if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
extent = arena_chunk_alloc(tsdn, arena);
if (extent != NULL) {
run = &arena_miscelm_get_mutable((arena_chunk_t *)
extent_addr_get(extent), map_bias)->run;
if (arena_run_split_large(tsdn, arena, iealloc(tsdn, run), run,
size, zero))
run = NULL;
@ -1100,8 +1089,8 @@ arena_run_alloc_small_helper(tsdn_t *tsdn, arena_t *arena, size_t size,
static arena_run_t *
arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind)
{
arena_chunk_t *chunk;
arena_run_t *run;
extent_t *extent;
assert(size <= arena_maxrun);
assert(size == PAGE_CEILING(size));
@ -1115,9 +1104,10 @@ arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind)
/*
* No usable runs. Create a new chunk from which to allocate the run.
*/
chunk = arena_chunk_alloc(tsdn, arena);
if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
extent = arena_chunk_alloc(tsdn, arena);
if (extent != NULL) {
run = &arena_miscelm_get_mutable(
(arena_chunk_t *)extent_addr_get(extent), map_bias)->run;
if (arena_run_split_small(tsdn, arena, iealloc(tsdn, run), run,
size, binind))
run = NULL;
@ -1420,6 +1410,8 @@ void
arena_maybe_purge(tsdn_t *tsdn, arena_t *arena)
{
malloc_mutex_assert_owner(tsdn, &arena->lock);
/* Don't recursively purge. */
if (arena->purging)
return;
@ -1484,7 +1476,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
if (rdelm == &chunkselm->rd) {
extent_t *chunkselm_next;
bool zero;
UNUSED void *chunk;
UNUSED extent_t *extent;
npages = extent_size_get(chunkselm) >> LG_PAGE;
if (opt_purge == purge_mode_decay && arena->ndirty -
@ -1492,16 +1484,12 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
break;
chunkselm_next = qr_next(chunkselm, cc_link);
/*
* Allocate. chunkselm remains valid due to the
* dalloc_extent=false argument to chunk_alloc_cache().
*/
/* Allocate. */
zero = false;
chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks,
extent_addr_get(chunkselm),
extent_size_get(chunkselm), chunksize, &zero,
false);
assert(chunk == extent_addr_get(chunkselm));
extent = arena_chunk_cache_alloc_locked(tsdn, arena,
chunk_hooks, extent_addr_get(chunkselm),
extent_size_get(chunkselm), chunksize, &zero);
assert(extent == chunkselm);
assert(zero == extent_zeroed_get(chunkselm));
extent_dirty_insert(chunkselm, purge_runs_sentinel,
purge_chunks_sentinel);
@ -1510,14 +1498,13 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
chunkselm = chunkselm_next;
} else {
extent_t *extent = iealloc(tsdn, rdelm);
arena_chunk_t *chunk =
(arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm);
arena_run_t *run = &miscelm->run;
size_t run_size =
arena_mapbits_unallocated_size_get(chunk, pageind);
arena_mapbits_unallocated_size_get((arena_chunk_t *)
extent_addr_get(extent), pageind);
npages = run_size >> LG_PAGE;
if (opt_purge == purge_mode_decay && arena->ndirty -
@ -1525,14 +1512,16 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
break;
assert(pageind + npages <= chunk_npages);
assert(arena_mapbits_dirty_get(chunk, pageind) ==
arena_mapbits_dirty_get(chunk, pageind+npages-1));
assert(arena_mapbits_dirty_get((arena_chunk_t *)
extent_addr_get(extent), pageind) ==
arena_mapbits_dirty_get((arena_chunk_t *)
extent_addr_get(extent), pageind+npages-1));
/*
* If purging the spare chunk's run, make it available
* prior to allocation.
*/
if (chunk == arena->spare)
if (extent == arena->spare)
arena_chunk_alloc(tsdn, arena);
/* Temporarily allocate the free dirty run. */
@ -1757,8 +1746,9 @@ arena_purge(tsdn_t *tsdn, arena_t *arena, bool all)
}
static void
arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk)
arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, extent_t *extent)
{
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
size_t pageind, npages;
cassert(config_prof);
@ -1773,10 +1763,10 @@ arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk)
if (arena_mapbits_large_get(chunk, pageind) != 0) {
void *ptr = (void *)((uintptr_t)chunk + (pageind
<< LG_PAGE));
size_t usize = isalloc(tsd_tsdn(tsd),
&chunk->extent, ptr, config_prof);
size_t usize = isalloc(tsd_tsdn(tsd), extent,
ptr, config_prof);
prof_free(tsd, &chunk->extent, ptr, usize);
prof_free(tsd, extent, ptr, usize);
npages = arena_mapbits_large_size_get(chunk,
pageind) >> LG_PAGE;
} else {
@ -1819,8 +1809,7 @@ arena_reset(tsd_t *tsd, arena_t *arena)
/* Remove large allocations from prof sample set. */
if (config_prof && opt_prof) {
ql_foreach(extent, &arena->achunks, ql_link) {
arena_achunk_prof_reset(tsd, arena,
extent_addr_get(extent));
arena_achunk_prof_reset(tsd, arena, extent);
}
}
@ -1845,7 +1834,7 @@ arena_reset(tsd_t *tsd, arena_t *arena)
/* Remove huge allocation from prof sample set. */
if (config_prof && opt_prof)
prof_free(tsd, extent, ptr, usize);
huge_dalloc(tsd_tsdn(tsd), extent, ptr);
huge_dalloc(tsd_tsdn(tsd), extent);
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx);
/* Cancel out unwanted effects on stats. */
if (config_stats)
@ -1883,8 +1872,7 @@ arena_reset(tsd_t *tsd, arena_t *arena)
for (extent = ql_last(&arena->achunks, ql_link); extent != NULL; extent
= ql_last(&arena->achunks, ql_link)) {
ql_remove(&arena->achunks, extent, ql_link);
arena_chunk_discard(tsd_tsdn(tsd), arena,
extent_addr_get(extent));
arena_chunk_discard(tsd_tsdn(tsd), arena, extent);
}
/* Spare. */
@ -2078,7 +2066,7 @@ arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
if (size == arena_maxrun) {
assert(run_ind == map_bias);
assert(run_pages == (arena_maxrun >> LG_PAGE));
arena_chunk_dalloc(tsdn, arena, chunk);
arena_chunk_dalloc(tsdn, arena, extent);
}
/*
@ -3113,10 +3101,12 @@ arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
} else {
return (huge_ralloc_no_move(tsdn, extent, ptr, oldsize,
usize_min, usize_max, zero));
} else if (oldsize >= chunksize && usize_max >= chunksize) {
return (huge_ralloc_no_move(tsdn, extent, usize_min, usize_max,
zero));
}
return (true);
}
static void *
@ -3138,42 +3128,41 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache)
{
void *ret;
size_t usize;
size_t usize, copysize;
usize = s2u(size);
if (unlikely(usize == 0 || size > HUGE_MAXCLASS))
return (NULL);
if (likely(usize <= large_maxclass)) {
size_t copysize;
/* Try to avoid moving the allocation. */
if (!arena_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, 0,
zero))
return (ptr);
/*
* size and oldsize are different enough that we need to move
* the object. In that case, fall back to allocating new space
* and copying.
*/
ret = arena_ralloc_move_helper(tsdn, arena, usize, alignment,
zero, tcache);
if (ret == NULL)
return (NULL);
/*
* Junk/zero-filling were already done by
* ipalloc()/arena_malloc().
*/
copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
} else {
ret = huge_ralloc(tsdn, arena, extent, ptr, oldsize, usize,
alignment, zero, tcache);
}
if (oldsize >= chunksize && usize >= chunksize) {
return (huge_ralloc(tsdn, arena, extent, usize, alignment, zero,
tcache));
}
/*
* size and oldsize are different enough that we need to move the
* object. In that case, fall back to allocating new space and copying.
*/
ret = arena_ralloc_move_helper(tsdn, arena, usize, alignment, zero,
tcache);
if (ret == NULL)
return (NULL);
/*
* Junk/zero-filling were already done by
* ipalloc()/arena_malloc().
*/
copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
return (ret);
}

View File

@ -58,7 +58,8 @@ static void chunk_record(tsdn_t *tsdn, arena_t *arena,
static void
extent_heaps_insert(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
{
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
size_t psz =
extent_size_quantize_floor(CHUNK_CEILING(extent_size_get(extent)));
pszind_t pind = psz2ind(psz);
extent_heap_insert(&extent_heaps[pind], extent);
}
@ -66,7 +67,8 @@ extent_heaps_insert(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
static void
extent_heaps_remove(extent_heap_t extent_heaps[NPSIZES], extent_t *extent)
{
size_t psz = extent_size_quantize_floor(extent_size_get(extent));
size_t psz =
extent_size_quantize_floor(CHUNK_CEILING(extent_size_get(extent)));
pszind_t pind = psz2ind(psz);
extent_heap_remove(&extent_heaps[pind], extent);
}
@ -211,7 +213,7 @@ chunk_register(tsdn_t *tsdn, const extent_t *extent)
extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent);
extent_rtree_release(tsdn, elm_a, elm_b);
if (config_prof && opt_prof) {
if (config_prof && opt_prof && extent_active_get(extent)) {
size_t nadd = (extent_size_get(extent) == 0) ? 1 :
extent_size_get(extent) / chunksize;
size_t cur = atomic_add_z(&curchunks, nadd);
@ -239,7 +241,7 @@ chunk_deregister(tsdn_t *tsdn, const extent_t *extent)
extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL);
extent_rtree_release(tsdn, elm_a, elm_b);
if (config_prof && opt_prof) {
if (config_prof && opt_prof && extent_active_get(extent)) {
size_t nsub = (extent_size_get(extent) == 0) ? 1 :
extent_size_get(extent) / chunksize;
assert(atomic_read_z(&curchunks) >= nsub);
@ -293,23 +295,15 @@ chunk_leak(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, bool cache,
}
}
static void *
static extent_t *
chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
extent_heap_t extent_heaps[NPSIZES], bool cache, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit, bool dalloc_extent)
size_t size, size_t alignment, bool *zero, bool *commit)
{
void *ret;
extent_t *extent;
size_t alloc_size, leadsize, trailsize;
bool zeroed, committed;
assert(new_addr == NULL || alignment == chunksize);
/*
* Cached chunks use the extent linkage embedded in their headers, in
* which case dalloc_extent is true, and new_addr is non-NULL because
* we're operating on a specific chunk.
*/
assert(dalloc_extent || new_addr != NULL);
alloc_size = CHUNK_CEILING(s2u(size + alignment - chunksize));
/* Beware size_t wrap-around. */
@ -338,99 +332,79 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
return (NULL);
}
extent_heaps_remove(extent_heaps, extent);
arena_chunk_cache_maybe_remove(arena, extent, cache);
leadsize = ALIGNMENT_CEILING((uintptr_t)extent_addr_get(extent),
alignment) - (uintptr_t)extent_addr_get(extent);
assert(new_addr == NULL || leadsize == 0);
assert(extent_size_get(extent) >= leadsize + size);
trailsize = extent_size_get(extent) - leadsize - size;
ret = (void *)((uintptr_t)extent_addr_get(extent) + leadsize);
zeroed = extent_zeroed_get(extent);
if (zeroed)
if (extent_zeroed_get(extent))
*zero = true;
committed = extent_committed_get(extent);
if (committed)
if (extent_committed_get(extent))
*commit = true;
/* Split the lead. */
if (leadsize != 0 &&
chunk_hooks->split(extent_addr_get(extent),
extent_size_get(extent), leadsize, size, false, arena->ind)) {
malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
return (NULL);
}
/* Remove extent from the heap. */
chunk_deregister(tsdn, extent);
extent_heaps_remove(extent_heaps, extent);
arena_chunk_cache_maybe_remove(arena, extent, cache);
if (leadsize != 0) {
/* Insert the leading space as a smaller chunk. */
extent_size_set(extent, leadsize);
if (chunk_register(tsdn, extent)) {
extent_t *lead = extent;
extent = chunk_split_wrapper(tsdn, arena, chunk_hooks, lead,
leadsize, size + trailsize);
if (extent == NULL) {
chunk_leak(tsdn, arena, chunk_hooks, cache,
extent_addr_get(extent), extent_size_get(extent));
arena_extent_dalloc(tsdn, arena, extent);
} else {
extent_heaps_insert(extent_heaps, extent);
arena_chunk_cache_maybe_insert(arena, extent, cache);
}
extent = NULL;
}
if (trailsize != 0) {
/* Split the trail. */
if (chunk_hooks->split(ret, size + trailsize, size,
trailsize, false, arena->ind)) {
if (dalloc_extent && extent != NULL)
arena_extent_dalloc(tsdn, arena, extent);
extent_addr_get(lead), extent_size_get(lead));
arena_extent_dalloc(tsdn, arena, lead);
malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
chunk_record(tsdn, arena, chunk_hooks, extent_heaps,
cache, ret, size + trailsize, zeroed, committed);
return (NULL);
}
/* Insert the trailing space as a smaller chunk. */
if (extent == NULL) {
extent = arena_extent_alloc(tsdn, arena);
if (extent == NULL) {
malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
chunk_record(tsdn, arena, chunk_hooks,
extent_heaps, cache, ret, size + trailsize,
zeroed, committed);
return (NULL);
}
}
extent_init(extent, arena, (void *)((uintptr_t)(ret) + size),
trailsize, false, zeroed, committed, false);
if (chunk_register(tsdn, extent)) {
extent_heaps_insert(extent_heaps, lead);
arena_chunk_cache_maybe_insert(arena, lead, cache);
}
/* Split the trail. */
if (trailsize != 0) {
extent_t *trail = chunk_split_wrapper(tsdn, arena, chunk_hooks,
extent, size, trailsize);
if (trail == NULL) {
chunk_leak(tsdn, arena, chunk_hooks, cache,
extent_addr_get(extent), extent_size_get(extent));
arena_extent_dalloc(tsdn, arena, extent);
} else {
extent_heaps_insert(extent_heaps, extent);
arena_chunk_cache_maybe_insert(arena, extent, cache);
malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
return (NULL);
}
extent = NULL;
extent_heaps_insert(extent_heaps, trail);
arena_chunk_cache_maybe_insert(arena, trail, cache);
}
if (!committed && chunk_hooks->commit(ret, size, 0, size, arena->ind)) {
if (!extent_committed_get(extent) &&
chunk_hooks->commit(extent_addr_get(extent),
extent_size_get(extent), 0, extent_size_get(extent), arena->ind)) {
malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
chunk_record(tsdn, arena, chunk_hooks, extent_heaps, cache, ret,
size, zeroed, committed);
chunk_record(tsdn, arena, chunk_hooks, extent_heaps, cache,
extent_addr_get(extent), extent_size_get(extent),
extent_zeroed_get(extent), extent_committed_get(extent));
arena_extent_dalloc(tsdn, arena, extent);
return (NULL);
}
extent_active_set(extent, true);
malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
assert(dalloc_extent || extent != NULL);
if (dalloc_extent && extent != NULL)
arena_extent_dalloc(tsdn, arena, extent);
if (*zero) {
if (!zeroed)
memset(ret, 0, size);
else if (config_debug) {
if (!extent_zeroed_get(extent)) {
memset(extent_addr_get(extent), 0,
extent_size_get(extent));
} else if (config_debug) {
size_t i;
size_t *p = (size_t *)(uintptr_t)ret;
size_t *p = (size_t *)(uintptr_t)
extent_addr_get(extent);
for (i = 0; i < size / sizeof(size_t); i++)
assert(p[i] == 0);
}
}
return (ret);
return (extent);
}
/*
@ -469,12 +443,11 @@ chunk_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
return (NULL);
}
void *
extent_t *
chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero,
bool dalloc_extent)
void *new_addr, size_t size, size_t alignment, bool *zero)
{
void *ret;
extent_t *extent;
bool commit;
assert(size != 0);
@ -483,12 +456,12 @@ chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
assert((alignment & chunksize_mask) == 0);
commit = true;
ret = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_cached,
true, new_addr, size, alignment, zero, &commit, dalloc_extent);
if (ret == NULL)
extent = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_cached,
true, new_addr, size, alignment, zero, &commit);
if (extent == NULL)
return (NULL);
assert(commit);
return (ret);
return (extent);
}
static arena_t *
@ -523,44 +496,51 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
return (ret);
}
static void *
static extent_t *
chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
{
void *ret;
extent_t *extent;
assert(size != 0);
assert((size & chunksize_mask) == 0);
assert(alignment != 0);
assert((alignment & chunksize_mask) == 0);
ret = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_retained,
false, new_addr, size, alignment, zero, commit, true);
extent = chunk_recycle(tsdn, arena, chunk_hooks, arena->chunks_retained,
false, new_addr, size, alignment, zero, commit);
if (config_stats && ret != NULL)
if (config_stats && extent != NULL)
arena->stats.retained -= size;
return (ret);
return (extent);
}
void *
extent_t *
chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
{
void *ret;
extent_t *extent;
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
ret = chunk_alloc_retained(tsdn, arena, chunk_hooks, new_addr, size,
extent = chunk_alloc_retained(tsdn, arena, chunk_hooks, new_addr, size,
alignment, zero, commit);
if (ret == NULL) {
ret = chunk_hooks->alloc(new_addr, size, alignment, zero,
commit, arena->ind);
if (ret == NULL)
if (extent == NULL) {
void *chunk;
extent = arena_extent_alloc(tsdn, arena);
if (extent == NULL)
return (NULL);
chunk = chunk_hooks->alloc(new_addr, size, alignment,
zero, commit, arena->ind);
if (chunk == NULL)
return (NULL);
extent_init(extent, arena, chunk, size, true, zero, commit,
false);
}
return (ret);
return (extent);
}
static bool
@ -668,7 +648,6 @@ chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
chunk_record(tsdn, arena, chunk_hooks, arena->chunks_cached, true,
chunk, size, false, committed);
arena_maybe_purge(tsdn, arena);
}
static bool
@ -779,6 +758,67 @@ chunk_split_default(void *chunk, size_t size, size_t size_a, size_t size_b,
return (false);
}
extent_t *
chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
extent_t *extent, size_t size_a, size_t size_b)
{
extent_t *trail;
rtree_elm_t *lead_elm_a, *lead_elm_b, *trail_elm_a, *trail_elm_b;
assert(CHUNK_CEILING(size_a) == size_a);
assert(CHUNK_CEILING(extent_size_get(extent)) == size_a +
CHUNK_CEILING(size_b));
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
trail = arena_extent_alloc(tsdn, arena);
if (trail == NULL)
goto label_error_a;
{
extent_t lead;
extent_init(&lead, arena, extent_addr_get(extent), size_a,
extent_active_get(extent), extent_zeroed_get(extent),
extent_committed_get(extent), extent_slab_get(extent));
if (extent_rtree_acquire(tsdn, &lead, false, true, &lead_elm_a,
&lead_elm_b))
goto label_error_b;
}
extent_init(trail, arena, (void *)((uintptr_t)extent_addr_get(extent) +
size_a), CHUNK_CEILING(size_b), extent_active_get(extent),
extent_zeroed_get(extent), extent_committed_get(extent),
extent_slab_get(extent));
if (extent_rtree_acquire(tsdn, trail, false, true, &trail_elm_a,
&trail_elm_b))
goto label_error_c;
if (chunk_hooks->split(extent_addr_get(extent), size_a +
CHUNK_CEILING(size_b), size_a, CHUNK_CEILING(size_b),
extent_committed_get(extent), arena->ind))
goto label_error_d;
extent_size_set(extent, size_a);
extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent);
extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail);
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
extent_rtree_release(tsdn, trail_elm_a, trail_elm_b);
return (trail);
label_error_d:
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
label_error_c:
extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
label_error_b:
arena_extent_dalloc(tsdn, arena, trail);
label_error_a:
return (NULL);
}
static bool
chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
bool committed, unsigned arena_ind)
@ -801,6 +841,7 @@ chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
{
rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
if (chunk_hooks->merge(extent_addr_get(a), extent_size_get(a),
extent_addr_get(b), extent_size_get(b), extent_committed_get(a),
arena->ind))

View File

@ -16,7 +16,6 @@ void *
huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero)
{
void *ret;
size_t ausize;
extent_t *extent;
bool is_zeroed;
@ -30,12 +29,6 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
return (NULL);
assert(ausize >= chunksize);
/* Allocate an extent with which to track the chunk. */
extent = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_t)),
CACHELINE, false, NULL, true, arena_ichoose(tsdn, arena));
if (extent == NULL)
return (NULL);
/*
* Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
* it is possible to make correct junk/zero fill decisions below.
@ -43,19 +36,17 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
is_zeroed = zero;
if (likely(!tsdn_null(tsdn)))
arena = arena_choose(tsdn_tsd(tsdn), arena);
if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL) {
idalloctm(tsdn, iealloc(tsdn, extent), extent, NULL, true,
true);
if (unlikely(arena == NULL) || (extent = arena_chunk_alloc_huge(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL)
return (NULL);
}
extent_init(extent, arena, ret, usize, true, is_zeroed, true, false);
if (usize < extent_size_get(extent))
extent_size_set(extent, usize);
if (chunk_register(tsdn, extent)) {
arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
idalloctm(tsdn, iealloc(tsdn, extent), extent, NULL, true,
true);
arena_chunk_dalloc_huge(tsdn, arena, extent_addr_get(extent),
usize);
arena_extent_dalloc(tsdn, arena, extent);
return (NULL);
}
@ -67,12 +58,12 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed)
memset(ret, 0, usize);
memset(extent_addr_get(extent), 0, usize);
} else if (config_fill && unlikely(opt_junk_alloc))
memset(ret, JEMALLOC_ALLOC_JUNK, usize);
memset(extent_addr_get(extent), JEMALLOC_ALLOC_JUNK, usize);
arena_decay_tick(tsdn, arena);
return (ret);
return (extent_addr_get(extent));
}
#ifdef JEMALLOC_JET
@ -99,11 +90,12 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
#endif
static void
huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero)
{
size_t usize, usize_next;
arena_t *arena;
arena_t *arena = extent_arena_get(extent);
size_t oldsize = extent_size_get(extent);
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
bool pre_zeroed, post_zeroed;
@ -115,20 +107,19 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, void *ptr,
if (oldsize == usize)
return;
arena = extent_arena_get(extent);
pre_zeroed = extent_zeroed_get(extent);
/* Fill if necessary (shrinking). */
if (oldsize > usize) {
size_t sdiff = oldsize - usize;
if (config_fill && unlikely(opt_junk_free)) {
memset((void *)((uintptr_t)ptr + usize),
JEMALLOC_FREE_JUNK, sdiff);
memset((void *)((uintptr_t)extent_addr_get(extent) +
usize), JEMALLOC_FREE_JUNK, sdiff);
post_zeroed = false;
} else {
post_zeroed = !chunk_purge_wrapper(tsdn, arena,
&chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
sdiff);
&chunk_hooks, extent_addr_get(extent),
CHUNK_CEILING(oldsize), usize, sdiff);
}
} else
post_zeroed = pre_zeroed;
@ -143,132 +134,157 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, void *ptr,
/* Update zeroed. */
extent_zeroed_set(extent, post_zeroed);
arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
arena_chunk_ralloc_huge_similar(tsdn, arena, extent, oldsize);
/* Fill if necessary (growing). */
if (oldsize < usize) {
if (zero || (config_fill && unlikely(opt_zero))) {
if (!pre_zeroed) {
memset((void *)((uintptr_t)ptr + oldsize), 0,
usize - oldsize);
memset((void *)
((uintptr_t)extent_addr_get(extent) +
oldsize), 0, usize - oldsize);
}
} else if (config_fill && unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)ptr + oldsize),
JEMALLOC_ALLOC_JUNK, usize - oldsize);
memset((void *)((uintptr_t)extent_addr_get(extent) +
oldsize), JEMALLOC_ALLOC_JUNK, usize - oldsize);
}
}
}
static bool
huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t usize)
huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
{
arena_t *arena;
chunk_hooks_t chunk_hooks;
size_t cdiff;
bool pre_zeroed, post_zeroed;
arena = extent_arena_get(extent);
pre_zeroed = extent_zeroed_get(extent);
chunk_hooks = chunk_hooks_get(tsdn, arena);
arena_t *arena = extent_arena_get(extent);
size_t oldsize = extent_size_get(extent);
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena);
size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
size_t sdiff = CHUNK_CEILING(usize) - usize;
assert(oldsize > usize);
/* Split excess chunks. */
cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize),
CHUNK_CEILING(usize), cdiff, true, arena->ind))
return (true);
if (cdiff != 0) {
extent_t *trail = chunk_split_wrapper(tsdn, arena, &chunk_hooks,
extent, CHUNK_CEILING(usize), cdiff);
if (trail == NULL)
return (true);
if (oldsize > usize) {
size_t sdiff = oldsize - usize;
if (config_fill && unlikely(opt_junk_free)) {
huge_dalloc_junk(tsdn, (void *)((uintptr_t)ptr + usize),
sdiff);
huge_dalloc_junk(tsdn, extent_addr_get(trail),
extent_size_get(trail));
}
arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks,
extent_addr_get(trail), extent_size_get(trail),
extent_committed_get(trail));
arena_extent_dalloc(tsdn, arena, trail);
}
/* Optionally fill trailing subchunk. */
if (sdiff != 0) {
bool post_zeroed;
if (config_fill && unlikely(opt_junk_free)) {
huge_dalloc_junk(tsdn,
(void *)((uintptr_t)extent_addr_get(extent) +
usize), sdiff);
post_zeroed = false;
} else {
post_zeroed = !chunk_purge_wrapper(tsdn, arena,
&chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
usize), CHUNK_CEILING(oldsize),
CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
&chunk_hooks, extent_addr_get(extent),
CHUNK_CEILING(usize), usize, sdiff);
if (config_fill && unlikely(opt_zero) && !post_zeroed) {
memset((void *)
((uintptr_t)extent_addr_get(extent) +
usize), 0, sdiff);
}
}
} else
post_zeroed = pre_zeroed;
extent_zeroed_set(extent, post_zeroed);
}
/* Update the size of the huge allocation. */
chunk_deregister(tsdn, extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize);
/* Update zeroed. */
extent_zeroed_set(extent, post_zeroed);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
chunk_reregister(tsdn, extent);
/* Zap the excess chunks. */
arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
arena_chunk_ralloc_huge_shrink(tsdn, arena, extent, oldsize);
return (false);
}
static bool
huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t usize, bool zero)
huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool zero)
{
arena_t *arena;
bool is_zeroed_subchunk;
arena_t *arena = extent_arena_get(extent);
size_t oldsize = extent_size_get(extent);
bool is_zeroed_subchunk = extent_zeroed_get(extent);
bool is_zeroed_chunk = false;
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena);
size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
void *nchunk =
(void *)CHUNK_CEILING((uintptr_t)extent_past_get(extent));
extent_t *trail;
arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
is_zeroed_subchunk = extent_zeroed_get(extent);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
if ((trail = arena_chunk_cache_alloc(tsdn, arena, &chunk_hooks, nchunk,
cdiff, chunksize, &is_zeroed_chunk)) == NULL) {
bool commit = true;
if ((trail = chunk_alloc_wrapper(tsdn, arena, &chunk_hooks,
nchunk, cdiff, chunksize, &is_zeroed_chunk, &commit)) ==
NULL)
return (true);
}
if (arena_chunk_ralloc_huge_expand(tsdn, arena, extent, usize))
if (chunk_merge_wrapper(tsdn, arena, &chunk_hooks, extent, trail)) {
arena_extent_dalloc(tsdn, arena, trail);
chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks,
extent_addr_get(trail), extent_size_get(trail),
extent_zeroed_get(trail), extent_committed_get(trail));
return (true);
}
if (zero || (config_fill && unlikely(opt_zero))) {
bool is_zeroed_chunk = extent_zeroed_get(extent);
if (!is_zeroed_subchunk) {
memset((void *)((uintptr_t)ptr + oldsize), 0,
CHUNK_CEILING(oldsize) - oldsize);
memset((void *)((uintptr_t)extent_addr_get(extent) +
oldsize), 0, CHUNK_CEILING(oldsize) - oldsize);
}
if (!is_zeroed_chunk) {
memset((void *)((uintptr_t)ptr +
memset((void *)((uintptr_t)extent_addr_get(extent) +
CHUNK_CEILING(oldsize)), 0, usize -
CHUNK_CEILING(oldsize));
}
} else if (config_fill && unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)ptr + oldsize), JEMALLOC_ALLOC_JUNK,
usize - oldsize);
memset((void *)((uintptr_t)extent_addr_get(extent) + oldsize),
JEMALLOC_ALLOC_JUNK, usize - oldsize);
}
if (usize < extent_size_get(extent))
extent_size_set(extent, usize);
arena_chunk_ralloc_huge_expand(tsdn, arena, extent, oldsize);
return (false);
}
bool
huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t usize_min, size_t usize_max, bool zero)
huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero)
{
assert(s2u(oldsize) == oldsize);
assert(s2u(extent_size_get(extent)) == extent_size_get(extent));
/* The following should have been caught by callers. */
assert(usize_min > 0 && usize_max <= HUGE_MAXCLASS);
/* Both allocation sizes must be huge to avoid a move. */
assert(extent_size_get(extent) >= chunksize && usize_max >= chunksize);
/* Both allocations must be huge to avoid a move. */
if (oldsize < chunksize || usize_max < chunksize)
return (true);
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(extent_size_get(extent))) {
/* Attempt to expand the allocation in-place. */
if (!huge_ralloc_no_move_expand(tsdn, extent, ptr, oldsize,
usize_max, zero)) {
if (!huge_ralloc_no_move_expand(tsdn, extent, usize_max,
zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
/* Try again, this time with usize_min. */
if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
extent, ptr, oldsize, usize_min, zero)) {
CHUNK_CEILING(extent_size_get(extent)) &&
huge_ralloc_no_move_expand(tsdn, extent, usize_min, zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
@ -278,18 +294,18 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
* Avoid moving the allocation if the existing chunk size accommodates
* the new size.
*/
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
huge_ralloc_no_move_similar(tsdn, extent, ptr, oldsize,
usize_min, usize_max, zero);
if (CHUNK_CEILING(extent_size_get(extent)) >= CHUNK_CEILING(usize_min)
&& CHUNK_CEILING(extent_size_get(extent)) <=
CHUNK_CEILING(usize_max)) {
huge_ralloc_no_move_similar(tsdn, extent, usize_min, usize_max,
zero);
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
/* Attempt to shrink the allocation in-place. */
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
if (!huge_ralloc_no_move_shrink(tsdn, extent, ptr, oldsize,
usize_max)) {
if (CHUNK_CEILING(extent_size_get(extent)) > CHUNK_CEILING(usize_max)) {
if (!huge_ralloc_no_move_shrink(tsdn, extent, usize_max)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
@ -308,22 +324,23 @@ huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
}
void *
huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t oldsize, size_t usize, size_t alignment, bool zero, tcache_t *tcache)
huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
size_t alignment, bool zero, tcache_t *tcache)
{
void *ret;
size_t copysize;
/* The following should have been caught by callers. */
assert(usize > 0 && usize <= HUGE_MAXCLASS);
/* Both allocation sizes must be huge to avoid a move. */
assert(extent_size_get(extent) >= chunksize && usize >= chunksize);
/* Try to avoid moving the allocation. */
if (!huge_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, usize,
zero))
return (ptr);
if (!huge_ralloc_no_move(tsdn, extent, usize, usize, zero))
return (extent_addr_get(extent));
/*
* usize and oldsize are different enough that we need to use a
* usize and old size are different enough that we need to use a
* different size class. In that case, fall back to allocating new
* space and copying.
*/
@ -331,14 +348,16 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
if (ret == NULL)
return (NULL);
copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
copysize = (usize < extent_size_get(extent)) ? usize :
extent_size_get(extent);
memcpy(ret, extent_addr_get(extent), copysize);
isdalloct(tsdn, extent, extent_addr_get(extent),
extent_size_get(extent), tcache, true);
return (ret);
}
void
huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr)
huge_dalloc(tsdn_t *tsdn, extent_t *extent)
{
arena_t *arena;
@ -352,13 +371,13 @@ huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr)
extent_size_get(extent));
arena_chunk_dalloc_huge(tsdn, extent_arena_get(extent),
extent_addr_get(extent), extent_size_get(extent));
idalloctm(tsdn, iealloc(tsdn, extent), extent, NULL, true, true);
arena_extent_dalloc(tsdn, arena, extent);
arena_decay_tick(tsdn, arena);
}
size_t
huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
huge_salloc(tsdn_t *tsdn, const extent_t *extent)
{
size_t size;
arena_t *arena;
@ -372,13 +391,11 @@ huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
}
prof_tctx_t *
huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent)
{
prof_tctx_t *tctx;
arena_t *arena;
assert(extent == iealloc(tsdn, ptr));
arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
tctx = extent_prof_tctx_get(extent);
@ -388,13 +405,10 @@ huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
}
void
huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx)
huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx)
{
arena_t *arena;
assert(extent == iealloc(tsdn, ptr));
arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_prof_tctx_set(extent, tctx);
@ -402,8 +416,8 @@ huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
}
void
huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr)
huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent)
{
huge_prof_tctx_set(tsdn, extent, ptr, (prof_tctx_t *)(uintptr_t)1U);
huge_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U);
}