project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move extent ehook calls into ehooks

Browse Source
This commit is contained in:
David Goldblatt 2019-12-02 14:19:22 -08:00 committed by David Goldblatt
parent ba8b9ecbcb
commit ae0d8e8591
13 changed files with 511 additions and 404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/jemalloc/internal/arena_externs.h
View File

@ -28,7 +28,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
bin_stats_data_t *bstats, arena_stats_large_t *lstats,
arena_stats_extents_t *estats);
void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent);
ehooks_t *ehooks, extent_t *extent);
#ifdef JEMALLOC_JET
size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
#endif
@ -72,7 +72,7 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, tcache_t *tcache,
hook_ralloc_args_t *hook_args);
dss_prec_t arena_dss_prec_get(arena_t *arena);
extent_hooks_t *arena_get_extent_hooks(arena_t *arena);
ehooks_t *arena_get_ehooks(arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena,
extent_hooks_t *extent_hooks);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);

9
include/jemalloc/internal/arena_inlines_b.h
View File

@ -14,11 +14,6 @@ arena_get_from_extent(extent_t *extent) {
ATOMIC_RELAXED);
}
JEMALLOC_ALWAYS_INLINE bool
arena_has_default_hooks(arena_t *arena) {
return (arena_get_extent_hooks(arena) == &extent_hooks_default);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
if (arena != NULL) {
@ -134,10 +129,10 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
/* Purge a single extent to retained / unmapped directly. */
JEMALLOC_ALWAYS_INLINE void
arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t *extent_hooks,
arena_decay_extent(tsdn_t *tsdn,arena_t *arena, ehooks_t *ehooks,
extent_t *extent) {
size_t extent_size = extent_size_get(extent);
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent);
extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
if (config_stats) {
/* Update stats accordingly. */
arena_stats_lock(tsdn, &arena->stats);

2
include/jemalloc/internal/base_externs.h
View File

@ -7,7 +7,7 @@ extern const char *metadata_thp_mode_names[];
base_t *b0get(void);
base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
void base_delete(tsdn_t *tsdn, base_t *base);
extent_hooks_t *base_extent_hooks_get(base_t *base);
ehooks_t *base_ehooks_get(base_t *base);
extent_hooks_t *base_extent_hooks_set(base_t *base,
extent_hooks_t *extent_hooks);
void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);

6
include/jemalloc/internal/base_structs.h
View File

@ -1,6 +1,7 @@
#ifndef JEMALLOC_INTERNAL_BASE_STRUCTS_H
#define JEMALLOC_INTERNAL_BASE_STRUCTS_H
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/extent.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/mutex.h"
@ -23,10 +24,9 @@ struct base_s {
unsigned ind;
/*
* User-configurable extent hook functions. Points to an
* extent_hooks_t.
* User-configurable extent hook functions.
*/
atomic_p_t extent_hooks;
ehooks_t ehooks;
/* Protects base_alloc() and base_stats_get() operations. */
malloc_mutex_t mtx;

147
include/jemalloc/internal/ehooks.h
View File

@ -1,4 +1,151 @@
#ifndef JEMALLOC_INTERNAL_EHOOKS_H
#define JEMALLOC_INTERNAL_EHOOKS_H
#include "jemalloc/internal/atomic.h"
extern const extent_hooks_t extent_hooks_default;
typedef struct ehooks_s ehooks_t;
struct ehooks_s {
/* Logically an extent_hooks_t *. */
atomic_p_t ptr;
};
void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks);
static inline void
ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
atomic_store_p(&ehooks->ptr, extent_hooks, ATOMIC_RELEASE);
}
static inline extent_hooks_t *
ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
}
static inline bool
ehooks_are_default(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks) == &extent_hooks_default;
}
static inline bool
ehooks_destroy_is_noop(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->destroy == NULL;
}
static inline bool
ehooks_purge_lazy_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->purge_lazy == NULL;
}
static inline bool
ehooks_purge_forced_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->purge_forced == NULL;
}
static inline bool
ehooks_split_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
}
static inline bool
ehooks_merge_will_fail(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
}
static inline void *
ehooks_alloc(ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool *commit, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
return extent_hooks->alloc(extent_hooks, new_addr, size, alignment,
zero, commit, arena_ind);
}
static inline bool
ehooks_dalloc(ehooks_t *ehooks, void *addr, size_t size, bool committed,
unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->dalloc == NULL) {
return true;
}
return extent_hooks->dalloc(extent_hooks, addr, size, committed,
arena_ind);
}
static inline void
ehooks_destroy(ehooks_t *ehooks, void *addr, size_t size, bool committed,
unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->destroy == NULL) {
return;
}
extent_hooks->destroy(extent_hooks, addr, size, committed, arena_ind);
}
static inline bool
ehooks_commit(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->commit == NULL) {
return true;
}
return extent_hooks->commit(extent_hooks, addr, size, offset, length,
arena_ind);
}
static inline bool
ehooks_decommit(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->decommit == NULL) {
return true;
}
return extent_hooks->decommit(extent_hooks, addr, size, offset, length,
arena_ind);
}
static inline bool
ehooks_purge_lazy(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->purge_lazy == NULL) {
return true;
}
return extent_hooks->purge_lazy(extent_hooks, addr, size, offset,
length, arena_ind);
}
static inline bool
ehooks_purge_forced(ehooks_t *ehooks, void *addr, size_t size, size_t offset,
size_t length, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->purge_forced == NULL) {
return true;
}
return extent_hooks->purge_forced(extent_hooks, addr, size, offset,
length, arena_ind);
}
static inline bool
ehooks_split(ehooks_t *ehooks, void *addr, size_t size, size_t size_a,
size_t size_b, bool committed, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->split == NULL) {
return true;
}
return extent_hooks->split(extent_hooks, addr, size, size_a, size_b,
committed, arena_ind);
}
static inline bool
ehooks_merge(ehooks_t *ehooks, void *addr_a, size_t size_a, void *addr_b,
size_t size_b, bool committed, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks->merge == NULL) {
return true;
}
return extent_hooks->merge(extent_hooks, addr_a, size_a, addr_b, size_b,
committed, arena_ind);
}
#endif /* JEMALLOC_INTERNAL_EHOOKS_H */

61
include/jemalloc/internal/extent_externs.h
View File

@ -1,6 +1,7 @@
#ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
#define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/mutex_pool.h"
#include "jemalloc/internal/ph.h"
@ -9,7 +10,6 @@
extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree;
extern const extent_hooks_t extent_hooks_default;
extern mutex_pool_t extent_mutex_pool;
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
@ -18,39 +18,34 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
ph_proto(, extent_avail_, extent_tree_t, extent_t)
ph_proto(, extent_heap_, extent_heap_t, extent_t)
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, eset_t *eset, void *new_addr,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
bool *zero, bool *commit);
void extents_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, eset_t *eset, extent_t *extent);
extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, eset_t *eset, size_t npages_min);
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit);
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit);
void extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, extent_t *extent);
extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, size_t npages_min);
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit);
void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent);
void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length);
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b);
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent);
void extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length);
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length);
bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length);
bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length);
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
size_t size_b, szind_t szind_b, bool slab_b);
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *a, extent_t *b);
bool extent_boot(void);

93
src/arena.c
View File

@ -4,6 +4,7 @@
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h"
#include "jemalloc/internal/mutex.h"
@ -252,13 +253,12 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
}
void
arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent) {
arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extents_dalloc(tsdn, arena, extent_hooks, &arena->eset_dirty,
extent);
extents_dalloc(tsdn, arena, ehooks, &arena->eset_dirty, extent);
if (arena_dirty_decay_ms_get(arena) == 0) {
arena_decay_dirty(tsdn, arena, false, true);
} else {
@ -426,7 +426,7 @@ arena_may_have_muzzy(arena_t *arena) {
extent_t *
arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero) {
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
ehooks_t *ehooks = arena_get_ehooks(arena);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
@ -434,19 +434,18 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
szind_t szind = sz_size2index(usize);
size_t mapped_add;
bool commit = true;
extent_t *extent = extents_alloc(tsdn, arena, extent_hooks,
extent_t *extent = extents_alloc(tsdn, arena, ehooks,
&arena->eset_dirty, NULL, usize, sz_large_pad, alignment, false,
szind, zero, &commit);
if (extent == NULL && arena_may_have_muzzy(arena)) {
extent = extents_alloc(tsdn, arena, extent_hooks,
&arena->eset_muzzy, NULL, usize, sz_large_pad, alignment,
false, szind, zero, &commit);
extent = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
NULL, usize, sz_large_pad, alignment, false, szind, zero,
&commit);
}
size_t size = usize + sz_large_pad;
if (extent == NULL) {
extent = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL,
usize, sz_large_pad, alignment, false, szind, zero,
&commit);
extent = extent_alloc_wrapper(tsdn, arena, ehooks, NULL, usize,
sz_large_pad, alignment, false, szind, zero, &commit);
if (config_stats) {
/*
* extent may be NULL on OOM, but in that case
@ -819,8 +818,8 @@ arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena,
static size_t
arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, eset_t *eset, size_t npages_limit,
size_t npages_decay_max, extent_list_t *decay_extents) {
ehooks_t *ehooks, eset_t *eset, size_t npages_limit,
size_t npages_decay_max, extent_list_t *decay_extents) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
@ -828,8 +827,8 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
size_t nstashed = 0;
extent_t *extent;
while (nstashed < npages_decay_max &&
(extent = extents_evict(tsdn, arena, extent_hooks, eset,
npages_limit)) != NULL) {
(extent = extents_evict(tsdn, arena, ehooks, eset, npages_limit))
!= NULL) {
extent_list_append(decay_extents, extent);
nstashed += extent_size_get(extent) >> LG_PAGE;
}
@ -837,9 +836,9 @@ arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
}
static size_t
arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, arena_decay_t *decay, eset_t *eset,
bool all, extent_list_t *decay_extents, bool is_background_thread) {
arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
arena_decay_t *decay, eset_t *eset, bool all, extent_list_t *decay_extents,
bool is_background_thread) {
size_t nmadvise, nunmapped;
size_t npurged;
@ -864,9 +863,9 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
case extent_state_dirty:
if (!all && muzzy_decay_ms != 0 &&
!extent_purge_lazy_wrapper(tsdn, arena,
extent_hooks, extent, 0,
ehooks, extent, 0,
extent_size_get(extent))) {
extents_dalloc(tsdn, arena, extent_hooks,
extents_dalloc(tsdn, arena, ehooks,
&arena->eset_muzzy, extent);
arena_background_thread_inactivity_check(tsdn,
arena, is_background_thread);
@ -874,8 +873,7 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
}
JEMALLOC_FALLTHROUGH;
case extent_state_muzzy:
extent_dalloc_wrapper(tsdn, arena, extent_hooks,
extent);
extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
if (config_stats) {
nunmapped += npages;
}
@ -923,17 +921,16 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
decay->purging = true;
malloc_mutex_unlock(tsdn, &decay->mtx);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
ehooks_t *ehooks = arena_get_ehooks(arena);
extent_list_t decay_extents;
extent_list_init(&decay_extents);
size_t npurge = arena_stash_decayed(tsdn, arena, extent_hooks, eset,
size_t npurge = arena_stash_decayed(tsdn, arena, ehooks, eset,
npages_limit, npages_decay_max, &decay_extents);
if (npurge != 0) {
size_t npurged = arena_decay_stashed(tsdn, arena,
extent_hooks, decay, eset, all, &decay_extents,
is_background_thread);
size_t npurged = arena_decay_stashed(tsdn, arena, ehooks, decay,
eset, all, &decay_extents, is_background_thread);
assert(npurged == npurge);
}
@ -1006,8 +1003,8 @@ static void
arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) {
arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, slab);
ehooks_t *ehooks = arena_get_ehooks(arena);
arena_extents_dirty_dalloc(tsdn, arena, ehooks, slab);
}
static void
@ -1159,11 +1156,11 @@ arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) {
* destroyed, or provide custom extent hooks that track retained
* dss-based extents for later reuse.
*/
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
ehooks_t *ehooks = arena_get_ehooks(arena);
extent_t *extent;
while ((extent = extents_evict(tsdn, arena, extent_hooks,
while ((extent = extents_evict(tsdn, arena, ehooks,
&arena->eset_retained, 0)) != NULL) {
extent_destroy_wrapper(tsdn, arena, extent_hooks, extent);
extent_destroy_wrapper(tsdn, arena, ehooks, extent);
}
}
@ -1204,9 +1201,8 @@ arena_destroy(tsd_t *tsd, arena_t *arena) {
}
static extent_t *
arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, const bin_info_t *bin_info,
szind_t szind) {
arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
const bin_info_t *bin_info, szind_t szind) {
extent_t *slab;
bool zero, commit;
@ -1215,7 +1211,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
zero = false;
commit = true;
slab = extent_alloc_wrapper(tsdn, arena, extent_hooks, NULL,
slab = extent_alloc_wrapper(tsdn, arena, ehooks, NULL,
bin_info->slab_size, 0, PAGE, true, szind, &zero, &commit);
if (config_stats && slab != NULL) {
@ -1232,21 +1228,20 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
ehooks_t *ehooks = arena_get_ehooks(arena);
szind_t szind = sz_size2index(bin_info->reg_size);
bool zero = false;
bool commit = true;
extent_t *slab = extents_alloc(tsdn, arena, extent_hooks,
&arena->eset_dirty, NULL, bin_info->slab_size, 0, PAGE, true,
binind, &zero, &commit);
extent_t *slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit);
if (slab == NULL && arena_may_have_muzzy(arena)) {
slab = extents_alloc(tsdn, arena, extent_hooks,
&arena->eset_muzzy, NULL, bin_info->slab_size, 0, PAGE,
true, binind, &zero, &commit);
slab = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero,
&commit);
}
if (slab == NULL) {
slab = arena_slab_alloc_hard(tsdn, arena, extent_hooks,
bin_info, szind);
slab = arena_slab_alloc_hard(tsdn, arena, ehooks, bin_info,
szind);
if (slab == NULL) {
return NULL;
}
@ -1846,9 +1841,9 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
return ret;
}
extent_hooks_t *
arena_get_extent_hooks(arena_t *arena) {
return base_extent_hooks_get(arena->base);
ehooks_t *
arena_get_ehooks(arena_t *arena) {
return base_ehooks_get(arena->base);
}
extent_hooks_t *

69
src/base.c
View File

@ -29,7 +29,7 @@ metadata_thp_madvise(void) {
}
static void *
base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) {
base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) {
void *addr;
bool zero = true;
bool commit = true;
@ -37,7 +37,7 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size)
/* Use huge page sizes and alignment regardless of opt_metadata_thp. */
assert(size == HUGEPAGE_CEILING(size));
size_t alignment = HUGEPAGE;
if (extent_hooks == &extent_hooks_default) {
if (ehooks_are_default(ehooks)) {
addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit);
if (have_madvise_huge && addr) {
pages_set_thp_state(addr, size);
@ -46,8 +46,8 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size)
/* No arena context as we are creating new arenas. */
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
pre_reentrancy(tsd, NULL);
addr = extent_hooks->alloc(extent_hooks, NULL, size, alignment,
&zero, &commit, ind);
addr = ehooks_alloc(ehooks, NULL, size, alignment, &zero,
&commit, ind);
post_reentrancy(tsd);
}
@ -55,7 +55,7 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size)
}
static void
base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr,
size_t size) {
/*
* Cascade through dalloc, decommit, purge_forced, and purge_lazy,
@ -67,7 +67,7 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
* may in fact want the end state of all associated virtual memory to be
* in some consistent-but-allocated state.
*/
if (extent_hooks == &extent_hooks_default) {
if (ehooks_are_default(ehooks)) {
if (!extent_dalloc_mmap(addr, size)) {
goto label_done;
}
@ -85,24 +85,16 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr,
} else {
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
pre_reentrancy(tsd, NULL);
if (extent_hooks->dalloc != NULL &&
!extent_hooks->dalloc(extent_hooks, addr, size, true,
ind)) {
if (!ehooks_dalloc(ehooks, addr, size, true, ind)) {
goto label_post_reentrancy;
}
if (extent_hooks->decommit != NULL &&
!extent_hooks->decommit(extent_hooks, addr, size, 0, size,
ind)) {
if (!ehooks_decommit(ehooks, addr, size, 0, size, ind)) {
goto label_post_reentrancy;
}
if (extent_hooks->purge_forced != NULL &&
!extent_hooks->purge_forced(extent_hooks, addr, size, 0,
size, ind)) {
if (!ehooks_purge_forced(ehooks, addr, size, 0, size, ind)) {
goto label_post_reentrancy;
}
if (extent_hooks->purge_lazy != NULL &&
!extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size,
ind)) {
if (!ehooks_purge_lazy(ehooks, addr, size, 0, size, ind)) {
goto label_post_reentrancy;
}
/* Nothing worked. That's the application's problem. */
@ -248,8 +240,8 @@ base_extent_bump_alloc(base_t *base, extent_t *extent, size_t size,
* On success a pointer to the initialized base_block_t header is returned.
*/
static base_block_t *
base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
unsigned ind, pszind_t *pind_last, size_t *extent_sn_next, size_t size,
base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind,
pszind_t *pind_last, size_t *extent_sn_next, size_t size,
size_t alignment) {
alignment = ALIGNMENT_CEILING(alignment, QUANTUM);
size_t usize = ALIGNMENT_CEILING(size, alignment);
@ -270,7 +262,7 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next));
size_t block_size = (min_block_size > next_block_size) ? min_block_size
: next_block_size;
base_block_t *block = (base_block_t *)base_map(tsdn, extent_hooks, ind,
base_block_t *block = (base_block_t *)base_map(tsdn, ehooks, ind,
block_size);
if (block == NULL) {
return NULL;
@ -311,13 +303,13 @@ static extent_t *
base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
malloc_mutex_assert_owner(tsdn, &base->mtx);
extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
ehooks_t *ehooks = base_ehooks_get(base);
/*
* Drop mutex during base_block_alloc(), because an extent hook will be
* called.
*/
malloc_mutex_unlock(tsdn, &base->mtx);
base_block_t *block = base_block_alloc(tsdn, base, extent_hooks,
base_block_t *block = base_block_alloc(tsdn, base, ehooks,
base_ind_get(base), &base->pind_last, &base->extent_sn_next, size,
alignment);
malloc_mutex_lock(tsdn, &base->mtx);
@ -353,7 +345,16 @@ base_t *
base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
pszind_t pind_last = 0;
size_t extent_sn_next = 0;
base_block_t *block = base_block_alloc(tsdn, NULL, extent_hooks, ind,
/*
* The base will contain the ehooks eventually, but it itself is
* allocated using them. So we use some stack ehooks to bootstrap its
* memory, and then initialize the ehooks within the base_t.
*/
ehooks_t fake_ehooks;
ehooks_init(&fake_ehooks, extent_hooks);
base_block_t *block = base_block_alloc(tsdn, NULL, &fake_ehooks, ind,
&pind_last, &extent_sn_next, sizeof(base_t), QUANTUM);
if (block == NULL) {
return NULL;
@ -365,10 +366,10 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
base_t *base = (base_t *)base_extent_bump_alloc_helper(&block->extent,
&gap_size, base_size, base_alignment);
base->ind = ind;
atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELAXED);
ehooks_init(&base->ehooks, extent_hooks);
if (malloc_mutex_init(&base->mtx, "base", WITNESS_RANK_BASE,
malloc_mutex_rank_exclusive)) {
base_unmap(tsdn, extent_hooks, ind, block, block->size);
base_unmap(tsdn, &fake_ehooks, ind, block, block->size);
return NULL;
}
base->pind_last = pind_last;
@ -397,26 +398,26 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
void
base_delete(tsdn_t *tsdn, base_t *base) {
extent_hooks_t *extent_hooks = base_extent_hooks_get(base);
ehooks_t *ehooks = base_ehooks_get(base);
base_block_t *next = base->blocks;
do {
base_block_t *block = next;
next = block->next;
base_unmap(tsdn, extent_hooks, base_ind_get(base), block,
base_unmap(tsdn, ehooks, base_ind_get(base), block,
block->size);
} while (next != NULL);
}
extent_hooks_t *
base_extent_hooks_get(base_t *base) {
return (extent_hooks_t *)atomic_load_p(&base->extent_hooks,
ATOMIC_ACQUIRE);
ehooks_t *
base_ehooks_get(base_t *base) {
return &base->ehooks;
}
extent_hooks_t *
base_extent_hooks_set(base_t *base, extent_hooks_t *extent_hooks) {
extent_hooks_t *old_extent_hooks = base_extent_hooks_get(base);
atomic_store_p(&base->extent_hooks, extent_hooks, ATOMIC_RELEASE);
extent_hooks_t *old_extent_hooks =
ehooks_get_extent_hooks_ptr(&base->ehooks);
ehooks_init(&base->ehooks, extent_hooks);
return old_extent_hooks;
}

5
src/ctl.c
View File

@ -2400,8 +2400,9 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
arena, new_extent_hooks);
READ(old_extent_hooks, extent_hooks_t *);
} else {
old_extent_hooks = arena_get_extent_hooks(
arena);
old_extent_hooks =
ehooks_get_extent_hooks_ptr(
arena_get_ehooks(arena));
READ(old_extent_hooks, extent_hooks_t *);
}
}

5
src/ehooks.c
View File

@ -1,3 +1,8 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/ehooks.h"
void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
ehooks_set_extent_hooks_ptr(ehooks, extent_hooks);
}

471
src/extent.c
View File

@ -28,40 +28,38 @@ static void extent_destroy_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, bool committed, unsigned arena_ind);
static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length, bool growing_retained);
static bool extent_decommit_default(extent_hooks_t *extent_hooks,
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length, bool growing_retained);
static bool extent_decommit_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
#ifdef PAGES_CAN_PURGE_LAZY
static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
#endif
static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length, bool growing_retained);
ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
bool growing_retained);
#ifdef PAGES_CAN_PURGE_FORCED
static bool extent_purge_forced_default(extent_hooks_t *extent_hooks,
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
#endif
static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length, bool growing_retained);
ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
bool growing_retained);
static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t size_a, size_t size_b, bool committed,
unsigned arena_ind);
static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
ehooks_t *ehooks, extent_t *extent, size_t size_a, szind_t szind_a,
bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
bool growing_retained);
static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
size_t size_a, void *addr_b, size_t size_b, bool committed,
unsigned arena_ind);
static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b,
bool growing_retained);
static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *a, extent_t *b, bool growing_retained);
const extent_hooks_t extent_hooks_default = {
const extent_hooks_t extent_hooks_default = {
extent_alloc_default,
extent_dalloc_default,
extent_destroy_default,
@ -97,16 +95,14 @@ static atomic_zu_t highpages;
*/
static void extent_deregister(tsdn_t *tsdn, extent_t *extent);
static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, eset_t *eset, void *new_addr,
size_t usize, size_t pad, size_t alignment, bool slab, szind_t szind,
bool *zero, bool *commit, bool growing_retained);
static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained);
static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset,
extent_t *extent, bool *coalesced, bool growing_retained);
static void extent_record(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, eset_t *eset, extent_t *extent,
bool growing_retained);
ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent,
bool *coalesced, bool growing_retained);
static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, extent_t *extent, bool growing_retained);
/******************************************************************************/
@ -224,13 +220,12 @@ extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp)
static bool
extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset,
extent_t *extent) {
extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent) {
extent_state_set(extent, extent_state_active);
bool coalesced;
extent = extent_try_coalesce(tsdn, arena, extent_hooks, rtree_ctx,
eset, extent, &coalesced, false);
extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset,
extent, &coalesced, false);
extent_state_set(extent, eset_state_get(eset));
if (!coalesced) {
@ -241,23 +236,23 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena,
}
extent_t *
extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
eset_t *eset, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
assert(size + pad != 0);
assert(alignment != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_t *extent = extent_recycle(tsdn, arena, extent_hooks, eset,
new_addr, size, pad, alignment, slab, szind, zero, commit, false);
extent_t *extent = extent_recycle(tsdn, arena, ehooks, eset, new_addr,
size, pad, alignment, slab, szind, zero, commit, false);
assert(extent == NULL || extent_dumpable_get(extent));
return extent;
}
void
extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
eset_t *eset, extent_t *extent) {
extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extent_t *extent) {
assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0);
assert(extent_dumpable_get(extent));
@ -267,12 +262,12 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false);
extent_record(tsdn, arena, extent_hooks, eset, extent, false);
extent_record(tsdn, arena, ehooks, eset, extent, false);
}
extent_t *
extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
eset_t *eset, size_t npages_min) {
extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
size_t npages_min) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@ -301,8 +296,8 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
break;
}
/* Try to coalesce. */
if (extent_try_delayed_coalesce(tsdn, arena, extent_hooks,
rtree_ctx, eset, extent)) {
if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx,
eset, extent)) {
break;
}
/*
@ -339,8 +334,8 @@ label_return:
* indicates OOM), e.g. when trying to split an existing extent.
*/
static void
extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
eset_t *eset, extent_t *extent, bool growing_retained) {
extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extent_t *extent, bool growing_retained) {
size_t sz = extent_size_get(extent);
if (config_stats) {
arena_stats_accum_zu(&arena->stats.abandoned_vm, sz);
@ -350,11 +345,10 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
* that this is only a virtual memory leak.
*/
if (eset_state_get(eset) == extent_state_dirty) {
if (extent_purge_lazy_impl(tsdn, arena, extent_hooks,
extent, 0, sz, growing_retained)) {
extent_purge_forced_impl(tsdn, arena, extent_hooks,
extent, 0, extent_size_get(extent),
growing_retained);
if (extent_purge_lazy_impl(tsdn, arena, ehooks, extent, 0, sz,
growing_retained)) {
extent_purge_forced_impl(tsdn, arena, ehooks, extent, 0,
extent_size_get(extent), growing_retained);
}
}
extent_dalloc(tsdn, arena, extent);
@ -581,10 +575,9 @@ extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) {
* given allocation request.
*/
static extent_t *
extent_recycle_extract(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
bool growing_retained) {
extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
size_t pad, size_t alignment, bool slab, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
assert(alignment > 0);
@ -668,8 +661,8 @@ typedef enum {
} extent_split_interior_result_t;
static extent_split_interior_result_t
extent_split_interior(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx,
extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx,
/* The result of splitting, in case of success. */
extent_t **extent, extent_t **lead, extent_t **trail,
/* The mess to clean up, in case of error. */
@ -693,9 +686,9 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena,
/* Split the lead. */
if (leadsize != 0) {
*lead = *extent;
*extent = extent_split_impl(tsdn, arena, extent_hooks,
*lead, leadsize, SC_NSIZES, false, esize + trailsize, szind,
slab, growing_retained);
*extent = extent_split_impl(tsdn, arena, ehooks, *lead,
leadsize, SC_NSIZES, false, esize + trailsize, szind, slab,
growing_retained);
if (*extent == NULL) {
*to_leak = *lead;
*lead = NULL;
@ -705,9 +698,8 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena,
/* Split the trail. */
if (trailsize != 0) {
*trail = extent_split_impl(tsdn, arena, extent_hooks, *extent,
esize, szind, slab, trailsize, SC_NSIZES, false,
growing_retained);
*trail = extent_split_impl(tsdn, arena, ehooks, *extent, esize,
szind, slab, trailsize, SC_NSIZES, false, growing_retained);
if (*trail == NULL) {
*to_leak = *extent;
*to_salvage = *lead;
@ -745,18 +737,18 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena,
* and put back into eset.
*/
static extent_t *
extent_recycle_split(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, extent_t *extent, bool growing_retained) {
extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
size_t pad, size_t alignment, bool slab, szind_t szind, extent_t *extent,
bool growing_retained) {
extent_t *lead;
extent_t *trail;
extent_t *to_leak;
extent_t *to_salvage;
extent_split_interior_result_t result = extent_split_interior(
tsdn, arena, extent_hooks, rtree_ctx, &extent, &lead, &trail,
&to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind,
tsdn, arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
&to_salvage, new_addr, size, pad, alignment, slab, szind,
growing_retained);
if (!maps_coalesce && result != extent_split_interior_ok
@ -790,8 +782,8 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena,
if (to_leak != NULL) {
void *leak = extent_base_get(to_leak);
extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, extent_hooks, eset,
to_leak, growing_retained);
extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak,
growing_retained);
assert(extent_lock_from_addr(tsdn, rtree_ctx, leak,
false) == NULL);
}
@ -807,7 +799,7 @@ extent_need_manual_zero(arena_t *arena) {
* default extent hooks installed (in which case the purge semantics may
* change); or 2) transparent huge pages enabled.
*/
return (!arena_has_default_hooks(arena) ||
return (!ehooks_are_default(arena_get_ehooks(arena)) ||
(opt_thp == thp_mode_always));
}
@ -816,10 +808,9 @@ extent_need_manual_zero(arena_t *arena) {
* in the given eset_t.
*/
static extent_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
eset_t *eset, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit,
bool growing_retained) {
extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
assert(new_addr == NULL || !slab);
@ -829,25 +820,25 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
extent_t *extent = extent_recycle_extract(tsdn, arena, extent_hooks,
extent_t *extent = extent_recycle_extract(tsdn, arena, ehooks,
rtree_ctx, eset, new_addr, size, pad, alignment, slab,
growing_retained);
if (extent == NULL) {
return NULL;
}
extent = extent_recycle_split(tsdn, arena, extent_hooks, rtree_ctx,
eset, new_addr, size, pad, alignment, slab, szind, extent,
extent = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset,
new_addr, size, pad, alignment, slab, szind, extent,
growing_retained);
if (extent == NULL) {
return NULL;
}
if (*commit && !extent_committed_get(extent)) {
if (extent_commit_impl(tsdn, arena, extent_hooks, extent,
0, extent_size_get(extent), growing_retained)) {
extent_record(tsdn, arena, extent_hooks, eset,
extent, growing_retained);
if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
extent_size_get(extent), growing_retained)) {
extent_record(tsdn, arena, ehooks, eset, extent,
growing_retained);
return NULL;
}
if (!extent_need_manual_zero(arena)) {
@ -985,9 +976,9 @@ extent_hook_post_reentrancy(tsdn_t *tsdn) {
* virtual memory ranges retained by each arena.
*/
static extent_t *
extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, size_t size, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit) {
extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
bool *zero, bool *commit) {
malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);
assert(pad == 0 || !slab);
assert(!*zero || !slab);
@ -1022,14 +1013,13 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
bool committed = false;
void *ptr;
if (extent_hooks == &extent_hooks_default) {
if (ehooks_are_default(ehooks)) {
ptr = extent_alloc_default_impl(tsdn, arena, NULL,
alloc_size, PAGE, &zeroed, &committed);
} else {
extent_hook_pre_reentrancy(tsdn, arena);
ptr = extent_hooks->alloc(extent_hooks, NULL,
alloc_size, PAGE, &zeroed, &committed,
arena_ind_get(arena));
ptr = ehooks_alloc(ehooks, NULL, alloc_size, PAGE, &zeroed,
&committed, arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);
}
@ -1060,18 +1050,17 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
extent_t *trail;
extent_t *to_leak;
extent_t *to_salvage;
extent_split_interior_result_t result = extent_split_interior(
tsdn, arena, extent_hooks, rtree_ctx, &extent, &lead, &trail,
&to_leak, &to_salvage, NULL, size, pad, alignment, slab, szind,
true);
extent_split_interior_result_t result = extent_split_interior(tsdn,
arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
&to_salvage, NULL, size, pad, alignment, slab, szind, true);
if (result == extent_split_interior_ok) {
if (lead != NULL) {
extent_record(tsdn, arena, extent_hooks,
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, lead, true);
}
if (trail != NULL) {
extent_record(tsdn, arena, extent_hooks,
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, trail, true);
}
} else {
@ -1084,21 +1073,21 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
if (config_prof) {
extent_gdump_add(tsdn, to_salvage);
}
extent_record(tsdn, arena, extent_hooks,
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, to_salvage, true);
}
if (to_leak != NULL) {
extent_deregister_no_gdump_sub(tsdn, to_leak);
extents_abandon_vm(tsdn, arena, extent_hooks,
extents_abandon_vm(tsdn, arena, ehooks,
&arena->eset_retained, to_leak, true);
}
goto label_err;
}
if (*commit && !extent_committed_get(extent)) {
if (extent_commit_impl(tsdn, arena, extent_hooks, extent, 0,
if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
extent_size_get(extent), true)) {
extent_record(tsdn, arena, extent_hooks,
extent_record(tsdn, arena, ehooks,
&arena->eset_retained, extent, true);
goto label_err;
}
@ -1151,15 +1140,15 @@ label_err:
}
static extent_t *
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
assert(size != 0);
assert(alignment != 0);
malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);
extent_t *extent = extent_recycle(tsdn, arena, extent_hooks,
extent_t *extent = extent_recycle(tsdn, arena, ehooks,
&arena->eset_retained, new_addr, size, pad, alignment, slab,
szind, zero, commit, true);
if (extent != NULL) {
@ -1168,8 +1157,8 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
extent_gdump_add(tsdn, extent);
}
} else if (opt_retain && new_addr == NULL) {
extent = extent_grow_retained(tsdn, arena, extent_hooks, size,
pad, alignment, slab, szind, zero, commit);
extent = extent_grow_retained(tsdn, arena, ehooks, size, pad,
alignment, slab, szind, zero, commit);
/* extent_grow_retained() always releases extent_grow_mtx. */
} else {
malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
@ -1180,9 +1169,9 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
}
static extent_t *
extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
size_t esize = size + pad;
extent_t *extent = extent_alloc(tsdn, arena);
if (extent == NULL) {
@ -1190,14 +1179,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
}
void *addr;
size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
if (extent_hooks == &extent_hooks_default) {
if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */
addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize,
palignment, zero, commit);
} else {
extent_hook_pre_reentrancy(tsdn, arena);
addr = extent_hooks->alloc(extent_hooks, new_addr,
esize, palignment, zero, commit, arena_ind_get(arena));
addr = ehooks_alloc(ehooks, new_addr, esize, palignment, zero,
commit, arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);
}
if (addr == NULL) {
@ -1219,14 +1208,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
}
extent_t *
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_t *extent = extent_alloc_retained(tsdn, arena, extent_hooks,
new_addr, size, pad, alignment, slab, szind, zero, commit);
extent_t *extent = extent_alloc_retained(tsdn, arena, ehooks, new_addr,
size, pad, alignment, slab, szind, zero, commit);
if (extent == NULL) {
if (opt_retain && new_addr != NULL) {
/*
@ -1237,7 +1226,7 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
*/
return NULL;
}
extent = extent_alloc_wrapper_hard(tsdn, arena, extent_hooks,
extent = extent_alloc_wrapper_hard(tsdn, arena, ehooks,
new_addr, size, pad, alignment, slab, szind, zero, commit);
}
@ -1266,15 +1255,14 @@ extent_can_coalesce(arena_t *arena, eset_t *eset, const extent_t *inner,
}
static bool
extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
eset_t *eset, extent_t *inner, extent_t *outer, bool forward,
bool growing_retained) {
extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extent_t *inner, extent_t *outer, bool forward, bool growing_retained) {
assert(extent_can_coalesce(arena, eset, inner, outer));
extent_activate_locked(tsdn, arena, eset, outer);
malloc_mutex_unlock(tsdn, &eset->mtx);
bool err = extent_merge_impl(tsdn, arena, extent_hooks,
bool err = extent_merge_impl(tsdn, arena, ehooks,
forward ? inner : outer, forward ? outer : inner, growing_retained);
malloc_mutex_lock(tsdn, &eset->mtx);
@ -1286,10 +1274,9 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
}
static extent_t *
extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset,
extent_t *extent, bool *coalesced, bool growing_retained,
bool inactive_only) {
extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
bool growing_retained, bool inactive_only) {
/*
* We avoid checking / locking inactive neighbors for large size
* classes, since they are eagerly coalesced on deallocation which can
@ -1318,7 +1305,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,
extent_unlock(tsdn, next);
if (can_coalesce && !extent_coalesce(tsdn, arena,
extent_hooks, eset, extent, next, true,
ehooks, eset, extent, next, true,
growing_retained)) {
if (eset->delay_coalesce) {
/* Do minimal coalescing. */
@ -1338,7 +1325,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,
extent_unlock(tsdn, prev);
if (can_coalesce && !extent_coalesce(tsdn, arena,
extent_hooks, eset, extent, prev, false,
ehooks, eset, extent, prev, false,
growing_retained)) {
extent = prev;
if (eset->delay_coalesce) {
@ -1358,19 +1345,19 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena,
}
static extent_t *
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset,
extent_t *extent, bool *coalesced, bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, extent_hooks, rtree_ctx,
eset, extent, coalesced, growing_retained, false);
extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
extent, coalesced, growing_retained, false);
}
static extent_t *
extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, rtree_ctx_t *rtree_ctx, eset_t *eset,
extent_t *extent, bool *coalesced, bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, extent_hooks, rtree_ctx,
eset, extent, coalesced, growing_retained, true);
extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
bool growing_retained) {
return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
extent, coalesced, growing_retained, true);
}
/*
@ -1378,8 +1365,8 @@ extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena,
* given eset_t (coalesces, deregisters slab interiors, the heap operations).
*/
static void
extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
eset_t *eset, extent_t *extent, bool growing_retained) {
extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
extent_t *extent, bool growing_retained) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
@ -1399,22 +1386,22 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
(uintptr_t)extent_base_get(extent), true) == extent);
if (!eset->delay_coalesce) {
extent = extent_try_coalesce(tsdn, arena, extent_hooks,
rtree_ctx, eset, extent, NULL, growing_retained);
extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx,
eset, extent, NULL, growing_retained);
} else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) {
assert(eset == &arena->eset_dirty);
/* Always coalesce large eset eagerly. */
bool coalesced;
do {
assert(extent_state_get(extent) == extent_state_active);
extent = extent_try_coalesce_large(tsdn, arena,
extent_hooks, rtree_ctx, eset, extent,
&coalesced, growing_retained);
extent = extent_try_coalesce_large(tsdn, arena, ehooks,
rtree_ctx, eset, extent, &coalesced,
growing_retained);
} while (coalesced);
if (extent_size_get(extent) >= oversize_threshold) {
/* Shortcut to purge the oversize extent eagerly. */
malloc_mutex_unlock(tsdn, &eset->mtx);
arena_decay_extent(tsdn, arena, extent_hooks, extent);
arena_decay_extent(tsdn, arena, ehooks, extent);
return;
}
}
@ -1425,7 +1412,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks,
void
extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
ehooks_t *ehooks = arena_get_ehooks(arena);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
@ -1434,7 +1421,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_dalloc(tsdn, arena, extent);
return;
}
extent_dalloc_wrapper(tsdn, arena, extent_hooks, extent);
extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
}
static bool
@ -1458,8 +1445,8 @@ extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
}
static bool
extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent) {
extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent) {
bool err;
assert(extent_base_get(extent) != NULL);
@ -1470,16 +1457,15 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
extent_addr_set(extent, extent_base_get(extent));
/* Try to deallocate. */
if (extent_hooks == &extent_hooks_default) {
if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */
err = extent_dalloc_default_impl(extent_base_get(extent),
extent_size_get(extent));
} else {
extent_hook_pre_reentrancy(tsdn, arena);
err = (extent_hooks->dalloc == NULL ||
extent_hooks->dalloc(extent_hooks,
extent_base_get(extent), extent_size_get(extent),
extent_committed_get(extent), arena_ind_get(arena)));
err = ehooks_dalloc(ehooks, extent_base_get(extent),
extent_size_get(extent), extent_committed_get(extent),
arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);
}
@ -1491,51 +1477,48 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
}
void
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent) {
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent) {
assert(extent_dumpable_get(extent));
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
/* Avoid calling the default extent_dalloc unless have to. */
if (extent_hooks != &extent_hooks_default || extent_may_dalloc()) {
if (!ehooks_are_default(ehooks) || extent_may_dalloc()) {
/*
* Deregister first to avoid a race with other allocating
* threads, and reregister if deallocation fails.
*/
extent_deregister(tsdn, extent);
if (!extent_dalloc_wrapper_try(tsdn, arena, extent_hooks,
extent)) {
if (!extent_dalloc_wrapper_try(tsdn, arena, ehooks, extent)) {
return;
}
extent_reregister(tsdn, extent);
}
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena);
}
/* Try to decommit; purge if that fails. */
bool zeroed;
if (!extent_committed_get(extent)) {
zeroed = true;
} else if (!extent_decommit_wrapper(tsdn, arena, extent_hooks, extent,
0, extent_size_get(extent))) {
} else if (!extent_decommit_wrapper(tsdn, arena, ehooks, extent, 0,
extent_size_get(extent))) {
zeroed = true;
} else if (extent_hooks->purge_forced != NULL &&
!extent_hooks->purge_forced(extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena_ind_get(arena))) {
} else if (!ehooks_purge_forced(ehooks, extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent),
arena_ind_get(arena))) {
zeroed = true;
} else if (extent_state_get(extent) == extent_state_muzzy ||
(extent_hooks->purge_lazy != NULL &&
!extent_hooks->purge_lazy(extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena_ind_get(arena)))) {
!ehooks_purge_lazy(ehooks, extent_base_get(extent),
extent_size_get(extent), 0, extent_size_get(extent),
arena_ind_get(arena))) {
zeroed = false;
} else {
zeroed = false;
}
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn);
}
extent_zeroed_set(extent, zeroed);
@ -1544,8 +1527,8 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_gdump_sub(tsdn, extent);
}
extent_record(tsdn, arena, extent_hooks, &arena->eset_retained,
extent, false);
extent_record(tsdn, arena, ehooks, &arena->eset_retained, extent,
false);
}
static void
@ -1562,8 +1545,8 @@ extent_destroy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
}
void
extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent) {
extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent) {
assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
@ -1575,15 +1558,15 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_addr_set(extent, extent_base_get(extent));
/* Try to destroy; silently fail otherwise. */
if (extent_hooks == &extent_hooks_default) {
if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */
extent_destroy_default_impl(extent_base_get(extent),
extent_size_get(extent));
} else if (extent_hooks->destroy != NULL) {
} else if (!ehooks_destroy_is_noop(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena);
extent_hooks->destroy(extent_hooks,
extent_base_get(extent), extent_size_get(extent),
extent_committed_get(extent), arena_ind_get(arena));
ehooks_destroy(ehooks, extent_base_get(extent),
extent_size_get(extent), extent_committed_get(extent),
arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);
}
@ -1598,19 +1581,17 @@ extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
}
static bool
extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length, bool growing_retained) {
extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena);
}
bool err = (extent_hooks->commit == NULL ||
extent_hooks->commit(extent_hooks, extent_base_get(extent),
extent_size_get(extent), offset, length, arena_ind_get(arena)));
if (extent_hooks != &extent_hooks_default) {
bool err = ehooks_commit(ehooks, extent_base_get(extent),
extent_size_get(extent), offset, length, arena_ind_get(arena));
if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn);
}
extent_committed_set(extent, extent_committed_get(extent) || !err);
@ -1618,11 +1599,11 @@ extent_commit_impl(tsdn_t *tsdn, arena_t *arena,
}
bool
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset,
size_t length) {
return extent_commit_impl(tsdn, arena, extent_hooks, extent, offset,
length, false);
return extent_commit_impl(tsdn, arena, ehooks, extent, offset, length,
false);
}
static bool
@ -1633,20 +1614,17 @@ extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
}
bool
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length) {
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena);
}
bool err = (extent_hooks->decommit == NULL ||
extent_hooks->decommit(extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena_ind_get(arena)));
if (extent_hooks != &extent_hooks_default) {
bool err = ehooks_decommit(ehooks, extent_base_get(extent),
extent_size_get(extent), offset, length, arena_ind_get(arena));
if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn);
}
extent_committed_set(extent, extent_committed_get(extent) && err);
@ -1668,22 +1646,20 @@ extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
#endif
static bool
extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length, bool growing_retained) {
extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks->purge_lazy == NULL) {
if (ehooks_purge_lazy_will_fail(ehooks)) {
return true;
}
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena);
}
bool err = extent_hooks->purge_lazy(extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena_ind_get(arena));
if (extent_hooks != &extent_hooks_default) {
bool err = ehooks_purge_lazy(ehooks, extent_base_get(extent),
extent_size_get(extent), offset, length, arena_ind_get(arena));
if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn);
}
@ -1691,11 +1667,10 @@ extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
}
bool
extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length) {
return extent_purge_lazy_impl(tsdn, arena, extent_hooks, extent,
offset, length, false);
extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length) {
return extent_purge_lazy_impl(tsdn, arena, ehooks, extent, offset,
length, false);
}
#ifdef PAGES_CAN_PURGE_FORCED
@ -1713,32 +1688,29 @@ extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
#endif
static bool
extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length, bool growing_retained) {
extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks->purge_forced == NULL) {
if (ehooks_purge_forced_will_fail(ehooks)) {
return true;
}
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena);
}
bool err = extent_hooks->purge_forced(extent_hooks,
extent_base_get(extent), extent_size_get(extent), offset, length,
arena_ind_get(arena));
if (extent_hooks != &extent_hooks_default) {
bool err = ehooks_purge_forced(ehooks, extent_base_get(extent),
extent_size_get(extent), offset, length, arena_ind_get(arena));
if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn);
}
return err;
}
bool
extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t offset,
size_t length) {
return extent_purge_forced_impl(tsdn, arena, extent_hooks, extent,
extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t offset, size_t length) {
return extent_purge_forced_impl(tsdn, arena, ehooks, extent,
offset, length, false);
}
@ -1765,15 +1737,14 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
* and returns the trail (except in case of error).
*/
static extent_t *
extent_split_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
bool growing_retained) {
extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained) {
assert(extent_size_get(extent) == size_a + size_b);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
if (extent_hooks->split == NULL) {
if (ehooks_split_will_fail(ehooks)) {
return NULL;
}
@ -1815,13 +1786,13 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena,
extent_lock2(tsdn, extent, trail);
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_pre_reentrancy(tsdn, arena);
}
bool err = extent_hooks->split(extent_hooks, extent_base_get(extent),
bool err = ehooks_split(ehooks, extent_base_get(extent),
size_a + size_b, size_a, size_b, extent_committed_get(extent),
arena_ind_get(arena));
if (extent_hooks != &extent_hooks_default) {
if (!ehooks_are_default(ehooks)) {
extent_hook_post_reentrancy(tsdn);
}
if (err) {
@ -1848,11 +1819,11 @@ label_error_a:
}
extent_t *
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) {
return extent_split_impl(tsdn, arena, extent_hooks, extent, size_a,
szind_a, slab_a, size_b, szind_b, slab_b, false);
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
size_t size_b, szind_t szind_b, bool slab_b) {
return extent_split_impl(tsdn, arena, ehooks, extent, size_a, szind_a,
slab_a, size_b, szind_b, slab_b, false);
}
static bool
@ -1914,28 +1885,26 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
}
static bool
extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b,
bool growing_retained) {
extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a,
extent_t *b, bool growing_retained) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
assert(extent_base_get(a) < extent_base_get(b));
if (extent_hooks->merge == NULL || extent_head_no_merge(a, b)) {
if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) {
return true;
}
bool err;
if (extent_hooks == &extent_hooks_default) {
if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */
err = extent_merge_default_impl(extent_base_get(a),
extent_base_get(b));
} else {
extent_hook_pre_reentrancy(tsdn, arena);
err = extent_hooks->merge(extent_hooks,
extent_base_get(a), extent_size_get(a), extent_base_get(b),
extent_size_get(b), extent_committed_get(a),
arena_ind_get(arena));
err = ehooks_merge(ehooks, extent_base_get(a),
extent_size_get(a), extent_base_get(b), extent_size_get(b),
extent_committed_get(a), arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);
}
@ -1991,9 +1960,9 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena,
}
bool
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t *extent_hooks, extent_t *a, extent_t *b) {
return extent_merge_impl(tsdn, arena, extent_hooks, a, b, false);
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *a, extent_t *b) {
return extent_merge_impl(tsdn, arena, ehooks, a, b, false);
}
bool

7
src/extent_dss.c
View File

@ -194,9 +194,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
*commit = pages_decommit(ret, size);
}
if (*zero && *commit) {
extent_hooks_t *extent_hooks =
arena_get_extent_hooks(arena);
extent_t extent;
ehooks_t *ehooks = arena_get_ehooks(
arena);
extent_init(&extent,
arena_ind_get(arena), ret, size,
@ -204,8 +204,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
extent_state_active, false, true,
true, EXTENT_NOT_HEAD);
if (extent_purge_forced_wrapper(tsdn,
arena, extent_hooks, &extent, 0,
size)) {
arena, ehooks, &extent, 0, size)) {
memset(ret, 0, size);
}
}

36
src/large.c
View File

@ -93,20 +93,20 @@ static bool
large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
arena_t *arena = arena_get_from_extent(extent);
size_t oldusize = extent_usize_get(extent);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
ehooks_t *ehooks = arena_get_ehooks(arena);
size_t diff = extent_size_get(extent) - (usize + sz_large_pad);
assert(oldusize > usize);
if (extent_hooks->split == NULL) {
if (ehooks_split_will_fail(ehooks)) {
return true;
}
/* Split excess pages. */
if (diff != 0) {
extent_t *trail = extent_split_wrapper(tsdn, arena,
extent_hooks, extent, usize + sz_large_pad,
sz_size2index(usize), false, diff, SC_NSIZES, false);
ehooks, extent, usize + sz_large_pad, sz_size2index(usize),
false, diff, SC_NSIZES, false);
if (trail == NULL) {
return true;
}
@ -116,7 +116,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
extent_size_get(trail));
}
arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, trail);
arena_extents_dirty_dalloc(tsdn, arena, ehooks, trail);
}
arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize);
@ -129,10 +129,10 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool zero) {
arena_t *arena = arena_get_from_extent(extent);
size_t oldusize = extent_usize_get(extent);
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
ehooks_t *ehooks = arena_get_ehooks(arena);
size_t trailsize = usize - oldusize;
if (extent_hooks->merge == NULL) {
if (ehooks_merge_will_fail(ehooks)) {
return true;
}
@ -149,17 +149,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool commit = true;
extent_t *trail;
bool new_mapping;
if ((trail = extents_alloc(tsdn, arena, extent_hooks,
&arena->eset_dirty, extent_past_get(extent), trailsize, 0,
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL
|| (trail = extents_alloc(tsdn, arena, extent_hooks,
&arena->eset_muzzy, extent_past_get(extent), trailsize, 0,
CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) {
if ((trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_dirty,
extent_past_get(extent), trailsize, 0, CACHELINE, false, SC_NSIZES,
&is_zeroed_trail, &commit)) != NULL
|| (trail = extents_alloc(tsdn, arena, ehooks, &arena->eset_muzzy,
extent_past_get(extent), trailsize, 0, CACHELINE, false, SC_NSIZES,
&is_zeroed_trail, &commit)) != NULL) {
if (config_stats) {
new_mapping = false;
}
} else {
if ((trail = extent_alloc_wrapper(tsdn, arena, extent_hooks,
if ((trail = extent_alloc_wrapper(tsdn, arena, ehooks,
extent_past_get(extent), trailsize, 0, CACHELINE, false,
SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) {
return true;
@ -169,8 +169,8 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
}
}
if (extent_merge_wrapper(tsdn, arena, extent_hooks, extent, trail)) {
extent_dalloc_wrapper(tsdn, arena, extent_hooks, trail);
if (extent_merge_wrapper(tsdn, arena, ehooks, extent, trail)) {
extent_dalloc_wrapper(tsdn, arena, ehooks, trail);
return true;
}
rtree_ctx_t rtree_ctx_fallback;
@ -339,8 +339,8 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
static void
large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = arena_get_extent_hooks(arena);
arena_extents_dirty_dalloc(tsdn, arena, extent_hooks, extent);
ehooks_t *ehooks = arena_get_ehooks(arena);
arena_extents_dirty_dalloc(tsdn, arena, ehooks, extent);
}
void