add636596a
Now that all merging go through try_acquire_edata_neighbor, the mergeablility checks (including head state checking) are done before reaching the merge hook. In other words, merge hook will never be called if the head state doesn't agree.
372 lines
13 KiB
C
372 lines
13 KiB
C
#ifndef JEMALLOC_INTERNAL_EHOOKS_H
|
|
#define JEMALLOC_INTERNAL_EHOOKS_H
|
|
|
|
#include "jemalloc/internal/atomic.h"
|
|
#include "jemalloc/internal/extent_mmap.h"
|
|
|
|
/*
|
|
* This module is the internal interface to the extent hooks (both
|
|
* user-specified and external). Eventually, this will give us the flexibility
|
|
* to use multiple different versions of user-visible extent-hook APIs under a
|
|
* single user interface.
|
|
*
|
|
* Current API expansions (not available to anyone but the default hooks yet):
|
|
* - Head state tracking. Hooks can decide whether or not to merge two
|
|
* extents based on whether or not one of them is the head (i.e. was
|
|
* allocated on its own). The later extent loses its "head" status.
|
|
*/
|
|
|
|
extern const extent_hooks_t ehooks_default_extent_hooks;
|
|
|
|
typedef struct ehooks_s ehooks_t;
|
|
struct ehooks_s {
|
|
/*
|
|
* The user-visible id that goes with the ehooks (i.e. that of the base
|
|
* they're a part of, the associated arena's index within the arenas
|
|
* array).
|
|
*/
|
|
unsigned ind;
|
|
/* Logically an extent_hooks_t *. */
|
|
atomic_p_t ptr;
|
|
};
|
|
|
|
extern const extent_hooks_t ehooks_default_extent_hooks;
|
|
|
|
/*
|
|
* These are not really part of the public API. Each hook has a fast-path for
|
|
* the default-hooks case that can avoid various small inefficiencies:
|
|
* - Forgetting tsd and then calling tsd_get within the hook.
|
|
* - Getting more state than necessary out of the extent_t.
|
|
* - Doing arena_ind -> arena -> arena_ind lookups.
|
|
* By making the calls to these functions visible to the compiler, it can move
|
|
* those extra bits of computation down below the fast-paths where they get ignored.
|
|
*/
|
|
void *ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size,
|
|
size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
|
|
bool ehooks_default_dalloc_impl(void *addr, size_t size);
|
|
void ehooks_default_destroy_impl(void *addr, size_t size);
|
|
bool ehooks_default_commit_impl(void *addr, size_t offset, size_t length);
|
|
bool ehooks_default_decommit_impl(void *addr, size_t offset, size_t length);
|
|
#ifdef PAGES_CAN_PURGE_LAZY
|
|
bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
|
|
#endif
|
|
#ifdef PAGES_CAN_PURGE_FORCED
|
|
bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
|
|
#endif
|
|
bool ehooks_default_split_impl();
|
|
/*
|
|
* Merge is the only default extent hook we declare -- see the comment in
|
|
* ehooks_merge.
|
|
*/
|
|
bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a,
|
|
size_t size_a, void *addr_b, size_t size_b, bool committed,
|
|
unsigned arena_ind);
|
|
bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b);
|
|
void ehooks_default_zero_impl(void *addr, size_t size);
|
|
|
|
/*
|
|
* We don't officially support reentrancy from wtihin the extent hooks. But
|
|
* various people who sit within throwing distance of the jemalloc team want
|
|
* that functionality in certain limited cases. The default reentrancy guards
|
|
* assert that we're not reentrant from a0 (since it's the bootstrap arena,
|
|
* where reentrant allocations would be redirected), which we would incorrectly
|
|
* trigger in cases where a0 has extent hooks (those hooks themselves can't be
|
|
* reentrant, then, but there are reasonable uses for such functionality, like
|
|
* putting internal metadata on hugepages). Therefore, we use the raw
|
|
* reentrancy guards.
|
|
*
|
|
* Eventually, we need to think more carefully about whether and where we
|
|
* support allocating from within extent hooks (and what that means for things
|
|
* like profiling, stats collection, etc.), and document what the guarantee is.
|
|
*/
|
|
static inline void
|
|
ehooks_pre_reentrancy(tsdn_t *tsdn) {
|
|
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
|
|
tsd_pre_reentrancy_raw(tsd);
|
|
}
|
|
|
|
static inline void
|
|
ehooks_post_reentrancy(tsdn_t *tsdn) {
|
|
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
|
|
tsd_post_reentrancy_raw(tsd);
|
|
}
|
|
|
|
/* Beginning of the public API. */
|
|
void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks, unsigned ind);
|
|
|
|
static inline unsigned
|
|
ehooks_ind_get(const ehooks_t *ehooks) {
|
|
return ehooks->ind;
|
|
}
|
|
|
|
static inline void
|
|
ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
|
|
atomic_store_p(&ehooks->ptr, extent_hooks, ATOMIC_RELEASE);
|
|
}
|
|
|
|
static inline extent_hooks_t *
|
|
ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
|
|
return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_are_default(ehooks_t *ehooks) {
|
|
return ehooks_get_extent_hooks_ptr(ehooks) ==
|
|
&ehooks_default_extent_hooks;
|
|
}
|
|
|
|
/*
|
|
* In some cases, a caller needs to allocate resources before attempting to call
|
|
* a hook. If that hook is doomed to fail, this is wasteful. We therefore
|
|
* include some checks for such cases.
|
|
*/
|
|
static inline bool
|
|
ehooks_dalloc_will_fail(ehooks_t *ehooks) {
|
|
if (ehooks_are_default(ehooks)) {
|
|
return opt_retain;
|
|
} else {
|
|
return ehooks_get_extent_hooks_ptr(ehooks)->dalloc == NULL;
|
|
}
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_split_will_fail(ehooks_t *ehooks) {
|
|
return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_merge_will_fail(ehooks_t *ehooks) {
|
|
return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
|
|
}
|
|
|
|
/*
|
|
* Some hooks are required to return zeroed memory in certain situations. In
|
|
* debug mode, we do some heuristic checks that they did what they were supposed
|
|
* to.
|
|
*
|
|
* This isn't really ehooks-specific (i.e. anyone can check for zeroed memory).
|
|
* But incorrect zero information indicates an ehook bug.
|
|
*/
|
|
static inline void
|
|
ehooks_debug_zero_check(void *addr, size_t size) {
|
|
assert(((uintptr_t)addr & PAGE_MASK) == 0);
|
|
assert((size & PAGE_MASK) == 0);
|
|
assert(size > 0);
|
|
if (config_debug) {
|
|
/* Check the whole first page. */
|
|
size_t *p = (size_t *)addr;
|
|
for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {
|
|
assert(p[i] == 0);
|
|
}
|
|
/*
|
|
* And 4 spots within. There's a tradeoff here; the larger
|
|
* this number, the more likely it is that we'll catch a bug
|
|
* where ehooks return a sparsely non-zero range. But
|
|
* increasing the number of checks also increases the number of
|
|
* page faults in debug mode. FreeBSD does much of their
|
|
* day-to-day development work in debug mode, so we don't want
|
|
* even the debug builds to be too slow.
|
|
*/
|
|
const size_t nchecks = 4;
|
|
assert(PAGE >= sizeof(size_t) * nchecks);
|
|
for (size_t i = 0; i < nchecks; ++i) {
|
|
assert(p[i * (size / sizeof(size_t) / nchecks)] == 0);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
static inline void *
|
|
ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
|
|
size_t alignment, bool *zero, bool *commit) {
|
|
bool orig_zero = *zero;
|
|
void *ret;
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
ret = ehooks_default_alloc_impl(tsdn, new_addr, size,
|
|
alignment, zero, commit, ehooks_ind_get(ehooks));
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
ret = extent_hooks->alloc(extent_hooks, new_addr, size,
|
|
alignment, zero, commit, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
}
|
|
assert(new_addr == NULL || ret == NULL || new_addr == ret);
|
|
assert(!orig_zero || *zero);
|
|
if (*zero && ret != NULL) {
|
|
ehooks_debug_zero_check(ret, size);
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_dalloc(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|
bool committed) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
return ehooks_default_dalloc_impl(addr, size);
|
|
} else if (extent_hooks->dalloc == NULL) {
|
|
return true;
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
bool err = extent_hooks->dalloc(extent_hooks, addr, size,
|
|
committed, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
static inline void
|
|
ehooks_destroy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|
bool committed) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
ehooks_default_destroy_impl(addr, size);
|
|
} else if (extent_hooks->destroy == NULL) {
|
|
/* Do nothing. */
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
extent_hooks->destroy(extent_hooks, addr, size, committed,
|
|
ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
}
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_commit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|
size_t offset, size_t length) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
bool err;
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
err = ehooks_default_commit_impl(addr, offset, length);
|
|
} else if (extent_hooks->commit == NULL) {
|
|
err = true;
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
err = extent_hooks->commit(extent_hooks, addr, size,
|
|
offset, length, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
}
|
|
if (!err) {
|
|
ehooks_debug_zero_check(addr, size);
|
|
}
|
|
return err;
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_decommit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|
size_t offset, size_t length) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
return ehooks_default_decommit_impl(addr, offset, length);
|
|
} else if (extent_hooks->decommit == NULL) {
|
|
return true;
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
bool err = extent_hooks->decommit(extent_hooks, addr, size,
|
|
offset, length, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_purge_lazy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|
size_t offset, size_t length) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
#ifdef PAGES_CAN_PURGE_LAZY
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
return ehooks_default_purge_lazy_impl(addr, offset, length);
|
|
}
|
|
#endif
|
|
if (extent_hooks->purge_lazy == NULL) {
|
|
return true;
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
bool err = extent_hooks->purge_lazy(extent_hooks, addr, size,
|
|
offset, length, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_purge_forced(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|
size_t offset, size_t length) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
/*
|
|
* It would be correct to have a ehooks_debug_zero_check call at the end
|
|
* of this function; purge_forced is required to zero. But checking
|
|
* would touch the page in question, which may have performance
|
|
* consequences (imagine the hooks are using hugepages, with a global
|
|
* zero page off). Even in debug mode, it's usually a good idea to
|
|
* avoid cases that can dramatically increase memory consumption.
|
|
*/
|
|
#ifdef PAGES_CAN_PURGE_FORCED
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
return ehooks_default_purge_forced_impl(addr, offset, length);
|
|
}
|
|
#endif
|
|
if (extent_hooks->purge_forced == NULL) {
|
|
return true;
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
bool err = extent_hooks->purge_forced(extent_hooks, addr, size,
|
|
offset, length, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_split(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|
size_t size_a, size_t size_b, bool committed) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
if (ehooks_are_default(ehooks)) {
|
|
return ehooks_default_split_impl();
|
|
} else if (extent_hooks->split == NULL) {
|
|
return true;
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
bool err = extent_hooks->split(extent_hooks, addr, size, size_a,
|
|
size_b, committed, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
static inline bool
|
|
ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a,
|
|
void *addr_b, size_t size_b, bool committed) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
return ehooks_default_merge_impl(tsdn, addr_a, addr_b);
|
|
} else if (extent_hooks->merge == NULL) {
|
|
return true;
|
|
} else {
|
|
ehooks_pre_reentrancy(tsdn);
|
|
bool err = extent_hooks->merge(extent_hooks, addr_a, size_a,
|
|
addr_b, size_b, committed, ehooks_ind_get(ehooks));
|
|
ehooks_post_reentrancy(tsdn);
|
|
return err;
|
|
}
|
|
}
|
|
|
|
static inline void
|
|
ehooks_zero(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size) {
|
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
|
ehooks_default_zero_impl(addr, size);
|
|
} else {
|
|
/*
|
|
* It would be correct to try using the user-provided purge
|
|
* hooks (since they are required to have zeroed the extent if
|
|
* they indicate success), but we don't necessarily know their
|
|
* cost. We'll be conservative and use memset.
|
|
*/
|
|
memset(addr, 0, size);
|
|
}
|
|
}
|
|
|
|
#endif /* JEMALLOC_INTERNAL_EHOOKS_H */
|