project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extent -> Ehooks: Move alloc hook.

Browse Source
This commit is contained in:
David Goldblatt 2019-12-02 16:42:44 -08:00 committed by David Goldblatt
parent 703fbc0ff5
commit dc8b4e6e13
4 changed files with 105 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
include/jemalloc/internal/ehooks.h
View File

@ -11,6 +11,26 @@ struct ehooks_s {
atomic_p_t ptr;
};
/* NOT PUBLIC. */
void *ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind);
void *ehooks_default_alloc(extent_hooks_t *extent_hooks, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit,
unsigned arena_ind);
static inline void
ehooks_pre_reentrancy(tsdn_t *tsdn) {
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
tsd_pre_reentrancy_raw(tsd);
}
static inline void
ehooks_post_reentrancy(tsdn_t *tsdn) {
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
tsd_post_reentrancy_raw(tsd);
}
/* PUBLIC. */
void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks);
static inline void
@ -54,11 +74,18 @@ ehooks_merge_will_fail(ehooks_t *ehooks) {
}
static inline void *
ehooks_alloc(ehooks_t *ehooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool *commit, unsigned arena_ind) {
ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
return extent_hooks->alloc(extent_hooks, new_addr, size, alignment,
if (extent_hooks == &extent_hooks_default) {
return ehooks_default_alloc_impl(tsdn, new_addr, size,
alignment, zero, commit, arena_ind);
}
ehooks_pre_reentrancy(tsdn);
void *ret = extent_hooks->alloc(extent_hooks, new_addr, size, alignment,
zero, commit, arena_ind);
ehooks_post_reentrancy(tsdn);
return ret;
}
static inline bool

6
src/base.c
View File

@ -43,12 +43,8 @@ base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) {
pages_set_thp_state(addr, size);
}
} else {
/* No arena context as we are creating new arenas. */
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
pre_reentrancy(tsd, NULL);
addr = ehooks_alloc(ehooks, NULL, size, alignment, &zero,
addr = ehooks_alloc(tsdn, ehooks, NULL, size, alignment, &zero,
&commit, ind);
post_reentrancy(tsd);
}
return addr;

69
src/ehooks.c
View File

@ -2,7 +2,76 @@
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/ehooks.h"
#include "jemalloc/internal/extent_mmap.h"
void ehooks_init(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
ehooks_set_extent_hooks_ptr(ehooks, extent_hooks);
}
/*
* If the caller specifies (!*zero), it is still possible to receive zeroed
* memory, in which case *zero is toggled to true. arena_extent_alloc() takes
* advantage of this to avoid demanding zeroed extents, but taking advantage of
* them if they are returned.
*/
static void *
extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec) {
void *ret;
assert(size != 0);
assert(alignment != 0);
/* "primary" dss. */
if (have_dss && dss_prec == dss_prec_primary && (ret =
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
commit)) != NULL) {
return ret;
}
/* mmap. */
if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit))
!= NULL) {
return ret;
}
/* "secondary" dss. */
if (have_dss && dss_prec == dss_prec_secondary && (ret =
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
commit)) != NULL) {
return ret;
}
/* All strategies for allocation failed. */
return NULL;
}
void *
ehooks_default_alloc_impl(tsdn_t *tsdn, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
arena_t *arena = arena_get(tsdn, arena_ind, false);
void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
commit, (dss_prec_t)atomic_load_u(&arena->dss_prec,
ATOMIC_RELAXED));
if (have_madvise_huge && ret) {
pages_set_thp_state(ret, size);
}
return ret;
}
void *
ehooks_default_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
tsdn_t *tsdn;
arena_t *arena;
tsdn = tsdn_fetch();
arena = arena_get(tsdn, arena_ind, false);
/*
* The arena we're allocating on behalf of must have been initialized
* already.
*/
assert(arena != NULL);
return ehooks_default_alloc_impl(tsdn, new_addr, size,
ALIGNMENT_CEILING(alignment, PAGE), zero, commit,
arena_ind_get(arena));
}

96
src/extent.c
View File

@ -19,9 +19,6 @@ mutex_pool_t extent_mutex_pool;
size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;
static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit,
unsigned arena_ind);
static bool extent_dalloc_default(extent_hooks_t *extent_hooks, void *addr,
size_t size, bool committed, unsigned arena_ind);
static void extent_destroy_default(extent_hooks_t *extent_hooks, void *addr,
@ -60,7 +57,7 @@ static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_t *a, extent_t *b, bool growing_retained);
const extent_hooks_t extent_hooks_default = {
extent_alloc_default,
ehooks_default_alloc,
extent_dalloc_default,
extent_destroy_default,
extent_commit_default,
@ -881,72 +878,6 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
return extent;
}
/*
* If the caller specifies (!*zero), it is still possible to receive zeroed
* memory, in which case *zero is toggled to true. arena_extent_alloc() takes
* advantage of this to avoid demanding zeroed extents, but taking advantage of
* them if they are returned.
*/
static void *
extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec) {
void *ret;
assert(size != 0);
assert(alignment != 0);
/* "primary" dss. */
if (have_dss && dss_prec == dss_prec_primary && (ret =
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
commit)) != NULL) {
return ret;
}
/* mmap. */
if ((ret = extent_alloc_mmap(new_addr, size, alignment, zero, commit))
!= NULL) {
return ret;
}
/* "secondary" dss. */
if (have_dss && dss_prec == dss_prec_secondary && (ret =
extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
commit)) != NULL) {
return ret;
}
/* All strategies for allocation failed. */
return NULL;
}
static void *
extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit) {
void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
commit, (dss_prec_t)atomic_load_u(&arena->dss_prec,
ATOMIC_RELAXED));
if (have_madvise_huge && ret) {
pages_set_thp_state(ret, size);
}
return ret;
}
static void *
extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
tsdn_t *tsdn;
arena_t *arena;
tsdn = tsdn_fetch();
arena = arena_get(tsdn, arena_ind, false);
/*
* The arena we're allocating on behalf of must have been initialized
* already.
*/
assert(arena != NULL);
return extent_alloc_default_impl(tsdn, arena, new_addr, size,
ALIGNMENT_CEILING(alignment, PAGE), zero, commit);
}
static void
extent_hook_pre_reentrancy(tsdn_t *tsdn, arena_t *arena) {
tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn);
@ -1012,16 +943,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
bool zeroed = false;
bool committed = false;
void *ptr;
if (ehooks_are_default(ehooks)) {
ptr = extent_alloc_default_impl(tsdn, arena, NULL,
alloc_size, PAGE, &zeroed, &committed);
} else {
extent_hook_pre_reentrancy(tsdn, arena);
ptr = ehooks_alloc(ehooks, NULL, alloc_size, PAGE, &zeroed,
&committed, arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);
}
void *ptr = ehooks_alloc(tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed,
&committed, arena_ind_get(arena));
extent_init(extent, arena_ind_get(arena), ptr, alloc_size, false,
SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed,
@ -1177,18 +1100,9 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
if (extent == NULL) {
return NULL;
}
void *addr;
size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
if (ehooks_are_default(ehooks)) {
/* Call directly to propagate tsdn. */
addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize,
palignment, zero, commit);
} else {
extent_hook_pre_reentrancy(tsdn, arena);
addr = ehooks_alloc(ehooks, new_addr, esize, palignment, zero,
commit, arena_ind_get(arena));
extent_hook_post_reentrancy(tsdn);
}
void *addr = ehooks_alloc(tsdn, ehooks, new_addr, esize, palignment,
zero, commit, arena_ind_get(arena));
if (addr == NULL) {
extent_dalloc(tsdn, arena, extent);
return NULL;