project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add experimental.arenas_create_ext mallctl

Browse Source 
This mallctl accepts an arena_config_t structure which
can be used to customize the behavior of the arena.
Right now it contains extent_hooks and a new option,
metadata_use_hooks, which controls whether the extent
hooks are also used for metadata allocation.

The medata_use_hooks option has two main use cases:

1. In heterogeneous memory systems, to avoid metadata
being placed on potentially slower memory.

2. Avoiding virtual memory from being leaked as a result
of metadata allocation failure originating in an extent hook.
This commit is contained in:
Piotr Balcer 2021-08-23 14:03:35 +02:00 committed by Qi Wang
parent a9031a0970
commit 7bb05e04be
17 changed files with 165 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/jemalloc/internal/arena_externs.h
View File

@ -97,7 +97,7 @@ bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena,
unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
bool arena_init_huge(void);
bool arena_is_huge(unsigned arena_ind);
arena_t *arena_choose_huge(tsd_t *tsd);

14
include/jemalloc/internal/arena_types.h
View File

@ -41,4 +41,18 @@ typedef enum {
*/
#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
struct arena_config_s {
/* extent hooks to be used for the arena */
struct extent_hooks_s *extent_hooks;
/*
* Use extent hooks for metadata (base) allocations when true.
*/
bool metadata_use_hooks;
};
typedef struct arena_config_s arena_config_t;
extern const arena_config_t arena_config_default;
#endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */

7
include/jemalloc/internal/base.h
View File

@ -46,6 +46,11 @@ struct base_s {
*/
ehooks_t ehooks;
/*
* Use user hooks for metadata when true.
*/
bool metadata_use_hooks;
/* Protects base_alloc() and base_stats_get() operations. */
malloc_mutex_t mtx;
@ -87,7 +92,7 @@ metadata_thp_enabled(void) {
base_t *b0get(void);
base_t *base_new(tsdn_t *tsdn, unsigned ind,
const extent_hooks_t *extent_hooks);
const extent_hooks_t *extent_hooks, bool metadata_use_hooks);
void base_delete(tsdn_t *tsdn, base_t *base);
ehooks_t *base_ehooks_get(base_t *base);
extent_hooks_t *base_extent_hooks_set(base_t *base,

5
include/jemalloc/internal/base_structs.h
View File

@ -25,6 +25,11 @@ struct base_s {
*/
ehooks_t ehooks;
/*
* Use user hooks for metadata when true.
*/
bool metadata_use_hooks;
/* Protects base_alloc() and base_stats_get() operations. */
malloc_mutex_t mtx;

2
include/jemalloc/internal/jemalloc_internal_externs.h
View File

@ -57,7 +57,7 @@ void *bootstrap_calloc(size_t num, size_t size);
void bootstrap_free(void *ptr);
void arena_set(unsigned ind, arena_t *arena);
unsigned narenas_total_get(void);
arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks);
arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind);
void iarena_cleanup(tsd_t *tsd);

2
include/jemalloc/internal/jemalloc_internal_inlines_a.h
View File

@ -66,7 +66,7 @@ arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) {
if (unlikely(ret == NULL)) {
if (init_if_missing) {
ret = arena_init(tsdn, ind,
(extent_hooks_t *)&ehooks_default_extent_hooks);
&arena_config_default);
}
}
return ret;

11
src/arena.c
View File

@ -48,6 +48,11 @@ static unsigned nbins_total;
static unsigned huge_arena_ind;
const arena_config_t arena_config_default = {
/* .extent_hooks = */ (extent_hooks_t *)&ehooks_default_extent_hooks,
/* .metadata_use_hooks = */ true,
};
/******************************************************************************/
/*
* Function prototypes for static functions that are referenced prior to
@ -1516,7 +1521,6 @@ arena_set_extent_hooks(tsd_t *tsd, arena_t *arena,
return ret;
}
dss_prec_t
arena_dss_prec_get(arena_t *arena) {
return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE);
@ -1583,7 +1587,7 @@ arena_nthreads_dec(arena_t *arena, bool internal) {
}
arena_t *
arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
arena_t *arena;
base_t *base;
unsigned i;
@ -1591,7 +1595,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
if (ind == 0) {
base = b0get();
} else {
base = base_new(tsdn, ind, extent_hooks);
base = base_new(tsdn, ind, config->extent_hooks,
config->metadata_use_hooks);
if (base == NULL) {
return NULL;
}

21
src/base.c
View File

@ -295,6 +295,12 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind,
return block;
}
static ehooks_t *
base_ehooks_get_for_metadata(base_t *base) {
return base->metadata_use_hooks ? &base->ehooks :
(struct ehooks_s *)&ehooks_default_extent_hooks;
}
/*
* Allocate an extent that is at least as large as specified size, with
* specified alignment.
@ -303,7 +309,7 @@ static edata_t *
base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) {
malloc_mutex_assert_owner(tsdn, &base->mtx);
ehooks_t *ehooks = base_ehooks_get(base);
ehooks_t *ehooks = base_ehooks_get_for_metadata(base);
/*
* Drop mutex during base_block_alloc(), because an extent hook will be
* called.
@ -342,7 +348,8 @@ b0get(void) {
}
base_t *
base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks) {
base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
bool metadata_use_hooks) {
pszind_t pind_last = 0;
size_t extent_sn_next = 0;
@ -352,7 +359,9 @@ base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks) {
* memory, and then initialize the ehooks within the base_t.
*/
ehooks_t fake_ehooks;
ehooks_init(&fake_ehooks, (extent_hooks_t *)extent_hooks, ind);
ehooks_init(&fake_ehooks, metadata_use_hooks ?
(extent_hooks_t *)extent_hooks :
(extent_hooks_t *)&ehooks_default_extent_hooks, ind);
base_block_t *block = base_block_alloc(tsdn, NULL, &fake_ehooks, ind,
&pind_last, &extent_sn_next, sizeof(base_t), QUANTUM);
@ -375,6 +384,7 @@ base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks) {
base->extent_sn_next = extent_sn_next;
base->blocks = block;
base->auto_thp_switched = false;
base->metadata_use_hooks = metadata_use_hooks;
for (szind_t i = 0; i < SC_NSIZES; i++) {
edata_heap_new(&base->avail[i]);
}
@ -397,7 +407,7 @@ base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks) {
void
base_delete(tsdn_t *tsdn, base_t *base) {
ehooks_t *ehooks = base_ehooks_get(base);
ehooks_t *ehooks = base_ehooks_get_for_metadata(base);
base_block_t *next = base->blocks;
do {
base_block_t *block = next;
@ -512,6 +522,7 @@ base_postfork_child(tsdn_t *tsdn, base_t *base) {
bool
base_boot(tsdn_t *tsdn) {
b0 = base_new(tsdn, 0, (extent_hooks_t *)&ehooks_default_extent_hooks);
b0 = base_new(tsdn, 0,
(extent_hooks_t *)&ehooks_default_extent_hooks, true);
return (b0 == NULL);
}

42
src/ctl.c
View File

@ -315,6 +315,7 @@ INDEX_PROTO(experimental_arenas_i)
CTL_PROTO(experimental_prof_recent_alloc_max)
CTL_PROTO(experimental_prof_recent_alloc_dump)
CTL_PROTO(experimental_batch_alloc)
CTL_PROTO(experimental_arenas_create_ext)
#define MUTEX_STATS_CTL_PROTO_GEN(n) \
CTL_PROTO(stats_##n##_num_ops) \
@ -870,6 +871,7 @@ static const ctl_named_node_t experimental_node[] = {
{NAME("hooks"), CHILD(named, experimental_hooks)},
{NAME("utilization"), CHILD(named, experimental_utilization)},
{NAME("arenas"), CHILD(indexed, experimental_arenas)},
{NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)},
{NAME("prof_recent"), CHILD(named, experimental_prof_recent)},
{NAME("batch_alloc"), CTL(experimental_batch_alloc)},
{NAME("thread"), CHILD(named, experimental_thread)}
@ -1242,7 +1244,7 @@ ctl_arena_refresh(tsdn_t *tsdn, arena_t *arena, ctl_arena_t *ctl_sdarena,
}
static unsigned
ctl_arena_init(tsd_t *tsd, extent_hooks_t *extent_hooks) {
ctl_arena_init(tsd_t *tsd, const arena_config_t *config) {
unsigned arena_ind;
ctl_arena_t *ctl_arena;
@ -1260,7 +1262,7 @@ ctl_arena_init(tsd_t *tsd, extent_hooks_t *extent_hooks) {
}
/* Initialize new arena. */
if (arena_init(tsd_tsdn(tsd), arena_ind, extent_hooks) == NULL) {
if (arena_init(tsd_tsdn(tsd), arena_ind, config) == NULL) {
return UINT_MAX;
}
@ -2881,8 +2883,11 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
extent_hooks_t *new_extent_hooks
JEMALLOC_CC_SILENCE_INIT(NULL);
WRITE(new_extent_hooks, extent_hooks_t *);
arena_config_t config = arena_config_default;
config.extent_hooks = new_extent_hooks;
arena = arena_init(tsd_tsdn(tsd), arena_ind,
new_extent_hooks);
&config);
if (arena == NULL) {
ret = EFAULT;
goto label_return;
@ -3069,15 +3074,14 @@ static int
arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
extent_hooks_t *extent_hooks;
unsigned arena_ind;
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
VERIFY_READ(unsigned);
extent_hooks = (extent_hooks_t *)&ehooks_default_extent_hooks;
WRITE(extent_hooks, extent_hooks_t *);
if ((arena_ind = ctl_arena_init(tsd, extent_hooks)) == UINT_MAX) {
arena_config_t config = arena_config_default;
WRITE(config.extent_hooks, extent_hooks_t *);
if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) {
ret = EAGAIN;
goto label_return;
}
@ -3089,6 +3093,30 @@ label_return:
return ret;
}
static int
experimental_arenas_create_ext_ctl(tsd_t *tsd,
const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
arena_config_t config = arena_config_default;
VERIFY_READ(unsigned);
WRITE(config, arena_config_t);
if ((arena_ind = ctl_arena_init(tsd, &config)) == UINT_MAX) {
ret = EAGAIN;
goto label_return;
}
READ(arena_ind, unsigned);
ret = 0;
label_return:
malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
return ret;
}
static int
arenas_lookup_ctl(tsd_t *tsd, const size_t *mib,
size_t miblen, void *oldp, size_t *oldlenp, void *newp,

14
src/jemalloc.c
View File

@ -384,7 +384,7 @@ narenas_total_get(void) {
/* Create a new arena and insert it into the arenas array at index ind. */
static arena_t *
arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
arena_init_locked(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
arena_t *arena;
assert(ind <= narenas_total_get());
@ -406,7 +406,7 @@ arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
}
/* Actually initialize the arena. */
arena = arena_new(tsdn, ind, extent_hooks);
arena = arena_new(tsdn, ind, config);
return arena;
}
@ -430,11 +430,11 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
}
arena_t *
arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
arena_t *arena;
malloc_mutex_lock(tsdn, &arenas_lock);
arena = arena_init_locked(tsdn, ind, extent_hooks);
arena = arena_init_locked(tsdn, ind, config);
malloc_mutex_unlock(tsdn, &arenas_lock);
arena_new_create_background_thread(tsdn, ind);
@ -570,8 +570,7 @@ arena_choose_hard(tsd_t *tsd, bool internal) {
choose[j] = first_null;
arena = arena_init_locked(tsd_tsdn(tsd),
choose[j],
(extent_hooks_t *)
&ehooks_default_extent_hooks);
&arena_config_default);
if (arena == NULL) {
malloc_mutex_unlock(tsd_tsdn(tsd),
&arenas_lock);
@ -1779,8 +1778,7 @@ malloc_init_hard_a0_locked() {
* Initialize one arena here. The rest are lazily created in
* arena_choose_hard().
*/
if (arena_init(TSDN_NULL, 0,
(extent_hooks_t *)&ehooks_default_extent_hooks) == NULL) {
if (arena_init(TSDN_NULL, 0, &arena_config_default) == NULL) {
return true;
}
a0 = arena_get(TSDN_NULL, 0, false);

53
test/integration/extent.c
View File

@ -2,6 +2,8 @@
#include "test/extent_hooks.h"
#include "jemalloc/internal/arena_types.h"
static void
test_extent_body(unsigned arena_ind) {
void *p;
@ -228,9 +230,58 @@ TEST_BEGIN(test_extent_auto_hook) {
}
TEST_END
static void
test_arenas_create_ext_base(arena_config_t config,
bool expect_hook_data, bool expect_hook_metadata)
{
unsigned arena, arena1;
void *ptr;
size_t sz = sizeof(unsigned);
extent_hooks_prep();
called_alloc = false;
expect_d_eq(mallctl("experimental.arenas_create_ext",
(void *)&arena, &sz, &config, sizeof(arena_config_t)), 0,
"Unexpected mallctl() failure");
expect_b_eq(called_alloc, expect_hook_metadata,
"expected hook metadata alloc mismatch");
called_alloc = false;
ptr = mallocx(42, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE);
expect_b_eq(called_alloc, expect_hook_data,
"expected hook data alloc mismatch");
expect_ptr_not_null(ptr, "Unexpected mallocx() failure");
expect_d_eq(mallctl("arenas.lookup", &arena1, &sz, &ptr, sizeof(ptr)),
0, "Unexpected mallctl() failure");
expect_u_eq(arena, arena1, "Unexpected arena index");
dallocx(ptr, 0);
}
TEST_BEGIN(test_arenas_create_ext_with_ehooks_no_metadata) {
arena_config_t config;
config.extent_hooks = &hooks;
config.metadata_use_hooks = false;
test_arenas_create_ext_base(config, true, false);
}
TEST_END
TEST_BEGIN(test_arenas_create_ext_with_ehooks_with_metadata) {
arena_config_t config;
config.extent_hooks = &hooks;
config.metadata_use_hooks = true;
test_arenas_create_ext_base(config, true, true);
}
TEST_END
int
main(void) {
return test(
test_extent_manual_hook,
test_extent_auto_hook);
test_extent_auto_hook,
test_arenas_create_ext_with_ehooks_no_metadata,
test_arenas_create_ext_with_ehooks_with_metadata);
}

7
test/unit/base.c
View File

@ -32,7 +32,8 @@ TEST_BEGIN(test_base_hooks_default) {
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
base = base_new(tsdn, 0,
(extent_hooks_t *)&ehooks_default_extent_hooks);
(extent_hooks_t *)&ehooks_default_extent_hooks,
/* metadata_use_hooks */ true);
if (config_stats) {
base_stats_get(tsdn, base, &allocated0, &resident, &mapped,
@ -74,7 +75,7 @@ TEST_BEGIN(test_base_hooks_null) {
memcpy(&hooks, &hooks_null, sizeof(extent_hooks_t));
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
base = base_new(tsdn, 0, &hooks);
base = base_new(tsdn, 0, &hooks, /* metadata_use_hooks */ true);
expect_ptr_not_null(base, "Unexpected base_new() failure");
if (config_stats) {
@ -120,7 +121,7 @@ TEST_BEGIN(test_base_hooks_not_null) {
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
did_alloc = false;
base = base_new(tsdn, 0, &hooks);
base = base_new(tsdn, 0, &hooks, /* metadata_use_hooks */ true);
expect_ptr_not_null(base, "Unexpected base_new() failure");
expect_true(did_alloc, "Expected alloc");

2
test/unit/edata_cache.c
View File

@ -5,7 +5,7 @@
static void
test_edata_cache_init(edata_cache_t *edata_cache) {
base_t *base = base_new(TSDN_NULL, /* ind */ 1,
&ehooks_default_extent_hooks);
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(base, "");
bool err = edata_cache_init(edata_cache, base);
assert_false(err, "");

2
test/unit/hpa.c
View File

@ -37,7 +37,7 @@ static hpa_shard_t *
create_test_data(hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
bool err;
base_t *base = base_new(TSDN_NULL, /* ind */ SHARD_IND,
&ehooks_default_extent_hooks);
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(base, "");
test_data_t *test_data = malloc(sizeof(test_data_t));

3
test/unit/pa.c
View File

@ -53,7 +53,8 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
assert_ptr_not_null(test_data, "");
init_test_extent_hooks(&test_data->hooks);
base_t *base = base_new(TSDN_NULL, /* ind */ 1, &test_data->hooks);
base_t *base = base_new(TSDN_NULL, /* ind */ 1,
&test_data->hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(base, "");
test_data->base = base;

15
test/unit/rtree.c
View File

@ -12,7 +12,8 @@ TEST_BEGIN(test_rtree_read_empty) {
tsdn = tsdn_fetch();
base_t *base = base_new(tsdn, 0, &ehooks_default_extent_hooks);
base_t *base = base_new(tsdn, 0,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
expect_ptr_not_null(base, "Unexpected base_new failure");
rtree_t *rtree = &test_rtree;
@ -52,7 +53,8 @@ TEST_BEGIN(test_rtree_extrema) {
tsdn_t *tsdn = tsdn_fetch();
base_t *base = base_new(tsdn, 0, &ehooks_default_extent_hooks);
base_t *base = base_new(tsdn, 0,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
expect_ptr_not_null(base, "Unexpected base_new failure");
rtree_t *rtree = &test_rtree;
@ -103,7 +105,8 @@ TEST_END
TEST_BEGIN(test_rtree_bits) {
tsdn_t *tsdn = tsdn_fetch();
base_t *base = base_new(tsdn, 0, &ehooks_default_extent_hooks);
base_t *base = base_new(tsdn, 0,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
expect_ptr_not_null(base, "Unexpected base_new failure");
uintptr_t keys[] = {PAGE, PAGE + 1,
@ -152,7 +155,8 @@ TEST_BEGIN(test_rtree_random) {
sfmt_t *sfmt = init_gen_rand(SEED);
tsdn_t *tsdn = tsdn_fetch();
base_t *base = base_new(tsdn, 0, &ehooks_default_extent_hooks);
base_t *base = base_new(tsdn, 0,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
expect_ptr_not_null(base, "Unexpected base_new failure");
uintptr_t keys[NSET];
@ -250,7 +254,8 @@ test_rtree_range_write(tsdn_t *tsdn, rtree_t *rtree, uintptr_t start,
TEST_BEGIN(test_rtree_range) {
tsdn_t *tsdn = tsdn_fetch();
base_t *base = base_new(tsdn, 0, &ehooks_default_extent_hooks);
base_t *base = base_new(tsdn, 0,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
expect_ptr_not_null(base, "Unexpected base_new failure");
rtree_t *rtree = &test_rtree;

4
test/unit/sec.c
View File

@ -42,7 +42,7 @@ test_sec_init(sec_t *sec, pai_t *fallback, size_t nshards, size_t max_alloc,
* short-running, and SECs are arena-scoped in reality.
*/
base_t *base = base_new(TSDN_NULL, /* ind */ 123,
&ehooks_default_extent_hooks);
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
bool err = sec_init(TSDN_NULL, sec, base, fallback, &opts);
assert_false(err, "Unexpected initialization failure");
@ -442,7 +442,7 @@ TEST_BEGIN(test_nshards_0) {
/* See the note above -- we can't use the real tsd. */
tsdn_t *tsdn = TSDN_NULL;
base_t *base = base_new(TSDN_NULL, /* ind */ 123,
&ehooks_default_extent_hooks);
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
sec_opts_t opts = SEC_OPTS_DEFAULT;
opts.nshards = 0;