Ehooks: Add head tracking.
This commit is contained in:
parent
09475bf8ac
commit
0704516245
@ -467,20 +467,12 @@ edata_prof_alloc_time_set(edata_t *edata, nstime_t *t) {
|
|||||||
|
|
||||||
static inline bool
|
static inline bool
|
||||||
edata_is_head_get(edata_t *edata) {
|
edata_is_head_get(edata_t *edata) {
|
||||||
if (maps_coalesce) {
|
|
||||||
not_reached();
|
|
||||||
}
|
|
||||||
|
|
||||||
return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) >>
|
return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) >>
|
||||||
EDATA_BITS_IS_HEAD_SHIFT);
|
EDATA_BITS_IS_HEAD_SHIFT);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void
|
static inline void
|
||||||
edata_is_head_set(edata_t *edata, bool is_head) {
|
edata_is_head_set(edata_t *edata, bool is_head) {
|
||||||
if (maps_coalesce) {
|
|
||||||
not_reached();
|
|
||||||
}
|
|
||||||
|
|
||||||
edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) |
|
edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) |
|
||||||
((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT);
|
((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT);
|
||||||
}
|
}
|
||||||
@ -502,9 +494,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
|
|||||||
edata_committed_set(edata, committed);
|
edata_committed_set(edata, committed);
|
||||||
edata_dumpable_set(edata, dumpable);
|
edata_dumpable_set(edata, dumpable);
|
||||||
ql_elm_new(edata, ql_link);
|
ql_elm_new(edata, ql_link);
|
||||||
if (!maps_coalesce) {
|
edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
|
||||||
edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
|
|
||||||
}
|
|
||||||
if (config_prof) {
|
if (config_prof) {
|
||||||
edata_prof_tctx_set(edata, NULL);
|
edata_prof_tctx_set(edata, NULL);
|
||||||
}
|
}
|
||||||
|
@ -1,16 +1,21 @@
|
|||||||
#ifndef JEMALLOC_INTERNAL_EHOOKS_H
|
#ifndef JEMALLOC_INTERNAL_EHOOKS_H
|
||||||
#define JEMALLOC_INTERNAL_EHOOKS_H
|
#define JEMALLOC_INTERNAL_EHOOKS_H
|
||||||
|
|
||||||
|
#include "jemalloc/internal/atomic.h"
|
||||||
|
#include "jemalloc/internal/extent_mmap.h"
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* This module is the internal interface to the extent hooks (both
|
* This module is the internal interface to the extent hooks (both
|
||||||
* user-specified and external). Eventually, this will give us the flexibility
|
* user-specified and external). Eventually, this will give us the flexibility
|
||||||
* to use multiple different versions of user-visible extent-hook APIs under a
|
* to use multiple different versions of user-visible extent-hook APIs under a
|
||||||
* single user interface.
|
* single user interface.
|
||||||
|
*
|
||||||
|
* Current API expansions (not available to anyone but the default hooks yet):
|
||||||
|
* - Head state tracking. Hooks can decide whether or not to merge two
|
||||||
|
* extents based on whether or not one of them is the head (i.e. was
|
||||||
|
* allocated on its own). The later extent loses its "head" status.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
#include "jemalloc/internal/atomic.h"
|
|
||||||
#include "jemalloc/internal/extent_mmap.h"
|
|
||||||
|
|
||||||
extern const extent_hooks_t ehooks_default_extent_hooks;
|
extern const extent_hooks_t ehooks_default_extent_hooks;
|
||||||
|
|
||||||
typedef struct ehooks_s ehooks_t;
|
typedef struct ehooks_s ehooks_t;
|
||||||
@ -43,7 +48,8 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
|
|||||||
bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
|
bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
|
||||||
#endif
|
#endif
|
||||||
bool ehooks_default_split_impl();
|
bool ehooks_default_split_impl();
|
||||||
bool ehooks_default_merge_impl(void *addr_a, void *addr_b);
|
bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, bool head_a,
|
||||||
|
void *addr_b, bool head_b);
|
||||||
void ehooks_default_zero_impl(void *addr, size_t size);
|
void ehooks_default_zero_impl(void *addr, size_t size);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@ -314,10 +320,12 @@ ehooks_split(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
|
|||||||
|
|
||||||
static inline bool
|
static inline bool
|
||||||
ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a,
|
ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a,
|
||||||
void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
|
bool head_a, void *addr_b, size_t size_b, bool head_b, bool committed,
|
||||||
|
unsigned arena_ind) {
|
||||||
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
|
||||||
if (extent_hooks == &ehooks_default_extent_hooks) {
|
if (extent_hooks == &ehooks_default_extent_hooks) {
|
||||||
return ehooks_default_merge_impl(addr_a, addr_b);
|
return ehooks_default_merge_impl(tsdn, addr_a, head_a, addr_b,
|
||||||
|
head_b);
|
||||||
} else if (extent_hooks->merge == NULL) {
|
} else if (extent_hooks->merge == NULL) {
|
||||||
return true;
|
return true;
|
||||||
} else {
|
} else {
|
||||||
|
@ -54,7 +54,6 @@ edata_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
|
|||||||
size_t size_b, szind_t szind_b, bool slab_b);
|
size_t size_b, szind_t szind_b, bool slab_b);
|
||||||
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
|
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
|
||||||
edata_t *a, edata_t *b);
|
edata_t *a, edata_t *b);
|
||||||
bool extent_head_no_merge(edata_t *a, edata_t *b);
|
|
||||||
|
|
||||||
bool extent_boot(void);
|
bool extent_boot(void);
|
||||||
|
|
||||||
|
60
src/ehooks.c
60
src/ehooks.c
@ -183,8 +183,51 @@ ehooks_default_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
|||||||
return ehooks_default_split_impl();
|
return ehooks_default_split_impl();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline bool
|
||||||
|
ehooks_same_sn(tsdn_t *tsdn, void *addr_a, void *addr_b) {
|
||||||
|
edata_t *a = iealloc(tsdn, addr_a);
|
||||||
|
edata_t *b = iealloc(tsdn, addr_b);
|
||||||
|
return edata_sn_comp(a, b) == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Returns true if the given extents can't be merged because of their head bit
|
||||||
|
* settings. Assumes the second extent has the higher address.
|
||||||
|
*/
|
||||||
|
static bool
|
||||||
|
ehooks_no_merge_heads(tsdn_t *tsdn, void *addr_a, bool head_a, void *addr_b,
|
||||||
|
bool head_b) {
|
||||||
|
/*
|
||||||
|
* When coalesce is not always allowed (Windows), only merge extents
|
||||||
|
* from the same VirtualAlloc region under opt.retain (in which case
|
||||||
|
* MEM_DECOMMIT is utilized for purging).
|
||||||
|
*/
|
||||||
|
if (maps_coalesce) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!opt_retain) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
/* If b is a head extent, disallow the cross-region merge. */
|
||||||
|
if (head_b) {
|
||||||
|
/*
|
||||||
|
* Additionally, sn should not overflow with retain; sanity
|
||||||
|
* check that different regions have unique sn.
|
||||||
|
*/
|
||||||
|
assert(!ehooks_same_sn(tsdn, addr_a, addr_b));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
assert(ehooks_same_sn(tsdn, addr_a, addr_b));
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
bool
|
bool
|
||||||
ehooks_default_merge_impl(void *addr_a, void *addr_b) {
|
ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, bool head_a, void *addr_b,
|
||||||
|
bool head_b) {
|
||||||
|
if (ehooks_no_merge_heads(tsdn, addr_a, head_a, addr_b, head_b)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if (!maps_coalesce && !opt_retain) {
|
if (!maps_coalesce && !opt_retain) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -198,15 +241,12 @@ ehooks_default_merge_impl(void *addr_a, void *addr_b) {
|
|||||||
static bool
|
static bool
|
||||||
ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
|
ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
|
||||||
void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
|
void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
|
||||||
if (!maps_coalesce) {
|
tsdn_t *tsdn = tsdn_fetch();
|
||||||
tsdn_t *tsdn = tsdn_fetch();
|
edata_t *a = iealloc(tsdn, addr_a);
|
||||||
edata_t *a = iealloc(tsdn, addr_a);
|
bool head_a = edata_is_head_get(a);
|
||||||
edata_t *b = iealloc(tsdn, addr_b);
|
edata_t *b = iealloc(tsdn, addr_b);
|
||||||
if (extent_head_no_merge(a, b)) {
|
bool head_b = edata_is_head_get(b);
|
||||||
return true;
|
return ehooks_default_merge_impl(tsdn, addr_a, head_a, addr_b, head_b);
|
||||||
}
|
|
||||||
}
|
|
||||||
return ehooks_default_merge_impl(addr_a, addr_b);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
|
@ -1493,38 +1493,6 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
|
|||||||
slab_a, size_b, szind_b, slab_b, false);
|
slab_a, size_b, szind_b, slab_b, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* Returns true if the given extents can't be merged because of their head bit
|
|
||||||
* settings. Assumes the second extent has the higher address.
|
|
||||||
*/
|
|
||||||
bool
|
|
||||||
extent_head_no_merge(edata_t *a, edata_t *b) {
|
|
||||||
assert(edata_base_get(a) < edata_base_get(b));
|
|
||||||
/*
|
|
||||||
* When coalesce is not always allowed (Windows), only merge extents
|
|
||||||
* from the same VirtualAlloc region under opt.retain (in which case
|
|
||||||
* MEM_DECOMMIT is utilized for purging).
|
|
||||||
*/
|
|
||||||
if (maps_coalesce) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!opt_retain) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
/* If b is a head extent, disallow the cross-region merge. */
|
|
||||||
if (edata_is_head_get(b)) {
|
|
||||||
/*
|
|
||||||
* Additionally, sn should not overflow with retain; sanity
|
|
||||||
* check that different regions have unique sn.
|
|
||||||
*/
|
|
||||||
assert(edata_sn_comp(a, b) != 0);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
assert(edata_sn_comp(a, b) == 0);
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool
|
static bool
|
||||||
extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
|
extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
|
||||||
edata_t *b, bool growing_retained) {
|
edata_t *b, bool growing_retained) {
|
||||||
@ -1532,13 +1500,10 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
|
|||||||
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
|
WITNESS_RANK_CORE, growing_retained ? 1 : 0);
|
||||||
assert(edata_base_get(a) < edata_base_get(b));
|
assert(edata_base_get(a) < edata_base_get(b));
|
||||||
|
|
||||||
if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool err = ehooks_merge(tsdn, ehooks, edata_base_get(a),
|
bool err = ehooks_merge(tsdn, ehooks, edata_base_get(a),
|
||||||
edata_size_get(a), edata_base_get(b), edata_size_get(b),
|
edata_size_get(a), edata_is_head_get(a), edata_base_get(b),
|
||||||
edata_committed_get(a), arena_ind_get(arena));
|
edata_size_get(b), edata_is_head_get(b), edata_committed_get(a),
|
||||||
|
arena_ind_get(arena));
|
||||||
|
|
||||||
if (err) {
|
if (err) {
|
||||||
return true;
|
return true;
|
||||||
|
Loading…
Reference in New Issue
Block a user