Refactor purging and splitting/merging.
Split purging into lazy and forced variants. Use the forced variant for
zeroing dss.
Add support for NULL function pointers as an opt-out mechanism for the
dalloc, commit, decommit, purge_lazy, purge_forced, split, and merge
fields of extent_hooks_t.
Add short-circuiting checks in large_ralloc_no_move_{shrink,expand}() so
that no attempt is made if splitting/merging is not supported.
This resolves #268.
This commit is contained in:
Jason Evans
parent
884fa22b8c
commit
a6e86810d8
@ -1512,7 +1512,8 @@ struct extent_hooks_s {
|
||||
extent_dalloc_t *dalloc;
|
||||
extent_commit_t *commit;
|
||||
extent_decommit_t *decommit;
|
||||
extent_purge_t *purge;
|
||||
extent_purge_t *purge_lazy;
|
||||
extent_purge_t *purge_forced;
|
||||
extent_split_t *split;
|
||||
extent_merge_t *merge;
|
||||
};]]></programlisting>
|
||||
@ -1522,13 +1523,12 @@ struct extent_hooks_s {
|
||||
mapped committed memory, in the simplest case followed by deallocation.
|
||||
However, there are performance and platform reasons to retain extents
|
||||
for later reuse. Cleanup attempts cascade from deallocation to decommit
|
||||
to purging, which gives the extent management functions opportunities to
|
||||
reject the most permanent cleanup operations in favor of less permanent
|
||||
(and often less costly) operations. The extent splitting and merging
|
||||
operations can also be opted out of, but this is mainly intended to
|
||||
support platforms on which virtual memory mappings provided by the
|
||||
operating system kernel do not automatically coalesce and split, e.g.
|
||||
Windows.</para>
|
||||
to lazy purging to forced purging, which gives the extent management
|
||||
functions opportunities to reject the most permanent cleanup operations
|
||||
in favor of less permanent (and often less costly) operations. All
|
||||
operations except allocation can be universally opted out of by setting
|
||||
the hook pointers to <constant>NULL</constant>, or selectively opted out
|
||||
of by returning failure.</para>
|
||||
|
||||
<funcsynopsis><funcprototype>
|
||||
<funcdef>typedef void *<function>(extent_alloc_t)</function></funcdef>
|
||||
@ -1641,14 +1641,17 @@ struct extent_hooks_s {
|
||||
</funcprototype></funcsynopsis>
|
||||
<literallayout></literallayout>
|
||||
<para>An extent purge function conforms to the
|
||||
<type>extent_purge_t</type> type and optionally discards physical pages
|
||||
<type>extent_purge_t</type> type and discards physical pages
|
||||
within the virtual memory mapping associated with an extent at given
|
||||
<parameter>addr</parameter> and <parameter>size</parameter> at
|
||||
<parameter>offset</parameter> bytes, extending for
|
||||
<parameter>length</parameter> on behalf of arena
|
||||
<parameter>arena_ind</parameter>, returning false if pages within the
|
||||
purged virtual memory range will be zero-filled the next time they are
|
||||
accessed.</para>
|
||||
<parameter>arena_ind</parameter>. A lazy extent purge function can
|
||||
delay purging indefinitely and leave the pages within the purged virtual
|
||||
memory range in an indeterminite state, whereas a forced extent purge
|
||||
function immediately purges, and the pages within the virtual memory
|
||||
range will be zero-filled the next time they are accessed. If the
|
||||
function returns true, this indicates failure to purge.</para>
|
||||
|
||||
<funcsynopsis><funcprototype>
|
||||
<funcdef>typedef bool <function>(extent_split_t)</function></funcdef>
|
||||
|
||||
@ -198,8 +198,8 @@ struct arena_s {
|
||||
|
||||
/*
|
||||
* Current count of pages within unused extents that are potentially
|
||||
* dirty, and for which madvise(... MADV_DONTNEED) has not been called.
|
||||
* By tracking this, we can institute a limit on how much dirty unused
|
||||
* dirty, and for which pages_purge_*() has not been called. By
|
||||
* tracking this, we can institute a limit on how much dirty unused
|
||||
* memory is mapped for each arena.
|
||||
*/
|
||||
size_t ndirty;
|
||||
|
||||
@ -133,7 +133,10 @@ bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length);
|
||||
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
|
||||
@ -24,6 +24,23 @@
|
||||
#define HUGEPAGE_CEILING(s) \
|
||||
(((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
|
||||
|
||||
/* PAGES_CAN_PURGE_LAZY is defined if lazy purging is supported. */
|
||||
#if defined(_WIN32) || defined(JEMALLOC_PURGE_MADVISE_FREE)
|
||||
# define PAGES_CAN_PURGE_LAZY
|
||||
#endif
|
||||
/*
|
||||
* PAGES_CAN_PURGE_FORCED is defined if forced purging is supported.
|
||||
*
|
||||
* The only supported way to hard-purge on Windows is to decommit and then
|
||||
* re-commit, but doing so is racy, and if re-commit fails it's a pain to
|
||||
* propagate the "poisoned" memory state. Since we typically decommit as the
|
||||
* next step after purging on Windows anyway, there's no point in adding such
|
||||
* complexity.
|
||||
*/
|
||||
#if !defined(_WIN32) && defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
|
||||
# define PAGES_CAN_PURGE_FORCED
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
@ -32,13 +49,29 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
static const bool pages_can_purge_lazy =
|
||||
#ifdef PAGES_CAN_PURGE_LAZY
|
||||
true
|
||||
#else
|
||||
false
|
||||
#endif
|
||||
;
|
||||
static const bool pages_can_purge_forced =
|
||||
#ifdef PAGES_CAN_PURGE_FORCED
|
||||
true
|
||||
#else
|
||||
false
|
||||
#endif
|
||||
;
|
||||
|
||||
void *pages_map(void *addr, size_t size, bool *commit);
|
||||
void pages_unmap(void *addr, size_t size);
|
||||
void *pages_trim(void *addr, size_t alloc_size, size_t leadsize,
|
||||
size_t size, bool *commit);
|
||||
bool pages_commit(void *addr, size_t size);
|
||||
bool pages_decommit(void *addr, size_t size);
|
||||
bool pages_purge(void *addr, size_t size);
|
||||
bool pages_purge_lazy(void *addr, size_t size);
|
||||
bool pages_purge_forced(void *addr, size_t size);
|
||||
bool pages_huge(void *addr, size_t size);
|
||||
bool pages_nohuge(void *addr, size_t size);
|
||||
void pages_boot(void);
|
||||
|
||||
@ -179,7 +179,8 @@ extent_merge_wrapper
|
||||
extent_past_get
|
||||
extent_prof_tctx_get
|
||||
extent_prof_tctx_set
|
||||
extent_purge_wrapper
|
||||
extent_purge_forced_wrapper
|
||||
extent_purge_lazy_wrapper
|
||||
extent_retained_get
|
||||
extent_ring_insert
|
||||
extent_ring_remove
|
||||
@ -327,7 +328,8 @@ pages_decommit
|
||||
pages_huge
|
||||
pages_map
|
||||
pages_nohuge
|
||||
pages_purge
|
||||
pages_purge_forced
|
||||
pages_purge_lazy
|
||||
pages_trim
|
||||
pages_unmap
|
||||
pind2sz
|
||||
|
||||
@ -61,7 +61,8 @@ struct extent_hooks_s {
|
||||
extent_dalloc_t *dalloc;
|
||||
extent_commit_t *commit;
|
||||
extent_decommit_t *decommit;
|
||||
extent_purge_t *purge;
|
||||
extent_purge_t *purge_lazy;
|
||||
extent_purge_t *purge_forced;
|
||||
extent_split_t *split;
|
||||
extent_merge_t *merge;
|
||||
};
|
||||
|
||||
134
src/extent.c
134
src/extent.c
@ -15,23 +15,47 @@ static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
static bool extent_decommit_default(extent_hooks_t *extent_hooks,
|
||||
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
static bool extent_purge_default(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
#ifdef PAGES_CAN_PURGE_LAZY
|
||||
static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks,
|
||||
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
#endif
|
||||
#ifdef PAGES_CAN_PURGE_FORCED
|
||||
static bool extent_purge_forced_default(extent_hooks_t *extent_hooks,
|
||||
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
#endif
|
||||
#ifdef JEMALLOC_MAPS_COALESCE
|
||||
static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t size_a, size_t size_b, bool committed,
|
||||
unsigned arena_ind);
|
||||
static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a,
|
||||
size_t size_a, void *addr_b, size_t size_b, bool committed,
|
||||
unsigned arena_ind);
|
||||
#endif
|
||||
|
||||
const extent_hooks_t extent_hooks_default = {
|
||||
extent_alloc_default,
|
||||
extent_dalloc_default,
|
||||
extent_commit_default,
|
||||
extent_decommit_default,
|
||||
extent_purge_default,
|
||||
extent_decommit_default
|
||||
#ifdef PAGES_CAN_PURGE_LAZY
|
||||
,
|
||||
extent_purge_lazy_default
|
||||
#else
|
||||
,
|
||||
NULL
|
||||
#endif
|
||||
#ifdef PAGES_CAN_PURGE_FORCED
|
||||
,
|
||||
extent_purge_forced_default
|
||||
#else
|
||||
,
|
||||
NULL
|
||||
#endif
|
||||
#ifdef JEMALLOC_MAPS_COALESCE
|
||||
,
|
||||
extent_split_default,
|
||||
extent_merge_default
|
||||
#endif
|
||||
};
|
||||
|
||||
/* Used exclusively for gdump triggering. */
|
||||
@ -395,8 +419,11 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
|
||||
* that this is only a virtual memory leak.
|
||||
*/
|
||||
if (cache) {
|
||||
extent_purge_wrapper(tsdn, arena, r_extent_hooks, extent, 0,
|
||||
extent_size_get(extent));
|
||||
if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks,
|
||||
extent, 0, extent_size_get(extent))) {
|
||||
extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks,
|
||||
extent, 0, extent_size_get(extent));
|
||||
}
|
||||
}
|
||||
extent_dalloc(tsdn, arena, extent);
|
||||
}
|
||||
@ -1023,7 +1050,7 @@ void
|
||||
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t **r_extent_hooks, extent_t *extent)
|
||||
{
|
||||
bool err;
|
||||
bool err, zeroed;
|
||||
|
||||
assert(extent_base_get(extent) != NULL);
|
||||
assert(extent_size_get(extent) != 0);
|
||||
@ -1041,9 +1068,10 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
err = extent_dalloc_default_impl(extent_base_get(extent),
|
||||
extent_size_get(extent));
|
||||
} else {
|
||||
err = (*r_extent_hooks)->dalloc(*r_extent_hooks,
|
||||
err = ((*r_extent_hooks)->dalloc == NULL ||
|
||||
(*r_extent_hooks)->dalloc(*r_extent_hooks,
|
||||
extent_base_get(extent), extent_size_get(extent),
|
||||
extent_committed_get(extent), arena->ind);
|
||||
extent_committed_get(extent), arena->ind));
|
||||
}
|
||||
|
||||
if (!err) {
|
||||
@ -1052,13 +1080,24 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
}
|
||||
extent_reregister(tsdn, extent);
|
||||
/* Try to decommit; purge if that fails. */
|
||||
if (extent_committed_get(extent)) {
|
||||
extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
|
||||
0, extent_size_get(extent));
|
||||
}
|
||||
extent_zeroed_set(extent, !extent_committed_get(extent) ||
|
||||
!(*r_extent_hooks)->purge(*r_extent_hooks, extent_base_get(extent),
|
||||
extent_size_get(extent), 0, extent_size_get(extent), arena->ind));
|
||||
if (!extent_committed_get(extent))
|
||||
zeroed = true;
|
||||
else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
|
||||
0, extent_size_get(extent)))
|
||||
zeroed = true;
|
||||
else if ((*r_extent_hooks)->purge_lazy != NULL &&
|
||||
!(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
|
||||
extent_base_get(extent), extent_size_get(extent), 0,
|
||||
extent_size_get(extent), arena->ind))
|
||||
zeroed = false;
|
||||
else if ((*r_extent_hooks)->purge_forced != NULL &&
|
||||
!(*r_extent_hooks)->purge_forced(*r_extent_hooks,
|
||||
extent_base_get(extent), extent_size_get(extent), 0,
|
||||
extent_size_get(extent), arena->ind))
|
||||
zeroed = true;
|
||||
else
|
||||
zeroed = false;
|
||||
extent_zeroed_set(extent, zeroed);
|
||||
|
||||
if (config_stats)
|
||||
arena->stats.retained += extent_size_get(extent);
|
||||
@ -1088,9 +1127,9 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
bool err;
|
||||
|
||||
extent_hooks_assure_initialized(arena, r_extent_hooks);
|
||||
err = (*r_extent_hooks)->commit(*r_extent_hooks,
|
||||
extent_base_get(extent), extent_size_get(extent), offset, length,
|
||||
arena->ind);
|
||||
err = ((*r_extent_hooks)->commit == NULL ||
|
||||
(*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent),
|
||||
extent_size_get(extent), offset, length, arena->ind));
|
||||
extent_committed_set(extent, extent_committed_get(extent) || !err);
|
||||
return (err);
|
||||
}
|
||||
@ -1115,15 +1154,17 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
|
||||
extent_hooks_assure_initialized(arena, r_extent_hooks);
|
||||
|
||||
err = (*r_extent_hooks)->decommit(*r_extent_hooks,
|
||||
err = ((*r_extent_hooks)->decommit == NULL ||
|
||||
(*r_extent_hooks)->decommit(*r_extent_hooks,
|
||||
extent_base_get(extent), extent_size_get(extent), offset, length,
|
||||
arena->ind);
|
||||
arena->ind));
|
||||
extent_committed_set(extent, extent_committed_get(extent) && err);
|
||||
return (err);
|
||||
}
|
||||
|
||||
#ifdef PAGES_CAN_PURGE_LAZY
|
||||
static bool
|
||||
extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
size_t offset, size_t length, unsigned arena_ind)
|
||||
{
|
||||
|
||||
@ -1133,22 +1174,55 @@ extent_purge_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
assert(length != 0);
|
||||
assert((length & PAGE_MASK) == 0);
|
||||
|
||||
return (pages_purge((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
return (pages_purge_lazy((void *)((uintptr_t)addr + (uintptr_t)offset),
|
||||
length));
|
||||
}
|
||||
#endif
|
||||
|
||||
bool
|
||||
extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(arena, r_extent_hooks);
|
||||
return ((*r_extent_hooks)->purge(*r_extent_hooks,
|
||||
return ((*r_extent_hooks)->purge_lazy == NULL ||
|
||||
(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
|
||||
extent_base_get(extent), extent_size_get(extent), offset, length,
|
||||
arena->ind));
|
||||
}
|
||||
|
||||
#ifdef PAGES_CAN_PURGE_FORCED
|
||||
static bool
|
||||
extent_purge_forced_default(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t offset, size_t length, unsigned arena_ind)
|
||||
{
|
||||
|
||||
assert(extent_hooks == &extent_hooks_default);
|
||||
assert(addr != NULL);
|
||||
assert((offset & PAGE_MASK) == 0);
|
||||
assert(length != 0);
|
||||
assert((length & PAGE_MASK) == 0);
|
||||
|
||||
return (pages_purge_forced((void *)((uintptr_t)addr +
|
||||
(uintptr_t)offset), length));
|
||||
}
|
||||
#endif
|
||||
|
||||
bool
|
||||
extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
||||
size_t length)
|
||||
{
|
||||
|
||||
extent_hooks_assure_initialized(arena, r_extent_hooks);
|
||||
return ((*r_extent_hooks)->purge_forced == NULL ||
|
||||
(*r_extent_hooks)->purge_forced(*r_extent_hooks,
|
||||
extent_base_get(extent), extent_size_get(extent), offset, length,
|
||||
arena->ind));
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_MAPS_COALESCE
|
||||
static bool
|
||||
extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
size_t size_a, size_t size_b, bool committed, unsigned arena_ind)
|
||||
@ -1160,6 +1234,7 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
return (true);
|
||||
return (false);
|
||||
}
|
||||
#endif
|
||||
|
||||
extent_t *
|
||||
extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
@ -1175,6 +1250,9 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
|
||||
extent_hooks_assure_initialized(arena, r_extent_hooks);
|
||||
|
||||
if ((*r_extent_hooks)->split == NULL)
|
||||
return (NULL);
|
||||
|
||||
trail = extent_alloc(tsdn, arena);
|
||||
if (trail == NULL)
|
||||
goto label_error_a;
|
||||
@ -1237,6 +1315,7 @@ extent_merge_default_impl(void *addr_a, void *addr_b)
|
||||
return (false);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_MAPS_COALESCE
|
||||
static bool
|
||||
extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
|
||||
void *addr_b, size_t size_b, bool committed, unsigned arena_ind)
|
||||
@ -1246,6 +1325,7 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
|
||||
|
||||
return (extent_merge_default_impl(addr_a, addr_b));
|
||||
}
|
||||
#endif
|
||||
|
||||
bool
|
||||
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
@ -1257,6 +1337,10 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
||||
rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
|
||||
|
||||
extent_hooks_assure_initialized(arena, r_extent_hooks);
|
||||
|
||||
if ((*r_extent_hooks)->merge == NULL)
|
||||
return (true);
|
||||
|
||||
if (*r_extent_hooks == &extent_hooks_default) {
|
||||
/* Call directly to propagate tsdn. */
|
||||
err = extent_merge_default_impl(extent_base_get(a),
|
||||
|
||||
@ -168,10 +168,20 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
|
||||
extent_dalloc_gap(tsdn, arena, gap);
|
||||
else
|
||||
extent_dalloc(tsdn, arena, gap);
|
||||
if (*zero)
|
||||
memset(ret, 0, size);
|
||||
if (!*commit)
|
||||
*commit = pages_decommit(ret, size);
|
||||
if (*zero && *commit) {
|
||||
extent_hooks_t *extent_hooks =
|
||||
EXTENT_HOOKS_INITIALIZER;
|
||||
extent_t extent;
|
||||
|
||||
extent_init(&extent, arena, ret, size,
|
||||
size, 0, true, false, true, false);
|
||||
if (extent_purge_forced_wrapper(tsdn,
|
||||
arena, &extent_hooks, &extent, 0,
|
||||
size))
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
return (ret);
|
||||
}
|
||||
/*
|
||||
|
||||
@ -110,6 +110,9 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
|
||||
|
||||
assert(oldusize > usize);
|
||||
|
||||
if (extent_hooks->split == NULL)
|
||||
return (true);
|
||||
|
||||
/* Split excess pages. */
|
||||
if (diff != 0) {
|
||||
extent_t *trail = extent_split_wrapper(tsdn, arena,
|
||||
@ -142,6 +145,9 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
|
||||
size_t trailsize = usize - extent_usize_get(extent);
|
||||
extent_t *trail;
|
||||
|
||||
if (extent_hooks->merge == NULL)
|
||||
return (true);
|
||||
|
||||
if ((trail = arena_extent_cache_alloc(tsdn, arena, &extent_hooks,
|
||||
extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail)) ==
|
||||
NULL) {
|
||||
|
||||
39
src/pages.c
39
src/pages.c
@ -163,33 +163,34 @@ pages_decommit(void *addr, size_t size)
|
||||
}
|
||||
|
||||
bool
|
||||
pages_purge(void *addr, size_t size)
|
||||
pages_purge_lazy(void *addr, size_t size)
|
||||
{
|
||||
bool unzeroed;
|
||||
|
||||
if (!pages_can_purge_lazy)
|
||||
return (true);
|
||||
|
||||
#ifdef _WIN32
|
||||
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
|
||||
unzeroed = true;
|
||||
#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
|
||||
defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
|
||||
# if defined(JEMALLOC_PURGE_MADVISE_FREE)
|
||||
# define JEMALLOC_MADV_PURGE MADV_FREE
|
||||
# define JEMALLOC_MADV_ZEROS false
|
||||
# elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
|
||||
# define JEMALLOC_MADV_PURGE MADV_DONTNEED
|
||||
# define JEMALLOC_MADV_ZEROS true
|
||||
#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
|
||||
madvise(addr, size, MADV_FREE);
|
||||
#else
|
||||
# error No madvise(2) flag defined for purging unused dirty pages
|
||||
not_reached();
|
||||
#endif
|
||||
int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
|
||||
unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
|
||||
# undef JEMALLOC_MADV_PURGE
|
||||
# undef JEMALLOC_MADV_ZEROS
|
||||
return (false);
|
||||
}
|
||||
|
||||
bool
|
||||
pages_purge_forced(void *addr, size_t size)
|
||||
{
|
||||
|
||||
if (!pages_can_purge_forced)
|
||||
return (true);
|
||||
|
||||
#if defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
|
||||
return (madvise(addr, size, MADV_DONTNEED) != 0);
|
||||
#else
|
||||
/* Last resort no-op. */
|
||||
unzeroed = true;
|
||||
not_reached();
|
||||
#endif
|
||||
return (unzeroed);
|
||||
}
|
||||
|
||||
bool
|
||||
|
||||
@ -13,7 +13,9 @@ static bool extent_commit(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
static bool extent_decommit(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
static bool extent_purge(extent_hooks_t *extent_hooks, void *addr,
|
||||
static bool extent_purge_lazy(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
static bool extent_purge_forced(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t offset, size_t length, unsigned arena_ind);
|
||||
static bool extent_split(extent_hooks_t *extent_hooks, void *addr,
|
||||
size_t size, size_t size_a, size_t size_b, bool committed,
|
||||
@ -27,7 +29,8 @@ static extent_hooks_t hooks = {
|
||||
extent_dalloc,
|
||||
extent_commit,
|
||||
extent_decommit,
|
||||
extent_purge,
|
||||
extent_purge_lazy,
|
||||
extent_purge_forced,
|
||||
extent_split,
|
||||
extent_merge
|
||||
};
|
||||
@ -42,7 +45,8 @@ static bool did_alloc;
|
||||
static bool did_dalloc;
|
||||
static bool did_commit;
|
||||
static bool did_decommit;
|
||||
static bool did_purge;
|
||||
static bool did_purge_lazy;
|
||||
static bool did_purge_forced;
|
||||
static bool tried_split;
|
||||
static bool did_split;
|
||||
static bool did_merge;
|
||||
@ -129,7 +133,7 @@ extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
extent_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
size_t offset, size_t length, unsigned arena_ind)
|
||||
{
|
||||
|
||||
@ -138,9 +142,29 @@ extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
offset, length, arena_ind);
|
||||
assert_ptr_eq(extent_hooks, new_hooks,
|
||||
"extent_hooks should be same as pointer used to set hooks");
|
||||
assert_ptr_eq(extent_hooks->purge, extent_purge, "Wrong hook function");
|
||||
did_purge = true;
|
||||
return (old_hooks->purge(old_hooks, addr, size, offset, length,
|
||||
assert_ptr_eq(extent_hooks->purge_lazy, extent_purge_lazy,
|
||||
"Wrong hook function");
|
||||
did_purge_lazy = true;
|
||||
return (old_hooks->purge_lazy == NULL ||
|
||||
old_hooks->purge_lazy(old_hooks, addr, size, offset, length,
|
||||
arena_ind));
|
||||
}
|
||||
|
||||
static bool
|
||||
extent_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
size_t offset, size_t length, unsigned arena_ind)
|
||||
{
|
||||
|
||||
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
|
||||
"length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
|
||||
offset, length, arena_ind);
|
||||
assert_ptr_eq(extent_hooks, new_hooks,
|
||||
"extent_hooks should be same as pointer used to set hooks");
|
||||
assert_ptr_eq(extent_hooks->purge_forced, extent_purge_forced,
|
||||
"Wrong hook function");
|
||||
did_purge_forced = true;
|
||||
return (old_hooks->purge_forced == NULL ||
|
||||
old_hooks->purge_forced(old_hooks, addr, size, offset, length,
|
||||
arena_ind));
|
||||
}
|
||||
|
||||
@ -158,8 +182,8 @@ extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
"extent_hooks should be same as pointer used to set hooks");
|
||||
assert_ptr_eq(extent_hooks->split, extent_split, "Wrong hook function");
|
||||
tried_split = true;
|
||||
err = old_hooks->split(old_hooks, addr, size, size_a, size_b, committed,
|
||||
arena_ind);
|
||||
err = (old_hooks->split == NULL || old_hooks->split(old_hooks, addr,
|
||||
size, size_a, size_b, committed, arena_ind));
|
||||
did_split = !err;
|
||||
return (err);
|
||||
}
|
||||
@ -177,8 +201,8 @@ extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
|
||||
assert_ptr_eq(extent_hooks, new_hooks,
|
||||
"extent_hooks should be same as pointer used to set hooks");
|
||||
assert_ptr_eq(extent_hooks->merge, extent_merge, "Wrong hook function");
|
||||
err = old_hooks->merge(old_hooks, addr_a, size_a, addr_b, size_b,
|
||||
committed, arena_ind);
|
||||
err = (old_hooks->merge == NULL || old_hooks->merge(old_hooks, addr_a,
|
||||
size_a, addr_b, size_b, committed, arena_ind));
|
||||
did_merge = !err;
|
||||
return (err);
|
||||
}
|
||||
@ -216,7 +240,10 @@ TEST_BEGIN(test_extent)
|
||||
"Unexpected commit error");
|
||||
assert_ptr_ne(old_hooks->decommit, extent_decommit,
|
||||
"Unexpected decommit error");
|
||||
assert_ptr_ne(old_hooks->purge, extent_purge, "Unexpected purge error");
|
||||
assert_ptr_ne(old_hooks->purge_lazy, extent_purge_lazy,
|
||||
"Unexpected purge_lazy error");
|
||||
assert_ptr_ne(old_hooks->purge_forced, extent_purge_forced,
|
||||
"Unexpected purge_forced error");
|
||||
assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error");
|
||||
assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error");
|
||||
|
||||
@ -240,7 +267,8 @@ TEST_BEGIN(test_extent)
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() error");
|
||||
did_dalloc = false;
|
||||
did_decommit = false;
|
||||
did_purge = false;
|
||||
did_purge_lazy = false;
|
||||
did_purge_forced = false;
|
||||
tried_split = false;
|
||||
did_split = false;
|
||||
xallocx_success_a = (xallocx(p, large0, 0, flags) == large0);
|
||||
@ -249,7 +277,8 @@ TEST_BEGIN(test_extent)
|
||||
if (xallocx_success_a) {
|
||||
assert_true(did_dalloc, "Expected dalloc");
|
||||
assert_false(did_decommit, "Unexpected decommit");
|
||||
assert_true(did_purge, "Expected purge");
|
||||
assert_true(did_purge_lazy || did_purge_forced,
|
||||
"Expected purge");
|
||||
}
|
||||
assert_true(tried_split, "Expected split");
|
||||
dallocx(p, flags);
|
||||
@ -300,8 +329,10 @@ TEST_BEGIN(test_extent)
|
||||
"Unexpected commit error");
|
||||
assert_ptr_eq(old_hooks->decommit, orig_hooks->decommit,
|
||||
"Unexpected decommit error");
|
||||
assert_ptr_eq(old_hooks->purge, orig_hooks->purge,
|
||||
"Unexpected purge error");
|
||||
assert_ptr_eq(old_hooks->purge_lazy, orig_hooks->purge_lazy,
|
||||
"Unexpected purge_lazy error");
|
||||
assert_ptr_eq(old_hooks->purge_forced, orig_hooks->purge_forced,
|
||||
"Unexpected purge_forced error");
|
||||
assert_ptr_eq(old_hooks->split, orig_hooks->split,
|
||||
"Unexpected split error");
|
||||
assert_ptr_eq(old_hooks->merge, orig_hooks->merge,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user