Implement dynamic per arena control over dirty page purging.

Add mallctls:
- arenas.lg_dirty_mult is initialized via opt.lg_dirty_mult, and can be
  modified to change the initial lg_dirty_mult setting for newly created
  arenas.
- arena.<i>.lg_dirty_mult controls an individual arena's dirty page
  purging threshold, and synchronously triggers any purging that may be
  necessary to maintain the constraint.
- arena.<i>.chunk.purge allows the per arena dirty page purging function
  to be replaced.

This resolves #93.
This commit is contained in:
Jason Evans
2015-03-18 18:55:33 -07:00
parent c9db461ffb
commit 8d6a3e8321
13 changed files with 460 additions and 99 deletions

View File

@@ -5,6 +5,7 @@
/* Data. */
ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
static ssize_t lg_dirty_mult_default;
arena_bin_info_t arena_bin_info[NBINS];
size_t map_bias;
@@ -1032,15 +1033,49 @@ arena_run_alloc_small(arena_t *arena, size_t size, index_t binind)
return (arena_run_alloc_small_helper(arena, size, binind));
}
static bool
arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult)
{
return (lg_dirty_mult >= -1 && lg_dirty_mult < (sizeof(size_t) << 3));
}
ssize_t
arena_lg_dirty_mult_get(arena_t *arena)
{
ssize_t lg_dirty_mult;
malloc_mutex_lock(&arena->lock);
lg_dirty_mult = arena->lg_dirty_mult;
malloc_mutex_unlock(&arena->lock);
return (lg_dirty_mult);
}
bool
arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult)
{
if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
return (true);
malloc_mutex_lock(&arena->lock);
arena->lg_dirty_mult = lg_dirty_mult;
arena_maybe_purge(arena);
malloc_mutex_unlock(&arena->lock);
return (false);
}
void
arena_maybe_purge(arena_t *arena)
{
size_t threshold;
/* Don't purge if the option is disabled. */
if (opt_lg_dirty_mult < 0)
if (arena->lg_dirty_mult < 0)
return;
threshold = (arena->nactive >> opt_lg_dirty_mult);
threshold = (arena->nactive >> arena->lg_dirty_mult);
threshold = threshold < chunk_npages ? chunk_npages : threshold;
/*
* Don't purge unless the number of purgeable pages exceeds the
@@ -1096,7 +1131,7 @@ arena_compute_npurge(arena_t *arena, bool all)
* purge.
*/
if (!all) {
size_t threshold = (arena->nactive >> opt_lg_dirty_mult);
size_t threshold = (arena->nactive >> arena->lg_dirty_mult);
threshold = threshold < chunk_npages ? chunk_npages : threshold;
npurge = arena->ndirty - threshold;
@@ -1192,6 +1227,7 @@ arena_purge_stashed(arena_t *arena,
extent_node_t *purge_chunks_sentinel)
{
size_t npurged, nmadvise;
chunk_purge_t *chunk_purge;
arena_runs_dirty_link_t *rdelm;
extent_node_t *chunkselm;
@@ -1199,6 +1235,7 @@ arena_purge_stashed(arena_t *arena,
nmadvise = 0;
npurged = 0;
chunk_purge = arena->chunk_purge;
malloc_mutex_unlock(&arena->lock);
for (rdelm = qr_next(purge_runs_sentinel, rd_link),
chunkselm = qr_next(purge_chunks_sentinel, cc_link);
@@ -1207,11 +1244,16 @@ arena_purge_stashed(arena_t *arena,
if (rdelm == &chunkselm->rd) {
size_t size = extent_node_size_get(chunkselm);
void *addr, *chunk;
size_t offset;
bool unzeroed;
npages = size >> LG_PAGE;
unzeroed = pages_purge(extent_node_addr_get(chunkselm),
size);
addr = extent_node_addr_get(chunkselm);
chunk = CHUNK_ADDR2BASE(addr);
offset = CHUNK_ADDR2OFFSET(addr);
unzeroed = chunk_purge_wrapper(arena, chunk_purge,
chunk, offset, size);
extent_node_zeroed_set(chunkselm, !unzeroed);
chunkselm = qr_next(chunkselm, cc_link);
} else {
@@ -1226,15 +1268,15 @@ arena_purge_stashed(arena_t *arena,
npages = run_size >> LG_PAGE;
assert(pageind + npages <= chunk_npages);
unzeroed = pages_purge((void *)((uintptr_t)chunk +
(pageind << LG_PAGE)), run_size);
unzeroed = chunk_purge_wrapper(arena, chunk_purge,
chunk, pageind << LG_PAGE, run_size);
flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0;
/*
* Set the unzeroed flag for all pages, now that
* pages_purge() has returned whether the pages were
* zeroed as a side effect of purging. This chunk map
* modification is safe even though the arena mutex
* chunk_purge_wrapper() has returned whether the pages
* were zeroed as a side effect of purging. This chunk
* map modification is safe even though the arena mutex
* isn't currently owned by this thread, because the run
* is marked as allocated, thus protecting it from being
* modified by any other thread. As long as these
@@ -1294,7 +1336,7 @@ arena_unstash_purged(arena_t *arena,
}
}
void
static void
arena_purge(arena_t *arena, bool all)
{
size_t npurge, npurgeable, npurged;
@@ -1309,7 +1351,7 @@ arena_purge(arena_t *arena, bool all)
size_t ndirty = arena_dirty_count(arena);
assert(ndirty == arena->ndirty);
}
assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all);
assert((arena->nactive >> arena->lg_dirty_mult) < arena->ndirty || all);
if (config_stats)
arena->stats.npurge++;
@@ -2596,6 +2638,23 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec)
return (false);
}
ssize_t
arena_lg_dirty_mult_default_get(void)
{
return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default));
}
bool
arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult)
{
if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
return (true);
atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult);
return (false);
}
void
arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive,
size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats,
@@ -2702,6 +2761,7 @@ arena_new(unsigned ind)
arena->spare = NULL;
arena->lg_dirty_mult = arena_lg_dirty_mult_default_get();
arena->nactive = 0;
arena->ndirty = 0;
@@ -2727,6 +2787,7 @@ arena_new(unsigned ind)
arena->chunk_alloc = chunk_alloc_default;
arena->chunk_dalloc = chunk_dalloc_default;
arena->chunk_purge = chunk_purge_default;
/* Initialize bins. */
for (i = 0; i < NBINS; i++) {
@@ -2860,6 +2921,8 @@ arena_boot(void)
size_t header_size;
unsigned i;
arena_lg_dirty_mult_default_set(opt_lg_dirty_mult);
/*
* Compute the header size such that it is large enough to contain the
* page map. The page map is biased to omit entries for the header

View File

@@ -391,8 +391,10 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad,
* pages have already been purged, so that this is only
* a virtual memory leak.
*/
if (cache)
pages_purge(chunk, size);
if (cache) {
chunk_purge_wrapper(arena, arena->chunk_purge,
chunk, 0, size);
}
goto label_return;
}
extent_node_init(node, arena, chunk, size, !unzeroed);
@@ -485,6 +487,37 @@ chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc, void *chunk,
JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size);
}
bool
chunk_purge_arena(arena_t *arena, void *chunk, size_t offset, size_t length)
{
assert(chunk != NULL);
assert(CHUNK_ADDR2BASE(chunk) == chunk);
assert((offset & PAGE_MASK) == 0);
assert(length != 0);
assert((length & PAGE_MASK) == 0);
return (pages_purge((void *)((uintptr_t)chunk + (uintptr_t)offset),
length));
}
bool
chunk_purge_default(void *chunk, size_t offset, size_t length,
unsigned arena_ind)
{
return (chunk_purge_arena(chunk_arena_get(arena_ind), chunk, offset,
length));
}
bool
chunk_purge_wrapper(arena_t *arena, chunk_purge_t *chunk_purge, void *chunk,
size_t offset, size_t length)
{
return (chunk_purge(chunk, offset, length, arena->ind));
}
static rtree_node_elm_t *
chunks_rtree_node_alloc(size_t nelms)
{

121
src/ctl.c
View File

@@ -116,8 +116,10 @@ CTL_PROTO(tcache_destroy)
CTL_PROTO(arena_i_purge)
static void arena_purge(unsigned arena_ind);
CTL_PROTO(arena_i_dss)
CTL_PROTO(arena_i_lg_dirty_mult)
CTL_PROTO(arena_i_chunk_alloc)
CTL_PROTO(arena_i_chunk_dalloc)
CTL_PROTO(arena_i_chunk_purge)
INDEX_PROTO(arena_i)
CTL_PROTO(arenas_bin_i_size)
CTL_PROTO(arenas_bin_i_nregs)
@@ -129,6 +131,7 @@ CTL_PROTO(arenas_hchunk_i_size)
INDEX_PROTO(arenas_hchunk_i)
CTL_PROTO(arenas_narenas)
CTL_PROTO(arenas_initialized)
CTL_PROTO(arenas_lg_dirty_mult)
CTL_PROTO(arenas_quantum)
CTL_PROTO(arenas_page)
CTL_PROTO(arenas_tcache_max)
@@ -283,12 +286,14 @@ static const ctl_named_node_t tcache_node[] = {
static const ctl_named_node_t chunk_node[] = {
{NAME("alloc"), CTL(arena_i_chunk_alloc)},
{NAME("dalloc"), CTL(arena_i_chunk_dalloc)}
{NAME("dalloc"), CTL(arena_i_chunk_dalloc)},
{NAME("purge"), CTL(arena_i_chunk_purge)}
};
static const ctl_named_node_t arena_i_node[] = {
{NAME("purge"), CTL(arena_i_purge)},
{NAME("dss"), CTL(arena_i_dss)},
{NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)},
{NAME("chunk"), CHILD(named, chunk)},
};
static const ctl_named_node_t super_arena_i_node[] = {
@@ -337,6 +342,7 @@ static const ctl_indexed_node_t arenas_hchunk_node[] = {
static const ctl_named_node_t arenas_node[] = {
{NAME("narenas"), CTL(arenas_narenas)},
{NAME("initialized"), CTL(arenas_initialized)},
{NAME("lg_dirty_mult"), CTL(arenas_lg_dirty_mult)},
{NAME("quantum"), CTL(arenas_quantum)},
{NAME("page"), CTL(arenas_page)},
{NAME("tcache_max"), CTL(arenas_tcache_max)},
@@ -1617,57 +1623,70 @@ label_return:
}
static int
arena_i_chunk_alloc_ctl(const size_t *mib, size_t miblen, void *oldp,
arena_i_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned arena_ind = mib[1];
arena_t *arena;
malloc_mutex_lock(&ctl_mtx);
if (arena_ind < narenas_total_get() && (arena = arena_get(tsd_fetch(),
arena_ind, false, true)) != NULL) {
malloc_mutex_lock(&arena->lock);
READ(arena->chunk_alloc, chunk_alloc_t *);
WRITE(arena->chunk_alloc, chunk_alloc_t *);
} else {
arena = arena_get(tsd_fetch(), arena_ind, false, (arena_ind == 0));
if (arena == NULL) {
ret = EFAULT;
goto label_outer_return;
goto label_return;
}
if (oldp != NULL && oldlenp != NULL) {
size_t oldval = arena_lg_dirty_mult_get(arena);
READ(oldval, ssize_t);
}
if (newp != NULL) {
if (newlen != sizeof(ssize_t)) {
ret = EINVAL;
goto label_return;
}
if (arena_lg_dirty_mult_set(arena, *(ssize_t *)newp)) {
ret = EFAULT;
goto label_return;
}
}
ret = 0;
label_return:
malloc_mutex_unlock(&arena->lock);
label_outer_return:
malloc_mutex_unlock(&ctl_mtx);
return (ret);
}
static int
arena_i_chunk_dalloc_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned arena_ind = mib[1];
arena_t *arena;
malloc_mutex_lock(&ctl_mtx);
if (arena_ind < narenas_total_get() && (arena = arena_get(tsd_fetch(),
arena_ind, false, true)) != NULL) {
malloc_mutex_lock(&arena->lock);
READ(arena->chunk_dalloc, chunk_dalloc_t *);
WRITE(arena->chunk_dalloc, chunk_dalloc_t *);
} else {
ret = EFAULT;
goto label_outer_return;
}
ret = 0;
label_return:
malloc_mutex_unlock(&arena->lock);
label_outer_return:
malloc_mutex_unlock(&ctl_mtx);
return (ret);
#define CHUNK_FUNC(n) \
static int \
arena_i_chunk_##n##_ctl(const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen) \
{ \
\
int ret; \
unsigned arena_ind = mib[1]; \
arena_t *arena; \
\
malloc_mutex_lock(&ctl_mtx); \
if (arena_ind < narenas_total_get() && (arena = \
arena_get(tsd_fetch(), arena_ind, false, true)) != NULL) { \
malloc_mutex_lock(&arena->lock); \
READ(arena->chunk_##n, chunk_##n##_t *); \
WRITE(arena->chunk_##n, chunk_##n##_t *); \
} else { \
ret = EFAULT; \
goto label_outer_return; \
} \
ret = 0; \
label_return: \
malloc_mutex_unlock(&arena->lock); \
label_outer_return: \
malloc_mutex_unlock(&ctl_mtx); \
return (ret); \
}
CHUNK_FUNC(alloc)
CHUNK_FUNC(dalloc)
CHUNK_FUNC(purge)
#undef CHUNK_FUNC
static const ctl_named_node_t *
arena_i_index(const size_t *mib, size_t miblen, size_t i)
@@ -1736,6 +1755,32 @@ label_return:
return (ret);
}
static int
arenas_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
if (oldp != NULL && oldlenp != NULL) {
size_t oldval = arena_lg_dirty_mult_default_get();
READ(oldval, ssize_t);
}
if (newp != NULL) {
if (newlen != sizeof(ssize_t)) {
ret = EINVAL;
goto label_return;
}
if (arena_lg_dirty_mult_default_set(*(ssize_t *)newp)) {
ret = EFAULT;
goto label_return;
}
}
ret = 0;
label_return:
return (ret);
}
CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t)

View File

@@ -124,9 +124,10 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
size_t size, size_t extra, bool zero)
{
size_t usize_next;
bool zeroed;
extent_node_t *node;
arena_t *arena;
chunk_purge_t *chunk_purge;
bool zeroed;
/* Increase usize to incorporate extra. */
while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize)
@@ -135,11 +136,18 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
if (oldsize == usize)
return;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
malloc_mutex_lock(&arena->lock);
chunk_purge = arena->chunk_purge;
malloc_mutex_unlock(&arena->lock);
/* Fill if necessary (shrinking). */
if (oldsize > usize) {
size_t sdiff = CHUNK_CEILING(usize) - usize;
zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr +
usize), sdiff) : true;
zeroed = (sdiff != 0) ? !chunk_purge_wrapper(arena, chunk_purge,
CHUNK_ADDR2BASE(ptr), CHUNK_ADDR2OFFSET(ptr), usize) : true;
if (config_fill && unlikely(opt_junk_free)) {
memset((void *)((uintptr_t)ptr + usize), 0x5a, oldsize -
usize);
@@ -148,8 +156,6 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
} else
zeroed = true;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
malloc_mutex_lock(&arena->huge_mtx);
/* Update the size of the huge allocation. */
assert(extent_node_size_get(node) != usize);
@@ -177,22 +183,29 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize,
static void
huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
{
size_t sdiff;
bool zeroed;
extent_node_t *node;
arena_t *arena;
chunk_purge_t *chunk_purge;
size_t sdiff;
bool zeroed;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
malloc_mutex_lock(&arena->lock);
chunk_purge = arena->chunk_purge;
malloc_mutex_unlock(&arena->lock);
sdiff = CHUNK_CEILING(usize) - usize;
zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + usize),
sdiff) : true;
zeroed = (sdiff != 0) ? !chunk_purge_wrapper(arena, chunk_purge,
CHUNK_ADDR2BASE((uintptr_t)ptr + usize),
CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff) : true;
if (config_fill && unlikely(opt_junk_free)) {
huge_dalloc_junk((void *)((uintptr_t)ptr + usize), oldsize -
usize);
zeroed = false;
}
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
malloc_mutex_lock(&arena->huge_mtx);
/* Update the size of the huge allocation. */
extent_node_size_set(node, usize);
@@ -291,8 +304,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
}
/* Attempt to expand the allocation in-place. */
if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra,
zero)) {
if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, zero)) {
if (extra == 0)
return (true);

View File

@@ -264,6 +264,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
{
unsigned nthreads;
const char *dss;
ssize_t lg_dirty_mult;
size_t page, pactive, pdirty, mapped;
size_t metadata_mapped, metadata_allocated;
uint64_t npurge, nmadvise, purged;
@@ -282,6 +283,15 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_I_GET("stats.arenas.0.dss", &dss, const char *);
malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n",
dss);
CTL_I_GET("stats.arenas.0.lg_dirty_mult", &lg_dirty_mult, ssize_t);
if (lg_dirty_mult >= 0) {
malloc_cprintf(write_cb, cbopaque,
"Min active:dirty page ratio: %u:1\n",
(1U << lg_dirty_mult));
} else {
malloc_cprintf(write_cb, cbopaque,
"Min active:dirty page ratio: N/A\n");
}
CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t);
CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t);
CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t);