project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement per arena base allocators.

Browse Source 
Add/rename related mallctls:
- Add stats.arenas.<i>.base .
- Rename stats.arenas.<i>.metadata to stats.arenas.<i>.internal .
- Add stats.arenas.<i>.resident .

Modify the arenas.extend mallctl to take an optional (extent_hooks_t *)
argument so that it is possible for all base allocations to be serviced
by the specified extent hooks.

This resolves #463.
This commit is contained in:
Jason Evans
2016-12-22 16:39:10 -06:00
parent a6e86810d8
commit a0dd3a4483
18 changed files with 957 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
test/integration/extent.c
View File

@@ -71,7 +71,7 @@ extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
assert_ptr_eq(extent_hooks->alloc, extent_alloc, "Wrong hook function");
did_alloc = true;
return (old_hooks->alloc(old_hooks, new_addr, size, alignment, zero,
commit, arena_ind));
commit, 0));
}
static bool
@@ -89,7 +89,7 @@ extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
did_dalloc = true;
if (!do_dalloc)
return (true);
return (old_hooks->dalloc(old_hooks, addr, size, committed, arena_ind));
return (old_hooks->dalloc(old_hooks, addr, size, committed, 0));
}
static bool
@@ -105,8 +105,7 @@ extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size,
"extent_hooks should be same as pointer used to set hooks");
assert_ptr_eq(extent_hooks->commit, extent_commit,
"Wrong hook function");
err = old_hooks->commit(old_hooks, addr, size, offset, length,
arena_ind);
err = old_hooks->commit(old_hooks, addr, size, offset, length, 0);
did_commit = !err;
return (err);
}
@@ -126,8 +125,7 @@ extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size,
"Wrong hook function");
if (!do_decommit)
return (true);
err = old_hooks->decommit(old_hooks, addr, size, offset, length,
arena_ind);
err = old_hooks->decommit(old_hooks, addr, size, offset, length, 0);
did_decommit = !err;
return (err);
}
@@ -146,8 +144,7 @@ extent_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size,
"Wrong hook function");
did_purge_lazy = true;
return (old_hooks->purge_lazy == NULL ||
old_hooks->purge_lazy(old_hooks, addr, size, offset, length,
arena_ind));
old_hooks->purge_lazy(old_hooks, addr, size, offset, length, 0));
}
static bool
@@ -164,8 +161,7 @@ extent_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t size,
"Wrong hook function");
did_purge_forced = true;
return (old_hooks->purge_forced == NULL ||
old_hooks->purge_forced(old_hooks, addr, size, offset, length,
arena_ind));
old_hooks->purge_forced(old_hooks, addr, size, offset, length, 0));
}
static bool
@@ -183,7 +179,7 @@ extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
assert_ptr_eq(extent_hooks->split, extent_split, "Wrong hook function");
tried_split = true;
err = (old_hooks->split == NULL || old_hooks->split(old_hooks, addr,
size, size_a, size_b, committed, arena_ind));
size, size_a, size_b, committed, 0));
did_split = !err;
return (err);
}
@@ -202,51 +198,23 @@ extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
"extent_hooks should be same as pointer used to set hooks");
assert_ptr_eq(extent_hooks->merge, extent_merge, "Wrong hook function");
err = (old_hooks->merge == NULL || old_hooks->merge(old_hooks, addr_a,
size_a, addr_b, size_b, committed, arena_ind));
size_a, addr_b, size_b, committed, 0));
did_merge = !err;
return (err);
}
TEST_BEGIN(test_extent)
static void
test_extent_body(unsigned arena_ind)
{
void *p;
size_t old_size, new_size, large0, large1, large2, sz;
unsigned arena_ind;
size_t large0, large1, large2, sz;
size_t purge_mib[3];
size_t purge_miblen;
int flags;
size_t hooks_mib[3], purge_mib[3];
size_t hooks_miblen, purge_miblen;
bool xallocx_success_a, xallocx_success_b, xallocx_success_c;
sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, &sz, NULL, 0),
0, "Unexpected mallctl() failure");
flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
/* Install custom extent hooks. */
hooks_miblen = sizeof(hooks_mib)/sizeof(size_t);
assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib,
&hooks_miblen), 0, "Unexpected mallctlnametomib() failure");
hooks_mib[1] = (size_t)arena_ind;
old_size = sizeof(extent_hooks_t *);
new_size = sizeof(extent_hooks_t *);
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, (void *)&old_hooks,
&old_size, (void *)&new_hooks, new_size), 0,
"Unexpected extent_hooks error");
orig_hooks = old_hooks;
assert_ptr_ne(old_hooks->alloc, extent_alloc, "Unexpected alloc error");
assert_ptr_ne(old_hooks->dalloc, extent_dalloc,
"Unexpected dalloc error");
assert_ptr_ne(old_hooks->commit, extent_commit,
"Unexpected commit error");
assert_ptr_ne(old_hooks->decommit, extent_decommit,
"Unexpected decommit error");
assert_ptr_ne(old_hooks->purge_lazy, extent_purge_lazy,
"Unexpected purge_lazy error");
assert_ptr_ne(old_hooks->purge_forced, extent_purge_forced,
"Unexpected purge_forced error");
assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error");
assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error");
/* Get large size classes. */
sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large0, &sz, NULL,
@@ -314,6 +282,45 @@ TEST_BEGIN(test_extent)
p = mallocx(42, flags);
assert_ptr_not_null(p, "Unexpected mallocx() error");
dallocx(p, flags);
}
TEST_BEGIN(test_extent_manual_hook)
{
unsigned arena_ind;
size_t old_size, new_size, sz;
size_t hooks_mib[3];
size_t hooks_miblen;
sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, &sz, NULL, 0),
0, "Unexpected mallctl() failure");
/* Install custom extent hooks. */
hooks_miblen = sizeof(hooks_mib)/sizeof(size_t);
assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib,
&hooks_miblen), 0, "Unexpected mallctlnametomib() failure");
hooks_mib[1] = (size_t)arena_ind;
old_size = sizeof(extent_hooks_t *);
new_size = sizeof(extent_hooks_t *);
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, (void *)&old_hooks,
&old_size, (void *)&new_hooks, new_size), 0,
"Unexpected extent_hooks error");
orig_hooks = old_hooks;
assert_ptr_ne(old_hooks->alloc, extent_alloc, "Unexpected alloc error");
assert_ptr_ne(old_hooks->dalloc, extent_dalloc,
"Unexpected dalloc error");
assert_ptr_ne(old_hooks->commit, extent_commit,
"Unexpected commit error");
assert_ptr_ne(old_hooks->decommit, extent_decommit,
"Unexpected decommit error");
assert_ptr_ne(old_hooks->purge_lazy, extent_purge_lazy,
"Unexpected purge_lazy error");
assert_ptr_ne(old_hooks->purge_forced, extent_purge_forced,
"Unexpected purge_forced error");
assert_ptr_ne(old_hooks->split, extent_split, "Unexpected split error");
assert_ptr_ne(old_hooks->merge, extent_merge, "Unexpected merge error");
test_extent_body(arena_ind);
/* Restore extent hooks. */
assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, NULL, NULL,
@@ -340,9 +347,25 @@ TEST_BEGIN(test_extent)
}
TEST_END
TEST_BEGIN(test_extent_auto_hook)
{
unsigned arena_ind;
size_t new_size, sz;
sz = sizeof(unsigned);
new_size = sizeof(extent_hooks_t *);
assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, &sz,
(void *)&new_hooks, new_size), 0, "Unexpected mallctl() failure");
test_extent_body(arena_ind);
}
TEST_END
int
main(void)
{
return (test(test_extent));
return (test(
test_extent_manual_hook,
test_extent_auto_hook));
}

274
test/unit/base.c Normal file
View File

@@ -0,0 +1,274 @@
#include "test/jemalloc_test.h"
static void *extent_alloc_hook(extent_hooks_t *extent_hooks, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit,
unsigned arena_ind);
static bool extent_dalloc_hook(extent_hooks_t *extent_hooks, void *addr,
size_t size, bool committed, unsigned arena_ind);
static bool extent_decommit_hook(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_purge_lazy_hook(extent_hooks_t *extent_hooks, void *addr,
size_t size, size_t offset, size_t length, unsigned arena_ind);
static bool extent_purge_forced_hook(extent_hooks_t *extent_hooks,
void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
static extent_hooks_t hooks_not_null = {
extent_alloc_hook,
extent_dalloc_hook,
NULL, /* commit */
extent_decommit_hook,
extent_purge_lazy_hook,
extent_purge_forced_hook,
NULL, /* split */
NULL /* merge */
};
static extent_hooks_t hooks_null = {
extent_alloc_hook,
NULL, /* dalloc */
NULL, /* commit */
NULL, /* decommit */
NULL, /* purge_lazy */
NULL, /* purge_forced */
NULL, /* split */
NULL /* merge */
};
static bool did_alloc;
static bool did_dalloc;
static bool did_decommit;
static bool did_purge_lazy;
static bool did_purge_forced;
#if 0
# define TRACE_HOOK(fmt, ...) malloc_printf(fmt, __VA_ARGS__)
#else
# define TRACE_HOOK(fmt, ...)
#endif
static void *
extent_alloc_hook(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind)
{
TRACE_HOOK("%s(extent_hooks=%p, new_addr=%p, size=%zu, alignment=%zu, "
"*zero=%s, *commit=%s, arena_ind=%u)\n", __func__, extent_hooks,
new_addr, size, alignment, *zero ? "true" : "false", *commit ?
"true" : "false", arena_ind);
did_alloc = true;
return (extent_hooks_default.alloc(
(extent_hooks_t *)&extent_hooks_default, new_addr, size, alignment,
zero, commit, 0));
}
static bool
extent_dalloc_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind)
{
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, committed=%s, "
"arena_ind=%u)\n", __func__, extent_hooks, addr, size, committed ?
"true" : "false", arena_ind);
did_dalloc = true;
return (true); /* Cause cascade. */
}
static bool
extent_decommit_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size,
offset, length, arena_ind);
did_decommit = true;
return (true); /* Cause cascade. */
}
static bool
extent_purge_lazy_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
offset, length, arena_ind);
did_purge_lazy = true;
return (true); /* Cause cascade. */
}
static bool
extent_purge_forced_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind)
{
TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
"length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
offset, length, arena_ind);
did_purge_forced = true;
return (true); /* Cause cascade. */
}
TEST_BEGIN(test_base_hooks_default)
{
tsdn_t *tsdn;
base_t *base;
size_t allocated0, allocated1, resident, mapped;
tsdn = tsdn_fetch();
base = base_new(tsdn, 0, (extent_hooks_t *)&extent_hooks_default);
base_stats_get(tsdn, base, &allocated0, &resident, &mapped);
assert_zu_ge(allocated0, sizeof(base_t),
"Base header should count as allocated");
assert_ptr_not_null(base_alloc(tsdn, base, 42, 1),
"Unexpected base_alloc() failure");
base_stats_get(tsdn, base, &allocated1, &resident, &mapped);
assert_zu_ge(allocated1 - allocated0, 42,
"At least 42 bytes were allocated by base_alloc()");
base_delete(base);
}
TEST_END
TEST_BEGIN(test_base_hooks_null)
{
tsdn_t *tsdn;
base_t *base;
size_t allocated0, allocated1, resident, mapped;
tsdn = tsdn_fetch();
base = base_new(tsdn, 0, (extent_hooks_t *)&hooks_null);
assert_ptr_not_null(base, "Unexpected base_new() failure");
base_stats_get(tsdn, base, &allocated0, &resident, &mapped);
assert_zu_ge(allocated0, sizeof(base_t),
"Base header should count as allocated");
assert_ptr_not_null(base_alloc(tsdn, base, 42, 1),
"Unexpected base_alloc() failure");
base_stats_get(tsdn, base, &allocated1, &resident, &mapped);
assert_zu_ge(allocated1 - allocated0, 42,
"At least 42 bytes were allocated by base_alloc()");
base_delete(base);
}
TEST_END
TEST_BEGIN(test_base_hooks_not_null)
{
tsdn_t *tsdn;
base_t *base;
void *p, *q, *r, *r_exp;
tsdn = tsdn_fetch();
did_alloc = false;
base = base_new(tsdn, 0, (extent_hooks_t *)&hooks_not_null);
assert_ptr_not_null(base, "Unexpected base_new() failure");
assert_true(did_alloc, "Expected alloc hook call");
/*
* Check for tight packing at specified alignment under simple
* conditions.
*/
{
const size_t alignments[] = {
1,
QUANTUM,
QUANTUM << 1,
CACHELINE,
CACHELINE << 1,
};
unsigned i;
for (i = 0; i < sizeof(alignments) / sizeof(size_t); i++) {
size_t alignment = alignments[i];
size_t align_ceil = ALIGNMENT_CEILING(alignment,
QUANTUM);
p = base_alloc(tsdn, base, 1, alignment);
assert_ptr_not_null(p,
"Unexpected base_alloc() failure");
assert_ptr_eq(p,
(void *)(ALIGNMENT_CEILING((uintptr_t)p,
alignment)), "Expected quantum alignment");
q = base_alloc(tsdn, base, alignment, alignment);
assert_ptr_not_null(q,
"Unexpected base_alloc() failure");
assert_ptr_eq((void *)((uintptr_t)p + align_ceil), q,
"Minimal allocation should take up %zu bytes",
align_ceil);
r = base_alloc(tsdn, base, 1, alignment);
assert_ptr_not_null(r,
"Unexpected base_alloc() failure");
assert_ptr_eq((void *)((uintptr_t)q + align_ceil), r,
"Minimal allocation should take up %zu bytes",
align_ceil);
}
}
/*
* Allocate an object that cannot fit in the first block, then verify
* that the first block's remaining space is considered for subsequent
* allocation.
*/
assert_zu_ge(extent_size_get(&base->blocks->extent), QUANTUM,
"Remainder insufficient for test");
/* Use up all but one quantum of block. */
while (extent_size_get(&base->blocks->extent) > QUANTUM) {
p = base_alloc(tsdn, base, QUANTUM, QUANTUM);
assert_ptr_not_null(p, "Unexpected base_alloc() failure");
}
r_exp = extent_addr_get(&base->blocks->extent);
assert_zu_eq(base->extent_sn_next, 1, "One extant block expected");
q = base_alloc(tsdn, base, QUANTUM + 1, QUANTUM);
assert_ptr_not_null(q, "Unexpected base_alloc() failure");
assert_ptr_ne(q, r_exp, "Expected allocation from new block");
assert_zu_eq(base->extent_sn_next, 2, "Two extant blocks expected");
r = base_alloc(tsdn, base, QUANTUM, QUANTUM);
assert_ptr_not_null(r, "Unexpected base_alloc() failure");
assert_ptr_eq(r, r_exp, "Expected allocation from first block");
assert_zu_eq(base->extent_sn_next, 2, "Two extant blocks expected");
/*
* Check for proper alignment support when normal blocks are too small.
*/
{
const size_t alignments[] = {
HUGEPAGE,
HUGEPAGE << 1
};
unsigned i;
for (i = 0; i < sizeof(alignments) / sizeof(size_t); i++) {
size_t alignment = alignments[i];
p = base_alloc(tsdn, base, QUANTUM, alignment);
assert_ptr_not_null(p,
"Unexpected base_alloc() failure");
assert_ptr_eq(p,
(void *)(ALIGNMENT_CEILING((uintptr_t)p,
alignment)), "Expected %zu-byte alignment",
alignment);
}
}
did_dalloc = did_decommit = did_purge_lazy = did_purge_forced = false;
base_delete(base);
assert_true(did_dalloc, "Expected dalloc hook call");
assert_true(did_decommit, "Expected decommit hook call");
assert_true(did_purge_lazy, "Expected purge_lazy hook call");
assert_true(did_purge_forced, "Expected purge_forced hook call");
}
TEST_END
int
main(void)
{
return (test(
test_base_hooks_default,
test_base_hooks_null,
test_base_hooks_not_null));
}