project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

San: Implement bump alloc

Browse Source 
The new allocator will be used to allocate guarded extents used as slabs
for guarded small allocations.
This commit is contained in:
Alex Lapenkou
2021-11-04 11:10:19 -07:00
committed by Alexander Lapenkov
parent 34b00f8969
commit 0f6da1257d
15 changed files with 521 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
test/include/test/arena_decay.h
View File

@@ -1,4 +1,4 @@
static unsigned
static inline unsigned
do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
unsigned arena_ind;
size_t sz = sizeof(unsigned);
@@ -24,7 +24,7 @@ do_arena_create(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
return arena_ind;
}
static void
static inline void
do_arena_destroy(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -35,14 +35,14 @@ do_arena_destroy(unsigned arena_ind) {
"Unexpected mallctlbymib() failure");
}
static void
static inline void
do_epoch(void) {
uint64_t epoch = 1;
expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
0, "Unexpected mallctl() failure");
}
static void
static inline void
do_purge(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -53,7 +53,7 @@ do_purge(unsigned arena_ind) {
"Unexpected mallctlbymib() failure");
}
static void
static inline void
do_decay(unsigned arena_ind) {
size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -64,7 +64,7 @@ do_decay(unsigned arena_ind) {
"Unexpected mallctlbymib() failure");
}
static uint64_t
static inline uint64_t
get_arena_npurge_impl(const char *mibname, unsigned arena_ind) {
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
@@ -78,32 +78,32 @@ get_arena_npurge_impl(const char *mibname, unsigned arena_ind) {
return npurge;
}
static uint64_t
static inline uint64_t
get_arena_dirty_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind);
}
static uint64_t
static inline uint64_t
get_arena_dirty_purged(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind);
}
static uint64_t
static inline uint64_t
get_arena_muzzy_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static uint64_t
static inline uint64_t
get_arena_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) +
get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static size_t
static inline size_t
get_arena_pdirty(unsigned arena_ind) {
do_epoch();
size_t mib[4];
@@ -118,7 +118,7 @@ get_arena_pdirty(unsigned arena_ind) {
return pdirty;
}
static size_t
static inline size_t
get_arena_pmuzzy(unsigned arena_ind) {
do_epoch();
size_t mib[4];
@@ -133,14 +133,14 @@ get_arena_pmuzzy(unsigned arena_ind) {
return pmuzzy;
}
static void *
static inline void *
do_mallocx(size_t size, int flags) {
void *p = mallocx(size, flags);
expect_ptr_not_null(p, "Unexpected mallocx() failure");
return p;
}
static void
static inline void
generate_dirty(unsigned arena_ind, size_t size) {
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
void *p = do_mallocx(size, flags);

2
test/unit/retained.c
View File

@@ -104,7 +104,7 @@ TEST_BEGIN(test_retained) {
arena_ind = do_arena_create(NULL);
sz = nallocx(HUGEPAGE, 0);
size_t guard_sz = san_enabled() ? PAGE_GUARDS_SIZE : 0;
size_t guard_sz = san_enabled() ? SAN_PAGE_GUARDS_SIZE : 0;
esz = sz + sz_large_pad + guard_sz;
atomic_store_u(&epoch, 0, ATOMIC_RELAXED);

2
test/unit/san.c
View File

@@ -122,7 +122,7 @@ TEST_BEGIN(test_guarded_decay) {
/* Verify that guarded extents as dirty. */
size_t sz1 = PAGE, sz2 = PAGE * 2;
/* W/o maps_coalesce, guarded extents are unguarded eagerly. */
size_t add_guard_size = maps_coalesce ? 0 : PAGE_GUARDS_SIZE;
size_t add_guard_size = maps_coalesce ? 0 : SAN_PAGE_GUARDS_SIZE;
generate_dirty(arena_ind, sz1);
verify_pdirty(arena_ind, sz1 + add_guard_size);
verify_pmuzzy(arena_ind, 0);

111
test/unit/san_bump.c Normal file
View File

@@ -0,0 +1,111 @@
#include "test/jemalloc_test.h"
#include "test/arena_decay.h"
#include "jemalloc/internal/arena_structs.h"
#include "jemalloc/internal/san_bump.h"
TEST_BEGIN(test_san_bump_alloc) {
test_skip_if(!maps_coalesce || !opt_retain);
tsdn_t *tsdn = tsdn_fetch();
san_bump_alloc_t sba;
san_bump_alloc_init(&sba);
unsigned arena_ind = do_arena_create(0, 0);
assert_u_ne(arena_ind, UINT_MAX, "Failed to create an arena");
arena_t *arena = arena_get(tsdn, arena_ind, false);
pac_t *pac = &arena->pa_shard.pac;
size_t alloc_size = PAGE * 16;
size_t alloc_n = alloc_size / sizeof(unsigned);
edata_t* edata = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac),
alloc_size, /* zero */ false);
expect_ptr_not_null(edata, "Failed to allocate edata");
expect_u_eq(edata_arena_ind_get(edata), arena_ind,
"Edata was assigned an incorrect arena id");
expect_zu_eq(edata_size_get(edata), alloc_size,
"Allocated edata of incorrect size");
expect_false(edata_slab_get(edata),
"Bump allocator incorrectly assigned 'slab' to true");
expect_true(edata_committed_get(edata), "Edata is not committed");
void *ptr = edata_addr_get(edata);
expect_ptr_not_null(ptr, "Edata was assigned an invalid address");
/* Test that memory is allocated; no guard pages are misplaced */
for (unsigned i = 0; i < alloc_n; ++i) {
((unsigned *)ptr)[i] = 1;
}
size_t alloc_size2 = PAGE * 28;
size_t alloc_n2 = alloc_size / sizeof(unsigned);
edata_t *edata2 = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac),
alloc_size2, /* zero */ true);
expect_ptr_not_null(edata2, "Failed to allocate edata");
expect_u_eq(edata_arena_ind_get(edata2), arena_ind,
"Edata was assigned an incorrect arena id");
expect_zu_eq(edata_size_get(edata2), alloc_size2,
"Allocated edata of incorrect size");
expect_false(edata_slab_get(edata2),
"Bump allocator incorrectly assigned 'slab' to true");
expect_true(edata_committed_get(edata2), "Edata is not committed");
void *ptr2 = edata_addr_get(edata2);
expect_ptr_not_null(ptr, "Edata was assigned an invalid address");
uintptr_t ptrdiff = ptr2 > ptr ? (uintptr_t)ptr2 - (uintptr_t)ptr
: (uintptr_t)ptr - (uintptr_t)ptr2;
size_t between_allocs = (size_t)ptrdiff - alloc_size;
expect_zu_ge(between_allocs, PAGE,
"Guard page between allocs is missing");
for (unsigned i = 0; i < alloc_n2; ++i) {
expect_u_eq(((unsigned *)ptr2)[i], 0, "Memory is not zeroed");
}
}
TEST_END
TEST_BEGIN(test_large_alloc_size) {
test_skip_if(!maps_coalesce || !opt_retain);
tsdn_t *tsdn = tsdn_fetch();
san_bump_alloc_t sba;
san_bump_alloc_init(&sba);
unsigned arena_ind = do_arena_create(0, 0);
assert_u_ne(arena_ind, UINT_MAX, "Failed to create an arena");
arena_t *arena = arena_get(tsdn, arena_ind, false);
pac_t *pac = &arena->pa_shard.pac;
size_t alloc_size = SBA_RETAINED_ALLOC_SIZE * 2;
edata_t* edata = san_bump_alloc(tsdn, &sba, pac, pac_ehooks_get(pac),
alloc_size, /* zero */ false);
expect_u_eq(edata_arena_ind_get(edata), arena_ind,
"Edata was assigned an incorrect arena id");
expect_zu_eq(edata_size_get(edata), alloc_size,
"Allocated edata of incorrect size");
expect_false(edata_slab_get(edata),
"Bump allocator incorrectly assigned 'slab' to true");
expect_true(edata_committed_get(edata), "Edata is not committed");
void *ptr = edata_addr_get(edata);
expect_ptr_not_null(ptr, "Edata was assigned an invalid address");
/* Test that memory is allocated; no guard pages are misplaced */
for (unsigned i = 0; i < alloc_size / PAGE; ++i) {
*((char *)ptr + PAGE * i) = 1;
}
}
TEST_END
int
main(void) {
return test(
test_san_bump_alloc,
test_large_alloc_size);
}