project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

edata_cache: Allow unbounded fast caching.

Browse Source 
The edata_cache_small had a fill/flush heuristic.  In retrospect, this was a
premature optimization; more testing indicates that an unbounded cache is
effectively fine here, and moreover we spend a nontrivial amount of time doing
unnecessary filling/flushing.

As the HPA takes on a larger and larger fraction of all allocations, any
theoretical differences in allocation patterns should shrink.  The HPA is more
efficient with its metadata in general, so it still comes out ahead on metadata
usage anyways.
This commit is contained in:
David Goldblatt
2021-07-23 15:29:43 -07:00
committed by David Goldblatt
parent d93eef2f40
commit 92a1e38f52
5 changed files with 99 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
test/unit/edata_cache.c
View File

@@ -47,38 +47,48 @@ TEST_BEGIN(test_edata_cache) {
}
TEST_END
TEST_BEGIN(test_edata_cache_small_simple) {
static size_t
ecf_count(edata_cache_fast_t *ecf) {
size_t count = 0;
edata_t *cur;
ql_foreach(cur, &ecf->list.head, ql_link_inactive) {
count++;
}
return count;
}
TEST_BEGIN(test_edata_cache_fast_simple) {
edata_cache_t ec;
edata_cache_small_t ecs;
edata_cache_fast_t ecf;
test_edata_cache_init(&ec);
edata_cache_small_init(&ecs, &ec);
edata_cache_fast_init(&ecf, &ec);
edata_t *ed1 = edata_cache_small_get(TSDN_NULL, &ecs);
edata_t *ed1 = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_ptr_not_null(ed1, "");
expect_zu_eq(ecs.count, 0, "");
expect_zu_eq(ecf_count(&ecf), 0, "");
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
edata_t *ed2 = edata_cache_small_get(TSDN_NULL, &ecs);
edata_t *ed2 = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_ptr_not_null(ed2, "");
expect_zu_eq(ecs.count, 0, "");
expect_zu_eq(ecf_count(&ecf), 0, "");
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
edata_cache_small_put(TSDN_NULL, &ecs, ed1);
expect_zu_eq(ecs.count, 1, "");
edata_cache_fast_put(TSDN_NULL, &ecf, ed1);
expect_zu_eq(ecf_count(&ecf), 1, "");
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
edata_cache_small_put(TSDN_NULL, &ecs, ed2);
expect_zu_eq(ecs.count, 2, "");
edata_cache_fast_put(TSDN_NULL, &ecf, ed2);
expect_zu_eq(ecf_count(&ecf), 2, "");
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
/* LIFO ordering. */
expect_ptr_eq(ed2, edata_cache_small_get(TSDN_NULL, &ecs), "");
expect_zu_eq(ecs.count, 1, "");
expect_ptr_eq(ed2, edata_cache_fast_get(TSDN_NULL, &ecf), "");
expect_zu_eq(ecf_count(&ecf), 1, "");
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
expect_ptr_eq(ed1, edata_cache_small_get(TSDN_NULL, &ecs), "");
expect_zu_eq(ecs.count, 0, "");
expect_ptr_eq(ed1, edata_cache_fast_get(TSDN_NULL, &ecf), "");
expect_zu_eq(ecf_count(&ecf), 0, "");
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
test_edata_cache_destroy(&ec);
@@ -87,41 +97,41 @@ TEST_END
TEST_BEGIN(test_edata_cache_fill) {
edata_cache_t ec;
edata_cache_small_t ecs;
edata_cache_fast_t ecf;
test_edata_cache_init(&ec);
edata_cache_small_init(&ecs, &ec);
edata_cache_fast_init(&ecf, &ec);
edata_t *allocs[EDATA_CACHE_SMALL_FILL * 2];
edata_t *allocs[EDATA_CACHE_FAST_FILL * 2];
/*
* If the fallback cache can't satisfy the request, we shouldn't do
* extra allocations until compelled to. Put half the fill goal in the
* fallback.
*/
for (int i = 0; i < EDATA_CACHE_SMALL_FILL / 2; i++) {
for (int i = 0; i < EDATA_CACHE_FAST_FILL / 2; i++) {
allocs[i] = edata_cache_get(TSDN_NULL, &ec);
}
for (int i = 0; i < EDATA_CACHE_SMALL_FILL / 2; i++) {
for (int i = 0; i < EDATA_CACHE_FAST_FILL / 2; i++) {
edata_cache_put(TSDN_NULL, &ec, allocs[i]);
}
expect_zu_eq(EDATA_CACHE_SMALL_FILL / 2,
expect_zu_eq(EDATA_CACHE_FAST_FILL / 2,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
allocs[0] = edata_cache_small_get(TSDN_NULL, &ecs);
expect_zu_eq(EDATA_CACHE_SMALL_FILL / 2 - 1, ecs.count,
allocs[0] = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_zu_eq(EDATA_CACHE_FAST_FILL / 2 - 1, ecf_count(&ecf),
"Should have grabbed all edatas available but no more.");
for (int i = 1; i < EDATA_CACHE_SMALL_FILL / 2; i++) {
allocs[i] = edata_cache_small_get(TSDN_NULL, &ecs);
for (int i = 1; i < EDATA_CACHE_FAST_FILL / 2; i++) {
allocs[i] = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_ptr_not_null(allocs[i], "");
}
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(0, ecf_count(&ecf), "");
/* When forced, we should alloc from the base. */
edata_t *edata = edata_cache_small_get(TSDN_NULL, &ecs);
edata_t *edata = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_ptr_not_null(edata, "");
expect_zu_eq(0, ecs.count, "Allocated more than necessary");
expect_zu_eq(0, ecf_count(&ecf), "Allocated more than necessary");
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED),
"Allocated more than necessary");
@@ -129,116 +139,78 @@ TEST_BEGIN(test_edata_cache_fill) {
* We should correctly fill in the common case where the fallback isn't
* exhausted, too.
*/
for (int i = 0; i < EDATA_CACHE_SMALL_FILL * 2; i++) {
for (int i = 0; i < EDATA_CACHE_FAST_FILL * 2; i++) {
allocs[i] = edata_cache_get(TSDN_NULL, &ec);
expect_ptr_not_null(allocs[i], "");
}
for (int i = 0; i < EDATA_CACHE_SMALL_FILL * 2; i++) {
for (int i = 0; i < EDATA_CACHE_FAST_FILL * 2; i++) {
edata_cache_put(TSDN_NULL, &ec, allocs[i]);
}
allocs[0] = edata_cache_small_get(TSDN_NULL, &ecs);
expect_zu_eq(EDATA_CACHE_SMALL_FILL - 1, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_FILL,
allocs[0] = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_zu_eq(EDATA_CACHE_FAST_FILL - 1, ecf_count(&ecf), "");
expect_zu_eq(EDATA_CACHE_FAST_FILL,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
for (int i = 1; i < EDATA_CACHE_SMALL_FILL; i++) {
expect_zu_eq(EDATA_CACHE_SMALL_FILL - i, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_FILL,
for (int i = 1; i < EDATA_CACHE_FAST_FILL; i++) {
expect_zu_eq(EDATA_CACHE_FAST_FILL - i, ecf_count(&ecf), "");
expect_zu_eq(EDATA_CACHE_FAST_FILL,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
allocs[i] = edata_cache_small_get(TSDN_NULL, &ecs);
allocs[i] = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_ptr_not_null(allocs[i], "");
}
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_FILL,
expect_zu_eq(0, ecf_count(&ecf), "");
expect_zu_eq(EDATA_CACHE_FAST_FILL,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
allocs[0] = edata_cache_small_get(TSDN_NULL, &ecs);
expect_zu_eq(EDATA_CACHE_SMALL_FILL - 1, ecs.count, "");
allocs[0] = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_zu_eq(EDATA_CACHE_FAST_FILL - 1, ecf_count(&ecf), "");
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
for (int i = 1; i < EDATA_CACHE_SMALL_FILL; i++) {
expect_zu_eq(EDATA_CACHE_SMALL_FILL - i, ecs.count, "");
for (int i = 1; i < EDATA_CACHE_FAST_FILL; i++) {
expect_zu_eq(EDATA_CACHE_FAST_FILL - i, ecf_count(&ecf), "");
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
allocs[i] = edata_cache_small_get(TSDN_NULL, &ecs);
allocs[i] = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_ptr_not_null(allocs[i], "");
}
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(0, ecf_count(&ecf), "");
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
test_edata_cache_destroy(&ec);
}
TEST_END
TEST_BEGIN(test_edata_cache_flush) {
edata_cache_t ec;
edata_cache_small_t ecs;
test_edata_cache_init(&ec);
edata_cache_small_init(&ecs, &ec);
edata_t *allocs[2 * EDATA_CACHE_SMALL_MAX + 2];
for (int i = 0; i < 2 * EDATA_CACHE_SMALL_MAX + 2; i++) {
allocs[i] = edata_cache_get(TSDN_NULL, &ec);
expect_ptr_not_null(allocs[i], "");
}
for (int i = 0; i < EDATA_CACHE_SMALL_MAX; i++) {
edata_cache_small_put(TSDN_NULL, &ecs, allocs[i]);
expect_zu_eq(i + 1, ecs.count, "");
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
}
edata_cache_small_put(TSDN_NULL, &ecs, allocs[EDATA_CACHE_SMALL_MAX]);
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_MAX + 1,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
for (int i = EDATA_CACHE_SMALL_MAX + 1;
i < 2 * EDATA_CACHE_SMALL_MAX + 1; i++) {
edata_cache_small_put(TSDN_NULL, &ecs, allocs[i]);
expect_zu_eq(i - EDATA_CACHE_SMALL_MAX, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_MAX + 1,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
}
edata_cache_small_put(TSDN_NULL, &ecs, allocs[2 * EDATA_CACHE_SMALL_MAX + 1]);
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(2 * EDATA_CACHE_SMALL_MAX + 2,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
test_edata_cache_destroy(&ec);
}
TEST_END
TEST_BEGIN(test_edata_cache_disable) {
edata_cache_t ec;
edata_cache_small_t ecs;
edata_cache_fast_t ecf;
test_edata_cache_init(&ec);
edata_cache_small_init(&ecs, &ec);
edata_cache_fast_init(&ecf, &ec);
for (int i = 0; i < EDATA_CACHE_SMALL_FILL; i++) {
for (int i = 0; i < EDATA_CACHE_FAST_FILL; i++) {
edata_t *edata = edata_cache_get(TSDN_NULL, &ec);
expect_ptr_not_null(edata, "");
edata_cache_small_put(TSDN_NULL, &ecs, edata);
edata_cache_fast_put(TSDN_NULL, &ecf, edata);
}
expect_zu_eq(EDATA_CACHE_SMALL_FILL, ecs.count, "");
expect_zu_eq(EDATA_CACHE_FAST_FILL, ecf_count(&ecf), "");
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
edata_cache_small_disable(TSDN_NULL, &ecs);
edata_cache_fast_disable(TSDN_NULL, &ecf);
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_FILL,
expect_zu_eq(0, ecf_count(&ecf), "");
expect_zu_eq(EDATA_CACHE_FAST_FILL,
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "Disabling should flush");
edata_t *edata = edata_cache_small_get(TSDN_NULL, &ecs);
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_FILL - 1,
edata_t *edata = edata_cache_fast_get(TSDN_NULL, &ecf);
expect_zu_eq(0, ecf_count(&ecf), "");
expect_zu_eq(EDATA_CACHE_FAST_FILL - 1,
atomic_load_zu(&ec.count, ATOMIC_RELAXED),
"Disabled ecs should forward on get");
"Disabled ecf should forward on get");
edata_cache_small_put(TSDN_NULL, &ecs, edata);
expect_zu_eq(0, ecs.count, "");
expect_zu_eq(EDATA_CACHE_SMALL_FILL,
edata_cache_fast_put(TSDN_NULL, &ecf, edata);
expect_zu_eq(0, ecf_count(&ecf), "");
expect_zu_eq(EDATA_CACHE_FAST_FILL,
atomic_load_zu(&ec.count, ATOMIC_RELAXED),
"Disabled ecs should forward on put");
"Disabled ecf should forward on put");
test_edata_cache_destroy(&ec);
}
@@ -248,8 +220,7 @@ int
main(void) {
return test(
test_edata_cache,
test_edata_cache_small_simple,
test_edata_cache_fast_simple,
test_edata_cache_fill,
test_edata_cache_flush,
test_edata_cache_disable);
}