2020-02-19 08:09:10 +08:00
|
|
|
#include "test/jemalloc_test.h"
|
|
|
|
|
|
|
|
#include "jemalloc/internal/edata_cache.h"
|
|
|
|
|
|
|
|
static void
|
|
|
|
test_edata_cache_init(edata_cache_t *edata_cache) {
|
|
|
|
base_t *base = base_new(TSDN_NULL, /* ind */ 1,
|
|
|
|
&ehooks_default_extent_hooks);
|
|
|
|
assert_ptr_not_null(base, "");
|
|
|
|
bool err = edata_cache_init(edata_cache, base);
|
|
|
|
assert_false(err, "");
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
test_edata_cache_destroy(edata_cache_t *edata_cache) {
|
|
|
|
base_delete(TSDN_NULL, edata_cache->base);
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_BEGIN(test_edata_cache) {
|
2020-02-19 09:21:40 +08:00
|
|
|
edata_cache_t ec;
|
|
|
|
test_edata_cache_init(&ec);
|
2020-02-19 08:09:10 +08:00
|
|
|
|
|
|
|
/* Get one */
|
2020-02-19 09:21:40 +08:00
|
|
|
edata_t *ed1 = edata_cache_get(TSDN_NULL, &ec);
|
2020-02-19 08:09:10 +08:00
|
|
|
assert_ptr_not_null(ed1, "");
|
|
|
|
|
|
|
|
/* Cache should be empty */
|
2020-02-19 09:21:40 +08:00
|
|
|
assert_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 08:09:10 +08:00
|
|
|
|
|
|
|
/* Get another */
|
2020-02-19 09:21:40 +08:00
|
|
|
edata_t *ed2 = edata_cache_get(TSDN_NULL, &ec);
|
2020-02-19 08:09:10 +08:00
|
|
|
assert_ptr_not_null(ed2, "");
|
|
|
|
|
|
|
|
/* Still empty */
|
2020-02-19 09:21:40 +08:00
|
|
|
assert_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 08:09:10 +08:00
|
|
|
|
|
|
|
/* Put one back, and the cache should now have one item */
|
2020-02-19 09:21:40 +08:00
|
|
|
edata_cache_put(TSDN_NULL, &ec, ed1);
|
|
|
|
assert_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 1, "");
|
2020-02-19 08:09:10 +08:00
|
|
|
|
|
|
|
/* Reallocating should reuse the item, and leave an empty cache. */
|
2020-02-19 09:21:40 +08:00
|
|
|
edata_t *ed1_again = edata_cache_get(TSDN_NULL, &ec);
|
2020-02-19 08:09:10 +08:00
|
|
|
assert_ptr_eq(ed1, ed1_again, "");
|
2020-02-19 09:21:40 +08:00
|
|
|
assert_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 08:09:10 +08:00
|
|
|
|
2020-02-19 09:21:40 +08:00
|
|
|
test_edata_cache_destroy(&ec);
|
|
|
|
}
|
|
|
|
TEST_END
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
static size_t
|
|
|
|
ecf_count(edata_cache_fast_t *ecf) {
|
|
|
|
size_t count = 0;
|
|
|
|
edata_t *cur;
|
|
|
|
ql_foreach(cur, &ecf->list.head, ql_link_inactive) {
|
|
|
|
count++;
|
|
|
|
}
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
TEST_BEGIN(test_edata_cache_fast_simple) {
|
2020-02-19 09:21:40 +08:00
|
|
|
edata_cache_t ec;
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_t ecf;
|
2020-02-19 09:21:40 +08:00
|
|
|
|
|
|
|
test_edata_cache_init(&ec);
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_init(&ecf, &ec);
|
2020-02-19 09:21:40 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_t *ed1 = edata_cache_fast_get(TSDN_NULL, &ecf);
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_ptr_not_null(ed1, "");
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(ecf_count(&ecf), 0, "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 09:21:40 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_t *ed2 = edata_cache_fast_get(TSDN_NULL, &ecf);
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_ptr_not_null(ed2, "");
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(ecf_count(&ecf), 0, "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 09:21:40 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_put(TSDN_NULL, &ecf, ed1);
|
|
|
|
expect_zu_eq(ecf_count(&ecf), 1, "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 09:21:40 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_put(TSDN_NULL, &ecf, ed2);
|
|
|
|
expect_zu_eq(ecf_count(&ecf), 2, "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 09:21:40 +08:00
|
|
|
|
2020-10-29 20:11:16 +08:00
|
|
|
/* LIFO ordering. */
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_ptr_eq(ed2, edata_cache_fast_get(TSDN_NULL, &ecf), "");
|
|
|
|
expect_zu_eq(ecf_count(&ecf), 1, "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
2020-02-19 09:21:40 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_ptr_eq(ed1, edata_cache_fast_get(TSDN_NULL, &ecf), "");
|
|
|
|
expect_zu_eq(ecf_count(&ecf), 0, "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(atomic_load_zu(&ec.count, ATOMIC_RELAXED), 0, "");
|
|
|
|
|
|
|
|
test_edata_cache_destroy(&ec);
|
|
|
|
}
|
|
|
|
TEST_END
|
|
|
|
|
|
|
|
TEST_BEGIN(test_edata_cache_fill) {
|
|
|
|
edata_cache_t ec;
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_t ecf;
|
2020-10-29 20:11:16 +08:00
|
|
|
|
|
|
|
test_edata_cache_init(&ec);
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_init(&ecf, &ec);
|
2020-10-29 20:11:16 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_t *allocs[EDATA_CACHE_FAST_FILL * 2];
|
2020-10-29 20:11:16 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* If the fallback cache can't satisfy the request, we shouldn't do
|
|
|
|
* extra allocations until compelled to. Put half the fill goal in the
|
|
|
|
* fallback.
|
|
|
|
*/
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 0; i < EDATA_CACHE_FAST_FILL / 2; i++) {
|
2020-10-29 20:11:16 +08:00
|
|
|
allocs[i] = edata_cache_get(TSDN_NULL, &ec);
|
|
|
|
}
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 0; i < EDATA_CACHE_FAST_FILL / 2; i++) {
|
2020-10-29 20:11:16 +08:00
|
|
|
edata_cache_put(TSDN_NULL, &ec, allocs[i]);
|
|
|
|
}
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL / 2,
|
2020-10-29 20:11:16 +08:00
|
|
|
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
allocs[0] = edata_cache_fast_get(TSDN_NULL, &ecf);
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL / 2 - 1, ecf_count(&ecf),
|
2020-10-29 20:11:16 +08:00
|
|
|
"Should have grabbed all edatas available but no more.");
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 1; i < EDATA_CACHE_FAST_FILL / 2; i++) {
|
|
|
|
allocs[i] = edata_cache_fast_get(TSDN_NULL, &ecf);
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_ptr_not_null(allocs[i], "");
|
|
|
|
}
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(0, ecf_count(&ecf), "");
|
2020-10-29 20:11:16 +08:00
|
|
|
|
|
|
|
/* When forced, we should alloc from the base. */
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_t *edata = edata_cache_fast_get(TSDN_NULL, &ecf);
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_ptr_not_null(edata, "");
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(0, ecf_count(&ecf), "Allocated more than necessary");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED),
|
|
|
|
"Allocated more than necessary");
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We should correctly fill in the common case where the fallback isn't
|
|
|
|
* exhausted, too.
|
|
|
|
*/
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 0; i < EDATA_CACHE_FAST_FILL * 2; i++) {
|
2020-10-29 20:11:16 +08:00
|
|
|
allocs[i] = edata_cache_get(TSDN_NULL, &ec);
|
|
|
|
expect_ptr_not_null(allocs[i], "");
|
|
|
|
}
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 0; i < EDATA_CACHE_FAST_FILL * 2; i++) {
|
2020-10-29 20:11:16 +08:00
|
|
|
edata_cache_put(TSDN_NULL, &ec, allocs[i]);
|
|
|
|
}
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
allocs[0] = edata_cache_fast_get(TSDN_NULL, &ecf);
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL - 1, ecf_count(&ecf), "");
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL,
|
2020-10-29 20:11:16 +08:00
|
|
|
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 1; i < EDATA_CACHE_FAST_FILL; i++) {
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL - i, ecf_count(&ecf), "");
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL,
|
2020-10-29 20:11:16 +08:00
|
|
|
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
2021-07-24 06:29:43 +08:00
|
|
|
allocs[i] = edata_cache_fast_get(TSDN_NULL, &ecf);
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_ptr_not_null(allocs[i], "");
|
|
|
|
}
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(0, ecf_count(&ecf), "");
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL,
|
2020-10-29 20:11:16 +08:00
|
|
|
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
allocs[0] = edata_cache_fast_get(TSDN_NULL, &ecf);
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL - 1, ecf_count(&ecf), "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 1; i < EDATA_CACHE_FAST_FILL; i++) {
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL - i, ecf_count(&ecf), "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
2021-07-24 06:29:43 +08:00
|
|
|
allocs[i] = edata_cache_fast_get(TSDN_NULL, &ecf);
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_ptr_not_null(allocs[i], "");
|
|
|
|
}
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(0, ecf_count(&ecf), "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
|
|
|
|
|
|
|
test_edata_cache_destroy(&ec);
|
|
|
|
}
|
|
|
|
TEST_END
|
|
|
|
|
|
|
|
TEST_BEGIN(test_edata_cache_disable) {
|
|
|
|
edata_cache_t ec;
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_t ecf;
|
2020-10-29 20:11:16 +08:00
|
|
|
|
|
|
|
test_edata_cache_init(&ec);
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_init(&ecf, &ec);
|
2020-10-29 20:11:16 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
for (int i = 0; i < EDATA_CACHE_FAST_FILL; i++) {
|
2020-10-29 20:11:16 +08:00
|
|
|
edata_t *edata = edata_cache_get(TSDN_NULL, &ec);
|
|
|
|
expect_ptr_not_null(edata, "");
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_put(TSDN_NULL, &ecf, edata);
|
2020-10-29 20:11:16 +08:00
|
|
|
}
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL, ecf_count(&ecf), "");
|
2020-10-29 20:11:16 +08:00
|
|
|
expect_zu_eq(0, atomic_load_zu(&ec.count, ATOMIC_RELAXED), "");
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_disable(TSDN_NULL, &ecf);
|
2020-10-29 20:11:16 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
expect_zu_eq(0, ecf_count(&ecf), "");
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL,
|
2020-10-29 20:11:16 +08:00
|
|
|
atomic_load_zu(&ec.count, ATOMIC_RELAXED), "Disabling should flush");
|
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_t *edata = edata_cache_fast_get(TSDN_NULL, &ecf);
|
|
|
|
expect_zu_eq(0, ecf_count(&ecf), "");
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL - 1,
|
2020-10-29 20:11:16 +08:00
|
|
|
atomic_load_zu(&ec.count, ATOMIC_RELAXED),
|
2021-07-24 06:29:43 +08:00
|
|
|
"Disabled ecf should forward on get");
|
2020-10-29 20:11:16 +08:00
|
|
|
|
2021-07-24 06:29:43 +08:00
|
|
|
edata_cache_fast_put(TSDN_NULL, &ecf, edata);
|
|
|
|
expect_zu_eq(0, ecf_count(&ecf), "");
|
|
|
|
expect_zu_eq(EDATA_CACHE_FAST_FILL,
|
2020-10-29 20:11:16 +08:00
|
|
|
atomic_load_zu(&ec.count, ATOMIC_RELAXED),
|
2021-07-24 06:29:43 +08:00
|
|
|
"Disabled ecf should forward on put");
|
2020-02-19 09:21:40 +08:00
|
|
|
|
|
|
|
test_edata_cache_destroy(&ec);
|
2020-02-19 08:09:10 +08:00
|
|
|
}
|
|
|
|
TEST_END
|
|
|
|
|
|
|
|
int
|
|
|
|
main(void) {
|
|
|
|
return test(
|
2020-02-19 09:21:40 +08:00
|
|
|
test_edata_cache,
|
2021-07-24 06:29:43 +08:00
|
|
|
test_edata_cache_fast_simple,
|
2020-10-29 20:11:16 +08:00
|
|
|
test_edata_cache_fill,
|
|
|
|
test_edata_cache_disable);
|
2020-02-19 08:09:10 +08:00
|
|
|
}
|