project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pre-generate ncached_max for all bins for better tcache_max tuning experience.

Browse Source
This commit is contained in:
guangli-dai
2023-08-22 16:31:54 -07:00
committed by Qi Wang
parent 36becb1302
commit 6b197fdd46
12 changed files with 417 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
test/unit/cache_bin.c
View File

@@ -106,11 +106,13 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_info_init(&info, ncached_max);
cache_bin_t bin;
test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/* Initialize to empty; should then have 0 elements. */
expect_d_eq(ncached_max, cache_bin_info_ncached_max(&info), "");
expect_true(cache_bin_ncached_get_local(&bin, &info) == 0, "");
expect_true(cache_bin_low_water_get(&bin, &info) == 0, "");
expect_d_eq(ncached_max, cache_bin_info_ncached_max_get(&bin,
&bin.bin_info), "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, "");
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success);
expect_false(success, "Shouldn't successfully allocate when empty");
@@ -127,14 +129,14 @@ TEST_BEGIN(test_cache_bin) {
void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0);
assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, &info) == i, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == i, "");
success = cache_bin_dalloc_easy(&bin, &ptrs[i]);
expect_true(success,
"Should be able to dalloc into a non-full cache bin.");
expect_true(cache_bin_low_water_get(&bin, &info) == 0,
expect_true(cache_bin_low_water_get(&bin, bin_info) == 0,
"Pushes and pops shouldn't change low water of zero.");
}
expect_true(cache_bin_ncached_get_local(&bin, &info) == ncached_max,
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max,
"");
success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]);
expect_false(success, "Shouldn't be able to dalloc into a full bin.");
@@ -142,9 +144,9 @@ TEST_BEGIN(test_cache_bin) {
cache_bin_low_water_set(&bin);
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_low_water_get(&bin, &info)
expect_true(cache_bin_low_water_get(&bin, bin_info)
== ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, &info)
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
== ncached_max - i, "");
/*
* This should fail -- the easy variant can't change the low
@@ -153,9 +155,9 @@ TEST_BEGIN(test_cache_bin) {
ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, "");
expect_false(success, "");
expect_true(cache_bin_low_water_get(&bin, &info)
expect_true(cache_bin_low_water_get(&bin, bin_info)
== ncached_max - i, "");
expect_true(cache_bin_ncached_get_local(&bin, &info)
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
== ncached_max - i, "");
/* This should succeed, though. */
@@ -163,13 +165,13 @@ TEST_BEGIN(test_cache_bin) {
expect_true(success, "");
expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1],
"Alloc should pop in stack order");
expect_true(cache_bin_low_water_get(&bin, &info)
expect_true(cache_bin_low_water_get(&bin, bin_info)
== ncached_max - i - 1, "");
expect_true(cache_bin_ncached_get_local(&bin, &info)
expect_true(cache_bin_ncached_get_local(&bin, bin_info)
== ncached_max - i - 1, "");
}
/* Now we're empty -- all alloc attempts should fail. */
expect_true(cache_bin_ncached_get_local(&bin, &info) == 0, "");
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == 0, "");
ptr = cache_bin_alloc_easy(&bin, &success);
expect_ptr_null(ptr, "");
expect_false(success, "");
@@ -185,7 +187,7 @@ TEST_BEGIN(test_cache_bin) {
for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) {
cache_bin_dalloc_easy(&bin, &ptrs[i]);
}
expect_true(cache_bin_ncached_get_local(&bin, &info) == ncached_max,
expect_true(cache_bin_ncached_get_local(&bin, bin_info) == ncached_max,
"");
for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) {
/*
@@ -202,60 +204,64 @@ TEST_BEGIN(test_cache_bin) {
expect_ptr_null(ptr, "");
/* We're going to test filling -- we must be empty to start. */
while (cache_bin_ncached_get_local(&bin, &info)) {
while (cache_bin_ncached_get_local(&bin, bin_info)) {
cache_bin_alloc(&bin, &success);
expect_true(success, "");
}
/* Test fill. */
/* Try to fill all, succeed fully. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max, ncached_max);
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max,
ncached_max);
/* Try to fill all, succeed partially. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max,
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max,
ncached_max / 2);
/* Try to fill all, fail completely. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max, 0);
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max, 0);
/* Try to fill some, succeed fully. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2,
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2,
ncached_max / 2);
/* Try to fill some, succeed partially. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2,
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2,
ncached_max / 4);
/* Try to fill some, fail completely. */
do_fill_test(&bin, &info, ptrs, ncached_max, ncached_max / 2, 0);
do_fill_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2, 0);
do_flush_test(&bin, &info, ptrs, ncached_max, ncached_max);
do_flush_test(&bin, &info, ptrs, ncached_max, ncached_max / 2);
do_flush_test(&bin, &info, ptrs, ncached_max, 0);
do_flush_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_test(&bin, &info, ptrs, ncached_max / 2, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max);
do_flush_test(&bin, bin_info, ptrs, ncached_max, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max, 0);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_test(&bin, bin_info, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max * 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, ncached_max / 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 1);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max, 0);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2,
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max,
ncached_max * 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max,
ncached_max / 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, ncached_max);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2,
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 4);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 2);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 1);
do_batch_alloc_test(&bin, &info, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, &info, ptrs, 2, ncached_max);
do_batch_alloc_test(&bin, &info, ptrs, 2, 2);
do_batch_alloc_test(&bin, &info, ptrs, 2, 1);
do_batch_alloc_test(&bin, &info, ptrs, 2, 0);
do_batch_alloc_test(&bin, &info, ptrs, 1, 2);
do_batch_alloc_test(&bin, &info, ptrs, 1, 1);
do_batch_alloc_test(&bin, &info, ptrs, 1, 0);
do_batch_alloc_test(&bin, &info, ptrs, 0, 2);
do_batch_alloc_test(&bin, &info, ptrs, 0, 1);
do_batch_alloc_test(&bin, &info, ptrs, 0, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, ncached_max / 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, ncached_max);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 2, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 1, 0);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 2);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 1);
do_batch_alloc_test(&bin, bin_info, ptrs, 0, 0);
free(ptrs);
}
@@ -328,6 +334,7 @@ TEST_BEGIN(test_cache_bin_stash) {
cache_bin_info_t info;
cache_bin_info_init(&info, ncached_max);
test_bin_init(&bin, &info);
cache_bin_info_t *bin_info = &bin.bin_info;
/*
* The content of this array is not accessed; instead the interior
@@ -337,10 +344,10 @@ TEST_BEGIN(test_cache_bin_stash) {
assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
bool ret;
for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
expect_true(cache_bin_ncached_get_local(&bin, &info) ==
expect_true(cache_bin_ncached_get_local(&bin, bin_info) ==
(i / 2 + i % 2), "Wrong ncached value");
expect_true(cache_bin_nstashed_get_local(&bin, &info) == i / 2,
"Wrong nstashed value");
expect_true(cache_bin_nstashed_get_local(&bin, bin_info) ==
i / 2, "Wrong nstashed value");
if (i % 2 == 0) {
cache_bin_dalloc_easy(&bin, &ptrs[i]);
} else {
@@ -362,18 +369,23 @@ TEST_BEGIN(test_cache_bin_stash) {
expect_true(diff % 2 == 0, "Should be able to alloc");
} else {
expect_false(ret, "Should not alloc stashed");
expect_true(cache_bin_nstashed_get_local(&bin, &info) ==
ncached_max / 2, "Wrong nstashed value");
expect_true(cache_bin_nstashed_get_local(&bin,
bin_info) == ncached_max / 2,
"Wrong nstashed value");
}
}
test_bin_init(&bin, &info);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max, 0);
do_flush_stashed_test(&bin, &info, ptrs, 0, ncached_max);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 2);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 4, ncached_max / 2);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 2, ncached_max / 4);
do_flush_stashed_test(&bin, &info, ptrs, ncached_max / 4, ncached_max / 4);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max, 0);
do_flush_stashed_test(&bin, bin_info, ptrs, 0, ncached_max);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4,
ncached_max / 2);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 2,
ncached_max / 4);
do_flush_stashed_test(&bin, bin_info, ptrs, ncached_max / 4,
ncached_max / 4);
}
TEST_END

44
test/unit/tcache_max.c
View File

@@ -76,8 +76,11 @@ tcache_bytes_read_local(void) {
size_t tcache_bytes = 0;
tsd_t *tsd = tsd_fetch();
tcache_t *tcache = tcache_get(tsd);
for (szind_t i = 0; i < tcache_nhbins_get(tcache); i++) {
for (szind_t i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
continue;
}
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&cache_bin->bin_info);
tcache_bytes += ncached * sz_index2size(i);
@@ -211,7 +214,7 @@ TEST_BEGIN(test_tcache_max) {
TEST_END
static size_t
tcache_max2nhbins(size_t tcache_max) {
tcache_max2nbins(size_t tcache_max) {
return sz_size2index(tcache_max) + 1;
}
@@ -241,23 +244,24 @@ validate_tcache_stack(tcache_t *tcache) {
static void *
tcache_check(void *arg) {
size_t old_tcache_max, new_tcache_max, min_tcache_max, sz;
unsigned tcache_nhbins;
unsigned tcache_nbins;
tsd_t *tsd = tsd_fetch();
tcache_t *tcache = tsd_tcachep_get(tsd);
tcache_slow_t *tcache_slow = tcache->tcache_slow;
sz = sizeof(size_t);
new_tcache_max = *(size_t *)arg;
min_tcache_max = 1;
/*
* Check the default tcache_max and tcache_nhbins of each thread's
* Check the default tcache_max and tcache_nbins of each thread's
* auto tcache.
*/
old_tcache_max = tcache_max_get(tcache);
old_tcache_max = tcache_max_get(tcache_slow);
expect_zu_eq(old_tcache_max, opt_tcache_max,
"Unexpected default value for tcache_max");
tcache_nhbins = tcache_nhbins_get(tcache);
expect_zu_eq(tcache_nhbins, (size_t)global_do_not_change_nhbins,
"Unexpected default value for tcache_nhbins");
tcache_nbins = tcache_nbins_get(tcache_slow);
expect_zu_eq(tcache_nbins, (size_t)global_do_not_change_nbins,
"Unexpected default value for tcache_nbins");
validate_tcache_stack(tcache);
/*
@@ -275,12 +279,12 @@ tcache_check(void *arg) {
assert_d_eq(mallctl("thread.tcache.max",
NULL, NULL, (void *)&temp_tcache_max, sz),.0,
"Unexpected.mallctl().failure");
old_tcache_max = tcache_max_get(tcache);
old_tcache_max = tcache_max_get(tcache_slow);
expect_zu_eq(old_tcache_max, TCACHE_MAXCLASS_LIMIT,
"Unexpected value for tcache_max");
tcache_nhbins = tcache_nhbins_get(tcache);
expect_zu_eq(tcache_nhbins, TCACHE_NBINS_MAX,
"Unexpected value for tcache_nhbins");
tcache_nbins = tcache_nbins_get(tcache_slow);
expect_zu_eq(tcache_nbins, TCACHE_NBINS_MAX,
"Unexpected value for tcache_nbins");
assert_d_eq(mallctl("thread.tcache.max",
(void *)&old_tcache_max, &sz,
(void *)&min_tcache_max, sz),.0,
@@ -294,10 +298,10 @@ tcache_check(void *arg) {
(void *)&e0, bool_sz), 0, "Unexpected mallctl() error");
expect_false(e1, "Unexpected previous tcache state");
min_tcache_max = sz_s2u(min_tcache_max);
expect_zu_eq(tcache_max_get(tcache), min_tcache_max,
expect_zu_eq(tcache_max_get(tcache_slow), min_tcache_max,
"Unexpected value for tcache_max");
expect_zu_eq(tcache_nhbins_get(tcache),
tcache_max2nhbins(min_tcache_max), "Unexpected value for nhbins");
expect_zu_eq(tcache_nbins_get(tcache_slow),
tcache_max2nbins(min_tcache_max), "Unexpected value for nbins");
assert_d_eq(mallctl("thread.tcache.max",
(void *)&old_tcache_max, &sz,
(void *)&new_tcache_max, sz),.0,
@@ -307,18 +311,18 @@ tcache_check(void *arg) {
validate_tcache_stack(tcache);
/*
* Check the thread's tcache_max and nhbins both through mallctl
* Check the thread's tcache_max and nbins both through mallctl
* and alloc tests.
*/
if (new_tcache_max > TCACHE_MAXCLASS_LIMIT) {
new_tcache_max = TCACHE_MAXCLASS_LIMIT;
}
old_tcache_max = tcache_max_get(tcache);
old_tcache_max = tcache_max_get(tcache_slow);
expect_zu_eq(old_tcache_max, new_tcache_max,
"Unexpected value for tcache_max");
tcache_nhbins = tcache_nhbins_get(tcache);
expect_zu_eq(tcache_nhbins, tcache_max2nhbins(new_tcache_max),
"Unexpected value for tcache_nhbins");
tcache_nbins = tcache_nbins_get(tcache_slow);
expect_zu_eq(tcache_nbins, tcache_max2nbins(new_tcache_max),
"Unexpected value for tcache_nbins");
for (unsigned alloc_option = alloc_option_start;
alloc_option < alloc_option_end;
alloc_option++) {