project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
e1dcc557d6
server-skynet-source-3rd-je.../test/unit/cache_bin.c

68 lines
2.6 KiB
C
Raw Normal View History

Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
#include "test/jemalloc_test.h"
cache_bin_t test_bin;
TEST_BEGIN(test_cache_bin) {
cache_bin_t *bin = &test_bin;
Fix incorrect usage of cassert.
2019-11-22 06:10:03 +08:00
assert(PAGE > TCACHE_NSLOTS_SMALL_MAX * sizeof(void *));
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
/* Page aligned to make sure lowbits not overflowable. */
void **stack = mallocx(PAGE, MALLOCX_TCACHE_NONE | MALLOCX_ALIGN(PAGE));
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_ptr_not_null(stack, "Unexpected mallocx failure");
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
/* Initialize to empty; bin 0. */
Cache bin: Make ncached_max a query on the info_t.
2020-02-27 09:23:47 +08:00
cache_bin_sz_t ncached_max = cache_bin_info_ncached_max(
&tcache_bin_info[0]);
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
void **empty_position = stack + ncached_max;
bin->cur_ptr.ptr = empty_position;
bin->low_water_position = bin->cur_ptr.lowbits;
bin->full_position = (uint32_t)(uintptr_t)stack;
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
expect_ptr_eq(cache_bin_empty_position_get(bin, &tcache_bin_info[0]),
Cache_bin: Shouldn't know about tcache. Instead, have it take the cache_bin_info_ts to use by pointer. While we're here, add a src file for the cache bin.
2020-02-26 04:14:48 +08:00
empty_position, "Incorrect empty position");
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
/* Not using expect_zu etc on cache_bin_sz_t since it may change. */
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
expect_true(cache_bin_ncached_get(bin, &tcache_bin_info[0]) == 0,
Cache_bin: Shouldn't know about tcache. Instead, have it take the cache_bin_info_ts to use by pointer. While we're here, add a src file for the cache bin.
2020-02-26 04:14:48 +08:00
"Incorrect cache size");
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
bool success;
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
void *ret = cache_bin_alloc_easy(bin, &success, &tcache_bin_info[0]);
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_false(success, "Empty cache bin should not alloc");
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
expect_true(cache_bin_low_water_get(bin, &tcache_bin_info[0]) == 0,
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
"Incorrect low water mark");
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
cache_bin_ncached_set(bin, 0, &tcache_bin_info[0]);
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_ptr_eq(bin->cur_ptr.ptr, empty_position, "Bin should be empty");
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
for (cache_bin_sz_t i = 1; i < ncached_max + 1; i++) {
success = cache_bin_dalloc_easy(bin, (void *)(uintptr_t)i);
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
expect_true(success && cache_bin_ncached_get(bin,
&tcache_bin_info[0]) == i, "Bin dalloc failure");
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
}
success = cache_bin_dalloc_easy(bin, (void *)1);
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_false(success, "Bin should be full");
expect_ptr_eq(bin->cur_ptr.ptr, stack, "Incorrect bin cur_ptr");
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
cache_bin_ncached_set(bin, ncached_max, &tcache_bin_info[0]);
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_ptr_eq(bin->cur_ptr.ptr, stack, "cur_ptr should not change");
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
/* Emulate low water after refill. */
bin->low_water_position = bin->full_position;
for (cache_bin_sz_t i = ncached_max; i > 0; i--) {
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
ret = cache_bin_alloc_easy(bin, &success, &tcache_bin_info[0]);
cache_bin_sz_t ncached = cache_bin_ncached_get(bin,
&tcache_bin_info[0]);
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_true(success && ncached == i - 1,
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
"Cache bin alloc failure");
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_ptr_eq(ret, (void *)(uintptr_t)i, "Bin alloc failure");
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
expect_true(cache_bin_low_water_get(bin, &tcache_bin_info[0])
Cache_bin: Shouldn't know about tcache. Instead, have it take the cache_bin_info_ts to use by pointer. While we're here, add a src file for the cache bin.
2020-02-26 04:14:48 +08:00
== ncached, "Incorrect low water mark");
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
}
Cache bin: Only take the relevant cache_bin_info_t Previously, we took an array of cache_bin_info_ts and an index, and dereferenced ourselves. But infos for other cache_bins aren't relevant to any particular cache bin, so that should be the caller's job.
2020-02-29 10:55:33 +08:00
ret = cache_bin_alloc_easy(bin, &success, &tcache_bin_info[0]);
Change assert_* to expect_* in tests ``` grep -Irl assert_ test/ | xargs sed -i \ 's/witness_assert/witness_do_not_replace/g'; grep -Irl assert_ test/ | xargs sed -i \ 's/malloc_mutex_assert_owner/malloc_mutex_do_not_replace_owner/g'; grep -Ir assert_ test/ | grep -o "[_a-zA-Z]*assert_[_a-zA-Z]*" | \ grep -v "^assert_"; # confirm no output grep -Irl assert_ test/ | xargs sed -i 's/assert_/expect_/g'; grep -Irl witness_do_not_replace test/ | xargs sed -i \ 's/witness_do_not_replace/witness_assert/g'; grep -Irl malloc_mutex_do_not_replace_owner test/ | xargs sed -i \ 's/malloc_mutex_do_not_replace_owner/malloc_mutex_assert_owner/g'; ```
2020-02-19 06:39:06 +08:00
expect_false(success, "Empty cache bin should not alloc.");
expect_ptr_eq(bin->cur_ptr.ptr, stack + ncached_max,
Redesign the cache bin metadata for fast path. Implement the pointer-based metadata for tcache bins -- - 3 pointers are maintained to represent each bin; - 2 of the pointers are compressed on 64-bit; - is_full / is_empty done through pointer comparison; Comparing to the previous counter based design -- - fast-path speed up ~15% in benchmarks - direct pointer comparison and de-reference - no need to access tcache_bin_info in common case
2019-08-10 13:12:47 +08:00
"Bin should be empty");
}
TEST_END
int
main(void) {
return test(test_cache_bin);
}
Reference in New Issue Copy Permalink