project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cache_bin: Shouldn't know about tcache.

Browse Source 
Instead, have it take the cache_bin_info_ts to use by pointer.  While we're
here, add a src file for the cache bin.
This commit is contained in:
David Goldblatt 2020-02-25 12:14:48 -08:00 committed by David Goldblatt
parent 79f1ee2fc0
commit 909c501b07
12 changed files with 101 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.in
View File

@ -102,6 +102,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/bin_info.c \
$(srcroot)src/bitmap.c \
$(srcroot)src/buf_writer.c \
$(srcroot)src/cache_bin.c \
$(srcroot)src/ckh.c \
$(srcroot)src/counter.c \
$(srcroot)src/ctl.c \

58
include/jemalloc/internal/cache_bin.h
View File

@ -35,7 +35,6 @@ struct cache_bin_info_s {
/* The size of the bin stack, i.e. ncached_max * sizeof(ptr). */
cache_bin_sz_t stack_size;
};
extern cache_bin_info_t *tcache_bin_info;
typedef struct cache_bin_s cache_bin_t;
struct cache_bin_s {
@ -115,29 +114,29 @@ struct cache_bin_array_descriptor_s {
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_ncached_max_get(szind_t ind) {
return tcache_bin_info[ind].stack_size / sizeof(void *);
cache_bin_ncached_max_get(szind_t ind, cache_bin_info_t *infos) {
return infos[ind].stack_size / sizeof(void *);
}
static inline cache_bin_sz_t
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind) {
cache_bin_sz_t n = (cache_bin_sz_t)((tcache_bin_info[ind].stack_size +
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind, cache_bin_info_t *infos) {
cache_bin_sz_t n = (cache_bin_sz_t)((infos[ind].stack_size +
bin->full_position - bin->cur_ptr.lowbits) / sizeof(void *));
assert(n <= cache_bin_ncached_max_get(ind));
assert(n <= cache_bin_ncached_max_get(ind, infos));
assert(n == 0 || *(bin->cur_ptr.ptr) != NULL);
return n;
}
static inline void **
cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind);
cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind,
cache_bin_info_t *infos) {
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind, infos);
/* Low bits overflow disallowed when allocating the space. */
assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits);
/* Can also be computed via (full_position + ncached_max) | highbits. */
uintptr_t lowbits = bin->full_position +
tcache_bin_info[ind].stack_size;
uintptr_t lowbits = bin->full_position + infos[ind].stack_size;
uintptr_t highbits = (uintptr_t)bin->cur_ptr.ptr &
~(((uint64_t)1 << 32) - 1);
assert(ret == (void **)(lowbits | highbits));
@ -147,32 +146,35 @@ cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
/* Returns the position of the bottom item on the stack; for convenience. */
static inline void **
cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind) {
void **bottom = cache_bin_empty_position_get(bin, ind) - 1;
assert(cache_bin_ncached_get(bin, ind) == 0 || *bottom != NULL);
cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind,
cache_bin_info_t *infos) {
void **bottom = cache_bin_empty_position_get(bin, ind, infos) - 1;
assert(cache_bin_ncached_get(bin, ind, infos) == 0 || *bottom != NULL);
return bottom;
}
/* Returns the numeric value of low water in [0, ncached]. */
static inline cache_bin_sz_t
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind);
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind,
cache_bin_info_t *infos) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind, infos);
cache_bin_sz_t low_water = ncached_max -
(cache_bin_sz_t)((bin->low_water_position - bin->full_position) /
sizeof(void *));
assert(low_water <= ncached_max);
assert(low_water <= cache_bin_ncached_get(bin, ind));
assert(low_water <= cache_bin_ncached_get(bin, ind, infos));
assert(bin->low_water_position >= bin->cur_ptr.lowbits);
return low_water;
}
static inline void
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n) {
bin->cur_ptr.lowbits = bin->full_position +
tcache_bin_info[ind].stack_size - n * sizeof(void *);
assert(n <= cache_bin_ncached_max_get(ind));
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n,
cache_bin_info_t *infos) {
bin->cur_ptr.lowbits = bin->full_position + infos[ind].stack_size
- n * sizeof(void *);
assert(n <= cache_bin_ncached_max_get(ind, infos));
assert(n == 0 || *bin->cur_ptr.ptr != NULL);
}
@ -188,7 +190,7 @@ cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor,
JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
const bool adjust_low_water) {
cache_bin_info_t *infos, const bool adjust_low_water) {
/*
* This may read from the empty position; however the loaded value won't
* be used. It's safe because the stack has one more slot reserved.
@ -197,14 +199,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
/*
* Check for both bin->ncached == 0 and ncached < low_water in a single
* branch. When adjust_low_water is true, this also avoids accessing
* tcache_bin_info (which is on a separate cacheline / page) in the
* common case.
* the cache_bin_info_ts (which is on a separate cacheline / page) in
* the common case.
*/
if (unlikely(bin->cur_ptr.lowbits > bin->low_water_position)) {
if (adjust_low_water) {
assert(ind != INVALID_SZIND);
uint32_t empty_position = bin->full_position +
tcache_bin_info[ind].stack_size;
infos[ind].stack_size;
if (unlikely(bin->cur_ptr.lowbits > empty_position)) {
/* Over-allocated; revert. */
bin->cur_ptr.ptr--;
@ -237,12 +239,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy_reduced(cache_bin_t *bin, bool *success) {
/* The szind parameter won't be used. */
return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND, false);
return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND,
/* infos */ NULL, false);
}
JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind) {
return cache_bin_alloc_easy_impl(bin, success, ind, true);
cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind,
cache_bin_info_t *infos) {
return cache_bin_alloc_easy_impl(bin, success, ind, infos, true);
}
#undef INVALID_SZIND

2
include/jemalloc/internal/tcache_externs.h
View File

@ -13,6 +13,8 @@ extern unsigned nhbins;
/* Maximum cached size class. */
extern size_t tcache_maxclass;
extern cache_bin_info_t *tcache_bin_info;
/*
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are

12
include/jemalloc/internal/tcache_inlines.h
View File

@ -36,7 +36,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
assert(binind < SC_NBINS);
bin = tcache_small_bin_get(tcache, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind,
tcache_bin_info);
assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) {
bool tcache_hard_success;
@ -79,7 +80,8 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
assert(binind >= SC_NBINS &&binind < nhbins);
bin = tcache_large_bin_get(tcache, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind,
tcache_bin_info);
assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) {
/*
@ -127,7 +129,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bin = tcache_small_bin_get(tcache, binind);
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
unsigned remain = cache_bin_ncached_max_get(binind,
tcache_bin_info) >> 1;
tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);
@ -145,7 +148,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bin = tcache_large_bin_get(tcache, binind);
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
unsigned remain = cache_bin_ncached_max_get(binind,
tcache_bin_info) >> 1;
tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);

1
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
View File

@ -42,6 +42,7 @@
<ClCompile Include="..\..\..\..\src\bin_info.c" />
<ClCompile Include="..\..\..\..\src\bitmap.c" />
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
<ClCompile Include="..\..\..\..\src\ckh.c" />
<ClCompile Include="..\..\..\..\src\counter.c" />
<ClCompile Include="..\..\..\..\src\ctl.c" />

3
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
View File

@ -25,6 +25,9 @@
<ClCompile Include="..\..\..\..\src\buf_writer.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\cache_bin.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\ckh.c">
<Filter>Source Files</Filter>
</ClCompile>

1
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
View File

@ -42,6 +42,7 @@
<ClCompile Include="..\..\..\..\src\bin_info.c" />
<ClCompile Include="..\..\..\..\src\bitmap.c" />
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
<ClCompile Include="..\..\..\..\src\ckh.c" />
<ClCompile Include="..\..\..\..\src\counter.c" />
<ClCompile Include="..\..\..\..\src\ctl.c" />

3
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
View File

@ -25,6 +25,9 @@
<ClCompile Include="..\..\..\..\src\buf_writer.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\cache_bin.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\ckh.c">
<Filter>Source Files</Filter>
</ClCompile>

18
src/arena.c
View File

@ -200,13 +200,14 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
for (szind_t i = 0; i < SC_NBINS; i++) {
cache_bin_t *tbin = &descriptor->bins_small[i];
arena_stats_accum_zu(&astats->tcache_bytes,
cache_bin_ncached_get(tbin, i) * sz_index2size(i));
cache_bin_ncached_get(tbin, i, tcache_bin_info)
* sz_index2size(i));
}
for (szind_t i = 0; i < nhbins - SC_NBINS; i++) {
cache_bin_t *tbin = &descriptor->bins_large[i];
arena_stats_accum_zu(&astats->tcache_bytes,
cache_bin_ncached_get(tbin, i + SC_NBINS) *
sz_index2size(i));
cache_bin_ncached_get(tbin, i + SC_NBINS,
tcache_bin_info) * sz_index2size(i));
}
}
malloc_mutex_prof_read(tsdn,
@ -1320,13 +1321,14 @@ arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind,
void
arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
cache_bin_t *tbin, szind_t binind) {
assert(cache_bin_ncached_get(tbin, binind) == 0);
assert(cache_bin_ncached_get(tbin, binind, tcache_bin_info) == 0);
tcache->bin_refilled[binind] = true;
const bin_info_t *bin_info = &bin_infos[binind];
const unsigned nfill = cache_bin_ncached_max_get(binind) >>
tcache->lg_fill_div[binind];
void **empty_position = cache_bin_empty_position_get(tbin, binind);
const unsigned nfill = cache_bin_ncached_max_get(binind,
tcache_bin_info) >> tcache->lg_fill_div[binind];
void **empty_position = cache_bin_empty_position_get(tbin, binind,
tcache_bin_info);
/*
* Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull
@ -1446,7 +1448,7 @@ label_refill:
fresh_slab = NULL;
}
cache_bin_ncached_set(tbin, binind, filled);
cache_bin_ncached_set(tbin, binind, filled, tcache_bin_info);
arena_decay_tick(tsdn, arena);
}

3
src/cache_bin.c Normal file
View File

@ -0,0 +1,3 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"

33
src/tcache.c
View File

@ -59,8 +59,10 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
is_small = false;
}
cache_bin_sz_t low_water = cache_bin_low_water_get(tbin, binind);
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind);
cache_bin_sz_t low_water = cache_bin_low_water_get(tbin, binind,
tcache_bin_info);
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind,
tcache_bin_info);
if (low_water > 0) {
/*
* Flush (ceiling) 3/4 of the objects below the low water mark.
@ -73,8 +75,8 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
* Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1.
*/
if ((cache_bin_ncached_max_get(binind) >>
(tcache->lg_fill_div[binind] + 1)) >= 1) {
if ((cache_bin_ncached_max_get(binind, tcache_bin_info)
>> (tcache->lg_fill_div[binind] + 1)) >= 1) {
tcache->lg_fill_div[binind]++;
}
} else {
@ -107,7 +109,8 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
assert(tcache->arena != NULL);
arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind);
ret = cache_bin_alloc_easy(tbin, tcache_success, binind);
ret = cache_bin_alloc_easy(tbin, tcache_success, binind,
tcache_bin_info);
return ret;
}
@ -126,7 +129,8 @@ tbin_edatas_lookup_size_check(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
* builds, avoid the branch in the loop.
*/
size_t szind_sum = binind * nflush;
void **bottom_item = cache_bin_bottom_item_get(tbin, binind);
void **bottom_item = cache_bin_bottom_item_get(tbin, binind,
tcache_bin_info);
for (unsigned i = 0 ; i < nflush; i++) {
emap_full_alloc_ctx_t full_alloc_ctx;
emap_full_alloc_ctx_lookup(tsd_tsdn(tsd), &emap_global,
@ -164,7 +168,8 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
} else {
assert(binind < nhbins);
}
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind);
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind,
tcache_bin_info);
assert((cache_bin_sz_t)rem <= ncached);
arena_t *tcache_arena = tcache->arena;
assert(tcache_arena != NULL);
@ -175,7 +180,8 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
* touched (it's just included to satisfy the no-zero-length rule).
*/
VARIABLE_ARRAY(edata_t *, item_edata, nflush + 1);
void **bottom_item = cache_bin_bottom_item_get(tbin, binind);
void **bottom_item = cache_bin_bottom_item_get(tbin, binind,
tcache_bin_info);
/* Look up edata once per item. */
if (config_opt_safety_checks) {
@ -340,7 +346,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
memmove(tbin->cur_ptr.ptr + (ncached - rem), tbin->cur_ptr.ptr, rem *
sizeof(void *));
cache_bin_ncached_set(tbin, binind, rem);
cache_bin_ncached_set(tbin, binind, rem, tcache_bin_info);
if (tbin->cur_ptr.lowbits > tbin->low_water_position) {
tbin->low_water_position = tbin->cur_ptr.lowbits;
}
@ -445,8 +451,9 @@ tcache_bin_init(cache_bin_t *bin, szind_t ind, uintptr_t *stack_cur) {
bin->low_water_position = bin->cur_ptr.lowbits;
bin->full_position = (uint32_t)(uintptr_t)full_position;
assert(bin->cur_ptr.lowbits - bin->full_position == bin_stack_size);
assert(cache_bin_ncached_get(bin, ind) == 0);
assert(cache_bin_empty_position_get(bin, ind) == empty_position);
assert(cache_bin_ncached_get(bin, ind, tcache_bin_info) == 0);
assert(cache_bin_empty_position_get(bin, ind, tcache_bin_info)
== empty_position);
return false;
}
@ -605,8 +612,8 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
if (tsd_tcache) {
/* Release the avail array for the TSD embedded auto tcache. */
cache_bin_t *bin = tcache_small_bin_get(tcache, 0);
assert(cache_bin_ncached_get(bin, 0) == 0);
assert(cache_bin_empty_position_get(bin, 0) ==
assert(cache_bin_ncached_get(bin, 0, tcache_bin_info) == 0);
assert(cache_bin_empty_position_get(bin, 0, tcache_bin_info) ==
bin->cur_ptr.ptr);
void *avail_array = (void *)((uintptr_t)bin->cur_ptr.ptr -
tcache_bin_info[0].stack_size);

33
test/unit/cache_bin.c
View File

@ -10,48 +10,51 @@ TEST_BEGIN(test_cache_bin) {
expect_ptr_not_null(stack, "Unexpected mallocx failure");
/* Initialize to empty; bin 0. */
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0);
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0,
tcache_bin_info);
void **empty_position = stack + ncached_max;
bin->cur_ptr.ptr = empty_position;
bin->low_water_position = bin->cur_ptr.lowbits;
bin->full_position = (uint32_t)(uintptr_t)stack;
expect_ptr_eq(cache_bin_empty_position_get(bin, 0), empty_position,
"Incorrect empty position");
expect_ptr_eq(cache_bin_empty_position_get(bin, 0, tcache_bin_info),
empty_position, "Incorrect empty position");
/* Not using expect_zu etc on cache_bin_sz_t since it may change. */
expect_true(cache_bin_ncached_get(bin, 0) == 0, "Incorrect cache size");
expect_true(cache_bin_ncached_get(bin, 0, tcache_bin_info) == 0,
"Incorrect cache size");
bool success;
void *ret = cache_bin_alloc_easy(bin, &success, 0);
void *ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info);
expect_false(success, "Empty cache bin should not alloc");
expect_true(cache_bin_low_water_get(bin, 0) == 0,
expect_true(cache_bin_low_water_get(bin, 0, tcache_bin_info) == 0,
"Incorrect low water mark");
cache_bin_ncached_set(bin, 0, 0);
cache_bin_ncached_set(bin, 0, 0, tcache_bin_info);
expect_ptr_eq(bin->cur_ptr.ptr, empty_position, "Bin should be empty");
for (cache_bin_sz_t i = 1; i < ncached_max + 1; i++) {
success = cache_bin_dalloc_easy(bin, (void *)(uintptr_t)i);
expect_true(success && cache_bin_ncached_get(bin, 0) == i,
"Bin dalloc failure");
expect_true(success && cache_bin_ncached_get(bin, 0,
tcache_bin_info) == i, "Bin dalloc failure");
}
success = cache_bin_dalloc_easy(bin, (void *)1);
expect_false(success, "Bin should be full");
expect_ptr_eq(bin->cur_ptr.ptr, stack, "Incorrect bin cur_ptr");
cache_bin_ncached_set(bin, 0, ncached_max);
cache_bin_ncached_set(bin, 0, ncached_max, tcache_bin_info);
expect_ptr_eq(bin->cur_ptr.ptr, stack, "cur_ptr should not change");
/* Emulate low water after refill. */
bin->low_water_position = bin->full_position;
for (cache_bin_sz_t i = ncached_max; i > 0; i--) {
ret = cache_bin_alloc_easy(bin, &success, 0);
cache_bin_sz_t ncached = cache_bin_ncached_get(bin, 0);
ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info);
cache_bin_sz_t ncached = cache_bin_ncached_get(bin, 0,
tcache_bin_info);
expect_true(success && ncached == i - 1,
"Cache bin alloc failure");
expect_ptr_eq(ret, (void *)(uintptr_t)i, "Bin alloc failure");
expect_true(cache_bin_low_water_get(bin, 0) == ncached,
"Incorrect low water mark");
expect_true(cache_bin_low_water_get(bin, 0, tcache_bin_info)
== ncached, "Incorrect low water mark");
}
ret = cache_bin_alloc_easy(bin, &success, 0);
ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info);
expect_false(success, "Empty cache bin should not alloc.");
expect_ptr_eq(bin->cur_ptr.ptr, stack + ncached_max,
"Bin should be empty");