project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cache_bin: Shouldn't know about tcache.

Browse Source 
Instead, have it take the cache_bin_info_ts to use by pointer.  While we're
here, add a src file for the cache bin.
This commit is contained in:
David Goldblatt
2020-02-25 12:14:48 -08:00
committed by David Goldblatt
parent 79f1ee2fc0
commit 909c501b07
12 changed files with 101 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
include/jemalloc/internal/cache_bin.h
View File

@@ -35,7 +35,6 @@ struct cache_bin_info_s {
/* The size of the bin stack, i.e. ncached_max * sizeof(ptr). */
cache_bin_sz_t stack_size;
};
extern cache_bin_info_t *tcache_bin_info;
typedef struct cache_bin_s cache_bin_t;
struct cache_bin_s {
@@ -115,29 +114,29 @@ struct cache_bin_array_descriptor_s {
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_ncached_max_get(szind_t ind) {
return tcache_bin_info[ind].stack_size / sizeof(void *);
cache_bin_ncached_max_get(szind_t ind, cache_bin_info_t *infos) {
return infos[ind].stack_size / sizeof(void *);
}
static inline cache_bin_sz_t
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind) {
cache_bin_sz_t n = (cache_bin_sz_t)((tcache_bin_info[ind].stack_size +
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind, cache_bin_info_t *infos) {
cache_bin_sz_t n = (cache_bin_sz_t)((infos[ind].stack_size +
bin->full_position - bin->cur_ptr.lowbits) / sizeof(void *));
assert(n <= cache_bin_ncached_max_get(ind));
assert(n <= cache_bin_ncached_max_get(ind, infos));
assert(n == 0 || *(bin->cur_ptr.ptr) != NULL);
return n;
}
static inline void **
cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind);
cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind,
cache_bin_info_t *infos) {
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind, infos);
/* Low bits overflow disallowed when allocating the space. */
assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits);
/* Can also be computed via (full_position + ncached_max) | highbits. */
uintptr_t lowbits = bin->full_position +
tcache_bin_info[ind].stack_size;
uintptr_t lowbits = bin->full_position + infos[ind].stack_size;
uintptr_t highbits = (uintptr_t)bin->cur_ptr.ptr &
~(((uint64_t)1 << 32) - 1);
assert(ret == (void **)(lowbits | highbits));
@@ -147,32 +146,35 @@ cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
/* Returns the position of the bottom item on the stack; for convenience. */
static inline void **
cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind) {
void **bottom = cache_bin_empty_position_get(bin, ind) - 1;
assert(cache_bin_ncached_get(bin, ind) == 0 || *bottom != NULL);
cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind,
cache_bin_info_t *infos) {
void **bottom = cache_bin_empty_position_get(bin, ind, infos) - 1;
assert(cache_bin_ncached_get(bin, ind, infos) == 0 || *bottom != NULL);
return bottom;
}
/* Returns the numeric value of low water in [0, ncached]. */
static inline cache_bin_sz_t
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind);
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind,
cache_bin_info_t *infos) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind, infos);
cache_bin_sz_t low_water = ncached_max -
(cache_bin_sz_t)((bin->low_water_position - bin->full_position) /
sizeof(void *));
assert(low_water <= ncached_max);
assert(low_water <= cache_bin_ncached_get(bin, ind));
assert(low_water <= cache_bin_ncached_get(bin, ind, infos));
assert(bin->low_water_position >= bin->cur_ptr.lowbits);
return low_water;
}
static inline void
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n) {
bin->cur_ptr.lowbits = bin->full_position +
tcache_bin_info[ind].stack_size - n * sizeof(void *);
assert(n <= cache_bin_ncached_max_get(ind));
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n,
cache_bin_info_t *infos) {
bin->cur_ptr.lowbits = bin->full_position + infos[ind].stack_size
- n * sizeof(void *);
assert(n <= cache_bin_ncached_max_get(ind, infos));
assert(n == 0 || *bin->cur_ptr.ptr != NULL);
}
@@ -188,7 +190,7 @@ cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor,
JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
const bool adjust_low_water) {
cache_bin_info_t *infos, const bool adjust_low_water) {
/*
* This may read from the empty position; however the loaded value won't
* be used. It's safe because the stack has one more slot reserved.
@@ -197,14 +199,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
/*
* Check for both bin->ncached == 0 and ncached < low_water in a single
* branch. When adjust_low_water is true, this also avoids accessing
* tcache_bin_info (which is on a separate cacheline / page) in the
* common case.
* the cache_bin_info_ts (which is on a separate cacheline / page) in
* the common case.
*/
if (unlikely(bin->cur_ptr.lowbits > bin->low_water_position)) {
if (adjust_low_water) {
assert(ind != INVALID_SZIND);
uint32_t empty_position = bin->full_position +
tcache_bin_info[ind].stack_size;
infos[ind].stack_size;
if (unlikely(bin->cur_ptr.lowbits > empty_position)) {
/* Over-allocated; revert. */
bin->cur_ptr.ptr--;
@@ -237,12 +239,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy_reduced(cache_bin_t *bin, bool *success) {
/* The szind parameter won't be used. */
return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND, false);
return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND,
/* infos */ NULL, false);
}
JEMALLOC_ALWAYS_INLINE void *
cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind) {
return cache_bin_alloc_easy_impl(bin, success, ind, true);
cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind,
cache_bin_info_t *infos) {
return cache_bin_alloc_easy_impl(bin, success, ind, infos, true);
}
#undef INVALID_SZIND

2
include/jemalloc/internal/tcache_externs.h
View File

@@ -13,6 +13,8 @@ extern unsigned nhbins;
/* Maximum cached size class. */
extern size_t tcache_maxclass;
extern cache_bin_info_t *tcache_bin_info;
/*
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are

12
include/jemalloc/internal/tcache_inlines.h
View File

@@ -36,7 +36,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
assert(binind < SC_NBINS);
bin = tcache_small_bin_get(tcache, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind,
tcache_bin_info);
assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) {
bool tcache_hard_success;
@@ -79,7 +80,8 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
assert(binind >= SC_NBINS &&binind < nhbins);
bin = tcache_large_bin_get(tcache, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind);
ret = cache_bin_alloc_easy(bin, &tcache_success, binind,
tcache_bin_info);
assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) {
/*
@@ -127,7 +129,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bin = tcache_small_bin_get(tcache, binind);
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
unsigned remain = cache_bin_ncached_max_get(binind,
tcache_bin_info) >> 1;
tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);
@@ -145,7 +148,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bin = tcache_large_bin_get(tcache, binind);
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
unsigned remain = cache_bin_ncached_max_get(binind,
tcache_bin_info) >> 1;
tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret);