project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Store ncached_max * ptr_size in tcache_bin_info.

Browse Source 
With the cache bin metadata switched to pointers, ncached_max is usually
accessed and timed by sizeof(ptr). Store the results in tcache_bin_info for
direct access, and add a helper function for the ncached_max value.
This commit is contained in:
Qi Wang 2019-08-14 13:08:06 -07:00 committed by Qi Wang
parent 7599c82d48
commit 937ca1db9f
5 changed files with 45 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
include/jemalloc/internal/cache_bin.h
View File

@ -35,8 +35,8 @@ struct cache_bin_stats_s {
*/ */
typedef struct cache_bin_info_s cache_bin_info_t; typedef struct cache_bin_info_s cache_bin_info_t;
struct cache_bin_info_s { struct cache_bin_info_s {
/* Upper limit on ncached. */ /* The size of the bin stack, i.e. ncached_max * sizeof(ptr). */
cache_bin_sz_t ncached_max; cache_bin_sz_t stack_size;
}; };
extern cache_bin_info_t *tcache_bin_info; extern cache_bin_info_t *tcache_bin_info;
@ -117,11 +117,18 @@ struct cache_bin_array_descriptor_s {
* None of the cache_bin_*_get / _set functions is used on the fast path, which * None of the cache_bin_*_get / _set functions is used on the fast path, which
* relies on pointer comparisons to determine if the cache is full / empty. * relies on pointer comparisons to determine if the cache is full / empty.
*/ */
/* Returns ncached_max: Upper limit on ncached. */
static inline cache_bin_sz_t
cache_bin_ncached_max_get(szind_t ind) {
return tcache_bin_info[ind].stack_size / sizeof(void *);
}
static inline cache_bin_sz_t static inline cache_bin_sz_t
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind) { cache_bin_ncached_get(cache_bin_t *bin, szind_t ind) {
cache_bin_sz_t n = tcache_bin_info[ind].ncached_max - cache_bin_sz_t n = (tcache_bin_info[ind].stack_size +
(bin->cur_ptr.lowbits - bin->full_position) / sizeof(void *); bin->full_position - bin->cur_ptr.lowbits) / sizeof(void *);
assert(n >= 0 && n <= tcache_bin_info[ind].ncached_max); assert(n >= 0 && n <= cache_bin_ncached_max_get(ind));
assert(n == 0 || *(bin->cur_ptr.ptr) != NULL); assert(n == 0 || *(bin->cur_ptr.ptr) != NULL);
return n; return n;
@ -132,14 +139,13 @@ cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind); void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind);
/* Low bits overflow disallowed when allocating the space. */ /* Low bits overflow disallowed when allocating the space. */
assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits); assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits);
assert(bin->full_position + tcache_bin_info[ind].ncached_max *
sizeof(void *) > bin->full_position);
/* Can also be computed via (full_position + ncached_max) | highbits. */ /* Can also be computed via (full_position + ncached_max) | highbits. */
assert(ret == (void **)((uintptr_t)(bin->full_position + uintptr_t lowbits = bin->full_position +
tcache_bin_info[ind].ncached_max * sizeof(void *)) | tcache_bin_info[ind].stack_size;
(uintptr_t)((uintptr_t)bin->cur_ptr.ptr & uintptr_t highbits = (uintptr_t)bin->cur_ptr.ptr &
~(((uint64_t)1 << 32) - 1)))); ~(((uint64_t)1 << 32) - 1);
assert(ret == (void **)(lowbits | highbits));
return ret; return ret;
} }
@ -156,10 +162,10 @@ cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind) {
/* Returns the numeric value of low water in [-1, ncached]. */ /* Returns the numeric value of low water in [-1, ncached]. */
static inline cache_bin_sz_t static inline cache_bin_sz_t
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) { cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) {
cache_bin_sz_t low_water = tcache_bin_info[ind].ncached_max - cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind);
cache_bin_sz_t low_water = ncached_max -
(bin->low_water_position - bin->full_position) / sizeof(void *); (bin->low_water_position - bin->full_position) / sizeof(void *);
assert(low_water >= -1 && low_water <= assert(low_water >= -1 && low_water <= ncached_max);
tcache_bin_info[ind].ncached_max);
assert(low_water <= cache_bin_ncached_get(bin, ind)); assert(low_water <= cache_bin_ncached_get(bin, ind));
assert(bin->low_water_position >= bin->cur_ptr.lowbits); assert(bin->low_water_position >= bin->cur_ptr.lowbits);
@ -169,8 +175,8 @@ cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) {
static inline void static inline void
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n) { cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n) {
bin->cur_ptr.lowbits = bin->full_position + bin->cur_ptr.lowbits = bin->full_position +
(tcache_bin_info[ind].ncached_max - n) * sizeof(void *); tcache_bin_info[ind].stack_size - n * sizeof(void *);
assert(n >= 0 && n <= tcache_bin_info[ind].ncached_max); assert(n >= 0 && n <= cache_bin_ncached_max_get(ind));
assert(n == 0 || *bin->cur_ptr.ptr != NULL); assert(n == 0 || *bin->cur_ptr.ptr != NULL);
} }
@ -197,7 +203,7 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success, cache_bin_sz_t ind) {
if (unlikely(bin->cur_ptr.lowbits >= bin->low_water_position)) { if (unlikely(bin->cur_ptr.lowbits >= bin->low_water_position)) {
bin->low_water_position = bin->cur_ptr.lowbits; bin->low_water_position = bin->cur_ptr.lowbits;
uint32_t empty_position = bin->full_position + uint32_t empty_position = bin->full_position +
tcache_bin_info[ind].ncached_max * sizeof(void *); tcache_bin_info[ind].stack_size;
if (bin->cur_ptr.lowbits > empty_position) { if (bin->cur_ptr.lowbits > empty_position) {
bin->cur_ptr.ptr--; bin->cur_ptr.ptr--;
assert(bin->cur_ptr.lowbits == empty_position); assert(bin->cur_ptr.lowbits == empty_position);

8
include/jemalloc/internal/tcache_inlines.h
View File

@ -174,8 +174,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bin = tcache_small_bin_get(tcache, binind); bin = tcache_small_bin_get(tcache, binind);
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
tcache_bin_flush_small(tsd, tcache, bin, binind, unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
tcache_bin_info[binind].ncached_max >> 1); tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr); bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret); assert(ret);
} }
@ -198,8 +198,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
bin = tcache_large_bin_get(tcache, binind); bin = tcache_large_bin_get(tcache, binind);
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
tcache_bin_flush_large(tsd, tcache, bin, binind, unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
tcache_bin_info[binind].ncached_max >> 1); tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr); bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret); assert(ret);
} }

2
src/arena.c
View File

@ -1392,7 +1392,7 @@ arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard);
void **empty_position = cache_bin_empty_position_get(tbin, binind); void **empty_position = cache_bin_empty_position_get(tbin, binind);
for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> for (i = 0, nfill = (cache_bin_ncached_max_get(binind) >>
tcache->lg_fill_div[binind]); i < nfill; i += cnt) { tcache->lg_fill_div[binind]); i < nfill; i += cnt) {
extent_t *slab; extent_t *slab;
if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) >

34
src/tcache.c
View File

@ -70,8 +70,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
* Reduce fill count by 2X. Limit lg_fill_div such that * Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1. * the fill count is always at least 1.
*/ */
cache_bin_info_t *tbin_info = &tcache_bin_info[binind]; if ((cache_bin_ncached_max_get(binind) >>
if ((tbin_info->ncached_max >>
(tcache->lg_fill_div[binind] + 1)) >= 1) { (tcache->lg_fill_div[binind] + 1)) >= 1) {
tcache->lg_fill_div[binind]++; tcache->lg_fill_div[binind]++;
} }
@ -431,8 +430,7 @@ tcache_bin_init(cache_bin_t *bin, szind_t ind, uintptr_t *stack_cur) {
* adjacent prefetch). * adjacent prefetch).
*/ */
void *full_position = (void *)*stack_cur; void *full_position = (void *)*stack_cur;
uint32_t bin_stack_size = tcache_bin_info[ind].ncached_max * uint32_t bin_stack_size = tcache_bin_info[ind].stack_size;
sizeof(void *);
*stack_cur += bin_stack_size; *stack_cur += bin_stack_size;
void *empty_position = (void *)*stack_cur; void *empty_position = (void *)*stack_cur;
@ -608,8 +606,8 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
assert(cache_bin_ncached_get(bin, 0) == 0); assert(cache_bin_ncached_get(bin, 0) == 0);
assert(cache_bin_empty_position_get(bin, 0) == assert(cache_bin_empty_position_get(bin, 0) ==
bin->cur_ptr.ptr); bin->cur_ptr.ptr);
void *avail_array = bin->cur_ptr.ptr - void *avail_array = (void *)((uintptr_t)bin->cur_ptr.ptr -
tcache_bin_info[0].ncached_max; tcache_bin_info[0].stack_size);
idalloctm(tsd_tsdn(tsd), avail_array, NULL, NULL, true, true); idalloctm(tsd_tsdn(tsd), avail_array, NULL, NULL, true, true);
} else { } else {
/* Release both the tcache struct and avail array. */ /* Release both the tcache struct and avail array. */
@ -810,27 +808,27 @@ tcache_boot(tsdn_t *tsdn) {
if (tcache_bin_info == NULL) { if (tcache_bin_info == NULL) {
return true; return true;
} }
unsigned i, stack_nelms; unsigned i, ncached_max;
stack_nelms = 0; total_stack_bytes = 0;
for (i = 0; i < SC_NBINS; i++) { for (i = 0; i < SC_NBINS; i++) {
if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
tcache_bin_info[i].ncached_max = ncached_max = TCACHE_NSLOTS_SMALL_MIN;
TCACHE_NSLOTS_SMALL_MIN;
} else if ((bin_infos[i].nregs << 1) <= } else if ((bin_infos[i].nregs << 1) <=
TCACHE_NSLOTS_SMALL_MAX) { TCACHE_NSLOTS_SMALL_MAX) {
tcache_bin_info[i].ncached_max = ncached_max = bin_infos[i].nregs << 1;
(bin_infos[i].nregs << 1);
} else { } else {
tcache_bin_info[i].ncached_max = ncached_max = TCACHE_NSLOTS_SMALL_MAX;
TCACHE_NSLOTS_SMALL_MAX;
} }
stack_nelms += tcache_bin_info[i].ncached_max; unsigned stack_size = ncached_max * sizeof(void *);
tcache_bin_info[i].stack_size = stack_size;
total_stack_bytes += stack_size;
} }
for (; i < nhbins; i++) { for (; i < nhbins; i++) {
tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; unsigned stack_size = TCACHE_NSLOTS_LARGE * sizeof(void *);
stack_nelms += tcache_bin_info[i].ncached_max; tcache_bin_info[i].stack_size = stack_size;
total_stack_bytes += stack_size;
} }
total_stack_bytes = stack_nelms * sizeof(void *) + total_stack_padding; total_stack_bytes += total_stack_padding;
return false; return false;
} }

2
test/unit/cache_bin.c
View File

@ -10,7 +10,7 @@ TEST_BEGIN(test_cache_bin) {
assert_ptr_not_null(stack, "Unexpected mallocx failure"); assert_ptr_not_null(stack, "Unexpected mallocx failure");
/* Initialize to empty; bin 0. */ /* Initialize to empty; bin 0. */
cache_bin_sz_t ncached_max = tcache_bin_info[0].ncached_max; cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0);
void **empty_position = stack + ncached_max; void **empty_position = stack + ncached_max;
bin->cur_ptr.ptr = empty_position; bin->cur_ptr.ptr = empty_position;
bin->low_water_position = bin->cur_ptr.lowbits; bin->low_water_position = bin->cur_ptr.lowbits;