Store ncached_max * ptr_size in tcache_bin_info.
With the cache bin metadata switched to pointers, ncached_max is usually accessed and timed by sizeof(ptr). Store the results in tcache_bin_info for direct access, and add a helper function for the ncached_max value.
This commit is contained in:
parent
7599c82d48
commit
937ca1db9f
@ -35,8 +35,8 @@ struct cache_bin_stats_s {
|
||||
*/
|
||||
typedef struct cache_bin_info_s cache_bin_info_t;
|
||||
struct cache_bin_info_s {
|
||||
/* Upper limit on ncached. */
|
||||
cache_bin_sz_t ncached_max;
|
||||
/* The size of the bin stack, i.e. ncached_max * sizeof(ptr). */
|
||||
cache_bin_sz_t stack_size;
|
||||
};
|
||||
extern cache_bin_info_t *tcache_bin_info;
|
||||
|
||||
@ -117,11 +117,18 @@ struct cache_bin_array_descriptor_s {
|
||||
* None of the cache_bin_*_get / _set functions is used on the fast path, which
|
||||
* relies on pointer comparisons to determine if the cache is full / empty.
|
||||
*/
|
||||
|
||||
/* Returns ncached_max: Upper limit on ncached. */
|
||||
static inline cache_bin_sz_t
|
||||
cache_bin_ncached_max_get(szind_t ind) {
|
||||
return tcache_bin_info[ind].stack_size / sizeof(void *);
|
||||
}
|
||||
|
||||
static inline cache_bin_sz_t
|
||||
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind) {
|
||||
cache_bin_sz_t n = tcache_bin_info[ind].ncached_max -
|
||||
(bin->cur_ptr.lowbits - bin->full_position) / sizeof(void *);
|
||||
assert(n >= 0 && n <= tcache_bin_info[ind].ncached_max);
|
||||
cache_bin_sz_t n = (tcache_bin_info[ind].stack_size +
|
||||
bin->full_position - bin->cur_ptr.lowbits) / sizeof(void *);
|
||||
assert(n >= 0 && n <= cache_bin_ncached_max_get(ind));
|
||||
assert(n == 0 || *(bin->cur_ptr.ptr) != NULL);
|
||||
|
||||
return n;
|
||||
@ -132,14 +139,13 @@ cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
|
||||
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind);
|
||||
/* Low bits overflow disallowed when allocating the space. */
|
||||
assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits);
|
||||
assert(bin->full_position + tcache_bin_info[ind].ncached_max *
|
||||
sizeof(void *) > bin->full_position);
|
||||
|
||||
/* Can also be computed via (full_position + ncached_max) | highbits. */
|
||||
assert(ret == (void **)((uintptr_t)(bin->full_position +
|
||||
tcache_bin_info[ind].ncached_max * sizeof(void *)) |
|
||||
(uintptr_t)((uintptr_t)bin->cur_ptr.ptr &
|
||||
~(((uint64_t)1 << 32) - 1))));
|
||||
uintptr_t lowbits = bin->full_position +
|
||||
tcache_bin_info[ind].stack_size;
|
||||
uintptr_t highbits = (uintptr_t)bin->cur_ptr.ptr &
|
||||
~(((uint64_t)1 << 32) - 1);
|
||||
assert(ret == (void **)(lowbits | highbits));
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -156,10 +162,10 @@ cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind) {
|
||||
/* Returns the numeric value of low water in [-1, ncached]. */
|
||||
static inline cache_bin_sz_t
|
||||
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) {
|
||||
cache_bin_sz_t low_water = tcache_bin_info[ind].ncached_max -
|
||||
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind);
|
||||
cache_bin_sz_t low_water = ncached_max -
|
||||
(bin->low_water_position - bin->full_position) / sizeof(void *);
|
||||
assert(low_water >= -1 && low_water <=
|
||||
tcache_bin_info[ind].ncached_max);
|
||||
assert(low_water >= -1 && low_water <= ncached_max);
|
||||
assert(low_water <= cache_bin_ncached_get(bin, ind));
|
||||
assert(bin->low_water_position >= bin->cur_ptr.lowbits);
|
||||
|
||||
@ -169,8 +175,8 @@ cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) {
|
||||
static inline void
|
||||
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n) {
|
||||
bin->cur_ptr.lowbits = bin->full_position +
|
||||
(tcache_bin_info[ind].ncached_max - n) * sizeof(void *);
|
||||
assert(n >= 0 && n <= tcache_bin_info[ind].ncached_max);
|
||||
tcache_bin_info[ind].stack_size - n * sizeof(void *);
|
||||
assert(n >= 0 && n <= cache_bin_ncached_max_get(ind));
|
||||
assert(n == 0 || *bin->cur_ptr.ptr != NULL);
|
||||
}
|
||||
|
||||
@ -197,7 +203,7 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success, cache_bin_sz_t ind) {
|
||||
if (unlikely(bin->cur_ptr.lowbits >= bin->low_water_position)) {
|
||||
bin->low_water_position = bin->cur_ptr.lowbits;
|
||||
uint32_t empty_position = bin->full_position +
|
||||
tcache_bin_info[ind].ncached_max * sizeof(void *);
|
||||
tcache_bin_info[ind].stack_size;
|
||||
if (bin->cur_ptr.lowbits > empty_position) {
|
||||
bin->cur_ptr.ptr--;
|
||||
assert(bin->cur_ptr.lowbits == empty_position);
|
||||
|
@ -174,8 +174,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
|
||||
|
||||
bin = tcache_small_bin_get(tcache, binind);
|
||||
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
|
||||
tcache_bin_flush_small(tsd, tcache, bin, binind,
|
||||
tcache_bin_info[binind].ncached_max >> 1);
|
||||
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
|
||||
tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
|
||||
bool ret = cache_bin_dalloc_easy(bin, ptr);
|
||||
assert(ret);
|
||||
}
|
||||
@ -198,8 +198,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
|
||||
|
||||
bin = tcache_large_bin_get(tcache, binind);
|
||||
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
|
||||
tcache_bin_flush_large(tsd, tcache, bin, binind,
|
||||
tcache_bin_info[binind].ncached_max >> 1);
|
||||
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
|
||||
tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
|
||||
bool ret = cache_bin_dalloc_easy(bin, ptr);
|
||||
assert(ret);
|
||||
}
|
||||
|
@ -1392,7 +1392,7 @@ arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
|
||||
bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard);
|
||||
|
||||
void **empty_position = cache_bin_empty_position_get(tbin, binind);
|
||||
for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >>
|
||||
for (i = 0, nfill = (cache_bin_ncached_max_get(binind) >>
|
||||
tcache->lg_fill_div[binind]); i < nfill; i += cnt) {
|
||||
extent_t *slab;
|
||||
if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) >
|
||||
|
34
src/tcache.c
34
src/tcache.c
@ -70,8 +70,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
|
||||
* Reduce fill count by 2X. Limit lg_fill_div such that
|
||||
* the fill count is always at least 1.
|
||||
*/
|
||||
cache_bin_info_t *tbin_info = &tcache_bin_info[binind];
|
||||
if ((tbin_info->ncached_max >>
|
||||
if ((cache_bin_ncached_max_get(binind) >>
|
||||
(tcache->lg_fill_div[binind] + 1)) >= 1) {
|
||||
tcache->lg_fill_div[binind]++;
|
||||
}
|
||||
@ -431,8 +430,7 @@ tcache_bin_init(cache_bin_t *bin, szind_t ind, uintptr_t *stack_cur) {
|
||||
* adjacent prefetch).
|
||||
*/
|
||||
void *full_position = (void *)*stack_cur;
|
||||
uint32_t bin_stack_size = tcache_bin_info[ind].ncached_max *
|
||||
sizeof(void *);
|
||||
uint32_t bin_stack_size = tcache_bin_info[ind].stack_size;
|
||||
|
||||
*stack_cur += bin_stack_size;
|
||||
void *empty_position = (void *)*stack_cur;
|
||||
@ -608,8 +606,8 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
|
||||
assert(cache_bin_ncached_get(bin, 0) == 0);
|
||||
assert(cache_bin_empty_position_get(bin, 0) ==
|
||||
bin->cur_ptr.ptr);
|
||||
void *avail_array = bin->cur_ptr.ptr -
|
||||
tcache_bin_info[0].ncached_max;
|
||||
void *avail_array = (void *)((uintptr_t)bin->cur_ptr.ptr -
|
||||
tcache_bin_info[0].stack_size);
|
||||
idalloctm(tsd_tsdn(tsd), avail_array, NULL, NULL, true, true);
|
||||
} else {
|
||||
/* Release both the tcache struct and avail array. */
|
||||
@ -810,27 +808,27 @@ tcache_boot(tsdn_t *tsdn) {
|
||||
if (tcache_bin_info == NULL) {
|
||||
return true;
|
||||
}
|
||||
unsigned i, stack_nelms;
|
||||
stack_nelms = 0;
|
||||
unsigned i, ncached_max;
|
||||
total_stack_bytes = 0;
|
||||
for (i = 0; i < SC_NBINS; i++) {
|
||||
if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
|
||||
tcache_bin_info[i].ncached_max =
|
||||
TCACHE_NSLOTS_SMALL_MIN;
|
||||
ncached_max = TCACHE_NSLOTS_SMALL_MIN;
|
||||
} else if ((bin_infos[i].nregs << 1) <=
|
||||
TCACHE_NSLOTS_SMALL_MAX) {
|
||||
tcache_bin_info[i].ncached_max =
|
||||
(bin_infos[i].nregs << 1);
|
||||
ncached_max = bin_infos[i].nregs << 1;
|
||||
} else {
|
||||
tcache_bin_info[i].ncached_max =
|
||||
TCACHE_NSLOTS_SMALL_MAX;
|
||||
ncached_max = TCACHE_NSLOTS_SMALL_MAX;
|
||||
}
|
||||
stack_nelms += tcache_bin_info[i].ncached_max;
|
||||
unsigned stack_size = ncached_max * sizeof(void *);
|
||||
tcache_bin_info[i].stack_size = stack_size;
|
||||
total_stack_bytes += stack_size;
|
||||
}
|
||||
for (; i < nhbins; i++) {
|
||||
tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
|
||||
stack_nelms += tcache_bin_info[i].ncached_max;
|
||||
unsigned stack_size = TCACHE_NSLOTS_LARGE * sizeof(void *);
|
||||
tcache_bin_info[i].stack_size = stack_size;
|
||||
total_stack_bytes += stack_size;
|
||||
}
|
||||
total_stack_bytes = stack_nelms * sizeof(void *) + total_stack_padding;
|
||||
total_stack_bytes += total_stack_padding;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
@ -10,7 +10,7 @@ TEST_BEGIN(test_cache_bin) {
|
||||
|
||||
assert_ptr_not_null(stack, "Unexpected mallocx failure");
|
||||
/* Initialize to empty; bin 0. */
|
||||
cache_bin_sz_t ncached_max = tcache_bin_info[0].ncached_max;
|
||||
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0);
|
||||
void **empty_position = stack + ncached_max;
|
||||
bin->cur_ptr.ptr = empty_position;
|
||||
bin->low_water_position = bin->cur_ptr.lowbits;
|
||||
|
Loading…
Reference in New Issue
Block a user