Cache_bin: Shouldn't know about tcache.
Instead, have it take the cache_bin_info_ts to use by pointer. While we're here, add a src file for the cache bin.
This commit is contained in:
parent
79f1ee2fc0
commit
909c501b07
@ -102,6 +102,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
|
|||||||
$(srcroot)src/bin_info.c \
|
$(srcroot)src/bin_info.c \
|
||||||
$(srcroot)src/bitmap.c \
|
$(srcroot)src/bitmap.c \
|
||||||
$(srcroot)src/buf_writer.c \
|
$(srcroot)src/buf_writer.c \
|
||||||
|
$(srcroot)src/cache_bin.c \
|
||||||
$(srcroot)src/ckh.c \
|
$(srcroot)src/ckh.c \
|
||||||
$(srcroot)src/counter.c \
|
$(srcroot)src/counter.c \
|
||||||
$(srcroot)src/ctl.c \
|
$(srcroot)src/ctl.c \
|
||||||
|
@ -35,7 +35,6 @@ struct cache_bin_info_s {
|
|||||||
/* The size of the bin stack, i.e. ncached_max * sizeof(ptr). */
|
/* The size of the bin stack, i.e. ncached_max * sizeof(ptr). */
|
||||||
cache_bin_sz_t stack_size;
|
cache_bin_sz_t stack_size;
|
||||||
};
|
};
|
||||||
extern cache_bin_info_t *tcache_bin_info;
|
|
||||||
|
|
||||||
typedef struct cache_bin_s cache_bin_t;
|
typedef struct cache_bin_s cache_bin_t;
|
||||||
struct cache_bin_s {
|
struct cache_bin_s {
|
||||||
@ -115,29 +114,29 @@ struct cache_bin_array_descriptor_s {
|
|||||||
|
|
||||||
/* Returns ncached_max: Upper limit on ncached. */
|
/* Returns ncached_max: Upper limit on ncached. */
|
||||||
static inline cache_bin_sz_t
|
static inline cache_bin_sz_t
|
||||||
cache_bin_ncached_max_get(szind_t ind) {
|
cache_bin_ncached_max_get(szind_t ind, cache_bin_info_t *infos) {
|
||||||
return tcache_bin_info[ind].stack_size / sizeof(void *);
|
return infos[ind].stack_size / sizeof(void *);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline cache_bin_sz_t
|
static inline cache_bin_sz_t
|
||||||
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind) {
|
cache_bin_ncached_get(cache_bin_t *bin, szind_t ind, cache_bin_info_t *infos) {
|
||||||
cache_bin_sz_t n = (cache_bin_sz_t)((tcache_bin_info[ind].stack_size +
|
cache_bin_sz_t n = (cache_bin_sz_t)((infos[ind].stack_size +
|
||||||
bin->full_position - bin->cur_ptr.lowbits) / sizeof(void *));
|
bin->full_position - bin->cur_ptr.lowbits) / sizeof(void *));
|
||||||
assert(n <= cache_bin_ncached_max_get(ind));
|
assert(n <= cache_bin_ncached_max_get(ind, infos));
|
||||||
assert(n == 0 || *(bin->cur_ptr.ptr) != NULL);
|
assert(n == 0 || *(bin->cur_ptr.ptr) != NULL);
|
||||||
|
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void **
|
static inline void **
|
||||||
cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
|
cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind,
|
||||||
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind);
|
cache_bin_info_t *infos) {
|
||||||
|
void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind, infos);
|
||||||
/* Low bits overflow disallowed when allocating the space. */
|
/* Low bits overflow disallowed when allocating the space. */
|
||||||
assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits);
|
assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits);
|
||||||
|
|
||||||
/* Can also be computed via (full_position + ncached_max) | highbits. */
|
/* Can also be computed via (full_position + ncached_max) | highbits. */
|
||||||
uintptr_t lowbits = bin->full_position +
|
uintptr_t lowbits = bin->full_position + infos[ind].stack_size;
|
||||||
tcache_bin_info[ind].stack_size;
|
|
||||||
uintptr_t highbits = (uintptr_t)bin->cur_ptr.ptr &
|
uintptr_t highbits = (uintptr_t)bin->cur_ptr.ptr &
|
||||||
~(((uint64_t)1 << 32) - 1);
|
~(((uint64_t)1 << 32) - 1);
|
||||||
assert(ret == (void **)(lowbits | highbits));
|
assert(ret == (void **)(lowbits | highbits));
|
||||||
@ -147,32 +146,35 @@ cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) {
|
|||||||
|
|
||||||
/* Returns the position of the bottom item on the stack; for convenience. */
|
/* Returns the position of the bottom item on the stack; for convenience. */
|
||||||
static inline void **
|
static inline void **
|
||||||
cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind) {
|
cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind,
|
||||||
void **bottom = cache_bin_empty_position_get(bin, ind) - 1;
|
cache_bin_info_t *infos) {
|
||||||
assert(cache_bin_ncached_get(bin, ind) == 0 || *bottom != NULL);
|
void **bottom = cache_bin_empty_position_get(bin, ind, infos) - 1;
|
||||||
|
assert(cache_bin_ncached_get(bin, ind, infos) == 0 || *bottom != NULL);
|
||||||
|
|
||||||
return bottom;
|
return bottom;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Returns the numeric value of low water in [0, ncached]. */
|
/* Returns the numeric value of low water in [0, ncached]. */
|
||||||
static inline cache_bin_sz_t
|
static inline cache_bin_sz_t
|
||||||
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) {
|
cache_bin_low_water_get(cache_bin_t *bin, szind_t ind,
|
||||||
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind);
|
cache_bin_info_t *infos) {
|
||||||
|
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind, infos);
|
||||||
cache_bin_sz_t low_water = ncached_max -
|
cache_bin_sz_t low_water = ncached_max -
|
||||||
(cache_bin_sz_t)((bin->low_water_position - bin->full_position) /
|
(cache_bin_sz_t)((bin->low_water_position - bin->full_position) /
|
||||||
sizeof(void *));
|
sizeof(void *));
|
||||||
assert(low_water <= ncached_max);
|
assert(low_water <= ncached_max);
|
||||||
assert(low_water <= cache_bin_ncached_get(bin, ind));
|
assert(low_water <= cache_bin_ncached_get(bin, ind, infos));
|
||||||
assert(bin->low_water_position >= bin->cur_ptr.lowbits);
|
assert(bin->low_water_position >= bin->cur_ptr.lowbits);
|
||||||
|
|
||||||
return low_water;
|
return low_water;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void
|
static inline void
|
||||||
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n) {
|
cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n,
|
||||||
bin->cur_ptr.lowbits = bin->full_position +
|
cache_bin_info_t *infos) {
|
||||||
tcache_bin_info[ind].stack_size - n * sizeof(void *);
|
bin->cur_ptr.lowbits = bin->full_position + infos[ind].stack_size
|
||||||
assert(n <= cache_bin_ncached_max_get(ind));
|
- n * sizeof(void *);
|
||||||
|
assert(n <= cache_bin_ncached_max_get(ind, infos));
|
||||||
assert(n == 0 || *bin->cur_ptr.ptr != NULL);
|
assert(n == 0 || *bin->cur_ptr.ptr != NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -188,7 +190,7 @@ cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor,
|
|||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE void *
|
JEMALLOC_ALWAYS_INLINE void *
|
||||||
cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
|
cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
|
||||||
const bool adjust_low_water) {
|
cache_bin_info_t *infos, const bool adjust_low_water) {
|
||||||
/*
|
/*
|
||||||
* This may read from the empty position; however the loaded value won't
|
* This may read from the empty position; however the loaded value won't
|
||||||
* be used. It's safe because the stack has one more slot reserved.
|
* be used. It's safe because the stack has one more slot reserved.
|
||||||
@ -197,14 +199,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
|
|||||||
/*
|
/*
|
||||||
* Check for both bin->ncached == 0 and ncached < low_water in a single
|
* Check for both bin->ncached == 0 and ncached < low_water in a single
|
||||||
* branch. When adjust_low_water is true, this also avoids accessing
|
* branch. When adjust_low_water is true, this also avoids accessing
|
||||||
* tcache_bin_info (which is on a separate cacheline / page) in the
|
* the cache_bin_info_ts (which is on a separate cacheline / page) in
|
||||||
* common case.
|
* the common case.
|
||||||
*/
|
*/
|
||||||
if (unlikely(bin->cur_ptr.lowbits > bin->low_water_position)) {
|
if (unlikely(bin->cur_ptr.lowbits > bin->low_water_position)) {
|
||||||
if (adjust_low_water) {
|
if (adjust_low_water) {
|
||||||
assert(ind != INVALID_SZIND);
|
assert(ind != INVALID_SZIND);
|
||||||
uint32_t empty_position = bin->full_position +
|
uint32_t empty_position = bin->full_position +
|
||||||
tcache_bin_info[ind].stack_size;
|
infos[ind].stack_size;
|
||||||
if (unlikely(bin->cur_ptr.lowbits > empty_position)) {
|
if (unlikely(bin->cur_ptr.lowbits > empty_position)) {
|
||||||
/* Over-allocated; revert. */
|
/* Over-allocated; revert. */
|
||||||
bin->cur_ptr.ptr--;
|
bin->cur_ptr.ptr--;
|
||||||
@ -237,12 +239,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind,
|
|||||||
JEMALLOC_ALWAYS_INLINE void *
|
JEMALLOC_ALWAYS_INLINE void *
|
||||||
cache_bin_alloc_easy_reduced(cache_bin_t *bin, bool *success) {
|
cache_bin_alloc_easy_reduced(cache_bin_t *bin, bool *success) {
|
||||||
/* The szind parameter won't be used. */
|
/* The szind parameter won't be used. */
|
||||||
return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND, false);
|
return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND,
|
||||||
|
/* infos */ NULL, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE void *
|
JEMALLOC_ALWAYS_INLINE void *
|
||||||
cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind) {
|
cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind,
|
||||||
return cache_bin_alloc_easy_impl(bin, success, ind, true);
|
cache_bin_info_t *infos) {
|
||||||
|
return cache_bin_alloc_easy_impl(bin, success, ind, infos, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
#undef INVALID_SZIND
|
#undef INVALID_SZIND
|
||||||
|
@ -13,6 +13,8 @@ extern unsigned nhbins;
|
|||||||
/* Maximum cached size class. */
|
/* Maximum cached size class. */
|
||||||
extern size_t tcache_maxclass;
|
extern size_t tcache_maxclass;
|
||||||
|
|
||||||
|
extern cache_bin_info_t *tcache_bin_info;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
|
* Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
|
||||||
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
|
* usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are
|
||||||
|
@ -36,7 +36,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
|||||||
|
|
||||||
assert(binind < SC_NBINS);
|
assert(binind < SC_NBINS);
|
||||||
bin = tcache_small_bin_get(tcache, binind);
|
bin = tcache_small_bin_get(tcache, binind);
|
||||||
ret = cache_bin_alloc_easy(bin, &tcache_success, binind);
|
ret = cache_bin_alloc_easy(bin, &tcache_success, binind,
|
||||||
|
tcache_bin_info);
|
||||||
assert(tcache_success == (ret != NULL));
|
assert(tcache_success == (ret != NULL));
|
||||||
if (unlikely(!tcache_success)) {
|
if (unlikely(!tcache_success)) {
|
||||||
bool tcache_hard_success;
|
bool tcache_hard_success;
|
||||||
@ -79,7 +80,8 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
|||||||
|
|
||||||
assert(binind >= SC_NBINS &&binind < nhbins);
|
assert(binind >= SC_NBINS &&binind < nhbins);
|
||||||
bin = tcache_large_bin_get(tcache, binind);
|
bin = tcache_large_bin_get(tcache, binind);
|
||||||
ret = cache_bin_alloc_easy(bin, &tcache_success, binind);
|
ret = cache_bin_alloc_easy(bin, &tcache_success, binind,
|
||||||
|
tcache_bin_info);
|
||||||
assert(tcache_success == (ret != NULL));
|
assert(tcache_success == (ret != NULL));
|
||||||
if (unlikely(!tcache_success)) {
|
if (unlikely(!tcache_success)) {
|
||||||
/*
|
/*
|
||||||
@ -127,7 +129,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
|
|||||||
|
|
||||||
bin = tcache_small_bin_get(tcache, binind);
|
bin = tcache_small_bin_get(tcache, binind);
|
||||||
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
|
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
|
||||||
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
|
unsigned remain = cache_bin_ncached_max_get(binind,
|
||||||
|
tcache_bin_info) >> 1;
|
||||||
tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
|
tcache_bin_flush_small(tsd, tcache, bin, binind, remain);
|
||||||
bool ret = cache_bin_dalloc_easy(bin, ptr);
|
bool ret = cache_bin_dalloc_easy(bin, ptr);
|
||||||
assert(ret);
|
assert(ret);
|
||||||
@ -145,7 +148,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
|
|||||||
|
|
||||||
bin = tcache_large_bin_get(tcache, binind);
|
bin = tcache_large_bin_get(tcache, binind);
|
||||||
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
|
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
|
||||||
unsigned remain = cache_bin_ncached_max_get(binind) >> 1;
|
unsigned remain = cache_bin_ncached_max_get(binind,
|
||||||
|
tcache_bin_info) >> 1;
|
||||||
tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
|
tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
|
||||||
bool ret = cache_bin_dalloc_easy(bin, ptr);
|
bool ret = cache_bin_dalloc_easy(bin, ptr);
|
||||||
assert(ret);
|
assert(ret);
|
||||||
|
@ -42,6 +42,7 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\bin_info.c" />
|
<ClCompile Include="..\..\..\..\src\bin_info.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\bitmap.c" />
|
<ClCompile Include="..\..\..\..\src\bitmap.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
|
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
|
||||||
|
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\ckh.c" />
|
<ClCompile Include="..\..\..\..\src\ckh.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\counter.c" />
|
<ClCompile Include="..\..\..\..\src\counter.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
||||||
|
@ -25,6 +25,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\buf_writer.c">
|
<ClCompile Include="..\..\..\..\src\buf_writer.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\cache_bin.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\ckh.c">
|
<ClCompile Include="..\..\..\..\src\ckh.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
@ -42,6 +42,7 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\bin_info.c" />
|
<ClCompile Include="..\..\..\..\src\bin_info.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\bitmap.c" />
|
<ClCompile Include="..\..\..\..\src\bitmap.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
|
<ClCompile Include="..\..\..\..\src\buf_writer.c" />
|
||||||
|
<ClCompile Include="..\..\..\..\src\cache_bin.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\ckh.c" />
|
<ClCompile Include="..\..\..\..\src\ckh.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\counter.c" />
|
<ClCompile Include="..\..\..\..\src\counter.c" />
|
||||||
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
<ClCompile Include="..\..\..\..\src\ctl.c" />
|
||||||
|
@ -25,6 +25,9 @@
|
|||||||
<ClCompile Include="..\..\..\..\src\buf_writer.c">
|
<ClCompile Include="..\..\..\..\src\buf_writer.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
<ClCompile Include="..\..\..\..\src\cache_bin.c">
|
||||||
|
<Filter>Source Files</Filter>
|
||||||
|
</ClCompile>
|
||||||
<ClCompile Include="..\..\..\..\src\ckh.c">
|
<ClCompile Include="..\..\..\..\src\ckh.c">
|
||||||
<Filter>Source Files</Filter>
|
<Filter>Source Files</Filter>
|
||||||
</ClCompile>
|
</ClCompile>
|
||||||
|
18
src/arena.c
18
src/arena.c
@ -200,13 +200,14 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
|
|||||||
for (szind_t i = 0; i < SC_NBINS; i++) {
|
for (szind_t i = 0; i < SC_NBINS; i++) {
|
||||||
cache_bin_t *tbin = &descriptor->bins_small[i];
|
cache_bin_t *tbin = &descriptor->bins_small[i];
|
||||||
arena_stats_accum_zu(&astats->tcache_bytes,
|
arena_stats_accum_zu(&astats->tcache_bytes,
|
||||||
cache_bin_ncached_get(tbin, i) * sz_index2size(i));
|
cache_bin_ncached_get(tbin, i, tcache_bin_info)
|
||||||
|
* sz_index2size(i));
|
||||||
}
|
}
|
||||||
for (szind_t i = 0; i < nhbins - SC_NBINS; i++) {
|
for (szind_t i = 0; i < nhbins - SC_NBINS; i++) {
|
||||||
cache_bin_t *tbin = &descriptor->bins_large[i];
|
cache_bin_t *tbin = &descriptor->bins_large[i];
|
||||||
arena_stats_accum_zu(&astats->tcache_bytes,
|
arena_stats_accum_zu(&astats->tcache_bytes,
|
||||||
cache_bin_ncached_get(tbin, i + SC_NBINS) *
|
cache_bin_ncached_get(tbin, i + SC_NBINS,
|
||||||
sz_index2size(i));
|
tcache_bin_info) * sz_index2size(i));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
malloc_mutex_prof_read(tsdn,
|
malloc_mutex_prof_read(tsdn,
|
||||||
@ -1320,13 +1321,14 @@ arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind,
|
|||||||
void
|
void
|
||||||
arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
|
arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
|
||||||
cache_bin_t *tbin, szind_t binind) {
|
cache_bin_t *tbin, szind_t binind) {
|
||||||
assert(cache_bin_ncached_get(tbin, binind) == 0);
|
assert(cache_bin_ncached_get(tbin, binind, tcache_bin_info) == 0);
|
||||||
tcache->bin_refilled[binind] = true;
|
tcache->bin_refilled[binind] = true;
|
||||||
|
|
||||||
const bin_info_t *bin_info = &bin_infos[binind];
|
const bin_info_t *bin_info = &bin_infos[binind];
|
||||||
const unsigned nfill = cache_bin_ncached_max_get(binind) >>
|
const unsigned nfill = cache_bin_ncached_max_get(binind,
|
||||||
tcache->lg_fill_div[binind];
|
tcache_bin_info) >> tcache->lg_fill_div[binind];
|
||||||
void **empty_position = cache_bin_empty_position_get(tbin, binind);
|
void **empty_position = cache_bin_empty_position_get(tbin, binind,
|
||||||
|
tcache_bin_info);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull
|
* Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull
|
||||||
@ -1446,7 +1448,7 @@ label_refill:
|
|||||||
fresh_slab = NULL;
|
fresh_slab = NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
cache_bin_ncached_set(tbin, binind, filled);
|
cache_bin_ncached_set(tbin, binind, filled, tcache_bin_info);
|
||||||
arena_decay_tick(tsdn, arena);
|
arena_decay_tick(tsdn, arena);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
3
src/cache_bin.c
Normal file
3
src/cache_bin.c
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||||
|
#include "jemalloc/internal/jemalloc_internal_includes.h"
|
||||||
|
|
33
src/tcache.c
33
src/tcache.c
@ -59,8 +59,10 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
|
|||||||
is_small = false;
|
is_small = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
cache_bin_sz_t low_water = cache_bin_low_water_get(tbin, binind);
|
cache_bin_sz_t low_water = cache_bin_low_water_get(tbin, binind,
|
||||||
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind);
|
tcache_bin_info);
|
||||||
|
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind,
|
||||||
|
tcache_bin_info);
|
||||||
if (low_water > 0) {
|
if (low_water > 0) {
|
||||||
/*
|
/*
|
||||||
* Flush (ceiling) 3/4 of the objects below the low water mark.
|
* Flush (ceiling) 3/4 of the objects below the low water mark.
|
||||||
@ -73,8 +75,8 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
|
|||||||
* Reduce fill count by 2X. Limit lg_fill_div such that
|
* Reduce fill count by 2X. Limit lg_fill_div such that
|
||||||
* the fill count is always at least 1.
|
* the fill count is always at least 1.
|
||||||
*/
|
*/
|
||||||
if ((cache_bin_ncached_max_get(binind) >>
|
if ((cache_bin_ncached_max_get(binind, tcache_bin_info)
|
||||||
(tcache->lg_fill_div[binind] + 1)) >= 1) {
|
>> (tcache->lg_fill_div[binind] + 1)) >= 1) {
|
||||||
tcache->lg_fill_div[binind]++;
|
tcache->lg_fill_div[binind]++;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@ -107,7 +109,8 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
|
|||||||
|
|
||||||
assert(tcache->arena != NULL);
|
assert(tcache->arena != NULL);
|
||||||
arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind);
|
arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind);
|
||||||
ret = cache_bin_alloc_easy(tbin, tcache_success, binind);
|
ret = cache_bin_alloc_easy(tbin, tcache_success, binind,
|
||||||
|
tcache_bin_info);
|
||||||
|
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
@ -126,7 +129,8 @@ tbin_edatas_lookup_size_check(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
|
|||||||
* builds, avoid the branch in the loop.
|
* builds, avoid the branch in the loop.
|
||||||
*/
|
*/
|
||||||
size_t szind_sum = binind * nflush;
|
size_t szind_sum = binind * nflush;
|
||||||
void **bottom_item = cache_bin_bottom_item_get(tbin, binind);
|
void **bottom_item = cache_bin_bottom_item_get(tbin, binind,
|
||||||
|
tcache_bin_info);
|
||||||
for (unsigned i = 0 ; i < nflush; i++) {
|
for (unsigned i = 0 ; i < nflush; i++) {
|
||||||
emap_full_alloc_ctx_t full_alloc_ctx;
|
emap_full_alloc_ctx_t full_alloc_ctx;
|
||||||
emap_full_alloc_ctx_lookup(tsd_tsdn(tsd), &emap_global,
|
emap_full_alloc_ctx_lookup(tsd_tsdn(tsd), &emap_global,
|
||||||
@ -164,7 +168,8 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
|
|||||||
} else {
|
} else {
|
||||||
assert(binind < nhbins);
|
assert(binind < nhbins);
|
||||||
}
|
}
|
||||||
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind);
|
cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind,
|
||||||
|
tcache_bin_info);
|
||||||
assert((cache_bin_sz_t)rem <= ncached);
|
assert((cache_bin_sz_t)rem <= ncached);
|
||||||
arena_t *tcache_arena = tcache->arena;
|
arena_t *tcache_arena = tcache->arena;
|
||||||
assert(tcache_arena != NULL);
|
assert(tcache_arena != NULL);
|
||||||
@ -175,7 +180,8 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
|
|||||||
* touched (it's just included to satisfy the no-zero-length rule).
|
* touched (it's just included to satisfy the no-zero-length rule).
|
||||||
*/
|
*/
|
||||||
VARIABLE_ARRAY(edata_t *, item_edata, nflush + 1);
|
VARIABLE_ARRAY(edata_t *, item_edata, nflush + 1);
|
||||||
void **bottom_item = cache_bin_bottom_item_get(tbin, binind);
|
void **bottom_item = cache_bin_bottom_item_get(tbin, binind,
|
||||||
|
tcache_bin_info);
|
||||||
|
|
||||||
/* Look up edata once per item. */
|
/* Look up edata once per item. */
|
||||||
if (config_opt_safety_checks) {
|
if (config_opt_safety_checks) {
|
||||||
@ -340,7 +346,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
|
|||||||
|
|
||||||
memmove(tbin->cur_ptr.ptr + (ncached - rem), tbin->cur_ptr.ptr, rem *
|
memmove(tbin->cur_ptr.ptr + (ncached - rem), tbin->cur_ptr.ptr, rem *
|
||||||
sizeof(void *));
|
sizeof(void *));
|
||||||
cache_bin_ncached_set(tbin, binind, rem);
|
cache_bin_ncached_set(tbin, binind, rem, tcache_bin_info);
|
||||||
if (tbin->cur_ptr.lowbits > tbin->low_water_position) {
|
if (tbin->cur_ptr.lowbits > tbin->low_water_position) {
|
||||||
tbin->low_water_position = tbin->cur_ptr.lowbits;
|
tbin->low_water_position = tbin->cur_ptr.lowbits;
|
||||||
}
|
}
|
||||||
@ -445,8 +451,9 @@ tcache_bin_init(cache_bin_t *bin, szind_t ind, uintptr_t *stack_cur) {
|
|||||||
bin->low_water_position = bin->cur_ptr.lowbits;
|
bin->low_water_position = bin->cur_ptr.lowbits;
|
||||||
bin->full_position = (uint32_t)(uintptr_t)full_position;
|
bin->full_position = (uint32_t)(uintptr_t)full_position;
|
||||||
assert(bin->cur_ptr.lowbits - bin->full_position == bin_stack_size);
|
assert(bin->cur_ptr.lowbits - bin->full_position == bin_stack_size);
|
||||||
assert(cache_bin_ncached_get(bin, ind) == 0);
|
assert(cache_bin_ncached_get(bin, ind, tcache_bin_info) == 0);
|
||||||
assert(cache_bin_empty_position_get(bin, ind) == empty_position);
|
assert(cache_bin_empty_position_get(bin, ind, tcache_bin_info)
|
||||||
|
== empty_position);
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -605,8 +612,8 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
|
|||||||
if (tsd_tcache) {
|
if (tsd_tcache) {
|
||||||
/* Release the avail array for the TSD embedded auto tcache. */
|
/* Release the avail array for the TSD embedded auto tcache. */
|
||||||
cache_bin_t *bin = tcache_small_bin_get(tcache, 0);
|
cache_bin_t *bin = tcache_small_bin_get(tcache, 0);
|
||||||
assert(cache_bin_ncached_get(bin, 0) == 0);
|
assert(cache_bin_ncached_get(bin, 0, tcache_bin_info) == 0);
|
||||||
assert(cache_bin_empty_position_get(bin, 0) ==
|
assert(cache_bin_empty_position_get(bin, 0, tcache_bin_info) ==
|
||||||
bin->cur_ptr.ptr);
|
bin->cur_ptr.ptr);
|
||||||
void *avail_array = (void *)((uintptr_t)bin->cur_ptr.ptr -
|
void *avail_array = (void *)((uintptr_t)bin->cur_ptr.ptr -
|
||||||
tcache_bin_info[0].stack_size);
|
tcache_bin_info[0].stack_size);
|
||||||
|
@ -10,48 +10,51 @@ TEST_BEGIN(test_cache_bin) {
|
|||||||
|
|
||||||
expect_ptr_not_null(stack, "Unexpected mallocx failure");
|
expect_ptr_not_null(stack, "Unexpected mallocx failure");
|
||||||
/* Initialize to empty; bin 0. */
|
/* Initialize to empty; bin 0. */
|
||||||
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0);
|
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0,
|
||||||
|
tcache_bin_info);
|
||||||
void **empty_position = stack + ncached_max;
|
void **empty_position = stack + ncached_max;
|
||||||
bin->cur_ptr.ptr = empty_position;
|
bin->cur_ptr.ptr = empty_position;
|
||||||
bin->low_water_position = bin->cur_ptr.lowbits;
|
bin->low_water_position = bin->cur_ptr.lowbits;
|
||||||
bin->full_position = (uint32_t)(uintptr_t)stack;
|
bin->full_position = (uint32_t)(uintptr_t)stack;
|
||||||
expect_ptr_eq(cache_bin_empty_position_get(bin, 0), empty_position,
|
expect_ptr_eq(cache_bin_empty_position_get(bin, 0, tcache_bin_info),
|
||||||
"Incorrect empty position");
|
empty_position, "Incorrect empty position");
|
||||||
/* Not using expect_zu etc on cache_bin_sz_t since it may change. */
|
/* Not using expect_zu etc on cache_bin_sz_t since it may change. */
|
||||||
expect_true(cache_bin_ncached_get(bin, 0) == 0, "Incorrect cache size");
|
expect_true(cache_bin_ncached_get(bin, 0, tcache_bin_info) == 0,
|
||||||
|
"Incorrect cache size");
|
||||||
|
|
||||||
bool success;
|
bool success;
|
||||||
void *ret = cache_bin_alloc_easy(bin, &success, 0);
|
void *ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info);
|
||||||
expect_false(success, "Empty cache bin should not alloc");
|
expect_false(success, "Empty cache bin should not alloc");
|
||||||
expect_true(cache_bin_low_water_get(bin, 0) == 0,
|
expect_true(cache_bin_low_water_get(bin, 0, tcache_bin_info) == 0,
|
||||||
"Incorrect low water mark");
|
"Incorrect low water mark");
|
||||||
|
|
||||||
cache_bin_ncached_set(bin, 0, 0);
|
cache_bin_ncached_set(bin, 0, 0, tcache_bin_info);
|
||||||
expect_ptr_eq(bin->cur_ptr.ptr, empty_position, "Bin should be empty");
|
expect_ptr_eq(bin->cur_ptr.ptr, empty_position, "Bin should be empty");
|
||||||
for (cache_bin_sz_t i = 1; i < ncached_max + 1; i++) {
|
for (cache_bin_sz_t i = 1; i < ncached_max + 1; i++) {
|
||||||
success = cache_bin_dalloc_easy(bin, (void *)(uintptr_t)i);
|
success = cache_bin_dalloc_easy(bin, (void *)(uintptr_t)i);
|
||||||
expect_true(success && cache_bin_ncached_get(bin, 0) == i,
|
expect_true(success && cache_bin_ncached_get(bin, 0,
|
||||||
"Bin dalloc failure");
|
tcache_bin_info) == i, "Bin dalloc failure");
|
||||||
}
|
}
|
||||||
success = cache_bin_dalloc_easy(bin, (void *)1);
|
success = cache_bin_dalloc_easy(bin, (void *)1);
|
||||||
expect_false(success, "Bin should be full");
|
expect_false(success, "Bin should be full");
|
||||||
expect_ptr_eq(bin->cur_ptr.ptr, stack, "Incorrect bin cur_ptr");
|
expect_ptr_eq(bin->cur_ptr.ptr, stack, "Incorrect bin cur_ptr");
|
||||||
|
|
||||||
cache_bin_ncached_set(bin, 0, ncached_max);
|
cache_bin_ncached_set(bin, 0, ncached_max, tcache_bin_info);
|
||||||
expect_ptr_eq(bin->cur_ptr.ptr, stack, "cur_ptr should not change");
|
expect_ptr_eq(bin->cur_ptr.ptr, stack, "cur_ptr should not change");
|
||||||
/* Emulate low water after refill. */
|
/* Emulate low water after refill. */
|
||||||
bin->low_water_position = bin->full_position;
|
bin->low_water_position = bin->full_position;
|
||||||
for (cache_bin_sz_t i = ncached_max; i > 0; i--) {
|
for (cache_bin_sz_t i = ncached_max; i > 0; i--) {
|
||||||
ret = cache_bin_alloc_easy(bin, &success, 0);
|
ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info);
|
||||||
cache_bin_sz_t ncached = cache_bin_ncached_get(bin, 0);
|
cache_bin_sz_t ncached = cache_bin_ncached_get(bin, 0,
|
||||||
|
tcache_bin_info);
|
||||||
expect_true(success && ncached == i - 1,
|
expect_true(success && ncached == i - 1,
|
||||||
"Cache bin alloc failure");
|
"Cache bin alloc failure");
|
||||||
expect_ptr_eq(ret, (void *)(uintptr_t)i, "Bin alloc failure");
|
expect_ptr_eq(ret, (void *)(uintptr_t)i, "Bin alloc failure");
|
||||||
expect_true(cache_bin_low_water_get(bin, 0) == ncached,
|
expect_true(cache_bin_low_water_get(bin, 0, tcache_bin_info)
|
||||||
"Incorrect low water mark");
|
== ncached, "Incorrect low water mark");
|
||||||
}
|
}
|
||||||
|
|
||||||
ret = cache_bin_alloc_easy(bin, &success, 0);
|
ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info);
|
||||||
expect_false(success, "Empty cache bin should not alloc.");
|
expect_false(success, "Empty cache bin should not alloc.");
|
||||||
expect_ptr_eq(bin->cur_ptr.ptr, stack + ncached_max,
|
expect_ptr_eq(bin->cur_ptr.ptr, stack + ncached_max,
|
||||||
"Bin should be empty");
|
"Bin should be empty");
|
||||||
|
Loading…
Reference in New Issue
Block a user