diff --git a/Makefile.in b/Makefile.in index 984bd724..b53846d2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -102,6 +102,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/bin_info.c \ $(srcroot)src/bitmap.c \ $(srcroot)src/buf_writer.c \ + $(srcroot)src/cache_bin.c \ $(srcroot)src/ckh.c \ $(srcroot)src/counter.c \ $(srcroot)src/ctl.c \ diff --git a/include/jemalloc/internal/cache_bin.h b/include/jemalloc/internal/cache_bin.h index 60feb15f..ec2fdf42 100644 --- a/include/jemalloc/internal/cache_bin.h +++ b/include/jemalloc/internal/cache_bin.h @@ -35,7 +35,6 @@ struct cache_bin_info_s { /* The size of the bin stack, i.e. ncached_max * sizeof(ptr). */ cache_bin_sz_t stack_size; }; -extern cache_bin_info_t *tcache_bin_info; typedef struct cache_bin_s cache_bin_t; struct cache_bin_s { @@ -115,29 +114,29 @@ struct cache_bin_array_descriptor_s { /* Returns ncached_max: Upper limit on ncached. */ static inline cache_bin_sz_t -cache_bin_ncached_max_get(szind_t ind) { - return tcache_bin_info[ind].stack_size / sizeof(void *); +cache_bin_ncached_max_get(szind_t ind, cache_bin_info_t *infos) { + return infos[ind].stack_size / sizeof(void *); } static inline cache_bin_sz_t -cache_bin_ncached_get(cache_bin_t *bin, szind_t ind) { - cache_bin_sz_t n = (cache_bin_sz_t)((tcache_bin_info[ind].stack_size + +cache_bin_ncached_get(cache_bin_t *bin, szind_t ind, cache_bin_info_t *infos) { + cache_bin_sz_t n = (cache_bin_sz_t)((infos[ind].stack_size + bin->full_position - bin->cur_ptr.lowbits) / sizeof(void *)); - assert(n <= cache_bin_ncached_max_get(ind)); + assert(n <= cache_bin_ncached_max_get(ind, infos)); assert(n == 0 || *(bin->cur_ptr.ptr) != NULL); return n; } static inline void ** -cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) { - void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind); +cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind, + cache_bin_info_t *infos) { + void **ret = bin->cur_ptr.ptr + cache_bin_ncached_get(bin, ind, infos); /* Low bits overflow disallowed when allocating the space. */ assert((uint32_t)(uintptr_t)ret >= bin->cur_ptr.lowbits); /* Can also be computed via (full_position + ncached_max) | highbits. */ - uintptr_t lowbits = bin->full_position + - tcache_bin_info[ind].stack_size; + uintptr_t lowbits = bin->full_position + infos[ind].stack_size; uintptr_t highbits = (uintptr_t)bin->cur_ptr.ptr & ~(((uint64_t)1 << 32) - 1); assert(ret == (void **)(lowbits | highbits)); @@ -147,32 +146,35 @@ cache_bin_empty_position_get(cache_bin_t *bin, szind_t ind) { /* Returns the position of the bottom item on the stack; for convenience. */ static inline void ** -cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind) { - void **bottom = cache_bin_empty_position_get(bin, ind) - 1; - assert(cache_bin_ncached_get(bin, ind) == 0 || *bottom != NULL); +cache_bin_bottom_item_get(cache_bin_t *bin, szind_t ind, + cache_bin_info_t *infos) { + void **bottom = cache_bin_empty_position_get(bin, ind, infos) - 1; + assert(cache_bin_ncached_get(bin, ind, infos) == 0 || *bottom != NULL); return bottom; } /* Returns the numeric value of low water in [0, ncached]. */ static inline cache_bin_sz_t -cache_bin_low_water_get(cache_bin_t *bin, szind_t ind) { - cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind); +cache_bin_low_water_get(cache_bin_t *bin, szind_t ind, + cache_bin_info_t *infos) { + cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(ind, infos); cache_bin_sz_t low_water = ncached_max - (cache_bin_sz_t)((bin->low_water_position - bin->full_position) / sizeof(void *)); assert(low_water <= ncached_max); - assert(low_water <= cache_bin_ncached_get(bin, ind)); + assert(low_water <= cache_bin_ncached_get(bin, ind, infos)); assert(bin->low_water_position >= bin->cur_ptr.lowbits); return low_water; } static inline void -cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n) { - bin->cur_ptr.lowbits = bin->full_position + - tcache_bin_info[ind].stack_size - n * sizeof(void *); - assert(n <= cache_bin_ncached_max_get(ind)); +cache_bin_ncached_set(cache_bin_t *bin, szind_t ind, cache_bin_sz_t n, + cache_bin_info_t *infos) { + bin->cur_ptr.lowbits = bin->full_position + infos[ind].stack_size + - n * sizeof(void *); + assert(n <= cache_bin_ncached_max_get(ind, infos)); assert(n == 0 || *bin->cur_ptr.ptr != NULL); } @@ -188,7 +190,7 @@ cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor, JEMALLOC_ALWAYS_INLINE void * cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind, - const bool adjust_low_water) { + cache_bin_info_t *infos, const bool adjust_low_water) { /* * This may read from the empty position; however the loaded value won't * be used. It's safe because the stack has one more slot reserved. @@ -197,14 +199,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind, /* * Check for both bin->ncached == 0 and ncached < low_water in a single * branch. When adjust_low_water is true, this also avoids accessing - * tcache_bin_info (which is on a separate cacheline / page) in the - * common case. + * the cache_bin_info_ts (which is on a separate cacheline / page) in + * the common case. */ if (unlikely(bin->cur_ptr.lowbits > bin->low_water_position)) { if (adjust_low_water) { assert(ind != INVALID_SZIND); uint32_t empty_position = bin->full_position + - tcache_bin_info[ind].stack_size; + infos[ind].stack_size; if (unlikely(bin->cur_ptr.lowbits > empty_position)) { /* Over-allocated; revert. */ bin->cur_ptr.ptr--; @@ -237,12 +239,14 @@ cache_bin_alloc_easy_impl(cache_bin_t *bin, bool *success, szind_t ind, JEMALLOC_ALWAYS_INLINE void * cache_bin_alloc_easy_reduced(cache_bin_t *bin, bool *success) { /* The szind parameter won't be used. */ - return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND, false); + return cache_bin_alloc_easy_impl(bin, success, INVALID_SZIND, + /* infos */ NULL, false); } JEMALLOC_ALWAYS_INLINE void * -cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind) { - return cache_bin_alloc_easy_impl(bin, success, ind, true); +cache_bin_alloc_easy(cache_bin_t *bin, bool *success, szind_t ind, + cache_bin_info_t *infos) { + return cache_bin_alloc_easy_impl(bin, success, ind, infos, true); } #undef INVALID_SZIND diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index db6f98bf..c5c8f485 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -13,6 +13,8 @@ extern unsigned nhbins; /* Maximum cached size class. */ extern size_t tcache_maxclass; +extern cache_bin_info_t *tcache_bin_info; + /* * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and * usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index ff06935d..dc6da949 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -36,7 +36,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, assert(binind < SC_NBINS); bin = tcache_small_bin_get(tcache, binind); - ret = cache_bin_alloc_easy(bin, &tcache_success, binind); + ret = cache_bin_alloc_easy(bin, &tcache_success, binind, + tcache_bin_info); assert(tcache_success == (ret != NULL)); if (unlikely(!tcache_success)) { bool tcache_hard_success; @@ -79,7 +80,8 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, assert(binind >= SC_NBINS &&binind < nhbins); bin = tcache_large_bin_get(tcache, binind); - ret = cache_bin_alloc_easy(bin, &tcache_success, binind); + ret = cache_bin_alloc_easy(bin, &tcache_success, binind, + tcache_bin_info); assert(tcache_success == (ret != NULL)); if (unlikely(!tcache_success)) { /* @@ -127,7 +129,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bin = tcache_small_bin_get(tcache, binind); if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { - unsigned remain = cache_bin_ncached_max_get(binind) >> 1; + unsigned remain = cache_bin_ncached_max_get(binind, + tcache_bin_info) >> 1; tcache_bin_flush_small(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); @@ -145,7 +148,8 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bin = tcache_large_bin_get(tcache, binind); if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { - unsigned remain = cache_bin_ncached_max_get(binind) >> 1; + unsigned remain = cache_bin_ncached_max_get(binind, + tcache_bin_info) >> 1; tcache_bin_flush_large(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index d98bb858..920d55ed 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -42,6 +42,7 @@ + diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index fd3e11c0..fe77170d 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -25,6 +25,9 @@ Source Files + + Source Files + Source Files diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index b59d411f..2db94010 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -42,6 +42,7 @@ + diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters index fd3e11c0..fe77170d 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -25,6 +25,9 @@ Source Files + + Source Files + Source Files diff --git a/src/arena.c b/src/arena.c index 0a9e4a98..5ca884b7 100644 --- a/src/arena.c +++ b/src/arena.c @@ -200,13 +200,14 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, for (szind_t i = 0; i < SC_NBINS; i++) { cache_bin_t *tbin = &descriptor->bins_small[i]; arena_stats_accum_zu(&astats->tcache_bytes, - cache_bin_ncached_get(tbin, i) * sz_index2size(i)); + cache_bin_ncached_get(tbin, i, tcache_bin_info) + * sz_index2size(i)); } for (szind_t i = 0; i < nhbins - SC_NBINS; i++) { cache_bin_t *tbin = &descriptor->bins_large[i]; arena_stats_accum_zu(&astats->tcache_bytes, - cache_bin_ncached_get(tbin, i + SC_NBINS) * - sz_index2size(i)); + cache_bin_ncached_get(tbin, i + SC_NBINS, + tcache_bin_info) * sz_index2size(i)); } } malloc_mutex_prof_read(tsdn, @@ -1320,13 +1321,14 @@ arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, cache_bin_t *tbin, szind_t binind) { - assert(cache_bin_ncached_get(tbin, binind) == 0); + assert(cache_bin_ncached_get(tbin, binind, tcache_bin_info) == 0); tcache->bin_refilled[binind] = true; const bin_info_t *bin_info = &bin_infos[binind]; - const unsigned nfill = cache_bin_ncached_max_get(binind) >> - tcache->lg_fill_div[binind]; - void **empty_position = cache_bin_empty_position_get(tbin, binind); + const unsigned nfill = cache_bin_ncached_max_get(binind, + tcache_bin_info) >> tcache->lg_fill_div[binind]; + void **empty_position = cache_bin_empty_position_get(tbin, binind, + tcache_bin_info); /* * Bin-local resources are used first: 1) bin->slabcur, and 2) nonfull @@ -1446,7 +1448,7 @@ label_refill: fresh_slab = NULL; } - cache_bin_ncached_set(tbin, binind, filled); + cache_bin_ncached_set(tbin, binind, filled, tcache_bin_info); arena_decay_tick(tsdn, arena); } diff --git a/src/cache_bin.c b/src/cache_bin.c new file mode 100644 index 00000000..454cb475 --- /dev/null +++ b/src/cache_bin.c @@ -0,0 +1,3 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + diff --git a/src/tcache.c b/src/tcache.c index c736f565..62905f14 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -59,8 +59,10 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { is_small = false; } - cache_bin_sz_t low_water = cache_bin_low_water_get(tbin, binind); - cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind); + cache_bin_sz_t low_water = cache_bin_low_water_get(tbin, binind, + tcache_bin_info); + cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind, + tcache_bin_info); if (low_water > 0) { /* * Flush (ceiling) 3/4 of the objects below the low water mark. @@ -73,8 +75,8 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { * Reduce fill count by 2X. Limit lg_fill_div such that * the fill count is always at least 1. */ - if ((cache_bin_ncached_max_get(binind) >> - (tcache->lg_fill_div[binind] + 1)) >= 1) { + if ((cache_bin_ncached_max_get(binind, tcache_bin_info) + >> (tcache->lg_fill_div[binind] + 1)) >= 1) { tcache->lg_fill_div[binind]++; } } else { @@ -107,7 +109,8 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, assert(tcache->arena != NULL); arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind); - ret = cache_bin_alloc_easy(tbin, tcache_success, binind); + ret = cache_bin_alloc_easy(tbin, tcache_success, binind, + tcache_bin_info); return ret; } @@ -126,7 +129,8 @@ tbin_edatas_lookup_size_check(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, * builds, avoid the branch in the loop. */ size_t szind_sum = binind * nflush; - void **bottom_item = cache_bin_bottom_item_get(tbin, binind); + void **bottom_item = cache_bin_bottom_item_get(tbin, binind, + tcache_bin_info); for (unsigned i = 0 ; i < nflush; i++) { emap_full_alloc_ctx_t full_alloc_ctx; emap_full_alloc_ctx_lookup(tsd_tsdn(tsd), &emap_global, @@ -164,7 +168,8 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, } else { assert(binind < nhbins); } - cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind); + cache_bin_sz_t ncached = cache_bin_ncached_get(tbin, binind, + tcache_bin_info); assert((cache_bin_sz_t)rem <= ncached); arena_t *tcache_arena = tcache->arena; assert(tcache_arena != NULL); @@ -175,7 +180,8 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, * touched (it's just included to satisfy the no-zero-length rule). */ VARIABLE_ARRAY(edata_t *, item_edata, nflush + 1); - void **bottom_item = cache_bin_bottom_item_get(tbin, binind); + void **bottom_item = cache_bin_bottom_item_get(tbin, binind, + tcache_bin_info); /* Look up edata once per item. */ if (config_opt_safety_checks) { @@ -340,7 +346,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, memmove(tbin->cur_ptr.ptr + (ncached - rem), tbin->cur_ptr.ptr, rem * sizeof(void *)); - cache_bin_ncached_set(tbin, binind, rem); + cache_bin_ncached_set(tbin, binind, rem, tcache_bin_info); if (tbin->cur_ptr.lowbits > tbin->low_water_position) { tbin->low_water_position = tbin->cur_ptr.lowbits; } @@ -445,8 +451,9 @@ tcache_bin_init(cache_bin_t *bin, szind_t ind, uintptr_t *stack_cur) { bin->low_water_position = bin->cur_ptr.lowbits; bin->full_position = (uint32_t)(uintptr_t)full_position; assert(bin->cur_ptr.lowbits - bin->full_position == bin_stack_size); - assert(cache_bin_ncached_get(bin, ind) == 0); - assert(cache_bin_empty_position_get(bin, ind) == empty_position); + assert(cache_bin_ncached_get(bin, ind, tcache_bin_info) == 0); + assert(cache_bin_empty_position_get(bin, ind, tcache_bin_info) + == empty_position); return false; } @@ -605,8 +612,8 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { if (tsd_tcache) { /* Release the avail array for the TSD embedded auto tcache. */ cache_bin_t *bin = tcache_small_bin_get(tcache, 0); - assert(cache_bin_ncached_get(bin, 0) == 0); - assert(cache_bin_empty_position_get(bin, 0) == + assert(cache_bin_ncached_get(bin, 0, tcache_bin_info) == 0); + assert(cache_bin_empty_position_get(bin, 0, tcache_bin_info) == bin->cur_ptr.ptr); void *avail_array = (void *)((uintptr_t)bin->cur_ptr.ptr - tcache_bin_info[0].stack_size); diff --git a/test/unit/cache_bin.c b/test/unit/cache_bin.c index f98a92c3..5ef108d0 100644 --- a/test/unit/cache_bin.c +++ b/test/unit/cache_bin.c @@ -10,48 +10,51 @@ TEST_BEGIN(test_cache_bin) { expect_ptr_not_null(stack, "Unexpected mallocx failure"); /* Initialize to empty; bin 0. */ - cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0); + cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(0, + tcache_bin_info); void **empty_position = stack + ncached_max; bin->cur_ptr.ptr = empty_position; bin->low_water_position = bin->cur_ptr.lowbits; bin->full_position = (uint32_t)(uintptr_t)stack; - expect_ptr_eq(cache_bin_empty_position_get(bin, 0), empty_position, - "Incorrect empty position"); + expect_ptr_eq(cache_bin_empty_position_get(bin, 0, tcache_bin_info), + empty_position, "Incorrect empty position"); /* Not using expect_zu etc on cache_bin_sz_t since it may change. */ - expect_true(cache_bin_ncached_get(bin, 0) == 0, "Incorrect cache size"); + expect_true(cache_bin_ncached_get(bin, 0, tcache_bin_info) == 0, + "Incorrect cache size"); bool success; - void *ret = cache_bin_alloc_easy(bin, &success, 0); + void *ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info); expect_false(success, "Empty cache bin should not alloc"); - expect_true(cache_bin_low_water_get(bin, 0) == 0, + expect_true(cache_bin_low_water_get(bin, 0, tcache_bin_info) == 0, "Incorrect low water mark"); - cache_bin_ncached_set(bin, 0, 0); + cache_bin_ncached_set(bin, 0, 0, tcache_bin_info); expect_ptr_eq(bin->cur_ptr.ptr, empty_position, "Bin should be empty"); for (cache_bin_sz_t i = 1; i < ncached_max + 1; i++) { success = cache_bin_dalloc_easy(bin, (void *)(uintptr_t)i); - expect_true(success && cache_bin_ncached_get(bin, 0) == i, - "Bin dalloc failure"); + expect_true(success && cache_bin_ncached_get(bin, 0, + tcache_bin_info) == i, "Bin dalloc failure"); } success = cache_bin_dalloc_easy(bin, (void *)1); expect_false(success, "Bin should be full"); expect_ptr_eq(bin->cur_ptr.ptr, stack, "Incorrect bin cur_ptr"); - cache_bin_ncached_set(bin, 0, ncached_max); + cache_bin_ncached_set(bin, 0, ncached_max, tcache_bin_info); expect_ptr_eq(bin->cur_ptr.ptr, stack, "cur_ptr should not change"); /* Emulate low water after refill. */ bin->low_water_position = bin->full_position; for (cache_bin_sz_t i = ncached_max; i > 0; i--) { - ret = cache_bin_alloc_easy(bin, &success, 0); - cache_bin_sz_t ncached = cache_bin_ncached_get(bin, 0); + ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info); + cache_bin_sz_t ncached = cache_bin_ncached_get(bin, 0, + tcache_bin_info); expect_true(success && ncached == i - 1, "Cache bin alloc failure"); expect_ptr_eq(ret, (void *)(uintptr_t)i, "Bin alloc failure"); - expect_true(cache_bin_low_water_get(bin, 0) == ncached, - "Incorrect low water mark"); + expect_true(cache_bin_low_water_get(bin, 0, tcache_bin_info) + == ncached, "Incorrect low water mark"); } - ret = cache_bin_alloc_easy(bin, &success, 0); + ret = cache_bin_alloc_easy(bin, &success, 0, tcache_bin_info); expect_false(success, "Empty cache bin should not alloc."); expect_ptr_eq(bin->cur_ptr.ptr, stack + ncached_max, "Bin should be empty");