From bacb6afc6c5a83c5bf2e5e04a6db99600046e971 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 21 Dec 2016 12:33:17 -0800 Subject: [PATCH] Simplify arena_slab_regind(). Rewrite arena_slab_regind() to provide sufficient constant data for the compiler to perform division strength reduction. This replaces more general manual strength reduction that was implemented before arena_bin_info was compile-time-constant. It would be possible to slightly improve on the compiler-generated division code by taking advantage of range limits that the compiler doesn't know about. --- Makefile.in | 1 + include/jemalloc/internal/arena.h | 3 + include/jemalloc/internal/private_symbols.txt | 1 + src/arena.c | 85 ++++++------------- test/unit/slab.c | 35 ++++++++ 5 files changed, 66 insertions(+), 59 deletions(-) create mode 100644 test/unit/slab.c diff --git a/Makefile.in b/Makefile.in index 7c1989b8..22b11349 100644 --- a/Makefile.in +++ b/Makefile.in @@ -184,6 +184,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/rtree.c \ $(srcroot)test/unit/SFMT.c \ $(srcroot)test/unit/size_classes.c \ + $(srcroot)test/unit/slab.c \ $(srcroot)test/unit/smoothstep.c \ $(srcroot)test/unit/stats.c \ $(srcroot)test/unit/ticker.c \ diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index ad400839..d6b1a2b0 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -271,6 +271,9 @@ void arena_extent_cache_maybe_insert(tsdn_t *tsdn, arena_t *arena, extent_t *extent, bool cache); void arena_extent_cache_maybe_remove(tsdn_t *tsdn, arena_t *arena, extent_t *extent, bool cache); +#ifdef JEMALLOC_JET +size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr); +#endif extent_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero); void arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 4560d702..63974880 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -66,6 +66,7 @@ arena_ralloc_no_move arena_reset arena_salloc arena_sdalloc +arena_slab_regind arena_stats_merge arena_tcache_fill_small arena_tdata_get diff --git a/src/arena.c b/src/arena.c index 488bfd47..73fea528 100644 --- a/src/arena.c +++ b/src/arena.c @@ -138,73 +138,41 @@ arena_slab_reg_alloc(tsdn_t *tsdn, extent_t *slab, return (ret); } -JEMALLOC_INLINE_C size_t -arena_slab_regind(extent_t *slab, const arena_bin_info_t *bin_info, - const void *ptr) +#ifndef JEMALLOC_JET +JEMALLOC_INLINE_C +#endif +size_t +arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr) { - size_t diff, interval, shift, regind; + size_t diff, regind; /* Freeing a pointer outside the slab can cause assertion failure. */ assert((uintptr_t)ptr >= (uintptr_t)extent_addr_get(slab)); assert((uintptr_t)ptr < (uintptr_t)extent_past_get(slab)); /* Freeing an interior pointer can cause assertion failure. */ assert(((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)) % - (uintptr_t)bin_info->reg_size == 0); + (uintptr_t)arena_bin_info[binind].reg_size == 0); - /* - * Avoid doing division with a variable divisor if possible. Using - * actual division here can reduce allocator throughput by over 20%! - */ + /* Avoid doing division with a variable divisor. */ diff = (size_t)((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)); - - /* Rescale (factor powers of 2 out of the numerator and denominator). */ - interval = bin_info->reg_size; - shift = ffs_zu(interval) - 1; - diff >>= shift; - interval >>= shift; - - if (interval == 1) { - /* The divisor was a power of 2. */ - regind = diff; - } else { - /* - * To divide by a number D that is not a power of two we - * multiply by (2^21 / D) and then right shift by 21 positions. - * - * X / D - * - * becomes - * - * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT - * - * We can omit the first three elements, because we never - * divide by 0, and 1 and 2 are both powers of two, which are - * handled above. - */ -#define SIZE_INV_SHIFT ((sizeof(size_t) << 3) - LG_SLAB_MAXREGS) -#define SIZE_INV(s) (((ZU(1) << SIZE_INV_SHIFT) / (s)) + 1) - static const size_t interval_invs[] = { - SIZE_INV(3), - SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), - SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), - SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), - SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), - SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), - SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), - SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) - }; - - if (likely(interval <= ((sizeof(interval_invs) / sizeof(size_t)) - + 2))) { - regind = (diff * interval_invs[interval - 3]) >> - SIZE_INV_SHIFT; - } else - regind = diff / interval; -#undef SIZE_INV -#undef SIZE_INV_SHIFT + switch (binind) { +#define REGIND_bin_yes(index, reg_size) \ + case index: \ + regind = diff / (reg_size); \ + assert(diff == regind * (reg_size)); \ + break; +#define REGIND_bin_no(index, reg_size) +#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ + lg_delta_lookup) \ + REGIND_bin_##bin(index, (1U<nregs); + + assert(regind < arena_bin_info[binind].nregs); return (regind); } @@ -215,7 +183,7 @@ arena_slab_reg_dalloc(tsdn_t *tsdn, extent_t *slab, { szind_t binind = slab_data->binind; const arena_bin_info_t *bin_info = &arena_bin_info[binind]; - size_t regind = arena_slab_regind(slab, bin_info, ptr); + size_t regind = arena_slab_regind(slab, binind, ptr); assert(slab_data->nfree < bin_info->nregs); /* Freeing an unallocated pointer can cause assertion failure. */ @@ -1022,7 +990,6 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, const arena_bin_info_t *bin_info; extent_t *slab; - bin_info = &arena_bin_info[binind]; if (bin->slabcur != NULL) { arena_bin_slabs_full_insert(bin, bin->slabcur); diff --git a/test/unit/slab.c b/test/unit/slab.c new file mode 100644 index 00000000..42e82a8b --- /dev/null +++ b/test/unit/slab.c @@ -0,0 +1,35 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_arena_slab_regind) +{ + szind_t binind; + + for (binind = 0; binind < NBINS; binind++) { + size_t regind; + extent_t slab; + const arena_bin_info_t *bin_info = &arena_bin_info[binind]; + extent_init(&slab, NULL, mallocx(bin_info->slab_size, + MALLOCX_LG_ALIGN(LG_PAGE)), bin_info->slab_size, 0, 0, true, + false, true, true); + assert_ptr_not_null(extent_addr_get(&slab), + "Unexpected malloc() failure"); + for (regind = 0; regind < bin_info->nregs; regind++) { + void *reg = (void *)((uintptr_t)extent_addr_get(&slab) + + (bin_info->reg_size * regind)); + assert_zu_eq(arena_slab_regind(&slab, binind, reg), + regind, + "Incorrect region index computed for size %zu", + bin_info->reg_size); + } + free(extent_addr_get(&slab)); + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_arena_slab_regind)); +}