From 7dcf77809c9886e3892e29954d90b838af1292c3 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Tue, 30 Nov 2021 15:58:03 -0800 Subject: [PATCH] Mark slab as true on sized dealloc fast path. For sized dealloc, fastpath only handles lookup-able sizes, which must be slabs. --- include/jemalloc/internal/sc.h | 11 ++++++++--- src/jemalloc.c | 10 +++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/include/jemalloc/internal/sc.h b/include/jemalloc/internal/sc.h index 031ffff4..8efd3249 100644 --- a/include/jemalloc/internal/sc.h +++ b/include/jemalloc/internal/sc.h @@ -248,16 +248,21 @@ /* The largest size class in the lookup table, and its binary log. */ #define SC_LG_MAX_LOOKUP 12 -#define SC_LOOKUP_MAXCLASS ((size_t)1 << SC_LG_MAX_LOOKUP) +#define SC_LOOKUP_MAXCLASS (1 << SC_LG_MAX_LOOKUP) /* Internal, only used for the definition of SC_SMALL_MAXCLASS. */ -#define SC_SMALL_MAX_BASE ((size_t)1 << (LG_PAGE + SC_LG_NGROUP - 1)) -#define SC_SMALL_MAX_DELTA ((size_t)1 << (LG_PAGE - 1)) +#define SC_SMALL_MAX_BASE (1 << (LG_PAGE + SC_LG_NGROUP - 1)) +#define SC_SMALL_MAX_DELTA (1 << (LG_PAGE - 1)) /* The largest size class allocated out of a slab. */ #define SC_SMALL_MAXCLASS (SC_SMALL_MAX_BASE \ + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) +/* The fastpath assumes all lookup-able sizes are small. */ +#if (SC_SMALL_MAXCLASS < SC_LOOKUP_MAXCLASS) +# error "Lookup table sizes must be small" +#endif + /* The smallest size class not allocated out of a slab. */ #define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP)) #define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP) diff --git a/src/jemalloc.c b/src/jemalloc.c index a7d43dc1..521f4ea4 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2946,9 +2946,17 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) { return false; } alloc_ctx.szind = sz_size2index_lookup(size); + /* Max lookup class must be small. */ + assert(alloc_ctx.szind < SC_NBINS); /* This is a dead store, except when opt size checking is on. */ - alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS); + alloc_ctx.slab = true; } + /* + * Currently the fastpath only handles small sizes. The branch on + * SC_LOOKUP_MAXCLASS makes sure of it. This lets us avoid checking + * tcache szind upper limit (i.e. tcache_maxclass) as well. + */ + assert(alloc_ctx.slab); uint64_t deallocated, threshold; te_free_fastpath_ctx(tsd, &deallocated, &threshold);