From 5e41ff9b740258bddebcbd5575e1670a15f8b1ae Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Thu, 15 Oct 2020 16:37:16 -0700 Subject: [PATCH] Add a hard limit on tcache max size class. For locality reasons, tcache bins are integrated in TSD. Allowing all size classes to be cached has little benefit, but takes up much thread local storage. In addition, it complicates the layout which we try hard to optimize. --- doc/jemalloc.xml.in | 4 ++-- include/jemalloc/internal/tcache_structs.h | 2 +- include/jemalloc/internal/tcache_types.h | 5 +++++ src/tcache.c | 12 ++++++------ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index f283fd37..8e9a5d8f 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1314,8 +1314,8 @@ malloc_conf = "xmalloc:true";]]> Maximum size class (log base 2) to cache in the thread-specific cache (tcache). At a minimum, all small size classes - are cached, and at a maximum all large size classes are cached. The - default maximum is 32 KiB (2^15). + are cached; and at a maximum, size classes up to 8 MiB can be cached. + The default maximum is 32 KiB (2^15). diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index 331bd247..176d73de 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -54,7 +54,7 @@ struct tcache_slow_s { struct tcache_s { tcache_slow_t *tcache_slow; - cache_bin_t bins[SC_NSIZES]; + cache_bin_t bins[TCACHE_NBINS_MAX]; }; /* Linkage for list of available (previously used) explicit tcache IDs. */ diff --git a/include/jemalloc/internal/tcache_types.h b/include/jemalloc/internal/tcache_types.h index fb311e72..583677ea 100644 --- a/include/jemalloc/internal/tcache_types.h +++ b/include/jemalloc/internal/tcache_types.h @@ -27,4 +27,9 @@ typedef struct tcaches_s tcaches_t; /* Used for explicit tcache only. Means flushed but not destroyed. */ #define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1) +#define TCACHE_LG_MAXCLASS_LIMIT 23 /* tcache_maxclass = 8M */ +#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT) +#define TCACHE_NBINS_MAX (SC_NBINS + SC_NGROUP * \ + (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS) + 1) + #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ diff --git a/src/tcache.c b/src/tcache.c index 06efe66a..63eddc2d 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -936,20 +936,20 @@ tcache_ncached_max_compute(szind_t szind) { bool tcache_boot(tsdn_t *tsdn, base_t *base) { /* If necessary, clamp opt_lg_tcache_max. */ - if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < - SC_SMALL_MAXCLASS) { + tcache_maxclass = opt_lg_tcache_max < 0 ? 0 : + ZU(1) << opt_lg_tcache_max; + if (tcache_maxclass < SC_SMALL_MAXCLASS) { tcache_maxclass = SC_SMALL_MAXCLASS; - } else { - tcache_maxclass = (ZU(1) << opt_lg_tcache_max); + } else if (tcache_maxclass > TCACHE_MAXCLASS_LIMIT) { + tcache_maxclass = TCACHE_MAXCLASS_LIMIT; } + nhbins = sz_size2index(tcache_maxclass) + 1; if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES, malloc_mutex_rank_exclusive)) { return true; } - nhbins = sz_size2index(tcache_maxclass) + 1; - /* Initialize tcache_bin_info. */ tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, base, nhbins * sizeof(cache_bin_info_t), CACHELINE);