diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 1ee63193..f044d322 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -10,6 +10,8 @@ extern unsigned opt_tcache_nslots_large; extern ssize_t opt_lg_tcache_shift; extern size_t opt_tcache_gc_incr_bytes; extern size_t opt_tcache_gc_delay_bytes; +extern unsigned opt_lg_tcache_flush_small_div; +extern unsigned opt_lg_tcache_flush_large_div; /* * Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index 5d49c4e3..1cba9186 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -110,7 +110,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin = &tcache->bins[binind]; if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { unsigned remain = cache_bin_info_ncached_max( - &tcache_bin_info[binind]) >> 1; + &tcache_bin_info[binind]) >> opt_lg_tcache_flush_small_div; tcache_bin_flush_small(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); @@ -128,7 +128,7 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin = &tcache->bins[binind]; if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { unsigned remain = cache_bin_info_ncached_max( - &tcache_bin_info[binind]) >> 1; + &tcache_bin_info[binind]) >> opt_lg_tcache_flush_large_div; tcache_bin_flush_large(tsd, tcache, bin, binind, remain); bool ret = cache_bin_dalloc_easy(bin, ptr); assert(ret); diff --git a/src/ctl.c b/src/ctl.c index c5964d85..be8be10f 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -110,6 +110,8 @@ CTL_PROTO(opt_tcache_nslots_large) CTL_PROTO(opt_lg_tcache_nslots_mul) CTL_PROTO(opt_tcache_gc_incr_bytes) CTL_PROTO(opt_tcache_gc_delay_bytes) +CTL_PROTO(opt_lg_tcache_flush_small_div) +CTL_PROTO(opt_lg_tcache_flush_large_div) CTL_PROTO(opt_thp) CTL_PROTO(opt_lg_extent_max_active_fit) CTL_PROTO(opt_prof) @@ -355,6 +357,10 @@ static const ctl_named_node_t opt_node[] = { {NAME("lg_tcache_nslots_mul"), CTL(opt_lg_tcache_nslots_mul)}, {NAME("tcache_gc_incr_bytes"), CTL(opt_tcache_gc_incr_bytes)}, {NAME("tcache_gc_delay_bytes"), CTL(opt_tcache_gc_delay_bytes)}, + {NAME("lg_tcache_flush_small_div"), + CTL(opt_lg_tcache_flush_small_div)}, + {NAME("lg_tcache_flush_large_div"), + CTL(opt_lg_tcache_flush_large_div)}, {NAME("thp"), CTL(opt_thp)}, {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)}, {NAME("prof"), CTL(opt_prof)}, @@ -1816,6 +1822,10 @@ CTL_RO_NL_GEN(opt_tcache_nslots_large, opt_tcache_nslots_large, unsigned) CTL_RO_NL_GEN(opt_lg_tcache_nslots_mul, opt_lg_tcache_nslots_mul, ssize_t) CTL_RO_NL_GEN(opt_tcache_gc_incr_bytes, opt_tcache_gc_incr_bytes, size_t) CTL_RO_NL_GEN(opt_tcache_gc_delay_bytes, opt_tcache_gc_delay_bytes, size_t) +CTL_RO_NL_GEN(opt_lg_tcache_flush_small_div, opt_lg_tcache_flush_small_div, + unsigned) +CTL_RO_NL_GEN(opt_lg_tcache_flush_large_div, opt_lg_tcache_flush_large_div, + unsigned) CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *) CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit, size_t) diff --git a/src/jemalloc.c b/src/jemalloc.c index 2903a412..74355d40 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1397,6 +1397,12 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], "tcache_gc_delay_bytes", 0, SIZE_T_MAX, CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, /* clip */ false) + CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_small_div, + "lg_tcache_flush_small_div", 1, 16, + CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) + CONF_HANDLE_UNSIGNED(opt_lg_tcache_flush_large_div, + "lg_tcache_flush_large_div", 1, 16, + CONF_CHECK_MIN, CONF_CHECK_MAX, /* clip */ true) /* * The runtime option of oversize_threshold remains diff --git a/src/stats.c b/src/stats.c index 8be69ca6..fb88e5a6 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1113,6 +1113,8 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_SSIZE_T("lg_tcache_nslots_mul") OPT_WRITE_SIZE_T("tcache_gc_incr_bytes") OPT_WRITE_SIZE_T("tcache_gc_delay_bytes") + OPT_WRITE_UNSIGNED("lg_tcache_flush_small_div") + OPT_WRITE_UNSIGNED("lg_tcache_flush_large_div") OPT_WRITE_CHAR_P("thp") OPT_WRITE_BOOL("prof") OPT_WRITE_CHAR_P("prof_prefix") diff --git a/src/tcache.c b/src/tcache.c index c9cb7853..2513ca33 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -53,6 +53,13 @@ size_t opt_tcache_gc_incr_bytes = 65536; */ size_t opt_tcache_gc_delay_bytes = 0; +/* + * When a cache bin is flushed because it's full, how much of it do we flush? + * By default, we flush half the maximum number of items. + */ +unsigned opt_lg_tcache_flush_small_div = 1; +unsigned opt_lg_tcache_flush_large_div = 1; + cache_bin_info_t *tcache_bin_info; /* Total stack size required (per tcache). Include the padding above. */