Add opt.bin_shards to specify number of bin shards.

The option uses the same format as "slab_sizes" to specify number of shards for
each bin size.

include/jemalloc/internal/bin.h

@@ -8,9 +8,7 @@
 #include "jemalloc/internal/sc.h"
 
 #define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH)
-
+#define N_BIN_SHARDS_DEFAULT 1
-extern unsigned opt_bin_shard_maxszind;
-extern unsigned opt_n_bin_shards;
 
 /*
  * A bin contains a set of extents that are currently being used for slab
@@ -93,8 +91,10 @@ struct bins_s {
 	bin_t *bin_shards;
 };
 
-void bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]);
+void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]);
-void bin_boot();
+bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size,
+    size_t end_size, size_t nshards);
+void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]);
 
 /* Initializes a bin to empty.  Returns true on error. */
 bool bin_init(bin_t *bin);

include/jemalloc/internal/extent_structs.h

@@ -128,10 +128,6 @@ struct extent_s {
 #define EXTENT_BITS_BINSHARD_SHIFT  (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT)
 #define EXTENT_BITS_BINSHARD_MASK  MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT)
 
-/* Will make dynamic options. */
-#define OPT_N_BIN_SHARDS (1)
-#define OPT_BIN_SHARD_MAXSZIND (0)
-
 #define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT)
 #define EXTENT_BITS_SN_MASK  (UINT64_MAX << EXTENT_BITS_SN_SHIFT)
 

src/arena.c

@@ -1346,8 +1346,7 @@ bin_t *
 arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     unsigned *binshard) {
 	bin_t *bin;
-	if (binind >= opt_bin_shard_maxszind || tsdn_null(tsdn) ||
+	if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) {
-	    tsd_arena_get(tsdn_tsd(tsdn)) == NULL) {
 		*binshard = 0;
 	} else {
 		*binshard = tsd_binshard_get(tsdn_tsd(tsdn)) %
@@ -1923,9 +1922,11 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 		}
 	}
 
-	size_t arena_size = sizeof(arena_t) +
+	unsigned nbins_total = 0;
-	    sizeof(bin_t) * opt_n_bin_shards * opt_bin_shard_maxszind +
+	for (i = 0; i < SC_NBINS; i++) {
-	    sizeof(bin_t) * (SC_NBINS - opt_bin_shard_maxszind);
+		nbins_total += bin_infos[i].n_shards;
+	}
+	size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total;
 	arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE);
 	if (arena == NULL) {
 		goto label_error;

src/bin.c

@@ -6,13 +6,11 @@
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/witness.h"
 
-unsigned opt_bin_shard_maxszind;
-unsigned opt_n_bin_shards;
-
 bin_info_t bin_infos[SC_NBINS];
 
-void
+static void
-bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]) {
+bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
+    bin_info_t bin_infos[SC_NBINS]) {
 	for (unsigned i = 0; i < SC_NBINS; i++) {
 		bin_info_t *bin_info = &bin_infos[i];
 		sc_t *sc = &sc_data->sc[i];
@@ -21,19 +19,49 @@ bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]) {
 		bin_info->slab_size = (sc->pgs << LG_PAGE);
 		bin_info->nregs =
 		    (uint32_t)(bin_info->slab_size / bin_info->reg_size);
-		bin_info->n_shards = (i < opt_bin_shard_maxszind) ? opt_n_bin_shards : 1;
+		bin_info->n_shards = bin_shard_sizes[i];
 		bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER(
 		    bin_info->nregs);
 		bin_info->bitmap_info = bitmap_info;
 	}
 }
 
+bool
+bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size,
+    size_t end_size, size_t nshards) {
+	if (nshards > BIN_SHARDS_MAX || nshards == 0) {
+		return true;
+	}
+
+	if (start_size > SC_SMALL_MAXCLASS) {
+		return false;
+	}
+	if (end_size > SC_SMALL_MAXCLASS) {
+		end_size = SC_SMALL_MAXCLASS;
+	}
+
+	/* Compute the index since this may happen before sz init. */
+	szind_t ind1 = sz_size2index_compute(start_size);
+	szind_t ind2 = sz_size2index_compute(end_size);
+	for (unsigned i = ind1; i <= ind2; i++) {
+		bin_shard_sizes[i] = (unsigned)nshards;
+	}
+
+	return false;
+}
+
 void
-bin_boot(sc_data_t *sc_data) {
+bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) {
+	/* Load the default number of shards. */
+	for (unsigned i = 0; i < SC_NBINS; i++) {
+		bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT;
+	}
+}
+
+void
+bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
 	assert(sc_data->initialized);
-	opt_bin_shard_maxszind = OPT_BIN_SHARD_MAXSZIND;
+	bin_infos_init(sc_data, bin_shard_sizes, bin_infos);
-	opt_n_bin_shards = OPT_N_BIN_SHARDS;
-	bin_infos_init(sc_data, bin_infos);
 }
 
 bool

src/jemalloc.c

@@ -767,9 +767,10 @@ init_opt_stats_print_opts(const char *v, size_t vlen) {
 	assert(opts_len == strlen(opt_stats_print_opts));
 }
 
+/* Reads the next size pair in a multi-sized option. */
 static bool
-malloc_conf_slab_sizes_next(const char **slab_size_segment_cur,
+malloc_conf_multi_sizes_next(const char **slab_size_segment_cur,
-    size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *pgs) {
+    size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) {
 	const char *cur = *slab_size_segment_cur;
 	char *end;
 	uintmax_t um;
@@ -797,7 +798,7 @@ malloc_conf_slab_sizes_next(const char **slab_size_segment_cur,
 	if (get_errno() != 0) {
 		return true;
 	}
-	*pgs = (size_t)um;
+	*new_size = (size_t)um;
 
 	/* Consume the separator if there is one. */
 	if (*end == '|') {
@@ -923,7 +924,7 @@ malloc_slow_flag_init(void) {
 }
 
 static void
-malloc_conf_init(sc_data_t *sc_data) {
+malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
 	unsigned i;
 	char buf[PATH_MAX + 1];
 	const char *opts, *k, *v;
@@ -1161,6 +1162,28 @@ malloc_conf_init(sc_data_t *sc_data) {
 			}
 			CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
 			    UINT_MAX, yes, no, false)
+			if (CONF_MATCH("bin_shards")) {
+				const char *bin_shards_segment_cur = v;
+				size_t vlen_left = vlen;
+				do {
+					size_t size_start;
+					size_t size_end;
+					size_t nshards;
+					bool err = malloc_conf_multi_sizes_next(
+					    &bin_shards_segment_cur, &vlen_left,
+					    &size_start, &size_end, &nshards);
+					if (err || bin_update_shard_size(
+					    bin_shard_sizes, size_start,
+					    size_end, nshards)) {
+						malloc_conf_error(
+						    "Invalid settings for "
+						    "bin_shards", k, klen, v,
+						    vlen);
+						break;
+					}
+				} while (vlen_left > 0);
+				continue;
+			}
 			CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
 			    "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
 			    QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
@@ -1256,7 +1279,7 @@ malloc_conf_init(sc_data_t *sc_data) {
 					size_t slab_start;
 					size_t slab_end;
 					size_t pgs;
-					err = malloc_conf_slab_sizes_next(
+					err = malloc_conf_multi_sizes_next(
 					    &slab_size_segment_cur,
 					    &vlen_left, &slab_start, &slab_end,
 					    &pgs);
@@ -1390,6 +1413,8 @@ malloc_init_hard_a0_locked() {
 	 * out of sc_data_global are final.
 	 */
 	sc_boot(&sc_data);
+	unsigned bin_shard_sizes[SC_NBINS];
+	bin_shard_sizes_boot(bin_shard_sizes);
 	/*
 	 * prof_boot0 only initializes opt_prof_prefix.  We need to do it before
 	 * we parse malloc_conf options, in case malloc_conf parsing overwrites
@@ -1398,9 +1423,9 @@ malloc_init_hard_a0_locked() {
 	if (config_prof) {
 		prof_boot0();
 	}
-	malloc_conf_init(&sc_data);
+	malloc_conf_init(&sc_data, bin_shard_sizes);
 	sz_boot(&sc_data);
-	bin_boot(&sc_data);
+	bin_boot(&sc_data, bin_shard_sizes);
 
 	if (opt_stats_print) {
 		/* Print statistics at exit. */