Add opt.bin_shards to specify number of bin shards.
The option uses the same format as "slab_sizes" to specify number of shards for each bin size.
This commit is contained in:
parent
37b8913925
commit
3f9f2833f6
@ -8,9 +8,7 @@
|
||||
#include "jemalloc/internal/sc.h"
|
||||
|
||||
#define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH)
|
||||
|
||||
extern unsigned opt_bin_shard_maxszind;
|
||||
extern unsigned opt_n_bin_shards;
|
||||
#define N_BIN_SHARDS_DEFAULT 1
|
||||
|
||||
/*
|
||||
* A bin contains a set of extents that are currently being used for slab
|
||||
@ -93,8 +91,10 @@ struct bins_s {
|
||||
bin_t *bin_shards;
|
||||
};
|
||||
|
||||
void bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]);
|
||||
void bin_boot();
|
||||
void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]);
|
||||
bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size,
|
||||
size_t end_size, size_t nshards);
|
||||
void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]);
|
||||
|
||||
/* Initializes a bin to empty. Returns true on error. */
|
||||
bool bin_init(bin_t *bin);
|
||||
|
@ -128,10 +128,6 @@ struct extent_s {
|
||||
#define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT)
|
||||
#define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT)
|
||||
|
||||
/* Will make dynamic options. */
|
||||
#define OPT_N_BIN_SHARDS (1)
|
||||
#define OPT_BIN_SHARD_MAXSZIND (0)
|
||||
|
||||
#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT)
|
||||
#define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT)
|
||||
|
||||
|
11
src/arena.c
11
src/arena.c
@ -1346,8 +1346,7 @@ bin_t *
|
||||
arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind,
|
||||
unsigned *binshard) {
|
||||
bin_t *bin;
|
||||
if (binind >= opt_bin_shard_maxszind || tsdn_null(tsdn) ||
|
||||
tsd_arena_get(tsdn_tsd(tsdn)) == NULL) {
|
||||
if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) {
|
||||
*binshard = 0;
|
||||
} else {
|
||||
*binshard = tsd_binshard_get(tsdn_tsd(tsdn)) %
|
||||
@ -1923,9 +1922,11 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
|
||||
}
|
||||
}
|
||||
|
||||
size_t arena_size = sizeof(arena_t) +
|
||||
sizeof(bin_t) * opt_n_bin_shards * opt_bin_shard_maxszind +
|
||||
sizeof(bin_t) * (SC_NBINS - opt_bin_shard_maxszind);
|
||||
unsigned nbins_total = 0;
|
||||
for (i = 0; i < SC_NBINS; i++) {
|
||||
nbins_total += bin_infos[i].n_shards;
|
||||
}
|
||||
size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total;
|
||||
arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE);
|
||||
if (arena == NULL) {
|
||||
goto label_error;
|
||||
|
48
src/bin.c
48
src/bin.c
@ -6,13 +6,11 @@
|
||||
#include "jemalloc/internal/sc.h"
|
||||
#include "jemalloc/internal/witness.h"
|
||||
|
||||
unsigned opt_bin_shard_maxszind;
|
||||
unsigned opt_n_bin_shards;
|
||||
|
||||
bin_info_t bin_infos[SC_NBINS];
|
||||
|
||||
void
|
||||
bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]) {
|
||||
static void
|
||||
bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
|
||||
bin_info_t bin_infos[SC_NBINS]) {
|
||||
for (unsigned i = 0; i < SC_NBINS; i++) {
|
||||
bin_info_t *bin_info = &bin_infos[i];
|
||||
sc_t *sc = &sc_data->sc[i];
|
||||
@ -21,19 +19,49 @@ bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]) {
|
||||
bin_info->slab_size = (sc->pgs << LG_PAGE);
|
||||
bin_info->nregs =
|
||||
(uint32_t)(bin_info->slab_size / bin_info->reg_size);
|
||||
bin_info->n_shards = (i < opt_bin_shard_maxszind) ? opt_n_bin_shards : 1;
|
||||
bin_info->n_shards = bin_shard_sizes[i];
|
||||
bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER(
|
||||
bin_info->nregs);
|
||||
bin_info->bitmap_info = bitmap_info;
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size,
|
||||
size_t end_size, size_t nshards) {
|
||||
if (nshards > BIN_SHARDS_MAX || nshards == 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (start_size > SC_SMALL_MAXCLASS) {
|
||||
return false;
|
||||
}
|
||||
if (end_size > SC_SMALL_MAXCLASS) {
|
||||
end_size = SC_SMALL_MAXCLASS;
|
||||
}
|
||||
|
||||
/* Compute the index since this may happen before sz init. */
|
||||
szind_t ind1 = sz_size2index_compute(start_size);
|
||||
szind_t ind2 = sz_size2index_compute(end_size);
|
||||
for (unsigned i = ind1; i <= ind2; i++) {
|
||||
bin_shard_sizes[i] = (unsigned)nshards;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void
|
||||
bin_boot(sc_data_t *sc_data) {
|
||||
bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) {
|
||||
/* Load the default number of shards. */
|
||||
for (unsigned i = 0; i < SC_NBINS; i++) {
|
||||
bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
|
||||
assert(sc_data->initialized);
|
||||
opt_bin_shard_maxszind = OPT_BIN_SHARD_MAXSZIND;
|
||||
opt_n_bin_shards = OPT_N_BIN_SHARDS;
|
||||
bin_infos_init(sc_data, bin_infos);
|
||||
bin_infos_init(sc_data, bin_shard_sizes, bin_infos);
|
||||
}
|
||||
|
||||
bool
|
||||
|
@ -767,9 +767,10 @@ init_opt_stats_print_opts(const char *v, size_t vlen) {
|
||||
assert(opts_len == strlen(opt_stats_print_opts));
|
||||
}
|
||||
|
||||
/* Reads the next size pair in a multi-sized option. */
|
||||
static bool
|
||||
malloc_conf_slab_sizes_next(const char **slab_size_segment_cur,
|
||||
size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *pgs) {
|
||||
malloc_conf_multi_sizes_next(const char **slab_size_segment_cur,
|
||||
size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) {
|
||||
const char *cur = *slab_size_segment_cur;
|
||||
char *end;
|
||||
uintmax_t um;
|
||||
@ -797,7 +798,7 @@ malloc_conf_slab_sizes_next(const char **slab_size_segment_cur,
|
||||
if (get_errno() != 0) {
|
||||
return true;
|
||||
}
|
||||
*pgs = (size_t)um;
|
||||
*new_size = (size_t)um;
|
||||
|
||||
/* Consume the separator if there is one. */
|
||||
if (*end == '|') {
|
||||
@ -923,7 +924,7 @@ malloc_slow_flag_init(void) {
|
||||
}
|
||||
|
||||
static void
|
||||
malloc_conf_init(sc_data_t *sc_data) {
|
||||
malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) {
|
||||
unsigned i;
|
||||
char buf[PATH_MAX + 1];
|
||||
const char *opts, *k, *v;
|
||||
@ -1161,6 +1162,28 @@ malloc_conf_init(sc_data_t *sc_data) {
|
||||
}
|
||||
CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
|
||||
UINT_MAX, yes, no, false)
|
||||
if (CONF_MATCH("bin_shards")) {
|
||||
const char *bin_shards_segment_cur = v;
|
||||
size_t vlen_left = vlen;
|
||||
do {
|
||||
size_t size_start;
|
||||
size_t size_end;
|
||||
size_t nshards;
|
||||
bool err = malloc_conf_multi_sizes_next(
|
||||
&bin_shards_segment_cur, &vlen_left,
|
||||
&size_start, &size_end, &nshards);
|
||||
if (err || bin_update_shard_size(
|
||||
bin_shard_sizes, size_start,
|
||||
size_end, nshards)) {
|
||||
malloc_conf_error(
|
||||
"Invalid settings for "
|
||||
"bin_shards", k, klen, v,
|
||||
vlen);
|
||||
break;
|
||||
}
|
||||
} while (vlen_left > 0);
|
||||
continue;
|
||||
}
|
||||
CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms,
|
||||
"dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) <
|
||||
QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) :
|
||||
@ -1256,7 +1279,7 @@ malloc_conf_init(sc_data_t *sc_data) {
|
||||
size_t slab_start;
|
||||
size_t slab_end;
|
||||
size_t pgs;
|
||||
err = malloc_conf_slab_sizes_next(
|
||||
err = malloc_conf_multi_sizes_next(
|
||||
&slab_size_segment_cur,
|
||||
&vlen_left, &slab_start, &slab_end,
|
||||
&pgs);
|
||||
@ -1390,6 +1413,8 @@ malloc_init_hard_a0_locked() {
|
||||
* out of sc_data_global are final.
|
||||
*/
|
||||
sc_boot(&sc_data);
|
||||
unsigned bin_shard_sizes[SC_NBINS];
|
||||
bin_shard_sizes_boot(bin_shard_sizes);
|
||||
/*
|
||||
* prof_boot0 only initializes opt_prof_prefix. We need to do it before
|
||||
* we parse malloc_conf options, in case malloc_conf parsing overwrites
|
||||
@ -1398,9 +1423,9 @@ malloc_init_hard_a0_locked() {
|
||||
if (config_prof) {
|
||||
prof_boot0();
|
||||
}
|
||||
malloc_conf_init(&sc_data);
|
||||
malloc_conf_init(&sc_data, bin_shard_sizes);
|
||||
sz_boot(&sc_data);
|
||||
bin_boot(&sc_data);
|
||||
bin_boot(&sc_data, bin_shard_sizes);
|
||||
|
||||
if (opt_stats_print) {
|
||||
/* Print statistics at exit. */
|
||||
|
Loading…
Reference in New Issue
Block a user