Create arena_bin_info_t.
Move read-only fields from arena_bin_t into arena_bin_info_t, primarily in order to avoid false cacheline sharing.
This commit is contained in:
@@ -71,6 +71,7 @@
|
||||
typedef struct arena_chunk_map_s arena_chunk_map_t;
|
||||
typedef struct arena_chunk_s arena_chunk_t;
|
||||
typedef struct arena_run_s arena_run_t;
|
||||
typedef struct arena_bin_info_s arena_bin_info_t;
|
||||
typedef struct arena_bin_s arena_bin_t;
|
||||
typedef struct arena_s arena_t;
|
||||
|
||||
@@ -218,6 +219,33 @@ struct arena_run_s {
|
||||
unsigned nfree;
|
||||
};
|
||||
|
||||
/*
|
||||
* Read-only information associated with each element for arena_t's bins array
|
||||
* is stored separately, partly to reduce memory usage (only one copy, rather
|
||||
* than one per arena), but mainly to avoid false cacheline sharing.
|
||||
*/
|
||||
struct arena_bin_info_s {
|
||||
/* Size of regions in a run for this bin's size class. */
|
||||
size_t reg_size;
|
||||
|
||||
/* Total size of a run for this bin's size class. */
|
||||
size_t run_size;
|
||||
|
||||
/* Total number of regions in a run for this bin's size class. */
|
||||
uint32_t nregs;
|
||||
|
||||
#ifdef JEMALLOC_PROF
|
||||
/*
|
||||
* Offset of first (prof_ctx_t *) in a run header for this bin's size
|
||||
* class, or 0 if (opt_prof == false).
|
||||
*/
|
||||
uint32_t ctx0_offset;
|
||||
#endif
|
||||
|
||||
/* Offset of first region in a run for this bin's size class. */
|
||||
uint32_t reg0_offset;
|
||||
};
|
||||
|
||||
struct arena_bin_s {
|
||||
/*
|
||||
* All operations on runcur, runs, and stats require that lock be
|
||||
@@ -242,26 +270,6 @@ struct arena_bin_s {
|
||||
*/
|
||||
arena_run_tree_t runs;
|
||||
|
||||
/* Size of regions in a run for this bin's size class. */
|
||||
size_t reg_size;
|
||||
|
||||
/* Total size of a run for this bin's size class. */
|
||||
size_t run_size;
|
||||
|
||||
/* Total number of regions in a run for this bin's size class. */
|
||||
uint32_t nregs;
|
||||
|
||||
#ifdef JEMALLOC_PROF
|
||||
/*
|
||||
* Offset of first (prof_ctx_t *) in a run header for this bin's size
|
||||
* class, or 0 if (opt_prof == false).
|
||||
*/
|
||||
uint32_t ctx0_offset;
|
||||
#endif
|
||||
|
||||
/* Offset of first region in a run for this bin's size class. */
|
||||
uint32_t reg0_offset;
|
||||
|
||||
#ifdef JEMALLOC_STATS
|
||||
/* Bin statistics. */
|
||||
malloc_bin_stats_t stats;
|
||||
@@ -398,6 +406,8 @@ extern ssize_t opt_lg_dirty_mult;
|
||||
extern uint8_t const *small_size2bin;
|
||||
#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN])
|
||||
|
||||
extern arena_bin_info_t *arena_bin_info;
|
||||
|
||||
/* Various bin-related settings. */
|
||||
#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */
|
||||
# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN))
|
||||
@@ -463,7 +473,8 @@ bool arena_boot(void);
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
unsigned arena_run_regind(arena_run_t *run, arena_bin_t *bin,
|
||||
size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
|
||||
unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
|
||||
const void *ptr, size_t size);
|
||||
# ifdef JEMALLOC_PROF
|
||||
prof_ctx_t *arena_prof_ctx_get(const void *ptr);
|
||||
@@ -473,8 +484,16 @@ void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
|
||||
JEMALLOC_INLINE size_t
|
||||
arena_bin_index(arena_t *arena, arena_bin_t *bin)
|
||||
{
|
||||
size_t binind = bin - arena->bins;
|
||||
assert(binind < nbins);
|
||||
return (binind);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE unsigned
|
||||
arena_run_regind(arena_run_t *run, arena_bin_t *bin, const void *ptr,
|
||||
arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr,
|
||||
size_t size)
|
||||
{
|
||||
unsigned shift, diff, regind;
|
||||
@@ -485,7 +504,8 @@ arena_run_regind(arena_run_t *run, arena_bin_t *bin, const void *ptr,
|
||||
* Avoid doing division with a variable divisor if possible. Using
|
||||
* actual division here can reduce allocator throughput by over 20%!
|
||||
*/
|
||||
diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
|
||||
diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run -
|
||||
bin_info->reg0_offset);
|
||||
|
||||
/* Rescale (factor powers of 2 out of the numerator and denominator). */
|
||||
shift = ffs(size) - 1;
|
||||
@@ -531,7 +551,7 @@ arena_run_regind(arena_run_t *run, arena_bin_t *bin, const void *ptr,
|
||||
#undef SIZE_INV_SHIFT
|
||||
}
|
||||
assert(diff == regind * size);
|
||||
assert(regind < bin->nregs);
|
||||
assert(regind < bin_info->nregs);
|
||||
|
||||
return (regind);
|
||||
}
|
||||
@@ -558,13 +578,15 @@ arena_prof_ctx_get(const void *ptr)
|
||||
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
|
||||
(uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) <<
|
||||
PAGE_SHIFT));
|
||||
arena_bin_t *bin = run->bin;
|
||||
size_t binind = arena_bin_index(chunk->arena, run->bin);
|
||||
arena_bin_info_t *bin_info = &arena_bin_info[binind];
|
||||
unsigned regind;
|
||||
|
||||
assert(run->magic == ARENA_RUN_MAGIC);
|
||||
regind = arena_run_regind(run, bin, ptr, bin->reg_size);
|
||||
regind = arena_run_regind(run, bin_info, ptr,
|
||||
bin_info->reg_size);
|
||||
ret = *(prof_ctx_t **)((uintptr_t)run +
|
||||
bin->ctx0_offset + (regind *
|
||||
bin_info->ctx0_offset + (regind *
|
||||
sizeof(prof_ctx_t *)));
|
||||
}
|
||||
} else
|
||||
@@ -593,11 +615,15 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
|
||||
PAGE_SHIFT));
|
||||
arena_bin_t *bin = run->bin;
|
||||
unsigned regind;
|
||||
size_t binind;
|
||||
arena_bin_info_t *bin_info;
|
||||
|
||||
assert(run->magic == ARENA_RUN_MAGIC);
|
||||
regind = arena_run_regind(run, bin, ptr, bin->reg_size);
|
||||
binind = arena_bin_index(chunk->arena, bin);
|
||||
bin_info = &arena_bin_info[binind];
|
||||
|
||||
*((prof_ctx_t **)((uintptr_t)run + bin->ctx0_offset
|
||||
*((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset
|
||||
+ (regind * sizeof(prof_ctx_t *)))) = ctx;
|
||||
} else
|
||||
assert((uintptr_t)ctx == (uintptr_t)1U);
|
||||
@@ -637,10 +663,17 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
|
||||
(uintptr_t)((pageind - (mapelm->bits >>
|
||||
PAGE_SHIFT)) << PAGE_SHIFT));
|
||||
assert(run->magic == ARENA_RUN_MAGIC);
|
||||
assert(((uintptr_t)ptr - ((uintptr_t)run +
|
||||
(uintptr_t)run->bin->reg0_offset)) %
|
||||
run->bin->reg_size == 0);
|
||||
bin = run->bin;
|
||||
#ifndef NDEBUG
|
||||
{
|
||||
size_t binind = arena_bin_index(arena, bin);
|
||||
arena_bin_info_t *bin_info =
|
||||
&arena_bin_info[binind];
|
||||
assert(((uintptr_t)ptr - ((uintptr_t)run +
|
||||
(uintptr_t)bin_info->reg0_offset)) %
|
||||
bin_info->reg_size == 0);
|
||||
}
|
||||
#endif
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
arena_dalloc_bin(arena, chunk, ptr, mapelm);
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
|
@@ -402,7 +402,7 @@ s2u(size_t size)
|
||||
{
|
||||
|
||||
if (size <= small_maxclass)
|
||||
return (arenas[0]->bins[SMALL_SIZE2BIN(size)].reg_size);
|
||||
return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size);
|
||||
if (size <= arena_maxclass)
|
||||
return (PAGE_CEILING(size));
|
||||
return (CHUNK_CEILING(size));
|
||||
@@ -446,10 +446,8 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p)
|
||||
}
|
||||
|
||||
if (usize <= arena_maxclass && alignment <= PAGE_SIZE) {
|
||||
if (usize <= small_maxclass) {
|
||||
return
|
||||
(arenas[0]->bins[SMALL_SIZE2BIN(usize)].reg_size);
|
||||
}
|
||||
if (usize <= small_maxclass)
|
||||
return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size);
|
||||
return (PAGE_CEILING(usize));
|
||||
} else {
|
||||
size_t run_size;
|
||||
|
@@ -232,7 +232,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
}
|
||||
assert(arena_salloc(ret) == tcache->arena->bins[binind].reg_size);
|
||||
assert(arena_salloc(ret) == arena_bin_info[binind].reg_size);
|
||||
|
||||
if (zero == false) {
|
||||
#ifdef JEMALLOC_FILL
|
||||
@@ -248,7 +248,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
|
||||
tbin->tstats.nrequests++;
|
||||
#endif
|
||||
#ifdef JEMALLOC_PROF
|
||||
tcache->prof_accumbytes += tcache->arena->bins[binind].reg_size;
|
||||
tcache->prof_accumbytes += arena_bin_info[binind].reg_size;
|
||||
#endif
|
||||
tcache_event(tcache);
|
||||
return (ret);
|
||||
@@ -331,7 +331,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
|
||||
|
||||
#ifdef JEMALLOC_FILL
|
||||
if (opt_junk)
|
||||
memset(ptr, 0x5a, bin->reg_size);
|
||||
memset(ptr, 0x5a, arena_bin_info[binind].reg_size);
|
||||
#endif
|
||||
|
||||
tbin = &tcache->tbins[binind];
|
||||
|
Reference in New Issue
Block a user