project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SC: Remove global data.

Browse Source 
The global data is mostly only used at initialization, or for easy access to
values we could compute statically.  Instead of consuming that space (and
risking TLB misses), we can just pass around a pointer to stack data during
bootstrapping.
This commit is contained in:
David Goldblatt
2018-07-19 17:08:10 -07:00
committed by David Goldblatt
parent 4bc48718b2
commit 3aba072cef
16 changed files with 73 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/arena.c
View File

@@ -1754,8 +1754,7 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit,
if (new_limit != NULL) {
size_t limit = *new_limit;
/* Grow no more than the new limit. */
if ((new_ind = sz_psz2ind(limit + 1) - 1)
>= sc_data_global.npsizes) {
if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) {
return true;
}
}
@@ -1899,7 +1898,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
}
arena->extent_grow_next = sz_psz2ind(HUGEPAGE);
arena->retain_grow_limit = sc_data_global.npsizes - 1;
arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS);
if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow",
WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
goto label_error;
@@ -2001,11 +2000,11 @@ arena_init_huge(void) {
}
void
arena_boot(void) {
arena_boot(sc_data_t *sc_data) {
arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms);
for (unsigned i = 0; i < SC_NBINS; i++) {
sc_t *sc = &sc_data_global.sc[i];
sc_t *sc = &sc_data->sc[i];
div_init(&arena_binind_div_info[i],
(1U << sc->lg_base) + (sc->ndelta << sc->lg_delta));
}

2
src/base.c
View File

@@ -262,7 +262,7 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks,
*/
size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size
+ usize));
pszind_t pind_next = (*pind_last + 1 < sc_data_global.npsizes) ?
pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ?
*pind_last + 1 : *pind_last;
size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next));
size_t block_size = (min_block_size > next_block_size) ? min_block_size

20
src/extent.c
View File

@@ -20,7 +20,7 @@ mutex_pool_t extent_mutex_pool;
size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;
static const bitmap_info_t extents_bitmap_info =
BITMAP_INFO_INITIALIZER(SC_NPSIZES_MAX+1);
BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);
static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit,
@@ -288,7 +288,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
malloc_mutex_rank_exclusive)) {
return true;
}
for (unsigned i = 0; i < sc_data_global.npsizes + 1; i++) {
for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
extent_heap_new(&extents->heaps[i]);
}
bitmap_init(extents->bitmap, &extents_bitmap_info, true);
@@ -375,7 +375,7 @@ extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size,
&extents_bitmap_info, (size_t)pind); i < pind_max; i =
(pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)i+1)) {
assert(i < sc_data_global.npsizes);
assert(i < SC_NPSIZES);
assert(!extent_heap_empty(&extents->heaps[i]));
extent_t *extent = extent_heap_first(&extents->heaps[i]);
uintptr_t base = (uintptr_t)extent_base_get(extent);
@@ -405,7 +405,7 @@ extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size));
pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)pind);
if (i < sc_data_global.npsizes + 1) {
if (i < SC_NPSIZES + 1) {
/*
* In order to reduce fragmentation, avoid reusing and splitting
* large extents for much smaller sizes.
@@ -434,7 +434,7 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size));
for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,
&extents_bitmap_info, (size_t)pind);
i < sc_data_global.npsizes + 1;
i < SC_NPSIZES + 1;
i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
(size_t)i+1)) {
assert(!extent_heap_empty(&extents->heaps[i]));
@@ -443,10 +443,10 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
if (ret == NULL || extent_snad_comp(extent, ret) < 0) {
ret = extent;
}
if (i == sc_data_global.npsizes) {
if (i == SC_NPSIZES) {
break;
}
assert(i < sc_data_global.npsizes);
assert(i < SC_NPSIZES);
}
return ret;
@@ -1249,13 +1249,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
while (alloc_size < alloc_size_min) {
egn_skip++;
if (arena->extent_grow_next + egn_skip ==
sc_data_global.npsizes) {
if (arena->extent_grow_next + egn_skip >=
sz_psz2ind(SC_LARGE_MAXCLASS)) {
/* Outside legal range. */
goto label_err;
}
assert(arena->extent_grow_next + egn_skip
< sc_data_global.npsizes);
alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
}

19
src/jemalloc.c
View File

@@ -920,7 +920,7 @@ malloc_slow_flag_init(void) {
}
static void
malloc_conf_init(void) {
malloc_conf_init(sc_data_t *sc_data) {
unsigned i;
char buf[PATH_MAX + 1];
const char *opts, *k, *v;
@@ -1254,7 +1254,7 @@ malloc_conf_init(void) {
&pgs);
if (!err) {
sc_data_update_slab_size(
&sc_data_global, slab_start,
sc_data, slab_start,
slab_end, (int)pgs);
} else {
malloc_conf_error(
@@ -1368,6 +1368,11 @@ static bool
malloc_init_hard_a0_locked() {
malloc_initializer = INITIALIZER;
JEMALLOC_DIAGNOSTIC_PUSH
JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
sc_data_t sc_data = {0};
JEMALLOC_DIAGNOSTIC_POP
/*
* Ordering here is somewhat tricky; we need sc_boot() first, since that
* determines what the size classes will be, and then
@@ -1375,10 +1380,10 @@ malloc_init_hard_a0_locked() {
* before sz_boot and bin_boot, which assume that the values they read
* out of sc_data_global are final.
*/
sc_boot();
malloc_conf_init();
sz_boot(&sc_data_global);
bin_boot(&sc_data_global);
sc_boot(&sc_data);
malloc_conf_init(&sc_data);
sz_boot(&sc_data);
bin_boot(&sc_data);
if (config_prof) {
prof_boot0();
@@ -1407,7 +1412,7 @@ malloc_init_hard_a0_locked() {
if (config_prof) {
prof_boot1();
}
arena_boot();
arena_boot(&sc_data);
if (tcache_boot(TSDN_NULL)) {
return true;
}

6
src/sc.c
View File

@@ -238,6 +238,8 @@ size_classes(
* touch the extra global cacheline. We assert, however, that the two
* computations are equivalent.
*/
assert(sc_data->npsizes == SC_NPSIZES);
assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS);
assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS);
assert(sc_data->large_minclass == SC_LARGE_MINCLASS);
assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS);
@@ -297,6 +299,6 @@ sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) {
}
void
sc_boot() {
sc_data_init(&sc_data_global);
sc_boot(sc_data_t *data) {
sc_data_init(data);
}

6
src/sz.c
View File

@@ -2,7 +2,7 @@
#include "jemalloc/internal/sz.h"
JEMALLOC_ALIGNED(CACHELINE)
size_t sz_pind2sz_tab[SC_NPSIZES_MAX+1];
size_t sz_pind2sz_tab[SC_NPSIZES+1];
static void
sz_boot_pind2sz_tab(const sc_data_t *sc_data) {
@@ -15,7 +15,9 @@ sz_boot_pind2sz_tab(const sc_data_t *sc_data) {
pind++;
}
}
sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE;
for (int i = pind; i <= (int)SC_NPSIZES; i++) {
sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE;
}
}
JEMALLOC_ALIGNED(CACHELINE)