project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Transition arena struct fields to C11 atomics

Browse Source
This commit is contained in:
David Goldblatt 2017-04-04 17:22:24 -07:00 committed by David Goldblatt
parent bc32ec3503
commit 56b72c7b17
3 changed files with 48 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/jemalloc/internal/arena_structs_b.h
View File

@ -134,7 +134,7 @@ struct arena_s {
* *
* Synchronization: atomic. * Synchronization: atomic.
*/ */
unsigned nthreads[2]; atomic_u_t nthreads[2];
/* /*
* When percpu_arena is enabled, to amortize the cost of reading / * When percpu_arena is enabled, to amortize the cost of reading /
@ -173,17 +173,21 @@ struct arena_s {
* *
* Synchronization: atomic. * Synchronization: atomic.
*/ */
size_t extent_sn_next; atomic_zu_t extent_sn_next;
/* Synchronization: atomic. */ /*
dss_prec_t dss_prec; * Represents a dss_prec_t, but atomically.
*
* Synchronization: atomic.
*/
atomic_u_t dss_prec;
/* /*
* Number of pages in active extents. * Number of pages in active extents.
* *
* Synchronization: atomic. * Synchronization: atomic.
*/ */
size_t nactive; atomic_zu_t nactive;
/* /*
* Extant large allocations. * Extant large allocations.
@ -222,7 +226,7 @@ struct arena_s {
* *
* Synchronization: atomic. * Synchronization: atomic.
*/ */
pszind_t extent_grow_next; atomic_u_t extent_grow_next;
/* /*
* Freelist of extent structures that were allocated via base_alloc(). * Freelist of extent structures that were allocated via base_alloc().

56
src/arena.c
View File

@ -13,10 +13,11 @@ const char *percpu_arena_mode_names[] = {
const char *opt_percpu_arena = OPT_PERCPU_ARENA_DEFAULT; const char *opt_percpu_arena = OPT_PERCPU_ARENA_DEFAULT;
percpu_arena_mode_t percpu_arena_mode = PERCPU_ARENA_MODE_DEFAULT; percpu_arena_mode_t percpu_arena_mode = PERCPU_ARENA_MODE_DEFAULT;
ssize_t opt_dirty_decay_time = DIRTY_DECAY_TIME_DEFAULT; ssize_t opt_dirty_decay_time = DIRTY_DECAY_TIME_DEFAULT;
ssize_t opt_muzzy_decay_time = MUZZY_DECAY_TIME_DEFAULT; ssize_t opt_muzzy_decay_time = MUZZY_DECAY_TIME_DEFAULT;
static ssize_t dirty_decay_time_default;
static ssize_t muzzy_decay_time_default; static atomic_zd_t dirty_decay_time_default;
static atomic_zd_t muzzy_decay_time_default;
const arena_bin_info_t arena_bin_info[NBINS] = { const arena_bin_info_t arena_bin_info[NBINS] = {
#define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \ #define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \
@ -197,7 +198,7 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
*dss = dss_prec_names[arena_dss_prec_get(arena)]; *dss = dss_prec_names[arena_dss_prec_get(arena)];
*dirty_decay_time = arena_dirty_decay_time_get(arena); *dirty_decay_time = arena_dirty_decay_time_get(arena);
*muzzy_decay_time = arena_muzzy_decay_time_get(arena); *muzzy_decay_time = arena_muzzy_decay_time_get(arena);
*nactive += atomic_read_zu(&arena->nactive); *nactive += atomic_load_zu(&arena->nactive, ATOMIC_RELAXED);
*ndirty += extents_npages_get(&arena->extents_dirty); *ndirty += extents_npages_get(&arena->extents_dirty);
*nmuzzy += extents_npages_get(&arena->extents_muzzy); *nmuzzy += extents_npages_get(&arena->extents_muzzy);
} }
@ -246,7 +247,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_stats_accum_zu(&astats->base, base_allocated); arena_stats_accum_zu(&astats->base, base_allocated);
arena_stats_accum_zu(&astats->internal, arena_internal_get(arena)); arena_stats_accum_zu(&astats->internal, arena_internal_get(arena));
arena_stats_accum_zu(&astats->resident, base_resident + arena_stats_accum_zu(&astats->resident, base_resident +
(((atomic_read_zu(&arena->nactive) + (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) +
extents_npages_get(&arena->extents_dirty) + extents_npages_get(&arena->extents_dirty) +
extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); extents_npages_get(&arena->extents_muzzy)) << LG_PAGE)));
@ -426,13 +427,13 @@ arena_slab_reg_dalloc(tsdn_t *tsdn, extent_t *slab,
static void static void
arena_nactive_add(arena_t *arena, size_t add_pages) { arena_nactive_add(arena_t *arena, size_t add_pages) {
atomic_add_zu(&arena->nactive, add_pages); atomic_fetch_add_zu(&arena->nactive, add_pages, ATOMIC_RELAXED);
} }
static void static void
arena_nactive_sub(arena_t *arena, size_t sub_pages) { arena_nactive_sub(arena_t *arena, size_t sub_pages) {
assert(atomic_read_zu(&arena->nactive) >= sub_pages); assert(atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) >= sub_pages);
atomic_sub_zu(&arena->nactive, sub_pages); atomic_fetch_sub_zu(&arena->nactive, sub_pages, ATOMIC_RELAXED);
} }
static void static void
@ -1100,7 +1101,7 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
} }
atomic_write_zu(&arena->nactive, 0); atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED);
} }
static void static void
@ -1750,7 +1751,7 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
dss_prec_t dss_prec_t
arena_dss_prec_get(arena_t *arena) { arena_dss_prec_get(arena_t *arena) {
return (dss_prec_t)atomic_read_u((unsigned *)&arena->dss_prec); return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE);
} }
bool bool
@ -1758,13 +1759,13 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) {
if (!have_dss) { if (!have_dss) {
return (dss_prec != dss_prec_disabled); return (dss_prec != dss_prec_disabled);
} }
atomic_write_u((unsigned *)&arena->dss_prec, dss_prec); atomic_store_u(&arena->dss_prec, (unsigned)dss_prec, ATOMIC_RELEASE);
return false; return false;
} }
ssize_t ssize_t
arena_dirty_decay_time_default_get(void) { arena_dirty_decay_time_default_get(void) {
return (ssize_t)atomic_read_zu((size_t *)&dirty_decay_time_default); return atomic_load_zd(&dirty_decay_time_default, ATOMIC_RELAXED);
} }
bool bool
@ -1772,14 +1773,13 @@ arena_dirty_decay_time_default_set(ssize_t decay_time) {
if (!arena_decay_time_valid(decay_time)) { if (!arena_decay_time_valid(decay_time)) {
return true; return true;
} }
atomic_write_zu((size_t *)&dirty_decay_time_default, atomic_store_zd(&dirty_decay_time_default, decay_time, ATOMIC_RELAXED);
(size_t)decay_time);
return false; return false;
} }
ssize_t ssize_t
arena_muzzy_decay_time_default_get(void) { arena_muzzy_decay_time_default_get(void) {
return (ssize_t)atomic_read_zu((size_t *)&muzzy_decay_time_default); return atomic_load_zd(&muzzy_decay_time_default, ATOMIC_RELAXED);
} }
bool bool
@ -1787,29 +1787,28 @@ arena_muzzy_decay_time_default_set(ssize_t decay_time) {
if (!arena_decay_time_valid(decay_time)) { if (!arena_decay_time_valid(decay_time)) {
return true; return true;
} }
atomic_write_zu((size_t *)&muzzy_decay_time_default, atomic_store_zd(&muzzy_decay_time_default, decay_time, ATOMIC_RELAXED);
(size_t)decay_time);
return false; return false;
} }
unsigned unsigned
arena_nthreads_get(arena_t *arena, bool internal) { arena_nthreads_get(arena_t *arena, bool internal) {
return atomic_read_u(&arena->nthreads[internal]); return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED);
} }
void void
arena_nthreads_inc(arena_t *arena, bool internal) { arena_nthreads_inc(arena_t *arena, bool internal) {
atomic_add_u(&arena->nthreads[internal], 1); atomic_fetch_add_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED);
} }
void void
arena_nthreads_dec(arena_t *arena, bool internal) { arena_nthreads_dec(arena_t *arena, bool internal) {
atomic_sub_u(&arena->nthreads[internal], 1); atomic_fetch_sub_u(&arena->nthreads[internal], 1, ATOMIC_RELAXED);
} }
size_t size_t
arena_extent_sn_next(arena_t *arena) { arena_extent_sn_next(arena_t *arena) {
return atomic_add_zu(&arena->extent_sn_next, 1) - 1; return atomic_fetch_add_zu(&arena->extent_sn_next, 1, ATOMIC_RELAXED);
} }
arena_t * arena_t *
@ -1832,7 +1831,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
goto label_error; goto label_error;
} }
arena->nthreads[0] = arena->nthreads[1] = 0; atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
arena->last_thd = NULL; arena->last_thd = NULL;
if (config_stats) { if (config_stats) {
@ -1867,11 +1867,12 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
(size_t)(uintptr_t)arena, ATOMIC_RELAXED); (size_t)(uintptr_t)arena, ATOMIC_RELAXED);
} }
arena->extent_sn_next = 0; atomic_store_zu(&arena->extent_sn_next, 0, ATOMIC_RELAXED);
arena->dss_prec = extent_dss_prec_get(); atomic_store_u(&arena->dss_prec, (unsigned)extent_dss_prec_get(),
ATOMIC_RELAXED);
atomic_write_zu(&arena->nactive, 0); atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED);
extent_list_init(&arena->large); extent_list_init(&arena->large);
if (malloc_mutex_init(&arena->large_mtx, "arena_large", if (malloc_mutex_init(&arena->large_mtx, "arena_large",
@ -1918,7 +1919,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
} }
if (!config_munmap) { if (!config_munmap) {
arena->extent_grow_next = psz2ind(HUGEPAGE); atomic_store_u(&arena->extent_grow_next, psz2ind(HUGEPAGE),
ATOMIC_RELAXED);
} }
extent_list_init(&arena->extent_freelist); extent_list_init(&arena->extent_freelist);

15
src/extent.c
View File

@ -913,7 +913,8 @@ extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
void *ret; void *ret;
ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
commit, arena->dss_prec); commit, (dss_prec_t)atomic_load_u(&arena->dss_prec,
ATOMIC_RELAXED));
return ret; return ret;
} }
@ -956,7 +957,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
* extent creation as a side effect. * extent creation as a side effect.
*/ */
size_t esize = size + pad; size_t esize = size + pad;
size_t alloc_size = pind2sz(atomic_read_u(&arena->extent_grow_next)); size_t alloc_size = pind2sz(atomic_load_u(&arena->extent_grow_next,
ATOMIC_RELAXED));
size_t alloc_size_min = esize + PAGE_CEILING(alignment) - PAGE; size_t alloc_size_min = esize + PAGE_CEILING(alignment) - PAGE;
/* Beware size_t wrap-around. */ /* Beware size_t wrap-around. */
if (alloc_size_min < esize) { if (alloc_size_min < esize) {
@ -972,7 +974,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
bool zeroed = false; bool zeroed = false;
bool committed = false; bool committed = false;
void *ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE, void *ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE,
&zeroed, &committed, arena->dss_prec); &zeroed, &committed, (dss_prec_t)atomic_load_u(&arena->dss_prec,
ATOMIC_RELAXED));
extent_init(extent, arena, ptr, alloc_size, false, NSIZES, extent_init(extent, arena, ptr, alloc_size, false, NSIZES,
arena_extent_sn_next(arena), extent_state_active, zeroed, arena_extent_sn_next(arena), extent_state_active, zeroed,
committed); committed);
@ -1078,14 +1081,14 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
* Increment extent_grow_next, but take care to do so atomically and * Increment extent_grow_next, but take care to do so atomically and
* bail out if the increment would exceed the legal range. * bail out if the increment would exceed the legal range.
*/ */
pszind_t egn = atomic_load_u(&arena->extent_grow_next, ATOMIC_RELAXED);
while (true) { while (true) {
pszind_t egn = atomic_read_u(&arena->extent_grow_next);
if (egn + 1 == NPSIZES) { if (egn + 1 == NPSIZES) {
break; break;
} }
assert(egn + 1 < NPSIZES); assert(egn + 1 < NPSIZES);
if (!atomic_cas_u(&arena->extent_grow_next, egn, egn + 1)) { if (atomic_compare_exchange_weak_u(&arena->extent_grow_next,
&egn, egn + 1, ATOMIC_RELAXED, ATOMIC_RELAXED)) {
break; break;
} }
} }