diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h index a5191d16..fe4e4eed 100644 --- a/include/jemalloc/internal/arena_structs_b.h +++ b/include/jemalloc/internal/arena_structs_b.h @@ -166,7 +166,7 @@ struct arena_s { * * Synchronization: atomic. */ - size_t offset_state; + atomic_zu_t offset_state; /* * Extent serial number generator state. diff --git a/include/jemalloc/internal/prng_inlines.h b/include/jemalloc/internal/prng_inlines.h index 646e07b7..3f06ccd4 100644 --- a/include/jemalloc/internal/prng_inlines.h +++ b/include/jemalloc/internal/prng_inlines.h @@ -6,14 +6,15 @@ uint32_t prng_state_next_u32(uint32_t state); uint64_t prng_state_next_u64(uint64_t state); size_t prng_state_next_zu(size_t state); -uint32_t prng_lg_range_u32(uint32_t *state, unsigned lg_range, +uint32_t prng_lg_range_u32(atomic_u32_t *state, unsigned lg_range, bool atomic); uint64_t prng_lg_range_u64(uint64_t *state, unsigned lg_range); -size_t prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic); +size_t prng_lg_range_zu(atomic_zu_t *state, unsigned lg_range, bool atomic); -uint32_t prng_range_u32(uint32_t *state, uint32_t range, bool atomic); +uint32_t prng_range_u32(atomic_u32_t *state, uint32_t range, + bool atomic); uint64_t prng_range_u64(uint64_t *state, uint64_t range); -size_t prng_range_zu(size_t *state, size_t range, bool atomic); +size_t prng_range_zu(atomic_zu_t *state, size_t range, bool atomic); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_)) @@ -39,22 +40,22 @@ prng_state_next_zu(size_t state) { } JEMALLOC_ALWAYS_INLINE uint32_t -prng_lg_range_u32(uint32_t *state, unsigned lg_range, bool atomic) { - uint32_t ret, state1; +prng_lg_range_u32(atomic_u32_t *state, unsigned lg_range, bool atomic) { + uint32_t ret, state0, state1; assert(lg_range > 0); assert(lg_range <= 32); - if (atomic) { - uint32_t state0; + state0 = atomic_load_u32(state, ATOMIC_RELAXED); + if (atomic) { do { - state0 = atomic_read_u32(state); state1 = prng_state_next_u32(state0); - } while (atomic_cas_u32(state, state0, state1)); + } while (!atomic_compare_exchange_weak_u32(state, &state0, + state1, ATOMIC_RELAXED, ATOMIC_RELAXED)); } else { - state1 = prng_state_next_u32(*state); - *state = state1; + state1 = prng_state_next_u32(state0); + atomic_store_u32(state, state1, ATOMIC_RELAXED); } ret = state1 >> (32 - lg_range); @@ -77,22 +78,22 @@ prng_lg_range_u64(uint64_t *state, unsigned lg_range) { } JEMALLOC_ALWAYS_INLINE size_t -prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic) { - size_t ret, state1; +prng_lg_range_zu(atomic_zu_t *state, unsigned lg_range, bool atomic) { + size_t ret, state0, state1; assert(lg_range > 0); assert(lg_range <= ZU(1) << (3 + LG_SIZEOF_PTR)); - if (atomic) { - size_t state0; + state0 = atomic_load_zu(state, ATOMIC_RELAXED); + if (atomic) { do { - state0 = atomic_read_zu(state); state1 = prng_state_next_zu(state0); - } while (atomic_cas_zu(state, state0, state1)); + } while (atomic_compare_exchange_weak_zu(state, &state0, + state1, ATOMIC_RELAXED, ATOMIC_RELAXED)); } else { - state1 = prng_state_next_zu(*state); - *state = state1; + state1 = prng_state_next_zu(state0); + atomic_store_zu(state, state1, ATOMIC_RELAXED); } ret = state1 >> ((ZU(1) << (3 + LG_SIZEOF_PTR)) - lg_range); @@ -100,7 +101,7 @@ prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic) { } JEMALLOC_ALWAYS_INLINE uint32_t -prng_range_u32(uint32_t *state, uint32_t range, bool atomic) { +prng_range_u32(atomic_u32_t *state, uint32_t range, bool atomic) { uint32_t ret; unsigned lg_range; @@ -136,7 +137,7 @@ prng_range_u64(uint64_t *state, uint64_t range) { } JEMALLOC_ALWAYS_INLINE size_t -prng_range_zu(size_t *state, size_t range, bool atomic) { +prng_range_zu(atomic_zu_t *state, size_t range, bool atomic) { size_t ret; unsigned lg_range; diff --git a/src/arena.c b/src/arena.c index b8ef4731..53bef36e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1863,8 +1863,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { * cost of test repeatability. For debug builds, instead use a * deterministic seed. */ - arena->offset_state = config_debug ? ind : - (size_t)(uintptr_t)arena; + atomic_store_zu(&arena->offset_state, config_debug ? ind : + (size_t)(uintptr_t)arena, ATOMIC_RELAXED); } arena->extent_sn_next = 0; diff --git a/test/unit/prng.c b/test/unit/prng.c index 74d9cf73..b5795c2f 100644 --- a/test/unit/prng.c +++ b/test/unit/prng.c @@ -2,31 +2,32 @@ static void test_prng_lg_range_u32(bool atomic) { - uint32_t sa, sb, ra, rb; + atomic_u32_t sa, sb; + uint32_t ra, rb; unsigned lg_range; - sa = 42; + atomic_store_u32(&sa, 42, ATOMIC_RELAXED); ra = prng_lg_range_u32(&sa, 32, atomic); - sa = 42; + atomic_store_u32(&sa, 42, ATOMIC_RELAXED); rb = prng_lg_range_u32(&sa, 32, atomic); assert_u32_eq(ra, rb, "Repeated generation should produce repeated results"); - sb = 42; + atomic_store_u32(&sb, 42, ATOMIC_RELAXED); rb = prng_lg_range_u32(&sb, 32, atomic); assert_u32_eq(ra, rb, "Equivalent generation should produce equivalent results"); - sa = 42; + atomic_store_u32(&sa, 42, ATOMIC_RELAXED); ra = prng_lg_range_u32(&sa, 32, atomic); rb = prng_lg_range_u32(&sa, 32, atomic); assert_u32_ne(ra, rb, "Full-width results must not immediately repeat"); - sa = 42; + atomic_store_u32(&sa, 42, ATOMIC_RELAXED); ra = prng_lg_range_u32(&sa, 32, atomic); for (lg_range = 31; lg_range > 0; lg_range--) { - sb = 42; + atomic_store_u32(&sb, 42, ATOMIC_RELAXED); rb = prng_lg_range_u32(&sb, lg_range, atomic); assert_u32_eq((rb & (UINT32_C(0xffffffff) << lg_range)), 0, "High order bits should be 0, lg_range=%u", lg_range); @@ -74,32 +75,33 @@ test_prng_lg_range_u64(void) { static void test_prng_lg_range_zu(bool atomic) { - size_t sa, sb, ra, rb; + atomic_zu_t sa, sb; + size_t ra, rb; unsigned lg_range; - sa = 42; + atomic_store_zu(&sa, 42, ATOMIC_RELAXED); ra = prng_lg_range_zu(&sa, ZU(1) << (3 + LG_SIZEOF_PTR), atomic); - sa = 42; + atomic_store_zu(&sa, 42, ATOMIC_RELAXED); rb = prng_lg_range_zu(&sa, ZU(1) << (3 + LG_SIZEOF_PTR), atomic); assert_zu_eq(ra, rb, "Repeated generation should produce repeated results"); - sb = 42; + atomic_store_zu(&sb, 42, ATOMIC_RELAXED); rb = prng_lg_range_zu(&sb, ZU(1) << (3 + LG_SIZEOF_PTR), atomic); assert_zu_eq(ra, rb, "Equivalent generation should produce equivalent results"); - sa = 42; + atomic_store_zu(&sa, 42, ATOMIC_RELAXED); ra = prng_lg_range_zu(&sa, ZU(1) << (3 + LG_SIZEOF_PTR), atomic); rb = prng_lg_range_zu(&sa, ZU(1) << (3 + LG_SIZEOF_PTR), atomic); assert_zu_ne(ra, rb, "Full-width results must not immediately repeat"); - sa = 42; + atomic_store_zu(&sa, 42, ATOMIC_RELAXED); ra = prng_lg_range_zu(&sa, ZU(1) << (3 + LG_SIZEOF_PTR), atomic); for (lg_range = (ZU(1) << (3 + LG_SIZEOF_PTR)) - 1; lg_range > 0; lg_range--) { - sb = 42; + atomic_store_zu(&sb, 42, ATOMIC_RELAXED); rb = prng_lg_range_zu(&sb, lg_range, atomic); assert_zu_eq((rb & (SIZE_T_MAX << lg_range)), 0, "High order bits should be 0, lg_range=%u", lg_range); @@ -142,10 +144,10 @@ test_prng_range_u32(bool atomic) { #define NREPS 10 for (range = 2; range < MAX_RANGE; range += RANGE_STEP) { - uint32_t s; + atomic_u32_t s; unsigned rep; - s = range; + atomic_store_u32(&s, range, ATOMIC_RELAXED); for (rep = 0; rep < NREPS; rep++) { uint32_t r = prng_range_u32(&s, range, atomic); @@ -182,10 +184,10 @@ test_prng_range_zu(bool atomic) { #define NREPS 10 for (range = 2; range < MAX_RANGE; range += RANGE_STEP) { - size_t s; + atomic_zu_t s; unsigned rep; - s = range; + atomic_store_zu(&s, range, ATOMIC_RELAXED); for (rep = 0; rep < NREPS; rep++) { size_t r = prng_range_zu(&s, range, atomic);