project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rename atomic_*_{uint32,uint64,u}() to atomic_*_{u32,u64,zu}().

Browse Source 
This change conforms to naming conventions throughout the codebase.
This commit is contained in:
Jason Evans 2016-11-07 11:27:48 -08:00
parent 2e46b13ad5
commit cda59f9970
8 changed files with 139 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/jemalloc/internal/arena.h
View File

@ -365,21 +365,21 @@ JEMALLOC_INLINE void
arena_metadata_add(arena_t *arena, size_t size) arena_metadata_add(arena_t *arena, size_t size)
{ {
atomic_add_z(&arena->stats.metadata, size); atomic_add_zu(&arena->stats.metadata, size);
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
arena_metadata_sub(arena_t *arena, size_t size) arena_metadata_sub(arena_t *arena, size_t size)
{ {
atomic_sub_z(&arena->stats.metadata, size); atomic_sub_zu(&arena->stats.metadata, size);
} }
JEMALLOC_INLINE size_t JEMALLOC_INLINE size_t
arena_metadata_get(arena_t *arena) arena_metadata_get(arena_t *arena)
{ {
return (atomic_read_z(&arena->stats.metadata)); return (atomic_read_zu(&arena->stats.metadata));
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool

198
include/jemalloc/internal/atomic.h
View File

@ -9,10 +9,12 @@
/******************************************************************************/ /******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS #ifdef JEMALLOC_H_EXTERNS
#define atomic_read_uint64(p) atomic_add_uint64(p, 0) #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
#define atomic_read_uint32(p) atomic_add_uint32(p, 0) #define atomic_read_u64(p) atomic_add_u64(p, 0)
#endif
#define atomic_read_u32(p) atomic_add_u32(p, 0)
#define atomic_read_p(p) atomic_add_p(p, NULL) #define atomic_read_p(p) atomic_add_p(p, NULL)
#define atomic_read_z(p) atomic_add_z(p, 0) #define atomic_read_zu(p) atomic_add_zu(p, 0)
#define atomic_read_u(p) atomic_add_u(p, 0) #define atomic_read_u(p) atomic_add_u(p, 0)
#endif /* JEMALLOC_H_EXTERNS */ #endif /* JEMALLOC_H_EXTERNS */
@ -41,22 +43,24 @@
*/ */
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); # if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); uint64_t atomic_add_u64(uint64_t *p, uint64_t x);
bool atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s); uint64_t atomic_sub_u64(uint64_t *p, uint64_t x);
void atomic_write_uint64(uint64_t *p, uint64_t x); bool atomic_cas_u64(uint64_t *p, uint64_t c, uint64_t s);
uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); void atomic_write_u64(uint64_t *p, uint64_t x);
uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); # endif
bool atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s); uint32_t atomic_add_u32(uint32_t *p, uint32_t x);
void atomic_write_uint32(uint32_t *p, uint32_t x); uint32_t atomic_sub_u32(uint32_t *p, uint32_t x);
bool atomic_cas_u32(uint32_t *p, uint32_t c, uint32_t s);
void atomic_write_u32(uint32_t *p, uint32_t x);
void *atomic_add_p(void **p, void *x); void *atomic_add_p(void **p, void *x);
void *atomic_sub_p(void **p, void *x); void *atomic_sub_p(void **p, void *x);
bool atomic_cas_p(void **p, void *c, void *s); bool atomic_cas_p(void **p, void *c, void *s);
void atomic_write_p(void **p, const void *x); void atomic_write_p(void **p, const void *x);
size_t atomic_add_z(size_t *p, size_t x); size_t atomic_add_zu(size_t *p, size_t x);
size_t atomic_sub_z(size_t *p, size_t x); size_t atomic_sub_zu(size_t *p, size_t x);
bool atomic_cas_z(size_t *p, size_t c, size_t s); bool atomic_cas_zu(size_t *p, size_t c, size_t s);
void atomic_write_z(size_t *p, size_t x); void atomic_write_zu(size_t *p, size_t x);
unsigned atomic_add_u(unsigned *p, unsigned x); unsigned atomic_add_u(unsigned *p, unsigned x);
unsigned atomic_sub_u(unsigned *p, unsigned x); unsigned atomic_sub_u(unsigned *p, unsigned x);
bool atomic_cas_u(unsigned *p, unsigned c, unsigned s); bool atomic_cas_u(unsigned *p, unsigned c, unsigned s);
@ -69,7 +73,7 @@ void atomic_write_u(unsigned *p, unsigned x);
#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
# if (defined(__amd64__) || defined(__x86_64__)) # if (defined(__amd64__) || defined(__x86_64__))
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x) atomic_add_u64(uint64_t *p, uint64_t x)
{ {
uint64_t t = x; uint64_t t = x;
@ -83,7 +87,7 @@ atomic_add_uint64(uint64_t *p, uint64_t x)
} }
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x) atomic_sub_u64(uint64_t *p, uint64_t x)
{ {
uint64_t t; uint64_t t;
@ -99,7 +103,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) atomic_cas_u64(uint64_t *p, uint64_t c, uint64_t s)
{ {
uint8_t success; uint8_t success;
@ -115,7 +119,7 @@ atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x) atomic_write_u64(uint64_t *p, uint64_t x)
{ {
asm volatile ( asm volatile (
@ -127,35 +131,35 @@ atomic_write_uint64(uint64_t *p, uint64_t x)
} }
# elif (defined(JEMALLOC_C11ATOMICS)) # elif (defined(JEMALLOC_C11ATOMICS))
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x) atomic_add_u64(uint64_t *p, uint64_t x)
{ {
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
return (atomic_fetch_add(a, x) + x); return (atomic_fetch_add(a, x) + x);
} }
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x) atomic_sub_u64(uint64_t *p, uint64_t x)
{ {
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
return (atomic_fetch_sub(a, x) - x); return (atomic_fetch_sub(a, x) - x);
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) atomic_cas_u64(uint64_t *p, uint64_t c, uint64_t s)
{ {
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
return (!atomic_compare_exchange_strong(a, &c, s)); return (!atomic_compare_exchange_strong(a, &c, s));
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x) atomic_write_u64(uint64_t *p, uint64_t x)
{ {
volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p;
atomic_store(a, x); atomic_store(a, x);
} }
# elif (defined(JEMALLOC_ATOMIC9)) # elif (defined(JEMALLOC_ATOMIC9))
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x) atomic_add_u64(uint64_t *p, uint64_t x)
{ {
/* /*
@ -168,7 +172,7 @@ atomic_add_uint64(uint64_t *p, uint64_t x)
} }
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x) atomic_sub_u64(uint64_t *p, uint64_t x)
{ {
assert(sizeof(uint64_t) == sizeof(unsigned long)); assert(sizeof(uint64_t) == sizeof(unsigned long));
@ -177,7 +181,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) atomic_cas_u64(uint64_t *p, uint64_t c, uint64_t s)
{ {
assert(sizeof(uint64_t) == sizeof(unsigned long)); assert(sizeof(uint64_t) == sizeof(unsigned long));
@ -186,7 +190,7 @@ atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x) atomic_write_u64(uint64_t *p, uint64_t x)
{ {
assert(sizeof(uint64_t) == sizeof(unsigned long)); assert(sizeof(uint64_t) == sizeof(unsigned long));
@ -195,53 +199,53 @@ atomic_write_uint64(uint64_t *p, uint64_t x)
} }
# elif (defined(JEMALLOC_OSATOMIC)) # elif (defined(JEMALLOC_OSATOMIC))
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x) atomic_add_u64(uint64_t *p, uint64_t x)
{ {
return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); return (OSAtomicAdd64((int64_t)x, (int64_t *)p));
} }
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x) atomic_sub_u64(uint64_t *p, uint64_t x)
{ {
return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) atomic_cas_u64(uint64_t *p, uint64_t c, uint64_t s)
{ {
return (!OSAtomicCompareAndSwap64(c, s, (int64_t *)p)); return (!OSAtomicCompareAndSwap64(c, s, (int64_t *)p));
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x) atomic_write_u64(uint64_t *p, uint64_t x)
{ {
uint64_t o; uint64_t o;
/*The documented OSAtomic*() API does not expose an atomic exchange. */ /*The documented OSAtomic*() API does not expose an atomic exchange. */
do { do {
o = atomic_read_uint64(p); o = atomic_read_u64(p);
} while (atomic_cas_uint64(p, o, x)); } while (atomic_cas_u64(p, o, x));
} }
# elif (defined(_MSC_VER)) # elif (defined(_MSC_VER))
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x) atomic_add_u64(uint64_t *p, uint64_t x)
{ {
return (InterlockedExchangeAdd64(p, x) + x); return (InterlockedExchangeAdd64(p, x) + x);
} }
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x) atomic_sub_u64(uint64_t *p, uint64_t x)
{ {
return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x); return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x);
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) atomic_cas_u64(uint64_t *p, uint64_t c, uint64_t s)
{ {
uint64_t o; uint64_t o;
@ -250,7 +254,7 @@ atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s)
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x) atomic_write_u64(uint64_t *p, uint64_t x)
{ {
InterlockedExchange64(p, x); InterlockedExchange64(p, x);
@ -258,28 +262,28 @@ atomic_write_uint64(uint64_t *p, uint64_t x)
# elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || \ # elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || \
defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x) atomic_add_u64(uint64_t *p, uint64_t x)
{ {
return (__sync_add_and_fetch(p, x)); return (__sync_add_and_fetch(p, x));
} }
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x) atomic_sub_u64(uint64_t *p, uint64_t x)
{ {
return (__sync_sub_and_fetch(p, x)); return (__sync_sub_and_fetch(p, x));
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) atomic_cas_u64(uint64_t *p, uint64_t c, uint64_t s)
{ {
return (!__sync_bool_compare_and_swap(p, c, s)); return (!__sync_bool_compare_and_swap(p, c, s));
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint64(uint64_t *p, uint64_t x) atomic_write_u64(uint64_t *p, uint64_t x)
{ {
__sync_lock_test_and_set(p, x); __sync_lock_test_and_set(p, x);
@ -293,7 +297,7 @@ atomic_write_uint64(uint64_t *p, uint64_t x)
/* 32-bit operations. */ /* 32-bit operations. */
#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x) atomic_add_u32(uint32_t *p, uint32_t x)
{ {
uint32_t t = x; uint32_t t = x;
@ -307,7 +311,7 @@ atomic_add_uint32(uint32_t *p, uint32_t x)
} }
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x) atomic_sub_u32(uint32_t *p, uint32_t x)
{ {
uint32_t t; uint32_t t;
@ -323,7 +327,7 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) atomic_cas_u32(uint32_t *p, uint32_t c, uint32_t s)
{ {
uint8_t success; uint8_t success;
@ -339,7 +343,7 @@ atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x) atomic_write_u32(uint32_t *p, uint32_t x)
{ {
asm volatile ( asm volatile (
@ -351,109 +355,109 @@ atomic_write_uint32(uint32_t *p, uint32_t x)
} }
# elif (defined(JEMALLOC_C11ATOMICS)) # elif (defined(JEMALLOC_C11ATOMICS))
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x) atomic_add_u32(uint32_t *p, uint32_t x)
{ {
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
return (atomic_fetch_add(a, x) + x); return (atomic_fetch_add(a, x) + x);
} }
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x) atomic_sub_u32(uint32_t *p, uint32_t x)
{ {
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
return (atomic_fetch_sub(a, x) - x); return (atomic_fetch_sub(a, x) - x);
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) atomic_cas_u32(uint32_t *p, uint32_t c, uint32_t s)
{ {
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
return (!atomic_compare_exchange_strong(a, &c, s)); return (!atomic_compare_exchange_strong(a, &c, s));
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x) atomic_write_u32(uint32_t *p, uint32_t x)
{ {
volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p;
atomic_store(a, x); atomic_store(a, x);
} }
#elif (defined(JEMALLOC_ATOMIC9)) #elif (defined(JEMALLOC_ATOMIC9))
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x) atomic_add_u32(uint32_t *p, uint32_t x)
{ {
return (atomic_fetchadd_32(p, x) + x); return (atomic_fetchadd_32(p, x) + x);
} }
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x) atomic_sub_u32(uint32_t *p, uint32_t x)
{ {
return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) atomic_cas_u32(uint32_t *p, uint32_t c, uint32_t s)
{ {
return (!atomic_cmpset_32(p, c, s)); return (!atomic_cmpset_32(p, c, s));
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x) atomic_write_u32(uint32_t *p, uint32_t x)
{ {
atomic_store_rel_32(p, x); atomic_store_rel_32(p, x);
} }
#elif (defined(JEMALLOC_OSATOMIC)) #elif (defined(JEMALLOC_OSATOMIC))
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x) atomic_add_u32(uint32_t *p, uint32_t x)
{ {
return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); return (OSAtomicAdd32((int32_t)x, (int32_t *)p));
} }
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x) atomic_sub_u32(uint32_t *p, uint32_t x)
{ {
return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p));
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) atomic_cas_u32(uint32_t *p, uint32_t c, uint32_t s)
{ {
return (!OSAtomicCompareAndSwap32(c, s, (int32_t *)p)); return (!OSAtomicCompareAndSwap32(c, s, (int32_t *)p));
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x) atomic_write_u32(uint32_t *p, uint32_t x)
{ {
uint32_t o; uint32_t o;
/*The documented OSAtomic*() API does not expose an atomic exchange. */ /*The documented OSAtomic*() API does not expose an atomic exchange. */
do { do {
o = atomic_read_uint32(p); o = atomic_read_u32(p);
} while (atomic_cas_uint32(p, o, x)); } while (atomic_cas_u32(p, o, x));
} }
#elif (defined(_MSC_VER)) #elif (defined(_MSC_VER))
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x) atomic_add_u32(uint32_t *p, uint32_t x)
{ {
return (InterlockedExchangeAdd(p, x) + x); return (InterlockedExchangeAdd(p, x) + x);
} }
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x) atomic_sub_u32(uint32_t *p, uint32_t x)
{ {
return (InterlockedExchangeAdd(p, -((int32_t)x)) - x); return (InterlockedExchangeAdd(p, -((int32_t)x)) - x);
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) atomic_cas_u32(uint32_t *p, uint32_t c, uint32_t s)
{ {
uint32_t o; uint32_t o;
@ -462,7 +466,7 @@ atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s)
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x) atomic_write_u32(uint32_t *p, uint32_t x)
{ {
InterlockedExchange(p, x); InterlockedExchange(p, x);
@ -470,28 +474,28 @@ atomic_write_uint32(uint32_t *p, uint32_t x)
#elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || \ #elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || \
defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x) atomic_add_u32(uint32_t *p, uint32_t x)
{ {
return (__sync_add_and_fetch(p, x)); return (__sync_add_and_fetch(p, x));
} }
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x) atomic_sub_u32(uint32_t *p, uint32_t x)
{ {
return (__sync_sub_and_fetch(p, x)); return (__sync_sub_and_fetch(p, x));
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) atomic_cas_u32(uint32_t *p, uint32_t c, uint32_t s)
{ {
return (!__sync_bool_compare_and_swap(p, c, s)); return (!__sync_bool_compare_and_swap(p, c, s));
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_uint32(uint32_t *p, uint32_t x) atomic_write_u32(uint32_t *p, uint32_t x)
{ {
__sync_lock_test_and_set(p, x); __sync_lock_test_and_set(p, x);
@ -507,9 +511,9 @@ atomic_add_p(void **p, void *x)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
return ((void *)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); return ((void *)atomic_add_u64((uint64_t *)p, (uint64_t)x));
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
return ((void *)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); return ((void *)atomic_add_u32((uint32_t *)p, (uint32_t)x));
#endif #endif
} }
@ -518,11 +522,9 @@ atomic_sub_p(void **p, void *x)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
return ((void *)atomic_add_uint64((uint64_t *)p, return ((void *)atomic_add_u64((uint64_t *)p, (uint64_t)-((int64_t)x)));
(uint64_t)-((int64_t)x)));
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
return ((void *)atomic_add_uint32((uint32_t *)p, return ((void *)atomic_add_u32((uint32_t *)p, (uint32_t)-((int32_t)x)));
(uint32_t)-((int32_t)x)));
#endif #endif
} }
@ -531,9 +533,9 @@ atomic_cas_p(void **p, void *c, void *s)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); return (atomic_cas_u64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); return (atomic_cas_u32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
#endif #endif
} }
@ -542,57 +544,55 @@ atomic_write_p(void **p, const void *x)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
atomic_write_uint64((uint64_t *)p, (uint64_t)x); atomic_write_u64((uint64_t *)p, (uint64_t)x);
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
atomic_write_uint32((uint32_t *)p, (uint32_t)x); atomic_write_u32((uint32_t *)p, (uint32_t)x);
#endif #endif
} }
/******************************************************************************/ /******************************************************************************/
/* size_t operations. */ /* size_t operations. */
JEMALLOC_INLINE size_t JEMALLOC_INLINE size_t
atomic_add_z(size_t *p, size_t x) atomic_add_zu(size_t *p, size_t x)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); return ((size_t)atomic_add_u64((uint64_t *)p, (uint64_t)x));
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); return ((size_t)atomic_add_u32((uint32_t *)p, (uint32_t)x));
#endif #endif
} }
JEMALLOC_INLINE size_t JEMALLOC_INLINE size_t
atomic_sub_z(size_t *p, size_t x) atomic_sub_zu(size_t *p, size_t x)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
return ((size_t)atomic_add_uint64((uint64_t *)p, return ((size_t)atomic_add_u64((uint64_t *)p, (uint64_t)-((int64_t)x)));
(uint64_t)-((int64_t)x)));
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
return ((size_t)atomic_add_uint32((uint32_t *)p, return ((size_t)atomic_add_u32((uint32_t *)p, (uint32_t)-((int32_t)x)));
(uint32_t)-((int32_t)x)));
#endif #endif
} }
JEMALLOC_INLINE bool JEMALLOC_INLINE bool
atomic_cas_z(size_t *p, size_t c, size_t s) atomic_cas_zu(size_t *p, size_t c, size_t s)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); return (atomic_cas_u64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); return (atomic_cas_u32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
#endif #endif
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
atomic_write_z(size_t *p, size_t x) atomic_write_zu(size_t *p, size_t x)
{ {
#if (LG_SIZEOF_PTR == 3) #if (LG_SIZEOF_PTR == 3)
atomic_write_uint64((uint64_t *)p, (uint64_t)x); atomic_write_u64((uint64_t *)p, (uint64_t)x);
#elif (LG_SIZEOF_PTR == 2) #elif (LG_SIZEOF_PTR == 2)
atomic_write_uint32((uint32_t *)p, (uint32_t)x); atomic_write_u32((uint32_t *)p, (uint32_t)x);
#endif #endif
} }
@ -603,9 +603,9 @@ atomic_add_u(unsigned *p, unsigned x)
{ {
#if (LG_SIZEOF_INT == 3) #if (LG_SIZEOF_INT == 3)
return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); return ((unsigned)atomic_add_u64((uint64_t *)p, (uint64_t)x));
#elif (LG_SIZEOF_INT == 2) #elif (LG_SIZEOF_INT == 2)
return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); return ((unsigned)atomic_add_u32((uint32_t *)p, (uint32_t)x));
#endif #endif
} }
@ -614,10 +614,10 @@ atomic_sub_u(unsigned *p, unsigned x)
{ {
#if (LG_SIZEOF_INT == 3) #if (LG_SIZEOF_INT == 3)
return ((unsigned)atomic_add_uint64((uint64_t *)p, return ((unsigned)atomic_add_u64((uint64_t *)p,
(uint64_t)-((int64_t)x))); (uint64_t)-((int64_t)x)));
#elif (LG_SIZEOF_INT == 2) #elif (LG_SIZEOF_INT == 2)
return ((unsigned)atomic_add_uint32((uint32_t *)p, return ((unsigned)atomic_add_u32((uint32_t *)p,
(uint32_t)-((int32_t)x))); (uint32_t)-((int32_t)x)));
#endif #endif
} }
@ -627,9 +627,9 @@ atomic_cas_u(unsigned *p, unsigned c, unsigned s)
{ {
#if (LG_SIZEOF_INT == 3) #if (LG_SIZEOF_INT == 3)
return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); return (atomic_cas_u64((uint64_t *)p, (uint64_t)c, (uint64_t)s));
#elif (LG_SIZEOF_INT == 2) #elif (LG_SIZEOF_INT == 2)
return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); return (atomic_cas_u32((uint32_t *)p, (uint32_t)c, (uint32_t)s));
#endif #endif
} }
@ -638,9 +638,9 @@ atomic_write_u(unsigned *p, unsigned x)
{ {
#if (LG_SIZEOF_INT == 3) #if (LG_SIZEOF_INT == 3)
atomic_write_uint64((uint64_t *)p, (uint64_t)x); atomic_write_u64((uint64_t *)p, (uint64_t)x);
#elif (LG_SIZEOF_INT == 2) #elif (LG_SIZEOF_INT == 2)
atomic_write_uint32((uint32_t *)p, (uint32_t)x); atomic_write_u32((uint32_t *)p, (uint32_t)x);
#endif #endif
} }

24
include/jemalloc/internal/private_symbols.txt
View File

@ -73,24 +73,24 @@ arenas
arenas_tdata_cleanup arenas_tdata_cleanup
atomic_add_p atomic_add_p
atomic_add_u atomic_add_u
atomic_add_uint32 atomic_add_u32
atomic_add_uint64 atomic_add_u64
atomic_add_z atomic_add_zu
atomic_cas_p atomic_cas_p
atomic_cas_u atomic_cas_u
atomic_cas_uint32 atomic_cas_u32
atomic_cas_uint64 atomic_cas_u64
atomic_cas_z atomic_cas_zu
atomic_sub_p atomic_sub_p
atomic_sub_u atomic_sub_u
atomic_sub_uint32 atomic_sub_u32
atomic_sub_uint64 atomic_sub_u64
atomic_sub_z atomic_sub_zu
atomic_write_p atomic_write_p
atomic_write_u atomic_write_u
atomic_write_uint32 atomic_write_u32
atomic_write_uint64 atomic_write_u64
atomic_write_z atomic_write_zu
base_alloc base_alloc
base_boot base_boot
base_postfork_child base_postfork_child

8
include/jemalloc/internal/prng.h
View File

@ -93,9 +93,9 @@ prng_lg_range_u32(uint32_t *state, unsigned lg_range, bool atomic)
uint32_t state0; uint32_t state0;
do { do {
state0 = atomic_read_uint32(state); state0 = atomic_read_u32(state);
state1 = prng_state_next_u32(state0); state1 = prng_state_next_u32(state0);
} while (atomic_cas_uint32(state, state0, state1)); } while (atomic_cas_u32(state, state0, state1));
} else { } else {
state1 = prng_state_next_u32(*state); state1 = prng_state_next_u32(*state);
*state = state1; *state = state1;
@ -133,9 +133,9 @@ prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic)
size_t state0; size_t state0;
do { do {
state0 = atomic_read_z(state); state0 = atomic_read_zu(state);
state1 = prng_state_next_zu(state0); state1 = prng_state_next_zu(state0);
} while (atomic_cas_z(state, state0, state1)); } while (atomic_cas_zu(state, state0, state1));
} else { } else {
state1 = prng_state_next_zu(*state); state1 = prng_state_next_zu(*state);
*state = state1; *state = state1;

2
include/jemalloc/internal/stats.h
View File

@ -101,7 +101,7 @@ struct arena_stats_s {
uint64_t purged; uint64_t purged;
/* Number of bytes currently allocated for internal metadata. */ /* Number of bytes currently allocated for internal metadata. */
size_t metadata; /* Protected via atomic_*_z(). */ size_t metadata; /* Protected via atomic_*_zu(). */
size_t allocated_large; size_t allocated_large;
uint64_t nmalloc_large; uint64_t nmalloc_large;

4
src/arena.c
View File

@ -1541,7 +1541,7 @@ ssize_t
arena_decay_time_default_get(void) arena_decay_time_default_get(void)
{ {
return ((ssize_t)atomic_read_z((size_t *)&decay_time_default)); return ((ssize_t)atomic_read_zu((size_t *)&decay_time_default));
} }
bool bool
@ -1550,7 +1550,7 @@ arena_decay_time_default_set(ssize_t decay_time)
if (!arena_decay_time_valid(decay_time)) if (!arena_decay_time_valid(decay_time))
return (true); return (true);
atomic_write_z((size_t *)&decay_time_default, (size_t)decay_time); atomic_write_zu((size_t *)&decay_time_default, (size_t)decay_time);
return (false); return (false);
} }

12
src/extent.c
View File

@ -291,14 +291,14 @@ extent_register(tsdn_t *tsdn, const extent_t *extent)
if (config_prof && opt_prof && extent_active_get(extent)) { if (config_prof && opt_prof && extent_active_get(extent)) {
size_t nadd = extent_size_get(extent) >> LG_PAGE; size_t nadd = extent_size_get(extent) >> LG_PAGE;
size_t cur = atomic_add_z(&curpages, nadd); size_t cur = atomic_add_zu(&curpages, nadd);
size_t high = atomic_read_z(&highpages); size_t high = atomic_read_zu(&highpages);
while (cur > high && atomic_cas_z(&highpages, high, cur)) { while (cur > high && atomic_cas_zu(&highpages, high, cur)) {
/* /*
* Don't refresh cur, because it may have decreased * Don't refresh cur, because it may have decreased
* since this thread lost the highpages update race. * since this thread lost the highpages update race.
*/ */
high = atomic_read_z(&highpages); high = atomic_read_zu(&highpages);
} }
if (cur > high && prof_gdump_get_unlocked()) if (cur > high && prof_gdump_get_unlocked())
prof_gdump(tsdn); prof_gdump(tsdn);
@ -347,8 +347,8 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent)
if (config_prof && opt_prof && extent_active_get(extent)) { if (config_prof && opt_prof && extent_active_get(extent)) {
size_t nsub = extent_size_get(extent) >> LG_PAGE; size_t nsub = extent_size_get(extent) >> LG_PAGE;
assert(atomic_read_z(&curpages) >= nsub); assert(atomic_read_zu(&curpages) >= nsub);
atomic_sub_z(&curpages, nsub); atomic_sub_zu(&curpages, nsub);
} }
} }

24
test/unit/atomic.c
View File

@ -65,23 +65,23 @@ typedef struct p##_test_s p##_test_t;
} \ } \
} while (0) } while (0)
TEST_STRUCT(uint64, uint64_t) TEST_STRUCT(u64, uint64_t)
TEST_BEGIN(test_atomic_uint64) TEST_BEGIN(test_atomic_u64)
{ {
#if !(LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) #if !(LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
test_skip("64-bit atomic operations not supported"); test_skip("64-bit atomic operations not supported");
#else #else
TEST_BODY(uint64, uint64_t, uint64_t, u64, FMTx64); TEST_BODY(u64, uint64_t, uint64_t, u64, FMTx64);
#endif #endif
} }
TEST_END TEST_END
TEST_STRUCT(uint32, uint32_t) TEST_STRUCT(u32, uint32_t)
TEST_BEGIN(test_atomic_uint32) TEST_BEGIN(test_atomic_u32)
{ {
TEST_BODY(uint32, uint32_t, uint32_t, u32, "#"FMTx32); TEST_BODY(u32, uint32_t, uint32_t, u32, "#"FMTx32);
} }
TEST_END TEST_END
@ -93,11 +93,11 @@ TEST_BEGIN(test_atomic_p)
} }
TEST_END TEST_END
TEST_STRUCT(z, size_t) TEST_STRUCT(zu, size_t)
TEST_BEGIN(test_atomic_z) TEST_BEGIN(test_atomic_zu)
{ {
TEST_BODY(z, size_t, size_t, zu, "#zx"); TEST_BODY(zu, size_t, size_t, zu, "#zx");
} }
TEST_END TEST_END
@ -114,9 +114,9 @@ main(void)
{ {
return (test( return (test(
test_atomic_uint64, test_atomic_u64,
test_atomic_uint32, test_atomic_u32,
test_atomic_p, test_atomic_p,
test_atomic_z, test_atomic_zu,
test_atomic_u)); test_atomic_u));
} }