Introduce a backport of C11 atomics
This introduces a backport of C11 atomics. It has four implementations; ranked in order of preference, they are: - GCC/Clang __atomic builtins - GCC/Clang __sync builtins - MSVC _Interlocked builtins - C11 atomics, from <stdatomic.h> The primary advantages are: - Close adherence to the standard API gives us a defined memory model. - Type safety: atomic objects are now separate types from non-atomic ones, so that it's impossible to mix up atomic and non-atomic updates (which is undefined behavior that compilers are starting to take advantage of). - Efficiency: we can specify ordering for operations, avoiding fences and atomic operations on strongly ordered architectures (example: `atomic_write_u32(ptr, val);` involves a CAS loop, whereas `atomic_store(ptr, val, ATOMIC_RELEASE);` is a plain store. This diff leaves in the current atomics API (implementing them in terms of the backport). This lets us transition uses over piecemeal. Testing: This is by nature hard to test. I've manually tested the first three options on Linux on gcc by futzing with the #defines manually, on freebsd with gcc and clang, on MSVC, and on OS X with clang. All of these were x86 machines though, and we don't have any test infrastructure set up for non-x86 platforms.
This commit is contained in:
David Goldblatt
committed by
David Goldblatt
David Goldblatt
parent
957b8c5f21
commit
d4ac7582f3
158
include/jemalloc/internal/atomic_msvc.h
Normal file
158
include/jemalloc/internal/atomic_msvc.h
Normal file
@@ -0,0 +1,158 @@
|
||||
#ifndef JEMALLOC_INTERNAL_ATOMIC_MSVC_H
|
||||
#define JEMALLOC_INTERNAL_ATOMIC_MSVC_H
|
||||
|
||||
#define ATOMIC_INIT(...) {__VA_ARGS__}
|
||||
|
||||
typedef enum {
|
||||
atomic_memory_order_relaxed,
|
||||
atomic_memory_order_acquire,
|
||||
atomic_memory_order_release,
|
||||
atomic_memory_order_acq_rel,
|
||||
atomic_memory_order_seq_cst
|
||||
} atomic_memory_order_t;
|
||||
|
||||
typedef char atomic_repr_0_t;
|
||||
typedef short atomic_repr_1_t;
|
||||
typedef long atomic_repr_2_t;
|
||||
typedef __int64 atomic_repr_3_t;
|
||||
|
||||
ATOMIC_INLINE void
|
||||
atomic_fence(atomic_memory_order_t mo) {
|
||||
_ReadWriteBarrier();
|
||||
# if defined(_M_ARM) || defined(_M_ARM64)
|
||||
/* ARM needs a barrier for everything but relaxed. */
|
||||
if (mo != atomic_memory_order_relaxed) {
|
||||
MemoryBarrier();
|
||||
}
|
||||
# elif defined(_M_IX86) || defined (_M_X64)
|
||||
/* x86 needs a barrier only for seq_cst. */
|
||||
if (mo == atomic_memory_order_seq_cst) {
|
||||
MemoryBarrier();
|
||||
}
|
||||
# else
|
||||
# error "Don't know how to create atomics for this platform for MSVC."
|
||||
# endif
|
||||
_ReadWriteBarrier();
|
||||
}
|
||||
|
||||
#define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_ ## lg_size ## _t
|
||||
|
||||
#define ATOMIC_CONCAT(a, b) ATOMIC_RAW_CONCAT(a, b)
|
||||
#define ATOMIC_RAW_CONCAT(a, b) a ## b
|
||||
|
||||
#define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) ATOMIC_CONCAT( \
|
||||
base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size))
|
||||
|
||||
#define ATOMIC_INTERLOCKED_SUFFIX(lg_size) \
|
||||
ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size)
|
||||
|
||||
#define ATOMIC_INTERLOCKED_SUFFIX_0 8
|
||||
#define ATOMIC_INTERLOCKED_SUFFIX_1 16
|
||||
#define ATOMIC_INTERLOCKED_SUFFIX_2
|
||||
#define ATOMIC_INTERLOCKED_SUFFIX_3 64
|
||||
|
||||
#define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \
|
||||
typedef struct { \
|
||||
ATOMIC_INTERLOCKED_REPR(lg_size) repr; \
|
||||
} atomic_##short_type##_t; \
|
||||
\
|
||||
ATOMIC_INLINE type \
|
||||
atomic_load_##short_type(const atomic_##short_type##_t *a, \
|
||||
atomic_memory_order_t mo) { \
|
||||
ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr; \
|
||||
if (mo != atomic_memory_order_relaxed) { \
|
||||
atomic_fence(atomic_memory_order_acquire); \
|
||||
} \
|
||||
return (type) ret; \
|
||||
} \
|
||||
\
|
||||
ATOMIC_INLINE void \
|
||||
atomic_store_##short_type(atomic_##short_type##_t *a, \
|
||||
type val, atomic_memory_order_t mo) { \
|
||||
if (mo != atomic_memory_order_relaxed) { \
|
||||
atomic_fence(atomic_memory_order_release); \
|
||||
} \
|
||||
a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size)) val; \
|
||||
if (mo == atomic_memory_order_seq_cst) { \
|
||||
atomic_fence(atomic_memory_order_seq_cst); \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
ATOMIC_INLINE type \
|
||||
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \
|
||||
atomic_memory_order_t mo) { \
|
||||
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange, \
|
||||
lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
|
||||
} \
|
||||
\
|
||||
ATOMIC_INLINE bool \
|
||||
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \
|
||||
type *expected, type desired, atomic_memory_order_t success_mo, \
|
||||
atomic_memory_order_t failure_mo) { \
|
||||
ATOMIC_INTERLOCKED_REPR(lg_size) e = \
|
||||
(ATOMIC_INTERLOCKED_REPR(lg_size))*expected; \
|
||||
ATOMIC_INTERLOCKED_REPR(lg_size) d = \
|
||||
(ATOMIC_INTERLOCKED_REPR(lg_size))desired; \
|
||||
ATOMIC_INTERLOCKED_REPR(lg_size) old = \
|
||||
ATOMIC_INTERLOCKED_NAME(_InterlockedCompareExchange, \
|
||||
lg_size)(&a->repr, d, e); \
|
||||
if (old == e) { \
|
||||
return true; \
|
||||
} else { \
|
||||
*expected = (type)old; \
|
||||
return false; \
|
||||
} \
|
||||
} \
|
||||
\
|
||||
ATOMIC_INLINE bool \
|
||||
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
|
||||
type *expected, type desired, atomic_memory_order_t success_mo, \
|
||||
atomic_memory_order_t failure_mo) { \
|
||||
/* We implement the weak version with strong semantics. */ \
|
||||
return atomic_compare_exchange_weak_##short_type(a, expected, \
|
||||
desired, success_mo, failure_mo); \
|
||||
}
|
||||
|
||||
|
||||
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \
|
||||
JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \
|
||||
\
|
||||
ATOMIC_INLINE type \
|
||||
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, \
|
||||
type val, atomic_memory_order_t mo) { \
|
||||
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchangeAdd, \
|
||||
lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
|
||||
} \
|
||||
\
|
||||
ATOMIC_INLINE type \
|
||||
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, \
|
||||
type val, atomic_memory_order_t mo) { \
|
||||
/* \
|
||||
* MSVC warns on negation of unsigned operands, but for us it \
|
||||
* gives exactly the right semantics (MAX_TYPE + 1 - operand). \
|
||||
*/ \
|
||||
__pragma(warning(push)) \
|
||||
__pragma(warning(disable: 4146)) \
|
||||
return atomic_fetch_add_##short_type(a, -val, mo); \
|
||||
__pragma(warning(pop)) \
|
||||
} \
|
||||
ATOMIC_INLINE type \
|
||||
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, \
|
||||
type val, atomic_memory_order_t mo) { \
|
||||
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedAnd, lg_size)( \
|
||||
&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
|
||||
} \
|
||||
ATOMIC_INLINE type \
|
||||
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, \
|
||||
type val, atomic_memory_order_t mo) { \
|
||||
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedOr, lg_size)( \
|
||||
&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
|
||||
} \
|
||||
ATOMIC_INLINE type \
|
||||
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, \
|
||||
type val, atomic_memory_order_t mo) { \
|
||||
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedXor, lg_size)( \
|
||||
&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_ATOMIC_MSVC_H */
|
||||
Reference in New Issue
Block a user