4f50f782fa
There are several benefits to this: 1. It's cleaner and more reliable to use the builtin to inform the compiler of assumptions instead of hoping that the optimizer understands your intentions. 2. `clang` will warn you if any of your assumptions would produce side-effects (which the compiler will discard). [This blog post](https://fastcompression.blogspot.com/2019/01/compiler-checked-contracts.html) by Yann Collet highlights that a hazard of using the `unreachable()`-based method of signaling assumptions is that it can sometimes result in additional instructions being generated (see [this Godbolt link](https://godbolt.org/z/lKNMs3) from the blog post for an example).
134 lines
3.0 KiB
C
134 lines
3.0 KiB
C
#ifndef JEMALLOC_INTERNAL_UTIL_H
|
|
#define JEMALLOC_INTERNAL_UTIL_H
|
|
|
|
#include "jemalloc/internal/jemalloc_preamble.h"
|
|
#include "jemalloc/internal/jemalloc_internal_types.h"
|
|
|
|
#define UTIL_INLINE static inline
|
|
|
|
/* Junk fill patterns. */
|
|
#ifndef JEMALLOC_ALLOC_JUNK
|
|
# define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)
|
|
#endif
|
|
#ifndef JEMALLOC_FREE_JUNK
|
|
# define JEMALLOC_FREE_JUNK ((uint8_t)0x5a)
|
|
#endif
|
|
|
|
/*
|
|
* Wrap a cpp argument that contains commas such that it isn't broken up into
|
|
* multiple arguments.
|
|
*/
|
|
#define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__
|
|
|
|
/* cpp macro definition stringification. */
|
|
#define STRINGIFY_HELPER(x) #x
|
|
#define STRINGIFY(x) STRINGIFY_HELPER(x)
|
|
|
|
/*
|
|
* Silence compiler warnings due to uninitialized values. This is used
|
|
* wherever the compiler fails to recognize that the variable is never used
|
|
* uninitialized.
|
|
*/
|
|
#define JEMALLOC_CC_SILENCE_INIT(v) = v
|
|
|
|
#ifdef __GNUC__
|
|
# define likely(x) __builtin_expect(!!(x), 1)
|
|
# define unlikely(x) __builtin_expect(!!(x), 0)
|
|
#else
|
|
# define likely(x) !!(x)
|
|
# define unlikely(x) !!(x)
|
|
#endif
|
|
|
|
#if !defined(JEMALLOC_INTERNAL_UNREACHABLE)
|
|
# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure
|
|
#endif
|
|
|
|
#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE()
|
|
|
|
/* Set error code. */
|
|
UTIL_INLINE void
|
|
set_errno(int errnum) {
|
|
#ifdef _WIN32
|
|
SetLastError(errnum);
|
|
#else
|
|
errno = errnum;
|
|
#endif
|
|
}
|
|
|
|
/* Get last error code. */
|
|
UTIL_INLINE int
|
|
get_errno(void) {
|
|
#ifdef _WIN32
|
|
return GetLastError();
|
|
#else
|
|
return errno;
|
|
#endif
|
|
}
|
|
|
|
#ifdef _MSC_VER
|
|
#define util_assume __assume
|
|
#elif defined(__clang__) && (__clang_major__ > 3 || \
|
|
(__clang_major__ == 3 && __clang_minor__ >= 6))
|
|
#define util_assume __builtin_assume
|
|
#else
|
|
#define util_assume(expr) \
|
|
do { \
|
|
if (!(expr)) { \
|
|
unreachable(); \
|
|
} \
|
|
} while(0)
|
|
#endif
|
|
|
|
/* ptr should be valid. */
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
util_prefetch_read(void *ptr) {
|
|
/*
|
|
* This should arguably be a config check; but any version of GCC so old
|
|
* that it doesn't support __builtin_prefetch is also too old to build
|
|
* jemalloc.
|
|
*/
|
|
#ifdef __GNUC__
|
|
if (config_debug) {
|
|
/* Enforce the "valid ptr" requirement. */
|
|
*(volatile char *)ptr;
|
|
}
|
|
__builtin_prefetch(ptr, /* read or write */ 0, /* locality hint */ 3);
|
|
#else
|
|
*(volatile char *)ptr;
|
|
#endif
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
util_prefetch_write(void *ptr) {
|
|
#ifdef __GNUC__
|
|
if (config_debug) {
|
|
*(volatile char *)ptr;
|
|
}
|
|
/*
|
|
* The only difference from the read variant is that this has a 1 as the
|
|
* second argument (the write hint).
|
|
*/
|
|
__builtin_prefetch(ptr, 1, 3);
|
|
#else
|
|
*(volatile char *)ptr;
|
|
#endif
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
util_prefetch_read_range(void *ptr, size_t sz) {
|
|
for (size_t i = 0; i < sz; i += CACHELINE) {
|
|
util_prefetch_read((void *)((byte_t *)ptr + i));
|
|
}
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE void
|
|
util_prefetch_write_range(void *ptr, size_t sz) {
|
|
for (size_t i = 0; i < sz; i += CACHELINE) {
|
|
util_prefetch_write((void *)((byte_t *)ptr + i));
|
|
}
|
|
}
|
|
|
|
#undef UTIL_INLINE
|
|
|
|
#endif /* JEMALLOC_INTERNAL_UTIL_H */
|