771 lines
21 KiB
C
771 lines
21 KiB
C
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_TYPES
|
|
|
|
/* Maximum number of malloc_tsd users with cleanup functions. */
|
|
#define MALLOC_TSD_CLEANUPS_MAX 2
|
|
|
|
typedef bool (*malloc_tsd_cleanup_t)(void);
|
|
|
|
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
|
|
!defined(_WIN32))
|
|
typedef struct tsd_init_block_s tsd_init_block_t;
|
|
typedef struct tsd_init_head_s tsd_init_head_t;
|
|
#endif
|
|
|
|
typedef struct tsd_s tsd_t;
|
|
typedef struct tsdn_s tsdn_t;
|
|
|
|
#define TSDN_NULL ((tsdn_t *)0)
|
|
|
|
typedef enum {
|
|
tsd_state_uninitialized,
|
|
tsd_state_nominal,
|
|
tsd_state_purgatory,
|
|
tsd_state_reincarnated
|
|
} tsd_state_t;
|
|
|
|
/*
|
|
* TLS/TSD-agnostic macro-based implementation of thread-specific data. There
|
|
* are five macros that support (at least) three use cases: file-private,
|
|
* library-private, and library-private inlined. Following is an example
|
|
* library-private tsd variable:
|
|
*
|
|
* In example.h:
|
|
* typedef struct {
|
|
* int x;
|
|
* int y;
|
|
* } example_t;
|
|
* #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0})
|
|
* malloc_tsd_types(example_, example_t)
|
|
* malloc_tsd_protos(, example_, example_t)
|
|
* malloc_tsd_externs(example_, example_t)
|
|
* In example.c:
|
|
* malloc_tsd_data(, example_, example_t, EX_INITIALIZER)
|
|
* malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER,
|
|
* example_tsd_cleanup)
|
|
*
|
|
* The result is a set of generated functions, e.g.:
|
|
*
|
|
* bool example_tsd_boot(void) {...}
|
|
* bool example_tsd_booted_get(void) {...}
|
|
* example_t *example_tsd_get() {...}
|
|
* void example_tsd_set(example_t *val) {...}
|
|
*
|
|
* Note that all of the functions deal in terms of (a_type *) rather than
|
|
* (a_type) so that it is possible to support non-pointer types (unlike
|
|
* pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is
|
|
* cast to (void *). This means that the cleanup function needs to cast the
|
|
* function argument to (a_type *), then dereference the resulting pointer to
|
|
* access fields, e.g.
|
|
*
|
|
* void
|
|
* example_tsd_cleanup(void *arg)
|
|
* {
|
|
* example_t *example = (example_t *)arg;
|
|
*
|
|
* example->x = 42;
|
|
* [...]
|
|
* if ([want the cleanup function to be called again])
|
|
* example_tsd_set(example);
|
|
* }
|
|
*
|
|
* If example_tsd_set() is called within example_tsd_cleanup(), it will be
|
|
* called again. This is similar to how pthreads TSD destruction works, except
|
|
* that pthreads only calls the cleanup function again if the value was set to
|
|
* non-NULL.
|
|
*/
|
|
|
|
/* malloc_tsd_types(). */
|
|
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
|
|
#define malloc_tsd_types(a_name, a_type)
|
|
#elif (defined(JEMALLOC_TLS))
|
|
#define malloc_tsd_types(a_name, a_type)
|
|
#elif (defined(_WIN32))
|
|
#define malloc_tsd_types(a_name, a_type) \
|
|
typedef struct { \
|
|
bool initialized; \
|
|
a_type val; \
|
|
} a_name##tsd_wrapper_t;
|
|
#else
|
|
#define malloc_tsd_types(a_name, a_type) \
|
|
typedef struct { \
|
|
bool initialized; \
|
|
a_type val; \
|
|
} a_name##tsd_wrapper_t;
|
|
#endif
|
|
|
|
/* malloc_tsd_protos(). */
|
|
#define malloc_tsd_protos(a_attr, a_name, a_type) \
|
|
a_attr bool \
|
|
a_name##tsd_boot0(void); \
|
|
a_attr void \
|
|
a_name##tsd_boot1(void); \
|
|
a_attr bool \
|
|
a_name##tsd_boot(void); \
|
|
a_attr bool \
|
|
a_name##tsd_booted_get(void); \
|
|
a_attr a_type * \
|
|
a_name##tsd_get(void); \
|
|
a_attr void \
|
|
a_name##tsd_set(a_type *val);
|
|
|
|
/* malloc_tsd_externs(). */
|
|
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
|
|
#define malloc_tsd_externs(a_name, a_type) \
|
|
extern __thread a_type a_name##tsd_tls; \
|
|
extern __thread bool a_name##tsd_initialized; \
|
|
extern bool a_name##tsd_booted;
|
|
#elif (defined(JEMALLOC_TLS))
|
|
#define malloc_tsd_externs(a_name, a_type) \
|
|
extern __thread a_type a_name##tsd_tls; \
|
|
extern pthread_key_t a_name##tsd_tsd; \
|
|
extern bool a_name##tsd_booted;
|
|
#elif (defined(_WIN32))
|
|
#define malloc_tsd_externs(a_name, a_type) \
|
|
extern DWORD a_name##tsd_tsd; \
|
|
extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
|
|
extern bool a_name##tsd_booted;
|
|
#else
|
|
#define malloc_tsd_externs(a_name, a_type) \
|
|
extern pthread_key_t a_name##tsd_tsd; \
|
|
extern tsd_init_head_t a_name##tsd_init_head; \
|
|
extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \
|
|
extern bool a_name##tsd_booted;
|
|
#endif
|
|
|
|
/* malloc_tsd_data(). */
|
|
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
|
|
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
|
a_attr __thread a_type JEMALLOC_TLS_MODEL \
|
|
a_name##tsd_tls = a_initializer; \
|
|
a_attr __thread bool JEMALLOC_TLS_MODEL \
|
|
a_name##tsd_initialized = false; \
|
|
a_attr bool a_name##tsd_booted = false;
|
|
#elif (defined(JEMALLOC_TLS))
|
|
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
|
a_attr __thread a_type JEMALLOC_TLS_MODEL \
|
|
a_name##tsd_tls = a_initializer; \
|
|
a_attr pthread_key_t a_name##tsd_tsd; \
|
|
a_attr bool a_name##tsd_booted = false;
|
|
#elif (defined(_WIN32))
|
|
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
|
a_attr DWORD a_name##tsd_tsd; \
|
|
a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
|
|
false, \
|
|
a_initializer \
|
|
}; \
|
|
a_attr bool a_name##tsd_booted = false;
|
|
#else
|
|
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
|
|
a_attr pthread_key_t a_name##tsd_tsd; \
|
|
a_attr tsd_init_head_t a_name##tsd_init_head = { \
|
|
ql_head_initializer(blocks), \
|
|
MALLOC_MUTEX_INITIALIZER \
|
|
}; \
|
|
a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \
|
|
false, \
|
|
a_initializer \
|
|
}; \
|
|
a_attr bool a_name##tsd_booted = false;
|
|
#endif
|
|
|
|
/* malloc_tsd_funcs(). */
|
|
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
|
|
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
|
|
a_cleanup) \
|
|
/* Initialization/cleanup. */ \
|
|
a_attr bool \
|
|
a_name##tsd_cleanup_wrapper(void) \
|
|
{ \
|
|
\
|
|
if (a_name##tsd_initialized) { \
|
|
a_name##tsd_initialized = false; \
|
|
a_cleanup(&a_name##tsd_tls); \
|
|
} \
|
|
return (a_name##tsd_initialized); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_boot0(void) \
|
|
{ \
|
|
\
|
|
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
|
malloc_tsd_cleanup_register( \
|
|
&a_name##tsd_cleanup_wrapper); \
|
|
} \
|
|
a_name##tsd_booted = true; \
|
|
return (false); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_boot1(void) \
|
|
{ \
|
|
\
|
|
/* Do nothing. */ \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_boot(void) \
|
|
{ \
|
|
\
|
|
return (a_name##tsd_boot0()); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_booted_get(void) \
|
|
{ \
|
|
\
|
|
return (a_name##tsd_booted); \
|
|
} \
|
|
/* Get/set. */ \
|
|
a_attr a_type * \
|
|
a_name##tsd_get(void) \
|
|
{ \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
return (&a_name##tsd_tls); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_set(a_type *val) \
|
|
{ \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
if (likely(&a_name##tsd_tls != val)) \
|
|
a_name##tsd_tls = (*val); \
|
|
if (a_cleanup != malloc_tsd_no_cleanup) \
|
|
a_name##tsd_initialized = true; \
|
|
}
|
|
#elif (defined(JEMALLOC_TLS))
|
|
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
|
|
a_cleanup) \
|
|
/* Initialization/cleanup. */ \
|
|
a_attr bool \
|
|
a_name##tsd_boot0(void) \
|
|
{ \
|
|
\
|
|
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
|
if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \
|
|
0) \
|
|
return (true); \
|
|
} \
|
|
a_name##tsd_booted = true; \
|
|
return (false); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_boot1(void) \
|
|
{ \
|
|
\
|
|
/* Do nothing. */ \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_boot(void) \
|
|
{ \
|
|
\
|
|
return (a_name##tsd_boot0()); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_booted_get(void) \
|
|
{ \
|
|
\
|
|
return (a_name##tsd_booted); \
|
|
} \
|
|
/* Get/set. */ \
|
|
a_attr a_type * \
|
|
a_name##tsd_get(void) \
|
|
{ \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
return (&a_name##tsd_tls); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_set(a_type *val) \
|
|
{ \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
if (likely(&a_name##tsd_tls != val)) \
|
|
a_name##tsd_tls = (*val); \
|
|
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
|
if (pthread_setspecific(a_name##tsd_tsd, \
|
|
(void *)(&a_name##tsd_tls))) { \
|
|
malloc_write("<jemalloc>: Error" \
|
|
" setting TSD for "#a_name"\n"); \
|
|
if (opt_abort) \
|
|
abort(); \
|
|
} \
|
|
} \
|
|
}
|
|
#elif (defined(_WIN32))
|
|
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
|
|
a_cleanup) \
|
|
/* Initialization/cleanup. */ \
|
|
a_attr bool \
|
|
a_name##tsd_cleanup_wrapper(void) \
|
|
{ \
|
|
DWORD error = GetLastError(); \
|
|
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
|
|
TlsGetValue(a_name##tsd_tsd); \
|
|
SetLastError(error); \
|
|
\
|
|
if (wrapper == NULL) \
|
|
return (false); \
|
|
if (a_cleanup != malloc_tsd_no_cleanup && \
|
|
wrapper->initialized) { \
|
|
wrapper->initialized = false; \
|
|
a_cleanup(&wrapper->val); \
|
|
if (wrapper->initialized) { \
|
|
/* Trigger another cleanup round. */ \
|
|
return (true); \
|
|
} \
|
|
} \
|
|
malloc_tsd_dalloc(wrapper); \
|
|
return (false); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
|
|
{ \
|
|
\
|
|
if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \
|
|
malloc_write("<jemalloc>: Error setting" \
|
|
" TSD for "#a_name"\n"); \
|
|
abort(); \
|
|
} \
|
|
} \
|
|
a_attr a_name##tsd_wrapper_t * \
|
|
a_name##tsd_wrapper_get(void) \
|
|
{ \
|
|
DWORD error = GetLastError(); \
|
|
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
|
|
TlsGetValue(a_name##tsd_tsd); \
|
|
SetLastError(error); \
|
|
\
|
|
if (unlikely(wrapper == NULL)) { \
|
|
wrapper = (a_name##tsd_wrapper_t *) \
|
|
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
|
if (wrapper == NULL) { \
|
|
malloc_write("<jemalloc>: Error allocating" \
|
|
" TSD for "#a_name"\n"); \
|
|
abort(); \
|
|
} else { \
|
|
wrapper->initialized = false; \
|
|
wrapper->val = a_initializer; \
|
|
} \
|
|
a_name##tsd_wrapper_set(wrapper); \
|
|
} \
|
|
return (wrapper); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_boot0(void) \
|
|
{ \
|
|
\
|
|
a_name##tsd_tsd = TlsAlloc(); \
|
|
if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \
|
|
return (true); \
|
|
if (a_cleanup != malloc_tsd_no_cleanup) { \
|
|
malloc_tsd_cleanup_register( \
|
|
&a_name##tsd_cleanup_wrapper); \
|
|
} \
|
|
a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
|
|
a_name##tsd_booted = true; \
|
|
return (false); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_boot1(void) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper; \
|
|
wrapper = (a_name##tsd_wrapper_t *) \
|
|
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
|
if (wrapper == NULL) { \
|
|
malloc_write("<jemalloc>: Error allocating" \
|
|
" TSD for "#a_name"\n"); \
|
|
abort(); \
|
|
} \
|
|
memcpy(wrapper, &a_name##tsd_boot_wrapper, \
|
|
sizeof(a_name##tsd_wrapper_t)); \
|
|
a_name##tsd_wrapper_set(wrapper); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_boot(void) \
|
|
{ \
|
|
\
|
|
if (a_name##tsd_boot0()) \
|
|
return (true); \
|
|
a_name##tsd_boot1(); \
|
|
return (false); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_booted_get(void) \
|
|
{ \
|
|
\
|
|
return (a_name##tsd_booted); \
|
|
} \
|
|
/* Get/set. */ \
|
|
a_attr a_type * \
|
|
a_name##tsd_get(void) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper; \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
wrapper = a_name##tsd_wrapper_get(); \
|
|
return (&wrapper->val); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_set(a_type *val) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper; \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
wrapper = a_name##tsd_wrapper_get(); \
|
|
if (likely(&wrapper->val != val)) \
|
|
wrapper->val = *(val); \
|
|
if (a_cleanup != malloc_tsd_no_cleanup) \
|
|
wrapper->initialized = true; \
|
|
}
|
|
#else
|
|
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
|
|
a_cleanup) \
|
|
/* Initialization/cleanup. */ \
|
|
a_attr void \
|
|
a_name##tsd_cleanup_wrapper(void *arg) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \
|
|
\
|
|
if (a_cleanup != malloc_tsd_no_cleanup && \
|
|
wrapper->initialized) { \
|
|
wrapper->initialized = false; \
|
|
a_cleanup(&wrapper->val); \
|
|
if (wrapper->initialized) { \
|
|
/* Trigger another cleanup round. */ \
|
|
if (pthread_setspecific(a_name##tsd_tsd, \
|
|
(void *)wrapper)) { \
|
|
malloc_write("<jemalloc>: Error" \
|
|
" setting TSD for "#a_name"\n"); \
|
|
if (opt_abort) \
|
|
abort(); \
|
|
} \
|
|
return; \
|
|
} \
|
|
} \
|
|
malloc_tsd_dalloc(wrapper); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
|
|
{ \
|
|
\
|
|
if (pthread_setspecific(a_name##tsd_tsd, \
|
|
(void *)wrapper)) { \
|
|
malloc_write("<jemalloc>: Error setting" \
|
|
" TSD for "#a_name"\n"); \
|
|
abort(); \
|
|
} \
|
|
} \
|
|
a_attr a_name##tsd_wrapper_t * \
|
|
a_name##tsd_wrapper_get(void) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
|
|
pthread_getspecific(a_name##tsd_tsd); \
|
|
\
|
|
if (unlikely(wrapper == NULL)) { \
|
|
tsd_init_block_t block; \
|
|
wrapper = tsd_init_check_recursion( \
|
|
&a_name##tsd_init_head, &block); \
|
|
if (wrapper) \
|
|
return (wrapper); \
|
|
wrapper = (a_name##tsd_wrapper_t *) \
|
|
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
|
block.data = wrapper; \
|
|
if (wrapper == NULL) { \
|
|
malloc_write("<jemalloc>: Error allocating" \
|
|
" TSD for "#a_name"\n"); \
|
|
abort(); \
|
|
} else { \
|
|
wrapper->initialized = false; \
|
|
wrapper->val = a_initializer; \
|
|
} \
|
|
a_name##tsd_wrapper_set(wrapper); \
|
|
tsd_init_finish(&a_name##tsd_init_head, &block); \
|
|
} \
|
|
return (wrapper); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_boot0(void) \
|
|
{ \
|
|
\
|
|
if (pthread_key_create(&a_name##tsd_tsd, \
|
|
a_name##tsd_cleanup_wrapper) != 0) \
|
|
return (true); \
|
|
a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \
|
|
a_name##tsd_booted = true; \
|
|
return (false); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_boot1(void) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper; \
|
|
wrapper = (a_name##tsd_wrapper_t *) \
|
|
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
|
|
if (wrapper == NULL) { \
|
|
malloc_write("<jemalloc>: Error allocating" \
|
|
" TSD for "#a_name"\n"); \
|
|
abort(); \
|
|
} \
|
|
memcpy(wrapper, &a_name##tsd_boot_wrapper, \
|
|
sizeof(a_name##tsd_wrapper_t)); \
|
|
a_name##tsd_wrapper_set(wrapper); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_boot(void) \
|
|
{ \
|
|
\
|
|
if (a_name##tsd_boot0()) \
|
|
return (true); \
|
|
a_name##tsd_boot1(); \
|
|
return (false); \
|
|
} \
|
|
a_attr bool \
|
|
a_name##tsd_booted_get(void) \
|
|
{ \
|
|
\
|
|
return (a_name##tsd_booted); \
|
|
} \
|
|
/* Get/set. */ \
|
|
a_attr a_type * \
|
|
a_name##tsd_get(void) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper; \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
wrapper = a_name##tsd_wrapper_get(); \
|
|
return (&wrapper->val); \
|
|
} \
|
|
a_attr void \
|
|
a_name##tsd_set(a_type *val) \
|
|
{ \
|
|
a_name##tsd_wrapper_t *wrapper; \
|
|
\
|
|
assert(a_name##tsd_booted); \
|
|
wrapper = a_name##tsd_wrapper_get(); \
|
|
if (likely(&wrapper->val != val)) \
|
|
wrapper->val = *(val); \
|
|
if (a_cleanup != malloc_tsd_no_cleanup) \
|
|
wrapper->initialized = true; \
|
|
}
|
|
#endif
|
|
|
|
#endif /* JEMALLOC_H_TYPES */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_STRUCTS
|
|
|
|
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
|
|
!defined(_WIN32))
|
|
struct tsd_init_block_s {
|
|
ql_elm(tsd_init_block_t) link;
|
|
pthread_t thread;
|
|
void *data;
|
|
};
|
|
struct tsd_init_head_s {
|
|
ql_head(tsd_init_block_t) blocks;
|
|
malloc_mutex_t lock;
|
|
};
|
|
#endif
|
|
|
|
#define MALLOC_TSD \
|
|
/* O(name, type, cleanup) */ \
|
|
O(tcache, tcache_t *, yes) \
|
|
O(thread_allocated, uint64_t, no) \
|
|
O(thread_deallocated, uint64_t, no) \
|
|
O(prof_tdata, prof_tdata_t *, yes) \
|
|
O(iarena, arena_t *, yes) \
|
|
O(arena, arena_t *, yes) \
|
|
O(arenas_tdata, arena_tdata_t *, yes) \
|
|
O(narenas_tdata, unsigned, no) \
|
|
O(arenas_tdata_bypass, bool, no) \
|
|
O(tcache_enabled, tcache_enabled_t, no) \
|
|
O(rtree_ctx, rtree_ctx_t, no) \
|
|
O(witnesses, witness_list_t, yes) \
|
|
O(rtree_elm_witnesses, rtree_elm_witness_tsd_t,no) \
|
|
O(witness_fork, bool, no) \
|
|
|
|
#define TSD_INITIALIZER { \
|
|
tsd_state_uninitialized, \
|
|
NULL, \
|
|
0, \
|
|
0, \
|
|
NULL, \
|
|
NULL, \
|
|
NULL, \
|
|
NULL, \
|
|
0, \
|
|
false, \
|
|
tcache_enabled_default, \
|
|
RTREE_CTX_INITIALIZER, \
|
|
ql_head_initializer(witnesses), \
|
|
RTREE_ELM_WITNESS_TSD_INITIALIZER, \
|
|
false \
|
|
}
|
|
|
|
struct tsd_s {
|
|
tsd_state_t state;
|
|
#define O(n, t, c) \
|
|
t n;
|
|
MALLOC_TSD
|
|
#undef O
|
|
};
|
|
|
|
/*
|
|
* Wrapper around tsd_t that makes it possible to avoid implicit conversion
|
|
* between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be
|
|
* explicitly converted to tsd_t, which is non-nullable.
|
|
*/
|
|
struct tsdn_s {
|
|
tsd_t tsd;
|
|
};
|
|
|
|
static const tsd_t tsd_initializer = TSD_INITIALIZER;
|
|
|
|
malloc_tsd_types(, tsd_t)
|
|
|
|
#endif /* JEMALLOC_H_STRUCTS */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_EXTERNS
|
|
|
|
void *malloc_tsd_malloc(size_t size);
|
|
void malloc_tsd_dalloc(void *wrapper);
|
|
void malloc_tsd_no_cleanup(void *arg);
|
|
void malloc_tsd_cleanup_register(bool (*f)(void));
|
|
tsd_t *malloc_tsd_boot0(void);
|
|
void malloc_tsd_boot1(void);
|
|
#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \
|
|
!defined(_WIN32))
|
|
void *tsd_init_check_recursion(tsd_init_head_t *head,
|
|
tsd_init_block_t *block);
|
|
void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
|
|
#endif
|
|
void tsd_cleanup(void *arg);
|
|
|
|
#endif /* JEMALLOC_H_EXTERNS */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_INLINES
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
|
|
|
|
tsd_t *tsd_fetch(void);
|
|
tsdn_t *tsd_tsdn(tsd_t *tsd);
|
|
bool tsd_nominal(tsd_t *tsd);
|
|
#define O(n, t, c) \
|
|
t *tsd_##n##p_get(tsd_t *tsd); \
|
|
t tsd_##n##_get(tsd_t *tsd); \
|
|
void tsd_##n##_set(tsd_t *tsd, t n);
|
|
MALLOC_TSD
|
|
#undef O
|
|
tsdn_t *tsdn_fetch(void);
|
|
bool tsdn_null(const tsdn_t *tsdn);
|
|
tsd_t *tsdn_tsd(tsdn_t *tsdn);
|
|
rtree_ctx_t *tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
|
|
malloc_tsd_externs(, tsd_t)
|
|
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
|
|
|
|
JEMALLOC_ALWAYS_INLINE tsd_t *
|
|
tsd_fetch(void)
|
|
{
|
|
tsd_t *tsd = tsd_get();
|
|
|
|
if (unlikely(tsd->state != tsd_state_nominal)) {
|
|
if (tsd->state == tsd_state_uninitialized) {
|
|
tsd->state = tsd_state_nominal;
|
|
/* Trigger cleanup handler registration. */
|
|
tsd_set(tsd);
|
|
} else if (tsd->state == tsd_state_purgatory) {
|
|
tsd->state = tsd_state_reincarnated;
|
|
tsd_set(tsd);
|
|
} else
|
|
assert(tsd->state == tsd_state_reincarnated);
|
|
}
|
|
|
|
return (tsd);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE tsdn_t *
|
|
tsd_tsdn(tsd_t *tsd)
|
|
{
|
|
|
|
return ((tsdn_t *)tsd);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
tsd_nominal(tsd_t *tsd)
|
|
{
|
|
|
|
return (tsd->state == tsd_state_nominal);
|
|
}
|
|
|
|
#define O(n, t, c) \
|
|
JEMALLOC_ALWAYS_INLINE t * \
|
|
tsd_##n##p_get(tsd_t *tsd) \
|
|
{ \
|
|
\
|
|
return (&tsd->n); \
|
|
} \
|
|
\
|
|
JEMALLOC_ALWAYS_INLINE t \
|
|
tsd_##n##_get(tsd_t *tsd) \
|
|
{ \
|
|
\
|
|
return (*tsd_##n##p_get(tsd)); \
|
|
} \
|
|
\
|
|
JEMALLOC_ALWAYS_INLINE void \
|
|
tsd_##n##_set(tsd_t *tsd, t n) \
|
|
{ \
|
|
\
|
|
assert(tsd->state == tsd_state_nominal); \
|
|
tsd->n = n; \
|
|
}
|
|
MALLOC_TSD
|
|
#undef O
|
|
|
|
JEMALLOC_ALWAYS_INLINE tsdn_t *
|
|
tsdn_fetch(void)
|
|
{
|
|
|
|
if (!tsd_booted_get())
|
|
return (NULL);
|
|
|
|
return (tsd_tsdn(tsd_fetch()));
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE bool
|
|
tsdn_null(const tsdn_t *tsdn)
|
|
{
|
|
|
|
return (tsdn == NULL);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE tsd_t *
|
|
tsdn_tsd(tsdn_t *tsdn)
|
|
{
|
|
|
|
assert(!tsdn_null(tsdn));
|
|
|
|
return (&tsdn->tsd);
|
|
}
|
|
|
|
JEMALLOC_ALWAYS_INLINE rtree_ctx_t *
|
|
tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback)
|
|
{
|
|
|
|
/*
|
|
* If tsd cannot be accessed, initialize the fallback rtree_ctx and
|
|
* return a pointer to it.
|
|
*/
|
|
if (unlikely(tsdn_null(tsdn))) {
|
|
static const rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
|
|
memcpy(fallback, &rtree_ctx, sizeof(rtree_ctx_t));
|
|
return (fallback);
|
|
}
|
|
return (tsd_rtree_ctxp_get(tsdn_tsd(tsdn)));
|
|
}
|
|
#endif
|
|
|
|
#endif /* JEMALLOC_H_INLINES */
|
|
/******************************************************************************/
|