project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Header refactoring: unify and de-catchall witness code.

Browse Source
This commit is contained in:
David Goldblatt 2017-05-22 19:32:04 -07:00 committed by David Goldblatt
parent 36195c8f4d
commit 9f822a1fd7
18 changed files with 602 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/jemalloc/internal/jemalloc_internal_includes.h
View File

@ -40,7 +40,6 @@
/* TYPES */ /* TYPES */
/******************************************************************************/ /******************************************************************************/
#include "jemalloc/internal/witness_types.h"
#include "jemalloc/internal/mutex_types.h" #include "jemalloc/internal/mutex_types.h"
#include "jemalloc/internal/extent_types.h" #include "jemalloc/internal/extent_types.h"
#include "jemalloc/internal/extent_dss_types.h" #include "jemalloc/internal/extent_dss_types.h"
@ -54,7 +53,6 @@
/* STRUCTS */ /* STRUCTS */
/******************************************************************************/ /******************************************************************************/
#include "jemalloc/internal/witness_structs.h"
#include "jemalloc/internal/mutex_structs.h" #include "jemalloc/internal/mutex_structs.h"
#include "jemalloc/internal/mutex_pool_structs.h" #include "jemalloc/internal/mutex_pool_structs.h"
#include "jemalloc/internal/arena_structs_a.h" #include "jemalloc/internal/arena_structs_a.h"
@ -72,7 +70,6 @@
/******************************************************************************/ /******************************************************************************/
#include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/jemalloc_internal_externs.h"
#include "jemalloc/internal/witness_externs.h"
#include "jemalloc/internal/mutex_externs.h" #include "jemalloc/internal/mutex_externs.h"
#include "jemalloc/internal/extent_externs.h" #include "jemalloc/internal/extent_externs.h"
#include "jemalloc/internal/extent_dss_externs.h" #include "jemalloc/internal/extent_dss_externs.h"
@ -89,7 +86,6 @@
/* INLINES */ /* INLINES */
/******************************************************************************/ /******************************************************************************/
#include "jemalloc/internal/witness_inlines.h"
#include "jemalloc/internal/mutex_inlines.h" #include "jemalloc/internal/mutex_inlines.h"
#include "jemalloc/internal/mutex_pool_inlines.h" #include "jemalloc/internal/mutex_pool_inlines.h"
#include "jemalloc/internal/jemalloc_internal_inlines_a.h" #include "jemalloc/internal/jemalloc_internal_inlines_a.h"

22
include/jemalloc/internal/jemalloc_internal_inlines_c.h
View File

@ -2,6 +2,7 @@
#define JEMALLOC_INTERNAL_INLINES_C_H #define JEMALLOC_INTERNAL_INLINES_C_H
#include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/witness.h"
JEMALLOC_ALWAYS_INLINE arena_t * JEMALLOC_ALWAYS_INLINE arena_t *
iaalloc(tsdn_t *tsdn, const void *ptr) { iaalloc(tsdn_t *tsdn, const void *ptr) {
@ -25,7 +26,8 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
assert(size != 0); assert(size != 0);
assert(!is_internal || tcache == NULL); assert(!is_internal || tcache == NULL);
assert(!is_internal || arena == NULL || arena_is_auto(arena)); assert(!is_internal || arena == NULL || arena_is_auto(arena));
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path);
if (config_stats && is_internal && likely(ret != NULL)) { if (config_stats && is_internal && likely(ret != NULL)) {
@ -49,7 +51,8 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
assert(usize == sa2u(usize, alignment)); assert(usize == sa2u(usize, alignment));
assert(!is_internal || tcache == NULL); assert(!is_internal || tcache == NULL);
assert(!is_internal || arena == NULL || arena_is_auto(arena)); assert(!is_internal || arena == NULL || arena_is_auto(arena));
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache);
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
@ -82,7 +85,8 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, alloc_ctx_t *alloc_ctx,
assert(ptr != NULL); assert(ptr != NULL);
assert(!is_internal || tcache == NULL); assert(!is_internal || tcache == NULL);
assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr))); assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr)));
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
if (config_stats && is_internal) { if (config_stats && is_internal) {
arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr));
} }
@ -100,7 +104,8 @@ idalloc(tsd_t *tsd, void *ptr) {
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
alloc_ctx_t *alloc_ctx, bool slow_path) { alloc_ctx_t *alloc_ctx, bool slow_path) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path); arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
} }
@ -108,7 +113,8 @@ JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, tcache_t *tcache, size_t extra, size_t alignment, bool zero, tcache_t *tcache,
arena_t *arena) { arena_t *arena) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
void *p; void *p;
size_t usize, copysize; size_t usize, copysize;
@ -146,7 +152,8 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
bool zero, tcache_t *tcache, arena_t *arena) { bool zero, tcache_t *tcache, arena_t *arena) {
assert(ptr != NULL); assert(ptr != NULL);
assert(size != 0); assert(size != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
!= 0) { != 0) {
@ -174,7 +181,8 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero) { size_t alignment, bool zero) {
assert(ptr != NULL); assert(ptr != NULL);
assert(size != 0); assert(size != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
!= 0) { != 0) {

14
include/jemalloc/internal/mutex_inlines.h
View File

@ -31,14 +31,14 @@ mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
/* Trylock: return false if the lock is successfully acquired. */ /* Trylock: return false if the lock is successfully acquired. */
static inline bool static inline bool
malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_assert_not_owner(tsdn, &mutex->witness); witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
if (isthreaded) { if (isthreaded) {
if (malloc_mutex_trylock_final(mutex)) { if (malloc_mutex_trylock_final(mutex)) {
return true; return true;
} }
mutex_owner_stats_update(tsdn, mutex); mutex_owner_stats_update(tsdn, mutex);
} }
witness_lock(tsdn, &mutex->witness); witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
return false; return false;
} }
@ -69,19 +69,19 @@ malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
static inline void static inline void
malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_assert_not_owner(tsdn, &mutex->witness); witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
if (isthreaded) { if (isthreaded) {
if (malloc_mutex_trylock_final(mutex)) { if (malloc_mutex_trylock_final(mutex)) {
malloc_mutex_lock_slow(mutex); malloc_mutex_lock_slow(mutex);
} }
mutex_owner_stats_update(tsdn, mutex); mutex_owner_stats_update(tsdn, mutex);
} }
witness_lock(tsdn, &mutex->witness); witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
} }
static inline void static inline void
malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_unlock(tsdn, &mutex->witness); witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
if (isthreaded) { if (isthreaded) {
MALLOC_MUTEX_UNLOCK(mutex); MALLOC_MUTEX_UNLOCK(mutex);
} }
@ -89,12 +89,12 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
static inline void static inline void
malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_assert_owner(tsdn, &mutex->witness); witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
} }
static inline void static inline void
malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_assert_not_owner(tsdn, &mutex->witness); witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
} }
/* Copy the prof data from mutex for processing. */ /* Copy the prof data from mutex for processing. */

1
include/jemalloc/internal/mutex_pool_inlines.h
View File

@ -4,6 +4,7 @@
#include "jemalloc/internal/hash.h" #include "jemalloc/internal/hash.h"
#include "jemalloc/internal/mutex_inlines.h" #include "jemalloc/internal/mutex_inlines.h"
#include "jemalloc/internal/mutex_pool_structs.h" #include "jemalloc/internal/mutex_pool_structs.h"
#include "jemalloc/internal/witness.h"
/* /*
* This file really combines "inlines" and "externs", but only transitionally. * This file really combines "inlines" and "externs", but only transitionally.

3
include/jemalloc/internal/mutex_structs.h
View File

@ -3,8 +3,7 @@
#include "jemalloc/internal/atomic.h" #include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/witness_types.h" #include "jemalloc/internal/witness.h"
#include "jemalloc/internal/witness_structs.h"
struct malloc_mutex_s { struct malloc_mutex_s {
union { union {

98
include/jemalloc/internal/tsd.h
View File

@ -10,8 +10,7 @@
#include "jemalloc/internal/tcache_types.h" #include "jemalloc/internal/tcache_types.h"
#include "jemalloc/internal/tcache_structs.h" #include "jemalloc/internal/tcache_structs.h"
#include "jemalloc/internal/util.h" #include "jemalloc/internal/util.h"
#include "jemalloc/internal/witness_types.h" #include "jemalloc/internal/witness.h"
#include "jemalloc/internal/witness_structs.h"
/* /*
* Thread-Specific-Data layout * Thread-Specific-Data layout
@ -52,30 +51,29 @@
typedef void (*test_callback_t)(int *); typedef void (*test_callback_t)(int *);
# define MALLOC_TSD_TEST_DATA_INIT 0x72b65c10 # define MALLOC_TSD_TEST_DATA_INIT 0x72b65c10
# define MALLOC_TEST_TSD \ # define MALLOC_TEST_TSD \
O(test_data, int) \ O(test_data, int, int) \
O(test_callback, test_callback_t) O(test_callback, test_callback_t, int)
# define MALLOC_TEST_TSD_INITIALIZER , MALLOC_TSD_TEST_DATA_INIT, NULL # define MALLOC_TEST_TSD_INITIALIZER , MALLOC_TSD_TEST_DATA_INIT, NULL
#else #else
# define MALLOC_TEST_TSD # define MALLOC_TEST_TSD
# define MALLOC_TEST_TSD_INITIALIZER # define MALLOC_TEST_TSD_INITIALIZER
#endif #endif
/* O(name, type, nullable type */
#define MALLOC_TSD \ #define MALLOC_TSD \
/* O(name, type) */ \ O(tcache_enabled, bool, bool) \
O(tcache_enabled, bool) \ O(arenas_tdata_bypass, bool, bool) \
O(arenas_tdata_bypass, bool) \ O(reentrancy_level, int8_t, int8_t) \
O(reentrancy_level, int8_t) \ O(narenas_tdata, uint32_t, uint32_t) \
O(narenas_tdata, uint32_t) \ O(thread_allocated, uint64_t, uint64_t) \
O(thread_allocated, uint64_t) \ O(thread_deallocated, uint64_t, uint64_t) \
O(thread_deallocated, uint64_t) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \
O(prof_tdata, prof_tdata_t *) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \
O(rtree_ctx, rtree_ctx_t) \ O(iarena, arena_t *, arena_t *) \
O(iarena, arena_t *) \ O(arena, arena_t *, arena_t *) \
O(arena, arena_t *) \ O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\
O(arenas_tdata, arena_tdata_t *) \ O(tcache, tcache_t, tcache_t) \
O(tcache, tcache_t) \ O(witness_tsd, witness_tsd_t, witness_tsdn_t) \
O(witnesses, witness_list_t) \
O(witness_fork, bool) \
MALLOC_TEST_TSD MALLOC_TEST_TSD
#define TSD_INITIALIZER { \ #define TSD_INITIALIZER { \
@ -92,8 +90,7 @@ typedef void (*test_callback_t)(int *);
NULL, \ NULL, \
NULL, \ NULL, \
TCACHE_ZERO_INITIALIZER, \ TCACHE_ZERO_INITIALIZER, \
ql_head_initializer(witnesses), \ WITNESS_TSD_INITIALIZER \
false \
MALLOC_TEST_TSD_INITIALIZER \ MALLOC_TEST_TSD_INITIALIZER \
} }
@ -119,7 +116,7 @@ struct tsd_s {
* setters below. * setters below.
*/ */
tsd_state_t state; tsd_state_t state;
#define O(n, t) \ #define O(n, t, nt) \
t use_a_getter_or_setter_instead_##n; t use_a_getter_or_setter_instead_##n;
MALLOC_TSD MALLOC_TSD
#undef O #undef O
@ -135,6 +132,22 @@ struct tsdn_s {
tsd_t tsd; tsd_t tsd;
}; };
#define TSDN_NULL ((tsdn_t *)0) #define TSDN_NULL ((tsdn_t *)0)
JEMALLOC_ALWAYS_INLINE tsdn_t *
tsd_tsdn(tsd_t *tsd) {
return (tsdn_t *)tsd;
}
JEMALLOC_ALWAYS_INLINE bool
tsdn_null(const tsdn_t *tsdn) {
return tsdn == NULL;
}
JEMALLOC_ALWAYS_INLINE tsd_t *
tsdn_tsd(tsdn_t *tsdn) {
assert(!tsdn_null(tsdn));
return &tsdn->tsd;
}
void *malloc_tsd_malloc(size_t size); void *malloc_tsd_malloc(size_t size);
void malloc_tsd_dalloc(void *wrapper); void malloc_tsd_dalloc(void *wrapper);
@ -166,7 +179,7 @@ void tsd_slow_update(tsd_t *tsd);
* foo. This omits some safety checks, and so can be used during tsd * foo. This omits some safety checks, and so can be used during tsd
* initialization and cleanup. * initialization and cleanup.
*/ */
#define O(n, t) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE t * \ JEMALLOC_ALWAYS_INLINE t * \
tsd_##n##p_get_unsafe(tsd_t *tsd) { \ tsd_##n##p_get_unsafe(tsd_t *tsd) { \
return &tsd->use_a_getter_or_setter_instead_##n; \ return &tsd->use_a_getter_or_setter_instead_##n; \
@ -175,7 +188,7 @@ MALLOC_TSD
#undef O #undef O
/* tsd_foop_get(tsd) returns a pointer to the thread-local instance of foo. */ /* tsd_foop_get(tsd) returns a pointer to the thread-local instance of foo. */
#define O(n, t) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE t * \ JEMALLOC_ALWAYS_INLINE t * \
tsd_##n##p_get(tsd_t *tsd) { \ tsd_##n##p_get(tsd_t *tsd) { \
assert(tsd->state == tsd_state_nominal || \ assert(tsd->state == tsd_state_nominal || \
@ -186,8 +199,24 @@ tsd_##n##p_get(tsd_t *tsd) { \
MALLOC_TSD MALLOC_TSD
#undef O #undef O
/*
* tsdn_foop_get(tsdn) returns either the thread-local instance of foo (if tsdn
* isn't NULL), or NULL (if tsdn is NULL), cast to the nullable pointer type.
*/
#define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE nt * \
tsdn_##n##p_get(tsdn_t *tsdn) { \
if (tsdn_null(tsdn)) { \
return NULL; \
} \
tsd_t *tsd = tsdn_tsd(tsdn); \
return (nt *)tsd_##n##p_get(tsd); \
}
MALLOC_TSD
#undef O
/* tsd_foo_get(tsd) returns the value of the thread-local instance of foo. */ /* tsd_foo_get(tsd) returns the value of the thread-local instance of foo. */
#define O(n, t) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE t \ JEMALLOC_ALWAYS_INLINE t \
tsd_##n##_get(tsd_t *tsd) { \ tsd_##n##_get(tsd_t *tsd) { \
return *tsd_##n##p_get(tsd); \ return *tsd_##n##p_get(tsd); \
@ -196,7 +225,7 @@ MALLOC_TSD
#undef O #undef O
/* tsd_foo_set(tsd, val) updates the thread-local instance of foo to be val. */ /* tsd_foo_set(tsd, val) updates the thread-local instance of foo to be val. */
#define O(n, t) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE void \ JEMALLOC_ALWAYS_INLINE void \
tsd_##n##_set(tsd_t *tsd, t val) { \ tsd_##n##_set(tsd_t *tsd, t val) { \
*tsd_##n##p_get(tsd) = val; \ *tsd_##n##p_get(tsd) = val; \
@ -243,11 +272,6 @@ tsd_fetch(void) {
return tsd_fetch_impl(true); return tsd_fetch_impl(true);
} }
JEMALLOC_ALWAYS_INLINE tsdn_t *
tsd_tsdn(tsd_t *tsd) {
return (tsdn_t *)tsd;
}
static inline bool static inline bool
tsd_nominal(tsd_t *tsd) { tsd_nominal(tsd_t *tsd) {
return (tsd->state <= tsd_state_nominal_max); return (tsd->state <= tsd_state_nominal_max);
@ -262,18 +286,6 @@ tsdn_fetch(void) {
return tsd_tsdn(tsd_fetch_impl(false)); return tsd_tsdn(tsd_fetch_impl(false));
} }
JEMALLOC_ALWAYS_INLINE bool
tsdn_null(const tsdn_t *tsdn) {
return tsdn == NULL;
}
JEMALLOC_ALWAYS_INLINE tsd_t *
tsdn_tsd(tsdn_t *tsdn) {
assert(!tsdn_null(tsdn));
return &tsdn->tsd;
}
JEMALLOC_ALWAYS_INLINE rtree_ctx_t * JEMALLOC_ALWAYS_INLINE rtree_ctx_t *
tsd_rtree_ctx(tsd_t *tsd) { tsd_rtree_ctx(tsd_t *tsd) {
return tsd_rtree_ctxp_get(tsd); return tsd_rtree_ctxp_get(tsd);

345
include/jemalloc/internal/witness.h Normal file
View File

@ -0,0 +1,345 @@
#ifndef JEMALLOC_INTERNAL_WITNESS_H
#define JEMALLOC_INTERNAL_WITNESS_H
#include "jemalloc/internal/ql.h"
/******************************************************************************/
/* LOCK RANKS */
/******************************************************************************/
/*
* Witnesses with rank WITNESS_RANK_OMIT are completely ignored by the witness
* machinery.
*/
#define WITNESS_RANK_OMIT 0U
#define WITNESS_RANK_MIN 1U
#define WITNESS_RANK_INIT 1U
#define WITNESS_RANK_CTL 1U
#define WITNESS_RANK_TCACHES 2U
#define WITNESS_RANK_ARENAS 3U
#define WITNESS_RANK_BACKGROUND_THREAD_GLOBAL 4U
#define WITNESS_RANK_PROF_DUMP 5U
#define WITNESS_RANK_PROF_BT2GCTX 6U
#define WITNESS_RANK_PROF_TDATAS 7U
#define WITNESS_RANK_PROF_TDATA 8U
#define WITNESS_RANK_PROF_GCTX 9U
#define WITNESS_RANK_BACKGROUND_THREAD 10U
/*
* Used as an argument to witness_assert_depth_to_rank() in order to validate
* depth excluding non-core locks with lower ranks. Since the rank argument to
* witness_assert_depth_to_rank() is inclusive rather than exclusive, this
* definition can have the same value as the minimally ranked core lock.
*/
#define WITNESS_RANK_CORE 11U
#define WITNESS_RANK_DECAY 11U
#define WITNESS_RANK_TCACHE_QL 12U
#define WITNESS_RANK_EXTENTS 13U
#define WITNESS_RANK_EXTENT_FREELIST 14U
#define WITNESS_RANK_EXTENT_POOL 15U
#define WITNESS_RANK_RTREE 16U
#define WITNESS_RANK_BASE 17U
#define WITNESS_RANK_ARENA_LARGE 18U
#define WITNESS_RANK_LEAF 0xffffffffU
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
#define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF
#define WITNESS_RANK_DSS WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_ACCUM WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF
/******************************************************************************/
/* PER-WITNESS DATA */
/******************************************************************************/
#if defined(JEMALLOC_DEBUG)
# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}}
#else
# define WITNESS_INITIALIZER(name, rank)
#endif
typedef struct witness_s witness_t;
typedef unsigned witness_rank_t;
typedef ql_head(witness_t) witness_list_t;
typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
void *);
struct witness_s {
/* Name, used for printing lock order reversal messages. */
const char *name;
/*
* Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses
* must be acquired in order of increasing rank.
*/
witness_rank_t rank;
/*
* If two witnesses are of equal rank and they have the samp comp
* function pointer, it is called as a last attempt to differentiate
* between witnesses of equal rank.
*/
witness_comp_t *comp;
/* Opaque data, passed to comp(). */
void *opaque;
/* Linkage for thread's currently owned locks. */
ql_elm(witness_t) link;
};
/******************************************************************************/
/* PER-THREAD DATA */
/******************************************************************************/
typedef struct witness_tsd_s witness_tsd_t;
struct witness_tsd_s {
witness_list_t witnesses;
bool forking;
};
#define WITNESS_TSD_INITIALIZER { ql_head_initializer(witnesses), false }
#define WITNESS_TSDN_NULL ((witness_tsdn_t *)0)
/******************************************************************************/
/* (PER-THREAD) NULLABILITY HELPERS */
/******************************************************************************/
typedef struct witness_tsdn_s witness_tsdn_t;
struct witness_tsdn_s {
witness_tsd_t witness_tsd;
};
JEMALLOC_ALWAYS_INLINE witness_tsdn_t *
witness_tsd_tsdn(witness_tsd_t *witness_tsd) {
return (witness_tsdn_t *)witness_tsd;
}
JEMALLOC_ALWAYS_INLINE bool
witness_tsdn_null(witness_tsdn_t *witness_tsdn) {
return witness_tsdn == NULL;
}
JEMALLOC_ALWAYS_INLINE witness_tsd_t *
witness_tsdn_tsd(witness_tsdn_t *witness_tsdn) {
assert(!witness_tsdn_null(witness_tsdn));
return &witness_tsdn->witness_tsd;
}
/******************************************************************************/
/* API */
/******************************************************************************/
void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
witness_comp_t *comp, void *opaque);
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
extern witness_lock_error_t *JET_MUTABLE witness_lock_error;
typedef void (witness_owner_error_t)(const witness_t *);
extern witness_owner_error_t *JET_MUTABLE witness_owner_error;
typedef void (witness_not_owner_error_t)(const witness_t *);
extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error;
typedef void (witness_depth_error_t)(const witness_list_t *,
witness_rank_t rank_inclusive, unsigned depth);
extern witness_depth_error_t *JET_MUTABLE witness_depth_error;
void witnesses_cleanup(witness_tsd_t *witness_tsd);
void witness_prefork(witness_tsd_t *witness_tsd);
void witness_postfork_parent(witness_tsd_t *witness_tsd);
void witness_postfork_child(witness_tsd_t *witness_tsd);
/* Helper, not intended for direct use. */
static inline bool
witness_owner(witness_tsd_t *witness_tsd, const witness_t *witness) {
witness_list_t *witnesses;
witness_t *w;
cassert(config_debug);
witnesses = &witness_tsd->witnesses;
ql_foreach(w, witnesses, link) {
if (w == witness) {
return true;
}
}
return false;
}
static inline void
witness_assert_owner(witness_tsdn_t *witness_tsdn, const witness_t *witness) {
witness_tsd_t *witness_tsd;
if (!config_debug) {
return;
}
if (witness_tsdn_null(witness_tsdn)) {
return;
}
witness_tsd = witness_tsdn_tsd(witness_tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
if (witness_owner(witness_tsd, witness)) {
return;
}
witness_owner_error(witness);
}
static inline void
witness_assert_not_owner(witness_tsdn_t *witness_tsdn,
const witness_t *witness) {
witness_tsd_t *witness_tsd;
witness_list_t *witnesses;
witness_t *w;
if (!config_debug) {
return;
}
if (witness_tsdn_null(witness_tsdn)) {
return;
}
witness_tsd = witness_tsdn_tsd(witness_tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
witnesses = &witness_tsd->witnesses;
ql_foreach(w, witnesses, link) {
if (w == witness) {
witness_not_owner_error(witness);
}
}
}
static inline void
witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn,
witness_rank_t rank_inclusive, unsigned depth) {
witness_tsd_t *witness_tsd;
unsigned d;
witness_list_t *witnesses;
witness_t *w;
if (!config_debug) {
return;
}
if (witness_tsdn_null(witness_tsdn)) {
return;
}
witness_tsd = witness_tsdn_tsd(witness_tsdn);
d = 0;
witnesses = &witness_tsd->witnesses;
w = ql_last(witnesses, link);
if (w != NULL) {
ql_reverse_foreach(w, witnesses, link) {
if (w->rank < rank_inclusive) {
break;
}
d++;
}
}
if (d != depth) {
witness_depth_error(witnesses, rank_inclusive, depth);
}
}
static inline void
witness_assert_depth(witness_tsdn_t *witness_tsdn, unsigned depth) {
witness_assert_depth_to_rank(witness_tsdn, WITNESS_RANK_MIN, depth);
}
static inline void
witness_assert_lockless(witness_tsdn_t *witness_tsdn) {
witness_assert_depth(witness_tsdn, 0);
}
static inline void
witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
witness_tsd_t *witness_tsd;
witness_list_t *witnesses;
witness_t *w;
if (!config_debug) {
return;
}
if (witness_tsdn_null(witness_tsdn)) {
return;
}
witness_tsd = witness_tsdn_tsd(witness_tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
witness_assert_not_owner(witness_tsdn, witness);
witnesses = &witness_tsd->witnesses;
w = ql_last(witnesses, link);
if (w == NULL) {
/* No other locks; do nothing. */
} else if (witness_tsd->forking && w->rank <= witness->rank) {
/* Forking, and relaxed ranking satisfied. */
} else if (w->rank > witness->rank) {
/* Not forking, rank order reversal. */
witness_lock_error(witnesses, witness);
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
witness->comp || w->comp(w, w->opaque, witness, witness->opaque) >
0)) {
/*
* Missing/incompatible comparison function, or comparison
* function indicates rank order reversal.
*/
witness_lock_error(witnesses, witness);
}
ql_elm_new(witness, link);
ql_tail_insert(witnesses, witness, link);
}
static inline void
witness_unlock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
witness_tsd_t *witness_tsd;
witness_list_t *witnesses;
if (!config_debug) {
return;
}
if (witness_tsdn_null(witness_tsdn)) {
return;
}
witness_tsd = witness_tsdn_tsd(witness_tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
/*
* Check whether owner before removal, rather than relying on
* witness_assert_owner() to abort, so that unit tests can test this
* function's failure mode without causing undefined behavior.
*/
if (witness_owner(witness_tsd, witness)) {
witnesses = &witness_tsd->witnesses;
ql_remove(witnesses, witness, link);
} else {
witness_assert_owner(witness_tsdn, witness);
}
}
#endif /* JEMALLOC_INTERNAL_WITNESS_H */

25
include/jemalloc/internal/witness_externs.h
View File

@ -1,25 +0,0 @@
#ifndef JEMALLOC_INTERNAL_WITNESS_EXTERNS_H
#define JEMALLOC_INTERNAL_WITNESS_EXTERNS_H
void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
witness_comp_t *comp, void *opaque);
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
extern witness_lock_error_t *JET_MUTABLE witness_lock_error;
typedef void (witness_owner_error_t)(const witness_t *);
extern witness_owner_error_t *JET_MUTABLE witness_owner_error;
typedef void (witness_not_owner_error_t)(const witness_t *);
extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error;
typedef void (witness_depth_error_t)(const witness_list_t *,
witness_rank_t rank_inclusive, unsigned depth);
extern witness_depth_error_t *JET_MUTABLE witness_depth_error;
void witnesses_cleanup(tsd_t *tsd);
void witness_prefork(tsd_t *tsd);
void witness_postfork_parent(tsd_t *tsd);
void witness_postfork_child(tsd_t *tsd);
#endif /* JEMALLOC_INTERNAL_WITNESS_EXTERNS_H */

188
include/jemalloc/internal/witness_inlines.h
View File

@ -1,188 +0,0 @@
#ifndef JEMALLOC_INTERNAL_WITNESS_INLINES_H
#define JEMALLOC_INTERNAL_WITNESS_INLINES_H
#include "jemalloc/internal/ql.h"
/* Helper, not intended for direct use. */
static inline bool
witness_owner(tsd_t *tsd, const witness_t *witness) {
witness_list_t *witnesses;
witness_t *w;
cassert(config_debug);
witnesses = tsd_witnessesp_get(tsd);
ql_foreach(w, witnesses, link) {
if (w == witness) {
return true;
}
}
return false;
}
static inline void
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) {
tsd_t *tsd;
if (!config_debug) {
return;
}
if (tsdn_null(tsdn)) {
return;
}
tsd = tsdn_tsd(tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
if (witness_owner(tsd, witness)) {
return;
}
witness_owner_error(witness);
}
static inline void
witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) {
tsd_t *tsd;
witness_list_t *witnesses;
witness_t *w;
if (!config_debug) {
return;
}
if (tsdn_null(tsdn)) {
return;
}
tsd = tsdn_tsd(tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
witnesses = tsd_witnessesp_get(tsd);
ql_foreach(w, witnesses, link) {
if (w == witness) {
witness_not_owner_error(witness);
}
}
}
static inline void
witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
unsigned depth) {
tsd_t *tsd;
unsigned d;
witness_list_t *witnesses;
witness_t *w;
if (!config_debug) {
return;
}
if (tsdn_null(tsdn)) {
return;
}
tsd = tsdn_tsd(tsdn);
d = 0;
witnesses = tsd_witnessesp_get(tsd);
w = ql_last(witnesses, link);
if (w != NULL) {
ql_reverse_foreach(w, witnesses, link) {
if (w->rank < rank_inclusive) {
break;
}
d++;
}
}
if (d != depth) {
witness_depth_error(witnesses, rank_inclusive, depth);
}
}
static inline void
witness_assert_depth(tsdn_t *tsdn, unsigned depth) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth);
}
static inline void
witness_assert_lockless(tsdn_t *tsdn) {
witness_assert_depth(tsdn, 0);
}
static inline void
witness_lock(tsdn_t *tsdn, witness_t *witness) {
tsd_t *tsd;
witness_list_t *witnesses;
witness_t *w;
if (!config_debug) {
return;
}
if (tsdn_null(tsdn)) {
return;
}
tsd = tsdn_tsd(tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
witness_assert_not_owner(tsdn, witness);
witnesses = tsd_witnessesp_get(tsd);
w = ql_last(witnesses, link);
if (w == NULL) {
/* No other locks; do nothing. */
} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
/* Forking, and relaxed ranking satisfied. */
} else if (w->rank > witness->rank) {
/* Not forking, rank order reversal. */
witness_lock_error(witnesses, witness);
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
witness->comp || w->comp(w, w->opaque, witness, witness->opaque) >
0)) {
/*
* Missing/incompatible comparison function, or comparison
* function indicates rank order reversal.
*/
witness_lock_error(witnesses, witness);
}
ql_elm_new(witness, link);
ql_tail_insert(witnesses, witness, link);
}
static inline void
witness_unlock(tsdn_t *tsdn, witness_t *witness) {
tsd_t *tsd;
witness_list_t *witnesses;
if (!config_debug) {
return;
}
if (tsdn_null(tsdn)) {
return;
}
tsd = tsdn_tsd(tsdn);
if (witness->rank == WITNESS_RANK_OMIT) {
return;
}
/*
* Check whether owner before removal, rather than relying on
* witness_assert_owner() to abort, so that unit tests can test this
* function's failure mode without causing undefined behavior.
*/
if (witness_owner(tsd, witness)) {
witnesses = tsd_witnessesp_get(tsd);
ql_remove(witnesses, witness, link);
} else {
witness_assert_owner(tsdn, witness);
}
}
#endif /* JEMALLOC_INTERNAL_WITNESS_INLINES_H */

28
include/jemalloc/internal/witness_structs.h
View File

@ -1,28 +0,0 @@
#ifndef JEMALLOC_INTERNAL_WITNESS_STRUCTS_H
#define JEMALLOC_INTERNAL_WITNESS_STRUCTS_H
struct witness_s {
/* Name, used for printing lock order reversal messages. */
const char *name;
/*
* Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses
* must be acquired in order of increasing rank.
*/
witness_rank_t rank;
/*
* If two witnesses are of equal rank and they have the samp comp
* function pointer, it is called as a last attempt to differentiate
* between witnesses of equal rank.
*/
witness_comp_t *comp;
/* Opaque data, passed to comp(). */
void *opaque;
/* Linkage for thread's currently owned locks. */
ql_elm(witness_t) link;
};
#endif /* JEMALLOC_INTERNAL_WITNESS_STRUCTS_H */

70
include/jemalloc/internal/witness_types.h
View File

@ -1,70 +0,0 @@
#ifndef JEMALLOC_INTERNAL_WITNESS_TYPES_H
#define JEMALLOC_INTERNAL_WITNESS_TYPES_H
#include "jemalloc/internal/ql.h"
typedef struct witness_s witness_t;
typedef unsigned witness_rank_t;
typedef ql_head(witness_t) witness_list_t;
typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
void *);
/*
* Lock ranks. Witnesses with rank WITNESS_RANK_OMIT are completely ignored by
* the witness machinery.
*/
#define WITNESS_RANK_OMIT 0U
#define WITNESS_RANK_MIN 1U
#define WITNESS_RANK_INIT 1U
#define WITNESS_RANK_CTL 1U
#define WITNESS_RANK_TCACHES 2U
#define WITNESS_RANK_ARENAS 3U
#define WITNESS_RANK_BACKGROUND_THREAD_GLOBAL 4U
#define WITNESS_RANK_PROF_DUMP 5U
#define WITNESS_RANK_PROF_BT2GCTX 6U
#define WITNESS_RANK_PROF_TDATAS 7U
#define WITNESS_RANK_PROF_TDATA 8U
#define WITNESS_RANK_PROF_GCTX 9U
#define WITNESS_RANK_BACKGROUND_THREAD 10U
/*
* Used as an argument to witness_assert_depth_to_rank() in order to validate
* depth excluding non-core locks with lower ranks. Since the rank argument to
* witness_assert_depth_to_rank() is inclusive rather than exclusive, this
* definition can have the same value as the minimally ranked core lock.
*/
#define WITNESS_RANK_CORE 11U
#define WITNESS_RANK_DECAY 11U
#define WITNESS_RANK_TCACHE_QL 12U
#define WITNESS_RANK_EXTENTS 13U
#define WITNESS_RANK_EXTENT_FREELIST 14U
#define WITNESS_RANK_EXTENT_POOL 15U
#define WITNESS_RANK_RTREE 16U
#define WITNESS_RANK_BASE 17U
#define WITNESS_RANK_ARENA_LARGE 18U
#define WITNESS_RANK_LEAF 0xffffffffU
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
#define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF
#define WITNESS_RANK_DSS WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_ACCUM WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF
#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF
#if defined(JEMALLOC_DEBUG)
# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}}
#else
# define WITNESS_INITIALIZER(name, rank)
#endif
#endif /* JEMALLOC_INTERNAL_WITNESS_TYPES_H */

18
src/arena.c
View File

@ -361,7 +361,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
void void
arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent) { extent_hooks_t **r_extent_hooks, extent_t *extent) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty, extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty,
extent); extent);
@ -497,7 +498,8 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool *zero) { size_t alignment, bool *zero) {
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
szind_t szind = size2index(usize); szind_t szind = size2index(usize);
size_t mapped_add; size_t mapped_add;
@ -892,7 +894,8 @@ static size_t
arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit, extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit,
extent_list_t *decay_extents) { extent_list_t *decay_extents) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
/* Stash extents according to npages_limit. */ /* Stash extents according to npages_limit. */
size_t nstashed = 0; size_t nstashed = 0;
@ -978,7 +981,8 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
static void static void
arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
extents_t *extents, bool all, size_t npages_limit) { extents_t *extents, bool all, size_t npages_limit) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 1);
malloc_mutex_assert_owner(tsdn, &decay->mtx); malloc_mutex_assert_owner(tsdn, &decay->mtx);
if (decay->purging) { if (decay->purging) {
@ -1253,7 +1257,8 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
extent_t *slab; extent_t *slab;
bool zero, commit; bool zero, commit;
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
zero = false; zero = false;
commit = true; commit = true;
@ -1271,7 +1276,8 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena,
static extent_t * static extent_t *
arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
const arena_bin_info_t *bin_info) { const arena_bin_info_t *bin_info) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
szind_t szind = size2index(bin_info->reg_size); szind_t szind = size2index(bin_info->reg_size);

54
src/extent.c
View File

@ -149,7 +149,8 @@ extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) {
extent_t * extent_t *
extent_alloc(tsdn_t *tsdn, arena_t *arena) { extent_alloc(tsdn_t *tsdn, arena_t *arena) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
extent_t *extent = extent_avail_first(&arena->extent_avail); extent_t *extent = extent_avail_first(&arena->extent_avail);
@ -164,7 +165,8 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) {
void void
extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
extent_avail_insert(&arena->extent_avail, extent); extent_avail_insert(&arena->extent_avail, extent);
@ -415,7 +417,8 @@ extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
assert(size + pad != 0); assert(size + pad != 0);
assert(alignment != 0); assert(alignment != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
return extent_recycle(tsdn, arena, r_extent_hooks, extents, new_addr, return extent_recycle(tsdn, arena, r_extent_hooks, extents, new_addr,
size, pad, alignment, slab, szind, zero, commit); size, pad, alignment, slab, szind, zero, commit);
@ -426,7 +429,8 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extents_t *extents, extent_t *extent) { extents_t *extents, extent_t *extent) {
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0); assert(extent_size_get(extent) != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false); extent_zeroed_set(extent, false);
@ -607,7 +611,8 @@ static void
extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) { extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) {
cassert(config_prof); cassert(config_prof);
/* prof_gdump() requirement. */ /* prof_gdump() requirement. */
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
if (opt_prof && extent_state_get(extent) == extent_state_active) { if (opt_prof && extent_state_get(extent) == extent_state_active) {
size_t nadd = extent_size_get(extent) >> LG_PAGE; size_t nadd = extent_size_get(extent) >> LG_PAGE;
@ -730,7 +735,8 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents,
bool locked, void *new_addr, size_t size, size_t pad, size_t alignment, bool locked, void *new_addr, size_t size, size_t pad, size_t alignment,
bool slab, bool *zero, bool *commit) { bool slab, bool *zero, bool *commit) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, locked ? 1 : 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, locked ? 1 : 0);
if (locked) { if (locked) {
malloc_mutex_assert_owner(tsdn, &extents->mtx); malloc_mutex_assert_owner(tsdn, &extents->mtx);
} }
@ -869,7 +875,8 @@ static extent_t *
extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extents_t *extents, void *new_addr, size_t size, size_t pad, extents_t *extents, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
assert(new_addr == NULL || !slab); assert(new_addr == NULL || !slab);
assert(pad == 0 || !slab); assert(pad == 0 || !slab);
assert(!*zero || !slab); assert(!*zero || !slab);
@ -1219,7 +1226,8 @@ extent_t *
extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad,
size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
@ -1385,7 +1393,8 @@ void
extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
if (extent_register(tsdn, extent)) { if (extent_register(tsdn, extent)) {
extents_leak(tsdn, arena, &extent_hooks, extents_leak(tsdn, arena, &extent_hooks,
@ -1418,7 +1427,8 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0); assert(extent_size_get(extent) != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_addr_set(extent, extent_base_get(extent)); extent_addr_set(extent, extent_base_get(extent));
@ -1445,7 +1455,8 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
void void
extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent) { extent_hooks_t **r_extent_hooks, extent_t *extent) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
/* /*
* Deregister first to avoid a race with other allocating threads, and * Deregister first to avoid a race with other allocating threads, and
@ -1508,7 +1519,8 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent) { extent_hooks_t **r_extent_hooks, extent_t *extent) {
assert(extent_base_get(extent) != NULL); assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0); assert(extent_size_get(extent) != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
/* Deregister first to avoid a race with other allocating threads. */ /* Deregister first to avoid a race with other allocating threads. */
extent_deregister(tsdn, extent); extent_deregister(tsdn, extent);
@ -1543,7 +1555,8 @@ bool
extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length) { size_t length) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
bool err = ((*r_extent_hooks)->commit == NULL || bool err = ((*r_extent_hooks)->commit == NULL ||
@ -1566,7 +1579,8 @@ bool
extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length) { size_t length) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
@ -1597,7 +1611,8 @@ bool
extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length) { size_t length) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
return ((*r_extent_hooks)->purge_lazy == NULL || return ((*r_extent_hooks)->purge_lazy == NULL ||
@ -1625,7 +1640,8 @@ bool
extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length) { size_t length) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
return ((*r_extent_hooks)->purge_forced == NULL || return ((*r_extent_hooks)->purge_forced == NULL ||
@ -1649,7 +1665,8 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) { szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) {
assert(extent_size_get(extent) == size_a + size_b); assert(extent_size_get(extent) == size_a + size_b);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);
@ -1742,7 +1759,8 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
bool bool
extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) { extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
extent_hooks_assure_initialized(arena, r_extent_hooks); extent_hooks_assure_initialized(arena, r_extent_hooks);

78
src/jemalloc.c
View File

@ -1755,7 +1755,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
*/ */
reentrancy_level = tsd_reentrancy_level_get(tsd); reentrancy_level = tsd_reentrancy_level_get(tsd);
if (reentrancy_level == 0) { if (reentrancy_level == 0) {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
} }
if (sopts->slow && unlikely(reentrancy_level > 0)) { if (sopts->slow && unlikely(reentrancy_level > 0)) {
/* /*
@ -1832,7 +1832,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
/* Success! */ /* Success! */
if (reentrancy_level == 0) { if (reentrancy_level == 0) {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
} }
*dopts->result = allocation; *dopts->result = allocation;
return 0; return 0;
@ -1847,7 +1847,7 @@ label_oom:
UTRACE(NULL, size, NULL); UTRACE(NULL, size, NULL);
} }
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
if (sopts->set_errno_on_error) { if (sopts->set_errno_on_error) {
set_errno(ENOMEM); set_errno(ENOMEM);
@ -1878,7 +1878,7 @@ label_invalid_alignment:
UTRACE(NULL, size, NULL); UTRACE(NULL, size, NULL);
} }
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
if (sopts->null_out_result_on_error) { if (sopts->null_out_result_on_error) {
*dopts->result = NULL; *dopts->result = NULL;
@ -2080,7 +2080,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
tsd_assert_fast(tsd); tsd_assert_fast(tsd);
} }
if (tsd_reentrancy_level_get(tsd) == 0) { if (tsd_reentrancy_level_get(tsd) == 0) {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
} else { } else {
assert(slow_path); assert(slow_path);
} }
@ -2120,7 +2120,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
tsd_assert_fast(tsd); tsd_assert_fast(tsd);
} }
if (tsd_reentrancy_level_get(tsd) == 0) { if (tsd_reentrancy_level_get(tsd) == 0) {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
} else { } else {
assert(slow_path); assert(slow_path);
} }
@ -2181,7 +2181,7 @@ je_realloc(void *ptr, size_t size) {
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
@ -2224,7 +2224,7 @@ je_realloc(void *ptr, size_t size) {
*tsd_thread_deallocatedp_get(tsd) += old_usize; *tsd_thread_deallocatedp_get(tsd) += old_usize;
} }
UTRACE(ptr, size, ret); UTRACE(ptr, size, ret);
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
return ret; return ret;
} }
@ -2234,7 +2234,8 @@ je_free(void *ptr) {
if (likely(ptr != NULL)) { if (likely(ptr != NULL)) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
if (tsd_reentrancy_level_get(tsd) == 0) { if (tsd_reentrancy_level_get(tsd) == 0) {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(
tsd)));
} }
tcache_t *tcache; tcache_t *tcache;
@ -2252,7 +2253,8 @@ je_free(void *ptr) {
ifree(tsd, ptr, tcache, true); ifree(tsd, ptr, tcache, true);
} }
if (tsd_reentrancy_level_get(tsd) == 0) { if (tsd_reentrancy_level_get(tsd) == 0) {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(
tsd)));
} }
} }
} }
@ -2513,7 +2515,7 @@ je_rallocx(void *ptr, size_t size, int flags) {
assert(size != 0); assert(size != 0);
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags); unsigned arena_ind = MALLOCX_ARENA_GET(flags);
@ -2569,7 +2571,7 @@ je_rallocx(void *ptr, size_t size, int flags) {
*tsd_thread_deallocatedp_get(tsd) += old_usize; *tsd_thread_deallocatedp_get(tsd) += old_usize;
} }
UTRACE(ptr, size, p); UTRACE(ptr, size, p);
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
return p; return p;
label_oom: label_oom:
if (config_xmalloc && unlikely(opt_xmalloc)) { if (config_xmalloc && unlikely(opt_xmalloc)) {
@ -2577,7 +2579,7 @@ label_oom:
abort(); abort();
} }
UTRACE(ptr, size, 0); UTRACE(ptr, size, 0);
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
return NULL; return NULL;
} }
@ -2669,7 +2671,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
assert(SIZE_T_MAX - size >= extra); assert(SIZE_T_MAX - size >= extra);
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
@ -2712,7 +2714,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
} }
label_not_resized: label_not_resized:
UTRACE(ptr, size, ptr); UTRACE(ptr, size, ptr);
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
return usize; return usize;
} }
@ -2726,7 +2728,7 @@ je_sallocx(const void *ptr, int flags) {
assert(ptr != NULL); assert(ptr != NULL);
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
if (config_debug || force_ivsalloc) { if (config_debug || force_ivsalloc) {
usize = ivsalloc(tsdn, ptr); usize = ivsalloc(tsdn, ptr);
@ -2735,7 +2737,7 @@ je_sallocx(const void *ptr, int flags) {
usize = isalloc(tsdn, ptr); usize = isalloc(tsdn, ptr);
} }
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
return usize; return usize;
} }
@ -2746,7 +2748,7 @@ je_dallocx(void *ptr, int flags) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
bool fast = tsd_fast(tsd); bool fast = tsd_fast(tsd);
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
tcache_t *tcache; tcache_t *tcache;
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
@ -2777,12 +2779,12 @@ je_dallocx(void *ptr, int flags) {
} else { } else {
ifree(tsd, ptr, tcache, true); ifree(tsd, ptr, tcache, true);
} }
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
} }
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
inallocx(tsdn_t *tsdn, size_t size, int flags) { inallocx(tsdn_t *tsdn, size_t size, int flags) {
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
size_t usize; size_t usize;
if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) { if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) {
@ -2790,7 +2792,7 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) {
} else { } else {
usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
} }
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
return usize; return usize;
} }
@ -2803,7 +2805,7 @@ je_sdallocx(void *ptr, size_t size, int flags) {
bool fast = tsd_fast(tsd); bool fast = tsd_fast(tsd);
size_t usize = inallocx(tsd_tsdn(tsd), size, flags); size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
assert(usize == isalloc(tsd_tsdn(tsd), ptr)); assert(usize == isalloc(tsd_tsdn(tsd), ptr));
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
tcache_t *tcache; tcache_t *tcache;
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
@ -2834,7 +2836,7 @@ je_sdallocx(void *ptr, size_t size, int flags) {
} else { } else {
isfree(tsd, ptr, usize, tcache, true); isfree(tsd, ptr, usize, tcache, true);
} }
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
} }
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@ -2850,14 +2852,14 @@ je_nallocx(size_t size, int flags) {
} }
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
usize = inallocx(tsdn, size, flags); usize = inallocx(tsdn, size, flags);
if (unlikely(usize > LARGE_MAXCLASS)) { if (unlikely(usize > LARGE_MAXCLASS)) {
return 0; return 0;
} }
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
return usize; return usize;
} }
@ -2872,9 +2874,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
} }
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
return ret; return ret;
} }
@ -2888,9 +2890,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) {
} }
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
ret = ctl_nametomib(tsdn, name, mibp, miblenp); ret = ctl_nametomib(tsdn, name, mibp, miblenp);
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
return ret; return ret;
} }
@ -2905,9 +2907,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
} }
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
return ret; return ret;
} }
@ -2917,9 +2919,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
tsdn_t *tsdn; tsdn_t *tsdn;
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
stats_print(write_cb, cbopaque, opts); stats_print(write_cb, cbopaque, opts);
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
} }
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@ -2930,7 +2932,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
if (unlikely(ptr == NULL)) { if (unlikely(ptr == NULL)) {
ret = 0; ret = 0;
@ -2943,7 +2945,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
} }
} }
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
return ret; return ret;
} }
@ -3000,7 +3002,7 @@ _malloc_prefork(void)
narenas = narenas_total_get(); narenas = narenas_total_get();
witness_prefork(tsd); witness_prefork(tsd_witness_tsdp_get(tsd));
/* Acquire all mutexes in a safe order. */ /* Acquire all mutexes in a safe order. */
ctl_prefork(tsd_tsdn(tsd)); ctl_prefork(tsd_tsdn(tsd));
tcache_prefork(tsd_tsdn(tsd)); tcache_prefork(tsd_tsdn(tsd));
@ -3067,7 +3069,7 @@ _malloc_postfork(void)
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_postfork_parent(tsd); witness_postfork_parent(tsd_witness_tsdp_get(tsd));
/* Release all mutexes, now that fork() has completed. */ /* Release all mutexes, now that fork() has completed. */
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena; arena_t *arena;
@ -3094,7 +3096,7 @@ jemalloc_postfork_child(void) {
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_postfork_child(tsd); witness_postfork_child(tsd_witness_tsdp_get(tsd));
/* Release all mutexes, now that fork() has completed. */ /* Release all mutexes, now that fork() has completed. */
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena; arena_t *arena;

4
src/tcache.c
View File

@ -568,7 +568,7 @@ label_return:
bool bool
tcaches_create(tsd_t *tsd, unsigned *r_ind) { tcaches_create(tsd_t *tsd, unsigned *r_ind) {
witness_assert_depth(tsd_tsdn(tsd), 0); witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
bool err; bool err;
@ -600,7 +600,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) {
err = false; err = false;
label_return: label_return:
witness_assert_depth(tsd_tsdn(tsd), 0); witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
return err; return err;
} }

2
src/tsd.c
View File

@ -140,7 +140,7 @@ tsd_do_data_cleanup(tsd_t *tsd) {
arena_cleanup(tsd); arena_cleanup(tsd);
arenas_tdata_cleanup(tsd); arenas_tdata_cleanup(tsd);
tcache_cleanup(tsd); tcache_cleanup(tsd);
witnesses_cleanup(tsd); witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd));
} }
void void

18
src/witness.c
View File

@ -63,38 +63,38 @@ witness_depth_error_t *JET_MUTABLE witness_depth_error =
witness_depth_error_impl; witness_depth_error_impl;
void void
witnesses_cleanup(tsd_t *tsd) { witnesses_cleanup(witness_tsd_t *witness_tsd) {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(witness_tsd_tsdn(witness_tsd));
/* Do nothing. */ /* Do nothing. */
} }
void void
witness_prefork(tsd_t *tsd) { witness_prefork(witness_tsd_t *witness_tsd) {
if (!config_debug) { if (!config_debug) {
return; return;
} }
tsd_witness_fork_set(tsd, true); witness_tsd->forking = true;
} }
void void
witness_postfork_parent(tsd_t *tsd) { witness_postfork_parent(witness_tsd_t *witness_tsd) {
if (!config_debug) { if (!config_debug) {
return; return;
} }
tsd_witness_fork_set(tsd, false); witness_tsd->forking = false;
} }
void void
witness_postfork_child(tsd_t *tsd) { witness_postfork_child(witness_tsd_t *witness_tsd) {
if (!config_debug) { if (!config_debug) {
return; return;
} }
#ifndef JEMALLOC_MUTEX_INIT_CB #ifndef JEMALLOC_MUTEX_INIT_CB
witness_list_t *witnesses; witness_list_t *witnesses;
witnesses = tsd_witnessesp_get(tsd); witnesses = &witness_tsd->witnesses;
ql_new(witnesses); ql_new(witnesses);
#endif #endif
tsd_witness_fork_set(tsd, false); witness_tsd->forking = false;
} }

168
test/unit/witness.c
View File

@ -55,95 +55,91 @@ witness_comp_reverse(const witness_t *a, void *oa, const witness_t *b,
TEST_BEGIN(test_witness) { TEST_BEGIN(test_witness) {
witness_t a, b; witness_t a, b;
tsdn_t *tsdn; witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER };
test_skip_if(!config_debug); test_skip_if(!config_debug);
tsdn = tsdn_fetch(); witness_assert_lockless(&witness_tsdn);
witness_assert_depth(&witness_tsdn, 0);
witness_assert_lockless(tsdn); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 0);
witness_assert_depth(tsdn, 0);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0);
witness_init(&a, "a", 1, NULL, NULL); witness_init(&a, "a", 1, NULL, NULL);
witness_assert_not_owner(tsdn, &a); witness_assert_not_owner(&witness_tsdn, &a);
witness_lock(tsdn, &a); witness_lock(&witness_tsdn, &a);
witness_assert_owner(tsdn, &a); witness_assert_owner(&witness_tsdn, &a);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 1);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 0); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)2U, 0);
witness_init(&b, "b", 2, NULL, NULL); witness_init(&b, "b", 2, NULL, NULL);
witness_assert_not_owner(tsdn, &b); witness_assert_not_owner(&witness_tsdn, &b);
witness_lock(tsdn, &b); witness_lock(&witness_tsdn, &b);
witness_assert_owner(tsdn, &b); witness_assert_owner(&witness_tsdn, &b);
witness_assert_depth(tsdn, 2); witness_assert_depth(&witness_tsdn, 2);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 2); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 2);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)2U, 1);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)3U, 0);
witness_unlock(tsdn, &a); witness_unlock(&witness_tsdn, &a);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 1);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)2U, 1);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)3U, 0);
witness_unlock(tsdn, &b); witness_unlock(&witness_tsdn, &b);
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_assert_depth(tsdn, 0); witness_assert_depth(&witness_tsdn, 0);
witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0); witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 0);
} }
TEST_END TEST_END
TEST_BEGIN(test_witness_comp) { TEST_BEGIN(test_witness_comp) {
witness_t a, b, c, d; witness_t a, b, c, d;
tsdn_t *tsdn; witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER };
test_skip_if(!config_debug); test_skip_if(!config_debug);
tsdn = tsdn_fetch(); witness_assert_lockless(&witness_tsdn);
witness_assert_lockless(tsdn);
witness_init(&a, "a", 1, witness_comp, &a); witness_init(&a, "a", 1, witness_comp, &a);
witness_assert_not_owner(tsdn, &a); witness_assert_not_owner(&witness_tsdn, &a);
witness_lock(tsdn, &a); witness_lock(&witness_tsdn, &a);
witness_assert_owner(tsdn, &a); witness_assert_owner(&witness_tsdn, &a);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
witness_init(&b, "b", 1, witness_comp, &b); witness_init(&b, "b", 1, witness_comp, &b);
witness_assert_not_owner(tsdn, &b); witness_assert_not_owner(&witness_tsdn, &b);
witness_lock(tsdn, &b); witness_lock(&witness_tsdn, &b);
witness_assert_owner(tsdn, &b); witness_assert_owner(&witness_tsdn, &b);
witness_assert_depth(tsdn, 2); witness_assert_depth(&witness_tsdn, 2);
witness_unlock(tsdn, &b); witness_unlock(&witness_tsdn, &b);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
witness_lock_error_orig = witness_lock_error; witness_lock_error_orig = witness_lock_error;
witness_lock_error = witness_lock_error_intercept; witness_lock_error = witness_lock_error_intercept;
saw_lock_error = false; saw_lock_error = false;
witness_init(&c, "c", 1, witness_comp_reverse, &c); witness_init(&c, "c", 1, witness_comp_reverse, &c);
witness_assert_not_owner(tsdn, &c); witness_assert_not_owner(&witness_tsdn, &c);
assert_false(saw_lock_error, "Unexpected witness lock error"); assert_false(saw_lock_error, "Unexpected witness lock error");
witness_lock(tsdn, &c); witness_lock(&witness_tsdn, &c);
assert_true(saw_lock_error, "Expected witness lock error"); assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &c); witness_unlock(&witness_tsdn, &c);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
saw_lock_error = false; saw_lock_error = false;
witness_init(&d, "d", 1, NULL, NULL); witness_init(&d, "d", 1, NULL, NULL);
witness_assert_not_owner(tsdn, &d); witness_assert_not_owner(&witness_tsdn, &d);
assert_false(saw_lock_error, "Unexpected witness lock error"); assert_false(saw_lock_error, "Unexpected witness lock error");
witness_lock(tsdn, &d); witness_lock(&witness_tsdn, &d);
assert_true(saw_lock_error, "Expected witness lock error"); assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &d); witness_unlock(&witness_tsdn, &d);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
witness_unlock(tsdn, &a); witness_unlock(&witness_tsdn, &a);
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_lock_error = witness_lock_error_orig; witness_lock_error = witness_lock_error_orig;
} }
@ -151,7 +147,7 @@ TEST_END
TEST_BEGIN(test_witness_reversal) { TEST_BEGIN(test_witness_reversal) {
witness_t a, b; witness_t a, b;
tsdn_t *tsdn; witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER };
test_skip_if(!config_debug); test_skip_if(!config_debug);
@ -159,24 +155,22 @@ TEST_BEGIN(test_witness_reversal) {
witness_lock_error = witness_lock_error_intercept; witness_lock_error = witness_lock_error_intercept;
saw_lock_error = false; saw_lock_error = false;
tsdn = tsdn_fetch(); witness_assert_lockless(&witness_tsdn);
witness_assert_lockless(tsdn);
witness_init(&a, "a", 1, NULL, NULL); witness_init(&a, "a", 1, NULL, NULL);
witness_init(&b, "b", 2, NULL, NULL); witness_init(&b, "b", 2, NULL, NULL);
witness_lock(tsdn, &b); witness_lock(&witness_tsdn, &b);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
assert_false(saw_lock_error, "Unexpected witness lock error"); assert_false(saw_lock_error, "Unexpected witness lock error");
witness_lock(tsdn, &a); witness_lock(&witness_tsdn, &a);
assert_true(saw_lock_error, "Expected witness lock error"); assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &a); witness_unlock(&witness_tsdn, &a);
witness_assert_depth(tsdn, 1); witness_assert_depth(&witness_tsdn, 1);
witness_unlock(tsdn, &b); witness_unlock(&witness_tsdn, &b);
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_lock_error = witness_lock_error_orig; witness_lock_error = witness_lock_error_orig;
} }
@ -184,7 +178,7 @@ TEST_END
TEST_BEGIN(test_witness_recursive) { TEST_BEGIN(test_witness_recursive) {
witness_t a; witness_t a;
tsdn_t *tsdn; witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER };
test_skip_if(!config_debug); test_skip_if(!config_debug);
@ -196,22 +190,20 @@ TEST_BEGIN(test_witness_recursive) {
witness_lock_error = witness_lock_error_intercept; witness_lock_error = witness_lock_error_intercept;
saw_lock_error = false; saw_lock_error = false;
tsdn = tsdn_fetch(); witness_assert_lockless(&witness_tsdn);
witness_assert_lockless(tsdn);
witness_init(&a, "a", 1, NULL, NULL); witness_init(&a, "a", 1, NULL, NULL);
witness_lock(tsdn, &a); witness_lock(&witness_tsdn, &a);
assert_false(saw_lock_error, "Unexpected witness lock error"); assert_false(saw_lock_error, "Unexpected witness lock error");
assert_false(saw_not_owner_error, "Unexpected witness not owner error"); assert_false(saw_not_owner_error, "Unexpected witness not owner error");
witness_lock(tsdn, &a); witness_lock(&witness_tsdn, &a);
assert_true(saw_lock_error, "Expected witness lock error"); assert_true(saw_lock_error, "Expected witness lock error");
assert_true(saw_not_owner_error, "Expected witness not owner error"); assert_true(saw_not_owner_error, "Expected witness not owner error");
witness_unlock(tsdn, &a); witness_unlock(&witness_tsdn, &a);
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_owner_error = witness_owner_error_orig; witness_owner_error = witness_owner_error_orig;
witness_lock_error = witness_lock_error_orig; witness_lock_error = witness_lock_error_orig;
@ -221,7 +213,7 @@ TEST_END
TEST_BEGIN(test_witness_unlock_not_owned) { TEST_BEGIN(test_witness_unlock_not_owned) {
witness_t a; witness_t a;
tsdn_t *tsdn; witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER };
test_skip_if(!config_debug); test_skip_if(!config_debug);
@ -229,17 +221,15 @@ TEST_BEGIN(test_witness_unlock_not_owned) {
witness_owner_error = witness_owner_error_intercept; witness_owner_error = witness_owner_error_intercept;
saw_owner_error = false; saw_owner_error = false;
tsdn = tsdn_fetch(); witness_assert_lockless(&witness_tsdn);
witness_assert_lockless(tsdn);
witness_init(&a, "a", 1, NULL, NULL); witness_init(&a, "a", 1, NULL, NULL);
assert_false(saw_owner_error, "Unexpected owner error"); assert_false(saw_owner_error, "Unexpected owner error");
witness_unlock(tsdn, &a); witness_unlock(&witness_tsdn, &a);
assert_true(saw_owner_error, "Expected owner error"); assert_true(saw_owner_error, "Expected owner error");
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_owner_error = witness_owner_error_orig; witness_owner_error = witness_owner_error_orig;
} }
@ -247,7 +237,7 @@ TEST_END
TEST_BEGIN(test_witness_depth) { TEST_BEGIN(test_witness_depth) {
witness_t a; witness_t a;
tsdn_t *tsdn; witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER };
test_skip_if(!config_debug); test_skip_if(!config_debug);
@ -255,26 +245,24 @@ TEST_BEGIN(test_witness_depth) {
witness_depth_error = witness_depth_error_intercept; witness_depth_error = witness_depth_error_intercept;
saw_depth_error = false; saw_depth_error = false;
tsdn = tsdn_fetch(); witness_assert_lockless(&witness_tsdn);
witness_assert_depth(&witness_tsdn, 0);
witness_assert_lockless(tsdn);
witness_assert_depth(tsdn, 0);
witness_init(&a, "a", 1, NULL, NULL); witness_init(&a, "a", 1, NULL, NULL);
assert_false(saw_depth_error, "Unexpected depth error"); assert_false(saw_depth_error, "Unexpected depth error");
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_assert_depth(tsdn, 0); witness_assert_depth(&witness_tsdn, 0);
witness_lock(tsdn, &a); witness_lock(&witness_tsdn, &a);
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_assert_depth(tsdn, 0); witness_assert_depth(&witness_tsdn, 0);
assert_true(saw_depth_error, "Expected depth error"); assert_true(saw_depth_error, "Expected depth error");
witness_unlock(tsdn, &a); witness_unlock(&witness_tsdn, &a);
witness_assert_lockless(tsdn); witness_assert_lockless(&witness_tsdn);
witness_assert_depth(tsdn, 0); witness_assert_depth(&witness_tsdn, 0);
witness_depth_error = witness_depth_error_orig; witness_depth_error = witness_depth_error_orig;
} }