346 lines
9.2 KiB
C
346 lines
9.2 KiB
C
|
#ifndef JEMALLOC_INTERNAL_WITNESS_H
|
||
|
|
#define JEMALLOC_INTERNAL_WITNESS_H
|
||
|
|
|
||
|
|
#include "jemalloc/internal/ql.h"
|
||
|
|
|
||
|
|
/******************************************************************************/
|
||
|
|
/* LOCK RANKS */
|
||
|
|
/******************************************************************************/
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Witnesses with rank WITNESS_RANK_OMIT are completely ignored by the witness
|
||
|
|
* machinery.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#define WITNESS_RANK_OMIT 0U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_MIN 1U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_INIT 1U
|
||
|
|
#define WITNESS_RANK_CTL 1U
|
||
|
|
#define WITNESS_RANK_TCACHES 2U
|
||
|
|
#define WITNESS_RANK_ARENAS 3U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_BACKGROUND_THREAD_GLOBAL 4U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_PROF_DUMP 5U
|
||
|
|
#define WITNESS_RANK_PROF_BT2GCTX 6U
|
||
|
|
#define WITNESS_RANK_PROF_TDATAS 7U
|
||
|
|
#define WITNESS_RANK_PROF_TDATA 8U
|
||
|
|
#define WITNESS_RANK_PROF_GCTX 9U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_BACKGROUND_THREAD 10U
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Used as an argument to witness_assert_depth_to_rank() in order to validate
|
||
|
|
* depth excluding non-core locks with lower ranks. Since the rank argument to
|
||
|
|
* witness_assert_depth_to_rank() is inclusive rather than exclusive, this
|
||
|
|
* definition can have the same value as the minimally ranked core lock.
|
||
|
|
*/
|
||
|
|
#define WITNESS_RANK_CORE 11U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_DECAY 11U
|
||
|
|
#define WITNESS_RANK_TCACHE_QL 12U
|
||
|
|
#define WITNESS_RANK_EXTENTS 13U
|
||
|
|
#define WITNESS_RANK_EXTENT_FREELIST 14U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_EXTENT_POOL 15U
|
||
|
|
#define WITNESS_RANK_RTREE 16U
|
||
|
|
#define WITNESS_RANK_BASE 17U
|
||
|
|
#define WITNESS_RANK_ARENA_LARGE 18U
|
||
|
|
|
||
|
|
#define WITNESS_RANK_LEAF 0xffffffffU
|
||
|
|
#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_DSS WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_PROF_ACCUM WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF
|
||
|
|
#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF
|
||
|
|
|
||
|
|
/******************************************************************************/
|
||
|
|
/* PER-WITNESS DATA */
|
||
|
|
/******************************************************************************/
|
||
|
|
#if defined(JEMALLOC_DEBUG)
|
||
|
|
# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}}
|
||
|
|
#else
|
||
|
|
# define WITNESS_INITIALIZER(name, rank)
|
||
|
|
#endif
|
||
|
|
|
||
|
|
typedef struct witness_s witness_t;
|
||
|
|
typedef unsigned witness_rank_t;
|
||
|
|
typedef ql_head(witness_t) witness_list_t;
|
||
|
|
typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
|
||
|
|
void *);
|
||
|
|
|
||
|
|
struct witness_s {
|
||
|
|
/* Name, used for printing lock order reversal messages. */
|
||
|
|
const char *name;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses
|
||
|
|
* must be acquired in order of increasing rank.
|
||
|
|
*/
|
||
|
|
witness_rank_t rank;
|
||
|
|
|
||
|
|
/*
|
||
|
|
* If two witnesses are of equal rank and they have the samp comp
|
||
|
|
* function pointer, it is called as a last attempt to differentiate
|
||
|
|
* between witnesses of equal rank.
|
||
|
|
*/
|
||
|
|
witness_comp_t *comp;
|
||
|
|
|
||
|
|
/* Opaque data, passed to comp(). */
|
||
|
|
void *opaque;
|
||
|
|
|
||
|
|
/* Linkage for thread's currently owned locks. */
|
||
|
|
ql_elm(witness_t) link;
|
||
|
|
};
|
||
|
|
|
||
|
|
/******************************************************************************/
|
||
|
|
/* PER-THREAD DATA */
|
||
|
|
/******************************************************************************/
|
||
|
|
typedef struct witness_tsd_s witness_tsd_t;
|
||
|
|
struct witness_tsd_s {
|
||
|
|
witness_list_t witnesses;
|
||
|
|
bool forking;
|
||
|
|
};
|
||
|
|
|
||
|
|
#define WITNESS_TSD_INITIALIZER { ql_head_initializer(witnesses), false }
|
||
|
|
#define WITNESS_TSDN_NULL ((witness_tsdn_t *)0)
|
||
|
|
|
||
|
|
/******************************************************************************/
|
||
|
|
/* (PER-THREAD) NULLABILITY HELPERS */
|
||
|
|
/******************************************************************************/
|
||
|
|
typedef struct witness_tsdn_s witness_tsdn_t;
|
||
|
|
struct witness_tsdn_s {
|
||
|
|
witness_tsd_t witness_tsd;
|
||
|
|
};
|
||
|
|
|
||
|
|
JEMALLOC_ALWAYS_INLINE witness_tsdn_t *
|
||
|
|
witness_tsd_tsdn(witness_tsd_t *witness_tsd) {
|
||
|
|
return (witness_tsdn_t *)witness_tsd;
|
||
|
|
}
|
||
|
|
|
||
|
|
JEMALLOC_ALWAYS_INLINE bool
|
||
|
|
witness_tsdn_null(witness_tsdn_t *witness_tsdn) {
|
||
|
|
return witness_tsdn == NULL;
|
||
|
|
}
|
||
|
|
|
||
|
|
JEMALLOC_ALWAYS_INLINE witness_tsd_t *
|
||
|
|
witness_tsdn_tsd(witness_tsdn_t *witness_tsdn) {
|
||
|
|
assert(!witness_tsdn_null(witness_tsdn));
|
||
|
|
return &witness_tsdn->witness_tsd;
|
||
|
|
}
|
||
|
|
|
||
|
|
/******************************************************************************/
|
||
|
|
/* API */
|
||
|
|
/******************************************************************************/
|
||
|
|
void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
|
||
|
|
witness_comp_t *comp, void *opaque);
|
||
|
|
|
||
|
|
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
|
||
|
|
extern witness_lock_error_t *JET_MUTABLE witness_lock_error;
|
||
|
|
|
||
|
|
typedef void (witness_owner_error_t)(const witness_t *);
|
||
|
|
extern witness_owner_error_t *JET_MUTABLE witness_owner_error;
|
||
|
|
|
||
|
|
typedef void (witness_not_owner_error_t)(const witness_t *);
|
||
|
|
extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error;
|
||
|
|
|
||
|
|
typedef void (witness_depth_error_t)(const witness_list_t *,
|
||
|
|
witness_rank_t rank_inclusive, unsigned depth);
|
||
|
|
extern witness_depth_error_t *JET_MUTABLE witness_depth_error;
|
||
|
|
|
||
|
|
void witnesses_cleanup(witness_tsd_t *witness_tsd);
|
||
|
|
void witness_prefork(witness_tsd_t *witness_tsd);
|
||
|
|
void witness_postfork_parent(witness_tsd_t *witness_tsd);
|
||
|
|
void witness_postfork_child(witness_tsd_t *witness_tsd);
|
||
|
|
|
||
|
|
/* Helper, not intended for direct use. */
|
||
|
|
static inline bool
|
||
|
|
witness_owner(witness_tsd_t *witness_tsd, const witness_t *witness) {
|
||
|
|
witness_list_t *witnesses;
|
||
|
|
witness_t *w;
|
||
|
|
|
||
|
|
cassert(config_debug);
|
||
|
|
|
||
|
|
witnesses = &witness_tsd->witnesses;
|
||
|
|
ql_foreach(w, witnesses, link) {
|
||
|
|
if (w == witness) {
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void
|
||
|
|
witness_assert_owner(witness_tsdn_t *witness_tsdn, const witness_t *witness) {
|
||
|
|
witness_tsd_t *witness_tsd;
|
||
|
|
|
||
|
|
if (!config_debug) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (witness_tsdn_null(witness_tsdn)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
witness_tsd = witness_tsdn_tsd(witness_tsdn);
|
||
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (witness_owner(witness_tsd, witness)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
witness_owner_error(witness);
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void
|
||
|
|
witness_assert_not_owner(witness_tsdn_t *witness_tsdn,
|
||
|
|
const witness_t *witness) {
|
||
|
|
witness_tsd_t *witness_tsd;
|
||
|
|
witness_list_t *witnesses;
|
||
|
|
witness_t *w;
|
||
|
|
|
||
|
|
if (!config_debug) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (witness_tsdn_null(witness_tsdn)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
witness_tsd = witness_tsdn_tsd(witness_tsdn);
|
||
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
witnesses = &witness_tsd->witnesses;
|
||
|
|
ql_foreach(w, witnesses, link) {
|
||
|
|
if (w == witness) {
|
||
|
|
witness_not_owner_error(witness);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void
|
||
|
|
witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn,
|
||
|
|
witness_rank_t rank_inclusive, unsigned depth) {
|
||
|
|
witness_tsd_t *witness_tsd;
|
||
|
|
unsigned d;
|
||
|
|
witness_list_t *witnesses;
|
||
|
|
witness_t *w;
|
||
|
|
|
||
|
|
if (!config_debug) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (witness_tsdn_null(witness_tsdn)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
witness_tsd = witness_tsdn_tsd(witness_tsdn);
|
||
|
|
|
||
|
|
d = 0;
|
||
|
|
witnesses = &witness_tsd->witnesses;
|
||
|
|
w = ql_last(witnesses, link);
|
||
|
|
if (w != NULL) {
|
||
|
|
ql_reverse_foreach(w, witnesses, link) {
|
||
|
|
if (w->rank < rank_inclusive) {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
d++;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (d != depth) {
|
||
|
|
witness_depth_error(witnesses, rank_inclusive, depth);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void
|
||
|
|
witness_assert_depth(witness_tsdn_t *witness_tsdn, unsigned depth) {
|
||
|
|
witness_assert_depth_to_rank(witness_tsdn, WITNESS_RANK_MIN, depth);
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void
|
||
|
|
witness_assert_lockless(witness_tsdn_t *witness_tsdn) {
|
||
|
|
witness_assert_depth(witness_tsdn, 0);
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void
|
||
|
|
witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
|
||
|
|
witness_tsd_t *witness_tsd;
|
||
|
|
witness_list_t *witnesses;
|
||
|
|
witness_t *w;
|
||
|
|
|
||
|
|
if (!config_debug) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (witness_tsdn_null(witness_tsdn)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
witness_tsd = witness_tsdn_tsd(witness_tsdn);
|
||
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
witness_assert_not_owner(witness_tsdn, witness);
|
||
|
|
|
||
|
|
witnesses = &witness_tsd->witnesses;
|
||
|
|
w = ql_last(witnesses, link);
|
||
|
|
if (w == NULL) {
|
||
|
|
/* No other locks; do nothing. */
|
||
|
|
} else if (witness_tsd->forking && w->rank <= witness->rank) {
|
||
|
|
/* Forking, and relaxed ranking satisfied. */
|
||
|
|
} else if (w->rank > witness->rank) {
|
||
|
|
/* Not forking, rank order reversal. */
|
||
|
|
witness_lock_error(witnesses, witness);
|
||
|
|
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
|
||
|
|
witness->comp || w->comp(w, w->opaque, witness, witness->opaque) >
|
||
|
|
0)) {
|
||
|
|
/*
|
||
|
|
* Missing/incompatible comparison function, or comparison
|
||
|
|
* function indicates rank order reversal.
|
||
|
|
*/
|
||
|
|
witness_lock_error(witnesses, witness);
|
||
|
|
}
|
||
|
|
|
||
|
|
ql_elm_new(witness, link);
|
||
|
|
ql_tail_insert(witnesses, witness, link);
|
||
|
|
}
|
||
|
|
|
||
|
|
static inline void
|
||
|
|
witness_unlock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
|
||
|
|
witness_tsd_t *witness_tsd;
|
||
|
|
witness_list_t *witnesses;
|
||
|
|
|
||
|
|
if (!config_debug) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (witness_tsdn_null(witness_tsdn)) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
witness_tsd = witness_tsdn_tsd(witness_tsdn);
|
||
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Check whether owner before removal, rather than relying on
|
||
|
|
* witness_assert_owner() to abort, so that unit tests can test this
|
||
|
|
* function's failure mode without causing undefined behavior.
|
||
|
|
*/
|
||
|
|
if (witness_owner(witness_tsd, witness)) {
|
||
|
|
witnesses = &witness_tsd->witnesses;
|
||
|
|
ql_remove(witnesses, witness, link);
|
||
|
|
} else {
|
||
|
|
witness_assert_owner(witness_tsdn, witness);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#endif /* JEMALLOC_INTERNAL_WITNESS_H */
|