2017-01-11 10:06:31 +08:00
|
|
|
#ifndef JEMALLOC_INTERNAL_WITNESS_INLINES_H
|
|
|
|
#define JEMALLOC_INTERNAL_WITNESS_INLINES_H
|
|
|
|
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
|
|
bool witness_owner(tsd_t *tsd, const witness_t *witness);
|
|
|
|
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
|
|
|
|
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
|
2017-01-22 07:12:03 +08:00
|
|
|
void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
|
|
|
|
unsigned depth);
|
|
|
|
void witness_assert_depth(tsdn_t *tsdn, unsigned depth);
|
2017-01-11 10:06:31 +08:00
|
|
|
void witness_assert_lockless(tsdn_t *tsdn);
|
|
|
|
void witness_lock(tsdn_t *tsdn, witness_t *witness);
|
|
|
|
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
|
|
|
|
/* Helper, not intended for direct use. */
|
|
|
|
JEMALLOC_INLINE bool
|
2017-01-16 08:56:30 +08:00
|
|
|
witness_owner(tsd_t *tsd, const witness_t *witness) {
|
2017-01-11 10:06:31 +08:00
|
|
|
witness_list_t *witnesses;
|
|
|
|
witness_t *w;
|
|
|
|
|
|
|
|
cassert(config_debug);
|
|
|
|
|
|
|
|
witnesses = tsd_witnessesp_get(tsd);
|
|
|
|
ql_foreach(w, witnesses, link) {
|
2017-01-16 08:56:30 +08:00
|
|
|
if (w == witness) {
|
2017-01-20 10:15:45 +08:00
|
|
|
return true;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
}
|
|
|
|
|
2017-01-20 10:15:45 +08:00
|
|
|
return false;
|
2017-01-11 10:06:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
JEMALLOC_INLINE void
|
2017-01-16 08:56:30 +08:00
|
|
|
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) {
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd_t *tsd;
|
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (!config_debug) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (tsdn_null(tsdn)) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd = tsdn_tsd(tsdn);
|
2017-01-16 08:56:30 +08:00
|
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (witness_owner(tsd, witness)) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
witness_owner_error(witness);
|
|
|
|
}
|
|
|
|
|
|
|
|
JEMALLOC_INLINE void
|
2017-01-16 08:56:30 +08:00
|
|
|
witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) {
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd_t *tsd;
|
|
|
|
witness_list_t *witnesses;
|
|
|
|
witness_t *w;
|
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (!config_debug) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (tsdn_null(tsdn)) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd = tsdn_tsd(tsdn);
|
2017-01-16 08:56:30 +08:00
|
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
|
|
|
witnesses = tsd_witnessesp_get(tsd);
|
|
|
|
ql_foreach(w, witnesses, link) {
|
2017-01-16 08:56:30 +08:00
|
|
|
if (w == witness) {
|
2017-01-11 10:06:31 +08:00
|
|
|
witness_not_owner_error(witness);
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
JEMALLOC_INLINE void
|
2017-01-22 07:12:03 +08:00
|
|
|
witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
|
|
|
|
unsigned depth) {
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd_t *tsd;
|
2017-01-22 07:12:03 +08:00
|
|
|
unsigned d;
|
2017-01-11 10:06:31 +08:00
|
|
|
witness_list_t *witnesses;
|
|
|
|
witness_t *w;
|
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (!config_debug) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (tsdn_null(tsdn)) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd = tsdn_tsd(tsdn);
|
|
|
|
|
2017-01-22 07:12:03 +08:00
|
|
|
d = 0;
|
2017-01-11 10:06:31 +08:00
|
|
|
witnesses = tsd_witnessesp_get(tsd);
|
|
|
|
w = ql_last(witnesses, link);
|
2017-01-16 08:56:30 +08:00
|
|
|
if (w != NULL) {
|
2017-01-22 07:12:03 +08:00
|
|
|
ql_reverse_foreach(w, witnesses, link) {
|
|
|
|
if (w->rank < rank_inclusive) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
d++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (d != depth) {
|
|
|
|
witness_depth_error(witnesses, rank_inclusive, depth);
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
}
|
|
|
|
|
2017-01-22 07:12:03 +08:00
|
|
|
JEMALLOC_INLINE void
|
|
|
|
witness_assert_depth(tsdn_t *tsdn, unsigned depth) {
|
|
|
|
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth);
|
|
|
|
}
|
|
|
|
|
|
|
|
JEMALLOC_INLINE void
|
|
|
|
witness_assert_lockless(tsdn_t *tsdn) {
|
|
|
|
witness_assert_depth(tsdn, 0);
|
|
|
|
}
|
|
|
|
|
2017-01-11 10:06:31 +08:00
|
|
|
JEMALLOC_INLINE void
|
2017-01-16 08:56:30 +08:00
|
|
|
witness_lock(tsdn_t *tsdn, witness_t *witness) {
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd_t *tsd;
|
|
|
|
witness_list_t *witnesses;
|
|
|
|
witness_t *w;
|
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (!config_debug) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (tsdn_null(tsdn)) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd = tsdn_tsd(tsdn);
|
2017-01-16 08:56:30 +08:00
|
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
|
|
|
witness_assert_not_owner(tsdn, witness);
|
|
|
|
|
|
|
|
witnesses = tsd_witnessesp_get(tsd);
|
|
|
|
w = ql_last(witnesses, link);
|
|
|
|
if (w == NULL) {
|
|
|
|
/* No other locks; do nothing. */
|
|
|
|
} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
|
|
|
|
/* Forking, and relaxed ranking satisfied. */
|
|
|
|
} else if (w->rank > witness->rank) {
|
|
|
|
/* Not forking, rank order reversal. */
|
|
|
|
witness_lock_error(witnesses, witness);
|
|
|
|
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
|
|
|
|
witness->comp || w->comp(w, w->opaque, witness, witness->opaque) >
|
|
|
|
0)) {
|
|
|
|
/*
|
|
|
|
* Missing/incompatible comparison function, or comparison
|
|
|
|
* function indicates rank order reversal.
|
|
|
|
*/
|
|
|
|
witness_lock_error(witnesses, witness);
|
|
|
|
}
|
|
|
|
|
|
|
|
ql_elm_new(witness, link);
|
|
|
|
ql_tail_insert(witnesses, witness, link);
|
|
|
|
}
|
|
|
|
|
|
|
|
JEMALLOC_INLINE void
|
2017-01-16 08:56:30 +08:00
|
|
|
witness_unlock(tsdn_t *tsdn, witness_t *witness) {
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd_t *tsd;
|
|
|
|
witness_list_t *witnesses;
|
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (!config_debug) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
2017-01-16 08:56:30 +08:00
|
|
|
if (tsdn_null(tsdn)) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
tsd = tsdn_tsd(tsdn);
|
2017-01-16 08:56:30 +08:00
|
|
|
if (witness->rank == WITNESS_RANK_OMIT) {
|
2017-01-11 10:06:31 +08:00
|
|
|
return;
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Check whether owner before removal, rather than relying on
|
|
|
|
* witness_assert_owner() to abort, so that unit tests can test this
|
|
|
|
* function's failure mode without causing undefined behavior.
|
|
|
|
*/
|
|
|
|
if (witness_owner(tsd, witness)) {
|
|
|
|
witnesses = tsd_witnessesp_get(tsd);
|
|
|
|
ql_remove(witnesses, witness, link);
|
2017-01-16 08:56:30 +08:00
|
|
|
} else {
|
2017-01-11 10:06:31 +08:00
|
|
|
witness_assert_owner(tsdn, witness);
|
2017-01-16 08:56:30 +08:00
|
|
|
}
|
2017-01-11 10:06:31 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#endif /* JEMALLOC_INTERNAL_WITNESS_INLINES_H */
|