Optimize witness fast path.
Short-circuit commonly called witness functions so that they only execute in debug builds, and remove equivalent guards from mutex functions. This avoids pointless code execution in witness_assert_lockless(), which is typically called twice per allocation/deallocation function invocation. Inline commonly called witness functions so that optimized builds can completely remove calls as dead code.
This commit is contained in:
Jason Evans
parent
7790a0ba40
commit
73d3d58dc2
@ -531,9 +531,9 @@ void jemalloc_postfork_child(void);
|
||||
#include "jemalloc/internal/smoothstep.h"
|
||||
#include "jemalloc/internal/stats.h"
|
||||
#include "jemalloc/internal/ctl.h"
|
||||
#include "jemalloc/internal/tsd.h"
|
||||
#include "jemalloc/internal/witness.h"
|
||||
#include "jemalloc/internal/mutex.h"
|
||||
#include "jemalloc/internal/tsd.h"
|
||||
#include "jemalloc/internal/mb.h"
|
||||
#include "jemalloc/internal/extent.h"
|
||||
#include "jemalloc/internal/base.h"
|
||||
|
||||
@ -81,7 +81,6 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded) {
|
||||
if (config_debug)
|
||||
witness_assert_not_owner(tsdn, &mutex->witness);
|
||||
#ifdef _WIN32
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
@ -94,7 +93,6 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
#else
|
||||
pthread_mutex_lock(&mutex->lock);
|
||||
#endif
|
||||
if (config_debug)
|
||||
witness_lock(tsdn, &mutex->witness);
|
||||
}
|
||||
}
|
||||
@ -104,7 +102,6 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded) {
|
||||
if (config_debug)
|
||||
witness_unlock(tsdn, &mutex->witness);
|
||||
#ifdef _WIN32
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
@ -124,7 +121,7 @@ JEMALLOC_INLINE void
|
||||
malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded && config_debug)
|
||||
if (isthreaded)
|
||||
witness_assert_owner(tsdn, &mutex->witness);
|
||||
}
|
||||
|
||||
@ -132,7 +129,7 @@ JEMALLOC_INLINE void
|
||||
malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
|
||||
{
|
||||
|
||||
if (isthreaded && config_debug)
|
||||
if (isthreaded)
|
||||
witness_assert_not_owner(tsdn, &mutex->witness);
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -74,24 +74,28 @@ void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
|
||||
extern witness_lock_error_t *witness_lock_error;
|
||||
#else
|
||||
void witness_lock_error(const witness_list_t *witnesses,
|
||||
const witness_t *witness);
|
||||
#endif
|
||||
void witness_lock(tsdn_t *tsdn, witness_t *witness);
|
||||
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_owner_error_t)(const witness_t *);
|
||||
extern witness_owner_error_t *witness_owner_error;
|
||||
#else
|
||||
void witness_owner_error(const witness_t *witness);
|
||||
#endif
|
||||
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_not_owner_error_t)(const witness_t *);
|
||||
extern witness_not_owner_error_t *witness_not_owner_error;
|
||||
#else
|
||||
void witness_not_owner_error(const witness_t *witness);
|
||||
#endif
|
||||
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (witness_lockless_error_t)(const witness_list_t *);
|
||||
extern witness_lockless_error_t *witness_lockless_error;
|
||||
#else
|
||||
void witness_lockless_error(const witness_list_t *witnesses);
|
||||
#endif
|
||||
void witness_assert_lockless(tsdn_t *tsdn);
|
||||
|
||||
void witnesses_cleanup(tsd_t *tsd);
|
||||
void witness_fork_cleanup(tsd_t *tsd);
|
||||
@ -103,5 +107,143 @@ void witness_postfork_child(tsd_t *tsd);
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
|
||||
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
|
||||
void witness_assert_lockless(tsdn_t *tsdn);
|
||||
void witness_lock(tsdn_t *tsdn, witness_t *witness);
|
||||
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
|
||||
JEMALLOC_INLINE void
|
||||
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_foreach(w, witnesses, link) {
|
||||
if (w == witness)
|
||||
return;
|
||||
}
|
||||
witness_owner_error(witness);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_foreach(w, witnesses, link) {
|
||||
if (w == witness)
|
||||
witness_not_owner_error(witness);
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_assert_lockless(tsdn_t *tsdn)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
w = ql_last(witnesses, link);
|
||||
if (w != NULL)
|
||||
witness_lockless_error(witnesses);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_lock(tsdn_t *tsdn, witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witness_assert_not_owner(tsdn, witness);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
w = ql_last(witnesses, link);
|
||||
if (w == NULL) {
|
||||
/* No other locks; do nothing. */
|
||||
} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
|
||||
/* Forking, and relaxed ranking satisfied. */
|
||||
} else if (w->rank > witness->rank) {
|
||||
/* Not forking, rank order reversal. */
|
||||
witness_lock_error(witnesses, witness);
|
||||
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
|
||||
witness->comp || w->comp(w, witness) > 0)) {
|
||||
/*
|
||||
* Missing/incompatible comparison function, or comparison
|
||||
* function indicates rank order reversal.
|
||||
*/
|
||||
witness_lock_error(witnesses, witness);
|
||||
}
|
||||
|
||||
ql_elm_new(witness, link);
|
||||
ql_tail_insert(witnesses, witness, link);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
witness_unlock(tsdn_t *tsdn, witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
|
||||
if (!config_debug)
|
||||
return;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witness_assert_owner(tsdn, witness);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_remove(witnesses, witness, link);
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
122
src/witness.c
122
src/witness.c
@ -15,7 +15,7 @@ witness_init(witness_t *witness, const char *name, witness_rank_t rank,
|
||||
#undef witness_lock_error
|
||||
#define witness_lock_error JEMALLOC_N(witness_lock_error_impl)
|
||||
#endif
|
||||
static void
|
||||
void
|
||||
witness_lock_error(const witness_list_t *witnesses, const witness_t *witness)
|
||||
{
|
||||
witness_t *w;
|
||||
@ -33,66 +33,11 @@ witness_lock_error(const witness_list_t *witnesses, const witness_t *witness)
|
||||
witness_lock_error_t *witness_lock_error = JEMALLOC_N(witness_lock_error_impl);
|
||||
#endif
|
||||
|
||||
void
|
||||
witness_lock(tsdn_t *tsdn, witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witness_assert_not_owner(tsdn, witness);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
w = ql_last(witnesses, link);
|
||||
if (w == NULL) {
|
||||
/* No other locks; do nothing. */
|
||||
} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
|
||||
/* Forking, and relaxed ranking satisfied. */
|
||||
} else if (w->rank > witness->rank) {
|
||||
/* Not forking, rank order reversal. */
|
||||
witness_lock_error(witnesses, witness);
|
||||
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
|
||||
witness->comp || w->comp(w, witness) > 0)) {
|
||||
/*
|
||||
* Missing/incompatible comparison function, or comparison
|
||||
* function indicates rank order reversal.
|
||||
*/
|
||||
witness_lock_error(witnesses, witness);
|
||||
}
|
||||
|
||||
ql_elm_new(witness, link);
|
||||
ql_tail_insert(witnesses, witness, link);
|
||||
}
|
||||
|
||||
void
|
||||
witness_unlock(tsdn_t *tsdn, witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witness_assert_owner(tsdn, witness);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_remove(witnesses, witness, link);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef witness_owner_error
|
||||
#define witness_owner_error JEMALLOC_N(witness_owner_error_impl)
|
||||
#endif
|
||||
static void
|
||||
void
|
||||
witness_owner_error(const witness_t *witness)
|
||||
{
|
||||
|
||||
@ -107,32 +52,11 @@ witness_owner_error_t *witness_owner_error =
|
||||
JEMALLOC_N(witness_owner_error_impl);
|
||||
#endif
|
||||
|
||||
void
|
||||
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_foreach(w, witnesses, link) {
|
||||
if (w == witness)
|
||||
return;
|
||||
}
|
||||
witness_owner_error(witness);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef witness_not_owner_error
|
||||
#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error_impl)
|
||||
#endif
|
||||
static void
|
||||
void
|
||||
witness_not_owner_error(const witness_t *witness)
|
||||
{
|
||||
|
||||
@ -147,31 +71,11 @@ witness_not_owner_error_t *witness_not_owner_error =
|
||||
JEMALLOC_N(witness_not_owner_error_impl);
|
||||
#endif
|
||||
|
||||
void
|
||||
witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
if (witness->rank == WITNESS_RANK_OMIT)
|
||||
return;
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
ql_foreach(w, witnesses, link) {
|
||||
if (w == witness)
|
||||
witness_not_owner_error(witness);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef witness_lockless_error
|
||||
#define witness_lockless_error JEMALLOC_N(witness_lockless_error_impl)
|
||||
#endif
|
||||
static void
|
||||
void
|
||||
witness_lockless_error(const witness_list_t *witnesses)
|
||||
{
|
||||
witness_t *w;
|
||||
@ -190,24 +94,6 @@ witness_lockless_error_t *witness_lockless_error =
|
||||
JEMALLOC_N(witness_lockless_error_impl);
|
||||
#endif
|
||||
|
||||
void
|
||||
witness_assert_lockless(tsdn_t *tsdn)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
witness_list_t *witnesses;
|
||||
witness_t *w;
|
||||
|
||||
if (tsdn_null(tsdn))
|
||||
return;
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
|
||||
witnesses = tsd_witnessesp_get(tsd);
|
||||
w = ql_last(witnesses, link);
|
||||
if (w != NULL) {
|
||||
witness_lockless_error(witnesses);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
witnesses_cleanup(tsd_t *tsd)
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user