Optimize witness fast path.

Short-circuit commonly called witness functions so that they only
execute in debug builds, and remove equivalent guards from mutex
functions.  This avoids pointless code execution in
witness_assert_lockless(), which is typically called twice per
allocation/deallocation function invocation.

Inline commonly called witness functions so that optimized builds can
completely remove calls as dead code.

include/jemalloc/internal/jemalloc_internal.h.in

@@ -531,9 +531,9 @@ void	jemalloc_postfork_child(void);
 #include "jemalloc/internal/smoothstep.h"
 #include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/tsd.h"
 #include "jemalloc/internal/witness.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/tsd.h"
 #include "jemalloc/internal/mb.h"
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/base.h"

include/jemalloc/internal/mutex.h

@@ -81,8 +81,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
 	if (isthreaded) {
-		if (config_debug)
-			witness_assert_not_owner(tsdn, &mutex->witness);
+		witness_assert_not_owner(tsdn, &mutex->witness);
 #ifdef _WIN32
 #  if _WIN32_WINNT >= 0x0600
 		AcquireSRWLockExclusive(&mutex->lock);
@@ -94,8 +93,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
 #else
 		pthread_mutex_lock(&mutex->lock);
 #endif
-		if (config_debug)
-			witness_lock(tsdn, &mutex->witness);
+		witness_lock(tsdn, &mutex->witness);
 	}
 }
 
@@ -104,8 +102,7 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
 	if (isthreaded) {
-		if (config_debug)
-			witness_unlock(tsdn, &mutex->witness);
+		witness_unlock(tsdn, &mutex->witness);
 #ifdef _WIN32
 #  if _WIN32_WINNT >= 0x0600
 		ReleaseSRWLockExclusive(&mutex->lock);
@@ -124,7 +121,7 @@ JEMALLOC_INLINE void
 malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
-	if (isthreaded && config_debug)
+	if (isthreaded)
 		witness_assert_owner(tsdn, &mutex->witness);
 }
 
@@ -132,7 +129,7 @@ JEMALLOC_INLINE void
 malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
-	if (isthreaded && config_debug)
+	if (isthreaded)
 		witness_assert_not_owner(tsdn, &mutex->witness);
 }
 #endif

include/jemalloc/internal/witness.h

@@ -74,24 +74,28 @@ void	witness_init(witness_t *witness, const char *name, witness_rank_t rank,
 #ifdef JEMALLOC_JET
 typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
 extern witness_lock_error_t *witness_lock_error;
+#else
+void	witness_lock_error(const witness_list_t *witnesses,
+    const witness_t *witness);
 #endif
-void	witness_lock(tsdn_t *tsdn, witness_t *witness);
-void	witness_unlock(tsdn_t *tsdn, witness_t *witness);
 #ifdef JEMALLOC_JET
 typedef void (witness_owner_error_t)(const witness_t *);
 extern witness_owner_error_t *witness_owner_error;
+#else
+void	witness_owner_error(const witness_t *witness);
 #endif
-void	witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
 #ifdef JEMALLOC_JET
 typedef void (witness_not_owner_error_t)(const witness_t *);
 extern witness_not_owner_error_t *witness_not_owner_error;
+#else
+void	witness_not_owner_error(const witness_t *witness);
 #endif
-void	witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
 #ifdef JEMALLOC_JET
 typedef void (witness_lockless_error_t)(const witness_list_t *);
 extern witness_lockless_error_t *witness_lockless_error;
+#else
+void	witness_lockless_error(const witness_list_t *witnesses);
 #endif
-void	witness_assert_lockless(tsdn_t *tsdn);
 
 void	witnesses_cleanup(tsd_t *tsd);
 void	witness_fork_cleanup(tsd_t *tsd);
@@ -103,5 +107,143 @@ void	witness_postfork_child(tsd_t *tsd);
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
+#ifndef JEMALLOC_ENABLE_INLINE
+void	witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
+void	witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
+void	witness_assert_lockless(tsdn_t *tsdn);
+void	witness_lock(tsdn_t *tsdn, witness_t *witness);
+void	witness_unlock(tsdn_t *tsdn, witness_t *witness);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
+JEMALLOC_INLINE void
+witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
+{
+	tsd_t *tsd;
+	witness_list_t *witnesses;
+	witness_t *w;
+
+	if (!config_debug)
+		return;
+
+	if (tsdn_null(tsdn))
+		return;
+	tsd = tsdn_tsd(tsdn);
+	if (witness->rank == WITNESS_RANK_OMIT)
+		return;
+
+	witnesses = tsd_witnessesp_get(tsd);
+	ql_foreach(w, witnesses, link) {
+		if (w == witness)
+			return;
+	}
+	witness_owner_error(witness);
+}
+
+JEMALLOC_INLINE void
+witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
+{
+	tsd_t *tsd;
+	witness_list_t *witnesses;
+	witness_t *w;
+
+	if (!config_debug)
+		return;
+
+	if (tsdn_null(tsdn))
+		return;
+	tsd = tsdn_tsd(tsdn);
+	if (witness->rank == WITNESS_RANK_OMIT)
+		return;
+
+	witnesses = tsd_witnessesp_get(tsd);
+	ql_foreach(w, witnesses, link) {
+		if (w == witness)
+			witness_not_owner_error(witness);
+	}
+}
+
+JEMALLOC_INLINE void
+witness_assert_lockless(tsdn_t *tsdn)
+{
+	tsd_t *tsd;
+	witness_list_t *witnesses;
+	witness_t *w;
+
+	if (!config_debug)
+		return;
+
+	if (tsdn_null(tsdn))
+		return;
+	tsd = tsdn_tsd(tsdn);
+
+	witnesses = tsd_witnessesp_get(tsd);
+	w = ql_last(witnesses, link);
+	if (w != NULL)
+		witness_lockless_error(witnesses);
+}
+
+JEMALLOC_INLINE void
+witness_lock(tsdn_t *tsdn, witness_t *witness)
+{
+	tsd_t *tsd;
+	witness_list_t *witnesses;
+	witness_t *w;
+
+	if (!config_debug)
+		return;
+
+	if (tsdn_null(tsdn))
+		return;
+	tsd = tsdn_tsd(tsdn);
+	if (witness->rank == WITNESS_RANK_OMIT)
+		return;
+
+	witness_assert_not_owner(tsdn, witness);
+
+	witnesses = tsd_witnessesp_get(tsd);
+	w = ql_last(witnesses, link);
+	if (w == NULL) {
+		/* No other locks; do nothing. */
+	} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
+		/* Forking, and relaxed ranking satisfied. */
+	} else if (w->rank > witness->rank) {
+		/* Not forking, rank order reversal. */
+		witness_lock_error(witnesses, witness);
+	} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
+	    witness->comp || w->comp(w, witness) > 0)) {
+		/*
+		 * Missing/incompatible comparison function, or comparison
+		 * function indicates rank order reversal.
+		 */
+		witness_lock_error(witnesses, witness);
+	}
+
+	ql_elm_new(witness, link);
+	ql_tail_insert(witnesses, witness, link);
+}
+
+JEMALLOC_INLINE void
+witness_unlock(tsdn_t *tsdn, witness_t *witness)
+{
+	tsd_t *tsd;
+	witness_list_t *witnesses;
+
+	if (!config_debug)
+		return;
+
+	if (tsdn_null(tsdn))
+		return;
+	tsd = tsdn_tsd(tsdn);
+	if (witness->rank == WITNESS_RANK_OMIT)
+		return;
+
+	witness_assert_owner(tsdn, witness);
+
+	witnesses = tsd_witnessesp_get(tsd);
+	ql_remove(witnesses, witness, link);
+}
+#endif
+
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/

src/witness.c

@@ -15,7 +15,7 @@ witness_init(witness_t *witness, const char *name, witness_rank_t rank,
 #undef witness_lock_error
 #define	witness_lock_error JEMALLOC_N(witness_lock_error_impl)
 #endif
-static void
+void
 witness_lock_error(const witness_list_t *witnesses, const witness_t *witness)
 {
 	witness_t *w;
@@ -33,66 +33,11 @@ witness_lock_error(const witness_list_t *witnesses, const witness_t *witness)
 witness_lock_error_t *witness_lock_error = JEMALLOC_N(witness_lock_error_impl);
 #endif
 
-void
-witness_lock(tsdn_t *tsdn, witness_t *witness)
-{
-	tsd_t *tsd;
-	witness_list_t *witnesses;
-	witness_t *w;
-
-	if (tsdn_null(tsdn))
-		return;
-	tsd = tsdn_tsd(tsdn);
-	if (witness->rank == WITNESS_RANK_OMIT)
-		return;
-
-	witness_assert_not_owner(tsdn, witness);
-
-	witnesses = tsd_witnessesp_get(tsd);
-	w = ql_last(witnesses, link);
-	if (w == NULL) {
-		/* No other locks; do nothing. */
-	} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
-		/* Forking, and relaxed ranking satisfied. */
-	} else if (w->rank > witness->rank) {
-		/* Not forking, rank order reversal. */
-		witness_lock_error(witnesses, witness);
-	} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
-	    witness->comp || w->comp(w, witness) > 0)) {
-		/*
-		 * Missing/incompatible comparison function, or comparison
-		 * function indicates rank order reversal.
-		 */
-		witness_lock_error(witnesses, witness);
-	}
-
-	ql_elm_new(witness, link);
-	ql_tail_insert(witnesses, witness, link);
-}
-
-void
-witness_unlock(tsdn_t *tsdn, witness_t *witness)
-{
-	tsd_t *tsd;
-	witness_list_t *witnesses;
-
-	if (tsdn_null(tsdn))
-		return;
-	tsd = tsdn_tsd(tsdn);
-	if (witness->rank == WITNESS_RANK_OMIT)
-		return;
-
-	witness_assert_owner(tsdn, witness);
-
-	witnesses = tsd_witnessesp_get(tsd);
-	ql_remove(witnesses, witness, link);
-}
-
 #ifdef JEMALLOC_JET
 #undef witness_owner_error
 #define	witness_owner_error JEMALLOC_N(witness_owner_error_impl)
 #endif
-static void
+void
 witness_owner_error(const witness_t *witness)
 {
 
@@ -107,32 +52,11 @@ witness_owner_error_t *witness_owner_error =
     JEMALLOC_N(witness_owner_error_impl);
 #endif
 
-void
-witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
-{
-	tsd_t *tsd;
-	witness_list_t *witnesses;
-	witness_t *w;
-
-	if (tsdn_null(tsdn))
-		return;
-	tsd = tsdn_tsd(tsdn);
-	if (witness->rank == WITNESS_RANK_OMIT)
-		return;
-
-	witnesses = tsd_witnessesp_get(tsd);
-	ql_foreach(w, witnesses, link) {
-		if (w == witness)
-			return;
-	}
-	witness_owner_error(witness);
-}
-
 #ifdef JEMALLOC_JET
 #undef witness_not_owner_error
 #define	witness_not_owner_error JEMALLOC_N(witness_not_owner_error_impl)
 #endif
-static void
+void
 witness_not_owner_error(const witness_t *witness)
 {
 
@@ -147,31 +71,11 @@ witness_not_owner_error_t *witness_not_owner_error =
     JEMALLOC_N(witness_not_owner_error_impl);
 #endif
 
-void
-witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
-{
-	tsd_t *tsd;
-	witness_list_t *witnesses;
-	witness_t *w;
-
-	if (tsdn_null(tsdn))
-		return;
-	tsd = tsdn_tsd(tsdn);
-	if (witness->rank == WITNESS_RANK_OMIT)
-		return;
-
-	witnesses = tsd_witnessesp_get(tsd);
-	ql_foreach(w, witnesses, link) {
-		if (w == witness)
-			witness_not_owner_error(witness);
-	}
-}
-
 #ifdef JEMALLOC_JET
 #undef witness_lockless_error
 #define	witness_lockless_error JEMALLOC_N(witness_lockless_error_impl)
 #endif
-static void
+void
 witness_lockless_error(const witness_list_t *witnesses)
 {
 	witness_t *w;
@@ -190,24 +94,6 @@ witness_lockless_error_t *witness_lockless_error =
     JEMALLOC_N(witness_lockless_error_impl);
 #endif
 
-void
-witness_assert_lockless(tsdn_t *tsdn)
-{
-	tsd_t *tsd;
-	witness_list_t *witnesses;
-	witness_t *w;
-
-	if (tsdn_null(tsdn))
-		return;
-	tsd = tsdn_tsd(tsdn);
-
-	witnesses = tsd_witnessesp_get(tsd);
-	w = ql_last(witnesses, link);
-	if (w != NULL) {
-		witness_lockless_error(witnesses);
-	}
-}
-
 void
 witnesses_cleanup(tsd_t *tsd)
 {