diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index c1c6c409..4dfe442c 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -612,14 +612,14 @@ valgrind_freelike_block valgrind_make_mem_defined valgrind_make_mem_noaccess valgrind_make_mem_undefined -witness_assert_lockless +witness_assert_lock_depth witness_assert_not_owner witness_assert_owner witness_fork_cleanup witness_init witness_lock witness_lock_error -witness_lockless_error +witness_lock_depth_error witness_not_owner_error witness_owner witness_owner_error diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index cdf15d79..dfd827f7 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -91,10 +91,12 @@ extern witness_not_owner_error_t *witness_not_owner_error; void witness_not_owner_error(const witness_t *witness); #endif #ifdef JEMALLOC_JET -typedef void (witness_lockless_error_t)(const witness_list_t *); -extern witness_lockless_error_t *witness_lockless_error; +typedef void (witness_lock_depth_error_t)(const witness_list_t *, + unsigned depth); +extern witness_lock_depth_error_t *witness_lock_depth_error; #else -void witness_lockless_error(const witness_list_t *witnesses); +void witness_lock_depth_error(const witness_list_t *witnesses, + unsigned depth); #endif void witnesses_cleanup(tsd_t *tsd); @@ -111,7 +113,7 @@ void witness_postfork_child(tsd_t *tsd); bool witness_owner(tsd_t *tsd, const witness_t *witness); void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); -void witness_assert_lockless(tsdn_t *tsdn); +void witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth); void witness_lock(tsdn_t *tsdn, witness_t *witness); void witness_unlock(tsdn_t *tsdn, witness_t *witness); #endif @@ -175,9 +177,10 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) } JEMALLOC_INLINE void -witness_assert_lockless(tsdn_t *tsdn) +witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth) { tsd_t *tsd; + unsigned d; witness_list_t *witnesses; witness_t *w; @@ -188,10 +191,16 @@ witness_assert_lockless(tsdn_t *tsdn) return; tsd = tsdn_tsd(tsdn); + d = 0; witnesses = tsd_witnessesp_get(tsd); w = ql_last(witnesses, link); - if (w != NULL) - witness_lockless_error(witnesses); + if (w != NULL) { + ql_foreach(w, witnesses, link) { + d++; + } + } + if (d != depth) + witness_lock_depth_error(witnesses, depth); } JEMALLOC_INLINE void diff --git a/src/jemalloc.c b/src/jemalloc.c index baead664..c08f7e2f 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1581,7 +1581,7 @@ ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize, tsd = tsd_fetch(); *tsdn = tsd_tsdn(tsd); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); ind = size2index(size); if (unlikely(ind >= NSIZES)) @@ -1619,7 +1619,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func, assert(usize == isalloc(tsdn, ret, config_prof)); *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize; } - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); } JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN @@ -1704,7 +1704,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) goto label_oom; } tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (size == 0) size = 1; @@ -1745,7 +1745,7 @@ label_return: UTRACE(0, size, result); JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize, false); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (ret); label_oom: assert(result == NULL); @@ -1755,7 +1755,7 @@ label_oom: abort(); } ret = ENOMEM; - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); goto label_return; } @@ -1873,7 +1873,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) size_t usize; UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -1901,7 +1901,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -1947,7 +1947,7 @@ je_realloc(void *ptr, size_t size) malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); if (config_valgrind && unlikely(in_valgrind)) { @@ -1994,7 +1994,7 @@ je_realloc(void *ptr, size_t size) UTRACE(ptr, size, ret); JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr, old_usize, old_rzsize, maybe, false); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (ret); } @@ -2005,12 +2005,12 @@ je_free(void *ptr) UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) { tsd_t *tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (likely(!malloc_slow)) ifree(tsd, ptr, tcache_get(tsd, false), false); else ifree(tsd, ptr, tcache_get(tsd, false), true); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); } } @@ -2239,7 +2239,7 @@ imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize, tsd = tsd_fetch(); *tsdn = tsd_tsdn(tsd); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (likely(flags == 0)) { szind_t ind = size2index(size); @@ -2374,7 +2374,7 @@ je_rallocx(void *ptr, size_t size, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); @@ -2421,7 +2421,7 @@ je_rallocx(void *ptr, size_t size, int flags) UTRACE(ptr, size, p); JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr, old_usize, old_rzsize, no, zero); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (p); label_oom: if (config_xmalloc && unlikely(opt_xmalloc)) { @@ -2429,7 +2429,7 @@ label_oom: abort(); } UTRACE(ptr, size, 0); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (NULL); } @@ -2525,7 +2525,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); @@ -2566,7 +2566,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) old_usize, old_rzsize, no, zero); label_not_resized: UTRACE(ptr, size, ptr); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (usize); } @@ -2581,14 +2581,14 @@ je_sallocx(const void *ptr, int flags) malloc_thread_init(); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); if (config_ivsalloc) usize = ivsalloc(tsdn, ptr, config_prof); else usize = isalloc(tsdn, ptr, config_prof); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (usize); } @@ -2602,7 +2602,7 @@ je_dallocx(void *ptr, int flags) assert(malloc_initialized() || IS_INITIALIZER); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) tcache = NULL; @@ -2616,7 +2616,7 @@ je_dallocx(void *ptr, int flags) ifree(tsd, ptr, tcache, false); else ifree(tsd, ptr, tcache, true); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); } JEMALLOC_ALWAYS_INLINE_C size_t @@ -2624,13 +2624,13 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { size_t usize; - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) usize = s2u(size); else usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (usize); } @@ -2647,7 +2647,7 @@ je_sdallocx(void *ptr, size_t size, int flags) usize = inallocx(tsd_tsdn(tsd), size, flags); assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof)); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) tcache = NULL; @@ -2661,7 +2661,7 @@ je_sdallocx(void *ptr, size_t size, int flags) isfree(tsd, ptr, usize, tcache, false); else isfree(tsd, ptr, usize, tcache, true); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2677,13 +2677,13 @@ je_nallocx(size_t size, int flags) return (0); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); usize = inallocx(tsdn, size, flags); if (unlikely(usize > HUGE_MAXCLASS)) return (0); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (usize); } @@ -2698,9 +2698,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, return (EAGAIN); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (ret); } @@ -2714,9 +2714,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) return (EAGAIN); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); ret = ctl_nametomib(tsdn, name, mibp, miblenp); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (ret); } @@ -2731,9 +2731,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, return (EAGAIN); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (ret); } @@ -2744,9 +2744,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, tsdn_t *tsdn; tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); stats_print(write_cb, cbopaque, opts); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2759,14 +2759,14 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) malloc_thread_init(); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); if (config_ivsalloc) ret = ivsalloc(tsdn, ptr, config_prof); else ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (ret); } diff --git a/src/witness.c b/src/witness.c index 23753f24..aaea88d4 100644 --- a/src/witness.c +++ b/src/witness.c @@ -71,15 +71,16 @@ witness_not_owner_error_t *witness_not_owner_error = #endif #ifdef JEMALLOC_JET -#undef witness_lockless_error -#define witness_lockless_error JEMALLOC_N(n_witness_lockless_error) +#undef witness_lock_depth_error +#define witness_lock_depth_error JEMALLOC_N(n_witness_lock_depth_error) #endif void -witness_lockless_error(const witness_list_t *witnesses) +witness_lock_depth_error(const witness_list_t *witnesses, unsigned depth) { witness_t *w; - malloc_printf(": Should not own any locks:"); + malloc_printf(": Should own %u lock%s:", depth, (depth != 1) ? + "s" : ""); ql_foreach(w, witnesses, link) { malloc_printf(" %s(%u)", w->name, w->rank); } @@ -87,17 +88,17 @@ witness_lockless_error(const witness_list_t *witnesses) abort(); } #ifdef JEMALLOC_JET -#undef witness_lockless_error -#define witness_lockless_error JEMALLOC_N(witness_lockless_error) -witness_lockless_error_t *witness_lockless_error = - JEMALLOC_N(n_witness_lockless_error); +#undef witness_lock_depth_error +#define witness_lock_depth_error JEMALLOC_N(witness_lock_depth_error) +witness_lock_depth_error_t *witness_lock_depth_error = + JEMALLOC_N(n_witness_lock_depth_error); #endif void witnesses_cleanup(tsd_t *tsd) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); /* Do nothing. */ } diff --git a/test/unit/witness.c b/test/unit/witness.c index ed172753..9d4a1718 100644 --- a/test/unit/witness.c +++ b/test/unit/witness.c @@ -3,12 +3,12 @@ static witness_lock_error_t *witness_lock_error_orig; static witness_owner_error_t *witness_owner_error_orig; static witness_not_owner_error_t *witness_not_owner_error_orig; -static witness_lockless_error_t *witness_lockless_error_orig; +static witness_lock_depth_error_t *witness_lock_depth_error_orig; static bool saw_lock_error; static bool saw_owner_error; static bool saw_not_owner_error; -static bool saw_lockless_error; +static bool saw_lock_depth_error; static void witness_lock_error_intercept(const witness_list_t *witnesses, @@ -33,10 +33,11 @@ witness_not_owner_error_intercept(const witness_t *witness) } static void -witness_lockless_error_intercept(const witness_list_t *witnesses) +witness_lock_depth_error_intercept(const witness_list_t *witnesses, + unsigned depth) { - saw_lockless_error = true; + saw_lock_depth_error = true; } static int @@ -66,22 +67,25 @@ TEST_BEGIN(test_witness) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); witness_assert_not_owner(tsdn, &a); witness_lock(tsdn, &a); witness_assert_owner(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_init(&b, "b", 2, NULL); witness_assert_not_owner(tsdn, &b); witness_lock(tsdn, &b); witness_assert_owner(tsdn, &b); + witness_assert_lock_depth(tsdn, 2); witness_unlock(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_unlock(tsdn, &b); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); } TEST_END @@ -94,18 +98,21 @@ TEST_BEGIN(test_witness_comp) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, witness_comp); witness_assert_not_owner(tsdn, &a); witness_lock(tsdn, &a); witness_assert_owner(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_init(&b, "b", 1, witness_comp); witness_assert_not_owner(tsdn, &b); witness_lock(tsdn, &b); witness_assert_owner(tsdn, &b); + witness_assert_lock_depth(tsdn, 2); witness_unlock(tsdn, &b); + witness_assert_lock_depth(tsdn, 1); witness_lock_error_orig = witness_lock_error; witness_lock_error = witness_lock_error_intercept; @@ -117,6 +124,7 @@ TEST_BEGIN(test_witness_comp) witness_lock(tsdn, &c); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &c); + witness_assert_lock_depth(tsdn, 1); saw_lock_error = false; @@ -126,10 +134,11 @@ TEST_BEGIN(test_witness_comp) witness_lock(tsdn, &d); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &d); + witness_assert_lock_depth(tsdn, 1); witness_unlock(tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_lock_error = witness_lock_error_orig; } @@ -148,20 +157,22 @@ TEST_BEGIN(test_witness_reversal) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); witness_init(&b, "b", 2, NULL); witness_lock(tsdn, &b); + witness_assert_lock_depth(tsdn, 1); assert_false(saw_lock_error, "Unexpected witness lock error"); witness_lock(tsdn, &a); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_unlock(tsdn, &b); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_lock_error = witness_lock_error_orig; } @@ -184,7 +195,7 @@ TEST_BEGIN(test_witness_recursive) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); @@ -197,7 +208,7 @@ TEST_BEGIN(test_witness_recursive) witness_unlock(tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_owner_error = witness_owner_error_orig; witness_lock_error = witness_lock_error_orig; @@ -218,7 +229,7 @@ TEST_BEGIN(test_witness_unlock_not_owned) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); @@ -226,41 +237,41 @@ TEST_BEGIN(test_witness_unlock_not_owned) witness_unlock(tsdn, &a); assert_true(saw_owner_error, "Expected owner error"); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_owner_error = witness_owner_error_orig; } TEST_END -TEST_BEGIN(test_witness_lockful) +TEST_BEGIN(test_witness_lock_depth) { witness_t a; tsdn_t *tsdn; test_skip_if(!config_debug); - witness_lockless_error_orig = witness_lockless_error; - witness_lockless_error = witness_lockless_error_intercept; - saw_lockless_error = false; + witness_lock_depth_error_orig = witness_lock_depth_error; + witness_lock_depth_error = witness_lock_depth_error_intercept; + saw_lock_depth_error = false; tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); - assert_false(saw_lockless_error, "Unexpected lockless error"); - witness_assert_lockless(tsdn); + assert_false(saw_lock_depth_error, "Unexpected lock_depth error"); + witness_assert_lock_depth(tsdn, 0); witness_lock(tsdn, &a); - witness_assert_lockless(tsdn); - assert_true(saw_lockless_error, "Expected lockless error"); + witness_assert_lock_depth(tsdn, 0); + assert_true(saw_lock_depth_error, "Expected lock_depth error"); witness_unlock(tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); - witness_lockless_error = witness_lockless_error_orig; + witness_lock_depth_error = witness_lock_depth_error_orig; } TEST_END @@ -274,5 +285,5 @@ main(void) test_witness_reversal, test_witness_recursive, test_witness_unlock_not_owned, - test_witness_lockful)); + test_witness_lock_depth)); }