project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Convert witness_assert_lockless() to witness_assert_lock_depth().

Browse Source 
This makes it possible to make lock state assertions about precisely
which locks are held.
This commit is contained in:
Jason Evans 2017-01-21 15:12:03 -08:00
parent c6943acb3c
commit dad74bd3c8
5 changed files with 102 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/jemalloc/internal/private_symbols.txt
View File

@ -612,14 +612,14 @@ valgrind_freelike_block
valgrind_make_mem_defined
valgrind_make_mem_noaccess
valgrind_make_mem_undefined
witness_assert_lockless
witness_assert_lock_depth
witness_assert_not_owner
witness_assert_owner
witness_fork_cleanup
witness_init
witness_lock
witness_lock_error
witness_lockless_error
witness_lock_depth_error
witness_not_owner_error
witness_owner
witness_owner_error

23
include/jemalloc/internal/witness.h
View File

@ -91,10 +91,12 @@ extern witness_not_owner_error_t *witness_not_owner_error;
void witness_not_owner_error(const witness_t *witness);
#endif
#ifdef JEMALLOC_JET
typedef void (witness_lockless_error_t)(const witness_list_t *);
extern witness_lockless_error_t *witness_lockless_error;
typedef void (witness_lock_depth_error_t)(const witness_list_t *,
unsigned depth);
extern witness_lock_depth_error_t *witness_lock_depth_error;
#else
void witness_lockless_error(const witness_list_t *witnesses);
void witness_lock_depth_error(const witness_list_t *witnesses,
unsigned depth);
#endif
void witnesses_cleanup(tsd_t *tsd);
@ -111,7 +113,7 @@ void witness_postfork_child(tsd_t *tsd);
bool witness_owner(tsd_t *tsd, const witness_t *witness);
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
void witness_assert_lockless(tsdn_t *tsdn);
void witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth);
void witness_lock(tsdn_t *tsdn, witness_t *witness);
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
#endif
@ -175,9 +177,10 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
}
JEMALLOC_INLINE void
witness_assert_lockless(tsdn_t *tsdn)
witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth)
{
tsd_t *tsd;
unsigned d;
witness_list_t *witnesses;
witness_t *w;
@ -188,10 +191,16 @@ witness_assert_lockless(tsdn_t *tsdn)
return;
tsd = tsdn_tsd(tsdn);
d = 0;
witnesses = tsd_witnessesp_get(tsd);
w = ql_last(witnesses, link);
if (w != NULL)
witness_lockless_error(witnesses);
if (w != NULL) {
ql_foreach(w, witnesses, link) {
d++;
}
}
if (d != depth)
witness_lock_depth_error(witnesses, depth);
}
JEMALLOC_INLINE void

74
src/jemalloc.c
View File

@ -1581,7 +1581,7 @@ ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
tsd = tsd_fetch();
*tsdn = tsd_tsdn(tsd);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
ind = size2index(size);
if (unlikely(ind >= NSIZES))
@ -1619,7 +1619,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
assert(usize == isalloc(tsdn, ret, config_prof));
*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
}
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
}
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
@ -1704,7 +1704,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
goto label_oom;
}
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
if (size == 0)
size = 1;
@ -1745,7 +1745,7 @@ label_return:
UTRACE(0, size, result);
JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize,
false);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
return (ret);
label_oom:
assert(result == NULL);
@ -1755,7 +1755,7 @@ label_oom:
abort();
}
ret = ENOMEM;
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
goto label_return;
}
@ -1873,7 +1873,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
size_t usize;
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
@ -1901,7 +1901,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
{
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
@ -1947,7 +1947,7 @@ je_realloc(void *ptr, size_t size)
malloc_thread_init();
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
if (config_valgrind && unlikely(in_valgrind)) {
@ -1994,7 +1994,7 @@ je_realloc(void *ptr, size_t size)
UTRACE(ptr, size, ret);
JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr,
old_usize, old_rzsize, maybe, false);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
return (ret);
}
@ -2005,12 +2005,12 @@ je_free(void *ptr)
UTRACE(ptr, 0, 0);
if (likely(ptr != NULL)) {
tsd_t *tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
if (likely(!malloc_slow))
ifree(tsd, ptr, tcache_get(tsd, false), false);
else
ifree(tsd, ptr, tcache_get(tsd, false), true);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
}
}
@ -2239,7 +2239,7 @@ imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
tsd = tsd_fetch();
*tsdn = tsd_tsdn(tsd);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
if (likely(flags == 0)) {
szind_t ind = size2index(size);
@ -2374,7 +2374,7 @@ je_rallocx(void *ptr, size_t size, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
malloc_thread_init();
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
@ -2421,7 +2421,7 @@ je_rallocx(void *ptr, size_t size, int flags)
UTRACE(ptr, size, p);
JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr,
old_usize, old_rzsize, no, zero);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
return (p);
label_oom:
if (config_xmalloc && unlikely(opt_xmalloc)) {
@ -2429,7 +2429,7 @@ label_oom:
abort();
}
UTRACE(ptr, size, 0);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
return (NULL);
}
@ -2525,7 +2525,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
malloc_thread_init();
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
@ -2566,7 +2566,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
old_usize, old_rzsize, no, zero);
label_not_resized:
UTRACE(ptr, size, ptr);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
return (usize);
}
@ -2581,14 +2581,14 @@ je_sallocx(const void *ptr, int flags)
malloc_thread_init();
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
if (config_ivsalloc)
usize = ivsalloc(tsdn, ptr, config_prof);
else
usize = isalloc(tsdn, ptr, config_prof);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
return (usize);
}
@ -2602,7 +2602,7 @@ je_dallocx(void *ptr, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
tcache = NULL;
@ -2616,7 +2616,7 @@ je_dallocx(void *ptr, int flags)
ifree(tsd, ptr, tcache, false);
else
ifree(tsd, ptr, tcache, true);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
}
JEMALLOC_ALWAYS_INLINE_C size_t
@ -2624,13 +2624,13 @@ inallocx(tsdn_t *tsdn, size_t size, int flags)
{
size_t usize;
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0))
usize = s2u(size);
else
usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
return (usize);
}
@ -2647,7 +2647,7 @@ je_sdallocx(void *ptr, size_t size, int flags)
usize = inallocx(tsd_tsdn(tsd), size, flags);
assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
tcache = NULL;
@ -2661,7 +2661,7 @@ je_sdallocx(void *ptr, size_t size, int flags)
isfree(tsd, ptr, usize, tcache, false);
else
isfree(tsd, ptr, usize, tcache, true);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
}
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@ -2677,13 +2677,13 @@ je_nallocx(size_t size, int flags)
return (0);
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
usize = inallocx(tsdn, size, flags);
if (unlikely(usize > HUGE_MAXCLASS))
return (0);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
return (usize);
}
@ -2698,9 +2698,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
return (EAGAIN);
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
return (ret);
}
@ -2714,9 +2714,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
return (EAGAIN);
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
ret = ctl_nametomib(tsdn, name, mibp, miblenp);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
return (ret);
}
@ -2731,9 +2731,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
return (EAGAIN);
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
return (ret);
}
@ -2744,9 +2744,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
tsdn_t *tsdn;
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
stats_print(write_cb, cbopaque, opts);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
}
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@ -2759,14 +2759,14 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
malloc_thread_init();
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
if (config_ivsalloc)
ret = ivsalloc(tsdn, ptr, config_prof);
else
ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
return (ret);
}

19
src/witness.c
View File

@ -71,15 +71,16 @@ witness_not_owner_error_t *witness_not_owner_error =
#endif
#ifdef JEMALLOC_JET
#undef witness_lockless_error
#define witness_lockless_error JEMALLOC_N(n_witness_lockless_error)
#undef witness_lock_depth_error
#define witness_lock_depth_error JEMALLOC_N(n_witness_lock_depth_error)
#endif
void
witness_lockless_error(const witness_list_t *witnesses)
witness_lock_depth_error(const witness_list_t *witnesses, unsigned depth)
{
witness_t *w;
malloc_printf("<jemalloc>: Should not own any locks:");
malloc_printf("<jemalloc>: Should own %u lock%s:", depth, (depth != 1) ?
"s" : "");
ql_foreach(w, witnesses, link) {
malloc_printf(" %s(%u)", w->name, w->rank);
}
@ -87,17 +88,17 @@ witness_lockless_error(const witness_list_t *witnesses)
abort();
}
#ifdef JEMALLOC_JET
#undef witness_lockless_error
#define witness_lockless_error JEMALLOC_N(witness_lockless_error)
witness_lockless_error_t *witness_lockless_error =
JEMALLOC_N(n_witness_lockless_error);
#undef witness_lock_depth_error
#define witness_lock_depth_error JEMALLOC_N(witness_lock_depth_error)
witness_lock_depth_error_t *witness_lock_depth_error =
JEMALLOC_N(n_witness_lock_depth_error);
#endif
void
witnesses_cleanup(tsd_t *tsd)
{
witness_assert_lockless(tsd_tsdn(tsd));
witness_assert_lock_depth(tsd_tsdn(tsd), 0);
/* Do nothing. */
}

63
test/unit/witness.c
View File

@ -3,12 +3,12 @@
static witness_lock_error_t *witness_lock_error_orig;
static witness_owner_error_t *witness_owner_error_orig;
static witness_not_owner_error_t *witness_not_owner_error_orig;
static witness_lockless_error_t *witness_lockless_error_orig;
static witness_lock_depth_error_t *witness_lock_depth_error_orig;
static bool saw_lock_error;
static bool saw_owner_error;
static bool saw_not_owner_error;
static bool saw_lockless_error;
static bool saw_lock_depth_error;
static void
witness_lock_error_intercept(const witness_list_t *witnesses,
@ -33,10 +33,11 @@ witness_not_owner_error_intercept(const witness_t *witness)
}
static void
witness_lockless_error_intercept(const witness_list_t *witnesses)
witness_lock_depth_error_intercept(const witness_list_t *witnesses,
unsigned depth)
{
saw_lockless_error = true;
saw_lock_depth_error = true;
}
static int
@ -66,22 +67,25 @@ TEST_BEGIN(test_witness)
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_init(&a, "a", 1, NULL);
witness_assert_not_owner(tsdn, &a);
witness_lock(tsdn, &a);
witness_assert_owner(tsdn, &a);
witness_assert_lock_depth(tsdn, 1);
witness_init(&b, "b", 2, NULL);
witness_assert_not_owner(tsdn, &b);
witness_lock(tsdn, &b);
witness_assert_owner(tsdn, &b);
witness_assert_lock_depth(tsdn, 2);
witness_unlock(tsdn, &a);
witness_assert_lock_depth(tsdn, 1);
witness_unlock(tsdn, &b);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
}
TEST_END
@ -94,18 +98,21 @@ TEST_BEGIN(test_witness_comp)
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_init(&a, "a", 1, witness_comp);
witness_assert_not_owner(tsdn, &a);
witness_lock(tsdn, &a);
witness_assert_owner(tsdn, &a);
witness_assert_lock_depth(tsdn, 1);
witness_init(&b, "b", 1, witness_comp);
witness_assert_not_owner(tsdn, &b);
witness_lock(tsdn, &b);
witness_assert_owner(tsdn, &b);
witness_assert_lock_depth(tsdn, 2);
witness_unlock(tsdn, &b);
witness_assert_lock_depth(tsdn, 1);
witness_lock_error_orig = witness_lock_error;
witness_lock_error = witness_lock_error_intercept;
@ -117,6 +124,7 @@ TEST_BEGIN(test_witness_comp)
witness_lock(tsdn, &c);
assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &c);
witness_assert_lock_depth(tsdn, 1);
saw_lock_error = false;
@ -126,10 +134,11 @@ TEST_BEGIN(test_witness_comp)
witness_lock(tsdn, &d);
assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &d);
witness_assert_lock_depth(tsdn, 1);
witness_unlock(tsdn, &a);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_lock_error = witness_lock_error_orig;
}
@ -148,20 +157,22 @@ TEST_BEGIN(test_witness_reversal)
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_init(&a, "a", 1, NULL);
witness_init(&b, "b", 2, NULL);
witness_lock(tsdn, &b);
witness_assert_lock_depth(tsdn, 1);
assert_false(saw_lock_error, "Unexpected witness lock error");
witness_lock(tsdn, &a);
assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &a);
witness_assert_lock_depth(tsdn, 1);
witness_unlock(tsdn, &b);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_lock_error = witness_lock_error_orig;
}
@ -184,7 +195,7 @@ TEST_BEGIN(test_witness_recursive)
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_init(&a, "a", 1, NULL);
@ -197,7 +208,7 @@ TEST_BEGIN(test_witness_recursive)
witness_unlock(tsdn, &a);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_owner_error = witness_owner_error_orig;
witness_lock_error = witness_lock_error_orig;
@ -218,7 +229,7 @@ TEST_BEGIN(test_witness_unlock_not_owned)
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_init(&a, "a", 1, NULL);
@ -226,41 +237,41 @@ TEST_BEGIN(test_witness_unlock_not_owned)
witness_unlock(tsdn, &a);
assert_true(saw_owner_error, "Expected owner error");
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_owner_error = witness_owner_error_orig;
}
TEST_END
TEST_BEGIN(test_witness_lockful)
TEST_BEGIN(test_witness_lock_depth)
{
witness_t a;
tsdn_t *tsdn;
test_skip_if(!config_debug);
witness_lockless_error_orig = witness_lockless_error;
witness_lockless_error = witness_lockless_error_intercept;
saw_lockless_error = false;
witness_lock_depth_error_orig = witness_lock_depth_error;
witness_lock_depth_error = witness_lock_depth_error_intercept;
saw_lock_depth_error = false;
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_init(&a, "a", 1, NULL);
assert_false(saw_lockless_error, "Unexpected lockless error");
witness_assert_lockless(tsdn);
assert_false(saw_lock_depth_error, "Unexpected lock_depth error");
witness_assert_lock_depth(tsdn, 0);
witness_lock(tsdn, &a);
witness_assert_lockless(tsdn);
assert_true(saw_lockless_error, "Expected lockless error");
witness_assert_lock_depth(tsdn, 0);
assert_true(saw_lock_depth_error, "Expected lock_depth error");
witness_unlock(tsdn, &a);
witness_assert_lockless(tsdn);
witness_assert_lock_depth(tsdn, 0);
witness_lockless_error = witness_lockless_error_orig;
witness_lock_depth_error = witness_lock_depth_error_orig;
}
TEST_END
@ -274,5 +285,5 @@ main(void)
test_witness_reversal,
test_witness_recursive,
test_witness_unlock_not_owned,
test_witness_lockful));
test_witness_lock_depth));
}