project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Witness assertions: only assert locklessness when non-reentrant.

Browse Source 
Previously we could still hit these assertions down error paths or in the
extended API.
This commit is contained in:
David Goldblatt 2017-05-31 09:43:43 -07:00 committed by David Goldblatt
parent 508f54b02b
commit fa35463d56
1 changed files with 62 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
src/jemalloc.c
View File

@ -627,6 +627,31 @@ stats_print_atexit(void) {
je_malloc_stats_print(NULL, NULL, opt_stats_print_opts); je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
} }
/*
* Ensure that we don't hold any locks upon entry to or exit from allocator
* code (in a "broad" sense that doesn't count a reentrant allocation as an
* entrance or exit).
*/
JEMALLOC_ALWAYS_INLINE void
check_entry_exit_locking(tsdn_t *tsdn) {
if (!config_debug) {
return;
}
if (tsdn_null(tsdn)) {
return;
}
tsd_t *tsd = tsdn_tsd(tsdn);
/*
* It's possible we hold locks at entry/exit if we're in a nested
* allocation.
*/
int8_t reentrancy_level = tsd_reentrancy_level_get(tsd);
if (reentrancy_level != 0) {
return;
}
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn));
}
/* /*
* End miscellaneous support functions. * End miscellaneous support functions.
*/ */
@ -1705,15 +1730,13 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
} }
} }
check_entry_exit_locking(tsd_tsdn(tsd));
/* /*
* If we need to handle reentrancy, we can do it out of a * If we need to handle reentrancy, we can do it out of a
* known-initialized arena (i.e. arena 0). * known-initialized arena (i.e. arena 0).
*/ */
reentrancy_level = tsd_reentrancy_level_get(tsd); reentrancy_level = tsd_reentrancy_level_get(tsd);
if (reentrancy_level == 0) {
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
}
if (sopts->slow && unlikely(reentrancy_level > 0)) { if (sopts->slow && unlikely(reentrancy_level > 0)) {
/* /*
* We should never specify particular arenas or tcaches from * We should never specify particular arenas or tcaches from
@ -1788,9 +1811,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
} }
/* Success! */ /* Success! */
if (reentrancy_level == 0) { check_entry_exit_locking(tsd_tsdn(tsd));
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd)));
}
*dopts->result = allocation; *dopts->result = allocation;
return 0; return 0;
@ -1804,7 +1825,7 @@ label_oom:
UTRACE(NULL, size, NULL); UTRACE(NULL, size, NULL);
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
if (sopts->set_errno_on_error) { if (sopts->set_errno_on_error) {
set_errno(ENOMEM); set_errno(ENOMEM);
@ -1835,7 +1856,7 @@ label_invalid_alignment:
UTRACE(NULL, size, NULL); UTRACE(NULL, size, NULL);
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
if (sopts->null_out_result_on_error) { if (sopts->null_out_result_on_error) {
*dopts->result = NULL; *dopts->result = NULL;
@ -2036,9 +2057,8 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
if (!slow_path) { if (!slow_path) {
tsd_assert_fast(tsd); tsd_assert_fast(tsd);
} }
if (tsd_reentrancy_level_get(tsd) == 0) { check_entry_exit_locking(tsd_tsdn(tsd));
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); if (tsd_reentrancy_level_get(tsd) != 0) {
} else {
assert(slow_path); assert(slow_path);
} }
@ -2076,9 +2096,8 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
if (!slow_path) { if (!slow_path) {
tsd_assert_fast(tsd); tsd_assert_fast(tsd);
} }
if (tsd_reentrancy_level_get(tsd) == 0) { check_entry_exit_locking(tsd_tsdn(tsd));
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); if (tsd_reentrancy_level_get(tsd) != 0) {
} else {
assert(slow_path); assert(slow_path);
} }
@ -2138,7 +2157,7 @@ je_realloc(void *ptr, size_t size) {
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
@ -2181,7 +2200,7 @@ je_realloc(void *ptr, size_t size) {
*tsd_thread_deallocatedp_get(tsd) += old_usize; *tsd_thread_deallocatedp_get(tsd) += old_usize;
} }
UTRACE(ptr, size, ret); UTRACE(ptr, size, ret);
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
return ret; return ret;
} }
@ -2190,10 +2209,7 @@ je_free(void *ptr) {
UTRACE(ptr, 0, 0); UTRACE(ptr, 0, 0);
if (likely(ptr != NULL)) { if (likely(ptr != NULL)) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
if (tsd_reentrancy_level_get(tsd) == 0) { check_entry_exit_locking(tsd_tsdn(tsd));
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(
tsd)));
}
tcache_t *tcache; tcache_t *tcache;
if (likely(tsd_fast(tsd))) { if (likely(tsd_fast(tsd))) {
@ -2209,10 +2225,7 @@ je_free(void *ptr) {
} }
ifree(tsd, ptr, tcache, true); ifree(tsd, ptr, tcache, true);
} }
if (tsd_reentrancy_level_get(tsd) == 0) { check_entry_exit_locking(tsd_tsdn(tsd));
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(
tsd)));
}
} }
} }
@ -2472,7 +2485,7 @@ je_rallocx(void *ptr, size_t size, int flags) {
assert(size != 0); assert(size != 0);
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags); unsigned arena_ind = MALLOCX_ARENA_GET(flags);
@ -2529,7 +2542,7 @@ je_rallocx(void *ptr, size_t size, int flags) {
*tsd_thread_deallocatedp_get(tsd) += old_usize; *tsd_thread_deallocatedp_get(tsd) += old_usize;
} }
UTRACE(ptr, size, p); UTRACE(ptr, size, p);
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
return p; return p;
label_oom: label_oom:
if (config_xmalloc && unlikely(opt_xmalloc)) { if (config_xmalloc && unlikely(opt_xmalloc)) {
@ -2537,7 +2550,7 @@ label_oom:
abort(); abort();
} }
UTRACE(ptr, size, 0); UTRACE(ptr, size, 0);
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
return NULL; return NULL;
} }
@ -2629,7 +2642,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
assert(SIZE_T_MAX - size >= extra); assert(SIZE_T_MAX - size >= extra);
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
@ -2672,7 +2685,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
} }
label_not_resized: label_not_resized:
UTRACE(ptr, size, ptr); UTRACE(ptr, size, ptr);
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
return usize; return usize;
} }
@ -2686,7 +2699,7 @@ je_sallocx(const void *ptr, int flags) {
assert(ptr != NULL); assert(ptr != NULL);
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
if (config_debug || force_ivsalloc) { if (config_debug || force_ivsalloc) {
usize = ivsalloc(tsdn, ptr); usize = ivsalloc(tsdn, ptr);
@ -2695,7 +2708,7 @@ je_sallocx(const void *ptr, int flags) {
usize = isalloc(tsdn, ptr); usize = isalloc(tsdn, ptr);
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
return usize; return usize;
} }
@ -2706,7 +2719,7 @@ je_dallocx(void *ptr, int flags) {
tsd_t *tsd = tsd_fetch(); tsd_t *tsd = tsd_fetch();
bool fast = tsd_fast(tsd); bool fast = tsd_fast(tsd);
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
tcache_t *tcache; tcache_t *tcache;
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
@ -2737,12 +2750,12 @@ je_dallocx(void *ptr, int flags) {
} else { } else {
ifree(tsd, ptr, tcache, true); ifree(tsd, ptr, tcache, true);
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
} }
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
inallocx(tsdn_t *tsdn, size_t size, int flags) { inallocx(tsdn_t *tsdn, size_t size, int flags) {
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
size_t usize; size_t usize;
if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) { if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) {
@ -2750,7 +2763,7 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) {
} else { } else {
usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); usize = sz_sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
return usize; return usize;
} }
@ -2763,7 +2776,7 @@ je_sdallocx(void *ptr, size_t size, int flags) {
bool fast = tsd_fast(tsd); bool fast = tsd_fast(tsd);
size_t usize = inallocx(tsd_tsdn(tsd), size, flags); size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
assert(usize == isalloc(tsd_tsdn(tsd), ptr)); assert(usize == isalloc(tsd_tsdn(tsd), ptr));
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
tcache_t *tcache; tcache_t *tcache;
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
@ -2794,7 +2807,7 @@ je_sdallocx(void *ptr, size_t size, int flags) {
} else { } else {
isfree(tsd, ptr, usize, tcache, true); isfree(tsd, ptr, usize, tcache, true);
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
} }
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@ -2810,14 +2823,14 @@ je_nallocx(size_t size, int flags) {
} }
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
usize = inallocx(tsdn, size, flags); usize = inallocx(tsdn, size, flags);
if (unlikely(usize > LARGE_MAXCLASS)) { if (unlikely(usize > LARGE_MAXCLASS)) {
return 0; return 0;
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
return usize; return usize;
} }
@ -2832,9 +2845,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
} }
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
return ret; return ret;
} }
@ -2848,9 +2861,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) {
} }
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
ret = ctl_nametomib(tsdn, name, mibp, miblenp); ret = ctl_nametomib(tsdn, name, mibp, miblenp);
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
return ret; return ret;
} }
@ -2865,9 +2878,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
} }
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); check_entry_exit_locking(tsd_tsdn(tsd));
return ret; return ret;
} }
@ -2877,9 +2890,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
tsdn_t *tsdn; tsdn_t *tsdn;
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
stats_print(write_cb, cbopaque, opts); stats_print(write_cb, cbopaque, opts);
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
} }
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@ -2890,7 +2903,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsdn = tsdn_fetch(); tsdn = tsdn_fetch();
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
if (unlikely(ptr == NULL)) { if (unlikely(ptr == NULL)) {
ret = 0; ret = 0;
@ -2903,7 +2916,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
} }
} }
witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); check_entry_exit_locking(tsdn);
return ret; return ret;
} }