diff --git a/include/jemalloc/internal/emap.h b/include/jemalloc/internal/emap.h index 3e397483..afb4983c 100644 --- a/include/jemalloc/internal/emap.h +++ b/include/jemalloc/internal/emap.h @@ -139,6 +139,9 @@ emap_assert_not_mapped(tsdn_t *tsdn, emap_t *emap, edata_t *edata) { static inline void emap_update_rtree_at_addr(tsdn_t *tsdn, rtree_t *rtree, edata_t *expected_edata, uintptr_t addr, extent_state_t state) { + witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE); + rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index 66dcf664..4cebb6e1 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -243,26 +243,13 @@ witness_assert_not_owner(witness_tsdn_t *witness_tsdn, } } -static inline void -witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn, - witness_rank_t rank_inclusive, unsigned depth) { - witness_tsd_t *witness_tsd; - unsigned d; - witness_list_t *witnesses; - witness_t *w; +/* Returns depth. Not intended for direct use. */ +static inline unsigned +witness_depth_to_rank(witness_list_t *witnesses, witness_rank_t rank_inclusive) +{ + unsigned d = 0; + witness_t *w = ql_last(witnesses, link); - if (!config_debug) { - return; - } - - if (witness_tsdn_null(witness_tsdn)) { - return; - } - witness_tsd = witness_tsdn_tsd(witness_tsdn); - - d = 0; - witnesses = &witness_tsd->witnesses; - w = ql_last(witnesses, link); if (w != NULL) { ql_reverse_foreach(w, witnesses, link) { if (w->rank < rank_inclusive) { @@ -271,6 +258,20 @@ witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn, d++; } } + + return d; +} + +static inline void +witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn, + witness_rank_t rank_inclusive, unsigned depth) { + if (!config_debug || witness_tsdn_null(witness_tsdn)) { + return; + } + + witness_list_t *witnesses = &witness_tsdn_tsd(witness_tsdn)->witnesses; + unsigned d = witness_depth_to_rank(witnesses, rank_inclusive); + if (d != depth) { witness_depth_error(witnesses, rank_inclusive, depth); } @@ -286,6 +287,21 @@ witness_assert_lockless(witness_tsdn_t *witness_tsdn) { witness_assert_depth(witness_tsdn, 0); } +static inline void +witness_assert_positive_depth_to_rank(witness_tsdn_t *witness_tsdn, + witness_rank_t rank_inclusive) { + if (!config_debug || witness_tsdn_null(witness_tsdn)) { + return; + } + + witness_list_t *witnesses = &witness_tsdn_tsd(witness_tsdn)->witnesses; + unsigned d = witness_depth_to_rank(witnesses, rank_inclusive); + + if (d == 0) { + witness_depth_error(witnesses, rank_inclusive, 1); + } +} + static inline void witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) { witness_tsd_t *witness_tsd;