diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h index 854fb1e2..24ea4162 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -146,7 +146,10 @@ tcache_get(tsd_t *tsd) { } static inline void -pre_reentrancy(tsd_t *tsd) { +pre_reentrancy(tsd_t *tsd, arena_t *arena) { + /* arena is the current context. Reentry from a0 is not allowed. */ + assert(arena != arena_get(tsd_tsdn(tsd), 0, false)); + bool fast = tsd_fast(tsd); ++*tsd_reentrancy_levelp_get(tsd); if (fast) { diff --git a/src/arena.c b/src/arena.c index 4e3bd6c1..b2830866 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2051,7 +2051,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { * is done enough that we should have tsd. */ assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + pre_reentrancy(tsdn_tsd(tsdn), arena); if (hooks_arena_new_hook) { hooks_arena_new_hook(); } diff --git a/src/background_thread.c b/src/background_thread.c index ab076fee..cc5100d1 100644 --- a/src/background_thread.c +++ b/src/background_thread.c @@ -316,7 +316,7 @@ background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) { &background_thread_lock); } - pre_reentrancy(tsd); + pre_reentrancy(tsd, NULL); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); bool has_thread; assert(info->state != background_thread_paused); @@ -408,7 +408,7 @@ label_restart: */ malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); - pre_reentrancy(tsd); + pre_reentrancy(tsd, NULL); int err = background_thread_create_signals_masked(&info->thread, NULL, background_thread_entry, (void *)(uintptr_t)i); post_reentrancy(tsd); @@ -557,7 +557,7 @@ background_thread_create(tsd_t *tsd, unsigned arena_ind) { return false; } - pre_reentrancy(tsd); + pre_reentrancy(tsd, NULL); /* * To avoid complications (besides reentrancy), create internal * background threads with the underlying pthread_create. diff --git a/src/base.c b/src/base.c index 22c94338..97078b13 100644 --- a/src/base.c +++ b/src/base.c @@ -25,11 +25,12 @@ base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) if (extent_hooks == &extent_hooks_default) { addr = extent_alloc_mmap(NULL, size, PAGE, &zero, &commit); } else { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + /* No arena context as we are creating new arenas. */ + tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); + pre_reentrancy(tsd, NULL); addr = extent_hooks->alloc(extent_hooks, NULL, size, PAGE, &zero, &commit, ind); - post_reentrancy(tsdn_tsd(tsdn)); + post_reentrancy(tsd); } return addr; @@ -64,8 +65,8 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, /* Nothing worked. This should never happen. */ not_reached(); } else { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); + pre_reentrancy(tsd, NULL); if (extent_hooks->dalloc != NULL && !extent_hooks->dalloc(extent_hooks, addr, size, true, ind)) { @@ -88,7 +89,7 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, } /* Nothing worked. That's the application's problem. */ label_done: - post_reentrancy(tsdn_tsd(tsdn)); + post_reentrancy(tsd); return; } } diff --git a/src/extent.c b/src/extent.c index 4b66dd88..fa45c84d 100644 --- a/src/extent.c +++ b/src/extent.c @@ -1025,6 +1025,18 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, alignment, zero, commit); } +static void +extent_hook_pre_reentrancy(tsdn_t *tsdn, arena_t *arena) { + tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); + pre_reentrancy(tsd, arena); +} + +static void +extent_hook_post_reentrancy(tsdn_t *tsdn) { + tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); + post_reentrancy(tsd); +} + /* * If virtual memory is retained, create increasingly larger extents from which * to split requested extents in order to limit the total number of disjoint @@ -1073,12 +1085,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, &zeroed, &committed, (dss_prec_t)atomic_load_u( &arena->dss_prec, ATOMIC_RELAXED)); } else { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); ptr = (*r_extent_hooks)->alloc(*r_extent_hooks, NULL, alloc_size, PAGE, &zeroed, &committed, arena_ind_get(arena)); - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } extent_init(extent, arena, ptr, alloc_size, false, NSIZES, @@ -1250,11 +1261,10 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, alignment, zero, commit); } else { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, esize, alignment, zero, commit, arena_ind_get(arena)); - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } if (addr == NULL) { extent_dalloc(tsdn, arena, extent); @@ -1492,13 +1502,12 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, err = extent_dalloc_default_impl(extent_base_get(extent), extent_size_get(extent)); } else { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); err = ((*r_extent_hooks)->dalloc == NULL || (*r_extent_hooks)->dalloc(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), extent_committed_get(extent), arena_ind_get(arena))); - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } if (!err) { @@ -1525,8 +1534,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_reregister(tsdn, extent); if (*r_extent_hooks != &extent_hooks_default) { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); } /* Try to decommit; purge if that fails. */ bool zeroed; @@ -1550,7 +1558,7 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, zeroed = false; } if (*r_extent_hooks != &extent_hooks_default) { - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } extent_zeroed_set(extent, zeroed); @@ -1595,12 +1603,11 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_destroy_default_impl(extent_base_get(extent), extent_size_get(extent)); } else if ((*r_extent_hooks)->destroy != NULL) { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); (*r_extent_hooks)->destroy(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), extent_committed_get(extent), arena_ind_get(arena)); - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } extent_dalloc(tsdn, arena, extent); @@ -1622,14 +1629,13 @@ extent_commit_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); if (*r_extent_hooks != &extent_hooks_default) { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); } bool err = ((*r_extent_hooks)->commit == NULL || (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena))); if (*r_extent_hooks != &extent_hooks_default) { - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } extent_committed_set(extent, extent_committed_get(extent) || !err); return err; @@ -1660,15 +1666,14 @@ extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); if (*r_extent_hooks != &extent_hooks_default) { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); } bool err = ((*r_extent_hooks)->decommit == NULL || (*r_extent_hooks)->decommit(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena))); if (*r_extent_hooks != &extent_hooks_default) { - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } extent_committed_set(extent, extent_committed_get(extent) && err); return err; @@ -1700,16 +1705,14 @@ extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, if ((*r_extent_hooks)->purge_lazy == NULL) { return true; } - if (*r_extent_hooks != &extent_hooks_default) { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); } bool err = (*r_extent_hooks)->purge_lazy(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena)); if (*r_extent_hooks != &extent_hooks_default) { - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } return err; @@ -1750,14 +1753,13 @@ extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, return true; } if (*r_extent_hooks != &extent_hooks_default) { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); } bool err = (*r_extent_hooks)->purge_forced(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena)); if (*r_extent_hooks != &extent_hooks_default) { - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } return err; } @@ -1829,14 +1831,13 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_lock2(tsdn, extent, trail); if (*r_extent_hooks != &extent_hooks_default) { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); } bool err = (*r_extent_hooks)->split(*r_extent_hooks, extent_base_get(extent), size_a + size_b, size_a, size_b, extent_committed_get(extent), arena_ind_get(arena)); if (*r_extent_hooks != &extent_hooks_default) { - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } if (err) { goto label_error_c; @@ -1908,13 +1909,12 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, err = extent_merge_default_impl(extent_base_get(a), extent_base_get(b)); } else { - assert(!tsdn_null(tsdn)); - pre_reentrancy(tsdn_tsd(tsdn)); + extent_hook_pre_reentrancy(tsdn, arena); err = (*r_extent_hooks)->merge(*r_extent_hooks, extent_base_get(a), extent_size_get(a), extent_base_get(b), extent_size_get(b), extent_committed_get(a), arena_ind_get(arena)); - post_reentrancy(tsdn_tsd(tsdn)); + extent_hook_post_reentrancy(tsdn); } if (err) { diff --git a/src/jemalloc.c b/src/jemalloc.c index aa79633c..7bf23105 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1476,7 +1476,7 @@ malloc_init_hard(void) { malloc_mutex_lock(tsd_tsdn(tsd), &init_lock); /* Set reentrancy level to 1 during init. */ - pre_reentrancy(tsd); + pre_reentrancy(tsd, NULL); /* Initialize narenas before prof_boot2 (for allocation). */ if (malloc_init_narenas() || background_thread_boot1(tsd_tsdn(tsd))) { UNLOCK_RETURN(tsd_tsdn(tsd), true, true) diff --git a/src/prof.c b/src/prof.c index 61dfa2ce..975722c4 100644 --- a/src/prof.c +++ b/src/prof.c @@ -1633,7 +1633,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, return true; } - pre_reentrancy(tsd); + pre_reentrancy(tsd, NULL); malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx); prof_gctx_tree_t gctxs;