project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Always adjust thread_(de)allocated

Browse Source
This commit is contained in:
Yinan Zhang 2019-08-22 15:56:47 -07:00
parent 57b81c078e
commit 49e6fbce78
1 changed files with 43 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
src/jemalloc.c
View File

@ -2031,16 +2031,14 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
/* Filled in by compute_size_with_overflow below. */ /* Filled in by compute_size_with_overflow below. */
size_t size = 0; size_t size = 0;
/* /*
* For unaligned allocations, we need only ind. For aligned * The zero initialization for ind is actually dead store, in that its
* allocations, or in case of stats or profiling we need usize. * value is reset before any branch on its value is taken. Sometimes
* * though, it's convenient to pass it as arguments before this point.
* These are actually dead stores, in that their values are reset before * To avoid undefined behavior then, we initialize it with dummy stores.
* any branch on their value is taken. Sometimes though, it's
* convenient to pass them as arguments before this point. To avoid
* undefined behavior then, we initialize them with dummy stores.
*/ */
szind_t ind = 0; szind_t ind = 0;
size_t usize = 0; /* usize will always be properly initialized. */
size_t usize;
/* Reentrancy is only checked on slow path. */ /* Reentrancy is only checked on slow path. */
int8_t reentrancy_level; int8_t reentrancy_level;
@ -2063,12 +2061,9 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
if (unlikely(ind >= SC_NSIZES)) { if (unlikely(ind >= SC_NSIZES)) {
goto label_oom; goto label_oom;
} }
if (config_stats || (config_prof && opt_prof) || sopts->usize) {
usize = sz_index2size(ind); usize = sz_index2size(ind);
assert(usize > 0 && usize <= SC_LARGE_MAXCLASS);
dopts->usize = usize; dopts->usize = usize;
assert(usize > 0 && usize
<= SC_LARGE_MAXCLASS);
}
} else { } else {
if (sopts->bump_empty_aligned_alloc) { if (sopts->bump_empty_aligned_alloc) {
if (unlikely(size == 0)) { if (unlikely(size == 0)) {
@ -2077,8 +2072,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
} }
usize = sz_sa2u(size, dopts->alignment); usize = sz_sa2u(size, dopts->alignment);
dopts->usize = usize; dopts->usize = usize;
if (unlikely(usize == 0 if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
|| usize > SC_LARGE_MAXCLASS)) {
goto label_oom; goto label_oom;
} }
} }
@ -2107,26 +2101,23 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
dopts->arena_ind = 0; dopts->arena_ind = 0;
} }
/*
* If dopts->alignment > 0, then ind is still 0, but usize was computed
* in the previous if statement. Down the positive alignment path,
* imalloc_no_sample and imalloc_sample will ignore ind.
*/
/* If profiling is on, get our profiling context. */ /* If profiling is on, get our profiling context. */
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
/*
* Note that if we're going down this path, usize must have been
* initialized in the previous if statement.
*/
prof_tctx_t *tctx = prof_alloc_prep( prof_tctx_t *tctx = prof_alloc_prep(
tsd, usize, prof_active_get_unlocked(), true); tsd, usize, prof_active_get_unlocked(), true);
alloc_ctx_t alloc_ctx; alloc_ctx_t alloc_ctx;
if (likely((uintptr_t)tctx == (uintptr_t)1U)) { if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
alloc_ctx.slab = (usize alloc_ctx.slab = (usize <= SC_SMALL_MAXCLASS);
<= SC_SMALL_MAXCLASS);
allocation = imalloc_no_sample( allocation = imalloc_no_sample(
sopts, dopts, tsd, usize, usize, ind); sopts, dopts, tsd, usize, usize, ind);
} else if ((uintptr_t)tctx > (uintptr_t)1U) { } else if ((uintptr_t)tctx > (uintptr_t)1U) {
/*
* Note that ind might still be 0 here. This is fine;
* imalloc_sample ignores ind if dopts->alignment > 0.
*/
allocation = imalloc_sample( allocation = imalloc_sample(
sopts, dopts, tsd, usize, ind); sopts, dopts, tsd, usize, ind);
alloc_ctx.slab = false; alloc_ctx.slab = false;
@ -2140,12 +2131,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
} }
prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx); prof_malloc(tsd_tsdn(tsd), allocation, usize, &alloc_ctx, tctx);
} else { } else {
/*
* If dopts->alignment > 0, then ind is still 0, but usize was
* computed in the previous if statement. Down the positive
* alignment path, imalloc_no_sample ignores ind and size
* (relying only on usize).
*/
allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize, allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
ind); ind);
if (unlikely(allocation == NULL)) { if (unlikely(allocation == NULL)) {
@ -2160,10 +2145,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
assert(dopts->alignment == 0 assert(dopts->alignment == 0
|| ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0)); || ((uintptr_t)allocation & (dopts->alignment - 1)) == ZU(0));
if (config_stats) {
assert(usize == isalloc(tsd_tsdn(tsd), allocation)); assert(usize == isalloc(tsd_tsdn(tsd), allocation));
*tsd_thread_allocatedp_get(tsd) += usize; *tsd_thread_allocatedp_get(tsd) += usize;
}
if (sopts->slow) { if (sopts->slow) {
UTRACE(0, size, allocation); UTRACE(0, size, allocation);
@ -2339,11 +2322,12 @@ je_malloc(size_t size) {
} }
szind_t ind = sz_size2index_lookup(size); szind_t ind = sz_size2index_lookup(size);
size_t usize; /* usize is always needed to increment thread_allocated. */
if (config_stats || config_prof) { size_t usize = sz_index2size(ind);
usize = sz_index2size(ind); /*
} * Fast path relies on size being a bin.
/* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */ * I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS
*/
assert(ind < SC_NBINS); assert(ind < SC_NBINS);
assert(size <= SC_SMALL_MAXCLASS); assert(size <= SC_SMALL_MAXCLASS);
@ -2373,8 +2357,8 @@ je_malloc(size_t size) {
void *ret = cache_bin_alloc_easy(bin, &tcache_success, ind); void *ret = cache_bin_alloc_easy(bin, &tcache_success, ind);
if (tcache_success) { if (tcache_success) {
if (config_stats) {
*tsd_thread_allocatedp_get(tsd) += usize; *tsd_thread_allocatedp_get(tsd) += usize;
if (config_stats) {
bin->tstats.nrequests++; bin->tstats.nrequests++;
} }
if (config_prof) { if (config_prof) {
@ -2573,16 +2557,11 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES); assert(alloc_ctx.szind != SC_NSIZES);
size_t usize; size_t usize = sz_index2size(alloc_ctx.szind);
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = sz_index2size(alloc_ctx.szind);
prof_free(tsd, ptr, usize, &alloc_ctx); prof_free(tsd, ptr, usize, &alloc_ctx);
} else if (config_stats) {
usize = sz_index2size(alloc_ctx.szind);
} }
if (config_stats) {
*tsd_thread_deallocatedp_get(tsd) += usize; *tsd_thread_deallocatedp_get(tsd) += usize;
}
if (likely(!slow_path)) { if (likely(!slow_path)) {
idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false, idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
@ -2638,9 +2617,8 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
prof_free(tsd, ptr, usize, ctx); prof_free(tsd, ptr, usize, ctx);
} }
if (config_stats) {
*tsd_thread_deallocatedp_get(tsd) += usize; *tsd_thread_deallocatedp_get(tsd) += usize;
}
if (likely(!slow_path)) { if (likely(!slow_path)) {
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false); isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
@ -2701,19 +2679,15 @@ je_realloc(void *ptr, size_t arg_size) {
assert(alloc_ctx.szind != SC_NSIZES); assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind); old_usize = sz_index2size(alloc_ctx.szind);
assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
if (config_prof && opt_prof) {
usize = sz_s2u(size); usize = sz_s2u(size);
if (unlikely(usize == 0 if (config_prof && opt_prof) {
|| usize > SC_LARGE_MAXCLASS)) { if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
ret = NULL; ret = NULL;
} else { } else {
ret = irealloc_prof(tsd, ptr, old_usize, usize, ret = irealloc_prof(tsd, ptr, old_usize, usize,
&alloc_ctx, &hook_args); &alloc_ctx, &hook_args);
} }
} else { } else {
if (config_stats) {
usize = sz_s2u(size);
}
ret = iralloc(tsd, ptr, old_usize, size, 0, false, ret = iralloc(tsd, ptr, old_usize, size, 0, false,
&hook_args); &hook_args);
} }
@ -2753,7 +2727,7 @@ je_realloc(void *ptr, size_t arg_size) {
} }
set_errno(ENOMEM); set_errno(ENOMEM);
} }
if (config_stats && likely(ret != NULL)) { if (likely(ret != NULL)) {
tsd_t *tsd; tsd_t *tsd;
assert(usize == isalloc(tsdn, ret)); assert(usize == isalloc(tsdn, ret));
@ -2852,10 +2826,8 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
return false; return false;
} }
if (config_stats) {
size_t usize = sz_index2size(alloc_ctx.szind); size_t usize = sz_index2size(alloc_ctx.szind);
*tsd_thread_deallocatedp_get(tsd) += usize; *tsd_thread_deallocatedp_get(tsd) += usize;
}
return true; return true;
} }
@ -3267,8 +3239,7 @@ je_rallocx(void *ptr, size_t size, int flags) {
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = (alignment == 0) ? usize = (alignment == 0) ?
sz_s2u(size) : sz_sa2u(size, alignment); sz_s2u(size) : sz_sa2u(size, alignment);
if (unlikely(usize == 0 if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
|| usize > SC_LARGE_MAXCLASS)) {
goto label_oom; goto label_oom;
} }
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
@ -3282,16 +3253,13 @@ je_rallocx(void *ptr, size_t size, int flags) {
if (unlikely(p == NULL)) { if (unlikely(p == NULL)) {
goto label_oom; goto label_oom;
} }
if (config_stats) {
usize = isalloc(tsd_tsdn(tsd), p); usize = isalloc(tsd_tsdn(tsd), p);
} }
}
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
if (config_stats) {
*tsd_thread_allocatedp_get(tsd) += usize; *tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize; *tsd_thread_deallocatedp_get(tsd) += old_usize;
}
UTRACE(ptr, size, p); UTRACE(ptr, size, p);
check_entry_exit_locking(tsd_tsdn(tsd)); check_entry_exit_locking(tsd_tsdn(tsd));
@ -3439,10 +3407,9 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
goto label_not_resized; goto label_not_resized;
} }
if (config_stats) {
*tsd_thread_allocatedp_get(tsd) += usize; *tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize; *tsd_thread_deallocatedp_get(tsd) += old_usize;
}
label_not_resized: label_not_resized:
if (unlikely(!tsd_fast(tsd))) { if (unlikely(!tsd_fast(tsd))) {
uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};