project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge realloc and rallocx pathways.

Browse Source
This commit is contained in:
David T. Goldblatt 2019-10-21 18:44:42 -07:00 committed by David Goldblatt
parent bd6e28d6a3
commit ee961c2310
1 changed files with 76 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

253
src/jemalloc.c
View File

@ -2506,56 +2506,6 @@ je_calloc(size_t num, size_t size) {
return ret; return ret;
} }
static void *
irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) {
void *p;
if (tctx == NULL) {
return NULL;
}
if (usize <= SC_SMALL_MAXCLASS) {
p = iralloc(tsd, old_ptr, old_usize,
SC_LARGE_MINCLASS, 0, false, hook_args);
if (p == NULL) {
return NULL;
}
arena_prof_promote(tsd_tsdn(tsd), p, usize);
} else {
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
hook_args);
}
return p;
}
JEMALLOC_ALWAYS_INLINE void *
irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) {
void *p;
bool prof_active;
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx);
tctx = prof_alloc_prep(tsd, usize, prof_active, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx,
hook_args);
} else {
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false,
hook_args);
}
if (unlikely(p == NULL)) {
prof_alloc_rollback(tsd, tctx, true);
return NULL;
}
prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
old_tctx);
return p;
}
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
if (!slow_path) { if (!slow_path) {
@ -2645,121 +2595,6 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
} }
} }
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_realloc(void *ptr, size_t arg_size) {
void *ret;
tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL);
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
size_t old_usize = 0;
size_t size = arg_size;
LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
if (unlikely(size == 0)) {
if (ptr != NULL) {
/* realloc(ptr, 0) is equivalent to free(ptr). */
UTRACE(ptr, 0, 0);
tcache_t *tcache;
tsd_t *tsd = tsd_fetch();
if (tsd_reentrancy_level_get(tsd) == 0) {
tcache = tcache_get(tsd);
} else {
tcache = NULL;
}
uintptr_t args[3] = {(uintptr_t)ptr, size};
hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
ifree(tsd, ptr, tcache, true);
LOG("core.realloc.exit", "result: %p", NULL);
return NULL;
}
size = 1;
}
if (likely(ptr != NULL)) {
assert(malloc_initialized() || IS_INITIALIZER);
tsd_t *tsd = tsd_fetch();
check_entry_exit_locking(tsd_tsdn(tsd));
hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr,
(uintptr_t)arg_size, 0, 0}};
alloc_ctx_t alloc_ctx;
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab);
assert(alloc_ctx.szind != SC_NSIZES);
old_usize = sz_index2size(alloc_ctx.szind);
assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
usize = sz_s2u(size);
if (config_prof && opt_prof) {
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
ret = NULL;
} else {
ret = irealloc_prof(tsd, ptr, old_usize, usize,
&alloc_ctx, &hook_args);
}
} else {
ret = iralloc(tsd, ptr, old_usize, size, 0, false,
&hook_args);
}
tsdn = tsd_tsdn(tsd);
} else {
/* realloc(NULL, size) is equivalent to malloc(size). */
static_opts_t sopts;
dynamic_opts_t dopts;
static_opts_init(&sopts);
dynamic_opts_init(&dopts);
sopts.null_out_result_on_error = true;
sopts.set_errno_on_error = true;
sopts.oom_string =
"<jemalloc>: Error in realloc(): out of memory\n";
dopts.result = &ret;
dopts.num_items = 1;
dopts.item_size = size;
imalloc(&sopts, &dopts);
if (sopts.slow) {
uintptr_t args[3] = {(uintptr_t)ptr, arg_size};
hook_invoke_alloc(hook_alloc_realloc, ret,
(uintptr_t)ret, args);
}
return ret;
}
if (unlikely(ret == NULL)) {
if (config_xmalloc && unlikely(opt_xmalloc)) {
malloc_write("<jemalloc>: Error in realloc(): "
"out of memory\n");
abort();
}
set_errno(ENOMEM);
}
if (likely(ret != NULL)) {
tsd_t *tsd;
assert(usize == isalloc(tsdn, ret));
tsd = tsdn_tsd(tsdn);
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
UTRACE(ptr, size, ret);
check_entry_exit_locking(tsdn);
LOG("core.realloc.exit", "result: %p", ret);
return ret;
}
JEMALLOC_NOINLINE JEMALLOC_NOINLINE
void void
free_default(void *ptr) { free_default(void *ptr) {
@ -3201,10 +3036,8 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
return p; return p;
} }
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN JEMALLOC_ALWAYS_INLINE void *
void JEMALLOC_NOTHROW * do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
JEMALLOC_ALLOC_SIZE(2)
je_rallocx(void *ptr, size_t size, int flags) {
void *p; void *p;
tsd_t *tsd; tsd_t *tsd;
size_t usize; size_t usize;
@ -3214,10 +3047,6 @@ je_rallocx(void *ptr, size_t size, int flags) {
arena_t *arena; arena_t *arena;
tcache_t *tcache; tcache_t *tcache;
LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
size, flags);
assert(ptr != NULL); assert(ptr != NULL);
assert(size != 0); assert(size != 0);
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
@ -3252,8 +3081,8 @@ je_rallocx(void *ptr, size_t size, int flags) {
old_usize = sz_index2size(alloc_ctx.szind); old_usize = sz_index2size(alloc_ctx.szind);
assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr));
hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags, hook_ralloc_args_t hook_args = {is_realloc, {(uintptr_t)ptr, size,
0}}; flags, 0}};
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = (alignment == 0) ? usize = (alignment == 0) ?
sz_s2u(size) : sz_sa2u(size, alignment); sz_s2u(size) : sz_sa2u(size, alignment);
@ -3281,7 +3110,6 @@ je_rallocx(void *ptr, size_t size, int flags) {
UTRACE(ptr, size, p); UTRACE(ptr, size, p);
check_entry_exit_locking(tsd_tsdn(tsd)); check_entry_exit_locking(tsd_tsdn(tsd));
LOG("core.rallocx.exit", "result: %p", p);
return p; return p;
label_oom: label_oom:
if (config_xmalloc && unlikely(opt_xmalloc)) { if (config_xmalloc && unlikely(opt_xmalloc)) {
@ -3291,10 +3119,81 @@ label_oom:
UTRACE(ptr, size, 0); UTRACE(ptr, size, 0);
check_entry_exit_locking(tsd_tsdn(tsd)); check_entry_exit_locking(tsd_tsdn(tsd));
LOG("core.rallocx.exit", "result: %p", NULL);
return NULL; return NULL;
} }
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_rallocx(void *ptr, size_t size, int flags) {
LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr,
size, flags);
void *ret = do_rallocx(ptr, size, flags, false);
LOG("core.rallocx.exit", "result: %p", ret);
return ret;
}
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
void JEMALLOC_NOTHROW *
JEMALLOC_ALLOC_SIZE(2)
je_realloc(void *ptr, size_t size) {
LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
if (likely(ptr != NULL && size != 0)) {
void *ret = do_rallocx(ptr, size, 0, true);
LOG("core.realloc.exit", "result: %p", ret);
return ret;
} else if (ptr != NULL && size == 0) {
/* realloc(ptr, 0) is equivalent to free(ptr). */
UTRACE(ptr, 0, 0);
tcache_t *tcache;
tsd_t *tsd = tsd_fetch();
check_entry_exit_locking(tsd_tsdn(tsd));
if (tsd_reentrancy_level_get(tsd) == 0) {
tcache = tcache_get(tsd);
} else {
tcache = NULL;
}
uintptr_t args[3] = {(uintptr_t)ptr, size};
hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
ifree(tsd, ptr, tcache, true);
check_entry_exit_locking(tsd_tsdn(tsd));
LOG("core.realloc.exit", "result: %p", NULL);
return NULL;
} else {
/* realloc(NULL, size) is equivalent to malloc(size). */
void *ret;
static_opts_t sopts;
dynamic_opts_t dopts;
static_opts_init(&sopts);
dynamic_opts_init(&dopts);
sopts.null_out_result_on_error = true;
sopts.set_errno_on_error = true;
sopts.oom_string =
"<jemalloc>: Error in realloc(): out of memory\n";
dopts.result = &ret;
dopts.num_items = 1;
dopts.item_size = size;
imalloc(&sopts, &dopts);
if (sopts.slow) {
uintptr_t args[3] = {(uintptr_t)ptr, size};
hook_invoke_alloc(hook_alloc_realloc, ret,
(uintptr_t)ret, args);
}
LOG("core.realloc.exit", "result: %p", ret);
return ret;
}
}
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
size_t extra, size_t alignment, bool zero) { size_t extra, size_t alignment, bool zero) {