project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extract profiling code from [re]allocation functions.

Browse Source 
Extract profiling code from malloc(), imemalign(), calloc(), realloc(),
mallocx(), rallocx(), and xallocx().  This slightly reduces the amount
of code compiled into the fast paths, but the primary benefit is the
combinatorial complexity reduction.

Simplify iralloc[t]() by creating a separate ixalloc() that handles the
no-move cases.

Further simplify [mrxn]allocx() (and by implication [mrn]allocm()) to
make request size overflows due to size class and/or alignment
constraints trigger undefined behavior (detected by debug-only
assertions).

Report ENOMEM rather than EINVAL if an OOM occurs during heap profiling
backtrace creation in imemalign().  This bug impacted posix_memalign()
and aligned_alloc().
This commit is contained in:
Jason Evans 2014-01-12 15:05:44 -08:00
parent 6b694c4d47
commit b2c31660be
10 changed files with 553 additions and 497 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
doc/jemalloc.xml.in
View File

@ -321,14 +321,16 @@
<para>The <function>mallocx<parameter/></function> function allocates at
least <parameter>size</parameter> bytes of memory, and returns a pointer
to the base address of the allocation. Behavior is undefined if
<parameter>size</parameter> is <constant>0</constant>.</para>
<parameter>size</parameter> is <constant>0</constant>, or if request size
overflows due to size class and/or alignment constraints.</para>
<para>The <function>rallocx<parameter/></function> function resizes the
allocation at <parameter>ptr</parameter> to be at least
<parameter>size</parameter> bytes, and returns a pointer to the base
address of the resulting allocation, which may or may not have moved from
its original location. Behavior is undefined if
<parameter>size</parameter> is <constant>0</constant>.</para>
<parameter>size</parameter> is <constant>0</constant>, or if request size
overflows due to size class and/or alignment constraints.</para>
<para>The <function>xallocx<parameter/></function> function resizes the
allocation at <parameter>ptr</parameter> in place to be at least
@ -355,8 +357,9 @@
<function>mallocx<parameter/></function> function, and returns the real
size of the allocation that would result from the equivalent
<function>mallocx<parameter/></function> function call. Behavior is
undefined if <parameter>size</parameter> is
<constant>0</constant>.</para>
undefined if <parameter>size</parameter> is <constant>0</constant>, or if
request size overflows due to size class and/or alignment
constraints.</para>
<para>The <function>mallctl<parameter/></function> function provides a
general interface for introspecting the memory allocator, as well as
@ -518,8 +521,9 @@ for (i = 0; i < nbins; i++) {
<parameter>*ptr</parameter> to the base address of the allocation, and
sets <parameter>*rsize</parameter> to the real size of the allocation if
<parameter>rsize</parameter> is not <constant>NULL</constant>. Behavior
is undefined if <parameter>size</parameter> is
<constant>0</constant>.</para>
is undefined if <parameter>size</parameter> is <constant>0</constant>, or
if request size overflows due to size class and/or alignment
constraints.</para>
<para>The <function>rallocm<parameter/></function> function resizes the
allocation at <parameter>*ptr</parameter> to be at least
@ -532,8 +536,9 @@ for (i = 0; i < nbins; i++) {
language="C">(<parameter>size</parameter> +
<parameter>extra</parameter>)</code> bytes, though inability to allocate
the extra byte(s) will not by itself result in failure. Behavior is
undefined if <parameter>size</parameter> is <constant>0</constant>, or if
<code language="C">(<parameter>size</parameter> +
undefined if <parameter>size</parameter> is <constant>0</constant>, if
request size overflows due to size class and/or alignment constraints, or
if <code language="C">(<parameter>size</parameter> +
<parameter>extra</parameter> &gt;
<constant>SIZE_T_MAX</constant>)</code>.</para>
@ -550,8 +555,9 @@ for (i = 0; i < nbins; i++) {
<parameter>rsize</parameter> is not <constant>NULL</constant> it sets
<parameter>*rsize</parameter> to the real size of the allocation that
would result from the equivalent <function>allocm<parameter/></function>
function call. Behavior is undefined if
<parameter>size</parameter> is <constant>0</constant>.</para>
function call. Behavior is undefined if <parameter>size</parameter> is
<constant>0</constant>, or if request size overflows due to size class
and/or alignment constraints.</para>
</refsect2>
</refsect1>
<refsect1 id="tuning">

2
include/jemalloc/internal/arena.h
View File

@ -436,7 +436,7 @@ void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t);
extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
#endif
void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero);
void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,

2
include/jemalloc/internal/huge.h
View File

@ -19,7 +19,7 @@ extern malloc_mutex_t huge_mtx;
void *huge_malloc(size_t size, bool zero);
void *huge_palloc(size_t size, size_t alignment, bool zero);
void *huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra);
void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_dalloc);

116
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -747,11 +747,15 @@ void idalloct(void *ptr, bool try_tcache);
void idalloc(void *ptr);
void iqalloct(void *ptr, bool try_tcache);
void iqalloc(void *ptr);
void *iralloct(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc,
void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena);
void *iralloct(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena);
void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero, bool no_move);
bool zero);
bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero);
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
#endif
@ -920,10 +924,42 @@ iqalloc(void *ptr)
}
JEMALLOC_ALWAYS_INLINE void *
iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena)
{
void *p;
size_t usize, copysize;
usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
if (p == NULL) {
if (extra == 0)
return (NULL);
/* Try again, without extra this time. */
usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
if (p == NULL)
return (NULL);
}
/*
* Copy at most size bytes (not size+extra), since the caller has no
* expectation that the extra bytes will be reliably preserved.
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize);
iqalloct(ptr, try_tcache_dalloc);
return (p);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
{
void *ret;
size_t oldsize;
assert(ptr != NULL);
@ -933,50 +969,14 @@ iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
!= 0) {
size_t usize, copysize;
/*
* Existing object alignment is inadequate; allocate new space
* and copy.
*/
if (no_move)
return (NULL);
usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena);
if (ret == NULL) {
if (extra == 0)
return (NULL);
/* Try again, without extra this time. */
usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
ret = ipalloct(usize, alignment, zero, try_tcache_alloc,
arena);
if (ret == NULL)
return (NULL);
}
/*
* Copy at most size bytes (not size+extra), since the caller
* has no expectation that the extra bytes will be reliably
* preserved.
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(ret, ptr, copysize);
iqalloct(ptr, try_tcache_dalloc);
return (ret);
return (iralloct_realign(ptr, oldsize, size, extra, alignment,
zero, try_tcache_alloc, try_tcache_dalloc, arena));
}
if (no_move) {
if (size <= arena_maxclass) {
return (arena_ralloc_no_move(ptr, oldsize, size,
extra, zero));
} else {
return (huge_ralloc_no_move(ptr, oldsize, size,
extra));
}
} else {
if (size + extra <= arena_maxclass) {
return (arena_ralloc(arena, ptr, oldsize, size, extra,
alignment, zero, try_tcache_alloc,
@ -986,15 +986,33 @@ iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
alignment, zero, try_tcache_dalloc));
}
}
}
JEMALLOC_ALWAYS_INLINE void *
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
bool no_move)
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
{
return (iralloct(ptr, size, extra, alignment, zero, no_move, true, true,
NULL));
return (iralloct(ptr, size, extra, alignment, zero, true, true, NULL));
}
JEMALLOC_ALWAYS_INLINE bool
ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero)
{
size_t oldsize;
assert(ptr != NULL);
assert(size != 0);
oldsize = isalloc(ptr, config_prof);
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
!= 0) {
/* Existing object alignment is inadequate. */
return (true);
}
if (size <= arena_maxclass)
return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
else
return (huge_ralloc_no_move(ptr, oldsize, size, extra));
}
malloc_tsd_externs(thread_allocated, thread_allocated_t)

2
include/jemalloc/internal/private_symbols.txt
View File

@ -223,9 +223,11 @@ iqalloc
iqalloct
iralloc
iralloct
iralloct_realign
isalloc
isthreaded
ivsalloc
ixalloc
jemalloc_postfork_child
jemalloc_postfork_parent
jemalloc_prefork

13
src/arena.c
View File

@ -2061,7 +2061,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra,
}
}
void *
bool
arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
bool zero)
{
@ -2077,19 +2077,19 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
SMALL_SIZE2BIN(size + extra) ==
SMALL_SIZE2BIN(oldsize)) || (size <= oldsize &&
size + extra >= oldsize))
return (ptr);
return (false);
} else {
assert(size <= arena_maxclass);
if (size + extra > SMALL_MAXCLASS) {
if (arena_ralloc_large(ptr, oldsize, size,
extra, zero) == false)
return (ptr);
return (false);
}
}
}
/* Reallocation would require a move. */
return (NULL);
return (true);
}
void *
@ -2101,9 +2101,8 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
size_t copysize;
/* Try to avoid moving the allocation. */
ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero);
if (ret != NULL)
return (ret);
if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false)
return (ptr);
/*
* size and oldsize are different enough that we need to move the

11
src/huge.c
View File

@ -78,7 +78,7 @@ huge_palloc(size_t size, size_t alignment, bool zero)
return (ret);
}
void *
bool
huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
{
@ -89,11 +89,11 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
&& CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
assert(CHUNK_CEILING(oldsize) == oldsize);
return (ptr);
return (false);
}
/* Reallocation would require a move. */
return (NULL);
return (true);
}
void *
@ -104,9 +104,8 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t copysize;
/* Try to avoid moving the allocation. */
ret = huge_ralloc_no_move(ptr, oldsize, size, extra);
if (ret != NULL)
return (ret);
if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false)
return (ptr);
/*
* size and oldsize are different enough that we need to use a

775
src/jemalloc.c
View File

File diff suppressed because it is too large Load Diff

33
test/integration/allocm.c
View File

@ -44,23 +44,6 @@ TEST_BEGIN(test_alignment_errors)
void *p;
size_t nsz, rsz, sz, alignment;
#if LG_SIZEOF_PTR == 3
alignment = UINT64_C(0x8000000000000000);
sz = UINT64_C(0x8000000000000000);
#else
alignment = 0x80000000LU;
sz = 0x80000000LU;
#endif
nsz = 0;
assert_d_ne(nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)), ALLOCM_SUCCESS,
"Expected error for nallocm(&nsz, %zu, %#x)",
sz, ALLOCM_ALIGN(alignment));
rsz = 0;
assert_d_ne(allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)),
ALLOCM_SUCCESS, "Expected error for allocm(&p, %zu, %#x)",
sz, ALLOCM_ALIGN(alignment));
assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch");
#if LG_SIZEOF_PTR == 3
alignment = UINT64_C(0x4000000000000000);
sz = UINT64_C(0x8400000000000001);
@ -75,22 +58,6 @@ TEST_BEGIN(test_alignment_errors)
assert_d_ne(allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)),
ALLOCM_SUCCESS, "Expected error for allocm(&p, %zu, %#x)",
sz, ALLOCM_ALIGN(alignment));
alignment = 0x10LU;
#if LG_SIZEOF_PTR == 3
sz = UINT64_C(0xfffffffffffffff0);
#else
sz = 0xfffffff0LU;
#endif
nsz = 0;
assert_d_ne(nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)), ALLOCM_SUCCESS,
"Expected error for nallocm(&nsz, %zu, %#x)",
sz, ALLOCM_ALIGN(alignment));
rsz = 0;
assert_d_ne(allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)),
ALLOCM_SUCCESS, "Expected error for allocm(&p, %zu, %#x)",
sz, ALLOCM_ALIGN(alignment));
assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch");
}
TEST_END

30
test/integration/mallocx.c
View File

@ -39,20 +39,6 @@ TEST_BEGIN(test_alignment_errors)
void *p;
size_t nsz, sz, alignment;
#if LG_SIZEOF_PTR == 3
alignment = UINT64_C(0x8000000000000000);
sz = UINT64_C(0x8000000000000000);
#else
alignment = 0x80000000LU;
sz = 0x80000000LU;
#endif
nsz = nallocx(sz, MALLOCX_ALIGN(alignment));
assert_zu_eq(nsz, 0, "Expected error for nallocx(%zu, %#x)", sz,
MALLOCX_ALIGN(alignment));
p = mallocx(sz, MALLOCX_ALIGN(alignment));
assert_ptr_null(p, "Expected error for mallocx(%zu, %#x)", sz,
MALLOCX_ALIGN(alignment));
#if LG_SIZEOF_PTR == 3
alignment = UINT64_C(0x4000000000000000);
sz = UINT64_C(0x8400000000000001);
@ -65,22 +51,6 @@ TEST_BEGIN(test_alignment_errors)
p = mallocx(sz, MALLOCX_ALIGN(alignment));
assert_ptr_null(p, "Expected error for mallocx(%zu, %#x)", sz,
MALLOCX_ALIGN(alignment));
alignment = 0x10LU;
#if LG_SIZEOF_PTR == 3
sz = UINT64_C(0xfffffffffffffff0);
#else
sz = 0xfffffff0LU;
#endif
nsz = nallocx(sz, MALLOCX_ALIGN(alignment));
assert_zu_eq(nsz, 0, "Expected error for nallocx(%zu, %#x)", sz,
MALLOCX_ALIGN(alignment));
nsz = nallocx(sz, MALLOCX_ALIGN(alignment));
assert_zu_eq(nsz, 0, "Expected error for nallocx(%zu, %#x)", sz,
MALLOCX_ALIGN(alignment));
p = mallocx(sz, MALLOCX_ALIGN(alignment));
assert_ptr_null(p, "Expected error for mallocx(%zu, %#x)", sz,
MALLOCX_ALIGN(alignment));
}
TEST_END