Implement Valgrind support, redzones, and quarantine.
Implement Valgrind support, as well as the redzone and quarantine features, which help Valgrind detect memory errors. Redzones are only implemented for small objects because the changes necessary to support redzones around large and huge objects are complicated by in-place reallocation, to the point that it isn't clear that the maintenance burden is worth the incremental improvement to Valgrind support. Merge arena_salloc() and arena_salloc_demote(). Refactor i[v]salloc() to expose the 'demote' option.
This commit is contained in:
Jason Evans
parent
a1ee7838e1
commit
122449b073
8
INSTALL
8
INSTALL
@ -113,8 +113,12 @@ any of the following arguments (not a definitive list) to 'configure':
|
||||
mmap(2).
|
||||
|
||||
--disable-fill
|
||||
Disable support for junk/zero filling of memory. See the "opt.junk"/
|
||||
"opt.zero" option documentation for usage details.
|
||||
Disable support for junk/zero filling of memory, quarantine, and redzones.
|
||||
See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option
|
||||
documentation for usage details.
|
||||
|
||||
--disable-valgrind
|
||||
Disable support for Valgrind.
|
||||
|
||||
--disable-experimental
|
||||
Disable support for the experimental API (*allocm()).
|
||||
|
||||
@ -49,9 +49,9 @@ CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/atomic.c \
|
||||
@srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \
|
||||
@srcroot@src/ckh.c @srcroot@src/ctl.c @srcroot@src/extent.c \
|
||||
@srcroot@src/hash.c @srcroot@src/huge.c @srcroot@src/mb.c \
|
||||
@srcroot@src/mutex.c @srcroot@src/prof.c @srcroot@src/rtree.c \
|
||||
@srcroot@src/stats.c @srcroot@src/tcache.c @srcroot@src/util.c \
|
||||
@srcroot@src/tsd.c
|
||||
@srcroot@src/mutex.c @srcroot@src/prof.c @srcroot@src/quarantine.c \
|
||||
@srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c \
|
||||
@srcroot@src/util.c @srcroot@src/tsd.c
|
||||
ifeq (macho, @abi@)
|
||||
CSRCS += @srcroot@src/zone.c
|
||||
endif
|
||||
|
||||
38
configure.ac
38
configure.ac
@ -685,7 +685,8 @@ AC_SUBST([enable_dss])
|
||||
|
||||
dnl Support the junk/zero filling option by default.
|
||||
AC_ARG_ENABLE([fill],
|
||||
[AS_HELP_STRING([--disable-fill], [Disable support for junk/zero filling])],
|
||||
[AS_HELP_STRING([--disable-fill],
|
||||
[Disable support for junk/zero filling, quarantine, and redzones])],
|
||||
[if test "x$enable_fill" = "xno" ; then
|
||||
enable_fill="0"
|
||||
else
|
||||
@ -727,6 +728,38 @@ if test "x$enable_utrace" = "x1" ; then
|
||||
fi
|
||||
AC_SUBST([enable_utrace])
|
||||
|
||||
dnl Support Valgrind by default.
|
||||
AC_ARG_ENABLE([valgrind],
|
||||
[AS_HELP_STRING([--disable-valgrind], [Disable support for Valgrind])],
|
||||
[if test "x$enable_valgrind" = "xno" ; then
|
||||
enable_valgrind="0"
|
||||
else
|
||||
enable_valgrind="1"
|
||||
fi
|
||||
],
|
||||
[enable_valgrind="1"]
|
||||
)
|
||||
if test "x$enable_valgrind" = "x1" ; then
|
||||
JE_COMPILABLE([valgrind], [
|
||||
#include <valgrind/valgrind.h>
|
||||
#include <valgrind/memcheck.h>
|
||||
|
||||
#if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__) \
|
||||
&& (__VALGRIND_MAJOR__ > 3 || (__VALGRIND_MAJOR__ == 3 && \
|
||||
__VALGRIND_MINOR__ >= 6))
|
||||
#else
|
||||
# error "Incompatible Valgrind version"
|
||||
#endif
|
||||
], [], [je_cv_valgrind])
|
||||
if test "x${je_cv_valgrind}" = "xno" ; then
|
||||
enable_valgrind="0"
|
||||
fi
|
||||
if test "x$enable_valgrind" = "x1" ; then
|
||||
AC_DEFINE([JEMALLOC_VALGRIND], [ ])
|
||||
fi
|
||||
fi
|
||||
AC_SUBST([enable_valgrind])
|
||||
|
||||
dnl Do not support the xmalloc option by default.
|
||||
AC_ARG_ENABLE([xmalloc],
|
||||
[AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])],
|
||||
@ -1088,8 +1121,9 @@ AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}])
|
||||
AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}])
|
||||
AC_MSG_RESULT([tcache : ${enable_tcache}])
|
||||
AC_MSG_RESULT([fill : ${enable_fill}])
|
||||
AC_MSG_RESULT([xmalloc : ${enable_xmalloc}])
|
||||
AC_MSG_RESULT([utrace : ${enable_utrace}])
|
||||
AC_MSG_RESULT([valgrind : ${enable_valgrind}])
|
||||
AC_MSG_RESULT([xmalloc : ${enable_xmalloc}])
|
||||
AC_MSG_RESULT([dss : ${enable_dss}])
|
||||
AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}])
|
||||
AC_MSG_RESULT([tls : ${enable_tls}])
|
||||
|
||||
@ -720,6 +720,16 @@ for (i = 0; i < nbins; i++) {
|
||||
build configuration.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry>
|
||||
<term>
|
||||
<mallctl>config.valgrind</mallctl>
|
||||
(<type>bool</type>)
|
||||
<literal>r-</literal>
|
||||
</term>
|
||||
<listitem><para><option>--enable-valgrind</option> was specified during
|
||||
build configuration.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry>
|
||||
<term>
|
||||
<mallctl>config.xmalloc</mallctl>
|
||||
@ -819,6 +829,47 @@ for (i = 0; i < nbins; i++) {
|
||||
configuration, in which case it is enabled by default.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.quarantine">
|
||||
<term>
|
||||
<mallctl>opt.quarantine</mallctl>
|
||||
(<type>size_t</type>)
|
||||
<literal>r-</literal>
|
||||
[<option>--enable-fill</option>]
|
||||
</term>
|
||||
<listitem><para>Per thread quarantine size in bytes. If non-zero, each
|
||||
thread maintains a FIFO object quarantine that stores up to the
|
||||
specified number of bytes of memory. The quarantined memory is not
|
||||
freed until it is released from quarantine, though it is immediately
|
||||
junk-filled if the <link
|
||||
linkend="opt.junk"><mallctl>opt.junk</mallctl></link> option is
|
||||
enabled. This feature is of particular use in combination with <ulink
|
||||
url="http://http://valgrind.org/">Valgrind</ulink>, which can detect
|
||||
attempts to access quarantined objects. This is intended for debugging
|
||||
and will impact performance negatively. The default quarantine size is
|
||||
0.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.redzone">
|
||||
<term>
|
||||
<mallctl>opt.redzone</mallctl>
|
||||
(<type>bool</type>)
|
||||
<literal>r-</literal>
|
||||
[<option>--enable-fill</option>]
|
||||
</term>
|
||||
<listitem><para>Redzones enabled/disabled. If enabled, small
|
||||
allocations have redzones before and after them. Furthermore, if the
|
||||
<link linkend="opt.junk"><mallctl>opt.junk</mallctl></link> option is
|
||||
enabled, the redzones are checked for corruption during deallocation.
|
||||
However, the primary intended purpose of this feature is to be used in
|
||||
combination with <ulink
|
||||
url="http://http://valgrind.org/">Valgrind</ulink>, which needs
|
||||
redzones in order to do effective buffer overflow/underflow detection.
|
||||
This option is intended for debugging and will impact performance
|
||||
negatively. This option is disabled by default unless
|
||||
<option>--enable-debug</option> is specified during configuration, in
|
||||
which case it is enabled by default.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.zero">
|
||||
<term>
|
||||
<mallctl>opt.zero</mallctl>
|
||||
@ -849,6 +900,25 @@ for (i = 0; i < nbins; i++) {
|
||||
is disabled by default.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.valgrind">
|
||||
<term>
|
||||
<mallctl>opt.valgrind</mallctl>
|
||||
(<type>bool</type>)
|
||||
<literal>r-</literal>
|
||||
[<option>--enable-valgrind</option>]
|
||||
</term>
|
||||
<listitem><para><ulink
|
||||
url="http://http://valgrind.org/">Valgrind</ulink> support
|
||||
enabled/disabled. If enabled, several other options are automatically
|
||||
modified during options processing to work well with Valgrind: <link
|
||||
linkend="opt.junk"><mallctl>opt.junk</mallctl></link> and <link
|
||||
linkend="opt.zero"><mallctl>opt.zero</mallctl></link> are set to false,
|
||||
<link linkend="opt.quarantine"><mallctl>opt.quarantine</mallctl></link>
|
||||
is set to 16 MiB, and <link
|
||||
linkend="opt.redzone"><mallctl>opt.redzone</mallctl></link> is set to
|
||||
true. This option is disabled by default.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.xmalloc">
|
||||
<term>
|
||||
<mallctl>opt.xmalloc</mallctl>
|
||||
@ -1764,10 +1834,11 @@ malloc_conf = "xmalloc:true";]]></programlisting>
|
||||
|
||||
<para>This implementation does not provide much detail about the problems
|
||||
it detects, because the performance impact for storing such information
|
||||
would be prohibitive. There are a number of allocator implementations
|
||||
available on the Internet which focus on detecting and pinpointing problems
|
||||
by trading performance for extra sanity checks and detailed
|
||||
diagnostics.</para>
|
||||
would be prohibitive. However, jemalloc does integrate with the most
|
||||
excellent <ulink url="http://http://valgrind.org/">Valgrind</ulink> tool if
|
||||
the <option>--enable-valgrind</option> configuration option is enabled and
|
||||
the <link linkend="opt.valgrind"><mallctl>opt.valgrind</mallctl></link>
|
||||
option is enabled.</para>
|
||||
</refsect1>
|
||||
<refsect1 id="diagnostic_messages">
|
||||
<title>DIAGNOSTIC MESSAGES</title>
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
* constraint is relaxed (ignored) for runs that are so small that the
|
||||
* per-region overhead is greater than:
|
||||
*
|
||||
* (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
|
||||
* (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP))
|
||||
*/
|
||||
#define RUN_BFP 12
|
||||
/* \/ Implicit binary fixed point. */
|
||||
@ -27,6 +27,12 @@
|
||||
#define LG_RUN_MAXREGS 11
|
||||
#define RUN_MAXREGS (1U << LG_RUN_MAXREGS)
|
||||
|
||||
/*
|
||||
* Minimum redzone size. Redzones may be larger than this if necessary to
|
||||
* preserve region alignment.
|
||||
*/
|
||||
#define REDZONE_MINSIZE 16
|
||||
|
||||
/*
|
||||
* The minimum ratio of active:dirty pages per arena is computed as:
|
||||
*
|
||||
@ -192,11 +198,50 @@ struct arena_run_s {
|
||||
* Read-only information associated with each element of arena_t's bins array
|
||||
* is stored separately, partly to reduce memory usage (only one copy, rather
|
||||
* than one per arena), but mainly to avoid false cacheline sharing.
|
||||
*
|
||||
* Each run has the following layout:
|
||||
*
|
||||
* /--------------------\
|
||||
* | arena_run_t header |
|
||||
* | ... |
|
||||
* bitmap_offset | bitmap |
|
||||
* | ... |
|
||||
* ctx0_offset | ctx map |
|
||||
* | ... |
|
||||
* |--------------------|
|
||||
* | redzone |
|
||||
* reg0_offset | region 0 |
|
||||
* | redzone |
|
||||
* |--------------------| \
|
||||
* | redzone | |
|
||||
* | region 1 | > reg_interval
|
||||
* | redzone | /
|
||||
* |--------------------|
|
||||
* | ... |
|
||||
* | ... |
|
||||
* | ... |
|
||||
* |--------------------|
|
||||
* | redzone |
|
||||
* | region nregs-1 |
|
||||
* | redzone |
|
||||
* |--------------------|
|
||||
* | alignment pad? |
|
||||
* \--------------------/
|
||||
*
|
||||
* reg_interval has at least the same minimum alignment as reg_size; this
|
||||
* preserves the alignment constraint that sa2u() depends on. Alignment pad is
|
||||
* either 0 or redzone_size; it is present only if needed to align reg0_offset.
|
||||
*/
|
||||
struct arena_bin_info_s {
|
||||
/* Size of regions in a run for this bin's size class. */
|
||||
size_t reg_size;
|
||||
|
||||
/* Redzone size. */
|
||||
size_t redzone_size;
|
||||
|
||||
/* Interval between regions (reg_size + (redzone_size << 1)). */
|
||||
size_t reg_interval;
|
||||
|
||||
/* Total size of a run for this bin's size class. */
|
||||
size_t run_size;
|
||||
|
||||
@ -357,13 +402,15 @@ void arena_purge_all(arena_t *arena);
|
||||
void arena_prof_accum(arena_t *arena, uint64_t accumbytes);
|
||||
void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
|
||||
size_t binind, uint64_t prof_accumbytes);
|
||||
void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info,
|
||||
bool zero);
|
||||
void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info);
|
||||
void *arena_malloc_small(arena_t *arena, size_t size, bool zero);
|
||||
void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
|
||||
void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size,
|
||||
size_t alignment, bool zero);
|
||||
size_t arena_salloc(const void *ptr);
|
||||
size_t arena_salloc(const void *ptr, bool demote);
|
||||
void arena_prof_promoted(const void *ptr, size_t size);
|
||||
size_t arena_salloc_demote(const void *ptr);
|
||||
void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
arena_chunk_map_t *mapelm);
|
||||
void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
|
||||
@ -408,7 +455,7 @@ JEMALLOC_INLINE unsigned
|
||||
arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
|
||||
{
|
||||
unsigned shift, diff, regind;
|
||||
size_t size;
|
||||
size_t interval;
|
||||
|
||||
/*
|
||||
* Freeing a pointer lower than region zero can cause assertion
|
||||
@ -425,12 +472,12 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
|
||||
bin_info->reg0_offset);
|
||||
|
||||
/* Rescale (factor powers of 2 out of the numerator and denominator). */
|
||||
size = bin_info->reg_size;
|
||||
shift = ffs(size) - 1;
|
||||
interval = bin_info->reg_interval;
|
||||
shift = ffs(interval) - 1;
|
||||
diff >>= shift;
|
||||
size >>= shift;
|
||||
interval >>= shift;
|
||||
|
||||
if (size == 1) {
|
||||
if (interval == 1) {
|
||||
/* The divisor was a power of 2. */
|
||||
regind = diff;
|
||||
} else {
|
||||
@ -442,7 +489,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
|
||||
*
|
||||
* becomes
|
||||
*
|
||||
* (X * size_invs[D - 3]) >> SIZE_INV_SHIFT
|
||||
* (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT
|
||||
*
|
||||
* We can omit the first three elements, because we never
|
||||
* divide by 0, and 1 and 2 are both powers of two, which are
|
||||
@ -450,7 +497,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
|
||||
*/
|
||||
#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS)
|
||||
#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1)
|
||||
static const unsigned size_invs[] = {
|
||||
static const unsigned interval_invs[] = {
|
||||
SIZE_INV(3),
|
||||
SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
|
||||
SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
|
||||
@ -461,14 +508,16 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
|
||||
SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
|
||||
};
|
||||
|
||||
if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2))
|
||||
regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT;
|
||||
else
|
||||
regind = diff / size;
|
||||
if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) +
|
||||
2)) {
|
||||
regind = (diff * interval_invs[interval - 3]) >>
|
||||
SIZE_INV_SHIFT;
|
||||
} else
|
||||
regind = diff / interval;
|
||||
#undef SIZE_INV
|
||||
#undef SIZE_INV_SHIFT
|
||||
}
|
||||
assert(diff == regind * size);
|
||||
assert(diff == regind * interval);
|
||||
assert(regind < bin_info->nregs);
|
||||
|
||||
return (regind);
|
||||
@ -610,7 +659,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
|
||||
&arena_bin_info[binind];
|
||||
assert(((uintptr_t)ptr - ((uintptr_t)run +
|
||||
(uintptr_t)bin_info->reg0_offset)) %
|
||||
bin_info->reg_size == 0);
|
||||
bin_info->reg_interval == 0);
|
||||
}
|
||||
malloc_mutex_lock(&bin->lock);
|
||||
arena_dalloc_bin(arena, chunk, ptr, mapelm);
|
||||
|
||||
@ -40,6 +40,11 @@
|
||||
#include <sys/ktrace.h>
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_VALGRIND
|
||||
#include <valgrind/valgrind.h>
|
||||
#include <valgrind/memcheck.h>
|
||||
#endif
|
||||
|
||||
#include "jemalloc/internal/private_namespace.h"
|
||||
|
||||
#ifdef JEMALLOC_CC_SILENCE
|
||||
@ -125,6 +130,13 @@ static const bool config_utrace =
|
||||
false
|
||||
#endif
|
||||
;
|
||||
static const bool config_valgrind =
|
||||
#ifdef JEMALLOC_VALGRIND
|
||||
true
|
||||
#else
|
||||
false
|
||||
#endif
|
||||
;
|
||||
static const bool config_xmalloc =
|
||||
#ifdef JEMALLOC_XMALLOC
|
||||
true
|
||||
@ -281,6 +293,77 @@ static const bool config_ivsalloc =
|
||||
#define PAGE_CEILING(s) \
|
||||
(((s) + PAGE_MASK) & ~PAGE_MASK)
|
||||
|
||||
#ifdef JEMALLOC_VALGRIND
|
||||
/*
|
||||
* The JEMALLOC_VALGRIND_*() macros must be macros rather than functions
|
||||
* so that when Valgrind reports errors, there are no extra stack frames
|
||||
* in the backtraces.
|
||||
*
|
||||
* The size that is reported to valgrind must be consistent through a chain of
|
||||
* malloc..realloc..realloc calls. Request size isn't recorded anywhere in
|
||||
* jemalloc, so it is critical that all callers of these macros provide usize
|
||||
* rather than request size. As a result, buffer overflow detection is
|
||||
* technically weakened for the standard API, though it is generally accepted
|
||||
* practice to consider any extra bytes reported by malloc_usable_size() as
|
||||
* usable space.
|
||||
*/
|
||||
#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \
|
||||
if (config_valgrind && opt_valgrind && cond) \
|
||||
VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \
|
||||
} while (0)
|
||||
#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \
|
||||
old_rzsize, zero) do { \
|
||||
if (config_valgrind && opt_valgrind) { \
|
||||
size_t rzsize = p2rz(ptr); \
|
||||
\
|
||||
if (ptr == old_ptr) { \
|
||||
VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \
|
||||
usize, rzsize); \
|
||||
if (zero && old_usize < usize) { \
|
||||
VALGRIND_MAKE_MEM_DEFINED( \
|
||||
(void *)((uintptr_t)ptr + \
|
||||
old_usize), usize - old_usize); \
|
||||
} \
|
||||
} else { \
|
||||
if (old_ptr != NULL) { \
|
||||
VALGRIND_FREELIKE_BLOCK(old_ptr, \
|
||||
old_rzsize); \
|
||||
} \
|
||||
if (ptr != NULL) { \
|
||||
size_t copy_size = (old_usize < usize) \
|
||||
? old_usize : usize; \
|
||||
size_t tail_size = usize - copy_size; \
|
||||
VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \
|
||||
rzsize, false); \
|
||||
if (copy_size > 0) { \
|
||||
VALGRIND_MAKE_MEM_DEFINED(ptr, \
|
||||
copy_size); \
|
||||
} \
|
||||
if (zero && tail_size > 0) { \
|
||||
VALGRIND_MAKE_MEM_DEFINED( \
|
||||
(void *)((uintptr_t)ptr + \
|
||||
copy_size), tail_size); \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} while (0)
|
||||
#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \
|
||||
if (config_valgrind && opt_valgrind) \
|
||||
VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \
|
||||
} while (0)
|
||||
#else
|
||||
#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
|
||||
#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)
|
||||
#define VALGRIND_FREELIKE_BLOCK(addr, rzB)
|
||||
#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len)
|
||||
#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len)
|
||||
#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero)
|
||||
#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \
|
||||
old_rzsize, zero)
|
||||
#define JEMALLOC_VALGRIND_FREE(ptr, rzsize)
|
||||
#endif
|
||||
|
||||
#include "jemalloc/internal/util.h"
|
||||
#include "jemalloc/internal/atomic.h"
|
||||
#include "jemalloc/internal/prng.h"
|
||||
@ -300,6 +383,7 @@ static const bool config_ivsalloc =
|
||||
#include "jemalloc/internal/rtree.h"
|
||||
#include "jemalloc/internal/tcache.h"
|
||||
#include "jemalloc/internal/hash.h"
|
||||
#include "jemalloc/internal/quarantine.h"
|
||||
#include "jemalloc/internal/prof.h"
|
||||
|
||||
#undef JEMALLOC_H_TYPES
|
||||
@ -325,6 +409,7 @@ static const bool config_ivsalloc =
|
||||
#include "jemalloc/internal/rtree.h"
|
||||
#include "jemalloc/internal/tcache.h"
|
||||
#include "jemalloc/internal/hash.h"
|
||||
#include "jemalloc/internal/quarantine.h"
|
||||
#include "jemalloc/internal/prof.h"
|
||||
|
||||
typedef struct {
|
||||
@ -343,7 +428,10 @@ typedef struct {
|
||||
|
||||
extern bool opt_abort;
|
||||
extern bool opt_junk;
|
||||
extern size_t opt_quarantine;
|
||||
extern bool opt_redzone;
|
||||
extern bool opt_utrace;
|
||||
extern bool opt_valgrind;
|
||||
extern bool opt_xmalloc;
|
||||
extern bool opt_zero;
|
||||
extern size_t opt_narenas;
|
||||
@ -385,6 +473,7 @@ void jemalloc_postfork_child(void);
|
||||
#include "jemalloc/internal/rtree.h"
|
||||
#include "jemalloc/internal/tcache.h"
|
||||
#include "jemalloc/internal/hash.h"
|
||||
#include "jemalloc/internal/quarantine.h"
|
||||
#include "jemalloc/internal/prof.h"
|
||||
|
||||
#undef JEMALLOC_H_EXTERNS
|
||||
@ -550,14 +639,18 @@ choose_arena(arena_t *arena)
|
||||
#include "jemalloc/internal/tcache.h"
|
||||
#include "jemalloc/internal/arena.h"
|
||||
#include "jemalloc/internal/hash.h"
|
||||
#include "jemalloc/internal/quarantine.h"
|
||||
|
||||
#ifndef JEMALLOC_ENABLE_INLINE
|
||||
void *imalloc(size_t size);
|
||||
void *icalloc(size_t size);
|
||||
void *ipalloc(size_t usize, size_t alignment, bool zero);
|
||||
size_t isalloc(const void *ptr);
|
||||
size_t ivsalloc(const void *ptr);
|
||||
size_t isalloc(const void *ptr, bool demote);
|
||||
size_t ivsalloc(const void *ptr, bool demote);
|
||||
size_t u2rz(size_t usize);
|
||||
size_t p2rz(const void *ptr);
|
||||
void idalloc(void *ptr);
|
||||
void iqalloc(void *ptr);
|
||||
void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
|
||||
bool zero, bool no_move);
|
||||
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
|
||||
@ -621,21 +714,25 @@ ipalloc(size_t usize, size_t alignment, bool zero)
|
||||
return (ret);
|
||||
}
|
||||
|
||||
/*
|
||||
* Typical usage:
|
||||
* void *ptr = [...]
|
||||
* size_t sz = isalloc(ptr, config_prof);
|
||||
*/
|
||||
JEMALLOC_INLINE size_t
|
||||
isalloc(const void *ptr)
|
||||
isalloc(const void *ptr, bool demote)
|
||||
{
|
||||
size_t ret;
|
||||
arena_chunk_t *chunk;
|
||||
|
||||
assert(ptr != NULL);
|
||||
/* Demotion only makes sense if config_prof is true. */
|
||||
assert(config_prof || demote == false);
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
if (chunk != ptr) {
|
||||
/* Region. */
|
||||
if (config_prof)
|
||||
ret = arena_salloc_demote(ptr);
|
||||
else
|
||||
ret = arena_salloc(ptr);
|
||||
ret = arena_salloc(ptr, demote);
|
||||
} else
|
||||
ret = huge_salloc(ptr);
|
||||
|
||||
@ -643,14 +740,36 @@ isalloc(const void *ptr)
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE size_t
|
||||
ivsalloc(const void *ptr)
|
||||
ivsalloc(const void *ptr, bool demote)
|
||||
{
|
||||
|
||||
/* Return 0 if ptr is not within a chunk managed by jemalloc. */
|
||||
if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL)
|
||||
return (0);
|
||||
|
||||
return (isalloc(ptr));
|
||||
return (isalloc(ptr, demote));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE size_t
|
||||
u2rz(size_t usize)
|
||||
{
|
||||
size_t ret;
|
||||
|
||||
if (usize <= SMALL_MAXCLASS) {
|
||||
size_t binind = SMALL_SIZE2BIN(usize);
|
||||
ret = arena_bin_info[binind].redzone_size;
|
||||
} else
|
||||
ret = 0;
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE size_t
|
||||
p2rz(const void *ptr)
|
||||
{
|
||||
size_t usize = isalloc(ptr, false);
|
||||
|
||||
return (u2rz(usize));
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
@ -667,6 +786,16 @@ idalloc(void *ptr)
|
||||
huge_dalloc(ptr, true);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void
|
||||
iqalloc(void *ptr)
|
||||
{
|
||||
|
||||
if (config_fill && opt_quarantine)
|
||||
quarantine(ptr);
|
||||
else
|
||||
idalloc(ptr);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE void *
|
||||
iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
|
||||
bool no_move)
|
||||
@ -677,14 +806,14 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
|
||||
assert(ptr != NULL);
|
||||
assert(size != 0);
|
||||
|
||||
oldsize = isalloc(ptr);
|
||||
oldsize = isalloc(ptr, config_prof);
|
||||
|
||||
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
|
||||
!= 0) {
|
||||
size_t usize, copysize;
|
||||
|
||||
/*
|
||||
* Existing object alignment is inadquate; allocate new space
|
||||
* Existing object alignment is inadequate; allocate new space
|
||||
* and copy.
|
||||
*/
|
||||
if (no_move)
|
||||
@ -711,7 +840,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
|
||||
*/
|
||||
copysize = (size < oldsize) ? size : oldsize;
|
||||
memcpy(ret, ptr, copysize);
|
||||
idalloc(ptr);
|
||||
iqalloc(ptr);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small)
|
||||
#define arena_bin_index JEMALLOC_N(arena_bin_index)
|
||||
#define arena_boot JEMALLOC_N(arena_boot)
|
||||
#define arena_dalloc JEMALLOC_N(arena_dalloc)
|
||||
#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin)
|
||||
#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
|
||||
#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large)
|
||||
#define arena_malloc JEMALLOC_N(arena_malloc)
|
||||
#define arena_malloc_large JEMALLOC_N(arena_malloc_large)
|
||||
@ -20,7 +22,6 @@
|
||||
#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)
|
||||
#define arena_run_regind JEMALLOC_N(arena_run_regind)
|
||||
#define arena_salloc JEMALLOC_N(arena_salloc)
|
||||
#define arena_salloc_demote JEMALLOC_N(arena_salloc_demote)
|
||||
#define arena_stats_merge JEMALLOC_N(arena_stats_merge)
|
||||
#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small)
|
||||
#define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index)
|
||||
@ -136,6 +137,7 @@
|
||||
#define idalloc JEMALLOC_N(idalloc)
|
||||
#define imalloc JEMALLOC_N(imalloc)
|
||||
#define ipalloc JEMALLOC_N(ipalloc)
|
||||
#define iqalloc JEMALLOC_N(iqalloc)
|
||||
#define iralloc JEMALLOC_N(iralloc)
|
||||
#define isalloc JEMALLOC_N(isalloc)
|
||||
#define ivsalloc JEMALLOC_N(ivsalloc)
|
||||
@ -176,6 +178,7 @@
|
||||
#define opt_utrace JEMALLOC_N(opt_utrace)
|
||||
#define opt_xmalloc JEMALLOC_N(opt_xmalloc)
|
||||
#define opt_zero JEMALLOC_N(opt_zero)
|
||||
#define p2rz JEMALLOC_N(p2rz)
|
||||
#define pow2_ceil JEMALLOC_N(pow2_ceil)
|
||||
#define prof_backtrace JEMALLOC_N(prof_backtrace)
|
||||
#define prof_boot0 JEMALLOC_N(prof_boot0)
|
||||
@ -195,6 +198,8 @@
|
||||
#define prof_tdata_init JEMALLOC_N(prof_tdata_init)
|
||||
#define prof_tdata_tls JEMALLOC_N(prof_tdata_tls)
|
||||
#define pthread_create JEMALLOC_N(pthread_create)
|
||||
#define quarantine JEMALLOC_N(quarantine)
|
||||
#define quarantine_boot JEMALLOC_N(quarantine_boot)
|
||||
#define register_zone JEMALLOC_N(register_zone)
|
||||
#define rtree_get JEMALLOC_N(rtree_get)
|
||||
#define rtree_get_locked JEMALLOC_N(rtree_get_locked)
|
||||
@ -229,3 +234,4 @@
|
||||
#define thread_allocated_get JEMALLOC_N(thread_allocated_get)
|
||||
#define thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard)
|
||||
#define thread_allocated_tls JEMALLOC_N(thread_allocated_tls)
|
||||
#define u2rz JEMALLOC_N(u2rz)
|
||||
|
||||
@ -378,7 +378,7 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt)
|
||||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
assert(size == isalloc(ptr));
|
||||
assert(size == isalloc(ptr, true));
|
||||
|
||||
if (opt_lg_prof_sample != 0) {
|
||||
if (prof_sample_accum_update(size)) {
|
||||
@ -427,7 +427,7 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
|
||||
assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U);
|
||||
|
||||
if (ptr != NULL) {
|
||||
assert(size == isalloc(ptr));
|
||||
assert(size == isalloc(ptr, true));
|
||||
if (opt_lg_prof_sample != 0) {
|
||||
if (prof_sample_accum_update(size)) {
|
||||
/*
|
||||
@ -500,7 +500,7 @@ prof_free(const void *ptr, size_t size)
|
||||
cassert(config_prof);
|
||||
|
||||
if ((uintptr_t)ctx > (uintptr_t)1) {
|
||||
assert(size == isalloc(ptr));
|
||||
assert(size == isalloc(ptr, true));
|
||||
prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt);
|
||||
|
||||
if (tcnt != NULL) {
|
||||
|
||||
24
include/jemalloc/internal/quarantine.h
Normal file
24
include/jemalloc/internal/quarantine.h
Normal file
@ -0,0 +1,24 @@
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_TYPES
|
||||
|
||||
/* Default per thread quarantine size if valgrind is enabled. */
|
||||
#define JEMALLOC_VALGRIND_QUARANTINE_DEFAULT (ZU(1) << 24)
|
||||
|
||||
#endif /* JEMALLOC_H_TYPES */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_STRUCTS
|
||||
|
||||
#endif /* JEMALLOC_H_STRUCTS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_EXTERNS
|
||||
|
||||
void quarantine(void *ptr);
|
||||
bool quarantine_boot(void);
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
#ifdef JEMALLOC_H_INLINES
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
/******************************************************************************/
|
||||
|
||||
@ -340,17 +340,24 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
}
|
||||
assert(arena_salloc(ret) == arena_bin_info[binind].reg_size);
|
||||
assert(arena_salloc(ret, false) == arena_bin_info[binind].reg_size);
|
||||
|
||||
if (zero == false) {
|
||||
if (config_fill) {
|
||||
if (opt_junk)
|
||||
memset(ret, 0xa5, size);
|
||||
else if (opt_zero)
|
||||
if (opt_junk) {
|
||||
arena_alloc_junk_small(ret,
|
||||
&arena_bin_info[binind], false);
|
||||
} else if (opt_zero)
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
} else
|
||||
} else {
|
||||
if (config_fill && opt_junk) {
|
||||
arena_alloc_junk_small(ret, &arena_bin_info[binind],
|
||||
true);
|
||||
}
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
|
||||
if (config_stats)
|
||||
tbin->tstats.nrequests++;
|
||||
@ -397,8 +404,10 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
|
||||
else if (opt_zero)
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
} else
|
||||
} else {
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
|
||||
if (config_stats)
|
||||
tbin->tstats.nrequests++;
|
||||
@ -422,7 +431,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
|
||||
size_t pageind, binind;
|
||||
arena_chunk_map_t *mapelm;
|
||||
|
||||
assert(arena_salloc(ptr) <= SMALL_MAXCLASS);
|
||||
assert(arena_salloc(ptr, false) <= SMALL_MAXCLASS);
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
arena = chunk->arena;
|
||||
@ -436,7 +445,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
|
||||
assert(binind < NBINS);
|
||||
|
||||
if (config_fill && opt_junk)
|
||||
memset(ptr, 0x5a, arena_bin_info[binind].reg_size);
|
||||
arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);
|
||||
|
||||
tbin = &tcache->tbins[binind];
|
||||
tbin_info = &tcache_bin_info[binind];
|
||||
@ -459,8 +468,8 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
|
||||
tcache_bin_info_t *tbin_info;
|
||||
|
||||
assert((size & PAGE_MASK) == 0);
|
||||
assert(arena_salloc(ptr) > SMALL_MAXCLASS);
|
||||
assert(arena_salloc(ptr) <= tcache_maxclass);
|
||||
assert(arena_salloc(ptr, false) > SMALL_MAXCLASS);
|
||||
assert(arena_salloc(ptr, false) <= tcache_maxclass);
|
||||
|
||||
binind = NBINS + (size >> LG_PAGE) - 1;
|
||||
|
||||
|
||||
@ -140,7 +140,6 @@ malloc_write(const char *s)
|
||||
|
||||
je_malloc_message(NULL, s);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_INLINES */
|
||||
|
||||
@ -148,7 +148,7 @@
|
||||
*/
|
||||
#undef JEMALLOC_DSS
|
||||
|
||||
/* Support memory filling (junk/zero). */
|
||||
/* Support memory filling (junk/zero/quarantine/redzone). */
|
||||
#undef JEMALLOC_FILL
|
||||
|
||||
/* Support the experimental API. */
|
||||
@ -157,6 +157,9 @@
|
||||
/* Support utrace(2)-based tracing. */
|
||||
#undef JEMALLOC_UTRACE
|
||||
|
||||
/* Support Valgrind. */
|
||||
#undef JEMALLOC_VALGRIND
|
||||
|
||||
/* Support optional abort() on OOM. */
|
||||
#undef JEMALLOC_XMALLOC
|
||||
|
||||
|
||||
208
src/arena.c
208
src/arena.c
@ -140,7 +140,7 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info)
|
||||
|
||||
regind = bitmap_sfu(bitmap, &bin_info->bitmap_info);
|
||||
ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset +
|
||||
(uintptr_t)(bin_info->reg_size * regind));
|
||||
(uintptr_t)(bin_info->reg_interval * regind));
|
||||
run->nfree--;
|
||||
if (regind == run->nextind)
|
||||
run->nextind++;
|
||||
@ -161,8 +161,8 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr)
|
||||
assert(run->nfree < bin_info->nregs);
|
||||
/* Freeing an interior pointer can cause assertion failure. */
|
||||
assert(((uintptr_t)ptr - ((uintptr_t)run +
|
||||
(uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size
|
||||
== 0);
|
||||
(uintptr_t)bin_info->reg0_offset)) %
|
||||
(uintptr_t)bin_info->reg_interval == 0);
|
||||
assert((uintptr_t)ptr >= (uintptr_t)run +
|
||||
(uintptr_t)bin_info->reg0_offset);
|
||||
/* Freeing an unallocated pointer can cause assertion failure. */
|
||||
@ -260,10 +260,18 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
|
||||
for (i = 0; i < need_pages; i++) {
|
||||
if ((chunk->map[run_ind+i-map_bias].bits
|
||||
& CHUNK_MAP_UNZEROED) != 0) {
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(
|
||||
(void *)((uintptr_t)
|
||||
chunk + ((run_ind+i) <<
|
||||
LG_PAGE)), PAGE);
|
||||
memset((void *)((uintptr_t)
|
||||
chunk + ((run_ind+i) <<
|
||||
LG_PAGE)), 0, PAGE);
|
||||
} else if (config_debug) {
|
||||
VALGRIND_MAKE_MEM_DEFINED(
|
||||
(void *)((uintptr_t)
|
||||
chunk + ((run_ind+i) <<
|
||||
LG_PAGE)), PAGE);
|
||||
arena_chunk_validate_zeroed(
|
||||
chunk, run_ind+i);
|
||||
}
|
||||
@ -273,6 +281,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
|
||||
* The run is dirty, so all pages must be
|
||||
* zeroed.
|
||||
*/
|
||||
VALGRIND_MAKE_MEM_UNDEFINED((void
|
||||
*)((uintptr_t)chunk + (run_ind <<
|
||||
LG_PAGE)), (need_pages << LG_PAGE));
|
||||
memset((void *)((uintptr_t)chunk + (run_ind <<
|
||||
LG_PAGE)), 0, (need_pages << LG_PAGE));
|
||||
}
|
||||
@ -1245,6 +1256,10 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind,
|
||||
ptr = arena_bin_malloc_hard(arena, bin);
|
||||
if (ptr == NULL)
|
||||
break;
|
||||
if (config_fill && opt_junk) {
|
||||
arena_alloc_junk_small(ptr, &arena_bin_info[binind],
|
||||
true);
|
||||
}
|
||||
/* Insert such that low regions get used first. */
|
||||
tbin->avail[nfill - 1 - i] = ptr;
|
||||
}
|
||||
@ -1259,6 +1274,55 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind,
|
||||
tbin->ncached = i;
|
||||
}
|
||||
|
||||
void
|
||||
arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero)
|
||||
{
|
||||
|
||||
if (zero) {
|
||||
size_t redzone_size = bin_info->redzone_size;
|
||||
memset((void *)((uintptr_t)ptr - redzone_size), 0xa5,
|
||||
redzone_size);
|
||||
memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5,
|
||||
redzone_size);
|
||||
} else {
|
||||
memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5,
|
||||
bin_info->reg_interval);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info)
|
||||
{
|
||||
size_t size = bin_info->reg_size;
|
||||
size_t redzone_size = bin_info->redzone_size;
|
||||
size_t i;
|
||||
bool error = false;
|
||||
|
||||
for (i = 1; i <= redzone_size; i++) {
|
||||
unsigned byte;
|
||||
if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) {
|
||||
error = true;
|
||||
malloc_printf("<jemalloc>: Corrupt redzone "
|
||||
"%zu byte%s before %p (size %zu), byte=%#x\n", i,
|
||||
(i == 1) ? "" : "s", ptr, size, byte);
|
||||
}
|
||||
}
|
||||
for (i = 0; i < redzone_size; i++) {
|
||||
unsigned byte;
|
||||
if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) {
|
||||
error = true;
|
||||
malloc_printf("<jemalloc>: Corrupt redzone "
|
||||
"%zu byte%s after end of %p (size %zu), byte=%#x\n",
|
||||
i, (i == 1) ? "" : "s", ptr, size, byte);
|
||||
}
|
||||
}
|
||||
if (opt_abort && error)
|
||||
abort();
|
||||
|
||||
memset((void *)((uintptr_t)ptr - redzone_size), 0x5a,
|
||||
bin_info->reg_interval);
|
||||
}
|
||||
|
||||
void *
|
||||
arena_malloc_small(arena_t *arena, size_t size, bool zero)
|
||||
{
|
||||
@ -1297,13 +1361,20 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
|
||||
|
||||
if (zero == false) {
|
||||
if (config_fill) {
|
||||
if (opt_junk)
|
||||
memset(ret, 0xa5, size);
|
||||
else if (opt_zero)
|
||||
if (opt_junk) {
|
||||
arena_alloc_junk_small(ret,
|
||||
&arena_bin_info[binind], false);
|
||||
} else if (opt_zero)
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
} else
|
||||
} else {
|
||||
if (config_fill && opt_junk) {
|
||||
arena_alloc_junk_small(ret, &arena_bin_info[binind],
|
||||
true);
|
||||
}
|
||||
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
|
||||
return (ret);
|
||||
}
|
||||
@ -1412,7 +1483,7 @@ arena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment,
|
||||
|
||||
/* Return the size of the allocation pointed to by ptr. */
|
||||
size_t
|
||||
arena_salloc(const void *ptr)
|
||||
arena_salloc(const void *ptr, bool demote)
|
||||
{
|
||||
size_t ret;
|
||||
arena_chunk_t *chunk;
|
||||
@ -1431,12 +1502,19 @@ arena_salloc(const void *ptr)
|
||||
size_t binind = arena_bin_index(chunk->arena, run->bin);
|
||||
arena_bin_info_t *bin_info = &arena_bin_info[binind];
|
||||
assert(((uintptr_t)ptr - ((uintptr_t)run +
|
||||
(uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size ==
|
||||
0);
|
||||
(uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval
|
||||
== 0);
|
||||
ret = bin_info->reg_size;
|
||||
} else {
|
||||
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
|
||||
ret = mapbits & ~PAGE_MASK;
|
||||
if (demote && prof_promote && ret == PAGE && (mapbits &
|
||||
CHUNK_MAP_CLASS_MASK) != 0) {
|
||||
size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >>
|
||||
CHUNK_MAP_CLASS_SHIFT) - 1;
|
||||
assert(binind < NBINS);
|
||||
ret = arena_bin_info[binind].reg_size;
|
||||
}
|
||||
assert(ret != 0);
|
||||
}
|
||||
|
||||
@ -1449,9 +1527,11 @@ arena_prof_promoted(const void *ptr, size_t size)
|
||||
arena_chunk_t *chunk;
|
||||
size_t pageind, binind;
|
||||
|
||||
assert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
assert(CHUNK_ADDR2BASE(ptr) != ptr);
|
||||
assert(isalloc(ptr) == PAGE);
|
||||
assert(isalloc(ptr, false) == PAGE);
|
||||
assert(isalloc(ptr, true) == PAGE);
|
||||
assert(size <= SMALL_MAXCLASS);
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
@ -1460,45 +1540,9 @@ arena_prof_promoted(const void *ptr, size_t size)
|
||||
assert(binind < NBINS);
|
||||
chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits &
|
||||
~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT);
|
||||
}
|
||||
|
||||
size_t
|
||||
arena_salloc_demote(const void *ptr)
|
||||
{
|
||||
size_t ret;
|
||||
arena_chunk_t *chunk;
|
||||
size_t pageind, mapbits;
|
||||
|
||||
assert(ptr != NULL);
|
||||
assert(CHUNK_ADDR2BASE(ptr) != ptr);
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
|
||||
mapbits = chunk->map[pageind-map_bias].bits;
|
||||
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
|
||||
if ((mapbits & CHUNK_MAP_LARGE) == 0) {
|
||||
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
|
||||
(uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE));
|
||||
size_t binind = arena_bin_index(chunk->arena, run->bin);
|
||||
arena_bin_info_t *bin_info = &arena_bin_info[binind];
|
||||
assert(((uintptr_t)ptr - ((uintptr_t)run +
|
||||
(uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size ==
|
||||
0);
|
||||
ret = bin_info->reg_size;
|
||||
} else {
|
||||
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
|
||||
ret = mapbits & ~PAGE_MASK;
|
||||
if (prof_promote && ret == PAGE && (mapbits &
|
||||
CHUNK_MAP_CLASS_MASK) != 0) {
|
||||
size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >>
|
||||
CHUNK_MAP_CLASS_SHIFT) - 1;
|
||||
assert(binind < NBINS);
|
||||
ret = arena_bin_info[binind].reg_size;
|
||||
}
|
||||
assert(ret != 0);
|
||||
}
|
||||
|
||||
return (ret);
|
||||
assert(isalloc(ptr, false) == PAGE);
|
||||
assert(isalloc(ptr, true) == size);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -1545,7 +1589,8 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
||||
run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
|
||||
past = (size_t)(PAGE_CEILING((uintptr_t)run +
|
||||
(uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind *
|
||||
bin_info->reg_size) - (uintptr_t)chunk) >> LG_PAGE);
|
||||
bin_info->reg_interval - bin_info->redzone_size) -
|
||||
(uintptr_t)chunk) >> LG_PAGE);
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
|
||||
/*
|
||||
@ -1617,7 +1662,7 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
size = bin_info->reg_size;
|
||||
|
||||
if (config_fill && opt_junk)
|
||||
memset(ptr, 0x5a, size);
|
||||
arena_dalloc_junk_small(ptr, bin_info);
|
||||
|
||||
arena_run_reg_dalloc(run, ptr);
|
||||
if (run->nfree == bin_info->nregs) {
|
||||
@ -1936,7 +1981,7 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
*/
|
||||
copysize = (size < oldsize) ? size : oldsize;
|
||||
memcpy(ret, ptr, copysize);
|
||||
idalloc(ptr);
|
||||
iqalloc(ptr);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -2007,16 +2052,40 @@ arena_new(arena_t *arena, unsigned ind)
|
||||
static size_t
|
||||
bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size)
|
||||
{
|
||||
size_t pad_size;
|
||||
size_t try_run_size, good_run_size;
|
||||
uint32_t try_nregs, good_nregs;
|
||||
uint32_t try_hdr_size, good_hdr_size;
|
||||
uint32_t try_bitmap_offset, good_bitmap_offset;
|
||||
uint32_t try_ctx0_offset, good_ctx0_offset;
|
||||
uint32_t try_reg0_offset, good_reg0_offset;
|
||||
uint32_t try_redzone0_offset, good_redzone0_offset;
|
||||
|
||||
assert(min_run_size >= PAGE);
|
||||
assert(min_run_size <= arena_maxclass);
|
||||
|
||||
/*
|
||||
* Determine redzone size based on minimum alignment and minimum
|
||||
* redzone size. Add padding to the end of the run if it is needed to
|
||||
* align the regions. The padding allows each redzone to be half the
|
||||
* minimum alignment; without the padding, each redzone would have to
|
||||
* be twice as large in order to maintain alignment.
|
||||
*/
|
||||
if (config_fill && opt_redzone) {
|
||||
size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1);
|
||||
if (align_min <= REDZONE_MINSIZE) {
|
||||
bin_info->redzone_size = REDZONE_MINSIZE;
|
||||
pad_size = 0;
|
||||
} else {
|
||||
bin_info->redzone_size = align_min >> 1;
|
||||
pad_size = bin_info->redzone_size;
|
||||
}
|
||||
} else {
|
||||
bin_info->redzone_size = 0;
|
||||
pad_size = 0;
|
||||
}
|
||||
bin_info->reg_interval = bin_info->reg_size +
|
||||
(bin_info->redzone_size << 1);
|
||||
|
||||
/*
|
||||
* Calculate known-valid settings before entering the run_size
|
||||
* expansion loop, so that the first part of the loop always copies
|
||||
@ -2028,7 +2097,8 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size)
|
||||
* header's mask length and the number of regions.
|
||||
*/
|
||||
try_run_size = min_run_size;
|
||||
try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size)
|
||||
try_nregs = ((try_run_size - sizeof(arena_run_t)) /
|
||||
bin_info->reg_interval)
|
||||
+ 1; /* Counter-act try_nregs-- in loop. */
|
||||
if (try_nregs > RUN_MAXREGS) {
|
||||
try_nregs = RUN_MAXREGS
|
||||
@ -2050,9 +2120,9 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size)
|
||||
try_hdr_size += try_nregs * sizeof(prof_ctx_t *);
|
||||
} else
|
||||
try_ctx0_offset = 0;
|
||||
try_reg0_offset = try_run_size - (try_nregs *
|
||||
bin_info->reg_size);
|
||||
} while (try_hdr_size > try_reg0_offset);
|
||||
try_redzone0_offset = try_run_size - (try_nregs *
|
||||
bin_info->reg_interval) - pad_size;
|
||||
} while (try_hdr_size > try_redzone0_offset);
|
||||
|
||||
/* run_size expansion loop. */
|
||||
do {
|
||||
@ -2064,12 +2134,12 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size)
|
||||
good_hdr_size = try_hdr_size;
|
||||
good_bitmap_offset = try_bitmap_offset;
|
||||
good_ctx0_offset = try_ctx0_offset;
|
||||
good_reg0_offset = try_reg0_offset;
|
||||
good_redzone0_offset = try_redzone0_offset;
|
||||
|
||||
/* Try more aggressive settings. */
|
||||
try_run_size += PAGE;
|
||||
try_nregs = ((try_run_size - sizeof(arena_run_t)) /
|
||||
bin_info->reg_size)
|
||||
try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) /
|
||||
bin_info->reg_interval)
|
||||
+ 1; /* Counter-act try_nregs-- in loop. */
|
||||
if (try_nregs > RUN_MAXREGS) {
|
||||
try_nregs = RUN_MAXREGS
|
||||
@ -2093,23 +2163,27 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size)
|
||||
try_hdr_size += try_nregs *
|
||||
sizeof(prof_ctx_t *);
|
||||
}
|
||||
try_reg0_offset = try_run_size - (try_nregs *
|
||||
bin_info->reg_size);
|
||||
} while (try_hdr_size > try_reg0_offset);
|
||||
try_redzone0_offset = try_run_size - (try_nregs *
|
||||
bin_info->reg_interval) - pad_size;
|
||||
} while (try_hdr_size > try_redzone0_offset);
|
||||
} while (try_run_size <= arena_maxclass
|
||||
&& try_run_size <= arena_maxclass
|
||||
&& RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX
|
||||
&& (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size
|
||||
&& RUN_MAX_OVRHD * (bin_info->reg_interval << 3) >
|
||||
RUN_MAX_OVRHD_RELAX
|
||||
&& (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size
|
||||
&& try_nregs < RUN_MAXREGS);
|
||||
|
||||
assert(good_hdr_size <= good_reg0_offset);
|
||||
assert(good_hdr_size <= good_redzone0_offset);
|
||||
|
||||
/* Copy final settings. */
|
||||
bin_info->run_size = good_run_size;
|
||||
bin_info->nregs = good_nregs;
|
||||
bin_info->bitmap_offset = good_bitmap_offset;
|
||||
bin_info->ctx0_offset = good_ctx0_offset;
|
||||
bin_info->reg0_offset = good_reg0_offset;
|
||||
bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size;
|
||||
|
||||
assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs
|
||||
* bin_info->reg_interval) + pad_size == bin_info->run_size);
|
||||
|
||||
return (good_run_size);
|
||||
}
|
||||
|
||||
15
src/ctl.c
15
src/ctl.c
@ -57,6 +57,7 @@ CTL_PROTO(config_stats)
|
||||
CTL_PROTO(config_tcache)
|
||||
CTL_PROTO(config_tls)
|
||||
CTL_PROTO(config_utrace)
|
||||
CTL_PROTO(config_valgrind)
|
||||
CTL_PROTO(config_xmalloc)
|
||||
CTL_PROTO(opt_abort)
|
||||
CTL_PROTO(opt_lg_chunk)
|
||||
@ -65,7 +66,10 @@ CTL_PROTO(opt_lg_dirty_mult)
|
||||
CTL_PROTO(opt_stats_print)
|
||||
CTL_PROTO(opt_junk)
|
||||
CTL_PROTO(opt_zero)
|
||||
CTL_PROTO(opt_quarantine)
|
||||
CTL_PROTO(opt_redzone)
|
||||
CTL_PROTO(opt_utrace)
|
||||
CTL_PROTO(opt_valgrind)
|
||||
CTL_PROTO(opt_xmalloc)
|
||||
CTL_PROTO(opt_tcache)
|
||||
CTL_PROTO(opt_lg_tcache_max)
|
||||
@ -179,6 +183,7 @@ static const ctl_node_t config_node[] = {
|
||||
{NAME("tcache"), CTL(config_tcache)},
|
||||
{NAME("tls"), CTL(config_tls)},
|
||||
{NAME("utrace"), CTL(config_utrace)},
|
||||
{NAME("valgrind"), CTL(config_valgrind)},
|
||||
{NAME("xmalloc"), CTL(config_xmalloc)}
|
||||
};
|
||||
|
||||
@ -190,7 +195,10 @@ static const ctl_node_t opt_node[] = {
|
||||
{NAME("stats_print"), CTL(opt_stats_print)},
|
||||
{NAME("junk"), CTL(opt_junk)},
|
||||
{NAME("zero"), CTL(opt_zero)},
|
||||
{NAME("quarantine"), CTL(opt_quarantine)},
|
||||
{NAME("redzone"), CTL(opt_redzone)},
|
||||
{NAME("utrace"), CTL(opt_utrace)},
|
||||
{NAME("valgrind"), CTL(opt_valgrind)},
|
||||
{NAME("xmalloc"), CTL(opt_xmalloc)},
|
||||
{NAME("tcache"), CTL(opt_tcache)},
|
||||
{NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)},
|
||||
@ -1050,7 +1058,8 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
|
||||
/* Set new arena association. */
|
||||
if (config_tcache) {
|
||||
tcache_t *tcache;
|
||||
if ((tcache = *tcache_tsd_get()) != NULL) {
|
||||
if ((uintptr_t)(tcache = *tcache_tsd_get()) >
|
||||
(uintptr_t)TCACHE_STATE_MAX) {
|
||||
tcache_arena_dissociate(tcache);
|
||||
tcache_arena_associate(tcache, arena);
|
||||
}
|
||||
@ -1085,6 +1094,7 @@ CTL_RO_BOOL_CONFIG_GEN(config_stats)
|
||||
CTL_RO_BOOL_CONFIG_GEN(config_tcache)
|
||||
CTL_RO_BOOL_CONFIG_GEN(config_tls)
|
||||
CTL_RO_BOOL_CONFIG_GEN(config_utrace)
|
||||
CTL_RO_BOOL_CONFIG_GEN(config_valgrind)
|
||||
CTL_RO_BOOL_CONFIG_GEN(config_xmalloc)
|
||||
|
||||
/******************************************************************************/
|
||||
@ -1096,7 +1106,10 @@ CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t)
|
||||
CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
|
||||
CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool)
|
||||
CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
|
||||
CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t)
|
||||
CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool)
|
||||
CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
|
||||
CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool)
|
||||
CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
|
||||
CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
|
||||
CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
|
||||
|
||||
@ -174,7 +174,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
#endif
|
||||
{
|
||||
memcpy(ret, ptr, copysize);
|
||||
idalloc(ptr);
|
||||
iqalloc(ptr);
|
||||
}
|
||||
return (ret);
|
||||
}
|
||||
|
||||
165
src/jemalloc.c
165
src/jemalloc.c
@ -14,14 +14,19 @@ const char *je_malloc_conf JEMALLOC_ATTR(visibility("default"));
|
||||
bool opt_abort = true;
|
||||
# ifdef JEMALLOC_FILL
|
||||
bool opt_junk = true;
|
||||
bool opt_redzone = true;
|
||||
# else
|
||||
bool opt_junk = false;
|
||||
bool opt_redzone = false;
|
||||
# endif
|
||||
#else
|
||||
bool opt_abort = false;
|
||||
bool opt_junk = false;
|
||||
bool opt_redzone = false;
|
||||
#endif
|
||||
size_t opt_quarantine = ZU(0);
|
||||
bool opt_utrace = false;
|
||||
bool opt_valgrind = false;
|
||||
bool opt_xmalloc = false;
|
||||
bool opt_zero = false;
|
||||
size_t opt_narenas = 0;
|
||||
@ -419,7 +424,7 @@ malloc_conf_init(void)
|
||||
|
||||
while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
|
||||
&vlen) == false) {
|
||||
#define CONF_HANDLE_BOOL(o, n) \
|
||||
#define CONF_HANDLE_BOOL_HIT(o, n, hit) \
|
||||
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
|
||||
klen) == 0) { \
|
||||
if (strncmp("true", v, vlen) == 0 && \
|
||||
@ -433,12 +438,19 @@ malloc_conf_init(void)
|
||||
"Invalid conf value", \
|
||||
k, klen, v, vlen); \
|
||||
} \
|
||||
hit = true; \
|
||||
} else \
|
||||
hit = false;
|
||||
#define CONF_HANDLE_BOOL(o, n) { \
|
||||
bool hit; \
|
||||
CONF_HANDLE_BOOL_HIT(o, n, hit); \
|
||||
if (hit) \
|
||||
continue; \
|
||||
}
|
||||
}
|
||||
#define CONF_HANDLE_SIZE_T(o, n, min, max) \
|
||||
if (sizeof(#n)-1 == klen && strncmp(#n, k, \
|
||||
klen) == 0) { \
|
||||
uintmax_t um; \
|
||||
uintmax_t um; \
|
||||
char *end; \
|
||||
\
|
||||
errno = 0; \
|
||||
@ -502,11 +514,30 @@ malloc_conf_init(void)
|
||||
CONF_HANDLE_BOOL(opt_stats_print, stats_print)
|
||||
if (config_fill) {
|
||||
CONF_HANDLE_BOOL(opt_junk, junk)
|
||||
CONF_HANDLE_SIZE_T(opt_quarantine, quarantine,
|
||||
0, SIZE_T_MAX)
|
||||
CONF_HANDLE_BOOL(opt_redzone, redzone)
|
||||
CONF_HANDLE_BOOL(opt_zero, zero)
|
||||
}
|
||||
if (config_utrace) {
|
||||
CONF_HANDLE_BOOL(opt_utrace, utrace)
|
||||
}
|
||||
if (config_valgrind) {
|
||||
bool hit;
|
||||
CONF_HANDLE_BOOL_HIT(opt_valgrind,
|
||||
valgrind, hit)
|
||||
if (config_fill && opt_valgrind && hit) {
|
||||
opt_junk = false;
|
||||
opt_zero = false;
|
||||
if (opt_quarantine == 0) {
|
||||
opt_quarantine =
|
||||
JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
|
||||
}
|
||||
opt_redzone = true;
|
||||
}
|
||||
if (hit)
|
||||
continue;
|
||||
}
|
||||
if (config_xmalloc) {
|
||||
CONF_HANDLE_BOOL(opt_xmalloc, xmalloc)
|
||||
}
|
||||
@ -662,6 +693,11 @@ malloc_init_hard(void)
|
||||
return (true);
|
||||
}
|
||||
|
||||
if (config_fill && quarantine_boot()) {
|
||||
malloc_mutex_unlock(&init_lock);
|
||||
return (true);
|
||||
}
|
||||
|
||||
if (config_prof && prof_boot2()) {
|
||||
malloc_mutex_unlock(&init_lock);
|
||||
return (true);
|
||||
@ -763,7 +799,7 @@ je_malloc(size_t size)
|
||||
} else
|
||||
ret = imalloc(size);
|
||||
} else {
|
||||
if (config_stats)
|
||||
if (config_stats || (config_valgrind && opt_valgrind))
|
||||
usize = s2u(size);
|
||||
ret = imalloc(size);
|
||||
}
|
||||
@ -780,10 +816,11 @@ label_oom:
|
||||
if (config_prof && opt_prof && ret != NULL)
|
||||
prof_malloc(ret, usize, cnt);
|
||||
if (config_stats && ret != NULL) {
|
||||
assert(usize == isalloc(ret));
|
||||
assert(usize == isalloc(ret, config_prof));
|
||||
thread_allocated_tsd_get()->allocated += usize;
|
||||
}
|
||||
UTRACE(0, size, ret);
|
||||
JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -872,7 +909,7 @@ imemalign(void **memptr, size_t alignment, size_t size,
|
||||
|
||||
label_return:
|
||||
if (config_stats && result != NULL) {
|
||||
assert(usize == isalloc(result));
|
||||
assert(usize == isalloc(result, config_prof));
|
||||
thread_allocated_tsd_get()->allocated += usize;
|
||||
}
|
||||
if (config_prof && opt_prof && result != NULL)
|
||||
@ -886,8 +923,10 @@ JEMALLOC_ATTR(visibility("default"))
|
||||
int
|
||||
je_posix_memalign(void **memptr, size_t alignment, size_t size)
|
||||
{
|
||||
|
||||
return imemalign(memptr, alignment, size, sizeof(void *));
|
||||
int ret = imemalign(memptr, alignment, size, sizeof(void *));
|
||||
JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
|
||||
config_prof), false);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
JEMALLOC_ATTR(malloc)
|
||||
@ -902,6 +941,8 @@ je_aligned_alloc(size_t alignment, size_t size)
|
||||
ret = NULL;
|
||||
errno = err;
|
||||
}
|
||||
JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
|
||||
false);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -956,7 +997,7 @@ je_calloc(size_t num, size_t size)
|
||||
} else
|
||||
ret = icalloc(num_size);
|
||||
} else {
|
||||
if (config_stats)
|
||||
if (config_stats || (config_valgrind && opt_valgrind))
|
||||
usize = s2u(num_size);
|
||||
ret = icalloc(num_size);
|
||||
}
|
||||
@ -974,10 +1015,11 @@ label_return:
|
||||
if (config_prof && opt_prof && ret != NULL)
|
||||
prof_malloc(ret, usize, cnt);
|
||||
if (config_stats && ret != NULL) {
|
||||
assert(usize == isalloc(ret));
|
||||
assert(usize == isalloc(ret, config_prof));
|
||||
thread_allocated_tsd_get()->allocated += usize;
|
||||
}
|
||||
UTRACE(0, num_size, ret);
|
||||
JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -988,19 +1030,30 @@ je_realloc(void *ptr, size_t size)
|
||||
void *ret;
|
||||
size_t usize;
|
||||
size_t old_size = 0;
|
||||
size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
|
||||
prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
|
||||
prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL);
|
||||
|
||||
if (size == 0) {
|
||||
if (ptr != NULL) {
|
||||
/* realloc(ptr, 0) is equivalent to free(p). */
|
||||
if (config_prof || config_stats)
|
||||
old_size = isalloc(ptr);
|
||||
if (config_prof) {
|
||||
old_size = isalloc(ptr, true);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
old_rzsize = p2rz(ptr);
|
||||
} else if (config_stats) {
|
||||
old_size = isalloc(ptr, false);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
old_rzsize = u2rz(old_size);
|
||||
} else if (config_valgrind && opt_valgrind) {
|
||||
old_size = isalloc(ptr, false);
|
||||
old_rzsize = u2rz(old_size);
|
||||
}
|
||||
if (config_prof && opt_prof) {
|
||||
old_ctx = prof_ctx_get(ptr);
|
||||
cnt = NULL;
|
||||
}
|
||||
idalloc(ptr);
|
||||
iqalloc(ptr);
|
||||
ret = NULL;
|
||||
goto label_return;
|
||||
} else
|
||||
@ -1010,8 +1063,18 @@ je_realloc(void *ptr, size_t size)
|
||||
if (ptr != NULL) {
|
||||
assert(malloc_initialized || IS_INITIALIZER);
|
||||
|
||||
if (config_prof || config_stats)
|
||||
old_size = isalloc(ptr);
|
||||
if (config_prof) {
|
||||
old_size = isalloc(ptr, true);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
old_rzsize = p2rz(ptr);
|
||||
} else if (config_stats) {
|
||||
old_size = isalloc(ptr, false);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
old_rzsize = u2rz(old_size);
|
||||
} else if (config_valgrind && opt_valgrind) {
|
||||
old_size = isalloc(ptr, false);
|
||||
old_rzsize = u2rz(old_size);
|
||||
}
|
||||
if (config_prof && opt_prof) {
|
||||
usize = s2u(size);
|
||||
old_ctx = prof_ctx_get(ptr);
|
||||
@ -1035,7 +1098,7 @@ je_realloc(void *ptr, size_t size)
|
||||
old_ctx = NULL;
|
||||
}
|
||||
} else {
|
||||
if (config_stats)
|
||||
if (config_stats || (config_valgrind && opt_valgrind))
|
||||
usize = s2u(size);
|
||||
ret = iralloc(ptr, size, 0, 0, false, false);
|
||||
}
|
||||
@ -1076,7 +1139,8 @@ label_oom:
|
||||
ret = imalloc(size);
|
||||
}
|
||||
} else {
|
||||
if (config_stats)
|
||||
if (config_stats || (config_valgrind &&
|
||||
opt_valgrind))
|
||||
usize = s2u(size);
|
||||
ret = imalloc(size);
|
||||
}
|
||||
@ -1097,12 +1161,13 @@ label_return:
|
||||
prof_realloc(ret, usize, cnt, old_size, old_ctx);
|
||||
if (config_stats && ret != NULL) {
|
||||
thread_allocated_t *ta;
|
||||
assert(usize == isalloc(ret));
|
||||
assert(usize == isalloc(ret, config_prof));
|
||||
ta = thread_allocated_tsd_get();
|
||||
ta->allocated += usize;
|
||||
ta->deallocated += old_size;
|
||||
}
|
||||
UTRACE(ptr, size, ret);
|
||||
JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -1114,18 +1179,21 @@ je_free(void *ptr)
|
||||
UTRACE(ptr, 0, 0);
|
||||
if (ptr != NULL) {
|
||||
size_t usize;
|
||||
size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
|
||||
|
||||
assert(malloc_initialized || IS_INITIALIZER);
|
||||
|
||||
if (config_prof && opt_prof) {
|
||||
usize = isalloc(ptr);
|
||||
usize = isalloc(ptr, config_prof);
|
||||
prof_free(ptr, usize);
|
||||
} else if (config_stats) {
|
||||
usize = isalloc(ptr);
|
||||
}
|
||||
} else if (config_stats || config_valgrind)
|
||||
usize = isalloc(ptr, config_prof);
|
||||
if (config_stats)
|
||||
thread_allocated_tsd_get()->deallocated += usize;
|
||||
idalloc(ptr);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
rzsize = p2rz(ptr);
|
||||
iqalloc(ptr);
|
||||
JEMALLOC_VALGRIND_FREE(ptr, rzsize);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1145,6 +1213,7 @@ je_memalign(size_t alignment, size_t size)
|
||||
{
|
||||
void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
|
||||
imemalign(&ret, alignment, size, 1);
|
||||
JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
@ -1157,6 +1226,7 @@ je_valloc(size_t size)
|
||||
{
|
||||
void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
|
||||
imemalign(&ret, PAGE, size, 1);
|
||||
JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
|
||||
return (ret);
|
||||
}
|
||||
#endif
|
||||
@ -1209,9 +1279,9 @@ je_malloc_usable_size(const void *ptr)
|
||||
assert(malloc_initialized || IS_INITIALIZER);
|
||||
|
||||
if (config_ivsalloc)
|
||||
ret = ivsalloc(ptr);
|
||||
ret = ivsalloc(ptr, config_prof);
|
||||
else
|
||||
ret = (ptr != NULL) ? isalloc(ptr) : 0;
|
||||
ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0;
|
||||
|
||||
return (ret);
|
||||
}
|
||||
@ -1336,10 +1406,11 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
|
||||
|
||||
*ptr = p;
|
||||
if (config_stats) {
|
||||
assert(usize == isalloc(p));
|
||||
assert(usize == isalloc(p, config_prof));
|
||||
thread_allocated_tsd_get()->allocated += usize;
|
||||
}
|
||||
UTRACE(0, size, p);
|
||||
JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero);
|
||||
return (ALLOCM_SUCCESS);
|
||||
label_oom:
|
||||
if (config_xmalloc && opt_xmalloc) {
|
||||
@ -1360,6 +1431,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
|
||||
void *p, *q;
|
||||
size_t usize;
|
||||
size_t old_size;
|
||||
size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
|
||||
size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
|
||||
& (SIZE_T_MAX-1));
|
||||
bool zero = flags & ALLOCM_ZERO;
|
||||
@ -1384,7 +1456,9 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
|
||||
size_t max_usize = (alignment == 0) ? s2u(size+extra) :
|
||||
sa2u(size+extra, alignment, NULL);
|
||||
prof_ctx_t *old_ctx = prof_ctx_get(p);
|
||||
old_size = isalloc(p);
|
||||
old_size = isalloc(p, true);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
old_rzsize = p2rz(p);
|
||||
PROF_ALLOC_PREP(1, max_usize, cnt);
|
||||
if (cnt == NULL)
|
||||
goto label_oom;
|
||||
@ -1403,27 +1477,33 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
|
||||
usize = max_usize;
|
||||
arena_prof_promoted(q, usize);
|
||||
} else
|
||||
usize = isalloc(q);
|
||||
usize = isalloc(q, config_prof);
|
||||
} else {
|
||||
q = iralloc(p, size, extra, alignment, zero, no_move);
|
||||
if (q == NULL)
|
||||
goto label_err;
|
||||
usize = isalloc(q);
|
||||
usize = isalloc(q, config_prof);
|
||||
}
|
||||
prof_realloc(q, usize, cnt, old_size, old_ctx);
|
||||
if (rsize != NULL)
|
||||
*rsize = usize;
|
||||
} else {
|
||||
if (config_stats)
|
||||
old_size = isalloc(p);
|
||||
if (config_stats) {
|
||||
old_size = isalloc(p, false);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
old_rzsize = u2rz(old_size);
|
||||
} else if (config_valgrind && opt_valgrind) {
|
||||
old_size = isalloc(p, false);
|
||||
old_rzsize = u2rz(old_size);
|
||||
}
|
||||
q = iralloc(p, size, extra, alignment, zero, no_move);
|
||||
if (q == NULL)
|
||||
goto label_err;
|
||||
if (config_stats)
|
||||
usize = isalloc(q);
|
||||
usize = isalloc(q, config_prof);
|
||||
if (rsize != NULL) {
|
||||
if (config_stats == false)
|
||||
usize = isalloc(q);
|
||||
usize = isalloc(q, config_prof);
|
||||
*rsize = usize;
|
||||
}
|
||||
}
|
||||
@ -1436,6 +1516,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
|
||||
ta->deallocated += old_size;
|
||||
}
|
||||
UTRACE(p, size, q);
|
||||
JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero);
|
||||
return (ALLOCM_SUCCESS);
|
||||
label_err:
|
||||
if (no_move) {
|
||||
@ -1462,10 +1543,10 @@ je_sallocm(const void *ptr, size_t *rsize, int flags)
|
||||
assert(malloc_initialized || IS_INITIALIZER);
|
||||
|
||||
if (config_ivsalloc)
|
||||
sz = ivsalloc(ptr);
|
||||
sz = ivsalloc(ptr, config_prof);
|
||||
else {
|
||||
assert(ptr != NULL);
|
||||
sz = isalloc(ptr);
|
||||
sz = isalloc(ptr, config_prof);
|
||||
}
|
||||
assert(rsize != NULL);
|
||||
*rsize = sz;
|
||||
@ -1479,21 +1560,25 @@ int
|
||||
je_dallocm(void *ptr, int flags)
|
||||
{
|
||||
size_t usize;
|
||||
size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
|
||||
|
||||
assert(ptr != NULL);
|
||||
assert(malloc_initialized || IS_INITIALIZER);
|
||||
|
||||
UTRACE(ptr, 0, 0);
|
||||
if (config_stats)
|
||||
usize = isalloc(ptr);
|
||||
if (config_stats || config_valgrind)
|
||||
usize = isalloc(ptr, config_prof);
|
||||
if (config_prof && opt_prof) {
|
||||
if (config_stats == false)
|
||||
usize = isalloc(ptr);
|
||||
if (config_stats == false && config_valgrind == false)
|
||||
usize = isalloc(ptr, config_prof);
|
||||
prof_free(ptr, usize);
|
||||
}
|
||||
if (config_stats)
|
||||
thread_allocated_tsd_get()->deallocated += usize;
|
||||
idalloc(ptr);
|
||||
if (config_valgrind && opt_valgrind)
|
||||
rzsize = p2rz(ptr);
|
||||
iqalloc(ptr);
|
||||
JEMALLOC_VALGRIND_FREE(ptr, rzsize);
|
||||
|
||||
return (ALLOCM_SUCCESS);
|
||||
}
|
||||
|
||||
163
src/quarantine.c
Normal file
163
src/quarantine.c
Normal file
@ -0,0 +1,163 @@
|
||||
#include "jemalloc/internal/jemalloc_internal.h"
|
||||
|
||||
/******************************************************************************/
|
||||
/* Data. */
|
||||
|
||||
typedef struct quarantine_s quarantine_t;
|
||||
|
||||
struct quarantine_s {
|
||||
size_t curbytes;
|
||||
size_t curobjs;
|
||||
size_t first;
|
||||
#define LG_MAXOBJS_INIT 10
|
||||
size_t lg_maxobjs;
|
||||
void *objs[1]; /* Dynamically sized ring buffer. */
|
||||
};
|
||||
|
||||
static void quarantine_cleanup(void *arg);
|
||||
|
||||
malloc_tsd_data(static, quarantine, quarantine_t *, NULL)
|
||||
malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL,
|
||||
quarantine_cleanup)
|
||||
|
||||
/******************************************************************************/
|
||||
/* Function prototypes for non-inline static functions. */
|
||||
|
||||
static quarantine_t *quarantine_init(size_t lg_maxobjs);
|
||||
static quarantine_t *quarantine_grow(quarantine_t *quarantine);
|
||||
static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
static quarantine_t *
|
||||
quarantine_init(size_t lg_maxobjs)
|
||||
{
|
||||
quarantine_t *quarantine;
|
||||
|
||||
quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
|
||||
((ZU(1) << lg_maxobjs) * sizeof(void *)));
|
||||
if (quarantine == NULL)
|
||||
return (NULL);
|
||||
quarantine->curbytes = 0;
|
||||
quarantine->curobjs = 0;
|
||||
quarantine->first = 0;
|
||||
quarantine->lg_maxobjs = lg_maxobjs;
|
||||
|
||||
quarantine_tsd_set(&quarantine);
|
||||
|
||||
return (quarantine);
|
||||
}
|
||||
|
||||
static quarantine_t *
|
||||
quarantine_grow(quarantine_t *quarantine)
|
||||
{
|
||||
quarantine_t *ret;
|
||||
|
||||
ret = quarantine_init(quarantine->lg_maxobjs + 1);
|
||||
if (ret == NULL)
|
||||
return (quarantine);
|
||||
|
||||
ret->curbytes = quarantine->curbytes;
|
||||
if (quarantine->first + quarantine->curobjs < (ZU(1) <<
|
||||
quarantine->lg_maxobjs)) {
|
||||
/* objs ring buffer data are contiguous. */
|
||||
memcpy(ret->objs, &quarantine->objs[quarantine->first],
|
||||
quarantine->curobjs * sizeof(void *));
|
||||
ret->curobjs = quarantine->curobjs;
|
||||
} else {
|
||||
/* objs ring buffer data wrap around. */
|
||||
size_t ncopy = (ZU(1) << quarantine->lg_maxobjs) -
|
||||
quarantine->first;
|
||||
memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy *
|
||||
sizeof(void *));
|
||||
ret->curobjs = ncopy;
|
||||
if (quarantine->curobjs != 0) {
|
||||
memcpy(&ret->objs[ret->curobjs], quarantine->objs,
|
||||
quarantine->curobjs - ncopy);
|
||||
}
|
||||
}
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static void
|
||||
quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
|
||||
{
|
||||
|
||||
while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) {
|
||||
void *ptr = quarantine->objs[quarantine->first];
|
||||
size_t usize = isalloc(ptr, config_prof);
|
||||
idalloc(ptr);
|
||||
quarantine->curbytes -= usize;
|
||||
quarantine->curobjs--;
|
||||
quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
|
||||
quarantine->lg_maxobjs) - 1);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
quarantine(void *ptr)
|
||||
{
|
||||
quarantine_t *quarantine;
|
||||
size_t usize = isalloc(ptr, config_prof);
|
||||
|
||||
assert(config_fill);
|
||||
assert(opt_quarantine);
|
||||
|
||||
quarantine = *quarantine_tsd_get();
|
||||
if (quarantine == NULL && (quarantine =
|
||||
quarantine_init(LG_MAXOBJS_INIT)) == NULL) {
|
||||
idalloc(ptr);
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* Drain one or more objects if the quarantine size limit would be
|
||||
* exceeded by appending ptr.
|
||||
*/
|
||||
if (quarantine->curbytes + usize > opt_quarantine) {
|
||||
size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine
|
||||
- usize : 0;
|
||||
quarantine_drain(quarantine, upper_bound);
|
||||
}
|
||||
/* Grow the quarantine ring buffer if it's full. */
|
||||
if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs))
|
||||
quarantine = quarantine_grow(quarantine);
|
||||
/* quarantine_grow() must free a slot if it fails to grow. */
|
||||
assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs));
|
||||
/* Append ptr if its size doesn't exceed the quarantine size. */
|
||||
if (quarantine->curbytes + usize <= opt_quarantine) {
|
||||
size_t offset = (quarantine->first + quarantine->curobjs) &
|
||||
((ZU(1) << quarantine->lg_maxobjs) - 1);
|
||||
quarantine->objs[offset] = ptr;
|
||||
quarantine->curbytes += usize;
|
||||
quarantine->curobjs++;
|
||||
if (opt_junk)
|
||||
memset(ptr, 0x5a, usize);
|
||||
} else {
|
||||
assert(quarantine->curbytes == 0);
|
||||
idalloc(ptr);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
quarantine_cleanup(void *arg)
|
||||
{
|
||||
quarantine_t *quarantine = *(quarantine_t **)arg;
|
||||
|
||||
if (quarantine != NULL) {
|
||||
quarantine_drain(quarantine, 0);
|
||||
idalloc(quarantine);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
quarantine_boot(void)
|
||||
{
|
||||
|
||||
assert(config_fill);
|
||||
|
||||
if (quarantine_tsd_boot())
|
||||
return (true);
|
||||
|
||||
return (false);
|
||||
}
|
||||
@ -382,8 +382,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
OPT_WRITE_SSIZE_T(lg_dirty_mult)
|
||||
OPT_WRITE_BOOL(stats_print)
|
||||
OPT_WRITE_BOOL(junk)
|
||||
OPT_WRITE_SIZE_T(quarantine)
|
||||
OPT_WRITE_BOOL(redzone)
|
||||
OPT_WRITE_BOOL(zero)
|
||||
OPT_WRITE_BOOL(utrace)
|
||||
OPT_WRITE_BOOL(valgrind)
|
||||
OPT_WRITE_BOOL(xmalloc)
|
||||
OPT_WRITE_BOOL(tcache)
|
||||
OPT_WRITE_SSIZE_T(lg_tcache_max)
|
||||
|
||||
@ -75,6 +75,10 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
|
||||
(uintptr_t)chunk) >> LG_PAGE;
|
||||
arena_chunk_map_t *mapelm =
|
||||
&chunk->map[pageind-map_bias];
|
||||
if (config_fill && opt_junk) {
|
||||
arena_alloc_junk_small(ptr,
|
||||
&arena_bin_info[binind], true);
|
||||
}
|
||||
arena_dalloc_bin(arena, chunk, ptr, mapelm);
|
||||
} else {
|
||||
/*
|
||||
@ -298,7 +302,7 @@ tcache_destroy(tcache_t *tcache)
|
||||
malloc_mutex_unlock(&tcache->arena->lock);
|
||||
}
|
||||
|
||||
tcache_size = arena_salloc(tcache);
|
||||
tcache_size = arena_salloc(tcache, false);
|
||||
if (tcache_size <= SMALL_MAXCLASS) {
|
||||
arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
|
||||
arena_t *arena = chunk->arena;
|
||||
|
||||
10
src/zone.c
10
src/zone.c
@ -56,7 +56,7 @@ zone_size(malloc_zone_t *zone, void *ptr)
|
||||
* not work in practice, we must check all pointers to assure that they
|
||||
* reside within a mapped chunk before determining size.
|
||||
*/
|
||||
return (ivsalloc(ptr));
|
||||
return (ivsalloc(ptr, config_prof));
|
||||
}
|
||||
|
||||
static void *
|
||||
@ -87,7 +87,7 @@ static void
|
||||
zone_free(malloc_zone_t *zone, void *ptr)
|
||||
{
|
||||
|
||||
if (ivsalloc(ptr) != 0) {
|
||||
if (ivsalloc(ptr, config_prof) != 0) {
|
||||
je_free(ptr);
|
||||
return;
|
||||
}
|
||||
@ -99,7 +99,7 @@ static void *
|
||||
zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
|
||||
{
|
||||
|
||||
if (ivsalloc(ptr) != 0)
|
||||
if (ivsalloc(ptr, config_prof) != 0)
|
||||
return (je_realloc(ptr, size));
|
||||
|
||||
return (realloc(ptr, size));
|
||||
@ -122,8 +122,8 @@ static void
|
||||
zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
|
||||
{
|
||||
|
||||
if (ivsalloc(ptr) != 0) {
|
||||
assert(ivsalloc(ptr) == size);
|
||||
if (ivsalloc(ptr, config_prof) != 0) {
|
||||
assert(ivsalloc(ptr, config_prof) == size);
|
||||
je_free(ptr);
|
||||
return;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user