diff --git a/INSTALL b/INSTALL index 8a825df9..a5942ec8 100644 --- a/INSTALL +++ b/INSTALL @@ -113,8 +113,12 @@ any of the following arguments (not a definitive list) to 'configure': mmap(2). --disable-fill - Disable support for junk/zero filling of memory. See the "opt.junk"/ - "opt.zero" option documentation for usage details. + Disable support for junk/zero filling of memory, quarantine, and redzones. + See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option + documentation for usage details. + +--disable-valgrind + Disable support for Valgrind. --disable-experimental Disable support for the experimental API (*allocm()). diff --git a/Makefile.in b/Makefile.in index 7df4fc60..8aa94253 100644 --- a/Makefile.in +++ b/Makefile.in @@ -49,9 +49,9 @@ CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/atomic.c \ @srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \ @srcroot@src/ckh.c @srcroot@src/ctl.c @srcroot@src/extent.c \ @srcroot@src/hash.c @srcroot@src/huge.c @srcroot@src/mb.c \ - @srcroot@src/mutex.c @srcroot@src/prof.c @srcroot@src/rtree.c \ - @srcroot@src/stats.c @srcroot@src/tcache.c @srcroot@src/util.c \ - @srcroot@src/tsd.c + @srcroot@src/mutex.c @srcroot@src/prof.c @srcroot@src/quarantine.c \ + @srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c \ + @srcroot@src/util.c @srcroot@src/tsd.c ifeq (macho, @abi@) CSRCS += @srcroot@src/zone.c endif diff --git a/configure.ac b/configure.ac index 8e94b5c9..a272ecd0 100644 --- a/configure.ac +++ b/configure.ac @@ -685,7 +685,8 @@ AC_SUBST([enable_dss]) dnl Support the junk/zero filling option by default. AC_ARG_ENABLE([fill], - [AS_HELP_STRING([--disable-fill], [Disable support for junk/zero filling])], + [AS_HELP_STRING([--disable-fill], + [Disable support for junk/zero filling, quarantine, and redzones])], [if test "x$enable_fill" = "xno" ; then enable_fill="0" else @@ -727,6 +728,38 @@ if test "x$enable_utrace" = "x1" ; then fi AC_SUBST([enable_utrace]) +dnl Support Valgrind by default. +AC_ARG_ENABLE([valgrind], + [AS_HELP_STRING([--disable-valgrind], [Disable support for Valgrind])], +[if test "x$enable_valgrind" = "xno" ; then + enable_valgrind="0" +else + enable_valgrind="1" +fi +], +[enable_valgrind="1"] +) +if test "x$enable_valgrind" = "x1" ; then + JE_COMPILABLE([valgrind], [ +#include +#include + +#if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__) \ + && (__VALGRIND_MAJOR__ > 3 || (__VALGRIND_MAJOR__ == 3 && \ + __VALGRIND_MINOR__ >= 6)) +#else +# error "Incompatible Valgrind version" +#endif +], [], [je_cv_valgrind]) + if test "x${je_cv_valgrind}" = "xno" ; then + enable_valgrind="0" + fi + if test "x$enable_valgrind" = "x1" ; then + AC_DEFINE([JEMALLOC_VALGRIND], [ ]) + fi +fi +AC_SUBST([enable_valgrind]) + dnl Do not support the xmalloc option by default. AC_ARG_ENABLE([xmalloc], [AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])], @@ -1088,8 +1121,9 @@ AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) AC_MSG_RESULT([tcache : ${enable_tcache}]) AC_MSG_RESULT([fill : ${enable_fill}]) -AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) AC_MSG_RESULT([utrace : ${enable_utrace}]) +AC_MSG_RESULT([valgrind : ${enable_valgrind}]) +AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) AC_MSG_RESULT([dss : ${enable_dss}]) AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) AC_MSG_RESULT([tls : ${enable_tls}]) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 8ae82621..a47c7635 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -720,6 +720,16 @@ for (i = 0; i < nbins; i++) { build configuration. + + + config.valgrind + (bool) + r- + + was specified during + build configuration. + + config.xmalloc @@ -819,6 +829,47 @@ for (i = 0; i < nbins; i++) { configuration, in which case it is enabled by default. + + + opt.quarantine + (size_t) + r- + [] + + Per thread quarantine size in bytes. If non-zero, each + thread maintains a FIFO object quarantine that stores up to the + specified number of bytes of memory. The quarantined memory is not + freed until it is released from quarantine, though it is immediately + junk-filled if the opt.junk option is + enabled. This feature is of particular use in combination with Valgrind, which can detect + attempts to access quarantined objects. This is intended for debugging + and will impact performance negatively. The default quarantine size is + 0. + + + + + opt.redzone + (bool) + r- + [] + + Redzones enabled/disabled. If enabled, small + allocations have redzones before and after them. Furthermore, if the + opt.junk option is + enabled, the redzones are checked for corruption during deallocation. + However, the primary intended purpose of this feature is to be used in + combination with Valgrind, which needs + redzones in order to do effective buffer overflow/underflow detection. + This option is intended for debugging and will impact performance + negatively. This option is disabled by default unless + is specified during configuration, in + which case it is enabled by default. + + opt.zero @@ -849,6 +900,25 @@ for (i = 0; i < nbins; i++) { is disabled by default. + + + opt.valgrind + (bool) + r- + [] + + Valgrind support + enabled/disabled. If enabled, several other options are automatically + modified during options processing to work well with Valgrind: opt.junk and opt.zero are set to false, + opt.quarantine + is set to 16 MiB, and opt.redzone is set to + true. This option is disabled by default. + + opt.xmalloc @@ -1764,10 +1834,11 @@ malloc_conf = "xmalloc:true";]]> This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information - would be prohibitive. There are a number of allocator implementations - available on the Internet which focus on detecting and pinpointing problems - by trading performance for extra sanity checks and detailed - diagnostics. + would be prohibitive. However, jemalloc does integrate with the most + excellent Valgrind tool if + the configuration option is enabled and + the opt.valgrind + option is enabled. DIAGNOSTIC MESSAGES diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index d25a2b1d..f52fac42 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -16,7 +16,7 @@ * constraint is relaxed (ignored) for runs that are so small that the * per-region overhead is greater than: * - * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) + * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) */ #define RUN_BFP 12 /* \/ Implicit binary fixed point. */ @@ -27,6 +27,12 @@ #define LG_RUN_MAXREGS 11 #define RUN_MAXREGS (1U << LG_RUN_MAXREGS) +/* + * Minimum redzone size. Redzones may be larger than this if necessary to + * preserve region alignment. + */ +#define REDZONE_MINSIZE 16 + /* * The minimum ratio of active:dirty pages per arena is computed as: * @@ -192,11 +198,50 @@ struct arena_run_s { * Read-only information associated with each element of arena_t's bins array * is stored separately, partly to reduce memory usage (only one copy, rather * than one per arena), but mainly to avoid false cacheline sharing. + * + * Each run has the following layout: + * + * /--------------------\ + * | arena_run_t header | + * | ... | + * bitmap_offset | bitmap | + * | ... | + * ctx0_offset | ctx map | + * | ... | + * |--------------------| + * | redzone | + * reg0_offset | region 0 | + * | redzone | + * |--------------------| \ + * | redzone | | + * | region 1 | > reg_interval + * | redzone | / + * |--------------------| + * | ... | + * | ... | + * | ... | + * |--------------------| + * | redzone | + * | region nregs-1 | + * | redzone | + * |--------------------| + * | alignment pad? | + * \--------------------/ + * + * reg_interval has at least the same minimum alignment as reg_size; this + * preserves the alignment constraint that sa2u() depends on. Alignment pad is + * either 0 or redzone_size; it is present only if needed to align reg0_offset. */ struct arena_bin_info_s { /* Size of regions in a run for this bin's size class. */ size_t reg_size; + /* Redzone size. */ + size_t redzone_size; + + /* Interval between regions (reg_size + (redzone_size << 1)). */ + size_t reg_interval; + /* Total size of a run for this bin's size class. */ size_t run_size; @@ -357,13 +402,15 @@ void arena_purge_all(arena_t *arena); void arena_prof_accum(arena_t *arena, uint64_t accumbytes); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, uint64_t prof_accumbytes); +void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, + bool zero); +void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); void *arena_malloc_small(arena_t *arena, size_t size, bool zero); void *arena_malloc_large(arena_t *arena, size_t size, bool zero); void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, bool zero); -size_t arena_salloc(const void *ptr); +size_t arena_salloc(const void *ptr, bool demote); void arena_prof_promoted(const void *ptr, size_t size); -size_t arena_salloc_demote(const void *ptr); void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_t *mapelm); void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); @@ -408,7 +455,7 @@ JEMALLOC_INLINE unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) { unsigned shift, diff, regind; - size_t size; + size_t interval; /* * Freeing a pointer lower than region zero can cause assertion @@ -425,12 +472,12 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) bin_info->reg0_offset); /* Rescale (factor powers of 2 out of the numerator and denominator). */ - size = bin_info->reg_size; - shift = ffs(size) - 1; + interval = bin_info->reg_interval; + shift = ffs(interval) - 1; diff >>= shift; - size >>= shift; + interval >>= shift; - if (size == 1) { + if (interval == 1) { /* The divisor was a power of 2. */ regind = diff; } else { @@ -442,7 +489,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) * * becomes * - * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT + * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT * * We can omit the first three elements, because we never * divide by 0, and 1 and 2 are both powers of two, which are @@ -450,7 +497,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) */ #define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) - static const unsigned size_invs[] = { + static const unsigned interval_invs[] = { SIZE_INV(3), SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), @@ -461,14 +508,16 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) }; - if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) - regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; - else - regind = diff / size; + if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + + 2)) { + regind = (diff * interval_invs[interval - 3]) >> + SIZE_INV_SHIFT; + } else + regind = diff / interval; #undef SIZE_INV #undef SIZE_INV_SHIFT } - assert(diff == regind * size); + assert(diff == regind * interval); assert(regind < bin_info->nregs); return (regind); @@ -610,7 +659,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) &arena_bin_info[binind]; assert(((uintptr_t)ptr - ((uintptr_t)run + (uintptr_t)bin_info->reg0_offset)) % - bin_info->reg_size == 0); + bin_info->reg_interval == 0); } malloc_mutex_lock(&bin->lock); arena_dalloc_bin(arena, chunk, ptr, mapelm); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 66dd357f..a16e5e27 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -40,6 +40,11 @@ #include #endif +#ifdef JEMALLOC_VALGRIND +#include +#include +#endif + #include "jemalloc/internal/private_namespace.h" #ifdef JEMALLOC_CC_SILENCE @@ -125,6 +130,13 @@ static const bool config_utrace = false #endif ; +static const bool config_valgrind = +#ifdef JEMALLOC_VALGRIND + true +#else + false +#endif + ; static const bool config_xmalloc = #ifdef JEMALLOC_XMALLOC true @@ -281,6 +293,77 @@ static const bool config_ivsalloc = #define PAGE_CEILING(s) \ (((s) + PAGE_MASK) & ~PAGE_MASK) +#ifdef JEMALLOC_VALGRIND +/* + * The JEMALLOC_VALGRIND_*() macros must be macros rather than functions + * so that when Valgrind reports errors, there are no extra stack frames + * in the backtraces. + * + * The size that is reported to valgrind must be consistent through a chain of + * malloc..realloc..realloc calls. Request size isn't recorded anywhere in + * jemalloc, so it is critical that all callers of these macros provide usize + * rather than request size. As a result, buffer overflow detection is + * technically weakened for the standard API, though it is generally accepted + * practice to consider any extra bytes reported by malloc_usable_size() as + * usable space. + */ +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ + if (config_valgrind && opt_valgrind && cond) \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ +} while (0) +#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ + old_rzsize, zero) do { \ + if (config_valgrind && opt_valgrind) { \ + size_t rzsize = p2rz(ptr); \ + \ + if (ptr == old_ptr) { \ + VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ + usize, rzsize); \ + if (zero && old_usize < usize) { \ + VALGRIND_MAKE_MEM_DEFINED( \ + (void *)((uintptr_t)ptr + \ + old_usize), usize - old_usize); \ + } \ + } else { \ + if (old_ptr != NULL) { \ + VALGRIND_FREELIKE_BLOCK(old_ptr, \ + old_rzsize); \ + } \ + if (ptr != NULL) { \ + size_t copy_size = (old_usize < usize) \ + ? old_usize : usize; \ + size_t tail_size = usize - copy_size; \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \ + rzsize, false); \ + if (copy_size > 0) { \ + VALGRIND_MAKE_MEM_DEFINED(ptr, \ + copy_size); \ + } \ + if (zero && tail_size > 0) { \ + VALGRIND_MAKE_MEM_DEFINED( \ + (void *)((uintptr_t)ptr + \ + copy_size), tail_size); \ + } \ + } \ + } \ + } \ +} while (0) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ + if (config_valgrind && opt_valgrind) \ + VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \ +} while (0) +#else +#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) +#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) +#define VALGRIND_FREELIKE_BLOCK(addr, rzB) +#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) +#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) +#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ + old_rzsize, zero) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) +#endif + #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" @@ -300,6 +383,7 @@ static const bool config_ivsalloc = #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_TYPES @@ -325,6 +409,7 @@ static const bool config_ivsalloc = #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" typedef struct { @@ -343,7 +428,10 @@ typedef struct { extern bool opt_abort; extern bool opt_junk; +extern size_t opt_quarantine; +extern bool opt_redzone; extern bool opt_utrace; +extern bool opt_valgrind; extern bool opt_xmalloc; extern bool opt_zero; extern size_t opt_narenas; @@ -385,6 +473,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_EXTERNS @@ -550,14 +639,18 @@ choose_arena(arena_t *arena) #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/arena.h" #include "jemalloc/internal/hash.h" +#include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE void *imalloc(size_t size); void *icalloc(size_t size); void *ipalloc(size_t usize, size_t alignment, bool zero); -size_t isalloc(const void *ptr); -size_t ivsalloc(const void *ptr); +size_t isalloc(const void *ptr, bool demote); +size_t ivsalloc(const void *ptr, bool demote); +size_t u2rz(size_t usize); +size_t p2rz(const void *ptr); void idalloc(void *ptr); +void iqalloc(void *ptr); void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, bool no_move); malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) @@ -621,21 +714,25 @@ ipalloc(size_t usize, size_t alignment, bool zero) return (ret); } +/* + * Typical usage: + * void *ptr = [...] + * size_t sz = isalloc(ptr, config_prof); + */ JEMALLOC_INLINE size_t -isalloc(const void *ptr) +isalloc(const void *ptr, bool demote) { size_t ret; arena_chunk_t *chunk; assert(ptr != NULL); + /* Demotion only makes sense if config_prof is true. */ + assert(config_prof || demote == false); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - if (config_prof) - ret = arena_salloc_demote(ptr); - else - ret = arena_salloc(ptr); + ret = arena_salloc(ptr, demote); } else ret = huge_salloc(ptr); @@ -643,14 +740,36 @@ isalloc(const void *ptr) } JEMALLOC_INLINE size_t -ivsalloc(const void *ptr) +ivsalloc(const void *ptr, bool demote) { /* Return 0 if ptr is not within a chunk managed by jemalloc. */ if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) return (0); - return (isalloc(ptr)); + return (isalloc(ptr, demote)); +} + +JEMALLOC_INLINE size_t +u2rz(size_t usize) +{ + size_t ret; + + if (usize <= SMALL_MAXCLASS) { + size_t binind = SMALL_SIZE2BIN(usize); + ret = arena_bin_info[binind].redzone_size; + } else + ret = 0; + + return (ret); +} + +JEMALLOC_INLINE size_t +p2rz(const void *ptr) +{ + size_t usize = isalloc(ptr, false); + + return (u2rz(usize)); } JEMALLOC_INLINE void @@ -667,6 +786,16 @@ idalloc(void *ptr) huge_dalloc(ptr, true); } +JEMALLOC_INLINE void +iqalloc(void *ptr) +{ + + if (config_fill && opt_quarantine) + quarantine(ptr); + else + idalloc(ptr); +} + JEMALLOC_INLINE void * iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, bool no_move) @@ -677,14 +806,14 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, assert(ptr != NULL); assert(size != 0); - oldsize = isalloc(ptr); + oldsize = isalloc(ptr, config_prof); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { size_t usize, copysize; /* - * Existing object alignment is inadquate; allocate new space + * Existing object alignment is inadequate; allocate new space * and copy. */ if (no_move) @@ -711,7 +840,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloc(ptr); return (ret); } diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h index de3042eb..a962192c 100644 --- a/include/jemalloc/internal/private_namespace.h +++ b/include/jemalloc/internal/private_namespace.h @@ -1,7 +1,9 @@ +#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small) #define arena_bin_index JEMALLOC_N(arena_bin_index) #define arena_boot JEMALLOC_N(arena_boot) #define arena_dalloc JEMALLOC_N(arena_dalloc) #define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) #define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) #define arena_malloc JEMALLOC_N(arena_malloc) #define arena_malloc_large JEMALLOC_N(arena_malloc_large) @@ -20,7 +22,6 @@ #define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) #define arena_run_regind JEMALLOC_N(arena_run_regind) #define arena_salloc JEMALLOC_N(arena_salloc) -#define arena_salloc_demote JEMALLOC_N(arena_salloc_demote) #define arena_stats_merge JEMALLOC_N(arena_stats_merge) #define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) #define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index) @@ -136,6 +137,7 @@ #define idalloc JEMALLOC_N(idalloc) #define imalloc JEMALLOC_N(imalloc) #define ipalloc JEMALLOC_N(ipalloc) +#define iqalloc JEMALLOC_N(iqalloc) #define iralloc JEMALLOC_N(iralloc) #define isalloc JEMALLOC_N(isalloc) #define ivsalloc JEMALLOC_N(ivsalloc) @@ -176,6 +178,7 @@ #define opt_utrace JEMALLOC_N(opt_utrace) #define opt_xmalloc JEMALLOC_N(opt_xmalloc) #define opt_zero JEMALLOC_N(opt_zero) +#define p2rz JEMALLOC_N(p2rz) #define pow2_ceil JEMALLOC_N(pow2_ceil) #define prof_backtrace JEMALLOC_N(prof_backtrace) #define prof_boot0 JEMALLOC_N(prof_boot0) @@ -195,6 +198,8 @@ #define prof_tdata_init JEMALLOC_N(prof_tdata_init) #define prof_tdata_tls JEMALLOC_N(prof_tdata_tls) #define pthread_create JEMALLOC_N(pthread_create) +#define quarantine JEMALLOC_N(quarantine) +#define quarantine_boot JEMALLOC_N(quarantine_boot) #define register_zone JEMALLOC_N(register_zone) #define rtree_get JEMALLOC_N(rtree_get) #define rtree_get_locked JEMALLOC_N(rtree_get_locked) @@ -229,3 +234,4 @@ #define thread_allocated_get JEMALLOC_N(thread_allocated_get) #define thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard) #define thread_allocated_tls JEMALLOC_N(thread_allocated_tls) +#define u2rz JEMALLOC_N(u2rz) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 34929e7e..a37bb448 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -378,7 +378,7 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) cassert(config_prof); assert(ptr != NULL); - assert(size == isalloc(ptr)); + assert(size == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { if (prof_sample_accum_update(size)) { @@ -427,7 +427,7 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); if (ptr != NULL) { - assert(size == isalloc(ptr)); + assert(size == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { if (prof_sample_accum_update(size)) { /* @@ -500,7 +500,7 @@ prof_free(const void *ptr, size_t size) cassert(config_prof); if ((uintptr_t)ctx > (uintptr_t)1) { - assert(size == isalloc(ptr)); + assert(size == isalloc(ptr, true)); prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt); if (tcnt != NULL) { diff --git a/include/jemalloc/internal/quarantine.h b/include/jemalloc/internal/quarantine.h new file mode 100644 index 00000000..38f3d696 --- /dev/null +++ b/include/jemalloc/internal/quarantine.h @@ -0,0 +1,24 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* Default per thread quarantine size if valgrind is enabled. */ +#define JEMALLOC_VALGRIND_QUARANTINE_DEFAULT (ZU(1) << 24) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +void quarantine(void *ptr); +bool quarantine_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 93e721d5..9d8c992d 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -340,17 +340,24 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) if (ret == NULL) return (NULL); } - assert(arena_salloc(ret) == arena_bin_info[binind].reg_size); + assert(arena_salloc(ret, false) == arena_bin_info[binind].reg_size); if (zero == false) { if (config_fill) { - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) + if (opt_junk) { + arena_alloc_junk_small(ret, + &arena_bin_info[binind], false); + } else if (opt_zero) memset(ret, 0, size); } - } else + } else { + if (config_fill && opt_junk) { + arena_alloc_junk_small(ret, &arena_bin_info[binind], + true); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } if (config_stats) tbin->tstats.nrequests++; @@ -397,8 +404,10 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) else if (opt_zero) memset(ret, 0, size); } - } else + } else { + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } if (config_stats) tbin->tstats.nrequests++; @@ -422,7 +431,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr) size_t pageind, binind; arena_chunk_map_t *mapelm; - assert(arena_salloc(ptr) <= SMALL_MAXCLASS); + assert(arena_salloc(ptr, false) <= SMALL_MAXCLASS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena = chunk->arena; @@ -436,7 +445,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr) assert(binind < NBINS); if (config_fill && opt_junk) - memset(ptr, 0x5a, arena_bin_info[binind].reg_size); + arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; @@ -459,8 +468,8 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) tcache_bin_info_t *tbin_info; assert((size & PAGE_MASK) == 0); - assert(arena_salloc(ptr) > SMALL_MAXCLASS); - assert(arena_salloc(ptr) <= tcache_maxclass); + assert(arena_salloc(ptr, false) > SMALL_MAXCLASS); + assert(arena_salloc(ptr, false) <= tcache_maxclass); binind = NBINS + (size >> LG_PAGE) - 1; diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index 3d3ea3ab..d360ae3f 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -140,7 +140,6 @@ malloc_write(const char *s) je_malloc_message(NULL, s); } - #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in index 8e7442d6..7770a7af 100644 --- a/include/jemalloc/jemalloc_defs.h.in +++ b/include/jemalloc/jemalloc_defs.h.in @@ -148,7 +148,7 @@ */ #undef JEMALLOC_DSS -/* Support memory filling (junk/zero). */ +/* Support memory filling (junk/zero/quarantine/redzone). */ #undef JEMALLOC_FILL /* Support the experimental API. */ @@ -157,6 +157,9 @@ /* Support utrace(2)-based tracing. */ #undef JEMALLOC_UTRACE +/* Support Valgrind. */ +#undef JEMALLOC_VALGRIND + /* Support optional abort() on OOM. */ #undef JEMALLOC_XMALLOC diff --git a/src/arena.c b/src/arena.c index c84aaf47..1d4f61ea 100644 --- a/src/arena.c +++ b/src/arena.c @@ -140,7 +140,7 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + - (uintptr_t)(bin_info->reg_size * regind)); + (uintptr_t)(bin_info->reg_interval * regind)); run->nfree--; if (regind == run->nextind) run->nextind++; @@ -161,8 +161,8 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) assert(run->nfree < bin_info->nregs); /* Freeing an interior pointer can cause assertion failure. */ assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size - == 0); + (uintptr_t)bin_info->reg0_offset)) % + (uintptr_t)bin_info->reg_interval == 0); assert((uintptr_t)ptr >= (uintptr_t)run + (uintptr_t)bin_info->reg0_offset); /* Freeing an unallocated pointer can cause assertion failure. */ @@ -260,10 +260,18 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, for (i = 0; i < need_pages; i++) { if ((chunk->map[run_ind+i-map_bias].bits & CHUNK_MAP_UNZEROED) != 0) { + VALGRIND_MAKE_MEM_UNDEFINED( + (void *)((uintptr_t) + chunk + ((run_ind+i) << + LG_PAGE)), PAGE); memset((void *)((uintptr_t) chunk + ((run_ind+i) << LG_PAGE)), 0, PAGE); } else if (config_debug) { + VALGRIND_MAKE_MEM_DEFINED( + (void *)((uintptr_t) + chunk + ((run_ind+i) << + LG_PAGE)), PAGE); arena_chunk_validate_zeroed( chunk, run_ind+i); } @@ -273,6 +281,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, * The run is dirty, so all pages must be * zeroed. */ + VALGRIND_MAKE_MEM_UNDEFINED((void + *)((uintptr_t)chunk + (run_ind << + LG_PAGE)), (need_pages << LG_PAGE)); memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, (need_pages << LG_PAGE)); } @@ -1245,6 +1256,10 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, ptr = arena_bin_malloc_hard(arena, bin); if (ptr == NULL) break; + if (config_fill && opt_junk) { + arena_alloc_junk_small(ptr, &arena_bin_info[binind], + true); + } /* Insert such that low regions get used first. */ tbin->avail[nfill - 1 - i] = ptr; } @@ -1259,6 +1274,55 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, tbin->ncached = i; } +void +arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) +{ + + if (zero) { + size_t redzone_size = bin_info->redzone_size; + memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, + redzone_size); + memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, + redzone_size); + } else { + memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, + bin_info->reg_interval); + } +} + +void +arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) +{ + size_t size = bin_info->reg_size; + size_t redzone_size = bin_info->redzone_size; + size_t i; + bool error = false; + + for (i = 1; i <= redzone_size; i++) { + unsigned byte; + if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) { + error = true; + malloc_printf(": Corrupt redzone " + "%zu byte%s before %p (size %zu), byte=%#x\n", i, + (i == 1) ? "" : "s", ptr, size, byte); + } + } + for (i = 0; i < redzone_size; i++) { + unsigned byte; + if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) { + error = true; + malloc_printf(": Corrupt redzone " + "%zu byte%s after end of %p (size %zu), byte=%#x\n", + i, (i == 1) ? "" : "s", ptr, size, byte); + } + } + if (opt_abort && error) + abort(); + + memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, + bin_info->reg_interval); +} + void * arena_malloc_small(arena_t *arena, size_t size, bool zero) { @@ -1297,13 +1361,20 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) if (zero == false) { if (config_fill) { - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) + if (opt_junk) { + arena_alloc_junk_small(ret, + &arena_bin_info[binind], false); + } else if (opt_zero) memset(ret, 0, size); } - } else + } else { + if (config_fill && opt_junk) { + arena_alloc_junk_small(ret, &arena_bin_info[binind], + true); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } return (ret); } @@ -1412,7 +1483,7 @@ arena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, /* Return the size of the allocation pointed to by ptr. */ size_t -arena_salloc(const void *ptr) +arena_salloc(const void *ptr, bool demote) { size_t ret; arena_chunk_t *chunk; @@ -1431,12 +1502,19 @@ arena_salloc(const void *ptr) size_t binind = arena_bin_index(chunk->arena, run->bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); + (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval + == 0); ret = bin_info->reg_size; } else { assert(((uintptr_t)ptr & PAGE_MASK) == 0); ret = mapbits & ~PAGE_MASK; + if (demote && prof_promote && ret == PAGE && (mapbits & + CHUNK_MAP_CLASS_MASK) != 0) { + size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> + CHUNK_MAP_CLASS_SHIFT) - 1; + assert(binind < NBINS); + ret = arena_bin_info[binind].reg_size; + } assert(ret != 0); } @@ -1449,9 +1527,11 @@ arena_prof_promoted(const void *ptr, size_t size) arena_chunk_t *chunk; size_t pageind, binind; + assert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - assert(isalloc(ptr) == PAGE); + assert(isalloc(ptr, false) == PAGE); + assert(isalloc(ptr, true) == PAGE); assert(size <= SMALL_MAXCLASS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); @@ -1460,45 +1540,9 @@ arena_prof_promoted(const void *ptr, size_t size) assert(binind < NBINS); chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); -} -size_t -arena_salloc_demote(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE)); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); - ret = bin_info->reg_size; - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = mapbits & ~PAGE_MASK; - if (prof_promote && ret == PAGE && (mapbits & - CHUNK_MAP_CLASS_MASK) != 0) { - size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> - CHUNK_MAP_CLASS_SHIFT) - 1; - assert(binind < NBINS); - ret = arena_bin_info[binind].reg_size; - } - assert(ret != 0); - } - - return (ret); + assert(isalloc(ptr, false) == PAGE); + assert(isalloc(ptr, true) == size); } static void @@ -1545,7 +1589,8 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); past = (size_t)(PAGE_CEILING((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * - bin_info->reg_size) - (uintptr_t)chunk) >> LG_PAGE); + bin_info->reg_interval - bin_info->redzone_size) - + (uintptr_t)chunk) >> LG_PAGE); malloc_mutex_lock(&arena->lock); /* @@ -1617,7 +1662,7 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, size = bin_info->reg_size; if (config_fill && opt_junk) - memset(ptr, 0x5a, size); + arena_dalloc_junk_small(ptr, bin_info); arena_run_reg_dalloc(run, ptr); if (run->nfree == bin_info->nregs) { @@ -1936,7 +1981,7 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloc(ptr); return (ret); } @@ -2007,16 +2052,40 @@ arena_new(arena_t *arena, unsigned ind) static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) { + size_t pad_size; size_t try_run_size, good_run_size; uint32_t try_nregs, good_nregs; uint32_t try_hdr_size, good_hdr_size; uint32_t try_bitmap_offset, good_bitmap_offset; uint32_t try_ctx0_offset, good_ctx0_offset; - uint32_t try_reg0_offset, good_reg0_offset; + uint32_t try_redzone0_offset, good_redzone0_offset; assert(min_run_size >= PAGE); assert(min_run_size <= arena_maxclass); + /* + * Determine redzone size based on minimum alignment and minimum + * redzone size. Add padding to the end of the run if it is needed to + * align the regions. The padding allows each redzone to be half the + * minimum alignment; without the padding, each redzone would have to + * be twice as large in order to maintain alignment. + */ + if (config_fill && opt_redzone) { + size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); + if (align_min <= REDZONE_MINSIZE) { + bin_info->redzone_size = REDZONE_MINSIZE; + pad_size = 0; + } else { + bin_info->redzone_size = align_min >> 1; + pad_size = bin_info->redzone_size; + } + } else { + bin_info->redzone_size = 0; + pad_size = 0; + } + bin_info->reg_interval = bin_info->reg_size + + (bin_info->redzone_size << 1); + /* * Calculate known-valid settings before entering the run_size * expansion loop, so that the first part of the loop always copies @@ -2028,7 +2097,8 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) * header's mask length and the number of regions. */ try_run_size = min_run_size; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) + try_nregs = ((try_run_size - sizeof(arena_run_t)) / + bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS @@ -2050,9 +2120,9 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_hdr_size += try_nregs * sizeof(prof_ctx_t *); } else try_ctx0_offset = 0; - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); + try_redzone0_offset = try_run_size - (try_nregs * + bin_info->reg_interval) - pad_size; + } while (try_hdr_size > try_redzone0_offset); /* run_size expansion loop. */ do { @@ -2064,12 +2134,12 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) good_hdr_size = try_hdr_size; good_bitmap_offset = try_bitmap_offset; good_ctx0_offset = try_ctx0_offset; - good_reg0_offset = try_reg0_offset; + good_redzone0_offset = try_redzone0_offset; /* Try more aggressive settings. */ try_run_size += PAGE; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / - bin_info->reg_size) + try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / + bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS @@ -2093,23 +2163,27 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_hdr_size += try_nregs * sizeof(prof_ctx_t *); } - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); + try_redzone0_offset = try_run_size - (try_nregs * + bin_info->reg_interval) - pad_size; + } while (try_hdr_size > try_redzone0_offset); } while (try_run_size <= arena_maxclass && try_run_size <= arena_maxclass - && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX - && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size + && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > + RUN_MAX_OVRHD_RELAX + && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size && try_nregs < RUN_MAXREGS); - assert(good_hdr_size <= good_reg0_offset); + assert(good_hdr_size <= good_redzone0_offset); /* Copy final settings. */ bin_info->run_size = good_run_size; bin_info->nregs = good_nregs; bin_info->bitmap_offset = good_bitmap_offset; bin_info->ctx0_offset = good_ctx0_offset; - bin_info->reg0_offset = good_reg0_offset; + bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; + + assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs + * bin_info->reg_interval) + pad_size == bin_info->run_size); return (good_run_size); } diff --git a/src/ctl.c b/src/ctl.c index a75ffef3..6be40561 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -57,6 +57,7 @@ CTL_PROTO(config_stats) CTL_PROTO(config_tcache) CTL_PROTO(config_tls) CTL_PROTO(config_utrace) +CTL_PROTO(config_valgrind) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) CTL_PROTO(opt_lg_chunk) @@ -65,7 +66,10 @@ CTL_PROTO(opt_lg_dirty_mult) CTL_PROTO(opt_stats_print) CTL_PROTO(opt_junk) CTL_PROTO(opt_zero) +CTL_PROTO(opt_quarantine) +CTL_PROTO(opt_redzone) CTL_PROTO(opt_utrace) +CTL_PROTO(opt_valgrind) CTL_PROTO(opt_xmalloc) CTL_PROTO(opt_tcache) CTL_PROTO(opt_lg_tcache_max) @@ -179,6 +183,7 @@ static const ctl_node_t config_node[] = { {NAME("tcache"), CTL(config_tcache)}, {NAME("tls"), CTL(config_tls)}, {NAME("utrace"), CTL(config_utrace)}, + {NAME("valgrind"), CTL(config_valgrind)}, {NAME("xmalloc"), CTL(config_xmalloc)} }; @@ -190,7 +195,10 @@ static const ctl_node_t opt_node[] = { {NAME("stats_print"), CTL(opt_stats_print)}, {NAME("junk"), CTL(opt_junk)}, {NAME("zero"), CTL(opt_zero)}, + {NAME("quarantine"), CTL(opt_quarantine)}, + {NAME("redzone"), CTL(opt_redzone)}, {NAME("utrace"), CTL(opt_utrace)}, + {NAME("valgrind"), CTL(opt_valgrind)}, {NAME("xmalloc"), CTL(opt_xmalloc)}, {NAME("tcache"), CTL(opt_tcache)}, {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, @@ -1050,7 +1058,8 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, /* Set new arena association. */ if (config_tcache) { tcache_t *tcache; - if ((tcache = *tcache_tsd_get()) != NULL) { + if ((uintptr_t)(tcache = *tcache_tsd_get()) > + (uintptr_t)TCACHE_STATE_MAX) { tcache_arena_dissociate(tcache); tcache_arena_associate(tcache, arena); } @@ -1085,6 +1094,7 @@ CTL_RO_BOOL_CONFIG_GEN(config_stats) CTL_RO_BOOL_CONFIG_GEN(config_tcache) CTL_RO_BOOL_CONFIG_GEN(config_tls) CTL_RO_BOOL_CONFIG_GEN(config_utrace) +CTL_RO_BOOL_CONFIG_GEN(config_valgrind) CTL_RO_BOOL_CONFIG_GEN(config_xmalloc) /******************************************************************************/ @@ -1096,7 +1106,10 @@ CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool) CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) +CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) +CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) +CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool) CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) diff --git a/src/huge.c b/src/huge.c index 43c8f3b0..daf0c622 100644 --- a/src/huge.c +++ b/src/huge.c @@ -174,7 +174,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, #endif { memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloc(ptr); } return (ret); } diff --git a/src/jemalloc.c b/src/jemalloc.c index cde998c4..237dd589 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -14,14 +14,19 @@ const char *je_malloc_conf JEMALLOC_ATTR(visibility("default")); bool opt_abort = true; # ifdef JEMALLOC_FILL bool opt_junk = true; +bool opt_redzone = true; # else bool opt_junk = false; +bool opt_redzone = false; # endif #else bool opt_abort = false; bool opt_junk = false; +bool opt_redzone = false; #endif +size_t opt_quarantine = ZU(0); bool opt_utrace = false; +bool opt_valgrind = false; bool opt_xmalloc = false; bool opt_zero = false; size_t opt_narenas = 0; @@ -419,7 +424,7 @@ malloc_conf_init(void) while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, &vlen) == false) { -#define CONF_HANDLE_BOOL(o, n) \ +#define CONF_HANDLE_BOOL_HIT(o, n, hit) \ if (sizeof(#n)-1 == klen && strncmp(#n, k, \ klen) == 0) { \ if (strncmp("true", v, vlen) == 0 && \ @@ -433,12 +438,19 @@ malloc_conf_init(void) "Invalid conf value", \ k, klen, v, vlen); \ } \ + hit = true; \ + } else \ + hit = false; +#define CONF_HANDLE_BOOL(o, n) { \ + bool hit; \ + CONF_HANDLE_BOOL_HIT(o, n, hit); \ + if (hit) \ continue; \ - } +} #define CONF_HANDLE_SIZE_T(o, n, min, max) \ if (sizeof(#n)-1 == klen && strncmp(#n, k, \ klen) == 0) { \ - uintmax_t um; \ + uintmax_t um; \ char *end; \ \ errno = 0; \ @@ -502,11 +514,30 @@ malloc_conf_init(void) CONF_HANDLE_BOOL(opt_stats_print, stats_print) if (config_fill) { CONF_HANDLE_BOOL(opt_junk, junk) + CONF_HANDLE_SIZE_T(opt_quarantine, quarantine, + 0, SIZE_T_MAX) + CONF_HANDLE_BOOL(opt_redzone, redzone) CONF_HANDLE_BOOL(opt_zero, zero) } if (config_utrace) { CONF_HANDLE_BOOL(opt_utrace, utrace) } + if (config_valgrind) { + bool hit; + CONF_HANDLE_BOOL_HIT(opt_valgrind, + valgrind, hit) + if (config_fill && opt_valgrind && hit) { + opt_junk = false; + opt_zero = false; + if (opt_quarantine == 0) { + opt_quarantine = + JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; + } + opt_redzone = true; + } + if (hit) + continue; + } if (config_xmalloc) { CONF_HANDLE_BOOL(opt_xmalloc, xmalloc) } @@ -662,6 +693,11 @@ malloc_init_hard(void) return (true); } + if (config_fill && quarantine_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (config_prof && prof_boot2()) { malloc_mutex_unlock(&init_lock); return (true); @@ -763,7 +799,7 @@ je_malloc(size_t size) } else ret = imalloc(size); } else { - if (config_stats) + if (config_stats || (config_valgrind && opt_valgrind)) usize = s2u(size); ret = imalloc(size); } @@ -780,10 +816,11 @@ label_oom: if (config_prof && opt_prof && ret != NULL) prof_malloc(ret, usize, cnt); if (config_stats && ret != NULL) { - assert(usize == isalloc(ret)); + assert(usize == isalloc(ret, config_prof)); thread_allocated_tsd_get()->allocated += usize; } UTRACE(0, size, ret); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); return (ret); } @@ -872,7 +909,7 @@ imemalign(void **memptr, size_t alignment, size_t size, label_return: if (config_stats && result != NULL) { - assert(usize == isalloc(result)); + assert(usize == isalloc(result, config_prof)); thread_allocated_tsd_get()->allocated += usize; } if (config_prof && opt_prof && result != NULL) @@ -886,8 +923,10 @@ JEMALLOC_ATTR(visibility("default")) int je_posix_memalign(void **memptr, size_t alignment, size_t size) { - - return imemalign(memptr, alignment, size, sizeof(void *)); + int ret = imemalign(memptr, alignment, size, sizeof(void *)); + JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, + config_prof), false); + return (ret); } JEMALLOC_ATTR(malloc) @@ -902,6 +941,8 @@ je_aligned_alloc(size_t alignment, size_t size) ret = NULL; errno = err; } + JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), + false); return (ret); } @@ -956,7 +997,7 @@ je_calloc(size_t num, size_t size) } else ret = icalloc(num_size); } else { - if (config_stats) + if (config_stats || (config_valgrind && opt_valgrind)) usize = s2u(num_size); ret = icalloc(num_size); } @@ -974,10 +1015,11 @@ label_return: if (config_prof && opt_prof && ret != NULL) prof_malloc(ret, usize, cnt); if (config_stats && ret != NULL) { - assert(usize == isalloc(ret)); + assert(usize == isalloc(ret, config_prof)); thread_allocated_tsd_get()->allocated += usize; } UTRACE(0, num_size, ret); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); return (ret); } @@ -988,19 +1030,30 @@ je_realloc(void *ptr, size_t size) void *ret; size_t usize; size_t old_size = 0; + size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL); if (size == 0) { if (ptr != NULL) { /* realloc(ptr, 0) is equivalent to free(p). */ - if (config_prof || config_stats) - old_size = isalloc(ptr); + if (config_prof) { + old_size = isalloc(ptr, true); + if (config_valgrind && opt_valgrind) + old_rzsize = p2rz(ptr); + } else if (config_stats) { + old_size = isalloc(ptr, false); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_size); + } else if (config_valgrind && opt_valgrind) { + old_size = isalloc(ptr, false); + old_rzsize = u2rz(old_size); + } if (config_prof && opt_prof) { old_ctx = prof_ctx_get(ptr); cnt = NULL; } - idalloc(ptr); + iqalloc(ptr); ret = NULL; goto label_return; } else @@ -1010,8 +1063,18 @@ je_realloc(void *ptr, size_t size) if (ptr != NULL) { assert(malloc_initialized || IS_INITIALIZER); - if (config_prof || config_stats) - old_size = isalloc(ptr); + if (config_prof) { + old_size = isalloc(ptr, true); + if (config_valgrind && opt_valgrind) + old_rzsize = p2rz(ptr); + } else if (config_stats) { + old_size = isalloc(ptr, false); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_size); + } else if (config_valgrind && opt_valgrind) { + old_size = isalloc(ptr, false); + old_rzsize = u2rz(old_size); + } if (config_prof && opt_prof) { usize = s2u(size); old_ctx = prof_ctx_get(ptr); @@ -1035,7 +1098,7 @@ je_realloc(void *ptr, size_t size) old_ctx = NULL; } } else { - if (config_stats) + if (config_stats || (config_valgrind && opt_valgrind)) usize = s2u(size); ret = iralloc(ptr, size, 0, 0, false, false); } @@ -1076,7 +1139,8 @@ label_oom: ret = imalloc(size); } } else { - if (config_stats) + if (config_stats || (config_valgrind && + opt_valgrind)) usize = s2u(size); ret = imalloc(size); } @@ -1097,12 +1161,13 @@ label_return: prof_realloc(ret, usize, cnt, old_size, old_ctx); if (config_stats && ret != NULL) { thread_allocated_t *ta; - assert(usize == isalloc(ret)); + assert(usize == isalloc(ret, config_prof)); ta = thread_allocated_tsd_get(); ta->allocated += usize; ta->deallocated += old_size; } UTRACE(ptr, size, ret); + JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false); return (ret); } @@ -1114,18 +1179,21 @@ je_free(void *ptr) UTRACE(ptr, 0, 0); if (ptr != NULL) { size_t usize; + size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); assert(malloc_initialized || IS_INITIALIZER); if (config_prof && opt_prof) { - usize = isalloc(ptr); + usize = isalloc(ptr, config_prof); prof_free(ptr, usize); - } else if (config_stats) { - usize = isalloc(ptr); - } + } else if (config_stats || config_valgrind) + usize = isalloc(ptr, config_prof); if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - idalloc(ptr); + if (config_valgrind && opt_valgrind) + rzsize = p2rz(ptr); + iqalloc(ptr); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); } } @@ -1145,6 +1213,7 @@ je_memalign(size_t alignment, size_t size) { void *ret JEMALLOC_CC_SILENCE_INIT(NULL); imemalign(&ret, alignment, size, 1); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } #endif @@ -1157,6 +1226,7 @@ je_valloc(size_t size) { void *ret JEMALLOC_CC_SILENCE_INIT(NULL); imemalign(&ret, PAGE, size, 1); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } #endif @@ -1209,9 +1279,9 @@ je_malloc_usable_size(const void *ptr) assert(malloc_initialized || IS_INITIALIZER); if (config_ivsalloc) - ret = ivsalloc(ptr); + ret = ivsalloc(ptr, config_prof); else - ret = (ptr != NULL) ? isalloc(ptr) : 0; + ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; return (ret); } @@ -1336,10 +1406,11 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags) *ptr = p; if (config_stats) { - assert(usize == isalloc(p)); + assert(usize == isalloc(p, config_prof)); thread_allocated_tsd_get()->allocated += usize; } UTRACE(0, size, p); + JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); return (ALLOCM_SUCCESS); label_oom: if (config_xmalloc && opt_xmalloc) { @@ -1360,6 +1431,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) void *p, *q; size_t usize; size_t old_size; + size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); bool zero = flags & ALLOCM_ZERO; @@ -1384,7 +1456,9 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) size_t max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment, NULL); prof_ctx_t *old_ctx = prof_ctx_get(p); - old_size = isalloc(p); + old_size = isalloc(p, true); + if (config_valgrind && opt_valgrind) + old_rzsize = p2rz(p); PROF_ALLOC_PREP(1, max_usize, cnt); if (cnt == NULL) goto label_oom; @@ -1403,27 +1477,33 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) usize = max_usize; arena_prof_promoted(q, usize); } else - usize = isalloc(q); + usize = isalloc(q, config_prof); } else { q = iralloc(p, size, extra, alignment, zero, no_move); if (q == NULL) goto label_err; - usize = isalloc(q); + usize = isalloc(q, config_prof); } prof_realloc(q, usize, cnt, old_size, old_ctx); if (rsize != NULL) *rsize = usize; } else { - if (config_stats) - old_size = isalloc(p); + if (config_stats) { + old_size = isalloc(p, false); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_size); + } else if (config_valgrind && opt_valgrind) { + old_size = isalloc(p, false); + old_rzsize = u2rz(old_size); + } q = iralloc(p, size, extra, alignment, zero, no_move); if (q == NULL) goto label_err; if (config_stats) - usize = isalloc(q); + usize = isalloc(q, config_prof); if (rsize != NULL) { if (config_stats == false) - usize = isalloc(q); + usize = isalloc(q, config_prof); *rsize = usize; } } @@ -1436,6 +1516,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) ta->deallocated += old_size; } UTRACE(p, size, q); + JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero); return (ALLOCM_SUCCESS); label_err: if (no_move) { @@ -1462,10 +1543,10 @@ je_sallocm(const void *ptr, size_t *rsize, int flags) assert(malloc_initialized || IS_INITIALIZER); if (config_ivsalloc) - sz = ivsalloc(ptr); + sz = ivsalloc(ptr, config_prof); else { assert(ptr != NULL); - sz = isalloc(ptr); + sz = isalloc(ptr, config_prof); } assert(rsize != NULL); *rsize = sz; @@ -1479,21 +1560,25 @@ int je_dallocm(void *ptr, int flags) { size_t usize; + size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); assert(ptr != NULL); assert(malloc_initialized || IS_INITIALIZER); UTRACE(ptr, 0, 0); - if (config_stats) - usize = isalloc(ptr); + if (config_stats || config_valgrind) + usize = isalloc(ptr, config_prof); if (config_prof && opt_prof) { - if (config_stats == false) - usize = isalloc(ptr); + if (config_stats == false && config_valgrind == false) + usize = isalloc(ptr, config_prof); prof_free(ptr, usize); } if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - idalloc(ptr); + if (config_valgrind && opt_valgrind) + rzsize = p2rz(ptr); + iqalloc(ptr); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); return (ALLOCM_SUCCESS); } diff --git a/src/quarantine.c b/src/quarantine.c new file mode 100644 index 00000000..89a25c6a --- /dev/null +++ b/src/quarantine.c @@ -0,0 +1,163 @@ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +typedef struct quarantine_s quarantine_t; + +struct quarantine_s { + size_t curbytes; + size_t curobjs; + size_t first; +#define LG_MAXOBJS_INIT 10 + size_t lg_maxobjs; + void *objs[1]; /* Dynamically sized ring buffer. */ +}; + +static void quarantine_cleanup(void *arg); + +malloc_tsd_data(static, quarantine, quarantine_t *, NULL) +malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL, + quarantine_cleanup) + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static quarantine_t *quarantine_init(size_t lg_maxobjs); +static quarantine_t *quarantine_grow(quarantine_t *quarantine); +static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); + +/******************************************************************************/ + +static quarantine_t * +quarantine_init(size_t lg_maxobjs) +{ + quarantine_t *quarantine; + + quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + + ((ZU(1) << lg_maxobjs) * sizeof(void *))); + if (quarantine == NULL) + return (NULL); + quarantine->curbytes = 0; + quarantine->curobjs = 0; + quarantine->first = 0; + quarantine->lg_maxobjs = lg_maxobjs; + + quarantine_tsd_set(&quarantine); + + return (quarantine); +} + +static quarantine_t * +quarantine_grow(quarantine_t *quarantine) +{ + quarantine_t *ret; + + ret = quarantine_init(quarantine->lg_maxobjs + 1); + if (ret == NULL) + return (quarantine); + + ret->curbytes = quarantine->curbytes; + if (quarantine->first + quarantine->curobjs < (ZU(1) << + quarantine->lg_maxobjs)) { + /* objs ring buffer data are contiguous. */ + memcpy(ret->objs, &quarantine->objs[quarantine->first], + quarantine->curobjs * sizeof(void *)); + ret->curobjs = quarantine->curobjs; + } else { + /* objs ring buffer data wrap around. */ + size_t ncopy = (ZU(1) << quarantine->lg_maxobjs) - + quarantine->first; + memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy * + sizeof(void *)); + ret->curobjs = ncopy; + if (quarantine->curobjs != 0) { + memcpy(&ret->objs[ret->curobjs], quarantine->objs, + quarantine->curobjs - ncopy); + } + } + + return (ret); +} + +static void +quarantine_drain(quarantine_t *quarantine, size_t upper_bound) +{ + + while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) { + void *ptr = quarantine->objs[quarantine->first]; + size_t usize = isalloc(ptr, config_prof); + idalloc(ptr); + quarantine->curbytes -= usize; + quarantine->curobjs--; + quarantine->first = (quarantine->first + 1) & ((ZU(1) << + quarantine->lg_maxobjs) - 1); + } +} + +void +quarantine(void *ptr) +{ + quarantine_t *quarantine; + size_t usize = isalloc(ptr, config_prof); + + assert(config_fill); + assert(opt_quarantine); + + quarantine = *quarantine_tsd_get(); + if (quarantine == NULL && (quarantine = + quarantine_init(LG_MAXOBJS_INIT)) == NULL) { + idalloc(ptr); + return; + } + /* + * Drain one or more objects if the quarantine size limit would be + * exceeded by appending ptr. + */ + if (quarantine->curbytes + usize > opt_quarantine) { + size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine + - usize : 0; + quarantine_drain(quarantine, upper_bound); + } + /* Grow the quarantine ring buffer if it's full. */ + if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) + quarantine = quarantine_grow(quarantine); + /* quarantine_grow() must free a slot if it fails to grow. */ + assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); + /* Append ptr if its size doesn't exceed the quarantine size. */ + if (quarantine->curbytes + usize <= opt_quarantine) { + size_t offset = (quarantine->first + quarantine->curobjs) & + ((ZU(1) << quarantine->lg_maxobjs) - 1); + quarantine->objs[offset] = ptr; + quarantine->curbytes += usize; + quarantine->curobjs++; + if (opt_junk) + memset(ptr, 0x5a, usize); + } else { + assert(quarantine->curbytes == 0); + idalloc(ptr); + } +} + +static void +quarantine_cleanup(void *arg) +{ + quarantine_t *quarantine = *(quarantine_t **)arg; + + if (quarantine != NULL) { + quarantine_drain(quarantine, 0); + idalloc(quarantine); + } +} + +bool +quarantine_boot(void) +{ + + assert(config_fill); + + if (quarantine_tsd_boot()) + return (true); + + return (false); +} diff --git a/src/stats.c b/src/stats.c index 0cd70b0d..4cad214f 100644 --- a/src/stats.c +++ b/src/stats.c @@ -382,8 +382,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, OPT_WRITE_SSIZE_T(lg_dirty_mult) OPT_WRITE_BOOL(stats_print) OPT_WRITE_BOOL(junk) + OPT_WRITE_SIZE_T(quarantine) + OPT_WRITE_BOOL(redzone) OPT_WRITE_BOOL(zero) OPT_WRITE_BOOL(utrace) + OPT_WRITE_BOOL(valgrind) OPT_WRITE_BOOL(xmalloc) OPT_WRITE_BOOL(tcache) OPT_WRITE_SSIZE_T(lg_tcache_max) diff --git a/src/tcache.c b/src/tcache.c index 99a657b6..be26b59c 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -75,6 +75,10 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; + if (config_fill && opt_junk) { + arena_alloc_junk_small(ptr, + &arena_bin_info[binind], true); + } arena_dalloc_bin(arena, chunk, ptr, mapelm); } else { /* @@ -298,7 +302,7 @@ tcache_destroy(tcache_t *tcache) malloc_mutex_unlock(&tcache->arena->lock); } - tcache_size = arena_salloc(tcache); + tcache_size = arena_salloc(tcache, false); if (tcache_size <= SMALL_MAXCLASS) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); arena_t *arena = chunk->arena; diff --git a/src/zone.c b/src/zone.c index a50c129c..cde5d49a 100644 --- a/src/zone.c +++ b/src/zone.c @@ -56,7 +56,7 @@ zone_size(malloc_zone_t *zone, void *ptr) * not work in practice, we must check all pointers to assure that they * reside within a mapped chunk before determining size. */ - return (ivsalloc(ptr)); + return (ivsalloc(ptr, config_prof)); } static void * @@ -87,7 +87,7 @@ static void zone_free(malloc_zone_t *zone, void *ptr) { - if (ivsalloc(ptr) != 0) { + if (ivsalloc(ptr, config_prof) != 0) { je_free(ptr); return; } @@ -99,7 +99,7 @@ static void * zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) { - if (ivsalloc(ptr) != 0) + if (ivsalloc(ptr, config_prof) != 0) return (je_realloc(ptr, size)); return (realloc(ptr, size)); @@ -122,8 +122,8 @@ static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) { - if (ivsalloc(ptr) != 0) { - assert(ivsalloc(ptr) == size); + if (ivsalloc(ptr, config_prof) != 0) { + assert(ivsalloc(ptr, config_prof) == size); je_free(ptr); return; }