From ecea0f6125ea87ee6fd82f16286b61eb8c0f5692 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 28 Apr 2010 12:00:59 -0700 Subject: [PATCH 1/4] Fix junk filling of cached large objects. Use the size argument to tcache_dalloc_large() to control the number of bytes set to 0x5a when junk filling is enabled, rather than accessing a non-existent arena bin. This bug was capable of corrupting an arbitrarily large memory region, depending on what followed the arena data structure in memory (typically zeroed memory, another arena_t, or a red-black tree node for a huge object). --- jemalloc/include/jemalloc/internal/tcache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jemalloc/include/jemalloc/internal/tcache.h b/jemalloc/include/jemalloc/internal/tcache.h index fa6c53f2..a8be436d 100644 --- a/jemalloc/include/jemalloc/internal/tcache.h +++ b/jemalloc/include/jemalloc/internal/tcache.h @@ -353,7 +353,7 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) #ifdef JEMALLOC_FILL if (opt_junk) - memset(ptr, 0x5a, arena->bins[binind].reg_size); + memset(ptr, 0x5a, size); #endif tbin = &tcache->tbins[binind]; From 2206e1acc12a078b0108052e3c1aca745887188e Mon Sep 17 00:00:00 2001 From: Jordan DeLong Date: Mon, 10 May 2010 14:17:00 -0700 Subject: [PATCH 2/4] Add MAP_NORESERVE support. Add MAP_NORESERVE to the chunk_mmap() case being used by chunk_swap_enable(), if the system supports it. --- .../include/jemalloc/internal/chunk_mmap.h | 1 + jemalloc/src/chunk_mmap.c | 43 +++++++++++++------ jemalloc/src/chunk_swap.c | 2 +- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/jemalloc/include/jemalloc/internal/chunk_mmap.h b/jemalloc/include/jemalloc/internal/chunk_mmap.h index 8fb90b77..dc52448c 100644 --- a/jemalloc/include/jemalloc/internal/chunk_mmap.h +++ b/jemalloc/include/jemalloc/internal/chunk_mmap.h @@ -10,6 +10,7 @@ #ifdef JEMALLOC_H_EXTERNS void *chunk_alloc_mmap(size_t size); +void *chunk_alloc_mmap_noreserve(size_t size); void chunk_dealloc_mmap(void *chunk, size_t size); #endif /* JEMALLOC_H_EXTERNS */ diff --git a/jemalloc/src/chunk_mmap.c b/jemalloc/src/chunk_mmap.c index 8f071138..d9f9e86d 100644 --- a/jemalloc/src/chunk_mmap.c +++ b/jemalloc/src/chunk_mmap.c @@ -23,14 +23,15 @@ static /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static void *pages_map(void *addr, size_t size); +static void *pages_map(void *addr, size_t size, bool noreserve); static void pages_unmap(void *addr, size_t size); -static void *chunk_alloc_mmap_slow(size_t size, bool unaligned); +static void *chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve); +static void *chunk_alloc_mmap_internal(size_t size, bool noreserve); /******************************************************************************/ static void * -pages_map(void *addr, size_t size) +pages_map(void *addr, size_t size, bool noreserve) { void *ret; @@ -38,8 +39,12 @@ pages_map(void *addr, size_t size) * We don't use MAP_FIXED here, because it can cause the *replacement* * of existing mappings, and we only want to create new mappings. */ - ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, - -1, 0); + int flags = MAP_PRIVATE | MAP_ANON; +#ifdef MAP_NORESERVE + if (noreserve) + flags |= MAP_NORESERVE; +#endif + ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0); assert(ret != NULL); if (ret == MAP_FAILED) @@ -83,7 +88,7 @@ pages_unmap(void *addr, size_t size) } static void * -chunk_alloc_mmap_slow(size_t size, bool unaligned) +chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve) { void *ret; size_t offset; @@ -92,7 +97,7 @@ chunk_alloc_mmap_slow(size_t size, bool unaligned) if (size + chunksize <= size) return (NULL); - ret = pages_map(NULL, size + chunksize); + ret = pages_map(NULL, size + chunksize, noreserve); if (ret == NULL) return (NULL); @@ -128,8 +133,8 @@ chunk_alloc_mmap_slow(size_t size, bool unaligned) return (ret); } -void * -chunk_alloc_mmap(size_t size) +static void * +chunk_alloc_mmap_internal(size_t size, bool noreserve) { void *ret; @@ -164,7 +169,7 @@ chunk_alloc_mmap(size_t size) if (mmap_unaligned == false) { size_t offset; - ret = pages_map(NULL, size); + ret = pages_map(NULL, size, noreserve); if (ret == NULL) return (NULL); @@ -173,13 +178,13 @@ chunk_alloc_mmap(size_t size) mmap_unaligned = true; /* Try to extend chunk boundary. */ if (pages_map((void *)((uintptr_t)ret + size), - chunksize - offset) == NULL) { + chunksize - offset, noreserve) == NULL) { /* * Extension failed. Clean up, then revert to * the reliable-but-expensive method. */ pages_unmap(ret, size); - ret = chunk_alloc_mmap_slow(size, true); + ret = chunk_alloc_mmap_slow(size, true, noreserve); } else { /* Clean up unneeded leading space. */ pages_unmap(ret, chunksize - offset); @@ -188,11 +193,23 @@ chunk_alloc_mmap(size_t size) } } } else - ret = chunk_alloc_mmap_slow(size, false); + ret = chunk_alloc_mmap_slow(size, false, noreserve); return (ret); } +void * +chunk_alloc_mmap(size_t size) +{ + return chunk_alloc_mmap_internal(size, false); +} + +void * +chunk_alloc_mmap_noreserve(size_t size) +{ + return chunk_alloc_mmap_internal(size, true); +} + void chunk_dealloc_mmap(void *chunk, size_t size) { diff --git a/jemalloc/src/chunk_swap.c b/jemalloc/src/chunk_swap.c index b8c880f0..ed9e414d 100644 --- a/jemalloc/src/chunk_swap.c +++ b/jemalloc/src/chunk_swap.c @@ -283,7 +283,7 @@ chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed) * Allocate a chunk-aligned region of anonymous memory, which will * be the final location for the memory-mapped files. */ - vaddr = chunk_alloc_mmap(cumsize); + vaddr = chunk_alloc_mmap_noreserve(cumsize); if (vaddr == NULL) { ret = true; goto RETURN; From ed3d152ea0aa39169a9629962a40c58e67b0b532 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 11 May 2010 12:00:22 -0700 Subject: [PATCH 3/4] Fix next_arena initialization. If there is more than one arena, initialize next_arena so that the first and second threads to allocate memory use arenas 0 and 1, rather than both using arena 0. --- jemalloc/src/jemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jemalloc/src/jemalloc.c b/jemalloc/src/jemalloc.c index aeab1408..bf2ace39 100644 --- a/jemalloc/src/jemalloc.c +++ b/jemalloc/src/jemalloc.c @@ -775,7 +775,7 @@ MALLOC_OUT: #endif #ifndef NO_TLS - next_arena = 0; + next_arena = (narenas > 0) ? 1 : 0; #endif /* Allocate and initialize arenas. */ From 7013d10a9e8e1a183542111747dc9c324560975a Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 11 May 2010 18:17:02 -0700 Subject: [PATCH 4/4] Avoid unnecessary isalloc() calls. When heap profiling is enabled but deactivated, there is no need to call isalloc(ptr) in prof_{malloc,realloc}(). Avoid these calls, so that profiling overhead under such conditions is negligible. --- jemalloc/src/prof.c | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/jemalloc/src/prof.c b/jemalloc/src/prof.c index 93904b8c..88e8f44a 100644 --- a/jemalloc/src/prof.c +++ b/jemalloc/src/prof.c @@ -623,13 +623,8 @@ static inline void prof_sample_accum_update(size_t size) { - if (opt_lg_prof_sample == 0) { - /* - * Don't bother with sampling logic, since sampling interval is - * 1. - */ - return; - } + /* Sampling logic is unnecessary if the interval is 1. */ + assert(opt_lg_prof_sample != 0); /* Take care to avoid integer overflow. */ if (size >= prof_sample_threshold - prof_sample_accum) { @@ -647,11 +642,15 @@ prof_sample_accum_update(size_t size) void prof_malloc(const void *ptr, prof_thr_cnt_t *cnt) { - size_t size = isalloc(ptr); + size_t size; assert(ptr != NULL); - prof_sample_accum_update(size); + if (opt_lg_prof_sample != 0) { + size = isalloc(ptr); + prof_sample_accum_update(size); + } else if ((uintptr_t)cnt > (uintptr_t)1U) + size = isalloc(ptr); if ((uintptr_t)cnt > (uintptr_t)1U) { prof_ctx_set(ptr, cnt->ctx); @@ -679,11 +678,18 @@ void prof_realloc(const void *ptr, prof_thr_cnt_t *cnt, const void *old_ptr, size_t old_size, prof_ctx_t *old_ctx) { - size_t size = isalloc(ptr); + size_t size; prof_thr_cnt_t *told_cnt; - if (ptr != NULL) - prof_sample_accum_update(size); + assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); + + if (ptr != NULL) { + if (opt_lg_prof_sample != 0) { + size = isalloc(ptr); + prof_sample_accum_update(size); + } else if ((uintptr_t)cnt > (uintptr_t)1U) + size = isalloc(ptr); + } if ((uintptr_t)old_ctx > (uintptr_t)1U) { told_cnt = prof_lookup(old_ctx->bt);