project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use rtree-based chunk lookups rather than pointer bit twiddling.

Browse Source 
Look up chunk metadata via the radix tree, rather than using
CHUNK_ADDR2BASE().

Propagate pointer's containing extent.

Minimize extent lookups by doing a single lookup (e.g. in free()) and
propagating the pointer's extent into nearly all the functions that may
need it.
This commit is contained in:
Jason Evans 2016-03-23 20:29:33 -07:00
parent 2d2b4e98c9
commit db72272bef
14 changed files with 548 additions and 504 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
include/jemalloc/internal/arena.h
View File

@ -526,13 +526,13 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero); szind_t ind, bool zero);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, tcache_t *tcache); size_t alignment, bool zero, tcache_t *tcache);
void arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size); void arena_prof_promoted(tsdn_t *tsdn, const extent_t *extent,
const void *ptr, size_t size);
void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm); arena_chunk_t *chunk, extent_t *extent, void *ptr,
void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_chunk_map_bits_t *bitselm);
void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm);
void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr, size_t pageind); extent_t *extent, void *ptr, size_t pageind);
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
typedef void (arena_dalloc_junk_large_t)(void *, size_t); typedef void (arena_dalloc_junk_large_t)(void *, size_t);
extern arena_dalloc_junk_large_t *arena_dalloc_junk_large; extern arena_dalloc_junk_large_t *arena_dalloc_junk_large;
@ -540,17 +540,17 @@ extern arena_dalloc_junk_large_t *arena_dalloc_junk_large;
void arena_dalloc_junk_large(void *ptr, size_t usize); void arena_dalloc_junk_large(void *ptr, size_t usize);
#endif #endif
void arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena, void arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, void *ptr); arena_chunk_t *chunk, extent_t *extent, void *ptr);
void arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr); extent_t *extent, void *ptr);
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t);
extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
#endif #endif
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, bool arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t size, size_t extra, bool zero); size_t oldsize, size_t size, size_t extra, bool zero);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t size, size_t alignment, bool zero, tcache_t *tcache); size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache);
dss_prec_t arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena); dss_prec_t arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena);
bool arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec); bool arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec);
ssize_t arena_lg_dirty_mult_default_get(void); ssize_t arena_lg_dirty_mult_default_get(void);
@ -637,20 +637,23 @@ szind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
size_t arena_run_regind(arena_run_t *run, const arena_bin_info_t *bin_info, size_t arena_run_regind(arena_run_t *run, const arena_bin_info_t *bin_info,
const void *ptr); const void *ptr);
prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr); prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
void arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, const void *ptr);
prof_tctx_t *tctx); void arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
void arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, size_t usize, prof_tctx_t *tctx);
const void *old_ptr, prof_tctx_t *old_tctx); void arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *old_tctx);
void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks); void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks);
void arena_decay_tick(tsdn_t *tsdn, arena_t *arena); void arena_decay_tick(tsdn_t *tsdn, arena_t *arena);
void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, tcache_t *tcache, bool slow_path); bool zero, tcache_t *tcache, bool slow_path);
arena_t *arena_aalloc(const void *ptr); arena_t *arena_aalloc(const void *ptr);
size_t arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote); size_t arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr,
void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); bool demote);
void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, void arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr,
bool slow_path); tcache_t *tcache, bool slow_path);
void arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
tcache_t *tcache, bool slow_path);
#endif #endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
@ -1042,7 +1045,9 @@ arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes)
return (ret); return (ret);
} }
} }
# endif /* JEMALLOC_ARENA_INLINE_A */
# ifdef JEMALLOC_ARENA_INLINE_B
JEMALLOC_ALWAYS_INLINE szind_t JEMALLOC_ALWAYS_INLINE szind_t
arena_ptr_small_binind_get(const void *ptr, size_t mapbits) arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
{ {
@ -1051,6 +1056,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
if (config_debug) { if (config_debug) {
const extent_t *extent;
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_t *arena; arena_t *arena;
size_t pageind; size_t pageind;
@ -1065,8 +1071,9 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
assert(binind != BININD_INVALID); assert(binind != BININD_INVALID);
assert(binind < NBINS); assert(binind < NBINS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); extent = iealloc(ptr);
arena = extent_arena_get(&chunk->extent); chunk = (arena_chunk_t *)extent_addr_get(extent);
arena = extent_arena_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
actual_mapbits = arena_mapbits_get(chunk, pageind); actual_mapbits = arena_mapbits_get(chunk, pageind);
assert(mapbits == actual_mapbits); assert(mapbits == actual_mapbits);
@ -1088,9 +1095,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
return (binind); return (binind);
} }
# endif /* JEMALLOC_ARENA_INLINE_A */
# ifdef JEMALLOC_ARENA_INLINE_B
JEMALLOC_INLINE szind_t JEMALLOC_INLINE szind_t
arena_bin_index(arena_t *arena, arena_bin_t *bin) arena_bin_index(arena_t *arena, arena_bin_t *bin)
{ {
@ -1172,16 +1177,15 @@ arena_run_regind(arena_run_t *run, const arena_bin_info_t *bin_info,
} }
JEMALLOC_INLINE prof_tctx_t * JEMALLOC_INLINE prof_tctx_t *
arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr) arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{ {
prof_tctx_t *ret; prof_tctx_t *ret;
arena_chunk_t *chunk;
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(extent_achunk_get(extent))) {
if (likely(chunk != ptr)) { arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind); size_t mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
@ -1193,22 +1197,21 @@ arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
ret = atomic_read_p(&elm->prof_tctx_pun); ret = atomic_read_p(&elm->prof_tctx_pun);
} }
} else } else
ret = huge_prof_tctx_get(tsdn, ptr); ret = huge_prof_tctx_get(tsdn, extent, ptr);
return (ret); return (ret);
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx) size_t usize, prof_tctx_t *tctx)
{ {
arena_chunk_t *chunk;
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(extent_achunk_get(extent))) {
if (likely(chunk != ptr)) { arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0); assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
@ -1231,12 +1234,12 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
assert(arena_mapbits_large_get(chunk, pageind) == 0); assert(arena_mapbits_large_get(chunk, pageind) == 0);
} }
} else } else
huge_prof_tctx_set(tsdn, ptr, tctx); huge_prof_tctx_set(tsdn, extent, ptr, tctx);
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
const void *old_ptr, prof_tctx_t *old_tctx) size_t usize, const void *old_ptr, prof_tctx_t *old_tctx)
{ {
cassert(config_prof); cassert(config_prof);
@ -1244,7 +1247,7 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr && if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr &&
(uintptr_t)old_tctx > (uintptr_t)1U))) { (uintptr_t)old_tctx > (uintptr_t)1U))) {
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
if (likely(chunk != ptr)) { if (likely(chunk != ptr)) {
size_t pageind; size_t pageind;
arena_chunk_map_misc_t *elm; arena_chunk_map_misc_t *elm;
@ -1259,7 +1262,7 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
atomic_write_p(&elm->prof_tctx_pun, atomic_write_p(&elm->prof_tctx_pun,
(prof_tctx_t *)(uintptr_t)1U); (prof_tctx_t *)(uintptr_t)1U);
} else } else
huge_prof_tctx_reset(tsdn, ptr); huge_prof_tctx_reset(tsdn, extent, ptr);
} }
} }
@ -1313,28 +1316,24 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
JEMALLOC_ALWAYS_INLINE arena_t * JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(const void *ptr) arena_aalloc(const void *ptr)
{ {
arena_chunk_t *chunk;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); return (extent_arena_get(iealloc(ptr)));
if (likely(chunk != ptr))
return (extent_arena_get(&chunk->extent));
else
return (huge_aalloc(ptr));
} }
/* Return the size of the allocation pointed to by ptr. */ /* Return the size of the allocation pointed to by ptr. */
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote) arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr, bool demote)
{ {
size_t ret; size_t ret;
arena_chunk_t *chunk;
size_t pageind; size_t pageind;
szind_t binind; szind_t binind;
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(extent_achunk_get(extent))) {
if (likely(chunk != ptr)) { const arena_chunk_t *chunk =
(const arena_chunk_t *)extent_addr_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0); assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
binind = arena_mapbits_binind_get(chunk, pageind); binind = arena_mapbits_binind_get(chunk, pageind);
@ -1367,22 +1366,23 @@ arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote)
ret = index2size(binind); ret = index2size(binind);
} }
} else } else
ret = huge_salloc(tsdn, ptr); ret = huge_salloc(tsdn, extent, ptr);
return (ret); return (ret);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
bool slow_path)
{ {
arena_chunk_t *chunk;
size_t pageind, mapbits; size_t pageind, mapbits;
assert(!tsdn_null(tsdn) || tcache == NULL); assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(extent_achunk_get(extent))) {
if (likely(chunk != ptr)) { arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapbits = arena_mapbits_get(chunk, pageind); mapbits = arena_mapbits_get(chunk, pageind);
assert(arena_mapbits_allocated_get(chunk, pageind) != 0); assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
@ -1395,7 +1395,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path)
binind, slow_path); binind, slow_path);
} else { } else {
arena_dalloc_small(tsdn, arena_dalloc_small(tsdn,
extent_arena_get(&chunk->extent), chunk, extent_arena_get(extent), chunk, extent,
ptr, pageind); ptr, pageind);
} }
} else { } else {
@ -1411,24 +1411,24 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path)
size - large_pad, slow_path); size - large_pad, slow_path);
} else { } else {
arena_dalloc_large(tsdn, arena_dalloc_large(tsdn,
extent_arena_get(&chunk->extent), chunk, extent_arena_get(extent), chunk, extent,
ptr); ptr);
} }
} }
} else } else
huge_dalloc(tsdn, ptr); huge_dalloc(tsdn, extent, ptr);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
bool slow_path) tcache_t *tcache, bool slow_path)
{ {
arena_chunk_t *chunk;
assert(!tsdn_null(tsdn) || tcache == NULL); assert(!tsdn_null(tsdn) || tcache == NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(extent_achunk_get(extent))) {
if (likely(chunk != ptr)) { arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
LG_PAGE; LG_PAGE;
@ -1443,7 +1443,8 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
pageind) - large_pad; pageind) - large_pad;
} }
} }
assert(s2u(size) == s2u(arena_salloc(tsdn, ptr, false))); assert(s2u(size) == s2u(arena_salloc(tsdn, extent, ptr,
false)));
if (likely(size <= SMALL_MAXCLASS)) { if (likely(size <= SMALL_MAXCLASS)) {
/* Small allocation. */ /* Small allocation. */
@ -1455,7 +1456,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
size_t pageind = ((uintptr_t)ptr - size_t pageind = ((uintptr_t)ptr -
(uintptr_t)chunk) >> LG_PAGE; (uintptr_t)chunk) >> LG_PAGE;
arena_dalloc_small(tsdn, arena_dalloc_small(tsdn,
extent_arena_get(&chunk->extent), chunk, extent_arena_get(extent), chunk, extent,
ptr, pageind); ptr, pageind);
} }
} else { } else {
@ -1467,12 +1468,12 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
size, slow_path); size, slow_path);
} else { } else {
arena_dalloc_large(tsdn, arena_dalloc_large(tsdn,
extent_arena_get(&chunk->extent), chunk, extent_arena_get(extent), chunk, extent,
ptr); ptr);
} }
} }
} else } else
huge_dalloc(tsdn, ptr); huge_dalloc(tsdn, extent, ptr);
} }
# endif /* JEMALLOC_ARENA_INLINE_B */ # endif /* JEMALLOC_ARENA_INLINE_B */
#endif #endif

2
include/jemalloc/internal/chunk.h
View File

@ -54,6 +54,8 @@ chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena,
bool chunk_register(tsdn_t *tsdn, const void *chunk, const extent_t *extent); bool chunk_register(tsdn_t *tsdn, const void *chunk, const extent_t *extent);
void chunk_deregister(const void *chunk, const extent_t *extent); void chunk_deregister(const void *chunk, const extent_t *extent);
void chunk_reregister(tsdn_t *tsdn, const void *chunk,
const extent_t *extent);
void *chunk_alloc_base(size_t size); void *chunk_alloc_base(size_t size);
void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,

22
include/jemalloc/internal/huge.h
View File

@ -12,20 +12,22 @@
void *huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero); void *huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, void *huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero); size_t alignment, bool zero);
bool huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, bool huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t usize_min, size_t usize_max, bool zero); size_t oldsize, size_t usize_min, size_t usize_max, bool zero);
void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t usize, size_t alignment, bool zero, tcache_t *tcache); size_t oldsize, size_t usize, size_t alignment, bool zero,
tcache_t *tcache);
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
typedef void (huge_dalloc_junk_t)(tsdn_t *, void *, size_t); typedef void (huge_dalloc_junk_t)(tsdn_t *, void *, size_t);
extern huge_dalloc_junk_t *huge_dalloc_junk; extern huge_dalloc_junk_t *huge_dalloc_junk;
#endif #endif
void huge_dalloc(tsdn_t *tsdn, void *ptr); void huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr);
arena_t *huge_aalloc(const void *ptr); size_t huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr);
size_t huge_salloc(tsdn_t *tsdn, const void *ptr); prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr); const void *ptr);
void huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx); void huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
void huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr); prof_tctx_t *tctx);
void huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr);
#endif /* JEMALLOC_H_EXTERNS */ #endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/ /******************************************************************************/

124
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -959,6 +959,20 @@ decay_ticker_get(tsd_t *tsd, unsigned ind)
#define JEMALLOC_ARENA_INLINE_A #define JEMALLOC_ARENA_INLINE_A
#include "jemalloc/internal/arena.h" #include "jemalloc/internal/arena.h"
#undef JEMALLOC_ARENA_INLINE_A #undef JEMALLOC_ARENA_INLINE_A
#ifndef JEMALLOC_ENABLE_INLINE
extent_t *iealloc(const void *ptr);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE extent_t *
iealloc(const void *ptr)
{
return (chunk_lookup(ptr, true));
}
#endif
#include "jemalloc/internal/tcache.h" #include "jemalloc/internal/tcache.h"
#define JEMALLOC_ARENA_INLINE_B #define JEMALLOC_ARENA_INLINE_B
#include "jemalloc/internal/arena.h" #include "jemalloc/internal/arena.h"
@ -968,7 +982,8 @@ decay_ticker_get(tsd_t *tsd, unsigned ind)
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
extent_t *iealloc(const void *ptr); extent_t *iealloc(const void *ptr);
arena_t *iaalloc(const void *ptr); arena_t *iaalloc(const void *ptr);
size_t isalloc(tsdn_t *tsdn, const void *ptr, bool demote); size_t isalloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr,
bool demote);
void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
tcache_t *tcache, bool is_metadata, arena_t *arena, bool slow_path); tcache_t *tcache, bool is_metadata, arena_t *arena, bool slow_path);
void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero,
@ -979,30 +994,23 @@ void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
tcache_t *tcache, arena_t *arena); tcache_t *tcache, arena_t *arena);
void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
size_t ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote); size_t ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote);
void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata, void idalloctm(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
bool slow_path); bool is_metadata, bool slow_path);
void idalloc(tsd_t *tsd, void *ptr); void idalloc(tsd_t *tsd, extent_t *extent, void *ptr);
void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, void isdalloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
bool slow_path); tcache_t *tcache, bool slow_path);
void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, void *iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t extra, size_t alignment, bool zero, tcache_t *tcache, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero,
arena_t *arena); tcache_t *tcache, arena_t *arena);
void *iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, void *iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); size_t size, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, void *iralloc(tsd_t *tsd, extent_t *extent, void *ptr, size_t oldsize,
size_t alignment, bool zero); size_t size, size_t alignment, bool zero);
bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, bool ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t extra, size_t alignment, bool zero); size_t size, size_t extra, size_t alignment, bool zero);
#endif #endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE extent_t *
iealloc(const void *ptr)
{
return (chunk_lookup(ptr, true));
}
JEMALLOC_ALWAYS_INLINE arena_t * JEMALLOC_ALWAYS_INLINE arena_t *
iaalloc(const void *ptr) iaalloc(const void *ptr)
{ {
@ -1016,17 +1024,18 @@ iaalloc(const void *ptr)
* Typical usage: * Typical usage:
* tsdn_t *tsdn = [...] * tsdn_t *tsdn = [...]
* void *ptr = [...] * void *ptr = [...]
* size_t sz = isalloc(tsdn, ptr, config_prof); * extent_t *extent = iealloc(ptr);
* size_t sz = isalloc(tsdn, extent, ptr, config_prof);
*/ */
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
isalloc(tsdn_t *tsdn, const void *ptr, bool demote) isalloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr, bool demote)
{ {
assert(ptr != NULL); assert(ptr != NULL);
/* Demotion only makes sense if config_prof is true. */ /* Demotion only makes sense if config_prof is true. */
assert(config_prof || !demote); assert(config_prof || !demote);
return (arena_salloc(tsdn, ptr, demote)); return (arena_salloc(tsdn, extent, ptr, demote));
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
@ -1041,8 +1050,8 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path);
if (config_stats && is_metadata && likely(ret != NULL)) { if (config_stats && is_metadata && likely(ret != NULL)) {
arena_metadata_allocated_add(iaalloc(ret), arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn,
isalloc(tsdn, ret, config_prof)); iealloc(ret), ret, config_prof));
} }
return (ret); return (ret);
} }
@ -1069,8 +1078,8 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache);
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
if (config_stats && is_metadata && likely(ret != NULL)) { if (config_stats && is_metadata && likely(ret != NULL)) {
arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn, ret, arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn,
config_prof)); iealloc(ret), ret, config_prof));
} }
return (ret); return (ret);
} }
@ -1104,43 +1113,45 @@ ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote)
assert(extent_addr_get(extent) == ptr || assert(extent_addr_get(extent) == ptr ||
extent_achunk_get(extent)); extent_achunk_get(extent));
return (isalloc(tsdn, ptr, demote)); return (isalloc(tsdn, extent, ptr, demote));
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata, idalloctm(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
bool slow_path) bool is_metadata, bool slow_path)
{ {
assert(ptr != NULL); assert(ptr != NULL);
assert(!is_metadata || tcache == NULL); assert(!is_metadata || tcache == NULL);
assert(!is_metadata || iaalloc(ptr)->ind < narenas_auto); assert(!is_metadata || iaalloc(ptr)->ind < narenas_auto);
if (config_stats && is_metadata) { if (config_stats && is_metadata) {
arena_metadata_allocated_sub(iaalloc(ptr), isalloc(tsdn, ptr, arena_metadata_allocated_sub(iaalloc(ptr), isalloc(tsdn, extent,
config_prof)); ptr, config_prof));
} }
arena_dalloc(tsdn, ptr, tcache, slow_path); arena_dalloc(tsdn, extent, ptr, tcache, slow_path);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
idalloc(tsd_t *tsd, void *ptr) idalloc(tsd_t *tsd, extent_t *extent, void *ptr)
{ {
idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true); idalloctm(tsd_tsdn(tsd), extent, ptr, tcache_get(tsd, false), false,
true);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, isdalloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
bool slow_path) tcache_t *tcache, bool slow_path)
{ {
arena_sdalloc(tsdn, ptr, size, tcache, slow_path); arena_sdalloc(tsdn, extent, ptr, size, tcache, slow_path);
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache,
arena_t *arena)
{ {
void *p; void *p;
size_t usize, copysize; size_t usize, copysize;
@ -1166,13 +1177,13 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
*/ */
copysize = (size < oldsize) ? size : oldsize; copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize); memcpy(p, ptr, copysize);
isdalloct(tsdn, ptr, oldsize, tcache, true); isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
return (p); return (p);
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
bool zero, tcache_t *tcache, arena_t *arena) size_t alignment, bool zero, tcache_t *tcache, arena_t *arena)
{ {
assert(ptr != NULL); assert(ptr != NULL);
@ -1184,26 +1195,26 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
* Existing object alignment is inadequate; allocate new space * Existing object alignment is inadequate; allocate new space
* and copy. * and copy.
*/ */
return (iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment, return (iralloct_realign(tsdn, extent, ptr, oldsize, size, 0,
zero, tcache, arena)); alignment, zero, tcache, arena));
} }
return (arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, return (arena_ralloc(tsdn, arena, extent, ptr, oldsize, size, alignment,
tcache)); zero, tcache));
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, iralloc(tsd_t *tsd, extent_t *extent, void *ptr, size_t oldsize, size_t size,
bool zero) size_t alignment, bool zero)
{ {
return (iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, return (iralloct(tsd_tsdn(tsd), extent, ptr, oldsize, size, alignment,
tcache_get(tsd, true), NULL)); zero, tcache_get(tsd, true), NULL));
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero) size_t extra, size_t alignment, bool zero)
{ {
assert(ptr != NULL); assert(ptr != NULL);
@ -1215,7 +1226,8 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
return (true); return (true);
} }
return (arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero)); return (arena_ralloc_no_move(tsdn, extent, ptr, oldsize, size, extra,
zero));
} }
#endif #endif

2
include/jemalloc/internal/private_symbols.txt
View File

@ -181,6 +181,7 @@ chunk_postfork_parent
chunk_prefork chunk_prefork
chunk_purge_wrapper chunk_purge_wrapper
chunk_register chunk_register
chunk_reregister
chunks_rtree chunks_rtree
chunksize chunksize
chunksize_mask chunksize_mask
@ -277,7 +278,6 @@ hash_rotl_64
hash_x64_128 hash_x64_128
hash_x86_128 hash_x86_128
hash_x86_32 hash_x86_32
huge_aalloc
huge_dalloc huge_dalloc
huge_dalloc_junk huge_dalloc_junk
huge_malloc huge_malloc

81
include/jemalloc/internal/prof.h
View File

@ -281,8 +281,8 @@ extern uint64_t prof_interval;
extern size_t lg_prof_sample; extern size_t lg_prof_sample;
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, void prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent,
prof_tctx_t *tctx); const void *ptr, size_t usize, prof_tctx_t *tctx);
void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
void bt_init(prof_bt_t *bt, void **vec); void bt_init(prof_bt_t *bt, void **vec);
void prof_backtrace(prof_bt_t *bt); void prof_backtrace(prof_bt_t *bt);
@ -330,21 +330,23 @@ void prof_sample_threshold_update(prof_tdata_t *tdata);
bool prof_active_get_unlocked(void); bool prof_active_get_unlocked(void);
bool prof_gdump_get_unlocked(void); bool prof_gdump_get_unlocked(void);
prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create);
prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const void *ptr); prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
void prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, const void *ptr);
prof_tctx_t *tctx); void prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
void prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, size_t usize, prof_tctx_t *tctx);
const void *old_ptr, prof_tctx_t *tctx); void prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *tctx);
bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
prof_tdata_t **tdata_out); prof_tdata_t **tdata_out);
prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
bool update); bool update);
void prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, void prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx); size_t usize, prof_tctx_t *tctx);
void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, void prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr,
prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr, size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated,
size_t old_usize, prof_tctx_t *old_tctx); const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(tsd_t *tsd, const void *ptr, size_t usize); void prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr,
size_t usize);
#endif #endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
@ -398,34 +400,35 @@ prof_tdata_get(tsd_t *tsd, bool create)
} }
JEMALLOC_ALWAYS_INLINE prof_tctx_t * JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_tctx_get(tsdn_t *tsdn, const void *ptr) prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{ {
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
return (arena_prof_tctx_get(tsdn, ptr)); return (arena_prof_tctx_get(tsdn, extent, ptr));
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
prof_tctx_t *tctx)
{ {
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
arena_prof_tctx_set(tsdn, ptr, usize, tctx); arena_prof_tctx_set(tsdn, extent, ptr, usize, tctx);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, const void *old_ptr, prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
prof_tctx_t *old_tctx) const void *old_ptr, prof_tctx_t *old_tctx)
{ {
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
arena_prof_tctx_reset(tsdn, ptr, usize, old_ptr, old_tctx); arena_prof_tctx_reset(tsdn, extent, ptr, usize, old_ptr, old_tctx);
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
@ -480,23 +483,26 @@ prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update)
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
prof_tctx_t *tctx)
{ {
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
assert(usize == isalloc(tsdn, ptr, true)); assert(usize == isalloc(tsdn, extent, ptr, true));
if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_malloc_sample_object(tsdn, ptr, usize, tctx); prof_malloc_sample_object(tsdn, extent, ptr, usize, tctx);
else else {
prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U); prof_tctx_set(tsdn, extent, ptr, usize,
(prof_tctx_t *)(uintptr_t)1U);
}
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
bool prof_active, bool updated, const void *old_ptr, size_t old_usize, prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
prof_tctx_t *old_tctx) size_t old_usize, prof_tctx_t *old_tctx)
{ {
bool sampled, old_sampled; bool sampled, old_sampled;
@ -504,7 +510,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
if (prof_active && !updated && ptr != NULL) { if (prof_active && !updated && ptr != NULL) {
assert(usize == isalloc(tsd_tsdn(tsd), ptr, true)); assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr, true));
if (prof_sample_accum_update(tsd, usize, true, NULL)) { if (prof_sample_accum_update(tsd, usize, true, NULL)) {
/* /*
* Don't sample. The usize passed to prof_alloc_prep() * Don't sample. The usize passed to prof_alloc_prep()
@ -520,22 +526,25 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
sampled = ((uintptr_t)tctx > (uintptr_t)1U); sampled = ((uintptr_t)tctx > (uintptr_t)1U);
old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U); old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
if (unlikely(sampled)) if (unlikely(sampled)) {
prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx); prof_malloc_sample_object(tsd_tsdn(tsd), extent, ptr, usize,
else tctx);
prof_tctx_reset(tsd_tsdn(tsd), ptr, usize, old_ptr, old_tctx); } else {
prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, usize, old_ptr,
old_tctx);
}
if (unlikely(old_sampled)) if (unlikely(old_sampled))
prof_free_sampled_object(tsd, old_usize, old_tctx); prof_free_sampled_object(tsd, old_usize, old_tctx);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
prof_free(tsd_t *tsd, const void *ptr, size_t usize) prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr, size_t usize)
{ {
prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr); prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr);
cassert(config_prof); cassert(config_prof);
assert(usize == isalloc(tsd_tsdn(tsd), ptr, true)); assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr, true));
if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_free_sampled_object(tsd, usize, tctx); prof_free_sampled_object(tsd, usize, tctx);

2
include/jemalloc/internal/tcache.h
View File

@ -371,7 +371,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
if (config_prof && usize == LARGE_MINCLASS) { if (config_prof && usize == LARGE_MINCLASS) {
arena_chunk_t *chunk = arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(ret); (arena_chunk_t *)extent_addr_get(iealloc(ret));
size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
LG_PAGE); LG_PAGE);
arena_mapbits_large_binind_set(chunk, pageind, arena_mapbits_large_binind_set(chunk, pageind,

245
src/arena.c
View File

@ -45,10 +45,10 @@ unsigned nhclasses; /* Number of huge size classes. */
static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena,
size_t ndirty_limit); size_t ndirty_limit);
static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool dirty, bool cleaned, bool decommitted); arena_run_t *run, bool dirty, bool cleaned, bool decommitted);
static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin); arena_chunk_t *chunk, extent_t *extent, arena_run_t *run, arena_bin_t *bin);
static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
arena_run_t *run, arena_bin_t *bin); arena_run_t *run, arena_bin_t *bin);
@ -264,9 +264,9 @@ arena_run_reg_alloc(arena_run_t *run, const arena_bin_info_t *bin_info)
} }
JEMALLOC_INLINE_C void JEMALLOC_INLINE_C void
arena_run_reg_dalloc(arena_run_t *run, void *ptr) arena_run_reg_dalloc(arena_run_t *run, extent_t *extent, void *ptr)
{ {
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind); size_t mapbits = arena_mapbits_get(chunk, pageind);
szind_t binind = arena_ptr_small_binind_get(ptr, mapbits); szind_t binind = arena_ptr_small_binind_get(ptr, mapbits);
@ -375,15 +375,15 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
} }
static bool static bool
arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, arena_run_split_large_helper(arena_t *arena, extent_t *extent, arena_run_t *run,
bool remove, bool zero) size_t size, bool remove, bool zero)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm; arena_chunk_map_misc_t *miscelm;
size_t flag_dirty, flag_decommitted, run_ind, need_pages; size_t flag_dirty, flag_decommitted, run_ind, need_pages;
size_t flag_unzeroed_mask; size_t flag_unzeroed_mask;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run); miscelm = arena_run_to_miscelm(run);
run_ind = arena_miscelm_to_pageind(miscelm); run_ind = arena_miscelm_to_pageind(miscelm);
flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
@ -439,22 +439,26 @@ arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,
} }
static bool static bool
arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) arena_run_split_large(arena_t *arena, extent_t *extent, arena_run_t *run,
size_t size, bool zero)
{ {
return (arena_run_split_large_helper(arena, run, size, true, zero)); return (arena_run_split_large_helper(arena, extent, run, size, true,
zero));
} }
static bool static bool
arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) arena_run_init_large(arena_t *arena, extent_t *extent, arena_run_t *run,
size_t size, bool zero)
{ {
return (arena_run_split_large_helper(arena, run, size, false, zero)); return (arena_run_split_large_helper(arena, extent, run, size, false,
zero));
} }
static bool static bool
arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, arena_run_split_small(arena_t *arena, extent_t *extent, arena_run_t *run,
szind_t binind) size_t size, szind_t binind)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm; arena_chunk_map_misc_t *miscelm;
@ -462,7 +466,7 @@ arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
assert(binind != BININD_INVALID); assert(binind != BININD_INVALID);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run); miscelm = arena_run_to_miscelm(run);
run_ind = arena_miscelm_to_pageind(miscelm); run_ind = arena_miscelm_to_pageind(miscelm);
flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
@ -1037,7 +1041,7 @@ arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
{ {
arena_run_t *run = arena_run_first_best_fit(arena, s2u(size)); arena_run_t *run = arena_run_first_best_fit(arena, s2u(size));
if (run != NULL) { if (run != NULL) {
if (arena_run_split_large(arena, run, size, zero)) if (arena_run_split_large(arena, iealloc(run), run, size, zero))
run = NULL; run = NULL;
} }
return (run); return (run);
@ -1063,7 +1067,7 @@ arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero)
chunk = arena_chunk_alloc(tsdn, arena); chunk = arena_chunk_alloc(tsdn, arena);
if (chunk != NULL) { if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run; run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
if (arena_run_split_large(arena, run, size, zero)) if (arena_run_split_large(arena, iealloc(run), run, size, zero))
run = NULL; run = NULL;
return (run); return (run);
} }
@ -1081,7 +1085,8 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind)
{ {
arena_run_t *run = arena_run_first_best_fit(arena, size); arena_run_t *run = arena_run_first_best_fit(arena, size);
if (run != NULL) { if (run != NULL) {
if (arena_run_split_small(arena, run, size, binind)) if (arena_run_split_small(arena, iealloc(run), run, size,
binind))
run = NULL; run = NULL;
} }
return (run); return (run);
@ -1108,7 +1113,8 @@ arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind)
chunk = arena_chunk_alloc(tsdn, arena); chunk = arena_chunk_alloc(tsdn, arena);
if (chunk != NULL) { if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run; run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
if (arena_run_split_small(arena, run, size, binind)) if (arena_run_split_small(arena, iealloc(run), run, size,
binind))
run = NULL; run = NULL;
return (run); return (run);
} }
@ -1435,8 +1441,9 @@ arena_dirty_count(arena_t *arena)
npages = extent_size_get(chunkselm) >> LG_PAGE; npages = extent_size_get(chunkselm) >> LG_PAGE;
chunkselm = qr_next(chunkselm, cc_link); chunkselm = qr_next(chunkselm, cc_link);
} else { } else {
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( extent_t *extent = iealloc(rdelm);
rdelm); arena_chunk_t *chunk =
(arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm = arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm); arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm); size_t pageind = arena_miscelm_to_pageind(miscelm);
@ -1497,8 +1504,9 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
LG_PAGE)); LG_PAGE));
chunkselm = chunkselm_next; chunkselm = chunkselm_next;
} else { } else {
extent_t *extent = iealloc(rdelm);
arena_chunk_t *chunk = arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm = arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm); arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm); size_t pageind = arena_miscelm_to_pageind(miscelm);
@ -1523,7 +1531,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
arena_chunk_alloc(tsdn, arena); arena_chunk_alloc(tsdn, arena);
/* Temporarily allocate the free dirty run. */ /* Temporarily allocate the free dirty run. */
arena_run_split_large(arena, run, run_size, false); arena_run_split_large(arena, extent, run, run_size,
false);
/* Stash. */ /* Stash. */
if (false) if (false)
qr_new(rdelm, rd_link); /* Redundant. */ qr_new(rdelm, rd_link); /* Redundant. */
@ -1577,8 +1586,9 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
} else { } else {
size_t pageind, run_size, flag_unzeroed, flags, i; size_t pageind, run_size, flag_unzeroed, flags, i;
bool decommitted; bool decommitted;
extent_t *extent = iealloc(rdelm);
arena_chunk_t *chunk = arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm = arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm); arena_rd_to_miscelm(rdelm);
pageind = arena_miscelm_to_pageind(miscelm); pageind = arena_miscelm_to_pageind(miscelm);
@ -1661,8 +1671,9 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr, chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr,
size, zeroed, committed); size, zeroed, committed);
} else { } else {
extent_t *extent = iealloc(rdelm);
arena_chunk_t *chunk = arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm = arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm); arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm); size_t pageind = arena_miscelm_to_pageind(miscelm);
@ -1670,7 +1681,7 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
pageind) != 0); pageind) != 0);
arena_run_t *run = &miscelm->run; arena_run_t *run = &miscelm->run;
qr_remove(rdelm, rd_link); qr_remove(rdelm, rd_link);
arena_run_dalloc(tsdn, arena, run, false, true, arena_run_dalloc(tsdn, arena, extent, run, false, true,
decommitted); decommitted);
} }
} }
@ -1755,10 +1766,10 @@ arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk)
if (arena_mapbits_large_get(chunk, pageind) != 0) { if (arena_mapbits_large_get(chunk, pageind) != 0) {
void *ptr = (void *)((uintptr_t)chunk + (pageind void *ptr = (void *)((uintptr_t)chunk + (pageind
<< LG_PAGE)); << LG_PAGE));
size_t usize = isalloc(tsd_tsdn(tsd), ptr, size_t usize = isalloc(tsd_tsdn(tsd),
config_prof); &chunk->extent, ptr, config_prof);
prof_free(tsd, ptr, usize); prof_free(tsd, &chunk->extent, ptr, usize);
npages = arena_mapbits_large_size_get(chunk, npages = arena_mapbits_large_size_get(chunk,
pageind) >> LG_PAGE; pageind) >> LG_PAGE;
} else { } else {
@ -1820,12 +1831,14 @@ arena_reset(tsd_t *tsd, arena_t *arena)
size_t usize; size_t usize;
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx); malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx);
if (config_stats || (config_prof && opt_prof)) if (config_stats || (config_prof && opt_prof)) {
usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); usize = isalloc(tsd_tsdn(tsd), extent, ptr,
config_prof);
}
/* Remove huge allocation from prof sample set. */ /* Remove huge allocation from prof sample set. */
if (config_prof && opt_prof) if (config_prof && opt_prof)
prof_free(tsd, ptr, usize); prof_free(tsd, extent, ptr, usize);
huge_dalloc(tsd_tsdn(tsd), ptr); huge_dalloc(tsd_tsdn(tsd), extent, ptr);
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx); malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx);
/* Cancel out unwanted effects on stats. */ /* Cancel out unwanted effects on stats. */
if (config_stats) if (config_stats)
@ -1997,14 +2010,14 @@ arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
} }
static void static void
arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty, arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
bool cleaned, bool decommitted) arena_run_t *run, bool dirty, bool cleaned, bool decommitted)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm; arena_chunk_map_misc_t *miscelm;
size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; size_t size, run_ind, run_pages, flag_dirty, flag_decommitted;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run); miscelm = arena_run_to_miscelm(run);
run_ind = arena_miscelm_to_pageind(miscelm); run_ind = arena_miscelm_to_pageind(miscelm);
assert(run_ind >= map_bias); assert(run_ind >= map_bias);
@ -2074,7 +2087,7 @@ arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty,
static void static void
arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
arena_run_t *run, size_t oldsize, size_t newsize) extent_t *extent, arena_run_t *run, size_t oldsize, size_t newsize)
{ {
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
size_t pageind = arena_miscelm_to_pageind(miscelm); size_t pageind = arena_miscelm_to_pageind(miscelm);
@ -2109,13 +2122,14 @@ arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
pageind+head_npages))); pageind+head_npages)));
arena_run_dalloc(tsdn, arena, run, false, false, (flag_decommitted != arena_run_dalloc(tsdn, arena, extent, run, false, false,
0)); (flag_decommitted != 0));
} }
static void static void
arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) extent_t *extent, arena_run_t *run, size_t oldsize, size_t newsize,
bool dirty)
{ {
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
size_t pageind = arena_miscelm_to_pageind(miscelm); size_t pageind = arena_miscelm_to_pageind(miscelm);
@ -2154,8 +2168,8 @@ arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages); tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages);
tail_run = &tail_miscelm->run; tail_run = &tail_miscelm->run;
arena_run_dalloc(tsdn, arena, tail_run, dirty, false, (flag_decommitted arena_run_dalloc(tsdn, arena, extent, tail_run, dirty, false,
!= 0)); (flag_decommitted != 0));
} }
static void static void
@ -2251,6 +2265,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin)
assert(bin->runcur->nfree > 0); assert(bin->runcur->nfree > 0);
ret = arena_run_reg_alloc(bin->runcur, bin_info); ret = arena_run_reg_alloc(bin->runcur, bin_info);
if (run != NULL) { if (run != NULL) {
extent_t *extent;
arena_chunk_t *chunk; arena_chunk_t *chunk;
/* /*
@ -2261,10 +2276,11 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin)
* arena_bin_lower_run() must be called, as if a region * arena_bin_lower_run() must be called, as if a region
* were just deallocated from the run. * were just deallocated from the run.
*/ */
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); extent = iealloc(run);
chunk = (arena_chunk_t *)extent_addr_get(extent);
if (run->nfree == bin_info->nregs) { if (run->nfree == bin_info->nregs) {
arena_dalloc_bin_run(tsdn, arena, chunk, run, arena_dalloc_bin_run(tsdn, arena, chunk, extent,
bin); run, bin);
} else } else
arena_bin_lower_run(arena, chunk, run, bin); arena_bin_lower_run(arena, chunk, run, bin);
} }
@ -2499,6 +2515,7 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
void *ret; void *ret;
size_t alloc_size, leadsize, trailsize; size_t alloc_size, leadsize, trailsize;
arena_run_t *run; arena_run_t *run;
extent_t *extent;
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm; arena_chunk_map_misc_t *miscelm;
void *rpages; void *rpages;
@ -2520,7 +2537,8 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
return (NULL); return (NULL);
} }
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); extent = iealloc(run);
chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run); miscelm = arena_run_to_miscelm(run);
rpages = arena_miscelm_to_rpages(miscelm); rpages = arena_miscelm_to_rpages(miscelm);
@ -2531,20 +2549,22 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
if (leadsize != 0) { if (leadsize != 0) {
arena_chunk_map_misc_t *head_miscelm = miscelm; arena_chunk_map_misc_t *head_miscelm = miscelm;
arena_run_t *head_run = run; arena_run_t *head_run = run;
extent_t *head_extent = extent;
miscelm = arena_miscelm_get_mutable(chunk, miscelm = arena_miscelm_get_mutable(chunk,
arena_miscelm_to_pageind(head_miscelm) + (leadsize >> arena_miscelm_to_pageind(head_miscelm) + (leadsize >>
LG_PAGE)); LG_PAGE));
run = &miscelm->run; run = &miscelm->run;
extent = iealloc(run);
arena_run_trim_head(tsdn, arena, chunk, head_run, alloc_size, arena_run_trim_head(tsdn, arena, chunk, head_extent, head_run,
alloc_size - leadsize); alloc_size, alloc_size - leadsize);
} }
if (trailsize != 0) { if (trailsize != 0) {
arena_run_trim_tail(tsdn, arena, chunk, run, usize + large_pad + arena_run_trim_tail(tsdn, arena, chunk, extent, run, usize +
trailsize, usize + large_pad, false); large_pad + trailsize, usize + large_pad, false);
} }
if (arena_run_init_large(arena, run, usize + large_pad, zero)) { if (arena_run_init_large(arena, extent, run, usize + large_pad, zero)) {
size_t run_ind = size_t run_ind =
arena_miscelm_to_pageind(arena_run_to_miscelm(run)); arena_miscelm_to_pageind(arena_run_to_miscelm(run));
bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0); bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0);
@ -2552,7 +2572,8 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
run_ind) != 0); run_ind) != 0);
assert(decommitted); /* Cause of OOM. */ assert(decommitted); /* Cause of OOM. */
arena_run_dalloc(tsdn, arena, run, dirty, false, decommitted); arena_run_dalloc(tsdn, arena, extent, run, dirty, false,
decommitted);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
return (NULL); return (NULL);
} }
@ -2616,7 +2637,8 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
} }
void void
arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size) arena_prof_promoted(tsdn_t *tsdn, const extent_t *extent, const void *ptr,
size_t size)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
size_t pageind; size_t pageind;
@ -2624,32 +2646,30 @@ arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size)
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
assert(CHUNK_ADDR2BASE(ptr) != ptr); assert(extent_addr_get(extent) != ptr);
assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); assert(isalloc(tsdn, extent, ptr, false) == LARGE_MINCLASS);
assert(isalloc(tsdn, ptr, true) == LARGE_MINCLASS); assert(isalloc(tsdn, extent, ptr, true) == LARGE_MINCLASS);
assert(size <= SMALL_MAXCLASS); assert(size <= SMALL_MAXCLASS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)extent_addr_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
binind = size2index(size); binind = size2index(size);
assert(binind < NBINS); assert(binind < NBINS);
arena_mapbits_large_binind_set(chunk, pageind, binind); arena_mapbits_large_binind_set(chunk, pageind, binind);
assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS); assert(isalloc(tsdn, extent, ptr, false) == LARGE_MINCLASS);
assert(isalloc(tsdn, ptr, true) == size); assert(isalloc(tsdn, extent, ptr, true) == size);
} }
static void static void
arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, arena_dissociate_bin_run(extent_t *extent, arena_run_t *run, arena_bin_t *bin)
arena_bin_t *bin)
{ {
/* Dissociate run from bin. */ /* Dissociate run from bin. */
if (run == bin->runcur) if (run == bin->runcur)
bin->runcur = NULL; bin->runcur = NULL;
else { else {
szind_t binind = arena_bin_index(extent_arena_get( szind_t binind = arena_bin_index(extent_arena_get(extent), bin);
&chunk->extent), bin);
const arena_bin_info_t *bin_info = &arena_bin_info[binind]; const arena_bin_info_t *bin_info = &arena_bin_info[binind];
/* /*
@ -2668,7 +2688,7 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
static void static void
arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
arena_run_t *run, arena_bin_t *bin) extent_t *extent, arena_run_t *run, arena_bin_t *bin)
{ {
assert(run != bin->runcur); assert(run != bin->runcur);
@ -2676,7 +2696,7 @@ arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
malloc_mutex_unlock(tsdn, &bin->lock); malloc_mutex_unlock(tsdn, &bin->lock);
/******************************/ /******************************/
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
arena_run_dalloc(tsdn, arena, run, true, false, false); arena_run_dalloc(tsdn, arena, extent, run, true, false, false);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
/****************************/ /****************************/
malloc_mutex_lock(tsdn, &bin->lock); malloc_mutex_lock(tsdn, &bin->lock);
@ -2707,7 +2727,7 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
static void static void
arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr, arena_chunk_map_bits_t *bitselm, bool junked) extent_t *extent, void *ptr, arena_chunk_map_bits_t *bitselm, bool junked)
{ {
size_t pageind, rpages_ind; size_t pageind, rpages_ind;
arena_run_t *run; arena_run_t *run;
@ -2725,10 +2745,10 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
if (!junked && config_fill && unlikely(opt_junk_free)) if (!junked && config_fill && unlikely(opt_junk_free))
arena_dalloc_junk_small(ptr, bin_info); arena_dalloc_junk_small(ptr, bin_info);
arena_run_reg_dalloc(run, ptr); arena_run_reg_dalloc(run, extent, ptr);
if (run->nfree == bin_info->nregs) { if (run->nfree == bin_info->nregs) {
arena_dissociate_bin_run(chunk, run, bin); arena_dissociate_bin_run(extent, run, bin);
arena_dalloc_bin_run(tsdn, arena, chunk, run, bin); arena_dalloc_bin_run(tsdn, arena, chunk, extent, run, bin);
} else if (run->nfree == 1 && run != bin->runcur) } else if (run->nfree == 1 && run != bin->runcur)
arena_bin_lower_run(arena, chunk, run, bin); arena_bin_lower_run(arena, chunk, run, bin);
@ -2740,15 +2760,17 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void void
arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm) arena_chunk_t *chunk, extent_t *extent, void *ptr,
arena_chunk_map_bits_t *bitselm)
{ {
arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, true); arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm,
true);
} }
void static void
arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
size_t pageind, arena_chunk_map_bits_t *bitselm) extent_t *extent, void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm)
{ {
arena_run_t *run; arena_run_t *run;
arena_bin_t *bin; arena_bin_t *bin;
@ -2758,13 +2780,14 @@ arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr,
run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run;
bin = &arena->bins[run->binind]; bin = &arena->bins[run->binind];
malloc_mutex_lock(tsdn, &bin->lock); malloc_mutex_lock(tsdn, &bin->lock);
arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, false); arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm,
false);
malloc_mutex_unlock(tsdn, &bin->lock); malloc_mutex_unlock(tsdn, &bin->lock);
} }
void void
arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr, size_t pageind) extent_t *extent, void *ptr, size_t pageind)
{ {
arena_chunk_map_bits_t *bitselm; arena_chunk_map_bits_t *bitselm;
@ -2774,7 +2797,7 @@ arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
pageind)) != BININD_INVALID); pageind)) != BININD_INVALID);
} }
bitselm = arena_bitselm_get_mutable(chunk, pageind); bitselm = arena_bitselm_get_mutable(chunk, pageind);
arena_dalloc_bin(tsdn, arena, chunk, ptr, pageind, bitselm); arena_dalloc_bin(tsdn, arena, chunk, extent, ptr, pageind, bitselm);
arena_decay_tick(tsdn, arena); arena_decay_tick(tsdn, arena);
} }
@ -2798,7 +2821,7 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large =
static void static void
arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, void *ptr, bool junked) arena_chunk_t *chunk, extent_t *extent, void *ptr, bool junked)
{ {
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk,
@ -2821,31 +2844,31 @@ arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena,
} }
} }
arena_run_dalloc(tsdn, arena, run, true, false, false); arena_run_dalloc(tsdn, arena, extent, run, true, false, false);
} }
void void
arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena, arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, void *ptr) arena_chunk_t *chunk, extent_t *extent, void *ptr)
{ {
arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, true); arena_dalloc_large_locked_impl(tsdn, arena, chunk, extent, ptr, true);
} }
void void
arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr) extent_t *extent, void *ptr)
{ {
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, false); arena_dalloc_large_locked_impl(tsdn, arena, chunk, extent, ptr, false);
malloc_mutex_unlock(tsdn, &arena->lock); malloc_mutex_unlock(tsdn, &arena->lock);
arena_decay_tick(tsdn, arena); arena_decay_tick(tsdn, arena);
} }
static void static void
arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr, size_t oldsize, size_t size) extent_t *extent, void *ptr, size_t oldsize, size_t size)
{ {
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk,
@ -2859,8 +2882,8 @@ arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
* allocations. * allocations.
*/ */
malloc_mutex_lock(tsdn, &arena->lock); malloc_mutex_lock(tsdn, &arena->lock);
arena_run_trim_tail(tsdn, arena, chunk, run, oldsize + large_pad, size + arena_run_trim_tail(tsdn, arena, chunk, extent, run, oldsize +
large_pad, true); large_pad, size + large_pad, true);
if (config_stats) { if (config_stats) {
szind_t oldindex = size2index(oldsize) - NBINS; szind_t oldindex = size2index(oldsize) - NBINS;
szind_t index = size2index(size) - NBINS; szind_t index = size2index(size) - NBINS;
@ -2916,7 +2939,8 @@ arena_ralloc_large_grow(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
goto label_fail; goto label_fail;
run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run; run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run;
if (arena_run_split_large(arena, run, splitsize, zero)) if (arena_run_split_large(arena, iealloc(run), run, splitsize,
zero))
goto label_fail; goto label_fail;
if (config_cache_oblivious && zero) { if (config_cache_oblivious && zero) {
@ -3005,8 +3029,8 @@ arena_ralloc_junk_large_t *arena_ralloc_junk_large =
* always fail if growing an object, and the following run is already in use. * always fail if growing an object, and the following run is already in use.
*/ */
static bool static bool
arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min, arena_ralloc_large(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t usize_max, bool zero) size_t usize_min, size_t usize_max, bool zero)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_t *arena; arena_t *arena;
@ -3016,8 +3040,8 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
return (false); return (false);
} }
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)extent_addr_get(extent);
arena = extent_arena_get(&chunk->extent); arena = extent_arena_get(extent);
if (oldsize < usize_max) { if (oldsize < usize_max) {
bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr, bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr,
@ -3026,10 +3050,12 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
if (unlikely(opt_junk_alloc)) { if (unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)ptr + oldsize), memset((void *)((uintptr_t)ptr + oldsize),
JEMALLOC_ALLOC_JUNK, JEMALLOC_ALLOC_JUNK,
isalloc(tsdn, ptr, config_prof) - oldsize); isalloc(tsdn, extent, ptr, config_prof) -
oldsize);
} else if (unlikely(opt_zero)) { } else if (unlikely(opt_zero)) {
memset((void *)((uintptr_t)ptr + oldsize), 0, memset((void *)((uintptr_t)ptr + oldsize), 0,
isalloc(tsdn, ptr, config_prof) - oldsize); isalloc(tsdn, extent, ptr, config_prof) -
oldsize);
} }
} }
return (ret); return (ret);
@ -3038,13 +3064,14 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
assert(oldsize > usize_max); assert(oldsize > usize_max);
/* Fill before shrinking in order avoid a race. */ /* Fill before shrinking in order avoid a race. */
arena_ralloc_junk_large(ptr, oldsize, usize_max); arena_ralloc_junk_large(ptr, oldsize, usize_max);
arena_ralloc_large_shrink(tsdn, arena, chunk, ptr, oldsize, usize_max); arena_ralloc_large_shrink(tsdn, arena, chunk, extent, ptr, oldsize,
usize_max);
return (false); return (false);
} }
bool bool
arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t extra, bool zero) size_t size, size_t extra, bool zero)
{ {
size_t usize_min, usize_max; size_t usize_min, usize_max;
@ -3057,8 +3084,6 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
usize_min = s2u(size); usize_min = s2u(size);
usize_max = s2u(size + extra); usize_max = s2u(size + extra);
if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) { if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) {
arena_chunk_t *chunk;
/* /*
* Avoid moving the allocation if the size class can be left the * Avoid moving the allocation if the size class can be left the
* same. * same.
@ -3073,17 +3098,16 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
} else { } else {
if (usize_max <= SMALL_MAXCLASS) if (usize_max <= SMALL_MAXCLASS)
return (true); return (true);
if (arena_ralloc_large(tsdn, ptr, oldsize, usize_min, if (arena_ralloc_large(tsdn, extent, ptr, oldsize,
usize_max, zero)) usize_min, usize_max, zero))
return (true); return (true);
} }
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena_decay_tick(tsdn, extent_arena_get(extent));
arena_decay_tick(tsdn, extent_arena_get(&chunk->extent));
return (false); return (false);
} else { } else {
return (huge_ralloc_no_move(tsdn, ptr, oldsize, usize_min, return (huge_ralloc_no_move(tsdn, extent, ptr, oldsize,
usize_max, zero)); usize_min, usize_max, zero));
} }
} }
@ -3102,8 +3126,8 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
} }
void * void *
arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t size, size_t alignment, bool zero, tcache_t *tcache) size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache)
{ {
void *ret; void *ret;
size_t usize; size_t usize;
@ -3116,7 +3140,8 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t copysize; size_t copysize;
/* Try to avoid moving the allocation. */ /* Try to avoid moving the allocation. */
if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero)) if (!arena_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, 0,
zero))
return (ptr); return (ptr);
/* /*
@ -3136,10 +3161,10 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
copysize = (usize < oldsize) ? usize : oldsize; copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize); memcpy(ret, ptr, copysize);
isdalloct(tsdn, ptr, oldsize, tcache, true); isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
} else { } else {
ret = huge_ralloc(tsdn, arena, ptr, oldsize, usize, alignment, ret = huge_ralloc(tsdn, arena, extent, ptr, oldsize, usize,
zero, tcache); alignment, zero, tcache);
} }
return (ret); return (ret);
} }

9
src/chunk.c
View File

@ -180,6 +180,15 @@ chunk_deregister(const void *chunk, const extent_t *extent)
} }
} }
void
chunk_reregister(tsdn_t *tsdn, const void *chunk, const extent_t *extent)
{
bool err;
err = chunk_register(tsdn, chunk, extent);
assert(!err);
}
/* /*
* Do first-best-fit chunk selection, i.e. select the lowest chunk that best * Do first-best-fit chunk selection, i.e. select the lowest chunk that best
* fits. * fits.

10
src/ckh.c
View File

@ -283,12 +283,12 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (!ckh_rebuild(ckh, tab)) { if (!ckh_rebuild(ckh, tab)) {
idalloctm(tsdn, tab, NULL, true, true); idalloctm(tsdn, iealloc(tab), tab, NULL, true, true);
break; break;
} }
/* Rebuilding failed, so back out partially rebuilt table. */ /* Rebuilding failed, so back out partially rebuilt table. */
idalloctm(tsdn, ckh->tab, NULL, true, true); idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
ckh->tab = tab; ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets; ckh->lg_curbuckets = lg_prevbuckets;
} }
@ -330,7 +330,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (!ckh_rebuild(ckh, tab)) { if (!ckh_rebuild(ckh, tab)) {
idalloctm(tsdn, tab, NULL, true, true); idalloctm(tsdn, iealloc(tab), tab, NULL, true, true);
#ifdef CKH_COUNT #ifdef CKH_COUNT
ckh->nshrinks++; ckh->nshrinks++;
#endif #endif
@ -338,7 +338,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
} }
/* Rebuilding failed, so back out partially rebuilt table. */ /* Rebuilding failed, so back out partially rebuilt table. */
idalloctm(tsdn, ckh->tab, NULL, true, true); idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
ckh->tab = tab; ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets; ckh->lg_curbuckets = lg_prevbuckets;
#ifdef CKH_COUNT #ifdef CKH_COUNT
@ -421,7 +421,7 @@ ckh_delete(tsdn_t *tsdn, ckh_t *ckh)
(unsigned long long)ckh->nrelocs); (unsigned long long)ckh->nrelocs);
#endif #endif
idalloctm(tsdn, ckh->tab, NULL, true, true); idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
if (config_debug) if (config_debug)
memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t)); memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
} }

149
src/huge.c
View File

@ -3,42 +3,6 @@
/******************************************************************************/ /******************************************************************************/
static extent_t *
huge_extent_get(const void *ptr)
{
extent_t *extent;
extent = chunk_lookup(ptr, true);
assert(!extent_achunk_get(extent));
return (extent);
}
static bool
huge_extent_set(tsdn_t *tsdn, const void *ptr, extent_t *extent)
{
assert(extent_addr_get(extent) == ptr);
assert(!extent_achunk_get(extent));
return (chunk_register(tsdn, ptr, extent));
}
static void
huge_extent_reset(tsdn_t *tsdn, const void *ptr, extent_t *extent)
{
bool err;
err = huge_extent_set(tsdn, ptr, extent);
assert(!err);
}
static void
huge_extent_unset(const void *ptr, const extent_t *extent)
{
chunk_deregister(ptr, extent);
}
void * void *
huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero) huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
{ {
@ -81,15 +45,15 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
arena = arena_choose(tsdn_tsd(tsdn), arena); arena = arena_choose(tsdn_tsd(tsdn), arena);
if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn, if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL) { arena, usize, alignment, &is_zeroed)) == NULL) {
idalloctm(tsdn, extent, NULL, true, true); idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
return (NULL); return (NULL);
} }
extent_init(extent, arena, ret, usize, is_zeroed, true); extent_init(extent, arena, ret, usize, is_zeroed, true);
if (huge_extent_set(tsdn, ret, extent)) { if (chunk_register(tsdn, ret, extent)) {
arena_chunk_dalloc_huge(tsdn, arena, ret, usize); arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
idalloctm(tsdn, extent, NULL, true, true); idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
return (NULL); return (NULL);
} }
@ -133,11 +97,10 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
#endif #endif
static void static void
huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize, huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t usize_min, size_t usize_max, bool zero) size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
{ {
size_t usize, usize_next; size_t usize, usize_next;
extent_t *extent;
arena_t *arena; arena_t *arena;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
bool pre_zeroed, post_zeroed; bool pre_zeroed, post_zeroed;
@ -150,7 +113,6 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
if (oldsize == usize) if (oldsize == usize)
return; return;
extent = huge_extent_get(ptr);
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
pre_zeroed = extent_zeroed_get(extent); pre_zeroed = extent_zeroed_get(extent);
@ -169,15 +131,15 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
} else } else
post_zeroed = pre_zeroed; post_zeroed = pre_zeroed;
malloc_mutex_lock(tsdn, &arena->huge_mtx);
/* Update the size of the huge allocation. */ /* Update the size of the huge allocation. */
assert(extent_size_get(extent) != usize); assert(extent_size_get(extent) != usize);
huge_extent_unset(ptr, extent); chunk_deregister(tsdn, ptr, extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize); extent_size_set(extent, usize);
huge_extent_reset(tsdn, ptr, extent); malloc_mutex_unlock(tsdn, &arena->huge_mtx);
chunk_reregister(tsdn, ptr, extent);
/* Update zeroed. */ /* Update zeroed. */
extent_zeroed_set(extent, post_zeroed); extent_zeroed_set(extent, post_zeroed);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize); arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
@ -196,16 +158,14 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
} }
static bool static bool
huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t usize) size_t oldsize, size_t usize)
{ {
extent_t *extent;
arena_t *arena; arena_t *arena;
chunk_hooks_t chunk_hooks; chunk_hooks_t chunk_hooks;
size_t cdiff; size_t cdiff;
bool pre_zeroed, post_zeroed; bool pre_zeroed, post_zeroed;
extent = huge_extent_get(ptr);
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
pre_zeroed = extent_zeroed_get(extent); pre_zeroed = extent_zeroed_get(extent);
chunk_hooks = chunk_hooks_get(tsdn, arena); chunk_hooks = chunk_hooks_get(tsdn, arena);
@ -233,14 +193,14 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
} else } else
post_zeroed = pre_zeroed; post_zeroed = pre_zeroed;
malloc_mutex_lock(tsdn, &arena->huge_mtx);
/* Update the size of the huge allocation. */ /* Update the size of the huge allocation. */
huge_extent_unset(ptr, extent); chunk_deregister(ptr, extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize); extent_size_set(extent, usize);
huge_extent_reset(tsdn, ptr, extent);
/* Update zeroed. */ /* Update zeroed. */
extent_zeroed_set(extent, post_zeroed); extent_zeroed_set(extent, post_zeroed);
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->huge_mtx);
chunk_reregister(tsdn, ptr, extent);
/* Zap the excess chunks. */ /* Zap the excess chunks. */
arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize); arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
@ -249,14 +209,12 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
} }
static bool static bool
huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize, huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t usize, bool zero) size_t oldsize, size_t usize, bool zero)
{ {
extent_t *extent;
arena_t *arena; arena_t *arena;
bool is_zeroed_subchunk, is_zeroed_chunk; bool is_zeroed_subchunk, is_zeroed_chunk;
extent = huge_extent_get(ptr);
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->huge_mtx);
is_zeroed_subchunk = extent_zeroed_get(extent); is_zeroed_subchunk = extent_zeroed_get(extent);
@ -272,12 +230,12 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
&is_zeroed_chunk)) &is_zeroed_chunk))
return (true); return (true);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
/* Update the size of the huge allocation. */ /* Update the size of the huge allocation. */
huge_extent_unset(ptr, extent); chunk_deregister(ptr, extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize); extent_size_set(extent, usize);
huge_extent_reset(tsdn, ptr, extent);
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->huge_mtx);
chunk_reregister(tsdn, ptr, extent);
if (zero || (config_fill && unlikely(opt_zero))) { if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed_subchunk) { if (!is_zeroed_subchunk) {
@ -298,8 +256,8 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
} }
bool bool
huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min, huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t usize_max, bool zero) size_t usize_min, size_t usize_max, bool zero)
{ {
assert(s2u(oldsize) == oldsize); assert(s2u(oldsize) == oldsize);
@ -312,16 +270,16 @@ huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) { if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
/* Attempt to expand the allocation in-place. */ /* Attempt to expand the allocation in-place. */
if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max, if (!huge_ralloc_no_move_expand(tsdn, extent, ptr, oldsize,
zero)) { usize_max, zero)) {
arena_decay_tick(tsdn, huge_aalloc(ptr)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} }
/* Try again, this time with usize_min. */ /* Try again, this time with usize_min. */
if (usize_min < usize_max && CHUNK_CEILING(usize_min) > if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn, CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
ptr, oldsize, usize_min, zero)) { extent, ptr, oldsize, usize_min, zero)) {
arena_decay_tick(tsdn, huge_aalloc(ptr)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} }
} }
@ -332,17 +290,17 @@ huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
*/ */
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min) if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) { && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min, huge_ralloc_no_move_similar(tsdn, extent, ptr, oldsize,
usize_max, zero); usize_min, usize_max, zero);
arena_decay_tick(tsdn, huge_aalloc(ptr)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} }
/* Attempt to shrink the allocation in-place. */ /* Attempt to shrink the allocation in-place. */
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) { if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize, if (!huge_ralloc_no_move_shrink(tsdn, extent, ptr, oldsize,
usize_max)) { usize_max)) {
arena_decay_tick(tsdn, huge_aalloc(ptr)); arena_decay_tick(tsdn, extent_arena_get(extent));
return (false); return (false);
} }
} }
@ -360,8 +318,8 @@ huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
} }
void * void *
huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t usize, size_t alignment, bool zero, tcache_t *tcache) size_t oldsize, size_t usize, size_t alignment, bool zero, tcache_t *tcache)
{ {
void *ret; void *ret;
size_t copysize; size_t copysize;
@ -370,7 +328,8 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
assert(usize > 0 && usize <= HUGE_MAXCLASS); assert(usize > 0 && usize <= HUGE_MAXCLASS);
/* Try to avoid moving the allocation. */ /* Try to avoid moving the allocation. */
if (!huge_ralloc_no_move(tsdn, ptr, oldsize, usize, usize, zero)) if (!huge_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, usize,
zero))
return (ptr); return (ptr);
/* /*
@ -384,19 +343,17 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
copysize = (usize < oldsize) ? usize : oldsize; copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize); memcpy(ret, ptr, copysize);
isdalloct(tsdn, ptr, oldsize, tcache, true); isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
return (ret); return (ret);
} }
void void
huge_dalloc(tsdn_t *tsdn, void *ptr) huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr)
{ {
extent_t *extent;
arena_t *arena; arena_t *arena;
extent = huge_extent_get(ptr);
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
huge_extent_unset(ptr, extent); chunk_deregister(ptr, extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->huge_mtx);
ql_remove(&arena->huge, extent, ql_link); ql_remove(&arena->huge, extent, ql_link);
malloc_mutex_unlock(tsdn, &arena->huge_mtx); malloc_mutex_unlock(tsdn, &arena->huge_mtx);
@ -405,26 +362,17 @@ huge_dalloc(tsdn_t *tsdn, void *ptr)
extent_size_get(extent)); extent_size_get(extent));
arena_chunk_dalloc_huge(tsdn, extent_arena_get(extent), arena_chunk_dalloc_huge(tsdn, extent_arena_get(extent),
extent_addr_get(extent), extent_size_get(extent)); extent_addr_get(extent), extent_size_get(extent));
idalloctm(tsdn, extent, NULL, true, true); idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
arena_decay_tick(tsdn, arena); arena_decay_tick(tsdn, arena);
} }
arena_t *
huge_aalloc(const void *ptr)
{
return (extent_arena_get(huge_extent_get(ptr)));
}
size_t size_t
huge_salloc(tsdn_t *tsdn, const void *ptr) huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{ {
size_t size; size_t size;
extent_t *extent;
arena_t *arena; arena_t *arena;
extent = huge_extent_get(ptr);
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->huge_mtx);
size = extent_size_get(extent); size = extent_size_get(extent);
@ -434,13 +382,13 @@ huge_salloc(tsdn_t *tsdn, const void *ptr)
} }
prof_tctx_t * prof_tctx_t *
huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr) huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{ {
prof_tctx_t *tctx; prof_tctx_t *tctx;
extent_t *extent;
arena_t *arena; arena_t *arena;
extent = huge_extent_get(ptr); assert(extent == iealloc(ptr));
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->huge_mtx);
tctx = extent_prof_tctx_get(extent); tctx = extent_prof_tctx_get(extent);
@ -450,12 +398,13 @@ huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
} }
void void
huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx)
{ {
extent_t *extent;
arena_t *arena; arena_t *arena;
extent = huge_extent_get(ptr); assert(extent == iealloc(ptr));
arena = extent_arena_get(extent); arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx); malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_prof_tctx_set(extent, tctx); extent_prof_tctx_set(extent, tctx);
@ -463,8 +412,8 @@ huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
} }
void void
huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr) huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr)
{ {
huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U); huge_prof_tctx_set(tsdn, extent, ptr, (prof_tctx_t *)(uintptr_t)1U);
} }

210
src/jemalloc.c
View File

@ -308,10 +308,10 @@ a0ialloc(size_t size, bool zero, bool is_metadata)
} }
static void static void
a0idalloc(void *ptr, bool is_metadata) a0idalloc(extent_t *extent, void *ptr, bool is_metadata)
{ {
idalloctm(TSDN_NULL, ptr, false, is_metadata, true); idalloctm(TSDN_NULL, extent, ptr, false, is_metadata, true);
} }
void * void *
@ -325,7 +325,7 @@ void
a0dalloc(void *ptr) a0dalloc(void *ptr)
{ {
a0idalloc(ptr, true); a0idalloc(iealloc(ptr), ptr, true);
} }
/* /*
@ -365,7 +365,7 @@ bootstrap_free(void *ptr)
if (unlikely(ptr == NULL)) if (unlikely(ptr == NULL))
return; return;
a0idalloc(ptr, false); a0idalloc(iealloc(ptr), ptr, false);
} }
static void static void
@ -1401,7 +1401,7 @@ ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero,
p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path); p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path);
if (p == NULL) if (p == NULL)
return (NULL); return (NULL);
arena_prof_promoted(tsd_tsdn(tsd), p, usize); arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
} else } else
p = ialloc(tsd, usize, ind, zero, slow_path); p = ialloc(tsd, usize, ind, zero, slow_path);
@ -1423,7 +1423,7 @@ ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path)
prof_alloc_rollback(tsd, tctx, true); prof_alloc_rollback(tsd, tctx, true);
return (NULL); return (NULL);
} }
prof_malloc(tsd_tsdn(tsd), p, usize, tctx); prof_malloc(tsd_tsdn(tsd), iealloc(p), p, usize, tctx);
return (p); return (p);
} }
@ -1482,7 +1482,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
set_errno(ENOMEM); set_errno(ENOMEM);
} }
if (config_stats && likely(ret != NULL)) { if (config_stats && likely(ret != NULL)) {
assert(usize == isalloc(tsdn, ret, config_prof)); assert(usize == isalloc(tsdn, iealloc(ret), ret, config_prof));
*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize; *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
} }
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn);
@ -1525,7 +1525,7 @@ imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
p = ipalloc(tsd, LARGE_MINCLASS, alignment, false); p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
if (p == NULL) if (p == NULL)
return (NULL); return (NULL);
arena_prof_promoted(tsd_tsdn(tsd), p, usize); arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
} else } else
p = ipalloc(tsd, usize, alignment, false); p = ipalloc(tsd, usize, alignment, false);
@ -1547,7 +1547,7 @@ imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
prof_alloc_rollback(tsd, tctx, true); prof_alloc_rollback(tsd, tctx, true);
return (NULL); return (NULL);
} }
prof_malloc(tsd_tsdn(tsd), p, usize, tctx); prof_malloc(tsd_tsdn(tsd), iealloc(p), p, usize, tctx);
return (p); return (p);
} }
@ -1604,7 +1604,8 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
ret = 0; ret = 0;
label_return: label_return:
if (config_stats && likely(result != NULL)) { if (config_stats && likely(result != NULL)) {
assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof)); assert(usize == isalloc(tsd_tsdn(tsd), iealloc(result), result,
config_prof));
*tsd_thread_allocatedp_get(tsd) += usize; *tsd_thread_allocatedp_get(tsd) += usize;
} }
UTRACE(0, size, result); UTRACE(0, size, result);
@ -1683,44 +1684,49 @@ je_calloc(size_t num, size_t size)
} }
static void * static void *
irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, irealloc_prof_sample(tsd_t *tsd, extent_t *extent, void *old_ptr,
prof_tctx_t *tctx) size_t old_usize, size_t usize, prof_tctx_t *tctx)
{ {
void *p; void *p;
if (tctx == NULL) if (tctx == NULL)
return (NULL); return (NULL);
if (usize <= SMALL_MAXCLASS) { if (usize <= SMALL_MAXCLASS) {
p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); p = iralloc(tsd, extent, old_ptr, old_usize, LARGE_MINCLASS, 0,
false);
if (p == NULL) if (p == NULL)
return (NULL); return (NULL);
arena_prof_promoted(tsd_tsdn(tsd), p, usize); arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
} else } else
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false);
return (p); return (p);
} }
JEMALLOC_ALWAYS_INLINE_C void * JEMALLOC_ALWAYS_INLINE_C void *
irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) irealloc_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize,
size_t usize)
{ {
void *p; void *p;
extent_t *e;
bool prof_active; bool prof_active;
prof_tctx_t *old_tctx, *tctx; prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked(); prof_active = prof_active_get_unlocked();
old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr);
tctx = prof_alloc_prep(tsd, usize, prof_active, true); tctx = prof_alloc_prep(tsd, usize, prof_active, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); p = irealloc_prof_sample(tsd, extent, old_ptr, old_usize, usize,
else tctx);
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); } else
p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false);
if (unlikely(p == NULL)) { if (unlikely(p == NULL)) {
prof_alloc_rollback(tsd, tctx, true); prof_alloc_rollback(tsd, tctx, true);
return (NULL); return (NULL);
} }
prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, e = (p == old_ptr) ? extent : iealloc(p);
old_tctx); prof_realloc(tsd, e, p, usize, tctx, prof_active, true,
old_ptr, old_usize, old_tctx);
return (p); return (p);
} }
@ -1728,6 +1734,7 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
JEMALLOC_INLINE_C void JEMALLOC_INLINE_C void
ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
{ {
extent_t *extent;
size_t usize; size_t usize;
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
@ -1735,22 +1742,24 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
assert(ptr != NULL); assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
extent = iealloc(ptr);
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
prof_free(tsd, ptr, usize); prof_free(tsd, extent, ptr, usize);
} else if (config_stats) } else if (config_stats)
usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
if (config_stats) if (config_stats)
*tsd_thread_deallocatedp_get(tsd) += usize; *tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path)) if (likely(!slow_path))
idalloctm(tsd_tsdn(tsd), ptr, tcache, false, false); idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, false);
else else
idalloctm(tsd_tsdn(tsd), ptr, tcache, false, true); idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, true);
} }
JEMALLOC_INLINE_C void JEMALLOC_INLINE_C void
isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) isfree(tsd_t *tsd, extent_t *extent, void *ptr, size_t usize, tcache_t *tcache,
bool slow_path)
{ {
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
@ -1759,14 +1768,14 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
if (config_prof && opt_prof) if (config_prof && opt_prof)
prof_free(tsd, ptr, usize); prof_free(tsd, extent, ptr, usize);
if (config_stats) if (config_stats)
*tsd_thread_deallocatedp_get(tsd) += usize; *tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path)) if (likely(!slow_path))
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, false); isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, false);
else else
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, true); isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, true);
} }
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
@ -1794,22 +1803,26 @@ je_realloc(void *ptr, size_t size)
if (likely(ptr != NULL)) { if (likely(ptr != NULL)) {
tsd_t *tsd; tsd_t *tsd;
extent_t *extent;
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); extent = iealloc(ptr);
old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = s2u(size); usize = s2u(size);
ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ? ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
NULL : irealloc_prof(tsd, ptr, old_usize, usize); NULL : irealloc_prof(tsd, extent, ptr, old_usize,
usize);
} else { } else {
if (config_stats) if (config_stats)
usize = s2u(size); usize = s2u(size);
ret = iralloc(tsd, ptr, old_usize, size, 0, false); ret = iralloc(tsd, extent, ptr, old_usize, size, 0,
false);
} }
tsdn = tsd_tsdn(tsd); tsdn = tsd_tsdn(tsd);
} else { } else {
@ -1832,7 +1845,7 @@ je_realloc(void *ptr, size_t size)
if (config_stats && likely(ret != NULL)) { if (config_stats && likely(ret != NULL)) {
tsd_t *tsd; tsd_t *tsd;
assert(usize == isalloc(tsdn, ret, config_prof)); assert(usize == isalloc(tsdn, iealloc(ret), ret, config_prof));
tsd = tsdn_tsd(tsdn); tsd = tsdn_tsd(tsdn);
*tsd_thread_allocatedp_get(tsd) += usize; *tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize; *tsd_thread_deallocatedp_get(tsd) += old_usize;
@ -1986,11 +1999,10 @@ imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
tcache, arena, slow_path); tcache, arena, slow_path);
if (p == NULL) if (p == NULL)
return (NULL); return (NULL);
arena_prof_promoted(tsdn, p, usize); arena_prof_promoted(tsdn, iealloc(p), p, usize);
} else { } else
p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena, p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena,
slow_path); slow_path);
}
return (p); return (p);
} }
@ -2021,7 +2033,7 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path)
prof_alloc_rollback(tsd, tctx, true); prof_alloc_rollback(tsd, tctx, true);
return (NULL); return (NULL);
} }
prof_malloc(tsd_tsdn(tsd), p, *usize, tctx); prof_malloc(tsd_tsdn(tsd), iealloc(p), p, *usize, tctx);
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
return (p); return (p);
@ -2109,46 +2121,47 @@ je_mallocx(size_t size, int flags)
} }
static void * static void *
irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, irallocx_prof_sample(tsdn_t *tsdn, extent_t *extent, void *old_ptr,
size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, size_t old_usize, size_t usize, size_t alignment, bool zero,
prof_tctx_t *tctx) tcache_t *tcache, arena_t *arena, prof_tctx_t *tctx)
{ {
void *p; void *p;
if (tctx == NULL) if (tctx == NULL)
return (NULL); return (NULL);
if (usize <= SMALL_MAXCLASS) { if (usize <= SMALL_MAXCLASS) {
p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS, p = iralloct(tsdn, extent, old_ptr, old_usize, LARGE_MINCLASS,
alignment, zero, tcache, arena); alignment, zero, tcache, arena);
if (p == NULL) if (p == NULL)
return (NULL); return (NULL);
arena_prof_promoted(tsdn, p, usize); arena_prof_promoted(tsdn, iealloc(p), p, usize);
} else { } else {
p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, p = iralloct(tsdn, extent, old_ptr, old_usize, usize, alignment,
tcache, arena); zero, tcache, arena);
} }
return (p); return (p);
} }
JEMALLOC_ALWAYS_INLINE_C void * JEMALLOC_ALWAYS_INLINE_C void *
irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, irallocx_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize,
size_t alignment, size_t *usize, bool zero, tcache_t *tcache, size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
arena_t *arena) arena_t *arena)
{ {
void *p; void *p;
extent_t *e;
bool prof_active; bool prof_active;
prof_tctx_t *old_tctx, *tctx; prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked(); prof_active = prof_active_get_unlocked();
old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr); old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr);
tctx = prof_alloc_prep(tsd, *usize, prof_active, true); tctx = prof_alloc_prep(tsd, *usize, prof_active, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, p = irallocx_prof_sample(tsd_tsdn(tsd), extent, old_ptr,
*usize, alignment, zero, tcache, arena, tctx); old_usize, *usize, alignment, zero, tcache, arena, tctx);
} else { } else {
p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, p = iralloct(tsd_tsdn(tsd), extent, old_ptr, old_usize, size,
zero, tcache, arena); alignment, zero, tcache, arena);
} }
if (unlikely(p == NULL)) { if (unlikely(p == NULL)) {
prof_alloc_rollback(tsd, tctx, true); prof_alloc_rollback(tsd, tctx, true);
@ -2164,9 +2177,11 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
* be the same as the current usize because of in-place large * be the same as the current usize because of in-place large
* reallocation. Therefore, query the actual value of usize. * reallocation. Therefore, query the actual value of usize.
*/ */
*usize = isalloc(tsd_tsdn(tsd), p, config_prof); e = extent;
} *usize = isalloc(tsd_tsdn(tsd), e, p, config_prof);
prof_realloc(tsd, p, *usize, tctx, prof_active, true, old_ptr, } else
e = iealloc(p);
prof_realloc(tsd, e, p, *usize, tctx, prof_active, true, old_ptr,
old_usize, old_tctx); old_usize, old_tctx);
return (p); return (p);
@ -2179,6 +2194,7 @@ je_rallocx(void *ptr, size_t size, int flags)
{ {
void *p; void *p;
tsd_t *tsd; tsd_t *tsd;
extent_t *extent;
size_t usize; size_t usize;
size_t old_usize; size_t old_usize;
size_t alignment = MALLOCX_ALIGN_GET(flags); size_t alignment = MALLOCX_ALIGN_GET(flags);
@ -2191,6 +2207,7 @@ je_rallocx(void *ptr, size_t size, int flags)
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
extent = iealloc(ptr);
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags); unsigned arena_ind = MALLOCX_ARENA_GET(flags);
@ -2208,23 +2225,25 @@ je_rallocx(void *ptr, size_t size, int flags)
} else } else
tcache = tcache_get(tsd, true); tcache = tcache_get(tsd, true);
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
goto label_oom; goto label_oom;
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment,
zero, tcache, arena); &usize, zero, tcache, arena);
if (unlikely(p == NULL)) if (unlikely(p == NULL))
goto label_oom; goto label_oom;
} else { } else {
p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, p = iralloct(tsd_tsdn(tsd), extent, ptr, old_usize, size,
zero, tcache, arena); alignment, zero, tcache, arena);
if (unlikely(p == NULL)) if (unlikely(p == NULL))
goto label_oom; goto label_oom;
if (config_stats) if (config_stats) {
usize = isalloc(tsd_tsdn(tsd), p, config_prof); usize = isalloc(tsd_tsdn(tsd), iealloc(p), p,
config_prof);
}
} }
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0)); assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
@ -2246,42 +2265,43 @@ label_oom:
} }
JEMALLOC_ALWAYS_INLINE_C size_t JEMALLOC_ALWAYS_INLINE_C size_t
ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, ixallocx_helper(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t old_usize,
size_t extra, size_t alignment, bool zero) size_t size, size_t extra, size_t alignment, bool zero)
{ {
size_t usize; size_t usize;
if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) if (ixalloc(tsdn, extent, ptr, old_usize, size, extra, alignment, zero))
return (old_usize); return (old_usize);
usize = isalloc(tsdn, ptr, config_prof); usize = isalloc(tsdn, extent, ptr, config_prof);
return (usize); return (usize);
} }
static size_t static size_t
ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, ixallocx_prof_sample(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx) size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero,
prof_tctx_t *tctx)
{ {
size_t usize; size_t usize;
if (tctx == NULL) if (tctx == NULL)
return (old_usize); return (old_usize);
usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment, usize = ixallocx_helper(tsdn, extent, ptr, old_usize, size, extra,
zero); alignment, zero);
return (usize); return (usize);
} }
JEMALLOC_ALWAYS_INLINE_C size_t JEMALLOC_ALWAYS_INLINE_C size_t
ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, ixallocx_prof(tsd_t *tsd, extent_t *extent, void *ptr, size_t old_usize,
size_t extra, size_t alignment, bool zero) size_t size, size_t extra, size_t alignment, bool zero)
{ {
size_t usize_max, usize; size_t usize_max, usize;
bool prof_active; bool prof_active;
prof_tctx_t *old_tctx, *tctx; prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked(); prof_active = prof_active_get_unlocked();
old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr); old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr);
/* /*
* usize isn't knowable before ixalloc() returns when extra is non-zero. * usize isn't knowable before ixalloc() returns when extra is non-zero.
* Therefore, compute its maximum possible value and use that in * Therefore, compute its maximum possible value and use that in
@ -2306,18 +2326,18 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize, usize = ixallocx_prof_sample(tsd_tsdn(tsd), extent, ptr,
size, extra, alignment, zero, tctx); old_usize, size, extra, alignment, zero, tctx);
} else { } else {
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize,
extra, alignment, zero); size, extra, alignment, zero);
} }
if (usize == old_usize) { if (usize == old_usize) {
prof_alloc_rollback(tsd, tctx, false); prof_alloc_rollback(tsd, tctx, false);
return (usize); return (usize);
} }
prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, prof_realloc(tsd, extent, ptr, usize, tctx, prof_active, false, ptr,
old_tctx); old_usize, old_tctx);
return (usize); return (usize);
} }
@ -2326,6 +2346,7 @@ JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_xallocx(void *ptr, size_t size, size_t extra, int flags) je_xallocx(void *ptr, size_t size, size_t extra, int flags)
{ {
tsd_t *tsd; tsd_t *tsd;
extent_t *extent;
size_t usize, old_usize; size_t usize, old_usize;
size_t alignment = MALLOCX_ALIGN_GET(flags); size_t alignment = MALLOCX_ALIGN_GET(flags);
bool zero = flags & MALLOCX_ZERO; bool zero = flags & MALLOCX_ZERO;
@ -2336,8 +2357,9 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
extent = iealloc(ptr);
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
/* /*
* The API explicitly absolves itself of protecting against (size + * The API explicitly absolves itself of protecting against (size +
@ -2356,11 +2378,11 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
extra = HUGE_MAXCLASS - size; extra = HUGE_MAXCLASS - size;
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra,
alignment, zero); alignment, zero);
} else { } else {
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size, usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize,
extra, alignment, zero); size, extra, alignment, zero);
} }
if (unlikely(usize == old_usize)) if (unlikely(usize == old_usize))
goto label_not_resized; goto label_not_resized;
@ -2390,7 +2412,7 @@ je_sallocx(const void *ptr, int flags)
if (config_ivsalloc) if (config_ivsalloc)
usize = ivsalloc(tsdn, ptr, config_prof); usize = ivsalloc(tsdn, ptr, config_prof);
else else
usize = isalloc(tsdn, ptr, config_prof); usize = isalloc(tsdn, iealloc(ptr), ptr, config_prof);
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn);
return (usize); return (usize);
@ -2442,14 +2464,16 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_sdallocx(void *ptr, size_t size, int flags) je_sdallocx(void *ptr, size_t size, int flags)
{ {
tsd_t *tsd; tsd_t *tsd;
tcache_t *tcache; extent_t *extent;
size_t usize; size_t usize;
tcache_t *tcache;
assert(ptr != NULL); assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER); assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch(); tsd = tsd_fetch();
extent = iealloc(ptr);
usize = inallocx(tsd_tsdn(tsd), size, flags); usize = inallocx(tsd_tsdn(tsd), size, flags);
assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof)); assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr, config_prof));
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
@ -2462,9 +2486,9 @@ je_sdallocx(void *ptr, size_t size, int flags)
UTRACE(ptr, 0, 0); UTRACE(ptr, 0, 0);
if (likely(!malloc_slow)) if (likely(!malloc_slow))
isfree(tsd, ptr, usize, tcache, false); isfree(tsd, extent, ptr, usize, tcache, false);
else else
isfree(tsd, ptr, usize, tcache, true); isfree(tsd, extent, ptr, usize, tcache, true);
witness_assert_lockless(tsd_tsdn(tsd)); witness_assert_lockless(tsd_tsdn(tsd));
} }
@ -2566,8 +2590,10 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
if (config_ivsalloc) if (config_ivsalloc)
ret = ivsalloc(tsdn, ptr, config_prof); ret = ivsalloc(tsdn, ptr, config_prof);
else else {
ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof); ret = (ptr == NULL) ? 0 : isalloc(tsdn, iealloc(ptr), ptr,
config_prof);
}
witness_assert_lockless(tsdn); witness_assert_lockless(tsdn);
return (ret); return (ret);

32
src/prof.c
View File

@ -223,11 +223,11 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
} }
void void
prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx) size_t usize, prof_tctx_t *tctx)
{ {
prof_tctx_set(tsdn, ptr, usize, tctx); prof_tctx_set(tsdn, extent, ptr, usize, tctx);
malloc_mutex_lock(tsdn, tctx->tdata->lock); malloc_mutex_lock(tsdn, tctx->tdata->lock);
tctx->cnts.curobjs++; tctx->cnts.curobjs++;
@ -596,7 +596,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
prof_leave(tsd, tdata_self); prof_leave(tsd, tdata_self);
/* Destroy gctx. */ /* Destroy gctx. */
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock); malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
idalloctm(tsd_tsdn(tsd), gctx, NULL, true, true); idalloctm(tsd_tsdn(tsd), iealloc(gctx), gctx, NULL, true, true);
} else { } else {
/* /*
* Compensate for increment in prof_tctx_destroy() or * Compensate for increment in prof_tctx_destroy() or
@ -707,7 +707,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
prof_tdata_destroy(tsd_tsdn(tsd), tdata, false); prof_tdata_destroy(tsd_tsdn(tsd), tdata, false);
if (destroy_tctx) if (destroy_tctx)
idalloctm(tsd_tsdn(tsd), tctx, NULL, true, true); idalloctm(tsd_tsdn(tsd), iealloc(tctx), tctx, NULL, true, true);
} }
static bool static bool
@ -736,7 +736,8 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
if (ckh_insert(tsd_tsdn(tsd), &bt2gctx, btkey.v, gctx.v)) { if (ckh_insert(tsd_tsdn(tsd), &bt2gctx, btkey.v, gctx.v)) {
/* OOM. */ /* OOM. */
prof_leave(tsd, tdata); prof_leave(tsd, tdata);
idalloctm(tsd_tsdn(tsd), gctx.v, NULL, true, true); idalloctm(tsd_tsdn(tsd), iealloc(gctx.v), gctx.v, NULL,
true, true);
return (true); return (true);
} }
new_gctx = true; new_gctx = true;
@ -816,7 +817,8 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt)
if (error) { if (error) {
if (new_gctx) if (new_gctx)
prof_gctx_try_destroy(tsd, tdata, gctx, tdata); prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
idalloctm(tsd_tsdn(tsd), ret.v, NULL, true, true); idalloctm(tsd_tsdn(tsd), iealloc(ret.v), ret.v, NULL,
true, true);
return (NULL); return (NULL);
} }
malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock); malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
@ -1238,7 +1240,8 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
to_destroy); to_destroy);
tctx_tree_remove(&gctx->tctxs, tctx_tree_remove(&gctx->tctxs,
to_destroy); to_destroy);
idalloctm(tsd_tsdn(tsd), to_destroy, idalloctm(tsd_tsdn(tsd),
iealloc(to_destroy), to_destroy,
NULL, true, true); NULL, true, true);
} else } else
next = NULL; next = NULL;
@ -1815,7 +1818,7 @@ prof_tdata_init_impl(tsdn_t *tsdn, uint64_t thr_uid, uint64_t thr_discrim,
if (ckh_new(tsdn, &tdata->bt2tctx, PROF_CKH_MINITEMS, if (ckh_new(tsdn, &tdata->bt2tctx, PROF_CKH_MINITEMS,
prof_bt_hash, prof_bt_keycomp)) { prof_bt_hash, prof_bt_keycomp)) {
idalloctm(tsdn, tdata, NULL, true, true); idalloctm(tsdn, iealloc(tdata), tdata, NULL, true, true);
return (NULL); return (NULL);
} }
@ -1878,10 +1881,12 @@ prof_tdata_destroy_locked(tsdn_t *tsdn, prof_tdata_t *tdata,
assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached)); assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
if (tdata->thread_name != NULL) if (tdata->thread_name != NULL) {
idalloctm(tsdn, tdata->thread_name, NULL, true, true); idalloctm(tsdn, iealloc(tdata->thread_name), tdata->thread_name,
NULL, true, true);
}
ckh_delete(tsdn, &tdata->bt2tctx); ckh_delete(tsdn, &tdata->bt2tctx);
idalloctm(tsdn, tdata, NULL, true, true); idalloctm(tsdn, iealloc(tdata), tdata, NULL, true, true);
} }
static void static void
@ -2075,7 +2080,8 @@ prof_thread_name_set(tsd_t *tsd, const char *thread_name)
return (EAGAIN); return (EAGAIN);
if (tdata->thread_name != NULL) { if (tdata->thread_name != NULL) {
idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, true, true); idalloctm(tsd_tsdn(tsd), iealloc(tdata->thread_name),
tdata->thread_name, NULL, true, true);
tdata->thread_name = NULL; tdata->thread_name = NULL;
} }
if (strlen(s) > 0) if (strlen(s) > 0)

31
src/tcache.c
View File

@ -27,7 +27,7 @@ size_t
tcache_salloc(tsdn_t *tsdn, const void *ptr) tcache_salloc(tsdn_t *tsdn, const void *ptr)
{ {
return (arena_salloc(tsdn, ptr, false)); return (arena_salloc(tsdn, iealloc(ptr), ptr, false));
} }
void void
@ -101,9 +101,8 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
assert(arena != NULL); assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena bin associated with the first object. */ /* Lock the arena bin associated with the first object. */
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( extent_t *extent = iealloc(*(tbin->avail - 1));
*(tbin->avail - 1)); arena_t *bin_arena = extent_arena_get(extent);
arena_t *bin_arena = extent_arena_get(&chunk->extent);
arena_bin_t *bin = &bin_arena->bins[binind]; arena_bin_t *bin = &bin_arena->bins[binind];
if (config_prof && bin_arena == arena) { if (config_prof && bin_arena == arena) {
@ -125,14 +124,17 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
for (i = 0; i < nflush; i++) { for (i = 0; i < nflush; i++) {
ptr = *(tbin->avail - 1 - i); ptr = *(tbin->avail - 1 - i);
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (extent_arena_get(&chunk->extent) == bin_arena) { extent = iealloc(ptr);
if (extent_arena_get(extent) == bin_arena) {
arena_chunk_t *chunk =
(arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr - size_t pageind = ((uintptr_t)ptr -
(uintptr_t)chunk) >> LG_PAGE; (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_bits_t *bitselm = arena_chunk_map_bits_t *bitselm =
arena_bitselm_get_mutable(chunk, pageind); arena_bitselm_get_mutable(chunk, pageind);
arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
bin_arena, chunk, ptr, bitselm); bin_arena, chunk, extent, ptr, bitselm);
} else { } else {
/* /*
* This object was allocated via a different * This object was allocated via a different
@ -183,9 +185,8 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
assert(arena != NULL); assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena associated with the first object. */ /* Lock the arena associated with the first object. */
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( extent_t *extent = iealloc(*(tbin->avail - 1));
*(tbin->avail - 1)); arena_t *locked_arena = extent_arena_get(extent);
arena_t *locked_arena = extent_arena_get(&chunk->extent);
UNUSED bool idump; UNUSED bool idump;
if (config_prof) if (config_prof)
@ -210,10 +211,12 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
for (i = 0; i < nflush; i++) { for (i = 0; i < nflush; i++) {
ptr = *(tbin->avail - 1 - i); ptr = *(tbin->avail - 1 - i);
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); extent = iealloc(ptr);
if (extent_arena_get(&chunk->extent) == locked_arena) { if (extent_arena_get(extent) == locked_arena) {
arena_chunk_t *chunk =
(arena_chunk_t *)extent_addr_get(extent);
arena_dalloc_large_junked_locked(tsd_tsdn(tsd), arena_dalloc_large_junked_locked(tsd_tsdn(tsd),
locked_arena, chunk, ptr); locked_arena, chunk, extent, ptr);
} else { } else {
/* /*
* This object was allocated via a different * This object was allocated via a different
@ -391,7 +394,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes)) arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes))
prof_idump(tsd_tsdn(tsd)); prof_idump(tsd_tsdn(tsd));
idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true); idalloctm(tsd_tsdn(tsd), iealloc(tcache), tcache, NULL, true, true);
} }
void void