project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use rtree-based chunk lookups rather than pointer bit twiddling.

Browse Source 
Look up chunk metadata via the radix tree, rather than using
CHUNK_ADDR2BASE().

Propagate pointer's containing extent.

Minimize extent lookups by doing a single lookup (e.g. in free()) and
propagating the pointer's extent into nearly all the functions that may
need it.
This commit is contained in:
Jason Evans
2016-03-23 20:29:33 -07:00
parent 2d2b4e98c9
commit db72272bef
14 changed files with 548 additions and 504 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
include/jemalloc/internal/arena.h
View File

@@ -526,13 +526,13 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, tcache_t *tcache);
void arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size);
void arena_prof_promoted(tsdn_t *tsdn, const extent_t *extent,
const void *ptr, size_t size);
void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm);
void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm);
arena_chunk_t *chunk, extent_t *extent, void *ptr,
arena_chunk_map_bits_t *bitselm);
void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr, size_t pageind);
extent_t *extent, void *ptr, size_t pageind);
#ifdef JEMALLOC_JET
typedef void (arena_dalloc_junk_large_t)(void *, size_t);
extern arena_dalloc_junk_large_t *arena_dalloc_junk_large;
@@ -540,17 +540,17 @@ extern arena_dalloc_junk_large_t *arena_dalloc_junk_large;
void arena_dalloc_junk_large(void *ptr, size_t usize);
#endif
void arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk, void *ptr);
arena_chunk_t *chunk, extent_t *extent, void *ptr);
void arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void *ptr);
extent_t *extent, void *ptr);
#ifdef JEMALLOC_JET
typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t);
extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
#endif
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize,
size_t size, size_t extra, bool zero);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, tcache_t *tcache);
bool arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t size, size_t extra, bool zero);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache);
dss_prec_t arena_dss_prec_get(tsdn_t *tsdn, arena_t *arena);
bool arena_dss_prec_set(tsdn_t *tsdn, arena_t *arena, dss_prec_t dss_prec);
ssize_t arena_lg_dirty_mult_default_get(void);
@@ -637,20 +637,23 @@ szind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
size_t arena_run_regind(arena_run_t *run, const arena_bin_info_t *bin_info,
const void *ptr);
prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr);
void arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx);
void arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
const void *old_ptr, prof_tctx_t *old_tctx);
prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
const void *ptr);
void arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx);
void arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *old_tctx);
void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks);
void arena_decay_tick(tsdn_t *tsdn, arena_t *arena);
void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, tcache_t *tcache, bool slow_path);
arena_t *arena_aalloc(const void *ptr);
size_t arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote);
void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
bool slow_path);
size_t arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr,
bool demote);
void arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr,
tcache_t *tcache, bool slow_path);
void arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
tcache_t *tcache, bool slow_path);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
@@ -1042,7 +1045,9 @@ arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes)
return (ret);
}
}
# endif /* JEMALLOC_ARENA_INLINE_A */
# ifdef JEMALLOC_ARENA_INLINE_B
JEMALLOC_ALWAYS_INLINE szind_t
arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
{
@@ -1051,6 +1056,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
if (config_debug) {
const extent_t *extent;
arena_chunk_t *chunk;
arena_t *arena;
size_t pageind;
@@ -1065,8 +1071,9 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
assert(binind != BININD_INVALID);
assert(binind < NBINS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
arena = extent_arena_get(&chunk->extent);
extent = iealloc(ptr);
chunk = (arena_chunk_t *)extent_addr_get(extent);
arena = extent_arena_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
actual_mapbits = arena_mapbits_get(chunk, pageind);
assert(mapbits == actual_mapbits);
@@ -1088,9 +1095,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
return (binind);
}
# endif /* JEMALLOC_ARENA_INLINE_A */
# ifdef JEMALLOC_ARENA_INLINE_B
JEMALLOC_INLINE szind_t
arena_bin_index(arena_t *arena, arena_bin_t *bin)
{
@@ -1172,16 +1177,15 @@ arena_run_regind(arena_run_t *run, const arena_bin_info_t *bin_info,
}
JEMALLOC_INLINE prof_tctx_t *
arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{
prof_tctx_t *ret;
arena_chunk_t *chunk;
cassert(config_prof);
assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr)) {
if (likely(extent_achunk_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
@@ -1193,22 +1197,21 @@ arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
ret = atomic_read_p(&elm->prof_tctx_pun);
}
} else
ret = huge_prof_tctx_get(tsdn, ptr);
ret = huge_prof_tctx_get(tsdn, extent, ptr);
return (ret);
}
JEMALLOC_INLINE void
arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx)
arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx)
{
arena_chunk_t *chunk;
cassert(config_prof);
assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr)) {
if (likely(extent_achunk_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
@@ -1231,12 +1234,12 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
assert(arena_mapbits_large_get(chunk, pageind) == 0);
}
} else
huge_prof_tctx_set(tsdn, ptr, tctx);
huge_prof_tctx_set(tsdn, extent, ptr, tctx);
}
JEMALLOC_INLINE void
arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
const void *old_ptr, prof_tctx_t *old_tctx)
arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *old_tctx)
{
cassert(config_prof);
@@ -1244,7 +1247,7 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr &&
(uintptr_t)old_tctx > (uintptr_t)1U))) {
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
if (likely(chunk != ptr)) {
size_t pageind;
arena_chunk_map_misc_t *elm;
@@ -1259,7 +1262,7 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
atomic_write_p(&elm->prof_tctx_pun,
(prof_tctx_t *)(uintptr_t)1U);
} else
huge_prof_tctx_reset(tsdn, ptr);
huge_prof_tctx_reset(tsdn, extent, ptr);
}
}
@@ -1313,28 +1316,24 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(const void *ptr)
{
arena_chunk_t *chunk;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr))
return (extent_arena_get(&chunk->extent));
else
return (huge_aalloc(ptr));
return (extent_arena_get(iealloc(ptr)));
}
/* Return the size of the allocation pointed to by ptr. */
JEMALLOC_ALWAYS_INLINE size_t
arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote)
arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr, bool demote)
{
size_t ret;
arena_chunk_t *chunk;
size_t pageind;
szind_t binind;
assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr)) {
if (likely(extent_achunk_get(extent))) {
const arena_chunk_t *chunk =
(const arena_chunk_t *)extent_addr_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
binind = arena_mapbits_binind_get(chunk, pageind);
@@ -1367,22 +1366,23 @@ arena_salloc(tsdn_t *tsdn, const void *ptr, bool demote)
ret = index2size(binind);
}
} else
ret = huge_salloc(tsdn, ptr);
ret = huge_salloc(tsdn, extent, ptr);
return (ret);
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path)
arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
bool slow_path)
{
arena_chunk_t *chunk;
size_t pageind, mapbits;
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr)) {
if (likely(extent_achunk_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapbits = arena_mapbits_get(chunk, pageind);
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
@@ -1395,7 +1395,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path)
binind, slow_path);
} else {
arena_dalloc_small(tsdn,
extent_arena_get(&chunk->extent), chunk,
extent_arena_get(extent), chunk, extent,
ptr, pageind);
}
} else {
@@ -1411,24 +1411,24 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path)
size - large_pad, slow_path);
} else {
arena_dalloc_large(tsdn,
extent_arena_get(&chunk->extent), chunk,
extent_arena_get(extent), chunk, extent,
ptr);
}
}
} else
huge_dalloc(tsdn, ptr);
huge_dalloc(tsdn, extent, ptr);
}
JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
bool slow_path)
arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
tcache_t *tcache, bool slow_path)
{
arena_chunk_t *chunk;
assert(!tsdn_null(tsdn) || tcache == NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (likely(chunk != ptr)) {
if (likely(extent_achunk_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
if (config_prof && opt_prof) {
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
LG_PAGE;
@@ -1443,7 +1443,8 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
pageind) - large_pad;
}
}
assert(s2u(size) == s2u(arena_salloc(tsdn, ptr, false)));
assert(s2u(size) == s2u(arena_salloc(tsdn, extent, ptr,
false)));
if (likely(size <= SMALL_MAXCLASS)) {
/* Small allocation. */
@@ -1455,7 +1456,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
size_t pageind = ((uintptr_t)ptr -
(uintptr_t)chunk) >> LG_PAGE;
arena_dalloc_small(tsdn,
extent_arena_get(&chunk->extent), chunk,
extent_arena_get(extent), chunk, extent,
ptr, pageind);
}
} else {
@@ -1467,12 +1468,12 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
size, slow_path);
} else {
arena_dalloc_large(tsdn,
extent_arena_get(&chunk->extent), chunk,
extent_arena_get(extent), chunk, extent,
ptr);
}
}
} else
huge_dalloc(tsdn, ptr);
huge_dalloc(tsdn, extent, ptr);
}
# endif /* JEMALLOC_ARENA_INLINE_B */
#endif

2
include/jemalloc/internal/chunk.h
View File

@@ -54,6 +54,8 @@ chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena,
bool chunk_register(tsdn_t *tsdn, const void *chunk, const extent_t *extent);
void chunk_deregister(const void *chunk, const extent_t *extent);
void chunk_reregister(tsdn_t *tsdn, const void *chunk,
const extent_t *extent);
void *chunk_alloc_base(size_t size);
void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,

22
include/jemalloc/internal/huge.h
View File

@@ -12,20 +12,22 @@
void *huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero);
bool huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize,
size_t usize_min, size_t usize_max, bool zero);
void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
bool huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t usize_min, size_t usize_max, bool zero);
void *huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t oldsize, size_t usize, size_t alignment, bool zero,
tcache_t *tcache);
#ifdef JEMALLOC_JET
typedef void (huge_dalloc_junk_t)(tsdn_t *, void *, size_t);
extern huge_dalloc_junk_t *huge_dalloc_junk;
#endif
void huge_dalloc(tsdn_t *tsdn, void *ptr);
arena_t *huge_aalloc(const void *ptr);
size_t huge_salloc(tsdn_t *tsdn, const void *ptr);
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr);
void huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx);
void huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr);
void huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr);
size_t huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr);
prof_tctx_t *huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
const void *ptr);
void huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
prof_tctx_t *tctx);
void huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/

124
include/jemalloc/internal/jemalloc_internal.h.in
View File

@@ -959,6 +959,20 @@ decay_ticker_get(tsd_t *tsd, unsigned ind)
#define JEMALLOC_ARENA_INLINE_A
#include "jemalloc/internal/arena.h"
#undef JEMALLOC_ARENA_INLINE_A
#ifndef JEMALLOC_ENABLE_INLINE
extent_t *iealloc(const void *ptr);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE extent_t *
iealloc(const void *ptr)
{
return (chunk_lookup(ptr, true));
}
#endif
#include "jemalloc/internal/tcache.h"
#define JEMALLOC_ARENA_INLINE_B
#include "jemalloc/internal/arena.h"
@@ -968,7 +982,8 @@ decay_ticker_get(tsd_t *tsd, unsigned ind)
#ifndef JEMALLOC_ENABLE_INLINE
extent_t *iealloc(const void *ptr);
arena_t *iaalloc(const void *ptr);
size_t isalloc(tsdn_t *tsdn, const void *ptr, bool demote);
size_t isalloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr,
bool demote);
void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
tcache_t *tcache, bool is_metadata, arena_t *arena, bool slow_path);
void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero,
@@ -979,30 +994,23 @@ void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
tcache_t *tcache, arena_t *arena);
void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
size_t ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote);
void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata,
bool slow_path);
void idalloc(tsd_t *tsd, void *ptr);
void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
bool slow_path);
void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, tcache_t *tcache,
arena_t *arena);
void *iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero);
bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero);
void idalloctm(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
bool is_metadata, bool slow_path);
void idalloc(tsd_t *tsd, extent_t *extent, void *ptr);
void isdalloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
tcache_t *tcache, bool slow_path);
void *iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr,
size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero,
tcache_t *tcache, arena_t *arena);
void *iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
void *iralloc(tsd_t *tsd, extent_t *extent, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero);
bool ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t size, size_t extra, size_t alignment, bool zero);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE extent_t *
iealloc(const void *ptr)
{
return (chunk_lookup(ptr, true));
}
JEMALLOC_ALWAYS_INLINE arena_t *
iaalloc(const void *ptr)
{
@@ -1016,17 +1024,18 @@ iaalloc(const void *ptr)
* Typical usage:
* tsdn_t *tsdn = [...]
* void *ptr = [...]
* size_t sz = isalloc(tsdn, ptr, config_prof);
* extent_t *extent = iealloc(ptr);
* size_t sz = isalloc(tsdn, extent, ptr, config_prof);
*/
JEMALLOC_ALWAYS_INLINE size_t
isalloc(tsdn_t *tsdn, const void *ptr, bool demote)
isalloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr, bool demote)
{
assert(ptr != NULL);
/* Demotion only makes sense if config_prof is true. */
assert(config_prof || !demote);
return (arena_salloc(tsdn, ptr, demote));
return (arena_salloc(tsdn, extent, ptr, demote));
}
JEMALLOC_ALWAYS_INLINE void *
@@ -1041,8 +1050,8 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path);
if (config_stats && is_metadata && likely(ret != NULL)) {
arena_metadata_allocated_add(iaalloc(ret),
isalloc(tsdn, ret, config_prof));
arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn,
iealloc(ret), ret, config_prof));
}
return (ret);
}
@@ -1069,8 +1078,8 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache);
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
if (config_stats && is_metadata && likely(ret != NULL)) {
arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn, ret,
config_prof));
arena_metadata_allocated_add(iaalloc(ret), isalloc(tsdn,
iealloc(ret), ret, config_prof));
}
return (ret);
}
@@ -1104,43 +1113,45 @@ ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote)
assert(extent_addr_get(extent) == ptr ||
extent_achunk_get(extent));
return (isalloc(tsdn, ptr, demote));
return (isalloc(tsdn, extent, ptr, demote));
}
JEMALLOC_ALWAYS_INLINE void
idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_metadata,
bool slow_path)
idalloctm(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
bool is_metadata, bool slow_path)
{
assert(ptr != NULL);
assert(!is_metadata || tcache == NULL);
assert(!is_metadata || iaalloc(ptr)->ind < narenas_auto);
if (config_stats && is_metadata) {
arena_metadata_allocated_sub(iaalloc(ptr), isalloc(tsdn, ptr,
config_prof));
arena_metadata_allocated_sub(iaalloc(ptr), isalloc(tsdn, extent,
ptr, config_prof));
}
arena_dalloc(tsdn, ptr, tcache, slow_path);
arena_dalloc(tsdn, extent, ptr, tcache, slow_path);
}
JEMALLOC_ALWAYS_INLINE void
idalloc(tsd_t *tsd, void *ptr)
idalloc(tsd_t *tsd, extent_t *extent, void *ptr)
{
idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd, false), false, true);
idalloctm(tsd_tsdn(tsd), extent, ptr, tcache_get(tsd, false), false,
true);
}
JEMALLOC_ALWAYS_INLINE void
isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
bool slow_path)
isdalloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
tcache_t *tcache, bool slow_path)
{
arena_sdalloc(tsdn, ptr, size, tcache, slow_path);
arena_sdalloc(tsdn, extent, ptr, size, tcache, slow_path);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena)
iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache,
arena_t *arena)
{
void *p;
size_t usize, copysize;
@@ -1166,13 +1177,13 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize);
isdalloct(tsdn, ptr, oldsize, tcache, true);
isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
return (p);
}
JEMALLOC_ALWAYS_INLINE void *
iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
bool zero, tcache_t *tcache, arena_t *arena)
iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero, tcache_t *tcache, arena_t *arena)
{
assert(ptr != NULL);
@@ -1184,26 +1195,26 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
* Existing object alignment is inadequate; allocate new space
* and copy.
*/
return (iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment,
zero, tcache, arena));
return (iralloct_realign(tsdn, extent, ptr, oldsize, size, 0,
alignment, zero, tcache, arena));
}
return (arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
tcache));
return (arena_ralloc(tsdn, arena, extent, ptr, oldsize, size, alignment,
zero, tcache));
}
JEMALLOC_ALWAYS_INLINE void *
iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
bool zero)
iralloc(tsd_t *tsd, extent_t *extent, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero)
{
return (iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero,
tcache_get(tsd, true), NULL));
return (iralloct(tsd_tsdn(tsd), extent, ptr, oldsize, size, alignment,
zero, tcache_get(tsd, true), NULL));
}
JEMALLOC_ALWAYS_INLINE bool
ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero)
ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
size_t extra, size_t alignment, bool zero)
{
assert(ptr != NULL);
@@ -1215,7 +1226,8 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
return (true);
}
return (arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero));
return (arena_ralloc_no_move(tsdn, extent, ptr, oldsize, size, extra,
zero));
}
#endif

2
include/jemalloc/internal/private_symbols.txt
View File

@@ -181,6 +181,7 @@ chunk_postfork_parent
chunk_prefork
chunk_purge_wrapper
chunk_register
chunk_reregister
chunks_rtree
chunksize
chunksize_mask
@@ -277,7 +278,6 @@ hash_rotl_64
hash_x64_128
hash_x86_128
hash_x86_32
huge_aalloc
huge_dalloc
huge_dalloc_junk
huge_malloc

81
include/jemalloc/internal/prof.h
View File

@@ -281,8 +281,8 @@ extern uint64_t prof_interval;
extern size_t lg_prof_sample;
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent,
const void *ptr, size_t usize, prof_tctx_t *tctx);
void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
void bt_init(prof_bt_t *bt, void **vec);
void prof_backtrace(prof_bt_t *bt);
@@ -330,21 +330,23 @@ void prof_sample_threshold_update(prof_tdata_t *tdata);
bool prof_active_get_unlocked(void);
bool prof_gdump_get_unlocked(void);
prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create);
prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const void *ptr);
void prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize,
const void *old_ptr, prof_tctx_t *tctx);
prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
const void *ptr);
void prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx);
void prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, const void *old_ptr, prof_tctx_t *tctx);
bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
prof_tdata_t **tdata_out);
prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
bool update);
void prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(tsd_t *tsd, const void *ptr, size_t usize);
void prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx);
void prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr,
size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated,
const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr,
size_t usize);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
@@ -398,34 +400,35 @@ prof_tdata_get(tsd_t *tsd, bool create)
}
JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_tctx_get(tsdn_t *tsdn, const void *ptr)
prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{
cassert(config_prof);
assert(ptr != NULL);
return (arena_prof_tctx_get(tsdn, ptr));
return (arena_prof_tctx_get(tsdn, extent, ptr));
}
JEMALLOC_ALWAYS_INLINE void
prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx)
prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
prof_tctx_t *tctx)
{
cassert(config_prof);
assert(ptr != NULL);
arena_prof_tctx_set(tsdn, ptr, usize, tctx);
arena_prof_tctx_set(tsdn, extent, ptr, usize, tctx);
}
JEMALLOC_ALWAYS_INLINE void
prof_tctx_reset(tsdn_t *tsdn, const void *ptr, size_t usize, const void *old_ptr,
prof_tctx_t *old_tctx)
prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
const void *old_ptr, prof_tctx_t *old_tctx)
{
cassert(config_prof);
assert(ptr != NULL);
arena_prof_tctx_reset(tsdn, ptr, usize, old_ptr, old_tctx);
arena_prof_tctx_reset(tsdn, extent, ptr, usize, old_ptr, old_tctx);
}
JEMALLOC_ALWAYS_INLINE bool
@@ -480,23 +483,26 @@ prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update)
}
JEMALLOC_ALWAYS_INLINE void
prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx)
prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
prof_tctx_t *tctx)
{
cassert(config_prof);
assert(ptr != NULL);
assert(usize == isalloc(tsdn, ptr, true));
assert(usize == isalloc(tsdn, extent, ptr, true));
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_malloc_sample_object(tsdn, ptr, usize, tctx);
else
prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U);
prof_malloc_sample_object(tsdn, extent, ptr, usize, tctx);
else {
prof_tctx_set(tsdn, extent, ptr, usize,
(prof_tctx_t *)(uintptr_t)1U);
}
}
JEMALLOC_ALWAYS_INLINE void
prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
bool prof_active, bool updated, const void *old_ptr, size_t old_usize,
prof_tctx_t *old_tctx)
prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
size_t old_usize, prof_tctx_t *old_tctx)
{
bool sampled, old_sampled;
@@ -504,7 +510,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
if (prof_active && !updated && ptr != NULL) {
assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));
assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr, true));
if (prof_sample_accum_update(tsd, usize, true, NULL)) {
/*
* Don't sample. The usize passed to prof_alloc_prep()
@@ -520,22 +526,25 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
sampled = ((uintptr_t)tctx > (uintptr_t)1U);
old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
if (unlikely(sampled))
prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx);
else
prof_tctx_reset(tsd_tsdn(tsd), ptr, usize, old_ptr, old_tctx);
if (unlikely(sampled)) {
prof_malloc_sample_object(tsd_tsdn(tsd), extent, ptr, usize,
tctx);
} else {
prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, usize, old_ptr,
old_tctx);
}
if (unlikely(old_sampled))
prof_free_sampled_object(tsd, old_usize, old_tctx);
}
JEMALLOC_ALWAYS_INLINE void
prof_free(tsd_t *tsd, const void *ptr, size_t usize)
prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr, size_t usize)
{
prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr);
cassert(config_prof);
assert(usize == isalloc(tsd_tsdn(tsd), ptr, true));
assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr, true));
if (unlikely((uintptr_t)tctx > (uintptr_t)1U))
prof_free_sampled_object(tsd, usize, tctx);

2
include/jemalloc/internal/tcache.h
View File

@@ -371,7 +371,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
if (config_prof && usize == LARGE_MINCLASS) {
arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(ret);
(arena_chunk_t *)extent_addr_get(iealloc(ret));
size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
LG_PAGE);
arena_mapbits_large_binind_set(chunk, pageind,