project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Further optimize and harden arena_salloc().

Browse Source 
Further optimize arena_salloc() to only look at the binind chunk map
bits in the common case.

Add more sanity checks to arena_salloc() that detect chunk map
inconsistencies for large allocations (whether due to allocator bugs or
application bugs).
This commit is contained in:
Jason Evans 2012-05-02 16:11:03 -07:00
parent 7bfecf412d
commit 80737c3323
3 changed files with 73 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
include/jemalloc/internal/arena.h
View File

@ -447,6 +447,7 @@ size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk,
size_t pageind); size_t pageind);
size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind);
@ -463,7 +464,7 @@ void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
size_t runind, size_t binind, size_t flags); size_t runind, size_t binind, size_t flags);
void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
size_t unzeroed); size_t unzeroed);
size_t arena_ptr_binind(const void *ptr, size_t mapbits); size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
const void *ptr); const void *ptr);
@ -533,6 +534,18 @@ arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind)
return (mapbits >> LG_PAGE); return (mapbits >> LG_PAGE);
} }
JEMALLOC_INLINE size_t
arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
size_t binind;
mapbits = arena_mapbits_get(chunk, pageind);
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
assert(binind < NBINS || binind == BININD_INVALID);
return (binind);
}
JEMALLOC_INLINE size_t JEMALLOC_INLINE size_t
arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind)
{ {
@ -644,25 +657,37 @@ arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
} }
JEMALLOC_INLINE size_t JEMALLOC_INLINE size_t
arena_ptr_binind(const void *ptr, size_t mapbits) arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
{ {
size_t binind; size_t binind;
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
if (config_debug) { if (config_debug) {
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena_chunk_t *chunk;
arena_t *arena = chunk->arena; arena_t *arena;
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind;
size_t actual_mapbits = arena_mapbits_get(chunk, pageind); size_t actual_mapbits;
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + arena_run_t *run;
(uintptr_t)((pageind - (actual_mapbits >> LG_PAGE)) << arena_bin_t *bin;
LG_PAGE)); arena_bin_t *bin = run->bin; size_t actual_binind;
size_t actual_binind = bin - arena->bins; arena_bin_info_t *bin_info;
arena_bin_info_t *bin_info = &arena_bin_info[actual_binind];
assert(binind != BININD_INVALID);
assert(binind < NBINS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
arena = chunk->arena;
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
actual_mapbits = arena_mapbits_get(chunk, pageind);
assert(mapbits == actual_mapbits); assert(mapbits == actual_mapbits);
assert(arena_mapbits_large_get(chunk, pageind) == 0);
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
(actual_mapbits >> LG_PAGE)) << LG_PAGE));
bin = run->bin;
actual_binind = bin - arena->bins;
assert(binind == actual_binind); assert(binind == actual_binind);
bin_info = &arena_bin_info[actual_binind];
assert(((uintptr_t)ptr - ((uintptr_t)run + assert(((uintptr_t)ptr - ((uintptr_t)run +
(uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval
== 0); == 0);
@ -775,7 +800,8 @@ arena_prof_ctx_get(const void *ptr)
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
(uintptr_t)((pageind - (mapbits >> LG_PAGE)) << (uintptr_t)((pageind - (mapbits >> LG_PAGE)) <<
LG_PAGE)); LG_PAGE));
size_t binind = arena_ptr_binind(ptr, mapbits); size_t binind = arena_ptr_small_binind_get(ptr,
mapbits);
arena_bin_info_t *bin_info = &arena_bin_info[binind]; arena_bin_info_t *bin_info = &arena_bin_info[binind];
unsigned regind; unsigned regind;
@ -813,7 +839,7 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
arena_bin_info_t *bin_info; arena_bin_info_t *bin_info;
unsigned regind; unsigned regind;
binind = arena_ptr_binind(ptr, mapbits); binind = arena_ptr_small_binind_get(ptr, mapbits);
bin_info = &arena_bin_info[binind]; bin_info = &arena_bin_info[binind];
regind = arena_run_regind(run, bin_info, ptr); regind = arena_run_regind(run, bin_info, ptr);
@ -861,7 +887,7 @@ arena_salloc(const void *ptr, bool demote)
{ {
size_t ret; size_t ret;
arena_chunk_t *chunk; arena_chunk_t *chunk;
size_t pageind, mapbits; size_t pageind, binind;
assert(ptr != NULL); assert(ptr != NULL);
assert(CHUNK_ADDR2BASE(ptr) != ptr); assert(CHUNK_ADDR2BASE(ptr) != ptr);
@ -869,25 +895,34 @@ arena_salloc(const void *ptr, bool demote)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0); assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
mapbits = arena_mapbits_get(chunk, pageind); binind = arena_mapbits_binind_get(chunk, pageind);
if ((mapbits & CHUNK_MAP_LARGE) == 0) { if (binind == BININD_INVALID || (config_prof && demote == false &&
size_t binind = arena_ptr_binind(ptr, mapbits); prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) {
ret = arena_bin_info[binind].reg_size; /*
} else { * Large allocation. In the common case (demote == true), and
* as this is an inline function, most callers will only end up
* looking at binind to determine that ptr is a small
* allocation.
*/
assert(((uintptr_t)ptr & PAGE_MASK) == 0); assert(((uintptr_t)ptr & PAGE_MASK) == 0);
ret = arena_mapbits_large_size_get(chunk, pageind); ret = arena_mapbits_large_size_get(chunk, pageind);
if (config_prof && demote && prof_promote && ret == PAGE &&
(mapbits & CHUNK_MAP_BININD_MASK) !=
CHUNK_MAP_BININD_MASK) {
size_t binind = ((mapbits & CHUNK_MAP_BININD_MASK) >>
CHUNK_MAP_BININD_SHIFT);
assert(binind < NBINS);
ret = arena_bin_info[binind].reg_size;
} else {
assert(demote == false || (mapbits &
CHUNK_MAP_BININD_MASK) == CHUNK_MAP_BININD_MASK);
}
assert(ret != 0); assert(ret != 0);
assert(pageind + (ret>>LG_PAGE) <= chunk_npages);
assert(ret == PAGE || arena_mapbits_large_size_get(chunk,
pageind+(ret>>LG_PAGE)-1) == 0);
assert(binind == arena_mapbits_binind_get(chunk,
pageind+(ret>>LG_PAGE)-1));
assert(arena_mapbits_dirty_get(chunk, pageind) ==
arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1));
} else {
/*
* Small allocation (possibly promoted to a large object due to
* prof_promote).
*/
assert(arena_mapbits_large_get(chunk, pageind) != 0 ||
arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
pageind)) == binind);
ret = arena_bin_info[binind].reg_size;
} }
return (ret); return (ret);
@ -912,8 +947,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
if (try_tcache && (tcache = tcache_get(false)) != NULL) { if (try_tcache && (tcache = tcache_get(false)) != NULL) {
size_t binind; size_t binind;
binind = arena_ptr_binind(ptr, mapbits); binind = arena_ptr_small_binind_get(ptr, mapbits);
assert(binind < NBINS);
tcache_dalloc_small(tcache, ptr, binind); tcache_dalloc_small(tcache, ptr, binind);
} else } else
arena_dalloc_small(arena, chunk, ptr, pageind); arena_dalloc_small(arena, chunk, ptr, pageind);

3
include/jemalloc/internal/private_namespace.h
View File

@ -16,6 +16,7 @@
#define arena_malloc_large JEMALLOC_N(arena_malloc_large) #define arena_malloc_large JEMALLOC_N(arena_malloc_large)
#define arena_malloc_small JEMALLOC_N(arena_malloc_small) #define arena_malloc_small JEMALLOC_N(arena_malloc_small)
#define arena_mapbits_allocated_get JEMALLOC_N(arena_mapbits_allocated_get) #define arena_mapbits_allocated_get JEMALLOC_N(arena_mapbits_allocated_get)
#define arena_mapbits_binind_get JEMALLOC_N(arena_mapbits_binind_get)
#define arena_mapbits_dirty_get JEMALLOC_N(arena_mapbits_dirty_get) #define arena_mapbits_dirty_get JEMALLOC_N(arena_mapbits_dirty_get)
#define arena_mapbits_get JEMALLOC_N(arena_mapbits_get) #define arena_mapbits_get JEMALLOC_N(arena_mapbits_get)
#define arena_mapbits_large_binind_set JEMALLOC_N(arena_mapbits_large_binind_set) #define arena_mapbits_large_binind_set JEMALLOC_N(arena_mapbits_large_binind_set)
@ -41,7 +42,7 @@
#define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) #define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get)
#define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) #define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set)
#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) #define arena_prof_promoted JEMALLOC_N(arena_prof_promoted)
#define arena_ptr_binind JEMALLOC_N(arena_ptr_binind) #define arena_ptr_small_binind_get JEMALLOC_N(arena_ptr_small_binind_get)
#define arena_purge_all JEMALLOC_N(arena_purge_all) #define arena_purge_all JEMALLOC_N(arena_purge_all)
#define arena_ralloc JEMALLOC_N(arena_ralloc) #define arena_ralloc JEMALLOC_N(arena_ralloc)
#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) #define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)

9
src/arena.c
View File

@ -154,7 +154,7 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr)
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind); size_t mapbits = arena_mapbits_get(chunk, pageind);
size_t binind = arena_ptr_binind(ptr, mapbits); size_t binind = arena_ptr_small_binind_get(ptr, mapbits);
arena_bin_info_t *bin_info = &arena_bin_info[binind]; arena_bin_info_t *bin_info = &arena_bin_info[binind];
unsigned regind = arena_run_regind(run, bin_info, ptr); unsigned regind = arena_run_regind(run, bin_info, ptr);
bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run +
@ -1581,7 +1581,7 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE));
bin = run->bin; bin = run->bin;
binind = arena_ptr_binind(ptr, mapelm->bits); binind = arena_ptr_small_binind_get(ptr, mapelm->bits);
bin_info = &arena_bin_info[binind]; bin_info = &arena_bin_info[binind];
if (config_fill || config_stats) if (config_fill || config_stats)
size = bin_info->reg_size; size = bin_info->reg_size;
@ -1624,8 +1624,9 @@ arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_chunk_map_t *mapelm; arena_chunk_map_t *mapelm;
if (config_debug) { if (config_debug) {
assert(arena_ptr_binind(ptr, arena_mapbits_get(chunk, pageind)) /* arena_ptr_small_binind_get() does extra sanity checking. */
!= BININD_INVALID); assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
pageind)) != BININD_INVALID);
} }
mapelm = arena_mapp_get(chunk, pageind); mapelm = arena_mapp_get(chunk, pageind);
arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm);