project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement cache-oblivious support for huge size classes.

Browse Source
This commit is contained in:
Jason Evans
2016-05-27 18:57:15 -07:00
parent 4731cd47f7
commit b46261d58b
12 changed files with 298 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
include/jemalloc/internal/arena.h
View File

@@ -688,7 +688,7 @@ JEMALLOC_ALWAYS_INLINE size_t
arena_miscelm_to_pageind(const extent_t *extent,
const arena_chunk_map_misc_t *miscelm)
{
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk +
map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias;
@@ -702,7 +702,7 @@ JEMALLOC_ALWAYS_INLINE void *
arena_miscelm_to_rpages(const extent_t *extent,
const arena_chunk_map_misc_t *miscelm)
{
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = arena_miscelm_to_pageind(extent, miscelm);
return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE)));
@@ -1065,7 +1065,7 @@ arena_ptr_small_binind_get(tsdn_t *tsdn, const void *ptr, size_t mapbits)
assert(binind != BININD_INVALID);
assert(binind < NBINS);
extent = iealloc(tsdn, ptr);
chunk = (arena_chunk_t *)extent_addr_get(extent);
chunk = (arena_chunk_t *)extent_base_get(extent);
arena = extent_arena_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
actual_mapbits = arena_mapbits_get(chunk, pageind);
@@ -1106,7 +1106,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
assert(ptr != NULL);
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
@@ -1132,7 +1132,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
assert(ptr != NULL);
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
@@ -1168,8 +1168,9 @@ arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr &&
(uintptr_t)old_tctx > (uintptr_t)1U))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
if (likely(chunk != ptr)) {
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk =
(arena_chunk_t *)extent_base_get(extent);
size_t pageind;
arena_chunk_map_misc_t *elm;
@@ -1253,7 +1254,7 @@ arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr, bool demote)
if (likely(extent_slab_get(extent))) {
const arena_chunk_t *chunk =
(const arena_chunk_t *)extent_addr_get(extent);
(const arena_chunk_t *)extent_base_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
@@ -1302,7 +1303,7 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
assert(ptr != NULL);
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapbits = arena_mapbits_get(chunk, pageind);
@@ -1349,7 +1350,7 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
assert(!tsdn_null(tsdn) || tcache == NULL);
if (likely(extent_slab_get(extent))) {
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
if (config_prof && opt_prof) {
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>

8
include/jemalloc/internal/chunk.h
View File

@@ -45,11 +45,11 @@ chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena,
const chunk_hooks_t *chunk_hooks);
extent_t *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool slab);
chunk_hooks_t *chunk_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab);
extent_t *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool *commit, bool slab);
chunk_hooks_t *chunk_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab);
void chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, extent_t *extent);
void chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,

42
include/jemalloc/internal/extent.h
View File

@@ -88,8 +88,10 @@ ph_proto(, extent_heap_, extent_heap_t, extent_t)
#ifndef JEMALLOC_ENABLE_INLINE
arena_t *extent_arena_get(const extent_t *extent);
void *extent_base_get(const extent_t *extent);
void *extent_addr_get(const extent_t *extent);
size_t extent_size_get(const extent_t *extent);
size_t extent_usize_get(const extent_t *extent);
void *extent_before_get(const extent_t *extent);
void *extent_last_get(const extent_t *extent);
void *extent_past_get(const extent_t *extent);
@@ -102,6 +104,7 @@ bool extent_slab_get(const extent_t *extent);
prof_tctx_t *extent_prof_tctx_get(const extent_t *extent);
void extent_arena_set(extent_t *extent, arena_t *arena);
void extent_addr_set(extent_t *extent, void *addr);
void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
void extent_size_set(extent_t *extent, size_t size);
void extent_active_set(extent_t *extent, bool active);
void extent_dirty_set(extent_t *extent, bool dirty);
@@ -125,10 +128,21 @@ extent_arena_get(const extent_t *extent)
return (extent->e_arena);
}
JEMALLOC_INLINE void *
extent_base_get(const extent_t *extent)
{
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent->e_slab);
return (PAGE_ADDR2BASE(extent->e_addr));
}
JEMALLOC_INLINE void *
extent_addr_get(const extent_t *extent)
{
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent->e_slab);
return (extent->e_addr);
}
@@ -139,6 +153,14 @@ extent_size_get(const extent_t *extent)
return (extent->e_size);
}
JEMALLOC_INLINE size_t
extent_usize_get(const extent_t *extent)
{
assert(!extent->e_slab);
return (extent->e_size - large_pad);
}
JEMALLOC_INLINE void *
extent_before_get(const extent_t *extent)
{
@@ -224,6 +246,24 @@ extent_addr_set(extent_t *extent, void *addr)
extent->e_addr = addr;
}
JEMALLOC_INLINE void
extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment)
{
assert(extent_base_get(extent) == extent_addr_get(extent));
if (alignment < PAGE) {
unsigned lg_range = LG_PAGE -
lg_floor(CACHELINE_CEILING(alignment));
uint64_t r =
prng_lg_range(&extent_arena_get(extent)->offset_state,
lg_range, true);
uintptr_t random_offset = ((uintptr_t)r) << lg_range;
extent->e_addr = (void *)((uintptr_t)extent->e_addr +
random_offset);
}
}
JEMALLOC_INLINE void
extent_size_set(extent_t *extent, size_t size)
{
@@ -278,6 +318,8 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
bool active, bool dirty, bool zeroed, bool committed, bool slab)
{
assert(addr == PAGE_ADDR2BASE(addr) || !slab);
extent_arena_set(extent, arena);
extent_addr_set(extent, addr);
extent_size_set(extent, size);

4
include/jemalloc/internal/private_symbols.txt
View File

@@ -206,10 +206,12 @@ dss_prec_names
extent_active_get
extent_active_set
extent_addr_get
extent_addr_randomize
extent_addr_set
extent_alloc
extent_arena_get
extent_arena_set
extent_base_get
extent_before_get
extent_committed_get
extent_committed_set
@@ -230,6 +232,7 @@ extent_size_quantize_ceil
extent_size_quantize_floor
extent_slab_get
extent_slab_set
extent_usize_get
extent_zeroed_get
extent_zeroed_set
ffs_llu
@@ -373,6 +376,7 @@ pow2_ceil_u64
pow2_ceil_zu
prng_lg_range
prng_range
prng_state_next
prof_active
prof_active_get
prof_active_get_unlocked

35
include/jemalloc/internal/prng.h
View File

@@ -35,28 +35,45 @@
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
uint64_t prng_lg_range(uint64_t *state, unsigned lg_range);
uint64_t prng_range(uint64_t *state, uint64_t range);
uint64_t prng_state_next(uint64_t state);
uint64_t prng_lg_range(uint64_t *state, unsigned lg_range, bool atomic);
uint64_t prng_range(uint64_t *state, uint64_t range, bool atomic);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_))
JEMALLOC_ALWAYS_INLINE uint64_t
prng_lg_range(uint64_t *state, unsigned lg_range)
prng_state_next(uint64_t state)
{
uint64_t ret;
return ((state * PRNG_A) + PRNG_C);
}
JEMALLOC_ALWAYS_INLINE uint64_t
prng_lg_range(uint64_t *state, unsigned lg_range, bool atomic)
{
uint64_t ret, state1;
assert(lg_range > 0);
assert(lg_range <= 64);
ret = (*state * PRNG_A) + PRNG_C;
*state = ret;
ret >>= (64 - lg_range);
if (atomic) {
uint64_t state0;
do {
state0 = atomic_read_uint64(state);
state1 = prng_state_next(state0);
} while (atomic_cas_uint64(state, state0, state1));
} else {
state1 = prng_state_next(*state);
*state = state1;
}
ret = state1 >> (64 - lg_range);
return (ret);
}
JEMALLOC_ALWAYS_INLINE uint64_t
prng_range(uint64_t *state, uint64_t range)
prng_range(uint64_t *state, uint64_t range, bool atomic)
{
uint64_t ret;
unsigned lg_range;
@@ -68,7 +85,7 @@ prng_range(uint64_t *state, uint64_t range)
/* Generate a result in [0..range) via repeated trial. */
do {
ret = prng_lg_range(state, lg_range);
ret = prng_lg_range(state, lg_range, atomic);
} while (ret >= range);
return (ret);