project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Determine rtree levels at compile time.

Browse Source 
Rather than dynamically building a table to aid per level computations,
define a constant table at compile time.  Omit both high and low
insignificant bits.  Use one to three tree levels, depending on the
number of significant bits.
This commit is contained in:
Jason Evans
2017-02-06 13:17:12 -08:00
parent ff4db5014e
commit f5cf9b19c8
9 changed files with 244 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/jemalloc/internal/jemalloc_internal_defs.h.in
View File

@@ -22,6 +22,13 @@
*/
#undef CPU_SPINWAIT
/*
* Number of significant bits in virtual addresses. This may be less than the
* total number of bits in a pointer, e.g. on x64, for which the uppermost 16
* bits are the same as bit 47.
*/
#undef LG_VADDR
/* Defined if C11 atomics are available. */
#undef JEMALLOC_C11ATOMICS

24
include/jemalloc/internal/rtree_externs.h
View File

@@ -1,7 +1,29 @@
#ifndef JEMALLOC_INTERNAL_RTREE_EXTERNS_H
#define JEMALLOC_INTERNAL_RTREE_EXTERNS_H
bool rtree_new(rtree_t *rtree, unsigned bits);
/*
* Split the bits into one to three partitions depending on number of
* significant bits. It the number of bits does not divide evenly into the
* number of levels, place one remainder bit per level starting at the leaf
* level.
*/
static const rtree_level_t rtree_levels[] = {
#if RTREE_NSB <= 10
{RTREE_NSB, RTREE_NHIB + RTREE_NSB}
#elif RTREE_NSB <= 36
{RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2},
{RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB}
#elif RTREE_NSB <= 52
{RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3},
{RTREE_NSB/3 + RTREE_NSB%3/2,
RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2},
{RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB}
#else
# error Unsupported number of significant virtual address bits
#endif
};
bool rtree_new(rtree_t *rtree);
#ifdef JEMALLOC_JET
typedef rtree_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
extern rtree_node_alloc_t *rtree_node_alloc;

22
include/jemalloc/internal/rtree_inlines.h
View File

@@ -2,8 +2,8 @@
#define JEMALLOC_INTERNAL_RTREE_INLINES_H
#ifndef JEMALLOC_ENABLE_INLINE
uintptr_t rtree_leafkey(rtree_t *rtree, uintptr_t key);
uintptr_t rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level);
uintptr_t rtree_leafkey(uintptr_t key);
uintptr_t rtree_subkey(uintptr_t key, unsigned level);
extent_t *rtree_elm_read(rtree_elm_t *elm, bool dependent);
void rtree_elm_write(rtree_elm_t *elm, const extent_t *extent);
rtree_elm_t *rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree,
@@ -25,21 +25,21 @@ void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_))
JEMALLOC_ALWAYS_INLINE uintptr_t
rtree_leafkey(rtree_t *rtree, uintptr_t key) {
rtree_leafkey(uintptr_t key) {
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
unsigned cumbits = (rtree->levels[rtree->height-1].cumbits -
rtree->levels[rtree->height-1].bits);
unsigned cumbits = (rtree_levels[RTREE_HEIGHT-1].cumbits -
rtree_levels[RTREE_HEIGHT-1].bits);
unsigned maskbits = ptrbits - cumbits;
uintptr_t mask = ~((ZU(1) << maskbits) - 1);
return (key & mask);
}
JEMALLOC_ALWAYS_INLINE uintptr_t
rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level) {
rtree_subkey(uintptr_t key, unsigned level) {
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
unsigned cumbits = rtree->levels[level].cumbits;
unsigned cumbits = rtree_levels[level].cumbits;
unsigned shiftbits = ptrbits - cumbits;
unsigned maskbits = rtree->levels[level].bits;
unsigned maskbits = rtree_levels[level].bits;
unsigned mask = (ZU(1) << maskbits) - 1;
return ((key >> shiftbits) & mask);
}
@@ -82,7 +82,7 @@ rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
assert(!dependent || !init_missing);
if (likely(key != 0)) {
uintptr_t leafkey = rtree_leafkey(rtree, key);
uintptr_t leafkey = rtree_leafkey(key);
#define RTREE_CACHE_CHECK(i) do { \
if (likely(rtree_ctx->cache[i].leafkey == leafkey)) { \
rtree_elm_t *leaf = rtree_ctx->cache[i].leaf; \
@@ -94,8 +94,8 @@ rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
rtree_ctx->cache[0].leafkey = leafkey; \
rtree_ctx->cache[0].leaf = leaf; \
\
uintptr_t subkey = rtree_subkey(rtree, \
key, rtree->height-1); \
uintptr_t subkey = rtree_subkey(key, \
RTREE_HEIGHT-1); \
return &leaf[subkey]; \
} \
} \

2
include/jemalloc/internal/rtree_structs.h
View File

@@ -41,12 +41,10 @@ struct rtree_ctx_s {
};
struct rtree_s {
unsigned height;
union {
void *root_pun;
rtree_elm_t *root;
};
rtree_level_t levels[RTREE_HEIGHT_MAX];
malloc_mutex_t init_lock;
};

17
include/jemalloc/internal/rtree_types.h
View File

@@ -16,15 +16,14 @@ typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t;
typedef struct rtree_ctx_s rtree_ctx_t;
typedef struct rtree_s rtree_t;
/*
* RTREE_BITS_PER_LEVEL must be a power of two that is no larger than the
* machine address width.
*/
#define LG_RTREE_BITS_PER_LEVEL 4
#define RTREE_BITS_PER_LEVEL (1U << LG_RTREE_BITS_PER_LEVEL)
/* Maximum rtree height. */
#define RTREE_HEIGHT_MAX \
((1U << (LG_SIZEOF_PTR+3)) / RTREE_BITS_PER_LEVEL)
/* Number of high insignificant bits. */
#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR)
/* Number of low insigificant bits. */
#define RTREE_NLIB LG_PAGE
/* Number of significant bits. */
#define RTREE_NSB (LG_VADDR - RTREE_NLIB)
/* Number of levels in radix tree. */
#define RTREE_HEIGHT (sizeof(rtree_levels)/sizeof(rtree_level_t))
/*
* Number of leafkey/leaf pairs to cache. Each entry supports an entire leaf,