project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Avoid atomic operations for dependent rtree reads.

Browse Source
This commit is contained in:
Jason Evans 2015-05-15 17:02:30 -07:00
parent c451831264
commit fd5f9e43c3
5 changed files with 43 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/jemalloc/internal/chunk.h
View File

@ -70,15 +70,15 @@ void chunk_postfork_child(void);
#ifdef JEMALLOC_H_INLINES #ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
extent_node_t *chunk_lookup(const void *chunk); extent_node_t *chunk_lookup(const void *chunk, bool dependent);
#endif #endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_CHUNK_C_)) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_CHUNK_C_))
JEMALLOC_INLINE extent_node_t * JEMALLOC_INLINE extent_node_t *
chunk_lookup(const void *chunk) chunk_lookup(const void *ptr, bool dependent)
{ {
return (rtree_get(&chunks_rtree, (uintptr_t)chunk)); return (rtree_get(&chunks_rtree, (uintptr_t)ptr, dependent));
} }
#endif #endif

2
include/jemalloc/internal/jemalloc_internal.h.in
View File

@ -948,7 +948,7 @@ ivsalloc(const void *ptr, bool demote)
extent_node_t *node; extent_node_t *node;
/* Return 0 if ptr is not within a chunk managed by jemalloc. */ /* Return 0 if ptr is not within a chunk managed by jemalloc. */
node = chunk_lookup(CHUNK_ADDR2BASE(ptr)); node = chunk_lookup(ptr, false);
if (node == NULL) if (node == NULL)
return (0); return (0);
/* Only arena chunks should be looked up via interior pointers. */ /* Only arena chunks should be looked up via interior pointers. */

29
include/jemalloc/internal/rtree.h
View File

@ -114,13 +114,14 @@ bool rtree_node_valid(rtree_node_elm_t *node);
rtree_node_elm_t *rtree_child_tryread(rtree_node_elm_t *elm); rtree_node_elm_t *rtree_child_tryread(rtree_node_elm_t *elm);
rtree_node_elm_t *rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, rtree_node_elm_t *rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm,
unsigned level); unsigned level);
extent_node_t *rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm); extent_node_t *rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm,
bool dependent);
void rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, void rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm,
const extent_node_t *val); const extent_node_t *val);
rtree_node_elm_t *rtree_subtree_tryread(rtree_t *rtree, unsigned level); rtree_node_elm_t *rtree_subtree_tryread(rtree_t *rtree, unsigned level);
rtree_node_elm_t *rtree_subtree_read(rtree_t *rtree, unsigned level); rtree_node_elm_t *rtree_subtree_read(rtree_t *rtree, unsigned level);
extent_node_t *rtree_get(rtree_t *rtree, uintptr_t key); extent_node_t *rtree_get(rtree_t *rtree, uintptr_t key, bool dependent);
bool rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val); bool rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val);
#endif #endif
@ -179,11 +180,26 @@ rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level)
} }
JEMALLOC_INLINE extent_node_t * JEMALLOC_INLINE extent_node_t *
rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm) rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm, bool dependent)
{ {
if (dependent) {
/*
* Reading a val on behalf of a pointer to a valid allocation is
* guaranteed to be a clean read even without synchronization,
* because the rtree update became visible in memory before the
* pointer came into existence.
*/
return (elm->val);
} else {
/*
* An arbitrary read, e.g. on behalf of ivsalloc(), may not be
* dependent on a previous rtree write, which means a stale read
* could result if synchronization were omitted here.
*/
return (atomic_read_p(&elm->pun)); return (atomic_read_p(&elm->pun));
} }
}
JEMALLOC_INLINE void JEMALLOC_INLINE void
rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, const extent_node_t *val) rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, const extent_node_t *val)
@ -216,7 +232,7 @@ rtree_subtree_read(rtree_t *rtree, unsigned level)
} }
JEMALLOC_INLINE extent_node_t * JEMALLOC_INLINE extent_node_t *
rtree_get(rtree_t *rtree, uintptr_t key) rtree_get(rtree_t *rtree, uintptr_t key, bool dependent)
{ {
uintptr_t subkey; uintptr_t subkey;
unsigned i, start_level; unsigned i, start_level;
@ -226,7 +242,7 @@ rtree_get(rtree_t *rtree, uintptr_t key)
for (i = start_level, node = rtree_subtree_tryread(rtree, start_level); for (i = start_level, node = rtree_subtree_tryread(rtree, start_level);
/**/; i++, node = child) { /**/; i++, node = child) {
if (unlikely(!rtree_node_valid(node))) if (!dependent && unlikely(!rtree_node_valid(node)))
return (NULL); return (NULL);
subkey = rtree_subkey(rtree, key, i); subkey = rtree_subkey(rtree, key, i);
if (i == rtree->height - 1) { if (i == rtree->height - 1) {
@ -234,7 +250,8 @@ rtree_get(rtree_t *rtree, uintptr_t key)
* node is a leaf, so it contains values rather than * node is a leaf, so it contains values rather than
* child pointers. * child pointers.
*/ */
return (rtree_val_read(rtree, &node[subkey])); return (rtree_val_read(rtree, &node[subkey],
dependent));
} }
assert(i < rtree->height - 1); assert(i < rtree->height - 1);
child = rtree_child_tryread(&node[subkey]); child = rtree_child_tryread(&node[subkey]);

2
src/huge.c
View File

@ -8,7 +8,7 @@ huge_node_get(const void *ptr)
{ {
extent_node_t *node; extent_node_t *node;
node = chunk_lookup(ptr); node = chunk_lookup(ptr, true);
assert(!extent_node_achunk_get(node)); assert(!extent_node_achunk_get(node));
return (node); return (node);

28
test/unit/rtree.c
View File

@ -22,7 +22,7 @@ TEST_BEGIN(test_rtree_get_empty)
rtree_t rtree; rtree_t rtree;
assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc), assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc),
"Unexpected rtree_new() failure"); "Unexpected rtree_new() failure");
assert_ptr_null(rtree_get(&rtree, 0), assert_ptr_null(rtree_get(&rtree, 0, false),
"rtree_get() should return NULL for empty tree"); "rtree_get() should return NULL for empty tree");
rtree_delete(&rtree); rtree_delete(&rtree);
} }
@ -40,11 +40,11 @@ TEST_BEGIN(test_rtree_extrema)
"Unexpected rtree_new() failure"); "Unexpected rtree_new() failure");
rtree_set(&rtree, 0, &node_a); rtree_set(&rtree, 0, &node_a);
assert_ptr_eq(rtree_get(&rtree, 0), &node_a, assert_ptr_eq(rtree_get(&rtree, 0, true), &node_a,
"rtree_get() should return previously set value"); "rtree_get() should return previously set value");
rtree_set(&rtree, ~((uintptr_t)0), &node_b); rtree_set(&rtree, ~((uintptr_t)0), &node_b);
assert_ptr_eq(rtree_get(&rtree, ~((uintptr_t)0)), &node_b, assert_ptr_eq(rtree_get(&rtree, ~((uintptr_t)0), true), &node_b,
"rtree_get() should return previously set value"); "rtree_get() should return previously set value");
rtree_delete(&rtree); rtree_delete(&rtree);
@ -68,15 +68,15 @@ TEST_BEGIN(test_rtree_bits)
for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
rtree_set(&rtree, keys[j], &node); rtree_set(&rtree, keys[j], &node);
for (k = 0; k < sizeof(keys)/sizeof(uintptr_t); k++) { for (k = 0; k < sizeof(keys)/sizeof(uintptr_t); k++) {
assert_ptr_eq(rtree_get(&rtree, keys[k]), &node, assert_ptr_eq(rtree_get(&rtree, keys[k], true),
"rtree_get() should return previously set " &node, "rtree_get() should return "
"value and ignore insignificant key bits; " "previously set value and ignore "
"i=%u, j=%u, k=%u, set key=%#"PRIxPTR", " "insignificant key bits; i=%u, j=%u, k=%u, "
"get key=%#"PRIxPTR, i, j, k, keys[j], "set key=%#"PRIxPTR", get key=%#"PRIxPTR, i,
keys[k]); j, k, keys[j], keys[k]);
} }
assert_ptr_null(rtree_get(&rtree, assert_ptr_null(rtree_get(&rtree,
(((uintptr_t)1) << (sizeof(uintptr_t)*8-i))), (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)), false),
"Only leftmost rtree leaf should be set; " "Only leftmost rtree leaf should be set; "
"i=%u, j=%u", i, j); "i=%u, j=%u", i, j);
rtree_set(&rtree, keys[j], NULL); rtree_set(&rtree, keys[j], NULL);
@ -107,21 +107,21 @@ TEST_BEGIN(test_rtree_random)
for (j = 0; j < NSET; j++) { for (j = 0; j < NSET; j++) {
keys[j] = (uintptr_t)gen_rand64(sfmt); keys[j] = (uintptr_t)gen_rand64(sfmt);
rtree_set(&rtree, keys[j], &node); rtree_set(&rtree, keys[j], &node);
assert_ptr_eq(rtree_get(&rtree, keys[j]), &node, assert_ptr_eq(rtree_get(&rtree, keys[j], true), &node,
"rtree_get() should return previously set value"); "rtree_get() should return previously set value");
} }
for (j = 0; j < NSET; j++) { for (j = 0; j < NSET; j++) {
assert_ptr_eq(rtree_get(&rtree, keys[j]), &node, assert_ptr_eq(rtree_get(&rtree, keys[j], true), &node,
"rtree_get() should return previously set value"); "rtree_get() should return previously set value");
} }
for (j = 0; j < NSET; j++) { for (j = 0; j < NSET; j++) {
rtree_set(&rtree, keys[j], NULL); rtree_set(&rtree, keys[j], NULL);
assert_ptr_null(rtree_get(&rtree, keys[j]), assert_ptr_null(rtree_get(&rtree, keys[j], true),
"rtree_get() should return previously set value"); "rtree_get() should return previously set value");
} }
for (j = 0; j < NSET; j++) { for (j = 0; j < NSET; j++) {
assert_ptr_null(rtree_get(&rtree, keys[j]), assert_ptr_null(rtree_get(&rtree, keys[j], true),
"rtree_get() should return previously set value"); "rtree_get() should return previously set value");
} }