project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rtree: Remove the per-field accessors.

Browse Source 
We instead split things into "edata" and "metadata".
This commit is contained in:
David Goldblatt 2020-03-19 17:58:44 -07:00 committed by David Goldblatt
parent 26e9a3103d
commit 79ae7f9211
5 changed files with 178 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
include/jemalloc/internal/emap.h
View File

@ -132,8 +132,7 @@ emap_edata_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr) {
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
return rtree_edata_read(tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr,
true);
return rtree_read(tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr).edata;
}
/* Fills in alloc_ctx with the info in the map. */
@ -143,8 +142,10 @@ emap_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_szind_slab_read(tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr,
true, &alloc_ctx->szind, &alloc_ctx->slab);
rtree_metadata_t metadata = rtree_metadata_read(tsdn, &emap->rtree,
rtree_ctx, (uintptr_t)ptr);
alloc_ctx->szind = metadata.szind;
alloc_ctx->slab = metadata.slab;
}
/* The pointer must be mapped. */
@ -154,9 +155,11 @@ emap_full_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
rtree_edata_szind_slab_read(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)ptr, true, &full_alloc_ctx->edata,
&full_alloc_ctx->szind, &full_alloc_ctx->slab);
rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)ptr);
full_alloc_ctx->edata = contents.edata;
full_alloc_ctx->szind = contents.metadata.szind;
full_alloc_ctx->slab = contents.metadata.slab;
}
/*
@ -170,24 +173,35 @@ emap_full_alloc_ctx_try_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
return rtree_edata_szind_slab_read(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)ptr, false, &full_alloc_ctx->edata,
&full_alloc_ctx->szind, &full_alloc_ctx->slab);
rtree_contents_t contents;
bool err = rtree_read_independent(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)ptr, &contents);
if (err) {
return true;
}
full_alloc_ctx->edata = contents.edata;
full_alloc_ctx->szind = contents.metadata.szind;
full_alloc_ctx->slab = contents.metadata.slab;
return false;
}
/*
* Fills in alloc_ctx, but only if it can be done easily (i.e. with a hit in the
* L1 rtree cache.
*
* Returns whether or not alloc_ctx was filled in.
* Returns true on error.
*/
JEMALLOC_ALWAYS_INLINE bool
emap_alloc_ctx_try_lookup_fast(tsd_t *tsd, emap_t *emap, const void *ptr,
emap_alloc_ctx_t *alloc_ctx) {
rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &emap->rtree,
rtree_ctx, (uintptr_t)ptr, &alloc_ctx->szind, &alloc_ctx->slab);
return res;
rtree_metadata_t metadata;
bool err = rtree_metadata_try_read_fast(tsd_tsdn(tsd), &emap->rtree,
rtree_ctx, (uintptr_t)ptr, &metadata);
if (err) {
return true;
}
alloc_ctx->szind = metadata.szind;
alloc_ctx->slab = metadata.slab;
return false;
}
#endif /* JEMALLOC_INTERNAL_EMAP_H */

215
include/jemalloc/internal/rtree.h
View File

@ -257,19 +257,29 @@ rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree,
#endif
}
static inline void
rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree,
rtree_leaf_elm_t *elm, szind_t szind, bool slab) {
assert(!slab || szind < SC_NBINS);
rtree_contents_t contents = rtree_leaf_elm_read(
tsdn, rtree, elm, /* dependent */ true);
/*
* The caller implicitly assures that it is the only writer to the szind
* and slab fields, and that the edata field cannot currently change.
*/
contents.metadata.slab = slab;
contents.metadata.szind = szind;
rtree_leaf_elm_write(tsdn, rtree, elm, contents);
/*
* Tries to look up the key in the L1 cache, returning it if there's a hit, or
* NULL if there's a miss.
* Key is allowed to be NULL; returns NULL in this case.
*/
JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t *
rtree_leaf_elm_lookup_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key) {
rtree_leaf_elm_t *elm;
size_t slot = rtree_cache_direct_map(key);
uintptr_t leafkey = rtree_leafkey(key);
assert(leafkey != RTREE_LEAFKEY_INVALID);
if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) {
rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf;
assert(leaf != NULL);
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1);
elm = &leaf[subkey];
return elm;
} else {
return NULL;
}
}
JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t *
@ -331,144 +341,79 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
dependent, init_missing);
}
/*
* Returns true on lookup failure.
*/
static inline bool
rtree_read_independent(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, rtree_contents_t *r_contents) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, /* dependent */ false, /* init_missing */ false);
if (elm == NULL) {
return true;
}
*r_contents = rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ false);
return false;
}
static inline rtree_contents_t
rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, /* dependent */ true, /* init_missing */ false);
assert(elm != NULL);
return rtree_leaf_elm_read(tsdn, rtree, elm, /* dependent */ true);
}
static inline rtree_metadata_t
rtree_metadata_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, /* dependent */ true, /* init_missing */ false);
assert(elm != NULL);
return rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ true).metadata;
}
/*
* Returns true on error.
*/
static inline bool
rtree_metadata_try_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, rtree_metadata_t *r_rtree_metadata) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup_fast(tsdn, rtree, rtree_ctx,
key);
if (elm == NULL) {
return true;
}
*r_rtree_metadata = rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ true).metadata;
return false;
}
static inline bool
rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
edata_t *edata, szind_t szind, bool slab) {
rtree_contents_t contents) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, false, true);
key, /* dependent */ false, /* init_missing */ true);
if (elm == NULL) {
return true;
}
rtree_contents_t contents;
contents.edata = edata;
contents.metadata.szind = szind;
contents.metadata.slab = slab;
rtree_leaf_elm_write(tsdn, rtree, elm, contents);
return false;
}
JEMALLOC_ALWAYS_INLINE rtree_leaf_elm_t *
rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
bool dependent) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, dependent, false);
if (!dependent && elm == NULL) {
return NULL;
}
assert(elm != NULL);
return elm;
}
JEMALLOC_ALWAYS_INLINE edata_t *
rtree_edata_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, bool dependent) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
if (!dependent && elm == NULL) {
return NULL;
}
return rtree_leaf_elm_read(tsdn, rtree, elm, dependent).edata;
}
JEMALLOC_ALWAYS_INLINE szind_t
rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, bool dependent) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
if (!dependent && elm == NULL) {
return SC_NSIZES;
}
return rtree_leaf_elm_read(tsdn, rtree, elm, dependent).metadata.szind;
}
/*
* rtree_slab_read() is intentionally omitted because slab is always read in
* conjunction with szind, which makes rtree_szind_slab_read() a better choice.
*/
JEMALLOC_ALWAYS_INLINE bool
rtree_edata_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree,
rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, edata_t **r_edata,
szind_t *r_szind, bool *r_slab) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
if (!dependent && elm == NULL) {
return true;
}
rtree_contents_t contents = rtree_leaf_elm_read(tsdn, rtree, elm,
dependent);
*r_edata = contents.edata;
*r_szind = contents.metadata.szind;
*r_slab = contents.metadata.slab;
return false;
}
/*
* Try to read szind_slab from the L1 cache. Returns true on a hit,
* and fills in r_szind and r_slab. Otherwise returns false.
*
* Key is allowed to be NULL in order to save an extra branch on the
* fastpath. returns false in this case.
*/
JEMALLOC_ALWAYS_INLINE bool
rtree_szind_slab_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, szind_t *r_szind, bool *r_slab) {
rtree_leaf_elm_t *elm;
size_t slot = rtree_cache_direct_map(key);
uintptr_t leafkey = rtree_leafkey(key);
assert(leafkey != RTREE_LEAFKEY_INVALID);
if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) {
rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf;
assert(leaf != NULL);
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1);
elm = &leaf[subkey];
rtree_contents_t contents = rtree_leaf_elm_read(tsdn, rtree,
elm, /* dependent */ true);
*r_szind = contents.metadata.szind;
*r_slab = contents.metadata.slab;
return true;
} else {
return false;
}
}
JEMALLOC_ALWAYS_INLINE bool
rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key,
dependent);
if (!dependent && elm == NULL) {
return true;
}
rtree_contents_t contents = rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ true);
*r_szind = contents.metadata.szind;
*r_slab = contents.metadata.slab;
return false;
}
static inline void
rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, szind_t szind, bool slab) {
assert(!slab || szind < SC_NBINS);
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true);
rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab);
}
static inline void
rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true);
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, /* dependent */ true, /* init_missing */ false);
assert(elm != NULL);
assert(rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ false).edata != NULL);
/* dependent */ true).edata != NULL);
rtree_contents_t contents;
contents.edata = NULL;
contents.metadata.szind = SC_NSIZES;

31
src/emap.c
View File

@ -171,9 +171,13 @@ emap_register_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
/* Register interior. */
for (size_t i = 1; i < (edata_size_get(edata) >> LG_PAGE) - 1; i++) {
rtree_contents_t contents;
contents.edata = edata;
contents.metadata.szind = szind;
contents.metadata.slab = true;
rtree_write(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata) + (uintptr_t)(i <<
LG_PAGE), edata, szind, true);
LG_PAGE), contents);
}
}
@ -200,13 +204,18 @@ emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
}
}
void emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind,
void
emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind,
bool slab) {
EMAP_DECLARE_RTREE_CTX;
if (szind != SC_NSIZES) {
rtree_szind_slab_update(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_addr_get(edata), szind, slab);
rtree_contents_t contents;
contents.edata = edata;
contents.metadata.szind = szind;
contents.metadata.slab = slab;
rtree_write(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_addr_get(edata), contents);
/*
* Recall that this is called only for active->inactive and
* inactive->active transitions (since only active extents have
@ -220,12 +229,12 @@ void emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind,
* call is coming in those cases, though.
*/
if (slab && edata_size_get(edata) > PAGE) {
rtree_szind_slab_update(tsdn,
&emap->rtree, rtree_ctx,
(uintptr_t)edata_past_get(edata) - (uintptr_t)PAGE,
szind, slab);
}
uintptr_t key = (uintptr_t)edata_past_get(edata)
- (uintptr_t)PAGE;
rtree_write(tsdn, &emap->rtree, rtree_ctx, key,
contents);
}
}
}
bool
@ -311,6 +320,6 @@ void
emap_do_assert_mapped(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
EMAP_DECLARE_RTREE_CTX;
assert(rtree_edata_read(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata), true) == edata);
assert(rtree_read(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_base_get(edata)).edata == edata);
}

4
src/jemalloc.c
View File

@ -2782,11 +2782,11 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
if (unlikely(tsd == NULL || !tsd_fast(tsd))) {
return false;
}
bool res = emap_alloc_ctx_try_lookup_fast(tsd,
bool err = emap_alloc_ctx_try_lookup_fast(tsd,
&arena_emap_global, ptr, &alloc_ctx);
/* Note: profiled objects will have alloc_ctx.slab set */
if (unlikely(!res || !alloc_ctx.slab)) {
if (unlikely(err || !alloc_ctx.slab)) {
return false;
}
assert(alloc_ctx.szind != SC_NSIZES);

73
test/unit/rtree.c
View File

@ -20,8 +20,9 @@ TEST_BEGIN(test_rtree_read_empty) {
rtree_ctx_data_init(&rtree_ctx);
expect_false(rtree_new(rtree, base, false),
"Unexpected rtree_new() failure");
expect_ptr_null(rtree_edata_read(tsdn, rtree, &rtree_ctx, PAGE,
false), "rtree_edata_read() should return NULL for empty tree");
rtree_contents_t contents;
expect_true(rtree_read_independent(tsdn, rtree, &rtree_ctx, PAGE,
&contents), "rtree_read_independent() should fail on empty rtree.");
base_delete(tsdn, base);
}
@ -50,21 +51,33 @@ TEST_BEGIN(test_rtree_extrema) {
expect_false(rtree_new(rtree, base, false),
"Unexpected rtree_new() failure");
expect_false(rtree_write(tsdn, rtree, &rtree_ctx, PAGE, &edata_a,
edata_szind_get(&edata_a), edata_slab_get(&edata_a)),
rtree_contents_t contents_a;
contents_a.edata = &edata_a;
contents_a.metadata.szind = edata_szind_get(&edata_a);
contents_a.metadata.slab = edata_slab_get(&edata_a);
expect_false(rtree_write(tsdn, rtree, &rtree_ctx, PAGE, contents_a),
"Unexpected rtree_write() failure");
rtree_szind_slab_update(tsdn, rtree, &rtree_ctx, PAGE,
edata_szind_get(&edata_a), edata_slab_get(&edata_a));
expect_ptr_eq(rtree_edata_read(tsdn, rtree, &rtree_ctx, PAGE, true),
&edata_a,
"rtree_edata_read() should return previously set value");
expect_false(rtree_write(tsdn, rtree, &rtree_ctx, PAGE, contents_a),
"Unexpected rtree_write() failure");
rtree_contents_t read_contents_a = rtree_read(tsdn, rtree, &rtree_ctx,
PAGE);
expect_true(contents_a.edata == read_contents_a.edata
&& contents_a.metadata.szind == read_contents_a.metadata.szind
&& contents_a.metadata.slab == read_contents_a.metadata.slab,
"rtree_read() should return previously set value");
rtree_contents_t contents_b;
contents_b.edata = &edata_b;
contents_b.metadata.szind = edata_szind_get_maybe_invalid(&edata_b);
contents_b.metadata.slab = edata_slab_get(&edata_b);
expect_false(rtree_write(tsdn, rtree, &rtree_ctx, ~((uintptr_t)0),
&edata_b, edata_szind_get_maybe_invalid(&edata_b),
edata_slab_get(&edata_b)), "Unexpected rtree_write() failure");
expect_ptr_eq(rtree_edata_read(tsdn, rtree, &rtree_ctx,
~((uintptr_t)0), true), &edata_b,
"rtree_edata_read() should return previously set value");
contents_b), "Unexpected rtree_write() failure");
rtree_contents_t read_contents_b = rtree_read(tsdn, rtree, &rtree_ctx,
~((uintptr_t)0));
assert_true(contents_b.edata == read_contents_b.edata
&& contents_b.metadata.szind == read_contents_b.metadata.szind
&& contents_b.metadata.slab == read_contents_b.metadata.slab,
"rtree_read() should return previously set value");
base_delete(tsdn, base);
}
@ -89,19 +102,23 @@ TEST_BEGIN(test_rtree_bits) {
"Unexpected rtree_new() failure");
for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) {
rtree_contents_t contents;
contents.edata = &edata;
contents.metadata.szind = SC_NSIZES;
contents.metadata.slab = false;
expect_false(rtree_write(tsdn, rtree, &rtree_ctx, keys[i],
&edata, SC_NSIZES, false),
"Unexpected rtree_write() failure");
contents), "Unexpected rtree_write() failure");
for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
expect_ptr_eq(rtree_edata_read(tsdn, rtree, &rtree_ctx,
keys[j], true), &edata,
expect_ptr_eq(rtree_read(tsdn, rtree, &rtree_ctx,
keys[j]).edata, &edata,
"rtree_edata_read() should return previously set "
"value and ignore insignificant key bits; i=%u, "
"j=%u, set key=%#"FMTxPTR", get key=%#"FMTxPTR, i,
j, keys[i], keys[j]);
}
expect_ptr_null(rtree_edata_read(tsdn, rtree, &rtree_ctx,
(((uintptr_t)2) << LG_PAGE), false),
expect_ptr_null(rtree_read(tsdn, rtree, &rtree_ctx,
(((uintptr_t)2) << LG_PAGE)).edata,
"Only leftmost rtree leaf should be set; i=%u", i);
rtree_clear(tsdn, rtree, &rtree_ctx, keys[i]);
}
@ -142,26 +159,26 @@ TEST_BEGIN(test_rtree_random) {
contents.metadata.szind = SC_NSIZES;
contents.metadata.slab = false;
rtree_leaf_elm_write(tsdn, rtree, elm, contents);
expect_ptr_eq(rtree_edata_read(tsdn, rtree, &rtree_ctx,
keys[i], true), &edata,
expect_ptr_eq(rtree_read(tsdn, rtree, &rtree_ctx,
keys[i]).edata, &edata,
"rtree_edata_read() should return previously set value");
}
for (unsigned i = 0; i < NSET; i++) {
expect_ptr_eq(rtree_edata_read(tsdn, rtree, &rtree_ctx,
keys[i], true), &edata,
expect_ptr_eq(rtree_read(tsdn, rtree, &rtree_ctx,
keys[i]).edata, &edata,
"rtree_edata_read() should return previously set value, "
"i=%u", i);
}
for (unsigned i = 0; i < NSET; i++) {
rtree_clear(tsdn, rtree, &rtree_ctx, keys[i]);
expect_ptr_null(rtree_edata_read(tsdn, rtree, &rtree_ctx,
keys[i], true),
expect_ptr_null(rtree_read(tsdn, rtree, &rtree_ctx,
keys[i]).edata,
"rtree_edata_read() should return previously set value");
}
for (unsigned i = 0; i < NSET; i++) {
expect_ptr_null(rtree_edata_read(tsdn, rtree, &rtree_ctx,
keys[i], true),
expect_ptr_null(rtree_read(tsdn, rtree, &rtree_ctx,
keys[i]).edata,
"rtree_edata_read() should return previously set value");
}