From 7c7b7020640488f26fb81143ab2ca7c74377580b Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Mon, 3 Feb 2020 13:27:21 -0800 Subject: [PATCH] Emap: Move over metadata splitting logic. --- include/jemalloc/internal/emap.h | 17 ++++++++++ src/emap.c | 53 ++++++++++++++++++++++++++++++++ src/extent.c | 44 ++++++-------------------- 3 files changed, 79 insertions(+), 35 deletions(-) diff --git a/include/jemalloc/internal/emap.h b/include/jemalloc/internal/emap.h index d28a5f79..e5188d49 100644 --- a/include/jemalloc/internal/emap.h +++ b/include/jemalloc/internal/emap.h @@ -70,4 +70,21 @@ void emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, void emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx, edata_t *edata); +typedef struct emap_split_prepare_s emap_split_prepare_t; +struct emap_split_prepare_s { + rtree_leaf_elm_t *lead_elm_a; + rtree_leaf_elm_t *lead_elm_b; + rtree_leaf_elm_t *trail_elm_a; + rtree_leaf_elm_t *trail_elm_b; +}; + +bool emap_split_prepare(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx, + emap_split_prepare_t *split_prepare, edata_t *edata, size_t size_a, + szind_t szind_a, bool slab_a, edata_t *trail, size_t size_b, + szind_t szind_b, bool slab_b, unsigned ind_b); +void emap_split_commit(tsdn_t *tsdn, emap_t *emap, + emap_split_prepare_t *split_prepare, edata_t *lead, size_t size_a, + szind_t szind_a, bool slab_a, edata_t *trail, size_t size_b, + szind_t szind_b, bool slab_b); + #endif /* JEMALLOC_INTERNAL_EMAP_H */ diff --git a/src/emap.c b/src/emap.c index 92814fcc..45673c23 100644 --- a/src/emap.c +++ b/src/emap.c @@ -183,3 +183,56 @@ emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx, LG_PAGE)); } } + +bool +emap_split_prepare(tsdn_t *tsdn, emap_t *emap, rtree_ctx_t *rtree_ctx, + emap_split_prepare_t *split_prepare, edata_t *edata, size_t size_a, + szind_t szind_a, bool slab_a, edata_t *trail, size_t size_b, + szind_t szind_b, bool slab_b, unsigned ind_b) { + /* + * Note that while the trail mostly inherits its attributes from the + * extent to be split, it maintains its own arena ind -- this allows + * cross-arena edata interactions, such as occur in the range ecache. + */ + edata_init(trail, ind_b, + (void *)((uintptr_t)edata_base_get(edata) + size_a), size_b, + slab_b, szind_b, edata_sn_get(edata), edata_state_get(edata), + edata_zeroed_get(edata), edata_committed_get(edata), + edata_dumpable_get(edata), EXTENT_NOT_HEAD); + + /* + * We use incorrect constants for things like arena ind, zero, dump, and + * commit state, and head status. This is a fake edata_t, used to + * facilitate a lookup. + */ + edata_t lead; + edata_init(&lead, 0U, edata_addr_get(edata), size_a, slab_a, szind_a, 0, + extent_state_active, false, false, false, EXTENT_NOT_HEAD); + + emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, &lead, false, true, + &split_prepare->lead_elm_a, &split_prepare->lead_elm_b); + emap_rtree_leaf_elms_lookup(tsdn, emap, rtree_ctx, trail, false, true, + &split_prepare->trail_elm_a, &split_prepare->trail_elm_b); + + if (split_prepare->lead_elm_a == NULL + || split_prepare->lead_elm_b == NULL + || split_prepare->trail_elm_a == NULL + || split_prepare->trail_elm_b == NULL) { + return true; + } + return false; +} + +void +emap_split_commit(tsdn_t *tsdn, emap_t *emap, + emap_split_prepare_t *split_prepare, edata_t *lead, size_t size_a, + szind_t szind_a, bool slab_a, edata_t *trail, size_t size_b, + szind_t szind_b, bool slab_b) { + edata_size_set(lead, size_a); + edata_szind_set(lead, szind_a); + + emap_rtree_write_acquired(tsdn, emap, split_prepare->lead_elm_a, + split_prepare->lead_elm_b, lead, szind_a, slab_a); + emap_rtree_write_acquired(tsdn, emap, split_prepare->trail_elm_a, + split_prepare->trail_elm_b, trail, szind_b, slab_b); +} diff --git a/src/extent.c b/src/extent.c index bb88c202..fa9bc41b 100644 --- a/src/extent.c +++ b/src/extent.c @@ -1276,53 +1276,27 @@ extent_split_impl(tsdn_t *tsdn, edata_cache_t *edata_cache, ehooks_t *ehooks, goto label_error_a; } - edata_init(trail, ehooks_ind_get(ehooks), - (void *)((uintptr_t)edata_base_get(edata) + size_a), size_b, - slab_b, szind_b, edata_sn_get(edata), edata_state_get(edata), - edata_zeroed_get(edata), edata_committed_get(edata), - edata_dumpable_get(edata), EXTENT_NOT_HEAD); - rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - rtree_leaf_elm_t *lead_elm_a, *lead_elm_b; - { - edata_t lead; - - edata_init(&lead, ehooks_ind_get(ehooks), - edata_addr_get(edata), size_a, - slab_a, szind_a, edata_sn_get(edata), - edata_state_get(edata), edata_zeroed_get(edata), - edata_committed_get(edata), edata_dumpable_get(edata), - EXTENT_NOT_HEAD); - - emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, - &lead, false, true, &lead_elm_a, &lead_elm_b); - } - rtree_leaf_elm_t *trail_elm_a, *trail_elm_b; - emap_rtree_leaf_elms_lookup(tsdn, &emap_global, rtree_ctx, trail, false, - true, &trail_elm_a, &trail_elm_b); - - if (lead_elm_a == NULL || lead_elm_b == NULL || trail_elm_a == NULL - || trail_elm_b == NULL) { + emap_split_prepare_t split_prepare; + bool err = emap_split_prepare(tsdn, &emap_global, rtree_ctx, + &split_prepare, edata, size_a, szind_a, slab_a, trail, size_b, + szind_b, slab_b, ehooks_ind_get(ehooks)); + if (err) { goto label_error_b; } emap_lock_edata2(tsdn, &emap_global, edata, trail); - bool err = ehooks_split(tsdn, ehooks, edata_base_get(edata), - size_a + size_b, size_a, size_b, edata_committed_get(edata)); + err = ehooks_split(tsdn, ehooks, edata_base_get(edata), size_a + size_b, + size_a, size_b, edata_committed_get(edata)); if (err) { goto label_error_c; } - edata_size_set(edata, size_a); - edata_szind_set(edata, szind_a); - - emap_rtree_write_acquired(tsdn, &emap_global, lead_elm_a, lead_elm_b, - edata, szind_a, slab_a); - emap_rtree_write_acquired(tsdn, &emap_global, trail_elm_a, trail_elm_b, - trail, szind_b, slab_b); + emap_split_commit(tsdn, &emap_global, &split_prepare, edata, size_a, + szind_a, slab_a, trail, size_b, szind_b, slab_b); emap_unlock_edata2(tsdn, &emap_global, edata, trail);