From 3093d9455eb179d75ec8a17b1073ee605fb1f0a9 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Thu, 25 Mar 2021 15:32:44 -0700 Subject: [PATCH] Move the edata mergeability related functions to extent.h. --- include/jemalloc/internal/emap.h | 20 -------- include/jemalloc/internal/extent.h | 73 ++++++++++++++++++++++++++++++ src/ehooks.c | 2 +- src/emap.c | 60 +----------------------- 4 files changed, 75 insertions(+), 80 deletions(-) diff --git a/include/jemalloc/internal/emap.h b/include/jemalloc/internal/emap.h index 364aefac..5a5dbb6d 100644 --- a/include/jemalloc/internal/emap.h +++ b/include/jemalloc/internal/emap.h @@ -224,26 +224,6 @@ extent_assert_can_expand(const edata_t *original, const edata_t *expand) { assert(edata_past_get(original) == edata_base_get(expand)); } -JEMALLOC_ALWAYS_INLINE bool -edata_neighbor_head_state_mergeable(bool edata_is_head, - bool neighbor_is_head, bool forward) { - /* - * Head states checking: disallow merging if the higher addr extent is a - * head extent. This helps preserve first-fit, and more importantly - * makes sure no merge across arenas. - */ - if (forward) { - if (neighbor_is_head) { - return false; - } - } else { - if (edata_is_head) { - return false; - } - } - return true; -} - JEMALLOC_ALWAYS_INLINE edata_t * emap_edata_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr) { EMAP_DECLARE_RTREE_CTX; diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 6a17ba60..b39e5ed5 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -51,4 +51,77 @@ bool extent_merge_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t extent_sn_next(pac_t *pac); bool extent_boot(void); +JEMALLOC_ALWAYS_INLINE bool +extent_neighbor_head_state_mergeable(bool edata_is_head, + bool neighbor_is_head, bool forward) { + /* + * Head states checking: disallow merging if the higher addr extent is a + * head extent. This helps preserve first-fit, and more importantly + * makes sure no merge across arenas. + */ + if (forward) { + if (neighbor_is_head) { + return false; + } + } else { + if (edata_is_head) { + return false; + } + } + return true; +} + +JEMALLOC_ALWAYS_INLINE bool +extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents, + extent_pai_t pai, extent_state_t expected_state, bool forward, + bool expanding) { + edata_t *neighbor = contents.edata; + if (neighbor == NULL) { + return false; + } + /* It's not safe to access *neighbor yet; must verify states first. */ + bool neighbor_is_head = contents.metadata.is_head; + if (!extent_neighbor_head_state_mergeable(edata_is_head_get(edata), + neighbor_is_head, forward)) { + return NULL; + } + extent_state_t neighbor_state = contents.metadata.state; + if (pai == EXTENT_PAI_PAC) { + if (neighbor_state != expected_state) { + return false; + } + /* From this point, it's safe to access *neighbor. */ + if (!expanding && (edata_committed_get(edata) != + edata_committed_get(neighbor))) { + /* + * Some platforms (e.g. Windows) require an explicit + * commit step (and writing to uncomitted memory is not + * allowed). + */ + return false; + } + } else { + if (neighbor_state == extent_state_active) { + return false; + } + /* From this point, it's safe to access *neighbor. */ + } + + assert(edata_pai_get(edata) == pai); + if (edata_pai_get(neighbor) != pai) { + return false; + } + if (opt_retain) { + assert(edata_arena_ind_get(edata) == + edata_arena_ind_get(neighbor)); + } else { + if (edata_arena_ind_get(edata) != + edata_arena_ind_get(neighbor)) { + return false; + } + } + + return true; +} + #endif /* JEMALLOC_INTERNAL_EXTENT_H */ diff --git a/src/ehooks.c b/src/ehooks.c index ca3ca209..535066e7 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -218,7 +218,7 @@ ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b) { bool head_b = edata_is_head_get(b); emap_assert_mapped(tsdn, &arena_emap_global, a); emap_assert_mapped(tsdn, &arena_emap_global, b); - assert(edata_neighbor_head_state_mergeable(head_a, head_b, + assert(extent_neighbor_head_state_mergeable(head_a, head_b, /* forward */ true)); } if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { diff --git a/src/emap.c b/src/emap.c index a1f402b8..1cc4fc81 100644 --- a/src/emap.c +++ b/src/emap.c @@ -48,64 +48,6 @@ emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata, emap_assert_mapped(tsdn, emap, edata); } -static inline bool -edata_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents, - extent_pai_t pai, extent_state_t expected_state, bool forward, - bool expanding) { - edata_t *neighbor = contents.edata; - if (neighbor == NULL) { - return false; - } - /* It's not safe to access *neighbor yet; must verify states first. */ - bool neighbor_is_head = contents.metadata.is_head; - if (!edata_neighbor_head_state_mergeable(edata_is_head_get(edata), - neighbor_is_head, forward)) { - return NULL; - } - extent_state_t neighbor_state = contents.metadata.state; - if (pai == EXTENT_PAI_PAC) { - if (neighbor_state != expected_state) { - return false; - } - /* From this point, it's safe to access *neighbor. */ - if (!expanding && (edata_committed_get(edata) != - edata_committed_get(neighbor))) { - /* - * Some platforms (e.g. Windows) require an explicit - * commit step (and writing to uncomitted memory is not - * allowed). - */ - return false; - } - } else { - if (neighbor_state == extent_state_active) { - return false; - } - /* From this point, it's safe to access *neighbor. */ - } - - assert(edata_pai_get(edata) == pai); - if (edata_pai_get(neighbor) != pai) { - return false; - } - if (opt_retain) { - assert(edata_arena_ind_get(edata) == - edata_arena_ind_get(neighbor)); - } else { - /* - * This isn't entirely safe with the presence of arena_reset / - * destroy, in which case the neighbor edata can be destoryed if - * it belongs to a manual arena. More on that later. - */ - if (edata_arena_ind_get(edata) != - edata_arena_ind_get(neighbor)) { - return false; - } - } - - return true; -} - static inline edata_t * emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_pai_t pai, extent_state_t expected_state, bool forward, @@ -142,7 +84,7 @@ emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata, rtree_contents_t neighbor_contents = rtree_leaf_elm_read(tsdn, &emap->rtree, elm, /* dependent */ true); - if (!edata_can_acquire_neighbor(edata, neighbor_contents, pai, + if (!extent_can_acquire_neighbor(edata, neighbor_contents, pai, expected_state, forward, expanding)) { return NULL; }