diff --git a/include/jemalloc/internal/hpdata.h b/include/jemalloc/internal/hpdata.h index 65cd073f..7cefb5cc 100644 --- a/include/jemalloc/internal/hpdata.h +++ b/include/jemalloc/internal/hpdata.h @@ -44,11 +44,12 @@ struct hpdata_s { ql_elm(hpdata_t) ql_link; }; - /* Number of currently free pages (regardless of contiguity). */ - size_t h_nfree; /* The length of the largest contiguous sequence of inactive pages. */ size_t h_longest_free_range; + /* Number of active pages. */ + size_t h_nactive; + /* A bitmap with bits set in the active pages. */ fb_group_t active_pages[FB_NGROUPS(HUGEPAGE_PAGES)]; }; @@ -84,17 +85,6 @@ hpdata_huge_set(hpdata_t *hpdata, bool huge) { hpdata->h_huge = huge; } -static inline size_t -hpdata_nfree_get(const hpdata_t *hpdata) { - return hpdata->h_nfree; -} - -static inline void -hpdata_nfree_set(hpdata_t *hpdata, size_t nfree) { - assert(nfree <= HUGEPAGE_PAGES); - hpdata->h_nfree = nfree; -} - static inline size_t hpdata_longest_free_range_get(const hpdata_t *hpdata) { return hpdata->h_longest_free_range; @@ -106,10 +96,15 @@ hpdata_longest_free_range_set(hpdata_t *hpdata, size_t longest_free_range) { hpdata->h_longest_free_range = longest_free_range; } +static inline size_t +hpdata_nactive_get(hpdata_t *hpdata) { + return hpdata->h_nactive; +} + static inline void hpdata_assert_empty(hpdata_t *hpdata) { assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES)); - assert(hpdata_nfree_get(hpdata) == HUGEPAGE_PAGES); + assert(hpdata->h_nactive == 0); } /* @@ -123,8 +118,8 @@ hpdata_consistent(hpdata_t *hpdata) { != hpdata_longest_free_range_get(hpdata)) { return false; } - if (fb_ucount(hpdata->active_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES) - != hpdata_nfree_get(hpdata)) { + if (fb_scount(hpdata->active_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES) + != hpdata->h_nactive) { return false; } return true; @@ -142,7 +137,7 @@ ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t); static inline bool hpdata_empty(hpdata_t *hpdata) { - return hpdata_nfree_get(hpdata) == HUGEPAGE_PAGES; + return hpdata->h_nactive == 0; } void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age); diff --git a/src/hpa.c b/src/hpa.c index a51f83ce..8bbe8a87 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -125,7 +125,7 @@ hpa_should_hugify(hpa_shard_t *shard, hpdata_t *ps) { * inactive. Eventually, this should be a malloc conf option. */ return !hpdata_huge_get(ps) - && hpdata_nfree_get(ps) < (HUGEPAGE / PAGE) * 5 / 100; + && hpdata_nactive_get(ps) >= (HUGEPAGE_PAGES) * 95 / 100; } /* Returns true on error. */ diff --git a/src/hpdata.c b/src/hpdata.c index a242efea..d513896a 100644 --- a/src/hpdata.c +++ b/src/hpdata.c @@ -22,7 +22,7 @@ hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) { hpdata_addr_set(hpdata, addr); hpdata_age_set(hpdata, age); hpdata_huge_set(hpdata, false); - hpdata_nfree_set(hpdata, HUGEPAGE_PAGES); + hpdata->h_nactive = 0; hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES); fb_init(hpdata->active_pages, HUGEPAGE_PAGES); @@ -72,7 +72,7 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) { /* We found a range; remember it. */ result = begin; fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages); - hpdata_nfree_set(hpdata, hpdata_nfree_get(hpdata) - npages); + hpdata->h_nactive += npages; /* * We might have shrunk the longest free range. We have to keep @@ -123,7 +123,7 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) { hpdata_longest_free_range_set(hpdata, new_range_len); } - hpdata_nfree_set(hpdata, hpdata_nfree_get(hpdata) + npages); + hpdata->h_nactive -= npages; hpdata_assert_consistent(hpdata); } diff --git a/src/psset.c b/src/psset.c index 7a5bd604..9fcdac22 100644 --- a/src/psset.c +++ b/src/psset.c @@ -57,8 +57,8 @@ psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps, size_t *ninactive_dst = hpdata_huge_get(ps) ? &binstats->ninactive_huge : &binstats->ninactive_nonhuge; - size_t ninactive = hpdata_nfree_get(ps); - size_t nactive = HUGEPAGE_PAGES - ninactive; + size_t nactive = hpdata_nactive_get(ps); + size_t ninactive = HUGEPAGE_PAGES - nactive; size_t mul = insert ? (size_t)1 : (size_t)-1; *npageslabs_dst += mul * 1;