From a559caf74aa5421f608a59bd2d38da688b1f2572 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Wed, 2 Dec 2020 14:21:36 -0800 Subject: [PATCH] hpdata: Strengthen assertions. Now that we have flat bitmap bit counting functions, we can easily assert that nfree is always correct. While we're tightening up this code, enforce consistency on API boundaries as well. --- include/jemalloc/internal/hpdata.h | 11 +++++++++-- src/hpdata.c | 9 ++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/include/jemalloc/internal/hpdata.h b/include/jemalloc/internal/hpdata.h index e8433c53..fdd6673f 100644 --- a/include/jemalloc/internal/hpdata.h +++ b/include/jemalloc/internal/hpdata.h @@ -114,8 +114,15 @@ hpdata_assert_empty(hpdata_t *hpdata) { static inline bool hpdata_consistent(hpdata_t *hpdata) { - return fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES) - == hpdata_longest_free_range_get(hpdata); + if(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES) + != hpdata_longest_free_range_get(hpdata)) { + return false; + } + if (fb_ucount(hpdata->active_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES) + != hpdata_nfree_get(hpdata)) { + return false; + } + return true; } static inline void diff --git a/src/hpdata.c b/src/hpdata.c index 847eb9da..a242efea 100644 --- a/src/hpdata.c +++ b/src/hpdata.c @@ -17,7 +17,6 @@ hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) { ph_gen(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t, ph_link, hpdata_age_comp) - void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) { hpdata_addr_set(hpdata, addr); @@ -26,10 +25,13 @@ hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) { hpdata_nfree_set(hpdata, HUGEPAGE_PAGES); hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES); fb_init(hpdata->active_pages, HUGEPAGE_PAGES); + + hpdata_assert_consistent(hpdata); } void * hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) { + hpdata_assert_consistent(hpdata); assert((sz & PAGE_MASK) == 0); size_t npages = sz >> LG_PAGE; assert(npages <= hpdata_longest_free_range_get(hpdata)); @@ -93,12 +95,15 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) { } hpdata_longest_free_range_set(hpdata, largest_unchosen_range); + hpdata_assert_consistent(hpdata); return (void *)( (uintptr_t)hpdata_addr_get(hpdata) + (result << LG_PAGE)); } void hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) { + hpdata_assert_consistent(hpdata); + assert(((uintptr_t)addr & PAGE_MASK) == 0); assert((sz & PAGE_MASK) == 0); size_t begin = ((uintptr_t)addr - (uintptr_t)hpdata_addr_get(hpdata)) >> LG_PAGE; @@ -119,4 +124,6 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) { } hpdata_nfree_set(hpdata, hpdata_nfree_get(hpdata) + npages); + + hpdata_assert_consistent(hpdata); }