diff --git a/include/jemalloc/internal/hpdata.h b/include/jemalloc/internal/hpdata.h index 7cefb5cc..5952a18f 100644 --- a/include/jemalloc/internal/hpdata.h +++ b/include/jemalloc/internal/hpdata.h @@ -52,6 +52,16 @@ struct hpdata_s { /* A bitmap with bits set in the active pages. */ fb_group_t active_pages[FB_NGROUPS(HUGEPAGE_PAGES)]; + + /* + * Number of dirty pages, and a bitmap tracking them. This really means + * "dirty" from the OS's point of view; it includes both active and + * inactive pages that have been touched by the user. + */ + size_t h_ndirty; + + /* The dirty pages (using the same definition as above). */ + fb_group_t dirty_pages[FB_NGROUPS(HUGEPAGE_PAGES)]; }; static inline void * @@ -80,11 +90,6 @@ hpdata_huge_get(const hpdata_t *hpdata) { return hpdata->h_huge; } -static inline void -hpdata_huge_set(hpdata_t *hpdata, bool huge) { - hpdata->h_huge = huge; -} - static inline size_t hpdata_longest_free_range_get(const hpdata_t *hpdata) { return hpdata->h_longest_free_range; @@ -122,6 +127,16 @@ hpdata_consistent(hpdata_t *hpdata) { != hpdata->h_nactive) { return false; } + if (fb_scount(hpdata->dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES) + != hpdata->h_ndirty) { + return false; + } + if (hpdata->h_ndirty < hpdata->h_nactive) { + return false; + } + if (hpdata->h_huge && hpdata->h_ndirty != HUGEPAGE_PAGES) { + return false; + } return true; } @@ -141,6 +156,7 @@ hpdata_empty(hpdata_t *hpdata) { } void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age); + /* * Given an hpdata which can serve an allocation request, pick and reserve an * offset within that allocation. @@ -148,4 +164,20 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age); void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz); void hpdata_unreserve(hpdata_t *hpdata, void *begin, size_t sz); +/* + * Tell the hpdata that it's now a hugepage (which, correspondingly, means that + * all its pages become dirty. + */ +void hpdata_hugify(hpdata_t *hpdata); +/* + * Tell the hpdata that it's no longer a hugepage (all its pages are still + * counted as dirty, though; an explicit purge call is required to change that). + */ +void hpdata_dehugify(hpdata_t *hpdata); +/* + * Tell the hpdata (which should be empty) that all dirty pages in it have been + * purged. + */ +void hpdata_purge(hpdata_t *hpdata); + #endif /* JEMALLOC_INTERNAL_HPDATA_H */ diff --git a/src/hpa.c b/src/hpa.c index 8bbe8a87..75636047 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -146,7 +146,10 @@ hpa_dehugify(hpdata_t *ps) { /* Purge, then dehugify while unbacked. */ pages_purge_forced(hpdata_addr_get(ps), HUGEPAGE); pages_nohuge(hpdata_addr_get(ps), HUGEPAGE); - hpdata_huge_set(ps, false); + + /* Update metadata. */ + hpdata_dehugify(ps); + hpdata_purge(ps); } static hpdata_t * @@ -297,7 +300,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) bool hugify = hpa_should_hugify(shard, ps); if (hugify) { - hpdata_huge_set(ps, true); + hpdata_hugify(ps); } psset_insert(&shard->psset, ps); diff --git a/src/hpdata.c b/src/hpdata.c index d513896a..8297158e 100644 --- a/src/hpdata.c +++ b/src/hpdata.c @@ -21,10 +21,12 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) { hpdata_addr_set(hpdata, addr); hpdata_age_set(hpdata, age); - hpdata_huge_set(hpdata, false); - hpdata->h_nactive = 0; + hpdata->h_huge = false; hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES); + hpdata->h_nactive = 0; fb_init(hpdata->active_pages, HUGEPAGE_PAGES); + hpdata->h_ndirty = 0; + fb_init(hpdata->dirty_pages, HUGEPAGE_PAGES); hpdata_assert_consistent(hpdata); } @@ -74,6 +76,15 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) { fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages); hpdata->h_nactive += npages; + /* + * We might be about to dirty some memory for the first time; update our + * count if so. + */ + size_t new_dirty = fb_ucount(hpdata->dirty_pages, HUGEPAGE_PAGES, + result, npages); + fb_set_range(hpdata->dirty_pages, HUGEPAGE_PAGES, result, npages); + hpdata->h_ndirty += new_dirty; + /* * We might have shrunk the longest free range. We have to keep * scanning until the end of the hpdata to be sure. @@ -127,3 +138,33 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) { hpdata_assert_consistent(hpdata); } + +void +hpdata_hugify(hpdata_t *hpdata) { + hpdata_assert_consistent(hpdata); + hpdata->h_huge = true; + fb_set_range(hpdata->dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES); + hpdata->h_ndirty = HUGEPAGE_PAGES; + hpdata_assert_consistent(hpdata); +} + +void +hpdata_dehugify(hpdata_t *hpdata) { + hpdata_assert_consistent(hpdata); + hpdata->h_huge = false; + hpdata_assert_consistent(hpdata); +} + +void +hpdata_purge(hpdata_t *hpdata) { + hpdata_assert_consistent(hpdata); + /* + * The hpdata must be empty; we don't (yet) support partial purges of + * hugepages. + */ + assert(hpdata->h_nactive == 0); + fb_unset_range(hpdata->dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES); + fb_init(hpdata->dirty_pages, HUGEPAGE_PAGES); + hpdata->h_ndirty = 0; + hpdata_assert_consistent(hpdata); +}