hpdata: track per-page dirty state.

include/jemalloc/internal/hpdata.h

@@ -52,6 +52,16 @@ struct hpdata_s {
 
 	/* A bitmap with bits set in the active pages. */
 	fb_group_t active_pages[FB_NGROUPS(HUGEPAGE_PAGES)];
+
+	/*
+	 * Number of dirty pages, and a bitmap tracking them.  This really means
+	 * "dirty" from the OS's point of view; it includes both active and
+	 * inactive pages that have been touched by the user.
+	 */
+	size_t h_ndirty;
+
+	/* The dirty pages (using the same definition as above). */
+	fb_group_t dirty_pages[FB_NGROUPS(HUGEPAGE_PAGES)];
 };
 
 static inline void *
@@ -80,11 +90,6 @@ hpdata_huge_get(const hpdata_t *hpdata) {
 	return hpdata->h_huge;
 }
 
-static inline void
-hpdata_huge_set(hpdata_t *hpdata, bool huge) {
-	hpdata->h_huge = huge;
-}
-
 static inline size_t
 hpdata_longest_free_range_get(const hpdata_t *hpdata) {
 	return hpdata->h_longest_free_range;
@@ -122,6 +127,16 @@ hpdata_consistent(hpdata_t *hpdata) {
 	    != hpdata->h_nactive) {
 		return false;
 	}
+	if (fb_scount(hpdata->dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES)
+	    != hpdata->h_ndirty) {
+		return false;
+	}
+	if (hpdata->h_ndirty < hpdata->h_nactive) {
+		return false;
+	}
+	if (hpdata->h_huge && hpdata->h_ndirty != HUGEPAGE_PAGES) {
+		return false;
+	}
 	return true;
 }
 
@@ -141,6 +156,7 @@ hpdata_empty(hpdata_t *hpdata) {
 }
 
 void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
+
 /*
  * Given an hpdata which can serve an allocation request, pick and reserve an
  * offset within that allocation.
@@ -148,4 +164,20 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
 void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void hpdata_unreserve(hpdata_t *hpdata, void *begin, size_t sz);
 
+/*
+ * Tell the hpdata that it's now a hugepage (which, correspondingly, means that
+ * all its pages become dirty.
+ */
+void hpdata_hugify(hpdata_t *hpdata);
+/*
+ * Tell the hpdata that it's no longer a hugepage (all its pages are still
+ * counted as dirty, though; an explicit purge call is required to change that).
+ */
+void hpdata_dehugify(hpdata_t *hpdata);
+/*
+ * Tell the hpdata (which should be empty) that all dirty pages in it have been
+ * purged.
+ */
+void hpdata_purge(hpdata_t *hpdata);
+
 #endif /* JEMALLOC_INTERNAL_HPDATA_H */

src/hpa.c

@@ -146,7 +146,10 @@ hpa_dehugify(hpdata_t *ps) {
 	/* Purge, then dehugify while unbacked. */
 	pages_purge_forced(hpdata_addr_get(ps), HUGEPAGE);
 	pages_nohuge(hpdata_addr_get(ps), HUGEPAGE);
-	hpdata_huge_set(ps, false);
+
+	/* Update metadata. */
+	hpdata_dehugify(ps);
+	hpdata_purge(ps);
 }
 
 static hpdata_t *
@@ -297,7 +300,7 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom)
 
 	bool hugify = hpa_should_hugify(shard, ps);
 	if (hugify) {
-		hpdata_huge_set(ps, true);
+		hpdata_hugify(ps);
 	}
 	psset_insert(&shard->psset, ps);
 

src/hpdata.c

@@ -21,10 +21,12 @@ void
 hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
 	hpdata_addr_set(hpdata, addr);
 	hpdata_age_set(hpdata, age);
-	hpdata_huge_set(hpdata, false);
+	hpdata->h_huge = false;
-	hpdata->h_nactive = 0;
 	hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
+	hpdata->h_nactive = 0;
 	fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
+	hpdata->h_ndirty = 0;
+	fb_init(hpdata->dirty_pages, HUGEPAGE_PAGES);
 
 	hpdata_assert_consistent(hpdata);
 }
@@ -74,6 +76,15 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
 	fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages);
 	hpdata->h_nactive += npages;
 
+	/*
+	 * We might be about to dirty some memory for the first time; update our
+	 * count if so.
+	 */
+	size_t new_dirty = fb_ucount(hpdata->dirty_pages,  HUGEPAGE_PAGES,
+	    result, npages);
+	fb_set_range(hpdata->dirty_pages, HUGEPAGE_PAGES, result, npages);
+	hpdata->h_ndirty += new_dirty;
+
 	/*
 	 * We might have shrunk the longest free range.  We have to keep
 	 * scanning until the end of the hpdata to be sure.
@@ -127,3 +138,33 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
 
 	hpdata_assert_consistent(hpdata);
 }
+
+void
+hpdata_hugify(hpdata_t *hpdata) {
+	hpdata_assert_consistent(hpdata);
+	hpdata->h_huge = true;
+	fb_set_range(hpdata->dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES);
+	hpdata->h_ndirty = HUGEPAGE_PAGES;
+	hpdata_assert_consistent(hpdata);
+}
+
+void
+hpdata_dehugify(hpdata_t *hpdata) {
+	hpdata_assert_consistent(hpdata);
+	hpdata->h_huge = false;
+	hpdata_assert_consistent(hpdata);
+}
+
+void
+hpdata_purge(hpdata_t *hpdata) {
+	hpdata_assert_consistent(hpdata);
+	/*
+	 * The hpdata must be empty; we don't (yet) support partial purges of
+	 * hugepages.
+	 */
+	assert(hpdata->h_nactive == 0);
+	fb_unset_range(hpdata->dirty_pages, HUGEPAGE_PAGES, 0, HUGEPAGE_PAGES);
+	fb_init(hpdata->dirty_pages, HUGEPAGE_PAGES);
+	hpdata->h_ndirty = 0;
+	hpdata_assert_consistent(hpdata);
+}