Use the flat bitmap for eset and psset bitmaps.

This is simpler (note that the eset field comment was actually incorrect!), and
slightly faster.

include/jemalloc/internal/eset.h

@@ -2,7 +2,7 @@
 #define JEMALLOC_INTERNAL_ESET_H
 
 #include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/bitmap.h"
+#include "jemalloc/internal/flat_bitmap.h"
 #include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
 
@@ -22,7 +22,7 @@ struct eset_s {
 	atomic_zu_t nbytes[SC_NPSIZES + 1];
 
 	/* Bitmap for which set bits correspond to non-empty heaps. */
-	bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
+	fb_group_t bitmap[FB_NGROUPS(SC_NPSIZES + 1)];
 
 	/* LRU of all extents in heaps. */
 	edata_list_inactive_t lru;

include/jemalloc/internal/psset.h

@@ -56,7 +56,8 @@ struct psset_s {
 	 * free run of pages in a pageslab.
 	 */
 	hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
-	bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
+	/* Bitmap for which set bits correspond to non-empty heaps. */
+	fb_group_t bitmap[FB_NGROUPS(PSSET_NPSIZES)];
 	/*
 	 * The sum of all bin stats in stats.  This lets us quickly answer
 	 * queries for the number of dirty, active, and retained pages in the

src/eset.c

@@ -3,15 +3,14 @@
 
 #include "jemalloc/internal/eset.h"
 
-const bitmap_info_t eset_bitmap_info =
+#define ESET_NPSIZES (SC_NPSIZES + 1)
-    BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);
 
 void
 eset_init(eset_t *eset, extent_state_t state) {
-	for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
+	for (unsigned i = 0; i < ESET_NPSIZES; i++) {
 		edata_heap_new(&eset->heaps[i]);
 	}
-	bitmap_init(eset->bitmap, &eset_bitmap_info, true);
+	fb_init(eset->bitmap, ESET_NPSIZES);
 	edata_list_inactive_init(&eset->lru);
 	atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
 	eset->state = state;
@@ -56,8 +55,7 @@ eset_insert(eset_t *eset, edata_t *edata) {
 	size_t psz = sz_psz_quantize_floor(size);
 	pszind_t pind = sz_psz2ind(psz);
 	if (edata_heap_empty(&eset->heaps[pind])) {
-		bitmap_unset(eset->bitmap, &eset_bitmap_info,
+		fb_set(eset->bitmap, ESET_NPSIZES, (size_t)pind);
-		    (size_t)pind);
 	}
 	edata_heap_insert(&eset->heaps[pind], edata);
 
@@ -92,8 +90,7 @@ eset_remove(eset_t *eset, edata_t *edata) {
 	}
 
 	if (edata_heap_empty(&eset->heaps[pind])) {
-		bitmap_set(eset->bitmap, &eset_bitmap_info,
+		fb_unset(eset->bitmap, ESET_NPSIZES, (size_t)pind);
-		    (size_t)pind);
 	}
 	edata_list_inactive_remove(&eset->lru, edata);
 	size_t npages = size >> LG_PAGE;
@@ -122,10 +119,10 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size,
         pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size));
         pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size));
 
-	for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
+	for (pszind_t i =
-	    &eset_bitmap_info, (size_t)pind); i < pind_max; i =
+	    (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
-	    (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
+	    i < pind_max;
-	    (size_t)i+1)) {
+	    i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
 		assert(i < SC_NPSIZES);
 		assert(!edata_heap_empty(&eset->heaps[i]));
 		edata_t *edata = edata_heap_first(&eset->heaps[i]);
@@ -171,11 +168,10 @@ eset_first_fit(eset_t *eset, size_t size, bool exact_only,
 		    edata_heap_first(&eset->heaps[pind]);
 	}
 
-	for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
+	for (pszind_t i =
-	    &eset_bitmap_info, (size_t)pind);
+	    (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
-	    i < SC_NPSIZES + 1;
+	    i < ESET_NPSIZES;
-	    i = (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
+	    i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
-	    (size_t)i+1)) {
 		assert(!edata_heap_empty(&eset->heaps[i]));
 		edata_t *edata = edata_heap_first(&eset->heaps[i]);
 		assert(edata_size_get(edata) >= size);

src/psset.c

@@ -5,15 +5,12 @@
 
 #include "jemalloc/internal/flat_bitmap.h"
 
-static const bitmap_info_t psset_bitmap_info =
-    BITMAP_INFO_INITIALIZER(PSSET_NPSIZES);
-
 void
 psset_init(psset_t *psset) {
 	for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
 		hpdata_age_heap_new(&psset->pageslabs[i]);
 	}
-	bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
+	fb_init(psset->bitmap, PSSET_NPSIZES);
 	memset(&psset->merged_stats, 0, sizeof(psset->merged_stats));
 	memset(&psset->stats, 0, sizeof(psset->stats));
 	hpdata_empty_list_init(&psset->empty);
@@ -101,14 +98,14 @@ static void
 psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
 	hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
 	if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
-		bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
+		fb_unset(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
 	}
 }
 
 static void
 psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
 	if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
-		bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
+		fb_set(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
 	}
 	hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
 }
@@ -266,7 +263,7 @@ psset_pick_alloc(psset_t *psset, size_t size) {
 	assert(size <= HUGEPAGE);
 
 	pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
-	pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
+	pszind_t pind = (pszind_t)fb_ffs(psset->bitmap, PSSET_NPSIZES,
 	    (size_t)min_pind);
 	if (pind == PSSET_NPSIZES) {
 		return hpdata_empty_list_first(&psset->empty);