Use a bitmap in extents_t to speed up search.

Rather than iteratively checking all sufficiently large heaps during
search, maintain and use a bitmap in order to skip empty heaps.

include/jemalloc/internal/bitmap_types.h

@@ -2,7 +2,13 @@
 #define JEMALLOC_INTERNAL_BITMAP_TYPES_H
 
 /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
+#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES
+/* Maximum bitmap bit count is determined by maximum regions per slab. */
 #  define LG_BITMAP_MAXBITS	LG_SLAB_MAXREGS
+#else
+/* Maximum bitmap bit count is determined by number of extent size classes. */
+#  define LG_BITMAP_MAXBITS	LG_CEIL_NSIZES
+#endif
 #define BITMAP_MAXBITS		(ZU(1) << LG_BITMAP_MAXBITS)
 
 typedef struct bitmap_level_s bitmap_level_t;

include/jemalloc/internal/extent_structs.h

@@ -102,6 +102,13 @@ struct extents_s {
 	 */
 	extent_heap_t		heaps[NPSIZES+1];
 
+	/*
+	 * Bitmap for which set bits correspond to non-empty heaps.
+	 *
+	 * Synchronization: mtx.
+	 */
+	bitmap_t		bitmap[BITMAP_GROUPS(NPSIZES+1)];
+
 	/*
 	 * LRU of all extents in heaps.
 	 *

src/extent.c

@@ -6,6 +6,9 @@
 
 rtree_t		extents_rtree;
 
+static const bitmap_info_t extents_bitmap_info =
+    BITMAP_INFO_INITIALIZER(NPSIZES+1);
+
 static void	*extent_alloc_default(extent_hooks_t *extent_hooks,
     void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
     unsigned arena_ind);
@@ -189,6 +192,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
 	for (unsigned i = 0; i < NPSIZES+1; i++) {
 		extent_heap_new(&extents->heaps[i]);
 	}
+	bitmap_init(extents->bitmap, &extents_bitmap_info, true);
 	extent_list_init(&extents->lru);
 	atomic_store_zu(&extents->npages, 0, ATOMIC_RELAXED);
 	extents->state = state;
@@ -215,6 +219,10 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent,
 	size_t size = extent_size_get(extent);
 	size_t psz = extent_size_quantize_floor(size);
 	pszind_t pind = psz2ind(psz);
+	if (extent_heap_empty(&extents->heaps[pind])) {
+		bitmap_unset(extents->bitmap, &extents_bitmap_info,
+		    (size_t)pind);
+	}
 	extent_heap_insert(&extents->heaps[pind], extent);
 	if (!preserve_lru) {
 		extent_list_append(&extents->lru, extent);
@@ -241,6 +249,10 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent,
 	size_t psz = extent_size_quantize_floor(size);
 	pszind_t pind = psz2ind(psz);
 	extent_heap_remove(&extents->heaps[pind], extent);
+	if (extent_heap_empty(&extents->heaps[pind])) {
+		bitmap_set(extents->bitmap, &extents_bitmap_info,
+		    (size_t)pind);
+	}
 	if (!preserve_lru) {
 		extent_list_remove(&extents->lru, extent);
 	}
@@ -261,13 +273,14 @@ static extent_t *
 extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
     size_t size) {
 	pszind_t pind = psz2ind(extent_size_quantize_ceil(size));
-	for (pszind_t i = pind; i < NPSIZES+1; i++) {
+	pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
+	    (size_t)pind);
+	if (i < NPSIZES+1) {
+		assert(!extent_heap_empty(&extents->heaps[i]));
 		extent_t *extent = extent_heap_any(&extents->heaps[i]);
-		if (extent != NULL) {
 		assert(extent_size_get(extent) >= size);
 		return extent;
 	}
-	}
 
 	return NULL;
 }
@@ -282,14 +295,20 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents,
 	extent_t *ret = NULL;
 
 	pszind_t pind = psz2ind(extent_size_quantize_ceil(size));
-	for (pszind_t i = pind; i < NPSIZES+1; i++) {
+	for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap,
+	    &extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i =
+	    (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info,
+	    (size_t)i+1)) {
+		assert(!extent_heap_empty(&extents->heaps[i]));
 		extent_t *extent = extent_heap_first(&extents->heaps[i]);
-		if (extent != NULL) {
 		assert(extent_size_get(extent) >= size);
 		if (ret == NULL || extent_snad_comp(extent, ret) < 0) {
 			ret = extent;
 		}
+		if (i == NPSIZES) {
+			break;
 		}
+		assert(i < NPSIZES);
 	}
 
 	return ret;