Fix bitmap_ffu() to work with 3+ levels.

include/jemalloc/internal/bitmap_inlines.h

@@ -83,52 +83,40 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
 	assert(min_bit < binfo->nbits);
 
 #ifdef BITMAP_USE_TREE
-	unsigned level = binfo->nlevels - 1;
+	size_t bit = 0;
-	size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS * (level+1));
+	for (unsigned level = binfo->nlevels; level--;) {
-	size_t bits_per_group = 1LU << lg_bits_per_group;
+		size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS * (level +
-	size_t bits_per_group_mask = bits_per_group - 1;
+		    1));
-	unsigned group_nmask = (min_bit & bits_per_group_mask) >> (level *
+		bitmap_t group = bitmap[binfo->levels[level].group_offset + (bit
-	    LG_BITMAP_GROUP_NBITS);
+		    >> lg_bits_per_group)];
+		unsigned group_nmask = ((min_bit > bit) ? (min_bit - bit) : 0)
+		    >> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS);
+		assert(group_nmask <= BITMAP_GROUP_NBITS);
 		bitmap_t group_mask = ~((1LU << group_nmask) - 1);
-	bitmap_t group = bitmap[binfo->levels[level].group_offset] & group_mask;
+		bitmap_t group_masked = group & group_mask;
+		if (group_masked == 0LU) {
 			if (group == 0LU) {
 				return binfo->nbits;
 			}
-	size_t bit = ffs_lu(group) - 1;
-
-	while (level > 0) {
-		level--;
-
-		lg_bits_per_group = (LG_BITMAP_GROUP_NBITS * (level+1));
-		bits_per_group = 1LU << lg_bits_per_group;
-		bits_per_group_mask = bits_per_group - 1;
-
-		group = bitmap[binfo->levels[level].group_offset + bit];
-		size_t cur_base = bit << lg_bits_per_group;
-		if (cur_base < min_bit) {
-			group_nmask = (min_bit & bits_per_group_mask) >> (level
-			    * LG_BITMAP_GROUP_NBITS);
-			group_mask = ~((1LU << group_nmask) - 1);
-			group &= group_mask;
-		}
-		if (group == 0LU) {
 			/*
-			 * If min_bit is not the first bit in its group, try
+			 * min_bit was preceded by one or more unset bits in
-			 * again starting at the first bit of the next group.
+			 * this group, but there are no other unset bits in this
-			 * This will only recurse at most once, since on
+			 * group.  Try again starting at the first bit of the
-			 * recursion, min_bit will be the first bit in its
+			 * next sibling.  This will recurse at most once per
-			 * group.
+			 * non-root level.
 			 */
-			size_t ceil_min_bit = (min_bit +
+			size_t sib_base = bit + (1U << lg_bits_per_group);
-			    BITMAP_GROUP_NBITS_MASK) & ~BITMAP_GROUP_NBITS_MASK;
+			assert(sib_base > min_bit);
-			if (ceil_min_bit != min_bit && ceil_min_bit <
+			assert(sib_base > bit);
-			    binfo->nbits) {
+			if (sib_base >= binfo->nbits) {
-				return bitmap_ffu(bitmap, binfo, ceil_min_bit);
-			}
 				return binfo->nbits;
 			}
-		bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffs_lu(group) - 1);
+			return bitmap_ffu(bitmap, binfo, sib_base);
 		}
+		bit += (ffs_lu(group_masked) - 1) << (lg_bits_per_group -
+		    LG_BITMAP_GROUP_NBITS);
+	}
+	assert(bit >= min_bit);
 	assert(bit < binfo->nbits);
 	return bit;
 #else

test/unit/bitmap.c

@@ -372,6 +372,33 @@ test_bitmap_xfu_body(const bitmap_info_t *binfo, size_t nbits) {
 		}
 	}
 
+	/*
+	 * Unset the last bit, bubble another unset bit through the bitmap, and
+	 * verify that bitmap_ffu() finds the correct bit for all four min_bit
+	 * cases.
+	 */
+	if (nbits >= 3) {
+		bitmap_unset(bitmap, binfo, nbits-1);
+		for (size_t i = 0; i < nbits-1; i++) {
+			bitmap_unset(bitmap, binfo, i);
+			if (i > 0) {
+				assert_zu_eq(bitmap_ffu(bitmap, binfo, i-1), i,
+				    "Unexpected first unset bit");
+			}
+			assert_zu_eq(bitmap_ffu(bitmap, binfo, i), i,
+			    "Unexpected first unset bit");
+			assert_zu_eq(bitmap_ffu(bitmap, binfo, i+1), nbits-1,
+			    "Unexpected first unset bit");
+			assert_zu_eq(bitmap_ffu(bitmap, binfo, nbits-1),
+			    nbits-1, "Unexpected first unset bit");
+
+			assert_zu_eq(bitmap_sfu(bitmap, binfo), i,
+			    "Unexpected first unset bit");
+		}
+		assert_zu_eq(bitmap_sfu(bitmap, binfo), nbits-1,
+		    "Unexpected first unset bit");
+	}
+
 	free(bitmap);
 }