Refactor prng* from cpp macros into inline functions.

Remove 32-bit variant, convert prng64() to prng_lg_range(), and add
prng_range().

Makefile.in

@@ -84,9 +84,10 @@ C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \
 	$(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \
 	$(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \
 	$(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/pages.c \
-	$(srcroot)src/prof.c $(srcroot)src/quarantine.c $(srcroot)src/rtree.c \
+	$(srcroot)src/prng.c $(srcroot)src/prof.c $(srcroot)src/quarantine.c \
-	$(srcroot)src/stats.c $(srcroot)src/tcache.c $(srcroot)src/ticker.c \
+	$(srcroot)src/rtree.c $(srcroot)src/stats.c $(srcroot)src/tcache.c \
-	$(srcroot)src/time.c $(srcroot)src/tsd.c $(srcroot)src/util.c
+	$(srcroot)src/ticker.c $(srcroot)src/time.c $(srcroot)src/tsd.c \
+	$(srcroot)src/util.c
 ifeq ($(enable_valgrind), 1)
 C_SRCS += $(srcroot)src/valgrind.c
 endif
@@ -129,6 +130,7 @@ TESTS_UNIT := $(srcroot)test/unit/atomic.c \
 	$(srcroot)test/unit/math.c \
 	$(srcroot)test/unit/mq.c \
 	$(srcroot)test/unit/mtx.c \
+	$(srcroot)test/unit/prng.c \
 	$(srcroot)test/unit/prof_accum.c \
 	$(srcroot)test/unit/prof_active.c \
 	$(srcroot)test/unit/prof_gdump.c \

include/jemalloc/internal/ckh.h

@@ -40,9 +40,7 @@ struct ckh_s {
 #endif
 
 	/* Used for pseudo-random number generation. */
-#define	CKH_A		1103515241
+	uint64_t	prng_state;
-#define	CKH_C		12347
-	uint32_t	prng_state;
 
 	/* Total number of items. */
 	size_t		count;

include/jemalloc/internal/jemalloc_internal.h.in

@@ -547,7 +547,7 @@ size2index_compute(size_t size)
 #if (NTBINS != 0)
 	if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
 		size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
-		size_t lg_ceil = lg_floor(pow2_ceil(size));
+		size_t lg_ceil = lg_floor(pow2_ceil_zu(size));
 		return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin);
 	}
 #endif
@@ -644,7 +644,7 @@ s2u_compute(size_t size)
 #if (NTBINS > 0)
 	if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
 		size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
-		size_t lg_ceil = lg_floor(pow2_ceil(size));
+		size_t lg_ceil = lg_floor(pow2_ceil_zu(size));
 		return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) :
 		    (ZU(1) << lg_ceil));
 	}

include/jemalloc/internal/private_symbols.txt

@@ -348,7 +348,9 @@ pages_map
 pages_purge
 pages_trim
 pages_unmap
-pow2_ceil
+pow2_ceil_u32
+pow2_ceil_u64
+pow2_ceil_zu
 prof_active_get
 prof_active_get_unlocked
 prof_active_set

include/jemalloc/internal/prng.h

@@ -18,31 +18,9 @@
  * proportional to bit position.  For example, the lowest bit has a cycle of 2,
  * the next has a cycle of 4, etc.  For this reason, we prefer to use the upper
  * bits.
- *
- * Macro parameters:
- *   uint32_t r          : Result.
- *   unsigned lg_range   : (0..32], number of least significant bits to return.
- *   uint32_t state      : Seed value.
- *   const uint32_t a, c : See above discussion.
  */
-#define	prng32(r, lg_range, state, a, c) do {				\
+#define	PRNG_A	UINT64_C(6364136223846793005)
-	assert((lg_range) > 0);						\
+#define	PRNG_C	UINT64_C(1442695040888963407)
-	assert((lg_range) <= 32);					\
-									\
-	r = (state * (a)) + (c);					\
-	state = r;							\
-	r >>= (32 - (lg_range));					\
-} while (false)
-
-/* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */
-#define	prng64(r, lg_range, state, a, c) do {				\
-	assert((lg_range) > 0);						\
-	assert((lg_range) <= 64);					\
-									\
-	r = (state * (a)) + (c);					\
-	state = r;							\
-	r >>= (64 - (lg_range));					\
-} while (false)
 
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
@@ -56,5 +34,46 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
+#ifndef JEMALLOC_ENABLE_INLINE
+uint64_t	prng_lg_range(uint64_t *state, unsigned lg_range);
+uint64_t	prng_range(uint64_t *state, uint64_t range);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_))
+JEMALLOC_ALWAYS_INLINE uint64_t
+prng_lg_range(uint64_t *state, unsigned lg_range)
+{
+	uint64_t ret;
+
+	assert(lg_range > 0);
+	assert(lg_range <= 64);
+
+	ret = (*state * PRNG_A) + PRNG_C;
+	*state = ret;
+	ret >>= (64 - lg_range);
+
+	return (ret);
+}
+
+JEMALLOC_ALWAYS_INLINE uint64_t
+prng_range(uint64_t *state, uint64_t range)
+{
+	uint64_t ret;
+	unsigned lg_range;
+
+	assert(range > 1);
+
+	/* Compute the ceiling of lg(range). */
+	lg_range = jemalloc_ffsl(pow2_ceil_u64(range)) - 1;
+
+	/* Generate a result in [0..range) via repeated trial. */
+	do {
+		ret = prng_lg_range(state, lg_range);
+	} while (ret >= range);
+
+	return (ret);
+}
+#endif
+
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/

include/jemalloc/internal/util.h

@@ -123,7 +123,9 @@ void	malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
 #ifndef JEMALLOC_ENABLE_INLINE
 int	jemalloc_ffsl(long bitmap);
 int	jemalloc_ffs(int bitmap);
-size_t	pow2_ceil(size_t x);
+uint64_t	pow2_ceil_u64(uint64_t x);
+uint32_t	pow2_ceil_u32(uint32_t x);
+size_t	pow2_ceil_zu(size_t x);
 size_t	lg_floor(size_t x);
 void	set_errno(int errnum);
 int	get_errno(void);
@@ -150,9 +152,8 @@ jemalloc_ffs(int bitmap)
 	return (JEMALLOC_INTERNAL_FFS(bitmap));
 }
 
-/* Compute the smallest power of 2 that is >= x. */
+JEMALLOC_INLINE uint64_t
-JEMALLOC_INLINE size_t
+pow2_ceil_u64(uint64_t x)
-pow2_ceil(size_t x)
 {
 
 	x--;
@@ -161,13 +162,37 @@ pow2_ceil(size_t x)
 	x |= x >> 4;
 	x |= x >> 8;
 	x |= x >> 16;
-#if (LG_SIZEOF_PTR == 3)
 	x |= x >> 32;
-#endif
 	x++;
 	return (x);
 }
 
+JEMALLOC_INLINE uint32_t
+pow2_ceil_u32(uint32_t x)
+{
+
+	x--;
+	x |= x >> 1;
+	x |= x >> 2;
+	x |= x >> 4;
+	x |= x >> 8;
+	x |= x >> 16;
+	x++;
+	return (x);
+}
+
+/* Compute the smallest power of 2 that is >= x. */
+JEMALLOC_INLINE size_t
+pow2_ceil_zu(size_t x)
+{
+
+#if (LG_SIZEOF_PTR == 3)
+	return (pow2_ceil_u64(x));
+#else
+	return (pow2_ceil_u32(x));
+#endif
+}
+
 #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
 JEMALLOC_INLINE size_t
 lg_floor(size_t x)

src/arena.c

@@ -2196,9 +2196,7 @@ arena_malloc_large(arena_t *arena, size_t size, szind_t binind, bool zero)
 		 * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64
 		 * for 4 KiB pages and 64-byte cachelines.
 		 */
-		prng64(r, LG_PAGE - LG_CACHELINE, arena->offset_state,
+		r = prng_lg_range(&arena->offset_state, LG_PAGE - LG_CACHELINE);
-		    UINT64_C(6364136223846793009),
-		    UINT64_C(1442695040888963409));
 		random_offset = ((uintptr_t)r) << LG_CACHELINE;
 	} else
 		random_offset = 0;

src/ckh.c

@@ -99,7 +99,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
 	 * Cycle through the cells in the bucket, starting at a random position.
 	 * The randomness avoids worst-case search overhead as buckets fill up.
 	 */
-	prng32(offset, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C);
+	offset = prng_lg_range(&ckh->prng_state, LG_CKH_BUCKET_CELLS);
 	for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
 		cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) +
 		    ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))];
@@ -141,7 +141,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 		 * were an item for which both hashes indicated the same
 		 * bucket.
 		 */
-		prng32(i, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C);
+		i = prng_lg_range(&ckh->prng_state, LG_CKH_BUCKET_CELLS);
 		cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
 		assert(cell->key != NULL);
 

src/prng.c

@@ -0,0 +1,2 @@
+#define	JEMALLOC_PRNG_C_
+#include "jemalloc/internal/jemalloc_internal.h"

src/prof.c

@@ -871,8 +871,7 @@ prof_sample_threshold_update(prof_tdata_t *tdata)
 	 *   pp 500
 	 *   (http://luc.devroye.org/rnbookindex.html)
 	 */
-	prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005),
+	r = prng_lg_range(&tdata->prng_state, 53);
-	    UINT64_C(1442695040888963407));
 	u = (double)r * (1.0/9007199254740992.0L);
 	tdata->bytes_until_sample = (uint64_t)(log(u) /
 	    log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))

test/unit/prng.c

@@ -0,0 +1,68 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_prng_lg_range)
+{
+	uint64_t sa, sb, ra, rb;
+	unsigned lg_range;
+
+	sa = 42;
+	ra = prng_lg_range(&sa, 64);
+	sa = 42;
+	rb = prng_lg_range(&sa, 64);
+	assert_u64_eq(ra, rb,
+	    "Repeated generation should produce repeated results");
+
+	sb = 42;
+	rb = prng_lg_range(&sb, 64);
+	assert_u64_eq(ra, rb,
+	    "Equivalent generation should produce equivalent results");
+
+	sa = 42;
+	ra = prng_lg_range(&sa, 64);
+	rb = prng_lg_range(&sa, 64);
+	assert_u64_ne(ra, rb,
+	    "Full-width results must not immediately repeat");
+
+	sa = 42;
+	ra = prng_lg_range(&sa, 64);
+	for (lg_range = 63; lg_range > 0; lg_range--) {
+		sb = 42;
+		rb = prng_lg_range(&sb, lg_range);
+		assert_u64_eq((rb & (UINT64_C(0xffffffffffffffff) << lg_range)),
+		    0, "High order bits should be 0, lg_range=%u", lg_range);
+		assert_u64_eq(rb, (ra >> (64 - lg_range)),
+		    "Expected high order bits of full-width result, "
+		    "lg_range=%u", lg_range);
+	}
+}
+TEST_END
+
+TEST_BEGIN(test_prng_range)
+{
+	uint64_t range;
+#define	MAX_RANGE	10000000
+#define	RANGE_STEP	97
+#define	NREPS		10
+
+	for (range = 2; range < MAX_RANGE; range += RANGE_STEP) {
+		uint64_t s;
+		unsigned rep;
+
+		s = range;
+		for (rep = 0; rep < NREPS; rep++) {
+			uint64_t r = prng_range(&s, range);
+
+			assert_u64_lt(r, range, "Out of range");
+		}
+	}
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_prng_lg_range,
+	    test_prng_range));
+}

test/unit/util.c

@@ -1,33 +1,54 @@
 #include "test/jemalloc_test.h"
 
-TEST_BEGIN(test_pow2_ceil)
+#define	TEST_POW2_CEIL(t, suf, pri) do {				\
+	unsigned i, pow2;						\
+	t x;								\
+									\
+	assert_zu_eq(pow2_ceil_##suf(0), 0, "Unexpected result");	\
+									\
+	for (i = 0; i < sizeof(t) * 8; i++) {				\
+		assert_zu_eq(pow2_ceil_##suf(((t)1) << i), ((t)1) << i,	\
+		    "Unexpected result");				\
+	}								\
+									\
+	for (i = 2; i < sizeof(t) * 8; i++) {				\
+		assert_zu_eq(pow2_ceil_##suf((((t)1) << i) - 1),	\
+		    ((t)1) << i, "Unexpected result");			\
+	}								\
+									\
+	for (i = 0; i < sizeof(t) * 8 - 1; i++) {			\
+		assert_zu_eq(pow2_ceil_##suf((((t)1) << i) + 1),	\
+		    ((t)1) << (i+1), "Unexpected result");		\
+	}								\
+									\
+	for (pow2 = 1; pow2 < 25; pow2++) {				\
+		for (x = (((t)1) << (pow2-1)) + 1; x <= ((t)1) << pow2;	\
+		    x++) {						\
+			assert_zu_eq(pow2_ceil_##suf(x),		\
+			    ((t)1) << pow2,				\
+			    "Unexpected result, x=%"pri, x);		\
+		}							\
+	}								\
+} while (0)
+
+TEST_BEGIN(test_pow2_ceil_u64)
 {
-	unsigned i, pow2;
-	size_t x;
 
-	assert_zu_eq(pow2_ceil(0), 0, "Unexpected result");
+	TEST_POW2_CEIL(uint64_t, u64, FMTu64);
+}
+TEST_END
 
-	for (i = 0; i < sizeof(size_t) * 8; i++) {
+TEST_BEGIN(test_pow2_ceil_u32)
-		assert_zu_eq(pow2_ceil(ZU(1) << i), ZU(1) << i,
+{
-		    "Unexpected result");
-	}
 
-	for (i = 2; i < sizeof(size_t) * 8; i++) {
+	TEST_POW2_CEIL(uint32_t, u32, FMTu32);
-		assert_zu_eq(pow2_ceil((ZU(1) << i) - 1), ZU(1) << i,
+}
-		    "Unexpected result");
+TEST_END
-	}
 
-	for (i = 0; i < sizeof(size_t) * 8 - 1; i++) {
+TEST_BEGIN(test_pow2_ceil_zu)
-		assert_zu_eq(pow2_ceil((ZU(1) << i) + 1), ZU(1) << (i+1),
+{
-		    "Unexpected result");
-	}
 
-	for (pow2 = 1; pow2 < 25; pow2++) {
+	TEST_POW2_CEIL(size_t, zu, "zu");
-		for (x = (ZU(1) << (pow2-1)) + 1; x <= ZU(1) << pow2; x++) {
-			assert_zu_eq(pow2_ceil(x), ZU(1) << pow2,
-			    "Unexpected result, x=%zu", x);
-		}
-	}
 }
 TEST_END
 
@@ -286,7 +307,9 @@ main(void)
 {
 
 	return (test(
-	    test_pow2_ceil,
+	    test_pow2_ceil_u64,
+	    test_pow2_ceil_u32,
+	    test_pow2_ceil_zu,
 	    test_malloc_strtoumax_no_endptr,
 	    test_malloc_strtoumax,
 	    test_malloc_snprintf_truncated,