Determine rtree levels at compile time.

Rather than dynamically building a table to aid per level computations,
define a constant table at compile time.  Omit both high and low
insignificant bits.  Use one to three tree levels, depending on the
number of significant bits.

configure.ac

@@ -406,6 +406,74 @@ case "${host_cpu}" in
 esac
 AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT])
 
+case "${host_cpu}" in
+  aarch64)
+    AC_MSG_CHECKING([number of significant virtual address bits])
+    LG_VADDR=48
+    AC_MSG_RESULT([$LG_VADDR])
+    ;;
+  x86_64)
+    AC_CACHE_CHECK([number of significant virtual address bits],
+                   [je_cv_lg_vaddr],
+                   AC_RUN_IFELSE([AC_LANG_PROGRAM(
+[[
+#include <stdio.h>
+#ifdef _WIN32
+#include <limits.h>
+#include <intrin.h>
+typedef unsigned __int32 uint32_t;
+#else
+#include <stdint.h>
+#endif
+]], [[
+	uint32_t r[[4]];
+	uint32_t eax_in = 0x80000008U;
+#ifdef _WIN32
+	__cpuid((int *)r, (int)eax_in);
+#else
+	asm volatile ("cpuid"
+	    : "=a" (r[[0]]), "=b" (r[[1]]), "=c" (r[[2]]), "=d" (r[[3]])
+	    : "a" (eax_in), "c" (0)
+	);
+#endif
+	uint32_t eax_out = r[[0]];
+	uint32_t vaddr = ((eax_out & 0x0000ff00U) >> 8);
+	FILE *f = fopen("conftest.out", "w");
+	if (f == NULL) {
+		return 1;
+	}
+	fprintf(f, "%u", vaddr);
+	fclose(f);
+	return 0;
+]])],
+                   [je_cv_lg_vaddr=`cat conftest.out`],
+                   [je_cv_lg_vaddr=error],
+                   [je_cv_lg_vaddr=57]))
+    if test "x${je_cv_lg_vaddr}" != "x" ; then
+      LG_VADDR="${je_cv_lg_vaddr}"
+    fi
+    if test "x${LG_VADDR}" != "xerror" ; then
+      AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR])
+    else
+      AC_MSG_ERROR([cannot determine number of significant virtual address bits])
+    fi
+    ;;
+  *)
+    AC_MSG_CHECKING([number of significant virtual address bits])
+    if test "x${LG_SIZEOF_PTR}" = "x3" ; then
+      LG_VADDR=64
+    elif test "x${LG_SIZEOF_PTR}" = "x2" ; then
+      LG_VADDR=32
+    elif test "x${LG_SIZEOF_PTR}" = "xLG_SIZEOF_PTR_WIN" ; then
+      LG_VADDR="(1U << (LG_SIZEOF_PTR_WIN+3))"
+    else
+      AC_MSG_ERROR([Unsupported lg(pointer size): ${LG_SIZEOF_PTR}])
+    fi
+    AC_MSG_RESULT([$LG_VADDR])
+    ;;
+esac
+AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR])
+
 LD_PRELOAD_VAR="LD_PRELOAD"
 so="so"
 importlib="${so}"

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -22,6 +22,13 @@
  */
 #undef CPU_SPINWAIT
 
+/*
+ * Number of significant bits in virtual addresses.  This may be less than the
+ * total number of bits in a pointer, e.g. on x64, for which the uppermost 16
+ * bits are the same as bit 47.
+ */
+#undef LG_VADDR
+
 /* Defined if C11 atomics are available. */
 #undef JEMALLOC_C11ATOMICS
 

include/jemalloc/internal/rtree_externs.h

@@ -1,7 +1,29 @@
 #ifndef JEMALLOC_INTERNAL_RTREE_EXTERNS_H
 #define JEMALLOC_INTERNAL_RTREE_EXTERNS_H
 
-bool rtree_new(rtree_t *rtree, unsigned bits);
+/*
+ * Split the bits into one to three partitions depending on number of
+ * significant bits.  It the number of bits does not divide evenly into the
+ * number of levels, place one remainder bit per level starting at the leaf
+ * level.
+ */
+static const rtree_level_t rtree_levels[] = {
+#if RTREE_NSB <= 10
+	{RTREE_NSB, RTREE_NHIB + RTREE_NSB}
+#elif RTREE_NSB <= 36
+	{RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2},
+	{RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB}
+#elif RTREE_NSB <= 52
+	{RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3},
+	{RTREE_NSB/3 + RTREE_NSB%3/2,
+	    RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2},
+	{RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB}
+#else
+#  error Unsupported number of significant virtual address bits
+#endif
+};
+
+bool rtree_new(rtree_t *rtree);
 #ifdef JEMALLOC_JET
 typedef rtree_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
 extern rtree_node_alloc_t *rtree_node_alloc;

include/jemalloc/internal/rtree_inlines.h

@@ -2,8 +2,8 @@
 #define JEMALLOC_INTERNAL_RTREE_INLINES_H
 
 #ifndef JEMALLOC_ENABLE_INLINE
-uintptr_t rtree_leafkey(rtree_t *rtree, uintptr_t key);
+uintptr_t rtree_leafkey(uintptr_t key);
-uintptr_t rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level);
+uintptr_t rtree_subkey(uintptr_t key, unsigned level);
 extent_t *rtree_elm_read(rtree_elm_t *elm, bool dependent);
 void rtree_elm_write(rtree_elm_t *elm, const extent_t *extent);
 rtree_elm_t *rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree,
@@ -25,21 +25,21 @@ void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_))
 JEMALLOC_ALWAYS_INLINE uintptr_t
-rtree_leafkey(rtree_t *rtree, uintptr_t key) {
+rtree_leafkey(uintptr_t key) {
 	unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
-	unsigned cumbits = (rtree->levels[rtree->height-1].cumbits -
+	unsigned cumbits = (rtree_levels[RTREE_HEIGHT-1].cumbits -
-	    rtree->levels[rtree->height-1].bits);
+	    rtree_levels[RTREE_HEIGHT-1].bits);
 	unsigned maskbits = ptrbits - cumbits;
 	uintptr_t mask = ~((ZU(1) << maskbits) - 1);
 	return (key & mask);
 }
 
 JEMALLOC_ALWAYS_INLINE uintptr_t
-rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level) {
+rtree_subkey(uintptr_t key, unsigned level) {
 	unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
-	unsigned cumbits = rtree->levels[level].cumbits;
+	unsigned cumbits = rtree_levels[level].cumbits;
 	unsigned shiftbits = ptrbits - cumbits;
-	unsigned maskbits = rtree->levels[level].bits;
+	unsigned maskbits = rtree_levels[level].bits;
 	unsigned mask = (ZU(1) << maskbits) - 1;
 	return ((key >> shiftbits) & mask);
 }
@@ -82,7 +82,7 @@ rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	assert(!dependent || !init_missing);
 
 	if (likely(key != 0)) {
-		uintptr_t leafkey = rtree_leafkey(rtree, key);
+		uintptr_t leafkey = rtree_leafkey(key);
 #define RTREE_CACHE_CHECK(i) do {					\
 		if (likely(rtree_ctx->cache[i].leafkey == leafkey)) {	\
 			rtree_elm_t *leaf = rtree_ctx->cache[i].leaf;	\
@@ -94,8 +94,8 @@ rtree_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 				rtree_ctx->cache[0].leafkey = leafkey;	\
 				rtree_ctx->cache[0].leaf = leaf;	\
 									\
-				uintptr_t subkey = rtree_subkey(rtree,	\
+				uintptr_t subkey = rtree_subkey(key,	\
-				    key, rtree->height-1);		\
+				    RTREE_HEIGHT-1);			\
 				return &leaf[subkey];			\
 			}						\
 		}							\

include/jemalloc/internal/rtree_structs.h

@@ -41,12 +41,10 @@ struct rtree_ctx_s {
 };
 
 struct rtree_s {
-	unsigned		height;
 	union {
 		void		*root_pun;
 		rtree_elm_t	*root;
 	};
-	rtree_level_t		levels[RTREE_HEIGHT_MAX];
 	malloc_mutex_t		init_lock;
 };
 

include/jemalloc/internal/rtree_types.h

@@ -16,15 +16,14 @@ typedef struct rtree_ctx_cache_elm_s rtree_ctx_cache_elm_t;
 typedef struct rtree_ctx_s rtree_ctx_t;
 typedef struct rtree_s rtree_t;
 
-/*
+/* Number of high insignificant bits. */
- * RTREE_BITS_PER_LEVEL must be a power of two that is no larger than the
+#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR)
- * machine address width.
+/* Number of low insigificant bits. */
- */
+#define RTREE_NLIB LG_PAGE
-#define LG_RTREE_BITS_PER_LEVEL	4
+/* Number of significant bits. */
-#define RTREE_BITS_PER_LEVEL	(1U << LG_RTREE_BITS_PER_LEVEL)
+#define RTREE_NSB (LG_VADDR - RTREE_NLIB)
-/* Maximum rtree height. */
+/* Number of levels in radix tree. */
-#define RTREE_HEIGHT_MAX						\
+#define RTREE_HEIGHT (sizeof(rtree_levels)/sizeof(rtree_level_t))
-    ((1U << (LG_SIZEOF_PTR+3)) / RTREE_BITS_PER_LEVEL)
 
 /*
  * Number of leafkey/leaf pairs to cache.  Each entry supports an entire leaf,

src/extent.c

@@ -1522,8 +1522,7 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
 
 bool
 extent_boot(void) {
-	if (rtree_new(&extents_rtree, (unsigned)((ZU(1) << (LG_SIZEOF_PTR+3)) -
+	if (rtree_new(&extents_rtree)) {
-	    LG_PAGE))) {
 		return true;
 	}
 

src/rtree.c

@@ -6,46 +6,11 @@
  * used.
  */
 bool
-rtree_new(rtree_t *rtree, unsigned bits) {
+rtree_new(rtree_t *rtree) {
-	unsigned bits_in_leaf, height, i;
-
-	assert(RTREE_HEIGHT_MAX == ((ZU(1) << (LG_SIZEOF_PTR+3)) /
-	    RTREE_BITS_PER_LEVEL));
-	assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3));
-
-	bits_in_leaf = (bits % RTREE_BITS_PER_LEVEL) == 0 ? RTREE_BITS_PER_LEVEL
-	    : (bits % RTREE_BITS_PER_LEVEL);
-	if (bits > bits_in_leaf) {
-		height = 1 + (bits - bits_in_leaf) / RTREE_BITS_PER_LEVEL;
-		if ((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf != bits) {
-			height++;
-		}
-	} else {
-		height = 1;
-	}
-	assert((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf == bits);
-
-	rtree->height = height;
-
 	rtree->root_pun = NULL;
-
+	if (malloc_mutex_init(&rtree->init_lock, "rtree", WITNESS_RANK_RTREE)) {
-	/* Root level. */
+		return true;
-	rtree->levels[0].bits = (height > 1) ? RTREE_BITS_PER_LEVEL :
-	    bits_in_leaf;
-	rtree->levels[0].cumbits = rtree->levels[0].bits;
-	/* Interior levels. */
-	for (i = 1; i < height-1; i++) {
-		rtree->levels[i].bits = RTREE_BITS_PER_LEVEL;
-		rtree->levels[i].cumbits = rtree->levels[i-1].cumbits +
-		    RTREE_BITS_PER_LEVEL;
 	}
-	/* Leaf level. */
-	if (height > 1) {
-		rtree->levels[height-1].bits = bits_in_leaf;
-		rtree->levels[height-1].cumbits = bits;
-	}
-
-	malloc_mutex_init(&rtree->init_lock, "rtree", WITNESS_RANK_RTREE);
 
 	return false;
 }
@@ -84,10 +49,10 @@ rtree_node_dalloc_t *rtree_node_dalloc = JEMALLOC_N(rtree_node_dalloc_impl);
 static void
 rtree_delete_subtree(tsdn_t *tsdn, rtree_t *rtree, rtree_elm_t *node,
     unsigned level) {
-	if (level + 1 < rtree->height) {
+	if (level + 1 < RTREE_HEIGHT) {
 		size_t nchildren, i;
 
-		nchildren = ZU(1) << rtree->levels[level].bits;
+		nchildren = ZU(1) << rtree_levels[level].bits;
 		for (i = 0; i < nchildren; i++) {
 			rtree_elm_t *child = node[i].child;
 			if (child != NULL) {
@@ -116,7 +81,7 @@ rtree_node_init(tsdn_t *tsdn, rtree_t *rtree, unsigned level,
 	node = atomic_read_p((void**)elmp);
 	if (node == NULL) {
 		node = rtree_node_alloc(tsdn, rtree, ZU(1) <<
-		    rtree->levels[level].bits);
+		    rtree_levels[level].bits);
 		if (node == NULL) {
 			malloc_mutex_unlock(tsdn, &rtree->init_lock);
 			return NULL;
@@ -186,24 +151,18 @@ rtree_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 	rtree_elm_t *node = init_missing ? rtree_subtree_read(tsdn, rtree,
 	    dependent) : rtree_subtree_tryread(rtree, dependent);
 
-#define RTREE_GET_BIAS	(RTREE_HEIGHT_MAX - rtree->height)
+#define RTREE_GET_SUBTREE(level) {					\
-	switch (RTREE_GET_BIAS) {
+		assert(level < RTREE_HEIGHT-1);				\
-#define RTREE_GET_SUBTREE(level)					\
-	case level: {							\
-		assert(level < (RTREE_HEIGHT_MAX-1));			\
 		if (!dependent && unlikely(!rtree_node_valid(node))) {	\
 			return NULL;					\
 		}							\
-		uintptr_t subkey = rtree_subkey(rtree, key, level -	\
+		uintptr_t subkey = rtree_subkey(key, level);		\
-		    RTREE_GET_BIAS);					\
 		node = init_missing ? rtree_child_read(tsdn, rtree,	\
-		    &node[subkey], level - RTREE_GET_BIAS, dependent) :	\
+		    &node[subkey], level, dependent) :			\
 		    rtree_child_tryread(&node[subkey], dependent);	\
-		/* Fall through. */					\
 	}
-#define RTREE_GET_LEAF(level)						\
+#define RTREE_GET_LEAF(level) {						\
-	case level: {							\
+		assert(level == RTREE_HEIGHT-1);			\
-		assert(level == (RTREE_HEIGHT_MAX-1));			\
 		if (!dependent && unlikely(!rtree_node_valid(node))) {	\
 			return NULL;					\
 		}							\
@@ -218,68 +177,27 @@ rtree_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
 				    sizeof(rtree_ctx_cache_elm_t) *	\
 				    (RTREE_CTX_NCACHE-1));		\
 			}						\
-			uintptr_t leafkey = rtree_leafkey(rtree, key);	\
+			uintptr_t leafkey = rtree_leafkey(key);		\
 			rtree_ctx->cache[0].leafkey = leafkey;		\
 			rtree_ctx->cache[0].leaf = node;		\
 		}							\
-		uintptr_t subkey = rtree_subkey(rtree, key, level -	\
+		uintptr_t subkey = rtree_subkey(key, level);		\
-		    RTREE_GET_BIAS);					\
 		return &node[subkey];					\
 	}
-#if RTREE_HEIGHT_MAX > 1
+	if (RTREE_HEIGHT > 1) {
-	RTREE_GET_SUBTREE(0)
+		RTREE_GET_SUBTREE(0)
-#endif
+	}
-#if RTREE_HEIGHT_MAX > 2
+	if (RTREE_HEIGHT > 2) {
-	RTREE_GET_SUBTREE(1)
+		RTREE_GET_SUBTREE(1)
-#endif
+	}
-#if RTREE_HEIGHT_MAX > 3
+	if (RTREE_HEIGHT > 3) {
-	RTREE_GET_SUBTREE(2)
+		for (unsigned i = 2; i < RTREE_HEIGHT-1; i++) {
-#endif
+			RTREE_GET_SUBTREE(i)
-#if RTREE_HEIGHT_MAX > 4
+		}
-	RTREE_GET_SUBTREE(3)
+	}
-#endif
+	RTREE_GET_LEAF(RTREE_HEIGHT-1)
-#if RTREE_HEIGHT_MAX > 5
-	RTREE_GET_SUBTREE(4)
-#endif
-#if RTREE_HEIGHT_MAX > 6
-	RTREE_GET_SUBTREE(5)
-#endif
-#if RTREE_HEIGHT_MAX > 7
-	RTREE_GET_SUBTREE(6)
-#endif
-#if RTREE_HEIGHT_MAX > 8
-	RTREE_GET_SUBTREE(7)
-#endif
-#if RTREE_HEIGHT_MAX > 9
-	RTREE_GET_SUBTREE(8)
-#endif
-#if RTREE_HEIGHT_MAX > 10
-	RTREE_GET_SUBTREE(9)
-#endif
-#if RTREE_HEIGHT_MAX > 11
-	RTREE_GET_SUBTREE(10)
-#endif
-#if RTREE_HEIGHT_MAX > 12
-	RTREE_GET_SUBTREE(11)
-#endif
-#if RTREE_HEIGHT_MAX > 13
-	RTREE_GET_SUBTREE(12)
-#endif
-#if RTREE_HEIGHT_MAX > 14
-	RTREE_GET_SUBTREE(13)
-#endif
-#if RTREE_HEIGHT_MAX > 15
-	RTREE_GET_SUBTREE(14)
-#endif
-#if RTREE_HEIGHT_MAX > 16
-#  error Unsupported RTREE_HEIGHT_MAX
-#endif
-	RTREE_GET_LEAF(RTREE_HEIGHT_MAX-1)
 #undef RTREE_GET_SUBTREE
 #undef RTREE_GET_LEAF
-	default: not_reached();
-	}
-#undef RTREE_GET_BIAS
 	not_reached();
 }
 
@@ -351,7 +269,7 @@ rtree_elm_witness_dalloc(tsd_t *tsd, witness_t *witness,
 			witness_init(&rew->witness, "rtree_elm",
 			    WITNESS_RANK_RTREE_ELM, rtree_elm_witness_comp,
 			    NULL);
-			    return;
+			return;
 		}
 	}
 	not_reached();

test/unit/rtree.c

@@ -33,31 +33,26 @@ rtree_node_dalloc_intercept(tsdn_t *tsdn, rtree_t *rtree, rtree_elm_t *node) {
 
 TEST_BEGIN(test_rtree_read_empty) {
 	tsdn_t *tsdn;
-	unsigned i;
 
 	tsdn = tsdn_fetch();
 
-	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
+	rtree_t rtree;
-		rtree_t rtree;
+	rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
-		rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
+	test_rtree = &rtree;
-		test_rtree = &rtree;
+	assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure");
-		assert_false(rtree_new(&rtree, i),
+	assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, 0, false),
-		    "Unexpected rtree_new() failure");
+	    "rtree_read() should return NULL for empty tree");
-		assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, 0, false),
+	rtree_delete(tsdn, &rtree);
-		    "rtree_read() should return NULL for empty tree");
+	test_rtree = NULL;
-		rtree_delete(tsdn, &rtree);
-		test_rtree = NULL;
-	}
 }
 TEST_END
 
 #define NTHREADS	8
-#define MAX_NBITS	18
+#define MAX_NBITS	30
 #define NITERS		1000
 #define SEED		42
 
 typedef struct {
-	unsigned	nbits;
 	rtree_t		rtree;
 	uint32_t	seed;
 } thd_start_arg_t;
@@ -77,7 +72,8 @@ thd_start(void *varg) {
 	tsdn = tsdn_fetch();
 
 	for (i = 0; i < NITERS; i++) {
-		uintptr_t key = (uintptr_t)gen_rand64(sfmt);
+		uintptr_t key = (uintptr_t)(gen_rand64(sfmt) & ((ZU(1) <<
+		    MAX_NBITS) - ZU(1)));
 		if (i % 2 == 0) {
 			rtree_elm_t *elm;
 
@@ -110,165 +106,134 @@ TEST_BEGIN(test_rtree_concurrent) {
 	thd_t thds[NTHREADS];
 	sfmt_t *sfmt;
 	tsdn_t *tsdn;
-	unsigned i, j;
 
 	sfmt = init_gen_rand(SEED);
 	tsdn = tsdn_fetch();
-	for (i = 1; i < MAX_NBITS; i++) {
+	test_rtree = &arg.rtree;
-		arg.nbits = i;
+	assert_false(rtree_new(&arg.rtree), "Unexpected rtree_new() failure");
-		test_rtree = &arg.rtree;
+	arg.seed = gen_rand32(sfmt);
-		assert_false(rtree_new(&arg.rtree, arg.nbits),
+	for (unsigned i = 0; i < NTHREADS; i++) {
-		    "Unexpected rtree_new() failure");
+		thd_create(&thds[i], thd_start, (void *)&arg);
-		arg.seed = gen_rand32(sfmt);
-		for (j = 0; j < NTHREADS; j++) {
-			thd_create(&thds[j], thd_start, (void *)&arg);
-		}
-		for (j = 0; j < NTHREADS; j++) {
-			thd_join(thds[j], NULL);
-		}
-		rtree_delete(tsdn, &arg.rtree);
-		test_rtree = NULL;
 	}
+	for (unsigned i = 0; i < NTHREADS; i++) {
+		thd_join(thds[i], NULL);
+	}
+	rtree_delete(tsdn, &arg.rtree);
+	test_rtree = NULL;
 	fini_gen_rand(sfmt);
 }
 TEST_END
 
 #undef NTHREADS
-#undef MAX_NBITS
 #undef NITERS
 #undef SEED
 
 TEST_BEGIN(test_rtree_extrema) {
-	unsigned i;
 	extent_t extent_a, extent_b;
 	tsdn_t *tsdn;
 
 	tsdn = tsdn_fetch();
 
-	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
+	rtree_t rtree;
-		rtree_t rtree;
+	rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
-		rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
+	test_rtree = &rtree;
-		test_rtree = &rtree;
+	assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure");
-		assert_false(rtree_new(&rtree, i),
-		    "Unexpected rtree_new() failure");
 
-		assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, 0,
+	assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, 0, &extent_a),
-		    &extent_a), "Unexpected rtree_write() failure, i=%u", i);
+	    "Unexpected rtree_write() failure");
-		assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, 0, true),
+	assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, 0, true), &extent_a,
-		    &extent_a,
+	    "rtree_read() should return previously set value");
-		    "rtree_read() should return previously set value, i=%u", i);
 
-		assert_false(rtree_write(tsdn, &rtree, &rtree_ctx,
+	assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, ~((uintptr_t)0),
-		    ~((uintptr_t)0), &extent_b),
+	    &extent_b), "Unexpected rtree_write() failure");
-		    "Unexpected rtree_write() failure, i=%u", i);
+	assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, ~((uintptr_t)0),
-		assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx,
+	    true), &extent_b,
-		    ~((uintptr_t)0), true), &extent_b,
+	    "rtree_read() should return previously set value");
-		    "rtree_read() should return previously set value, i=%u", i);
 
-		rtree_delete(tsdn, &rtree);
+	rtree_delete(tsdn, &rtree);
-		test_rtree = NULL;
+	test_rtree = NULL;
-	}
 }
 TEST_END
 
 TEST_BEGIN(test_rtree_bits) {
-	tsdn_t *tsdn;
+	tsdn_t *tsdn = tsdn_fetch();
-	unsigned i, j, k;
 
-	tsdn = tsdn_fetch();
+	uintptr_t keys[] = {0, 1, (((uintptr_t)1) << LG_PAGE) - 1};
 
-	for (i = 1; i < (sizeof(uintptr_t) << 3); i++) {
+	extent_t extent;
-		uintptr_t keys[] = {0, 1,
+	rtree_t rtree;
-		    (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1};
+	rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
-		extent_t extent;
-		rtree_t rtree;
-		rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
 
-		test_rtree = &rtree;
+	test_rtree = &rtree;
-		assert_false(rtree_new(&rtree, i),
+	assert_false(rtree_new(&rtree),
-		    "Unexpected rtree_new() failure");
+	    "Unexpected rtree_new() failure");
 
-		for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
+	for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) {
-			assert_false(rtree_write(tsdn, &rtree, &rtree_ctx,
+		assert_false(rtree_write(tsdn, &rtree, &rtree_ctx, keys[i],
-			    keys[j], &extent),
+		    &extent), "Unexpected rtree_write() failure");
-			    "Unexpected rtree_write() failure");
+		for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) {
-			for (k = 0; k < sizeof(keys)/sizeof(uintptr_t); k++) {
+			assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx,
-				assert_ptr_eq(rtree_read(tsdn, &rtree,
+			    keys[j], true), &extent,
-				    &rtree_ctx, keys[k], true), &extent,
+			    "rtree_read() should return previously set "
-				    "rtree_read() should return previously set "
+			    "value and ignore insignificant key bits; "
-				    "value and ignore insignificant key bits; "
+			    "i=%u, j=%u, set key=%#"FMTxPTR", get "
-				    "i=%u, j=%u, k=%u, set key=%#"FMTxPTR", "
+			    "key=%#"FMTxPTR, i, j, keys[i], keys[j]);
-				    "get key=%#"FMTxPTR, i, j, k, keys[j],
-				    keys[k]);
-			}
-			assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx,
-			    (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)), false),
-			    "Only leftmost rtree leaf should be set; "
-			    "i=%u, j=%u", i, j);
-			rtree_clear(tsdn, &rtree, &rtree_ctx, keys[j]);
 		}
-
+		assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx,
-		rtree_delete(tsdn, &rtree);
+		    (((uintptr_t)1) << LG_PAGE), false),
-		test_rtree = NULL;
+		    "Only leftmost rtree leaf should be set; i=%u", i);
+		rtree_clear(tsdn, &rtree, &rtree_ctx, keys[i]);
 	}
+
+	rtree_delete(tsdn, &rtree);
+	test_rtree = NULL;
 }
 TEST_END
 
 TEST_BEGIN(test_rtree_random) {
-	unsigned i;
-	sfmt_t *sfmt;
-	tsdn_t *tsdn;
 #define NSET 16
 #define SEED 42
+	sfmt_t *sfmt = init_gen_rand(SEED);
+	tsdn_t *tsdn = tsdn_fetch();
+	uintptr_t keys[NSET];
+	extent_t extent;
+	rtree_t rtree;
+	rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
+	rtree_elm_t *elm;
 
-	sfmt = init_gen_rand(SEED);
+	test_rtree = &rtree;
-	tsdn = tsdn_fetch();
+	assert_false(rtree_new(&rtree), "Unexpected rtree_new() failure");
-	for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) {
-		uintptr_t keys[NSET];
-		extent_t extent;
-		unsigned j;
-		rtree_t rtree;
-		rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
-		rtree_elm_t *elm;
 
-		test_rtree = &rtree;
+	for (unsigned i = 0; i < NSET; i++) {
-		assert_false(rtree_new(&rtree, i),
+		keys[i] = (uintptr_t)gen_rand64(sfmt);
-		    "Unexpected rtree_new() failure");
+		elm = rtree_elm_acquire(tsdn, &rtree, &rtree_ctx, keys[i],
-
+		    false, true);
-		for (j = 0; j < NSET; j++) {
+		assert_ptr_not_null(elm,
-			keys[j] = (uintptr_t)gen_rand64(sfmt);
+		    "Unexpected rtree_elm_acquire() failure");
-			elm = rtree_elm_acquire(tsdn, &rtree, &rtree_ctx,
+		rtree_elm_write_acquired(tsdn, &rtree, elm, &extent);
-			    keys[j], false, true);
+		rtree_elm_release(tsdn, &rtree, elm);
-			assert_ptr_not_null(elm,
+		assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i],
-			    "Unexpected rtree_elm_acquire() failure");
+		    true), &extent,
-			rtree_elm_write_acquired(tsdn, &rtree, elm, &extent);
+		    "rtree_read() should return previously set value");
-			rtree_elm_release(tsdn, &rtree, elm);
-			assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx,
-			    keys[j], true), &extent,
-			    "rtree_read() should return previously set value");
-		}
-		for (j = 0; j < NSET; j++) {
-			assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx,
-			    keys[j], true), &extent,
-			    "rtree_read() should return previously set value, "
-			    "j=%u", j);
-		}
-
-		for (j = 0; j < NSET; j++) {
-			rtree_clear(tsdn, &rtree, &rtree_ctx, keys[j]);
-			assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx,
-			    keys[j], true),
-			    "rtree_read() should return previously set value");
-		}
-		for (j = 0; j < NSET; j++) {
-			assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx,
-			    keys[j], true),
-			    "rtree_read() should return previously set value");
-		}
-
-		rtree_delete(tsdn, &rtree);
-		test_rtree = NULL;
 	}
+	for (unsigned i = 0; i < NSET; i++) {
+		assert_ptr_eq(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i],
+		    true), &extent,
+		    "rtree_read() should return previously set value, i=%u", i);
+	}
+
+	for (unsigned i = 0; i < NSET; i++) {
+		rtree_clear(tsdn, &rtree, &rtree_ctx, keys[i]);
+		assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i],
+		    true), "rtree_read() should return previously set value");
+	}
+	for (unsigned i = 0; i < NSET; i++) {
+		assert_ptr_null(rtree_read(tsdn, &rtree, &rtree_ctx, keys[i],
+		    true), "rtree_read() should return previously set value");
+	}
+
+	rtree_delete(tsdn, &rtree);
+	test_rtree = NULL;
 	fini_gen_rand(sfmt);
 #undef NSET
 #undef SEED