Refactor prng* from cpp macros into inline functions.

Remove 32-bit variant, convert prng64() to prng_lg_range(), and add
prng_range().
This commit is contained in:
Jason Evans 2016-02-09 16:28:40 -08:00 committed by Jason Evans
parent c87ab25d18
commit 34676d3369
12 changed files with 205 additions and 69 deletions

View File

@ -84,9 +84,10 @@ C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \
$(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \ $(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \
$(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \ $(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \
$(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/pages.c \ $(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/pages.c \
$(srcroot)src/prof.c $(srcroot)src/quarantine.c $(srcroot)src/rtree.c \ $(srcroot)src/prng.c $(srcroot)src/prof.c $(srcroot)src/quarantine.c \
$(srcroot)src/stats.c $(srcroot)src/tcache.c $(srcroot)src/ticker.c \ $(srcroot)src/rtree.c $(srcroot)src/stats.c $(srcroot)src/tcache.c \
$(srcroot)src/time.c $(srcroot)src/tsd.c $(srcroot)src/util.c $(srcroot)src/ticker.c $(srcroot)src/time.c $(srcroot)src/tsd.c \
$(srcroot)src/util.c
ifeq ($(enable_valgrind), 1) ifeq ($(enable_valgrind), 1)
C_SRCS += $(srcroot)src/valgrind.c C_SRCS += $(srcroot)src/valgrind.c
endif endif
@ -129,6 +130,7 @@ TESTS_UNIT := $(srcroot)test/unit/atomic.c \
$(srcroot)test/unit/math.c \ $(srcroot)test/unit/math.c \
$(srcroot)test/unit/mq.c \ $(srcroot)test/unit/mq.c \
$(srcroot)test/unit/mtx.c \ $(srcroot)test/unit/mtx.c \
$(srcroot)test/unit/prng.c \
$(srcroot)test/unit/prof_accum.c \ $(srcroot)test/unit/prof_accum.c \
$(srcroot)test/unit/prof_active.c \ $(srcroot)test/unit/prof_active.c \
$(srcroot)test/unit/prof_gdump.c \ $(srcroot)test/unit/prof_gdump.c \

View File

@ -40,9 +40,7 @@ struct ckh_s {
#endif #endif
/* Used for pseudo-random number generation. */ /* Used for pseudo-random number generation. */
#define CKH_A 1103515241 uint64_t prng_state;
#define CKH_C 12347
uint32_t prng_state;
/* Total number of items. */ /* Total number of items. */
size_t count; size_t count;

View File

@ -547,7 +547,7 @@ size2index_compute(size_t size)
#if (NTBINS != 0) #if (NTBINS != 0)
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
size_t lg_ceil = lg_floor(pow2_ceil(size)); size_t lg_ceil = lg_floor(pow2_ceil_zu(size));
return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin);
} }
#endif #endif
@ -644,7 +644,7 @@ s2u_compute(size_t size)
#if (NTBINS > 0) #if (NTBINS > 0)
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
size_t lg_ceil = lg_floor(pow2_ceil(size)); size_t lg_ceil = lg_floor(pow2_ceil_zu(size));
return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) :
(ZU(1) << lg_ceil)); (ZU(1) << lg_ceil));
} }

View File

@ -348,7 +348,9 @@ pages_map
pages_purge pages_purge
pages_trim pages_trim
pages_unmap pages_unmap
pow2_ceil pow2_ceil_u32
pow2_ceil_u64
pow2_ceil_zu
prof_active_get prof_active_get
prof_active_get_unlocked prof_active_get_unlocked
prof_active_set prof_active_set

View File

@ -18,31 +18,9 @@
* proportional to bit position. For example, the lowest bit has a cycle of 2, * proportional to bit position. For example, the lowest bit has a cycle of 2,
* the next has a cycle of 4, etc. For this reason, we prefer to use the upper * the next has a cycle of 4, etc. For this reason, we prefer to use the upper
* bits. * bits.
*
* Macro parameters:
* uint32_t r : Result.
* unsigned lg_range : (0..32], number of least significant bits to return.
* uint32_t state : Seed value.
* const uint32_t a, c : See above discussion.
*/ */
#define prng32(r, lg_range, state, a, c) do { \ #define PRNG_A UINT64_C(6364136223846793005)
assert((lg_range) > 0); \ #define PRNG_C UINT64_C(1442695040888963407)
assert((lg_range) <= 32); \
\
r = (state * (a)) + (c); \
state = r; \
r >>= (32 - (lg_range)); \
} while (false)
/* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */
#define prng64(r, lg_range, state, a, c) do { \
assert((lg_range) > 0); \
assert((lg_range) <= 64); \
\
r = (state * (a)) + (c); \
state = r; \
r >>= (64 - (lg_range)); \
} while (false)
#endif /* JEMALLOC_H_TYPES */ #endif /* JEMALLOC_H_TYPES */
/******************************************************************************/ /******************************************************************************/
@ -56,5 +34,46 @@
/******************************************************************************/ /******************************************************************************/
#ifdef JEMALLOC_H_INLINES #ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
uint64_t prng_lg_range(uint64_t *state, unsigned lg_range);
uint64_t prng_range(uint64_t *state, uint64_t range);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_))
JEMALLOC_ALWAYS_INLINE uint64_t
prng_lg_range(uint64_t *state, unsigned lg_range)
{
uint64_t ret;
assert(lg_range > 0);
assert(lg_range <= 64);
ret = (*state * PRNG_A) + PRNG_C;
*state = ret;
ret >>= (64 - lg_range);
return (ret);
}
JEMALLOC_ALWAYS_INLINE uint64_t
prng_range(uint64_t *state, uint64_t range)
{
uint64_t ret;
unsigned lg_range;
assert(range > 1);
/* Compute the ceiling of lg(range). */
lg_range = jemalloc_ffsl(pow2_ceil_u64(range)) - 1;
/* Generate a result in [0..range) via repeated trial. */
do {
ret = prng_lg_range(state, lg_range);
} while (ret >= range);
return (ret);
}
#endif
#endif /* JEMALLOC_H_INLINES */ #endif /* JEMALLOC_H_INLINES */
/******************************************************************************/ /******************************************************************************/

View File

@ -123,7 +123,9 @@ void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
int jemalloc_ffsl(long bitmap); int jemalloc_ffsl(long bitmap);
int jemalloc_ffs(int bitmap); int jemalloc_ffs(int bitmap);
size_t pow2_ceil(size_t x); uint64_t pow2_ceil_u64(uint64_t x);
uint32_t pow2_ceil_u32(uint32_t x);
size_t pow2_ceil_zu(size_t x);
size_t lg_floor(size_t x); size_t lg_floor(size_t x);
void set_errno(int errnum); void set_errno(int errnum);
int get_errno(void); int get_errno(void);
@ -150,9 +152,8 @@ jemalloc_ffs(int bitmap)
return (JEMALLOC_INTERNAL_FFS(bitmap)); return (JEMALLOC_INTERNAL_FFS(bitmap));
} }
/* Compute the smallest power of 2 that is >= x. */ JEMALLOC_INLINE uint64_t
JEMALLOC_INLINE size_t pow2_ceil_u64(uint64_t x)
pow2_ceil(size_t x)
{ {
x--; x--;
@ -161,13 +162,37 @@ pow2_ceil(size_t x)
x |= x >> 4; x |= x >> 4;
x |= x >> 8; x |= x >> 8;
x |= x >> 16; x |= x >> 16;
#if (LG_SIZEOF_PTR == 3)
x |= x >> 32; x |= x >> 32;
#endif
x++; x++;
return (x); return (x);
} }
JEMALLOC_INLINE uint32_t
pow2_ceil_u32(uint32_t x)
{
x--;
x |= x >> 1;
x |= x >> 2;
x |= x >> 4;
x |= x >> 8;
x |= x >> 16;
x++;
return (x);
}
/* Compute the smallest power of 2 that is >= x. */
JEMALLOC_INLINE size_t
pow2_ceil_zu(size_t x)
{
#if (LG_SIZEOF_PTR == 3)
return (pow2_ceil_u64(x));
#else
return (pow2_ceil_u32(x));
#endif
}
#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) #if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__))
JEMALLOC_INLINE size_t JEMALLOC_INLINE size_t
lg_floor(size_t x) lg_floor(size_t x)

View File

@ -2196,9 +2196,7 @@ arena_malloc_large(arena_t *arena, size_t size, szind_t binind, bool zero)
* that is a multiple of the cacheline size, e.g. [0 .. 63) * 64 * that is a multiple of the cacheline size, e.g. [0 .. 63) * 64
* for 4 KiB pages and 64-byte cachelines. * for 4 KiB pages and 64-byte cachelines.
*/ */
prng64(r, LG_PAGE - LG_CACHELINE, arena->offset_state, r = prng_lg_range(&arena->offset_state, LG_PAGE - LG_CACHELINE);
UINT64_C(6364136223846793009),
UINT64_C(1442695040888963409));
random_offset = ((uintptr_t)r) << LG_CACHELINE; random_offset = ((uintptr_t)r) << LG_CACHELINE;
} else } else
random_offset = 0; random_offset = 0;

View File

@ -99,7 +99,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
* Cycle through the cells in the bucket, starting at a random position. * Cycle through the cells in the bucket, starting at a random position.
* The randomness avoids worst-case search overhead as buckets fill up. * The randomness avoids worst-case search overhead as buckets fill up.
*/ */
prng32(offset, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C); offset = prng_lg_range(&ckh->prng_state, LG_CKH_BUCKET_CELLS);
for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) +
((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))];
@ -141,7 +141,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
* were an item for which both hashes indicated the same * were an item for which both hashes indicated the same
* bucket. * bucket.
*/ */
prng32(i, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C); i = prng_lg_range(&ckh->prng_state, LG_CKH_BUCKET_CELLS);
cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
assert(cell->key != NULL); assert(cell->key != NULL);

2
src/prng.c Normal file
View File

@ -0,0 +1,2 @@
#define JEMALLOC_PRNG_C_
#include "jemalloc/internal/jemalloc_internal.h"

View File

@ -871,8 +871,7 @@ prof_sample_threshold_update(prof_tdata_t *tdata)
* pp 500 * pp 500
* (http://luc.devroye.org/rnbookindex.html) * (http://luc.devroye.org/rnbookindex.html)
*/ */
prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005), r = prng_lg_range(&tdata->prng_state, 53);
UINT64_C(1442695040888963407));
u = (double)r * (1.0/9007199254740992.0L); u = (double)r * (1.0/9007199254740992.0L);
tdata->bytes_until_sample = (uint64_t)(log(u) / tdata->bytes_until_sample = (uint64_t)(log(u) /
log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample))))

68
test/unit/prng.c Normal file
View File

@ -0,0 +1,68 @@
#include "test/jemalloc_test.h"
TEST_BEGIN(test_prng_lg_range)
{
uint64_t sa, sb, ra, rb;
unsigned lg_range;
sa = 42;
ra = prng_lg_range(&sa, 64);
sa = 42;
rb = prng_lg_range(&sa, 64);
assert_u64_eq(ra, rb,
"Repeated generation should produce repeated results");
sb = 42;
rb = prng_lg_range(&sb, 64);
assert_u64_eq(ra, rb,
"Equivalent generation should produce equivalent results");
sa = 42;
ra = prng_lg_range(&sa, 64);
rb = prng_lg_range(&sa, 64);
assert_u64_ne(ra, rb,
"Full-width results must not immediately repeat");
sa = 42;
ra = prng_lg_range(&sa, 64);
for (lg_range = 63; lg_range > 0; lg_range--) {
sb = 42;
rb = prng_lg_range(&sb, lg_range);
assert_u64_eq((rb & (UINT64_C(0xffffffffffffffff) << lg_range)),
0, "High order bits should be 0, lg_range=%u", lg_range);
assert_u64_eq(rb, (ra >> (64 - lg_range)),
"Expected high order bits of full-width result, "
"lg_range=%u", lg_range);
}
}
TEST_END
TEST_BEGIN(test_prng_range)
{
uint64_t range;
#define MAX_RANGE 10000000
#define RANGE_STEP 97
#define NREPS 10
for (range = 2; range < MAX_RANGE; range += RANGE_STEP) {
uint64_t s;
unsigned rep;
s = range;
for (rep = 0; rep < NREPS; rep++) {
uint64_t r = prng_range(&s, range);
assert_u64_lt(r, range, "Out of range");
}
}
}
TEST_END
int
main(void)
{
return (test(
test_prng_lg_range,
test_prng_range));
}

View File

@ -1,33 +1,54 @@
#include "test/jemalloc_test.h" #include "test/jemalloc_test.h"
TEST_BEGIN(test_pow2_ceil) #define TEST_POW2_CEIL(t, suf, pri) do { \
unsigned i, pow2; \
t x; \
\
assert_zu_eq(pow2_ceil_##suf(0), 0, "Unexpected result"); \
\
for (i = 0; i < sizeof(t) * 8; i++) { \
assert_zu_eq(pow2_ceil_##suf(((t)1) << i), ((t)1) << i, \
"Unexpected result"); \
} \
\
for (i = 2; i < sizeof(t) * 8; i++) { \
assert_zu_eq(pow2_ceil_##suf((((t)1) << i) - 1), \
((t)1) << i, "Unexpected result"); \
} \
\
for (i = 0; i < sizeof(t) * 8 - 1; i++) { \
assert_zu_eq(pow2_ceil_##suf((((t)1) << i) + 1), \
((t)1) << (i+1), "Unexpected result"); \
} \
\
for (pow2 = 1; pow2 < 25; pow2++) { \
for (x = (((t)1) << (pow2-1)) + 1; x <= ((t)1) << pow2; \
x++) { \
assert_zu_eq(pow2_ceil_##suf(x), \
((t)1) << pow2, \
"Unexpected result, x=%"pri, x); \
} \
} \
} while (0)
TEST_BEGIN(test_pow2_ceil_u64)
{ {
unsigned i, pow2;
size_t x;
assert_zu_eq(pow2_ceil(0), 0, "Unexpected result"); TEST_POW2_CEIL(uint64_t, u64, FMTu64);
}
TEST_END
for (i = 0; i < sizeof(size_t) * 8; i++) { TEST_BEGIN(test_pow2_ceil_u32)
assert_zu_eq(pow2_ceil(ZU(1) << i), ZU(1) << i, {
"Unexpected result");
}
for (i = 2; i < sizeof(size_t) * 8; i++) { TEST_POW2_CEIL(uint32_t, u32, FMTu32);
assert_zu_eq(pow2_ceil((ZU(1) << i) - 1), ZU(1) << i, }
"Unexpected result"); TEST_END
}
for (i = 0; i < sizeof(size_t) * 8 - 1; i++) { TEST_BEGIN(test_pow2_ceil_zu)
assert_zu_eq(pow2_ceil((ZU(1) << i) + 1), ZU(1) << (i+1), {
"Unexpected result");
}
for (pow2 = 1; pow2 < 25; pow2++) { TEST_POW2_CEIL(size_t, zu, "zu");
for (x = (ZU(1) << (pow2-1)) + 1; x <= ZU(1) << pow2; x++) {
assert_zu_eq(pow2_ceil(x), ZU(1) << pow2,
"Unexpected result, x=%zu", x);
}
}
} }
TEST_END TEST_END
@ -286,7 +307,9 @@ main(void)
{ {
return (test( return (test(
test_pow2_ceil, test_pow2_ceil_u64,
test_pow2_ceil_u32,
test_pow2_ceil_zu,
test_malloc_strtoumax_no_endptr, test_malloc_strtoumax_no_endptr,
test_malloc_strtoumax, test_malloc_strtoumax,
test_malloc_snprintf_truncated, test_malloc_snprintf_truncated,