diff --git a/jemalloc/include/jemalloc/internal/arena.h b/jemalloc/include/jemalloc/internal/arena.h index 94b7f3d9..b80c118d 100644 --- a/jemalloc/include/jemalloc/internal/arena.h +++ b/jemalloc/include/jemalloc/internal/arena.h @@ -58,6 +58,10 @@ #define RUN_MAX_OVRHD 0x0000003dU #define RUN_MAX_OVRHD_RELAX 0x00001800U +/* Maximum number of regions in one run. */ +#define LG_RUN_MAXREGS 11 +#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) + /* * The minimum ratio of active:dirty pages per arena is computed as: * @@ -556,8 +560,8 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) * divide by 0, and 1 and 2 are both powers of two, which are * handled above. */ -#define SIZE_INV_SHIFT 21 -#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) +#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) +#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) static const unsigned size_invs[] = { SIZE_INV(3), SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), diff --git a/jemalloc/include/jemalloc/internal/atomic.h b/jemalloc/include/jemalloc/internal/atomic.h index f1f0c2b2..821c2efb 100644 --- a/jemalloc/include/jemalloc/internal/atomic.h +++ b/jemalloc/include/jemalloc/internal/atomic.h @@ -70,7 +70,9 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); } #else -# error "Missing implementation for 64-bit atomic operations" +# if (LG_SIZEOF_PTR == 3) +# error "Missing implementation for 64-bit atomic operations" +# endif #endif /* 32-bit operations. */ diff --git a/jemalloc/include/jemalloc/internal/bitmap.h b/jemalloc/include/jemalloc/internal/bitmap.h index 4bb2212c..605ebac5 100644 --- a/jemalloc/include/jemalloc/internal/bitmap.h +++ b/jemalloc/include/jemalloc/internal/bitmap.h @@ -2,7 +2,7 @@ #ifdef JEMALLOC_H_TYPES /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ -#define LG_BITMAP_MAXBITS 18 +#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS typedef struct bitmap_level_s bitmap_level_t; typedef struct bitmap_info_s bitmap_info_t; diff --git a/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in b/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in index fc944a8c..f82385d0 100644 --- a/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in +++ b/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in @@ -224,9 +224,9 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" -#include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" +#include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" diff --git a/jemalloc/src/arena.c b/jemalloc/src/arena.c index 0f4f12aa..0693f360 100644 --- a/jemalloc/src/arena.c +++ b/jemalloc/src/arena.c @@ -2427,6 +2427,7 @@ small_size2bin_init_hard(void) * *) bin_info->run_size >= min_run_size * *) bin_info->run_size <= arena_maxclass * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). + * *) bin_info->nregs <= RUN_MAXREGS * * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also * calculated here, since these settings are all interdependent. @@ -2459,6 +2460,10 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_run_size = min_run_size; try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) + 1; /* Counter-act try_nregs-- in loop. */ + if (try_nregs > RUN_MAXREGS) { + try_nregs = RUN_MAXREGS + + 1; /* Counter-act try_nregs-- in loop. */ + } do { try_nregs--; try_hdr_size = sizeof(arena_run_t); @@ -2500,6 +2505,10 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) + 1; /* Counter-act try_nregs-- in loop. */ + if (try_nregs > RUN_MAXREGS) { + try_nregs = RUN_MAXREGS + + 1; /* Counter-act try_nregs-- in loop. */ + } do { try_nregs--; try_hdr_size = sizeof(arena_run_t); @@ -2526,7 +2535,8 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) } while (try_run_size <= arena_maxclass && try_run_size <= arena_maxclass && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX - && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size); + && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size + && try_nregs < RUN_MAXREGS); assert(good_hdr_size <= good_reg0_offset); diff --git a/jemalloc/test/bitmap.c b/jemalloc/test/bitmap.c index 7a017c86..adfaacfe 100644 --- a/jemalloc/test/bitmap.c +++ b/jemalloc/test/bitmap.c @@ -13,7 +13,11 @@ */ #include "../src/bitmap.c" -#define MAXBITS 4500 +#if (LG_BITMAP_MAXBITS > 12) +# define MAXBITS 4500 +#else +# define MAXBITS (1U << LG_BITMAP_MAXBITS) +#endif static void test_bitmap_size(void)