#define JEMALLOC_ARENA_C_ #include "jemalloc/internal/jemalloc_internal.h" /******************************************************************************/ /* Data. */ ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; arena_bin_info_t arena_bin_info[NBINS]; JEMALLOC_ALIGNED(CACHELINE) const uint32_t small_bin2size_tab[NBINS] = { #define B2S_bin_yes(size) \ size, #define B2S_bin_no(size) #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ B2S_bin_##bin((ZU(1)<bits); } static inline int arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) { uintptr_t a_mapelm = (uintptr_t)a; uintptr_t b_mapelm = (uintptr_t)b; assert(a != NULL); assert(b != NULL); return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); } /* Generate red-black tree functions. */ rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, u.rb_link, arena_run_comp) static inline int arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) { int ret; size_t a_size; size_t b_size = arena_mapelm_to_bits(b) & ~PAGE_MASK; uintptr_t a_mapelm = (uintptr_t)a; uintptr_t b_mapelm = (uintptr_t)b; if (a_mapelm & CHUNK_MAP_KEY) a_size = a_mapelm & ~PAGE_MASK; else a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; ret = (a_size > b_size) - (a_size < b_size); if (ret == 0) { if (!(a_mapelm & CHUNK_MAP_KEY)) ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); else { /* * Treat keys as if they are lower than anything else. */ ret = -1; } } return (ret); } /* Generate red-black tree functions. */ rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, u.rb_link, arena_avail_comp) static inline int arena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) { assert(a != NULL); assert(b != NULL); /* * Short-circuit for self comparison. The following comparison code * would come to the same result, but at the cost of executing the slow * path. */ if (a == b) return (0); /* * Order such that chunks with higher fragmentation are "less than" * those with lower fragmentation -- purging order is from "least" to * "greatest". Fragmentation is measured as: * * mean current avail run size * -------------------------------- * mean defragmented avail run size * * navail * ----------- * nruns_avail nruns_avail-nruns_adjac * = ========================= = ----------------------- * navail nruns_avail * ----------------------- * nruns_avail-nruns_adjac * * The following code multiplies away the denominator prior to * comparison, in order to avoid division. * */ { size_t a_val = (a->nruns_avail - a->nruns_adjac) * b->nruns_avail; size_t b_val = (b->nruns_avail - b->nruns_adjac) * a->nruns_avail; if (a_val < b_val) return (1); if (a_val > b_val) return (-1); } /* * Break ties by chunk address. For fragmented chunks, report lower * addresses as "lower", so that fragmentation reduction happens first * at lower addresses. However, use the opposite ordering for * unfragmented chunks, in order to increase the chances of * re-allocating dirty runs. */ { uintptr_t a_chunk = (uintptr_t)a; uintptr_t b_chunk = (uintptr_t)b; int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); if (a->nruns_adjac == 0) { assert(b->nruns_adjac == 0); ret = -ret; } return (ret); } } /* Generate red-black tree functions. */ rb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, dirty_link, arena_chunk_dirty_comp) static inline bool arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) { bool ret; if (pageind-1 < map_bias) ret = false; else { ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); assert(ret == false || arena_mapbits_dirty_get(chunk, pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); } return (ret); } static inline bool arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) { bool ret; if (pageind+npages == chunk_npages) ret = false; else { assert(pageind+npages < chunk_npages); ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) != arena_mapbits_dirty_get(chunk, pageind+npages)); } return (ret); } static inline bool arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) { return (arena_avail_adjac_pred(chunk, pageind) || arena_avail_adjac_succ(chunk, pageind, npages)); } static void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) { assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); /* * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be * removed and reinserted even if the run to be inserted is clean. */ if (chunk->ndirty != 0) arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) chunk->nruns_adjac++; if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) chunk->nruns_adjac++; chunk->nruns_avail++; assert(chunk->nruns_avail > chunk->nruns_adjac); if (arena_mapbits_dirty_get(chunk, pageind) != 0) { arena->ndirty += npages; chunk->ndirty += npages; } if (chunk->ndirty != 0) arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, pageind)); } static void arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) { assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); /* * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be * removed and reinserted even if the run to be removed is clean. */ if (chunk->ndirty != 0) arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) chunk->nruns_adjac--; if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) chunk->nruns_adjac--; chunk->nruns_avail--; assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail == 0 && chunk->nruns_adjac == 0)); if (arena_mapbits_dirty_get(chunk, pageind) != 0) { arena->ndirty -= npages; chunk->ndirty -= npages; } if (chunk->ndirty != 0) arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, pageind)); } static inline void * arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) { void *ret; unsigned regind; bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); assert(run->nfree > 0); assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(bin_info->reg_interval * regind)); run->nfree--; if (regind == run->nextind) run->nextind++; assert(regind < run->nextind); return (ret); } static inline void arena_run_reg_dalloc(arena_run_t *run, void *ptr) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t mapbits = arena_mapbits_get(chunk, pageind); size_t binind = arena_ptr_small_binind_get(ptr, mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind = arena_run_regind(run, bin_info, ptr); bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); assert(run->nfree < bin_info->nregs); /* Freeing an interior pointer can cause assertion failure. */ assert(((uintptr_t)ptr - ((uintptr_t)run + (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_interval == 0); assert((uintptr_t)ptr >= (uintptr_t)run + (uintptr_t)bin_info->reg0_offset); /* Freeing an unallocated pointer can cause assertion failure. */ assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); bitmap_unset(bitmap, &bin_info->bitmap_info, regind); run->nfree++; } static inline void arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), (npages << LG_PAGE)); memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, (npages << LG_PAGE)); } static inline void arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) { JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), PAGE); } static inline void arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) { size_t i; UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); arena_run_page_mark_zeroed(chunk, run_ind); for (i = 0; i < PAGE / sizeof(size_t); i++) assert(p[i] == 0); } static void arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) { if (config_stats) { ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - sub_pages) << LG_PAGE); if (cactive_diff != 0) stats_cactive_add(cactive_diff); } } static void arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, size_t flag_dirty, size_t need_pages) { size_t total_pages, rem_pages; total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> LG_PAGE; assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == flag_dirty); assert(need_pages <= total_pages); rem_pages = total_pages - need_pages; arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); arena_cactive_update(arena, need_pages, 0); arena->nactive += need_pages; /* Keep track of trailing unused pages for later use. */ if (rem_pages > 0) { if (flag_dirty != 0) { arena_mapbits_unallocated_set(chunk, run_ind+need_pages, (rem_pages << LG_PAGE), flag_dirty); arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, (rem_pages << LG_PAGE), flag_dirty); } else { arena_mapbits_unallocated_set(chunk, run_ind+need_pages, (rem_pages << LG_PAGE), arena_mapbits_unzeroed_get(chunk, run_ind+need_pages)); arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, (rem_pages << LG_PAGE), arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1)); } arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, false, true); } } static void arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, bool remove, bool zero) { arena_chunk_t *chunk; size_t flag_dirty, run_ind, need_pages, i; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); need_pages = (size >> LG_PAGE); assert(need_pages > 0); if (remove) { arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); } if (zero) { if (flag_dirty == 0) { /* * The run is clean, so some pages may be zeroed (i.e. * never before touched). */ for (i = 0; i < need_pages; i++) { if (arena_mapbits_unzeroed_get(chunk, run_ind+i) != 0) arena_run_zero(chunk, run_ind+i, 1); else if (config_debug) { arena_run_page_validate_zeroed(chunk, run_ind+i); } else { arena_run_page_mark_zeroed(chunk, run_ind+i); } } } else { /* The run is dirty, so all pages must be zeroed. */ arena_run_zero(chunk, run_ind, need_pages); } } else { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } /* * Set the last element first, in case the run only contains one page * (i.e. both statements set the same element). */ arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); } static void arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) { arena_run_split_large_helper(arena, run, size, true, zero); } static void arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) { arena_run_split_large_helper(arena, run, size, false, zero); } static void arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, size_t binind) { arena_chunk_t *chunk; size_t flag_dirty, run_ind, need_pages, i; assert(binind != BININD_INVALID); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); need_pages = (size >> LG_PAGE); assert(need_pages > 0); arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); /* * Propagate the dirty and unzeroed flags to the allocated small run, * so that arena_dalloc_bin_run() has the ability to conditionally trim * clean pages. */ arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); /* * The first page will always be dirtied during small run * initialization, so a validation failure here would not actually * cause an observable failure. */ if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, run_ind) == 0) arena_run_page_validate_zeroed(chunk, run_ind); for (i = 1; i < need_pages - 1; i++) { arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) arena_run_page_validate_zeroed(chunk, run_ind+i); } arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, binind, flag_dirty); if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) == 0) arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } static arena_chunk_t * arena_chunk_init_spare(arena_t *arena) { arena_chunk_t *chunk; assert(arena->spare != NULL); chunk = arena->spare; arena->spare = NULL; assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == arena_maxclass); assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == arena_maxclass); assert(arena_mapbits_dirty_get(chunk, map_bias) == arena_mapbits_dirty_get(chunk, chunk_npages-1)); return (chunk); } static arena_chunk_t * arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, bool *zero) { arena_chunk_t *chunk; chunk_alloc_t *chunk_alloc; chunk_dalloc_t *chunk_dalloc; chunk_alloc = arena->chunk_alloc; chunk_dalloc = arena->chunk_dalloc; malloc_mutex_unlock(&arena->lock); chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, size, alignment, zero); malloc_mutex_lock(&arena->lock); if (config_stats && chunk != NULL) arena->stats.mapped += chunksize; return (chunk); } void * arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, bool *zero) { void *ret; chunk_alloc_t *chunk_alloc; chunk_dalloc_t *chunk_dalloc; malloc_mutex_lock(&arena->lock); chunk_alloc = arena->chunk_alloc; chunk_dalloc = arena->chunk_dalloc; if (config_stats) { /* Optimistically update stats prior to unlocking. */ arena->stats.mapped += size; arena->stats.allocated_huge += size; arena->stats.nmalloc_huge++; arena->stats.nrequests_huge++; } arena->nactive += (size >> LG_PAGE); malloc_mutex_unlock(&arena->lock); ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, size, alignment, zero); if (config_stats) { if (ret != NULL) stats_cactive_add(size); else { /* Revert optimistic stats updates. */ malloc_mutex_lock(&arena->lock); arena->stats.mapped -= size; arena->stats.allocated_huge -= size; arena->stats.nmalloc_huge--; malloc_mutex_unlock(&arena->lock); } } return (ret); } static arena_chunk_t * arena_chunk_init_hard(arena_t *arena) { arena_chunk_t *chunk; bool zero; size_t unzeroed, i; assert(arena->spare == NULL); zero = false; chunk = arena_chunk_alloc_internal(arena, chunksize, chunksize, &zero); if (chunk == NULL) return (NULL); chunk->arena = arena; /* * Claim that no pages are in use, since the header is merely overhead. */ chunk->ndirty = 0; chunk->nruns_avail = 0; chunk->nruns_adjac = 0; /* * Initialize the map to contain one maximal free untouched run. Mark * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. */ unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, unzeroed); /* * There is no need to initialize the internal page map entries unless * the chunk is not zeroed. */ if (zero == false) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( (void *)arena_mapp_get(chunk, map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, map_bias+1))); for (i = map_bias+1; i < chunk_npages-1; i++) arena_mapbits_unzeroed_set(chunk, i, unzeroed); } else { JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, map_bias+1))); if (config_debug) { for (i = map_bias+1; i < chunk_npages-1; i++) { assert(arena_mapbits_unzeroed_get(chunk, i) == unzeroed); } } } arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, unzeroed); return (chunk); } static arena_chunk_t * arena_chunk_alloc(arena_t *arena) { arena_chunk_t *chunk; if (arena->spare != NULL) chunk = arena_chunk_init_spare(arena); else { chunk = arena_chunk_init_hard(arena); if (chunk == NULL) return (NULL); } /* Insert the run into the runs_avail tree. */ arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, false, false); return (chunk); } static void arena_chunk_dalloc_internal(arena_t *arena, arena_chunk_t *chunk) { chunk_dalloc_t *chunk_dalloc; chunk_dalloc = arena->chunk_dalloc; malloc_mutex_unlock(&arena->lock); chunk_dalloc((void *)chunk, chunksize, arena->ind); malloc_mutex_lock(&arena->lock); if (config_stats) arena->stats.mapped -= chunksize; } void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size) { chunk_dalloc_t *chunk_dalloc; malloc_mutex_lock(&arena->lock); chunk_dalloc = arena->chunk_dalloc; if (config_stats) { arena->stats.mapped -= size; arena->stats.allocated_huge -= size; arena->stats.ndalloc_huge++; stats_cactive_sub(size); } arena->nactive -= (size >> LG_PAGE); malloc_mutex_unlock(&arena->lock); chunk_dalloc(chunk, size, arena->ind); } static void arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) { assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == arena_maxclass); assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == arena_maxclass); assert(arena_mapbits_dirty_get(chunk, map_bias) == arena_mapbits_dirty_get(chunk, chunk_npages-1)); /* * Remove run from the runs_avail tree, so that the arena does not use * it. */ arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, false, false); if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; arena->spare = chunk; arena_chunk_dalloc_internal(arena, spare); } else arena->spare = chunk; } static arena_run_t * arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { arena_run_t *run; arena_chunk_map_t *mapelm; arena_chunk_map_t *key; key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (mapelm != NULL) { arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); size_t pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << LG_PAGE)); arena_run_split_large(arena, run, size, zero); return (run); } return (NULL); } static arena_run_t * arena_run_alloc_large(arena_t *arena, size_t size, bool zero) { arena_chunk_t *chunk; arena_run_t *run; assert(size <= arena_maxclass); assert((size & PAGE_MASK) == 0); /* Search the arena's chunks for the lowest best fit. */ run = arena_run_alloc_large_helper(arena, size, zero); if (run != NULL) return (run); /* * No usable runs. Create a new chunk from which to allocate the run. */ chunk = arena_chunk_alloc(arena); if (chunk != NULL) { run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); arena_run_split_large(arena, run, size, zero); return (run); } /* * arena_chunk_alloc() failed, but another thread may have made * sufficient memory available while this one dropped arena->lock in * arena_chunk_alloc(), so search one more time. */ return (arena_run_alloc_large_helper(arena, size, zero)); } static arena_run_t * arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) { arena_run_t *run; arena_chunk_map_t *mapelm; arena_chunk_map_t *key; key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (mapelm != NULL) { arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); size_t pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << LG_PAGE)); arena_run_split_small(arena, run, size, binind); return (run); } return (NULL); } static arena_run_t * arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) { arena_chunk_t *chunk; arena_run_t *run; assert(size <= arena_maxclass); assert((size & PAGE_MASK) == 0); assert(binind != BININD_INVALID); /* Search the arena's chunks for the lowest best fit. */ run = arena_run_alloc_small_helper(arena, size, binind); if (run != NULL) return (run); /* * No usable runs. Create a new chunk from which to allocate the run. */ chunk = arena_chunk_alloc(arena); if (chunk != NULL) { run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); arena_run_split_small(arena, run, size, binind); return (run); } /* * arena_chunk_alloc() failed, but another thread may have made * sufficient memory available while this one dropped arena->lock in * arena_chunk_alloc(), so search one more time. */ return (arena_run_alloc_small_helper(arena, size, binind)); } static inline void arena_maybe_purge(arena_t *arena) { size_t npurgeable, threshold; /* Don't purge if the option is disabled. */ if (opt_lg_dirty_mult < 0) return; /* Don't purge if all dirty pages are already being purged. */ if (arena->ndirty <= arena->npurgatory) return; npurgeable = arena->ndirty - arena->npurgatory; threshold = (arena->nactive >> opt_lg_dirty_mult); /* * Don't purge unless the number of purgeable pages exceeds the * threshold. */ if (npurgeable <= threshold) return; arena_purge(arena, false); } static arena_chunk_t * chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) { size_t *ndirty = (size_t *)arg; assert(chunk->ndirty != 0); *ndirty += chunk->ndirty; return (NULL); } static size_t arena_compute_npurgatory(arena_t *arena, bool all) { size_t npurgatory, npurgeable; /* * Compute the minimum number of pages that this thread should try to * purge. */ npurgeable = arena->ndirty - arena->npurgatory; if (all == false) { size_t threshold = (arena->nactive >> opt_lg_dirty_mult); npurgatory = npurgeable - threshold; } else npurgatory = npurgeable; return (npurgatory); } static void arena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, arena_chunk_mapelms_t *mapelms) { size_t pageind, npages; /* * Temporarily allocate free dirty runs within chunk. If all is false, * only operate on dirty runs that are fragments; otherwise operate on * all dirty runs. */ for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); if (arena_mapbits_allocated_get(chunk, pageind) == 0) { size_t run_size = arena_mapbits_unallocated_size_get(chunk, pageind); npages = run_size >> LG_PAGE; assert(pageind + npages <= chunk_npages); assert(arena_mapbits_dirty_get(chunk, pageind) == arena_mapbits_dirty_get(chunk, pageind+npages-1)); if (arena_mapbits_dirty_get(chunk, pageind) != 0 && (all || arena_avail_adjac(chunk, pageind, npages))) { arena_run_t *run = (arena_run_t *)((uintptr_t) chunk + (uintptr_t)(pageind << LG_PAGE)); arena_run_split_large(arena, run, run_size, false); /* Append to list for later processing. */ ql_elm_new(mapelm, u.ql_link); ql_tail_insert(mapelms, mapelm, u.ql_link); } } else { /* Skip run. */ if (arena_mapbits_large_get(chunk, pageind) != 0) { npages = arena_mapbits_large_size_get(chunk, pageind) >> LG_PAGE; } else { size_t binind; arena_bin_info_t *bin_info; arena_run_t *run = (arena_run_t *)((uintptr_t) chunk + (uintptr_t)(pageind << LG_PAGE)); assert(arena_mapbits_small_runind_get(chunk, pageind) == 0); binind = arena_bin_index(arena, run->bin); bin_info = &arena_bin_info[binind]; npages = bin_info->run_size >> LG_PAGE; } } } assert(pageind == chunk_npages); assert(chunk->ndirty == 0 || all == false); assert(chunk->nruns_adjac == 0); } static size_t arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, arena_chunk_mapelms_t *mapelms) { size_t npurged, pageind, npages, nmadvise; arena_chunk_map_t *mapelm; malloc_mutex_unlock(&arena->lock); if (config_stats) nmadvise = 0; npurged = 0; ql_foreach(mapelm, mapelms, u.ql_link) { bool unzeroed; size_t flag_unzeroed, i; pageind = arena_mapelm_to_pageind(mapelm); npages = arena_mapbits_large_size_get(chunk, pageind) >> LG_PAGE; assert(pageind + npages <= chunk_npages); unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << LG_PAGE)), (npages << LG_PAGE)); flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; /* * Set the unzeroed flag for all pages, now that pages_purge() * has returned whether the pages were zeroed as a side effect * of purging. This chunk map modification is safe even though * the arena mutex isn't currently owned by this thread, * because the run is marked as allocated, thus protecting it * from being modified by any other thread. As long as these * writes don't perturb the first and last elements' * CHUNK_MAP_ALLOCATED bits, behavior is well defined. */ for (i = 0; i < npages; i++) { arena_mapbits_unzeroed_set(chunk, pageind+i, flag_unzeroed); } npurged += npages; if (config_stats) nmadvise++; } malloc_mutex_lock(&arena->lock); if (config_stats) arena->stats.nmadvise += nmadvise; return (npurged); } static void arena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, arena_chunk_mapelms_t *mapelms) { arena_chunk_map_t *mapelm; size_t pageind; /* Deallocate runs. */ for (mapelm = ql_first(mapelms); mapelm != NULL; mapelm = ql_first(mapelms)) { arena_run_t *run; pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << LG_PAGE)); ql_remove(mapelms, mapelm, u.ql_link); arena_run_dalloc(arena, run, false, true); } } static inline size_t arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) { size_t npurged; arena_chunk_mapelms_t mapelms; ql_new(&mapelms); /* * If chunk is the spare, temporarily re-allocate it, 1) so that its * run is reinserted into runs_avail, and 2) so that it cannot be * completely discarded by another thread while arena->lock is dropped * by this thread. Note that the arena_run_dalloc() call will * implicitly deallocate the chunk, so no explicit action is required * in this function to deallocate the chunk. * * Note that once a chunk contains dirty pages, it cannot again contain * a single run unless 1) it is a dirty run, or 2) this function purges * dirty pages and causes the transition to a single clean run. Thus * (chunk == arena->spare) is possible, but it is not possible for * this function to be called on the spare unless it contains a dirty * run. */ if (chunk == arena->spare) { assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); arena_chunk_alloc(arena); } if (config_stats) arena->stats.purged += chunk->ndirty; /* * Operate on all dirty runs if there is no clean/dirty run * fragmentation. */ if (chunk->nruns_adjac == 0) all = true; arena_chunk_stash_dirty(arena, chunk, all, &mapelms); npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); arena_chunk_unstash_purged(arena, chunk, &mapelms); return (npurged); } static void arena_purge(arena_t *arena, bool all) { arena_chunk_t *chunk; size_t npurgatory; if (config_debug) { size_t ndirty = 0; arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, chunks_dirty_iter_cb, (void *)&ndirty); assert(ndirty == arena->ndirty); } assert(arena->ndirty > arena->npurgatory || all); assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - arena->npurgatory) || all); if (config_stats) arena->stats.npurge++; /* * Add the minimum number of pages this thread should try to purge to * arena->npurgatory. This will keep multiple threads from racing to * reduce ndirty below the threshold. */ npurgatory = arena_compute_npurgatory(arena, all); arena->npurgatory += npurgatory; while (npurgatory > 0) { size_t npurgeable, npurged, nunpurged; /* Get next chunk with dirty pages. */ chunk = arena_chunk_dirty_first(&arena->chunks_dirty); if (chunk == NULL) { /* * This thread was unable to purge as many pages as * originally intended, due to races with other threads * that either did some of the purging work, or re-used * dirty pages. */ arena->npurgatory -= npurgatory; return; } npurgeable = chunk->ndirty; assert(npurgeable != 0); if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { /* * This thread will purge all the dirty pages in chunk, * so set npurgatory to reflect this thread's intent to * purge the pages. This tends to reduce the chances * of the following scenario: * * 1) This thread sets arena->npurgatory such that * (arena->ndirty - arena->npurgatory) is at the * threshold. * 2) This thread drops arena->lock. * 3) Another thread causes one or more pages to be * dirtied, and immediately determines that it must * purge dirty pages. * * If this scenario *does* play out, that's okay, * because all of the purging work being done really * needs to happen. */ arena->npurgatory += npurgeable - npurgatory; npurgatory = npurgeable; } /* * Keep track of how many pages are purgeable, versus how many * actually get purged, and adjust counters accordingly. */ arena->npurgatory -= npurgeable; npurgatory -= npurgeable; npurged = arena_chunk_purge(arena, chunk, all); nunpurged = npurgeable - npurged; arena->npurgatory += nunpurged; npurgatory += nunpurged; } } void arena_purge_all(arena_t *arena) { malloc_mutex_lock(&arena->lock); arena_purge(arena, true); malloc_mutex_unlock(&arena->lock); } static void arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) { size_t size = *p_size; size_t run_ind = *p_run_ind; size_t run_pages = *p_run_pages; /* Try to coalesce forward. */ if (run_ind + run_pages < chunk_npages && arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages); size_t nrun_pages = nrun_size >> LG_PAGE; /* * Remove successor from runs_avail; the coalesced run is * inserted later. */ assert(arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages+nrun_pages-1) == nrun_size); assert(arena_mapbits_dirty_get(chunk, run_ind+run_pages+nrun_pages-1) == flag_dirty); arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, false, true); size += nrun_size; run_pages += nrun_pages; arena_mapbits_unallocated_size_set(chunk, run_ind, size); arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, size); } /* Try to coalesce backward. */ if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) { size_t prun_size = arena_mapbits_unallocated_size_get(chunk, run_ind-1); size_t prun_pages = prun_size >> LG_PAGE; run_ind -= prun_pages; /* * Remove predecessor from runs_avail; the coalesced run is * inserted later. */ assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == prun_size); assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); arena_avail_remove(arena, chunk, run_ind, prun_pages, true, false); size += prun_size; run_pages += prun_pages; arena_mapbits_unallocated_size_set(chunk, run_ind, size); arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, size); } *p_size = size; *p_run_ind = run_ind; *p_run_pages = run_pages; } static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) { arena_chunk_t *chunk; size_t size, run_ind, run_pages, flag_dirty; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); assert(run_ind >= map_bias); assert(run_ind < chunk_npages); if (arena_mapbits_large_get(chunk, run_ind) != 0) { size = arena_mapbits_large_size_get(chunk, run_ind); assert(size == PAGE || arena_mapbits_large_size_get(chunk, run_ind+(size>>LG_PAGE)-1) == 0); } else { size_t binind = arena_bin_index(arena, run->bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; size = bin_info->run_size; } run_pages = (size >> LG_PAGE); arena_cactive_update(arena, 0, run_pages); arena->nactive -= run_pages; /* * The run is dirty if the caller claims to have dirtied it, as well as * if it was already dirty before being allocated and the caller * doesn't claim to have cleaned it. */ assert(arena_mapbits_dirty_get(chunk, run_ind) == arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) dirty = true; flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; /* Mark pages as unallocated in the chunk map. */ if (dirty) { arena_mapbits_unallocated_set(chunk, run_ind, size, CHUNK_MAP_DIRTY); arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, CHUNK_MAP_DIRTY); } else { arena_mapbits_unallocated_set(chunk, run_ind, size, arena_mapbits_unzeroed_get(chunk, run_ind)); arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); } arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, flag_dirty); /* Insert into runs_avail, now that coalescing is complete. */ assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); assert(arena_mapbits_dirty_get(chunk, run_ind) == arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); /* Deallocate chunk if it is now completely unused. */ if (size == arena_maxclass) { assert(run_ind == map_bias); assert(run_pages == (arena_maxclass >> LG_PAGE)); arena_chunk_dalloc(arena, chunk); } /* * It is okay to do dirty page processing here even if the chunk was * deallocated above, since in that case it is the spare. Waiting * until after possible chunk deallocation to do dirty processing * allows for an old spare to be fully deallocated, thus decreasing the * chances of spuriously crossing the dirty page purging threshold. */ if (dirty) arena_maybe_purge(arena); } static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize) { size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; size_t head_npages = (oldsize - newsize) >> LG_PAGE; size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); assert(oldsize > newsize); /* * Update the chunk map so that arena_run_dalloc() can treat the * leading run as separately allocated. Set the last element of each * run first, in case of single-page runs. */ assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); if (config_debug) { UNUSED size_t tail_npages = newsize >> LG_PAGE; assert(arena_mapbits_large_size_get(chunk, pageind+head_npages+tail_npages-1) == 0); assert(arena_mapbits_dirty_get(chunk, pageind+head_npages+tail_npages-1) == flag_dirty); } arena_mapbits_large_set(chunk, pageind+head_npages, newsize, flag_dirty); arena_run_dalloc(arena, run, false, false); } static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) { size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; size_t head_npages = newsize >> LG_PAGE; size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); assert(oldsize > newsize); /* * Update the chunk map so that arena_run_dalloc() can treat the * trailing run as separately allocated. Set the last element of each * run first, in case of single-page runs. */ assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); if (config_debug) { UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; assert(arena_mapbits_large_size_get(chunk, pageind+head_npages+tail_npages-1) == 0); assert(arena_mapbits_dirty_get(chunk, pageind+head_npages+tail_npages-1) == flag_dirty); } arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, flag_dirty); arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), dirty, false); } static arena_run_t * arena_bin_runs_first(arena_bin_t *bin) { arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); if (mapelm != NULL) { arena_chunk_t *chunk; size_t pageind; arena_run_t *run; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); return (run); } return (NULL); } static void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); arena_run_tree_insert(&bin->runs, mapelm); } static void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); arena_run_tree_remove(&bin->runs, mapelm); } static arena_run_t * arena_bin_nonfull_run_tryget(arena_bin_t *bin) { arena_run_t *run = arena_bin_runs_first(bin); if (run != NULL) { arena_bin_runs_remove(bin, run); if (config_stats) bin->stats.reruns++; } return (run); } static arena_run_t * arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) { arena_run_t *run; size_t binind; arena_bin_info_t *bin_info; /* Look for a usable run. */ run = arena_bin_nonfull_run_tryget(bin); if (run != NULL) return (run); /* No existing runs have any space available. */ binind = arena_bin_index(arena, bin); bin_info = &arena_bin_info[binind]; /* Allocate a new run. */ malloc_mutex_unlock(&bin->lock); /******************************/ malloc_mutex_lock(&arena->lock); run = arena_run_alloc_small(arena, bin_info->run_size, binind); if (run != NULL) { bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); /* Initialize run internals. */ run->bin = bin; run->nextind = 0; run->nfree = bin_info->nregs; bitmap_init(bitmap, &bin_info->bitmap_info); } malloc_mutex_unlock(&arena->lock); /********************************/ malloc_mutex_lock(&bin->lock); if (run != NULL) { if (config_stats) { bin->stats.nruns++; bin->stats.curruns++; } return (run); } /* * arena_run_alloc_small() failed, but another thread may have made * sufficient memory available while this one dropped bin->lock above, * so search one more time. */ run = arena_bin_nonfull_run_tryget(bin); if (run != NULL) return (run); return (NULL); } /* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ static void * arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) { void *ret; size_t binind; arena_bin_info_t *bin_info; arena_run_t *run; binind = arena_bin_index(arena, bin); bin_info = &arena_bin_info[binind]; bin->runcur = NULL; run = arena_bin_nonfull_run_get(arena, bin); if (bin->runcur != NULL && bin->runcur->nfree > 0) { /* * Another thread updated runcur while this one ran without the * bin lock in arena_bin_nonfull_run_get(). */ assert(bin->runcur->nfree > 0); ret = arena_run_reg_alloc(bin->runcur, bin_info); if (run != NULL) { arena_chunk_t *chunk; /* * arena_run_alloc_small() may have allocated run, or * it may have pulled run from the bin's run tree. * Therefore it is unsafe to make any assumptions about * how run has previously been used, and * arena_bin_lower_run() must be called, as if a region * were just deallocated from the run. */ chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); if (run->nfree == bin_info->nregs) arena_dalloc_bin_run(arena, chunk, run, bin); else arena_bin_lower_run(arena, chunk, run, bin); } return (ret); } if (run == NULL) return (NULL); bin->runcur = run; assert(bin->runcur->nfree > 0); return (arena_run_reg_alloc(bin->runcur, bin_info)); } void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, uint64_t prof_accumbytes) { unsigned i, nfill; arena_bin_t *bin; arena_run_t *run; void *ptr; assert(tbin->ncached == 0); if (config_prof && arena_prof_accum(arena, prof_accumbytes)) prof_idump(); bin = &arena->bins[binind]; malloc_mutex_lock(&bin->lock); for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> tbin->lg_fill_div); i < nfill; i++) { if ((run = bin->runcur) != NULL && run->nfree > 0) ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); else ptr = arena_bin_malloc_hard(arena, bin); if (ptr == NULL) break; if (config_fill && opt_junk) { arena_alloc_junk_small(ptr, &arena_bin_info[binind], true); } /* Insert such that low regions get used first. */ tbin->avail[nfill - 1 - i] = ptr; } if (config_stats) { bin->stats.allocated += i * arena_bin_info[binind].reg_size; bin->stats.nmalloc += i; bin->stats.nrequests += tbin->tstats.nrequests; bin->stats.nfills++; tbin->tstats.nrequests = 0; } malloc_mutex_unlock(&bin->lock); tbin->ncached = i; } void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) { if (zero) { size_t redzone_size = bin_info->redzone_size; memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, redzone_size); memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, redzone_size); } else { memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, bin_info->reg_interval); } } #ifdef JEMALLOC_JET #undef arena_redzone_corruption #define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) #endif static void arena_redzone_corruption(void *ptr, size_t usize, bool after, size_t offset, uint8_t byte) { malloc_printf(": Corrupt redzone %zu byte%s %s %p " "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", after ? "after" : "before", ptr, usize, byte); } #ifdef JEMALLOC_JET #undef arena_redzone_corruption #define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) arena_redzone_corruption_t *arena_redzone_corruption = JEMALLOC_N(arena_redzone_corruption_impl); #endif static void arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) { size_t size = bin_info->reg_size; size_t redzone_size = bin_info->redzone_size; size_t i; bool error = false; for (i = 1; i <= redzone_size; i++) { uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); if (*byte != 0xa5) { error = true; arena_redzone_corruption(ptr, size, false, i, *byte); if (reset) *byte = 0xa5; } } for (i = 0; i < redzone_size; i++) { uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); if (*byte != 0xa5) { error = true; arena_redzone_corruption(ptr, size, true, i, *byte); if (reset) *byte = 0xa5; } } if (opt_abort && error) abort(); } #ifdef JEMALLOC_JET #undef arena_dalloc_junk_small #define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) #endif void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) { size_t redzone_size = bin_info->redzone_size; arena_redzones_validate(ptr, bin_info, false); memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, bin_info->reg_interval); } #ifdef JEMALLOC_JET #undef arena_dalloc_junk_small #define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) arena_dalloc_junk_small_t *arena_dalloc_junk_small = JEMALLOC_N(arena_dalloc_junk_small_impl); #endif void arena_quarantine_junk_small(void *ptr, size_t usize) { size_t binind; arena_bin_info_t *bin_info; cassert(config_fill); assert(opt_junk); assert(opt_quarantine); assert(usize <= SMALL_MAXCLASS); binind = small_size2bin(usize); bin_info = &arena_bin_info[binind]; arena_redzones_validate(ptr, bin_info, true); } void * arena_malloc_small(arena_t *arena, size_t size, bool zero) { void *ret; arena_bin_t *bin; arena_run_t *run; size_t binind; binind = small_size2bin(size); assert(binind < NBINS); bin = &arena->bins[binind]; size = small_bin2size(binind); malloc_mutex_lock(&bin->lock); if ((run = bin->runcur) != NULL && run->nfree > 0) ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); else ret = arena_bin_malloc_hard(arena, bin); if (ret == NULL) { malloc_mutex_unlock(&bin->lock); return (NULL); } if (config_stats) { bin->stats.allocated += size; bin->stats.nmalloc++; bin->stats.nrequests++; } malloc_mutex_unlock(&bin->lock); if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) prof_idump(); if (zero == false) { if (config_fill) { if (opt_junk) { arena_alloc_junk_small(ret, &arena_bin_info[binind], false); } else if (opt_zero) memset(ret, 0, size); } JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { if (config_fill && opt_junk) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); } return (ret); } void * arena_malloc_large(arena_t *arena, size_t size, bool zero) { void *ret; UNUSED bool idump; /* Large allocation. */ size = PAGE_CEILING(size); malloc_mutex_lock(&arena->lock); ret = (void *)arena_run_alloc_large(arena, size, zero); if (ret == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); } if (config_stats) { arena->stats.nmalloc_large++; arena->stats.nrequests_large++; arena->stats.allocated_large += size; arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } if (config_prof) idump = arena_prof_accum_locked(arena, size); malloc_mutex_unlock(&arena->lock); if (config_prof && idump) prof_idump(); if (zero == false) { if (config_fill) { if (opt_junk) memset(ret, 0xa5, size); else if (opt_zero) memset(ret, 0, size); } } return (ret); } /* Only handles large allocations that require more than page alignment. */ void * arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) { void *ret; size_t alloc_size, leadsize, trailsize; arena_run_t *run; arena_chunk_t *chunk; assert((size & PAGE_MASK) == 0); alignment = PAGE_CEILING(alignment); alloc_size = size + alignment - PAGE; malloc_mutex_lock(&arena->lock); run = arena_run_alloc_large(arena, alloc_size, false); if (run == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); } chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - (uintptr_t)run; assert(alloc_size >= leadsize + size); trailsize = alloc_size - leadsize - size; ret = (void *)((uintptr_t)run + leadsize); if (leadsize != 0) { arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - leadsize); } if (trailsize != 0) { arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, false); } arena_run_init_large(arena, (arena_run_t *)ret, size, zero); if (config_stats) { arena->stats.nmalloc_large++; arena->stats.nrequests_large++; arena->stats.allocated_large += size; arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } malloc_mutex_unlock(&arena->lock); if (config_fill && zero == false) { if (opt_junk) memset(ret, 0xa5, size); else if (opt_zero) memset(ret, 0, size); } return (ret); } void arena_prof_promoted(const void *ptr, size_t size) { arena_chunk_t *chunk; size_t pageind, binind; cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); assert(isalloc(ptr, false) == PAGE); assert(isalloc(ptr, true) == PAGE); assert(size <= SMALL_MAXCLASS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; binind = small_size2bin(size); assert(binind < NBINS); arena_mapbits_large_binind_set(chunk, pageind, binind); assert(isalloc(ptr, false) == PAGE); assert(isalloc(ptr, true) == size); } static void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin) { /* Dissociate run from bin. */ if (run == bin->runcur) bin->runcur = NULL; else { size_t binind = arena_bin_index(chunk->arena, bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; if (bin_info->nregs != 1) { /* * This block's conditional is necessary because if the * run only contains one region, then it never gets * inserted into the non-full runs tree. */ arena_bin_runs_remove(bin, run); } } } static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin) { size_t binind; arena_bin_info_t *bin_info; size_t npages, run_ind, past; assert(run != bin->runcur); assert(arena_run_tree_search(&bin->runs, arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) == NULL); binind = arena_bin_index(chunk->arena, run->bin); bin_info = &arena_bin_info[binind]; malloc_mutex_unlock(&bin->lock); /******************************/ npages = bin_info->run_size >> LG_PAGE; run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); past = (size_t)(PAGE_CEILING((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * bin_info->reg_interval - bin_info->redzone_size) - (uintptr_t)chunk) >> LG_PAGE); malloc_mutex_lock(&arena->lock); /* * If the run was originally clean, and some pages were never touched, * trim the clean pages before deallocating the dirty portion of the * run. */ assert(arena_mapbits_dirty_get(chunk, run_ind) == arena_mapbits_dirty_get(chunk, run_ind+npages-1)); if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < npages) { /* Trim clean pages. Convert to large run beforehand. */ assert(npages > 0); arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), ((past - run_ind) << LG_PAGE), false); /* npages = past - run_ind; */ } arena_run_dalloc(arena, run, true, false); malloc_mutex_unlock(&arena->lock); /****************************/ malloc_mutex_lock(&bin->lock); if (config_stats) bin->stats.curruns--; } static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin) { /* * Make sure that if bin->runcur is non-NULL, it refers to the lowest * non-full run. It is okay to NULL runcur out rather than proactively * keeping it pointing at the lowest non-full run. */ if ((uintptr_t)run < (uintptr_t)bin->runcur) { /* Switch runcur. */ if (bin->runcur->nfree > 0) arena_bin_runs_insert(bin, bin->runcur); bin->runcur = run; if (config_stats) bin->stats.reruns++; } else arena_bin_runs_insert(bin, run); } void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_t *mapelm) { size_t pageind; arena_run_t *run; arena_bin_t *bin; arena_bin_info_t *bin_info; size_t size, binind; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); bin = run->bin; binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); bin_info = &arena_bin_info[binind]; if (config_fill || config_stats) size = bin_info->reg_size; if (config_fill && opt_junk) arena_dalloc_junk_small(ptr, bin_info); arena_run_reg_dalloc(run, ptr); if (run->nfree == bin_info->nregs) { arena_dissociate_bin_run(chunk, run, bin); arena_dalloc_bin_run(arena, chunk, run, bin); } else if (run->nfree == 1 && run != bin->runcur) arena_bin_lower_run(arena, chunk, run, bin); if (config_stats) { bin->stats.allocated -= size; bin->stats.ndalloc++; } } void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind, arena_chunk_map_t *mapelm) { arena_run_t *run; arena_bin_t *bin; run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); bin = run->bin; malloc_mutex_lock(&bin->lock); arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); malloc_mutex_unlock(&bin->lock); } void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind) { arena_chunk_map_t *mapelm; if (config_debug) { /* arena_ptr_small_binind_get() does extra sanity checking. */ assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)) != BININD_INVALID); } mapelm = arena_mapp_get(chunk, pageind); arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); } #ifdef JEMALLOC_JET #undef arena_dalloc_junk_large #define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) #endif static void arena_dalloc_junk_large(void *ptr, size_t usize) { if (config_fill && opt_junk) memset(ptr, 0x5a, usize); } #ifdef JEMALLOC_JET #undef arena_dalloc_junk_large #define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) arena_dalloc_junk_large_t *arena_dalloc_junk_large = JEMALLOC_N(arena_dalloc_junk_large_impl); #endif void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) { if (config_fill || config_stats) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t usize = arena_mapbits_large_size_get(chunk, pageind); arena_dalloc_junk_large(ptr, usize); if (config_stats) { arena->stats.ndalloc_large++; arena->stats.allocated_large -= usize; arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; } } arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); } void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) { malloc_mutex_lock(&arena->lock); arena_dalloc_large_locked(arena, chunk, ptr); malloc_mutex_unlock(&arena->lock); } static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t oldsize, size_t size) { assert(size < oldsize); /* * Shrink the run, and make trailing pages available for other * allocations. */ malloc_mutex_lock(&arena->lock); arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, true); if (config_stats) { arena->stats.ndalloc_large++; arena->stats.allocated_large -= oldsize; arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; arena->stats.nmalloc_large++; arena->stats.nrequests_large++; arena->stats.allocated_large += size; arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } malloc_mutex_unlock(&arena->lock); } static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t npages = oldsize >> LG_PAGE; size_t followsize; assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); /* Try to extend the run. */ assert(size + extra > oldsize); malloc_mutex_lock(&arena->lock); if (pageind + npages < chunk_npages && arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && (followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages)) >= size - oldsize) { /* * The next run is available and sufficiently large. Split the * following run, then merge the first part with the existing * allocation. */ size_t flag_dirty; size_t splitsize = (oldsize + followsize <= size + extra) ? followsize : size + extra - oldsize; arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + ((pageind+npages) << LG_PAGE)), splitsize, zero); size = oldsize + splitsize; npages = size >> LG_PAGE; /* * Mark the extended run as dirty if either portion of the run * was dirty before allocation. This is rather pedantic, * because there's not actually any sequence of events that * could cause the resulting run to be passed to * arena_run_dalloc() with the dirty argument set to false * (which is when dirty flag consistency would really matter). */ flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | arena_mapbits_dirty_get(chunk, pageind+npages-1); arena_mapbits_large_set(chunk, pageind, size, flag_dirty); arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); if (config_stats) { arena->stats.ndalloc_large++; arena->stats.allocated_large -= oldsize; arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; arena->stats.nmalloc_large++; arena->stats.nrequests_large++; arena->stats.allocated_large += size; arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } malloc_mutex_unlock(&arena->lock); return (false); } malloc_mutex_unlock(&arena->lock); return (true); } #ifdef JEMALLOC_JET #undef arena_ralloc_junk_large #define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) #endif static void arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) { if (config_fill && opt_junk) { memset((void *)((uintptr_t)ptr + usize), 0x5a, old_usize - usize); } } #ifdef JEMALLOC_JET #undef arena_ralloc_junk_large #define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) arena_ralloc_junk_large_t *arena_ralloc_junk_large = JEMALLOC_N(arena_ralloc_junk_large_impl); #endif /* * Try to resize a large allocation, in order to avoid copying. This will * always fail if growing an object, and the following run is already in use. */ static bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { size_t psize; psize = PAGE_CEILING(size + extra); if (psize == oldsize) { /* Same size class. */ return (false); } else { arena_chunk_t *chunk; arena_t *arena; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena = chunk->arena; if (psize < oldsize) { /* Fill before shrinking in order avoid a race. */ arena_ralloc_junk_large(ptr, oldsize, psize); arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, psize); return (false); } else { bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, PAGE_CEILING(size), psize - PAGE_CEILING(size), zero); if (config_fill && ret == false && zero == false) { if (opt_junk) { memset((void *)((uintptr_t)ptr + oldsize), 0xa5, isalloc(ptr, config_prof) - oldsize); } else if (opt_zero) { memset((void *)((uintptr_t)ptr + oldsize), 0, isalloc(ptr, config_prof) - oldsize); } } return (ret); } } } bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { /* * Avoid moving the allocation if the size class can be left the same. */ if (oldsize <= arena_maxclass) { if (oldsize <= SMALL_MAXCLASS) { assert(arena_bin_info[small_size2bin(oldsize)].reg_size == oldsize); if ((size + extra <= SMALL_MAXCLASS && small_size2bin(size + extra) == small_size2bin(oldsize)) || (size <= oldsize && size + extra >= oldsize)) return (false); } else { assert(size <= arena_maxclass); if (size + extra > SMALL_MAXCLASS) { if (arena_ralloc_large(ptr, oldsize, size, extra, zero) == false) return (false); } } } /* Reallocation would require a move. */ return (true); } void * arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc) { void *ret; size_t copysize; /* Try to avoid moving the allocation. */ if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) return (ptr); /* * size and oldsize are different enough that we need to move the * object. In that case, fall back to allocating new space and * copying. */ if (alignment != 0) { size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); } else ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment != 0) { size_t usize = sa2u(size, alignment); if (usize == 0) return (NULL); ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); } else ret = arena_malloc(arena, size, zero, try_tcache_alloc); if (ret == NULL) return (NULL); } /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ /* * Copy at most size bytes (not size+extra), since the caller has no * expectation that the extra bytes will be reliably preserved. */ copysize = (size < oldsize) ? size : oldsize; JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); iqalloct(ptr, try_tcache_dalloc); return (ret); } dss_prec_t arena_dss_prec_get(arena_t *arena) { dss_prec_t ret; malloc_mutex_lock(&arena->lock); ret = arena->dss_prec; malloc_mutex_unlock(&arena->lock); return (ret); } bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) { if (have_dss == false) return (dss_prec != dss_prec_disabled); malloc_mutex_lock(&arena->lock); arena->dss_prec = dss_prec; malloc_mutex_unlock(&arena->lock); return (false); } void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats) { unsigned i; malloc_mutex_lock(&arena->lock); *dss = dss_prec_names[arena->dss_prec]; *nactive += arena->nactive; *ndirty += arena->ndirty; astats->mapped += arena->stats.mapped; astats->npurge += arena->stats.npurge; astats->nmadvise += arena->stats.nmadvise; astats->purged += arena->stats.purged; astats->allocated_large += arena->stats.allocated_large; astats->nmalloc_large += arena->stats.nmalloc_large; astats->ndalloc_large += arena->stats.ndalloc_large; astats->nrequests_large += arena->stats.nrequests_large; astats->allocated_huge += arena->stats.allocated_huge; astats->nmalloc_huge += arena->stats.nmalloc_huge; astats->ndalloc_huge += arena->stats.ndalloc_huge; astats->nrequests_huge += arena->stats.nrequests_huge; for (i = 0; i < nlclasses; i++) { lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; lstats[i].nrequests += arena->stats.lstats[i].nrequests; lstats[i].curruns += arena->stats.lstats[i].curruns; } malloc_mutex_unlock(&arena->lock); for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(&bin->lock); bstats[i].allocated += bin->stats.allocated; bstats[i].nmalloc += bin->stats.nmalloc; bstats[i].ndalloc += bin->stats.ndalloc; bstats[i].nrequests += bin->stats.nrequests; if (config_tcache) { bstats[i].nfills += bin->stats.nfills; bstats[i].nflushes += bin->stats.nflushes; } bstats[i].nruns += bin->stats.nruns; bstats[i].reruns += bin->stats.reruns; bstats[i].curruns += bin->stats.curruns; malloc_mutex_unlock(&bin->lock); } } bool arena_new(arena_t *arena, unsigned ind) { unsigned i; arena_bin_t *bin; arena->ind = ind; arena->nthreads = 0; arena->chunk_alloc = chunk_alloc_default; arena->chunk_dalloc = chunk_dalloc_default; if (malloc_mutex_init(&arena->lock)) return (true); if (config_stats) { memset(&arena->stats, 0, sizeof(arena_stats_t)); arena->stats.lstats = (malloc_large_stats_t *)base_alloc(nlclasses * sizeof(malloc_large_stats_t)); if (arena->stats.lstats == NULL) return (true); memset(arena->stats.lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); if (config_tcache) ql_new(&arena->tcache_ql); } if (config_prof) arena->prof_accumbytes = 0; arena->dss_prec = chunk_dss_prec_get(); /* Initialize chunks. */ arena_chunk_dirty_new(&arena->chunks_dirty); arena->spare = NULL; arena->nactive = 0; arena->ndirty = 0; arena->npurgatory = 0; arena_avail_tree_new(&arena->runs_avail); /* Initialize bins. */ for (i = 0; i < NBINS; i++) { bin = &arena->bins[i]; if (malloc_mutex_init(&bin->lock)) return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); if (config_stats) memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } return (false); } /* * Calculate bin_info->run_size such that it meets the following constraints: * * *) bin_info->run_size >= min_run_size * *) bin_info->run_size <= arena_maxclass * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). * *) bin_info->nregs <= RUN_MAXREGS * * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also * calculated here, since these settings are all interdependent. */ static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) { size_t pad_size; size_t try_run_size, good_run_size; uint32_t try_nregs, good_nregs; uint32_t try_hdr_size, good_hdr_size; uint32_t try_bitmap_offset, good_bitmap_offset; uint32_t try_redzone0_offset, good_redzone0_offset; assert(min_run_size >= PAGE); assert(min_run_size <= arena_maxclass); /* * Determine redzone size based on minimum alignment and minimum * redzone size. Add padding to the end of the run if it is needed to * align the regions. The padding allows each redzone to be half the * minimum alignment; without the padding, each redzone would have to * be twice as large in order to maintain alignment. */ if (config_fill && opt_redzone) { size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - 1); if (align_min <= REDZONE_MINSIZE) { bin_info->redzone_size = REDZONE_MINSIZE; pad_size = 0; } else { bin_info->redzone_size = align_min >> 1; pad_size = bin_info->redzone_size; } } else { bin_info->redzone_size = 0; pad_size = 0; } bin_info->reg_interval = bin_info->reg_size + (bin_info->redzone_size << 1); /* * Calculate known-valid settings before entering the run_size * expansion loop, so that the first part of the loop always copies * valid settings. * * The do..while loop iteratively reduces the number of regions until * the run header and the regions no longer overlap. A closed formula * would be quite messy, since there is an interdependency between the * header's mask length and the number of regions. */ try_run_size = min_run_size; try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS + 1; /* Counter-act try_nregs-- in loop. */ } do { try_nregs--; try_hdr_size = sizeof(arena_run_t); /* Pad to a long boundary. */ try_hdr_size = LONG_CEILING(try_hdr_size); try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); try_redzone0_offset = try_run_size - (try_nregs * bin_info->reg_interval) - pad_size; } while (try_hdr_size > try_redzone0_offset); /* run_size expansion loop. */ do { /* * Copy valid settings before trying more aggressive settings. */ good_run_size = try_run_size; good_nregs = try_nregs; good_hdr_size = try_hdr_size; good_bitmap_offset = try_bitmap_offset; good_redzone0_offset = try_redzone0_offset; /* Try more aggressive settings. */ try_run_size += PAGE; try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS + 1; /* Counter-act try_nregs-- in loop. */ } do { try_nregs--; try_hdr_size = sizeof(arena_run_t); /* Pad to a long boundary. */ try_hdr_size = LONG_CEILING(try_hdr_size); try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); try_redzone0_offset = try_run_size - (try_nregs * bin_info->reg_interval) - pad_size; } while (try_hdr_size > try_redzone0_offset); } while (try_run_size <= arena_maxclass && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > RUN_MAX_OVRHD_RELAX && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size && try_nregs < RUN_MAXREGS); assert(good_hdr_size <= good_redzone0_offset); /* Copy final settings. */ bin_info->run_size = good_run_size; bin_info->nregs = good_nregs; bin_info->bitmap_offset = good_bitmap_offset; bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs * bin_info->reg_interval) + pad_size == bin_info->run_size); return (good_run_size); } static void bin_info_init(void) { arena_bin_info_t *bin_info; size_t prev_run_size = PAGE; #define BIN_INFO_INIT_bin_yes(index, size) \ bin_info = &arena_bin_info[index]; \ bin_info->reg_size = size; \ prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); #define BIN_INFO_INIT_bin_no(index, size) #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ BIN_INFO_INIT_bin_##bin(index, (ZU(1)<= the result * from (2), and will always be correct. */ map_bias = 0; for (i = 0; i < 3; i++) { header_size = offsetof(arena_chunk_t, map) + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) != 0); } assert(map_bias > 0); arena_maxclass = chunksize - (map_bias << LG_PAGE); bin_info_init(); } void arena_prefork(arena_t *arena) { unsigned i; malloc_mutex_prefork(&arena->lock); for (i = 0; i < NBINS; i++) malloc_mutex_prefork(&arena->bins[i].lock); } void arena_postfork_parent(arena_t *arena) { unsigned i; for (i = 0; i < NBINS; i++) malloc_mutex_postfork_parent(&arena->bins[i].lock); malloc_mutex_postfork_parent(&arena->lock); } void arena_postfork_child(arena_t *arena) { unsigned i; for (i = 0; i < NBINS; i++) malloc_mutex_postfork_child(&arena->bins[i].lock); malloc_mutex_postfork_child(&arena->lock); }