From 498856f44a30b31fe713a18eb2fc7c6ecf3a9f63 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 29 May 2016 18:34:50 -0700 Subject: [PATCH] Move slabs out of chunks. --- doc/jemalloc.xml.in | 98 +- include/jemalloc/internal/arena.h | 795 +------- include/jemalloc/internal/bitmap.h | 2 +- include/jemalloc/internal/extent.h | 86 +- include/jemalloc/internal/private_symbols.txt | 45 +- include/jemalloc/internal/prof.h | 44 +- include/jemalloc/internal/size_classes.sh | 22 +- include/jemalloc/internal/stats.h | 14 +- include/jemalloc/internal/tcache.h | 2 +- src/arena.c | 1633 ++++------------- src/base.c | 3 +- src/chunk.c | 11 +- src/chunk_dss.c | 2 +- src/ctl.c | 37 +- src/huge.c | 4 +- src/jemalloc.c | 46 +- src/stats.c | 45 +- src/tcache.c | 8 +- test/unit/extent_quantize.c | 10 +- test/unit/mallctl.c | 3 +- test/unit/stats.c | 18 +- 21 files changed, 596 insertions(+), 2332 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index efb4bfe4..923097d4 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -509,26 +509,20 @@ for (i = 0; i < nbins; i++) { In addition to multiple arenas, unless is specified during configuration, this - allocator supports thread-specific caching for small and large objects, in - order to make it possible to completely avoid synchronization for most - allocation requests. Such caching allows very fast allocation in the - common case, but it increases memory usage and fragmentation, since a - bounded number of objects can remain allocated in each thread cache. + allocator supports thread-specific caching, in order to make it possible to + completely avoid synchronization for most allocation requests. Such caching + allows very fast allocation in the common case, but it increases memory + usage and fragmentation, since a bounded number of objects can remain + allocated in each thread cache. - Memory is conceptually broken into equal-sized chunks, where the chunk - size is a power of two that is greater than the page size. Chunks are - always aligned to multiples of the chunk size. This alignment makes it - possible to find metadata for user objects very quickly. User objects are - broken into three categories according to size: small, large, and huge. - Multiple small and large objects can reside within a single chunk, whereas - huge objects each have one or more chunks backing them. Each chunk that - contains small and/or large objects tracks its contents as runs of - contiguous pages (unused, backing a set of small objects, or backing one - large object). The combination of chunk alignment and chunk page maps makes - it possible to determine all metadata regarding small and large allocations - in constant time. + Memory is conceptually broken into extents. Extents are always + aligned to multiples of the page size. This alignment makes it possible to + find metadata for user objects quickly. User objects are broken into two + categories according to size: small and large. Contiguous small objects + comprise a slab, which resides within a single extent, whereas large objects + each have their own extents backing them. - Small objects are managed in groups by page runs. Each run maintains + Small objects are managed in groups by slabs. Each slab maintains a bitmap to track which regions are in use. Allocation requests that are no more than half the quantum (8 or 16, depending on architecture) are rounded up to the nearest power of two that is at least opt.lg_chunk option), and - huge size classes extend from the chunk size up to the largest size class - that does not exceed PTRDIFF_MAX. + are smaller than four times the page size, and large size classes extend + from four times the page size up to the largest size class that does not + exceed PTRDIFF_MAX. Allocations are packed tightly together, which can be an issue for multi-threaded applications. If you need to assure that allocations do not @@ -560,18 +552,16 @@ for (i = 0; i < nbins; i++) { trivially succeeds in place as long as the pre-size and post-size both round up to the same size class. No other API guarantees are made regarding in-place resizing, but the current implementation also tries to resize large - and huge allocations in place, as long as the pre-size and post-size are - both large or both huge. In such cases shrinkage always succeeds for large - size classes, but for huge size classes the chunk allocator must support - splitting (see arena.<i>.chunk_hooks). - Growth only succeeds if the trailing memory is currently available, and - additionally for huge size classes the chunk allocator must support - merging. + Growth only succeeds if the trailing memory is currently available, and the + extent allocator supports merging. - Assuming 2 MiB chunks, 4 KiB pages, and a 16-byte quantum on a - 64-bit system, the size classes in each category are as shown in . + Assuming 4 KiB pages and a 16-byte quantum on a 64-bit system, the + size classes in each category are as shown in . Size classes @@ -625,7 +615,7 @@ for (i = 0; i < nbins; i++) { [10 KiB, 12 KiB, 14 KiB] - Large + Large 2 KiB [16 KiB] @@ -655,12 +645,7 @@ for (i = 0; i < nbins; i++) { 256 KiB - [1280 KiB, 1536 KiB, 1792 KiB] - - - Huge - 256 KiB - [2 MiB] + [1280 KiB, 1536 KiB, 1792 KiB, 2 MiB] 512 KiB @@ -1875,16 +1860,16 @@ typedef struct { (uint32_t) r- - Number of regions per page run. + Number of regions per slab. - + - arenas.bin.<i>.run_size + arenas.bin.<i>.slab_size (size_t) r- - Number of bytes per page run. + Number of bytes per slab. @@ -2185,7 +2170,7 @@ typedef struct { (size_t) r- - Number of pages in active runs. + Number of pages in active extents. @@ -2194,8 +2179,9 @@ typedef struct { (size_t) r- - Number of pages within unused runs that are potentially - dirty, and for which madvise... + Number of pages within unused extents that are + potentially dirty, and for which + madvise... MADV_DONTNEED or similar has not been called. @@ -2483,35 +2469,35 @@ typedef struct { Cumulative number of tcache flushes. - + - stats.arenas.<i>.bins.<j>.nruns + stats.arenas.<i>.bins.<j>.nslabs (uint64_t) r- [] - Cumulative number of runs created. + Cumulative number of slabs created. - + - stats.arenas.<i>.bins.<j>.nreruns + stats.arenas.<i>.bins.<j>.nreslabs (uint64_t) r- [] - Cumulative number of times the current run from which + Cumulative number of times the current slab from which to allocate changed. - + - stats.arenas.<i>.bins.<j>.curruns + stats.arenas.<i>.bins.<j>.curslabs (size_t) r- [] - Current number of runs. + Current number of slabs. diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index b0c4b5f3..d66548f2 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -3,9 +3,9 @@ #define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) -/* Maximum number of regions in one run. */ -#define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) -#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) +/* Maximum number of regions in one slab. */ +#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN) +#define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) /* * The minimum ratio of active:dirty pages per arena is computed as: @@ -29,12 +29,7 @@ typedef enum { /* Number of event ticks between time checks. */ #define DECAY_NTICKS_PER_UPDATE 1000 -typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; -typedef struct arena_avail_links_s arena_avail_links_t; -typedef struct arena_run_s arena_run_t; -typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; -typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; -typedef struct arena_chunk_s arena_chunk_t; +typedef struct arena_slab_data_s arena_slab_data_t; typedef struct arena_bin_info_s arena_bin_info_t; typedef struct arena_bin_s arena_bin_t; typedef struct arena_s arena_t; @@ -45,152 +40,25 @@ typedef struct arena_tdata_s arena_tdata_t; #ifdef JEMALLOC_H_STRUCTS #ifdef JEMALLOC_ARENA_STRUCTS_A -struct arena_run_s { - /* Index of bin this run is associated with. */ +struct arena_slab_data_s { + /* Index of bin this slab is associated with. */ szind_t binind; - /* Number of free regions in run. */ + /* Number of free regions in slab. */ unsigned nfree; /* Per region allocated/deallocated bitmap. */ bitmap_t bitmap[BITMAP_GROUPS_MAX]; }; - -/* Each element of the chunk map corresponds to one page within the chunk. */ -struct arena_chunk_map_bits_s { - /* - * Run address (or size) and various flags are stored together. The bit - * layout looks like (assuming 32-bit system): - * - * ???????? ???????? ???nnnnn nnndumla - * - * ? : Unallocated: Run address for first/last pages, unset for internal - * pages. - * Small: Run page offset. - * Large: Run page count for first page, unset for trailing pages. - * n : binind for small size class, BININD_INVALID for large size class. - * d : dirty? - * u : unzeroed? - * m : decommitted? - * l : large? - * a : allocated? - * - * Following are example bit patterns for the three types of runs. - * - * p : run page offset - * s : run size - * n : binind for size class; large objects set these to BININD_INVALID - * x : don't care - * - : 0 - * + : 1 - * [DUMLA] : bit set - * [dumla] : bit unset - * - * Unallocated (clean): - * ssssssss ssssssss sss+++++ +++dum-a - * xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx - * ssssssss ssssssss sss+++++ +++dUm-a - * - * Unallocated (dirty): - * ssssssss ssssssss sss+++++ +++D-m-a - * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx - * ssssssss ssssssss sss+++++ +++D-m-a - * - * Small: - * pppppppp pppppppp pppnnnnn nnnd---A - * pppppppp pppppppp pppnnnnn nnn----A - * pppppppp pppppppp pppnnnnn nnnd---A - * - * Large: - * ssssssss ssssssss sss+++++ +++D--LA - * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx - * -------- -------- ---+++++ +++D--LA - * - * Large (sampled, size <= LARGE_MINCLASS): - * ssssssss ssssssss sssnnnnn nnnD--LA - * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx - * -------- -------- ---+++++ +++D--LA - * - * Large (not sampled, size == LARGE_MINCLASS): - * ssssssss ssssssss sss+++++ +++D--LA - * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx - * -------- -------- ---+++++ +++D--LA - */ - size_t bits; -#define CHUNK_MAP_ALLOCATED ((size_t)0x01U) -#define CHUNK_MAP_LARGE ((size_t)0x02U) -#define CHUNK_MAP_STATE_MASK ((size_t)0x3U) - -#define CHUNK_MAP_DECOMMITTED ((size_t)0x04U) -#define CHUNK_MAP_UNZEROED ((size_t)0x08U) -#define CHUNK_MAP_DIRTY ((size_t)0x10U) -#define CHUNK_MAP_FLAGS_MASK ((size_t)0x1cU) - -#define CHUNK_MAP_BININD_SHIFT 5 -#define BININD_INVALID ((size_t)0xffU) -#define CHUNK_MAP_BININD_MASK (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) -#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK - -#define CHUNK_MAP_RUNIND_SHIFT (CHUNK_MAP_BININD_SHIFT + 8) -#define CHUNK_MAP_SIZE_SHIFT (CHUNK_MAP_RUNIND_SHIFT - LG_PAGE) -#define CHUNK_MAP_SIZE_MASK \ - (~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK)) -}; - -struct arena_runs_dirty_link_s { - qr(arena_runs_dirty_link_t) rd_link; -}; - -/* - * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just - * like arena_chunk_map_bits_t. Two separate arrays are stored within each - * chunk header in order to improve cache locality. - */ -struct arena_chunk_map_misc_s { - /* - * Linkage for run heaps. There are two disjoint uses: - * - * 1) arena_t's runs_avail heaps. - * 2) arena_run_t conceptually uses this linkage for in-use non-full - * runs, rather than directly embedding linkage. - */ - phn(arena_chunk_map_misc_t) ph_link; - - union { - /* Linkage for list of dirty runs. */ - arena_runs_dirty_link_t rd; - - /* Profile counters, used for large object runs. */ - union { - void *prof_tctx_pun; - prof_tctx_t *prof_tctx; - }; - - /* Small region run metadata. */ - arena_run_t run; - }; -}; -typedef ph(arena_chunk_map_misc_t) arena_run_heap_t; #endif /* JEMALLOC_ARENA_STRUCTS_A */ #ifdef JEMALLOC_ARENA_STRUCTS_B -/* Arena chunk header. */ -struct arena_chunk_s { - /* - * Map of pages within chunk that keeps track of free/large/small. The - * first map_bias entries are omitted, since the chunk header does not - * need to be tracked in the map. This omission saves a header page - * for common chunk sizes (e.g. 4 MiB). - */ - arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ -}; - /* * Read-only information associated with each element of arena_t's bins array * is stored separately, partly to reduce memory usage (only one copy, rather * than one per arena), but mainly to avoid false cacheline sharing. * - * Each run has the following layout: + * Each slab has the following layout: * * /--------------------\ * | region 0 | @@ -205,45 +73,42 @@ struct arena_chunk_s { * \--------------------/ */ struct arena_bin_info_s { - /* Size of regions in a run for this bin's size class. */ + /* Size of regions in a slab for this bin's size class. */ size_t reg_size; - /* Total size of a run for this bin's size class. */ - size_t run_size; + /* Total size of a slab for this bin's size class. */ + size_t slab_size; - /* Total number of regions in a run for this bin's size class. */ + /* Total number of regions in a slab for this bin's size class. */ uint32_t nregs; /* - * Metadata used to manipulate bitmaps for runs associated with this + * Metadata used to manipulate bitmaps for slabs associated with this * bin. */ bitmap_info_t bitmap_info; }; struct arena_bin_s { - /* - * All operations on runcur, runs, and stats require that lock be - * locked. Run allocation/deallocation are protected by the arena lock, - * which may be acquired while holding one or more bin locks, but not - * vise versa. - */ + /* All operations on arena_bin_t fields require lock ownership. */ malloc_mutex_t lock; /* - * Current run being used to service allocations of this bin's size - * class. + * Current slab being used to service allocations of this bin's size + * class. slabcur is independent of slabs_{nonfull,full}; whenever + * slabcur is reassigned, the previous slab must be deallocated or + * inserted into slabs_{nonfull,full}. */ - arena_run_t *runcur; + extent_t *slabcur; /* - * Heap of non-full runs. This heap is used when looking for an - * existing run when runcur is no longer usable. We choose the - * non-full run that is lowest in memory; this policy tends to keep - * objects packed well, and it can also help reduce the number of - * almost-empty chunks. + * Heap of non-full slabs. This heap is used to assure that new + * allocations come from the non-full slab that is lowest in memory. */ - arena_run_heap_t runs; + extent_heap_t slabs_nonfull; + + /* Ring sentinel used to track full slabs. */ + extent_t slabs_full; /* Bin statistics. */ malloc_bin_stats_t stats; @@ -272,7 +137,7 @@ struct arena_s { * perspective: * 1) Thread assignment (modifies nthreads) is synchronized via atomics. * 2) Bin-related operations are protected by bin locks. - * 3) Chunk- and run-related operations are protected by this mutex. + * 3) Chunk-related operations are protected by this mutex. */ malloc_mutex_t lock; @@ -294,32 +159,17 @@ struct arena_s { dss_prec_t dss_prec; - /* Extant arena chunks. */ - ql_head(extent_t) achunks; - - /* - * In order to avoid rapid chunk allocation/deallocation when an arena - * oscillates right on the cusp of needing a new chunk, cache the most - * recently freed chunk. The spare is left in the arena's chunk trees - * until it is deleted. - * - * There is one spare chunk per arena, rather than one spare total, in - * order to avoid interactions between multiple threads that could make - * a single spare inadequate. - */ - extent_t *spare; - /* Minimum ratio (log base 2) of nactive:ndirty. */ ssize_t lg_dirty_mult; /* True if a thread is currently executing arena_purge_to_limit(). */ bool purging; - /* Number of pages in active runs and huge regions. */ + /* Number of pages in active extents. */ size_t nactive; /* - * Current count of pages within unused runs that are potentially + * Current count of pages within unused extents that are potentially * dirty, and for which madvise(... MADV_DONTNEED) has not been called. * By tracking this, we can institute a limit on how much dirty unused * memory is mapped for each arena. @@ -327,35 +177,10 @@ struct arena_s { size_t ndirty; /* - * Unused dirty memory this arena manages. Dirty memory is conceptually - * tracked as an arbitrarily interleaved LRU of dirty runs and cached - * chunks, but the list linkage is actually semi-duplicated in order to - * avoid extra arena_chunk_map_misc_t space overhead. - * - * LRU-----------------------------------------------------------MRU - * - * /-- arena ---\ - * | | - * | | - * |------------| /-- chunk --\ - * ...->|chunks_cache|<--------------------------->| /------\ |<--... - * |------------| | |extent| | - * | | | | | | - * | | /- run -\ /- run -\ | | | | - * | | | | | | | | | | - * | | | | | | | | | | - * |------------| |-------| |-------| | |------| | - * ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----... - * |------------| |-------| |-------| | |------| | - * | | | | | | | | | | - * | | | | | | | \------/ | - * | | \-------/ \-------/ | | - * | | | | - * | | | | - * \------------/ \-----------/ + * Ring sentinel used to track unused dirty memory. Dirty memory is + * managed as an LRU of cached extents. */ - arena_runs_dirty_link_t runs_dirty; - extent_t chunks_cache; + extent_t extents_dirty; /* * Approximate time in seconds from the creation of a set of unused @@ -424,16 +249,8 @@ struct arena_s { /* User-configurable chunk hook functions. */ chunk_hooks_t chunk_hooks; - /* bins is used to store trees of free regions. */ + /* bins is used to store heaps of free regions. */ arena_bin_t bins[NBINS]; - - /* - * Size-segregated address-ordered heaps of this arena's available runs, - * used for first-best-fit run allocation. Runs are quantized, i.e. - * they reside in the last heap which corresponds to a size class less - * than or equal to the run size. - */ - arena_run_heap_t runs_avail[NPSIZES]; }; /* Used in conjunction with tsd for fast arena-related context lookup. */ @@ -461,15 +278,6 @@ extern ssize_t opt_decay_time; extern const arena_bin_info_t arena_bin_info[NBINS]; -extern size_t map_bias; /* Number of arena chunk header pages. */ -extern size_t map_misc_offset; -extern size_t arena_maxrun; /* Max run size for arenas. */ - -#ifdef JEMALLOC_JET -typedef size_t (run_quantize_t)(size_t); -extern run_quantize_t *run_quantize_floor; -extern run_quantize_t *run_quantize_ceil; -#endif extent_t *arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, bool *zero); @@ -514,10 +322,9 @@ void arena_prof_promote(tsdn_t *tsdn, extent_t *extent, const void *ptr, void arena_dalloc_promoted(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache, bool slow_path); void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, - arena_chunk_t *chunk, extent_t *extent, void *ptr, - arena_chunk_map_bits_t *bitselm); -void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, - extent_t *extent, void *ptr, size_t pageind); + extent_t *extent, void *ptr); +void arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, extent_t *extent, + void *ptr); bool arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr, @@ -552,70 +359,19 @@ void arena_postfork_child(tsdn_t *tsdn, arena_t *arena); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -arena_chunk_map_bits_t *arena_bitselm_get_mutable(arena_chunk_t *chunk, - size_t pageind); -const arena_chunk_map_bits_t *arena_bitselm_get_const( - const arena_chunk_t *chunk, size_t pageind); -arena_chunk_map_misc_t *arena_miscelm_get_mutable(arena_chunk_t *chunk, - size_t pageind); -const arena_chunk_map_misc_t *arena_miscelm_get_const( - const arena_chunk_t *chunk, size_t pageind); -size_t arena_miscelm_to_pageind(const extent_t *extent, - const arena_chunk_map_misc_t *miscelm); -void *arena_miscelm_to_rpages(const extent_t *extent, - const arena_chunk_map_misc_t *miscelm); -arena_chunk_map_misc_t *arena_rd_to_miscelm(const extent_t *extent, - arena_runs_dirty_link_t *rd); -arena_chunk_map_misc_t *arena_run_to_miscelm(const extent_t *extent, - arena_run_t *run); -size_t *arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind); -const size_t *arena_mapbitsp_get_const(const arena_chunk_t *chunk, - size_t pageind); -size_t arena_mapbitsp_read(const size_t *mapbitsp); -size_t arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind); -size_t arena_mapbits_size_decode(size_t mapbits); -size_t arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, - size_t pageind); -size_t arena_mapbits_large_size_get(const arena_chunk_t *chunk, - size_t pageind); -size_t arena_mapbits_small_runind_get(const arena_chunk_t *chunk, - size_t pageind); -szind_t arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind); -size_t arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind); -size_t arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind); -size_t arena_mapbits_decommitted_get(const arena_chunk_t *chunk, - size_t pageind); -size_t arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind); -size_t arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind); -void arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); -size_t arena_mapbits_size_encode(size_t size); -void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, - size_t size, size_t flags); -void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, - size_t size); -void arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, - size_t flags); -void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, - size_t size, size_t flags); -void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, - szind_t binind); -void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, - size_t runind, szind_t binind, size_t flags); void arena_metadata_allocated_add(arena_t *arena, size_t size); void arena_metadata_allocated_sub(arena_t *arena, size_t size); size_t arena_metadata_allocated_get(arena_t *arena); bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes); -szind_t arena_ptr_small_binind_get(tsdn_t *tsdn, const void *ptr, - size_t mapbits); szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr); void arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize, prof_tctx_t *tctx); void arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, - size_t usize, const void *old_ptr, prof_tctx_t *old_tctx); + prof_tctx_t *tctx); void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks); void arena_decay_tick(tsdn_t *tsdn, arena_t *arena); void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, @@ -630,330 +386,6 @@ void arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size, #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) # ifdef JEMALLOC_ARENA_INLINE_A -JEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * -arena_bitselm_get_mutable(arena_chunk_t *chunk, size_t pageind) -{ - - assert(pageind >= map_bias); - assert(pageind < chunk_npages); - - return (&chunk->map_bits[pageind-map_bias]); -} - -JEMALLOC_ALWAYS_INLINE const arena_chunk_map_bits_t * -arena_bitselm_get_const(const arena_chunk_t *chunk, size_t pageind) -{ - - return (arena_bitselm_get_mutable((arena_chunk_t *)chunk, pageind)); -} - -JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * -arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind) -{ - - assert(pageind >= map_bias); - assert(pageind < chunk_npages); - - return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + - (uintptr_t)map_misc_offset) + pageind-map_bias); -} - -JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t * -arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind) -{ - - return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind)); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_miscelm_to_pageind(const extent_t *extent, - const arena_chunk_map_misc_t *miscelm) -{ - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + - map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; - - assert(pageind >= map_bias); - assert(pageind < chunk_npages); - - return (pageind); -} - -JEMALLOC_ALWAYS_INLINE void * -arena_miscelm_to_rpages(const extent_t *extent, - const arena_chunk_map_misc_t *miscelm) -{ - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t pageind = arena_miscelm_to_pageind(extent, miscelm); - - return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); -} - -JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * -arena_rd_to_miscelm(const extent_t *extent, arena_runs_dirty_link_t *rd) -{ - arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t - *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); - - assert(arena_miscelm_to_pageind(extent, miscelm) >= map_bias); - assert(arena_miscelm_to_pageind(extent, miscelm) < chunk_npages); - - return (miscelm); -} - -JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * -arena_run_to_miscelm(const extent_t *extent, arena_run_t *run) -{ - arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t - *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); - - assert(arena_miscelm_to_pageind(extent, miscelm) >= map_bias); - assert(arena_miscelm_to_pageind(extent, miscelm) < chunk_npages); - - return (miscelm); -} - -JEMALLOC_ALWAYS_INLINE size_t * -arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind) -{ - - return (&arena_bitselm_get_mutable(chunk, pageind)->bits); -} - -JEMALLOC_ALWAYS_INLINE const size_t * -arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind) -{ - - return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind)); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbitsp_read(const size_t *mapbitsp) -{ - - return (*mapbitsp); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind) -{ - - return (arena_mapbitsp_read(arena_mapbitsp_get_const(chunk, pageind))); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_size_decode(size_t mapbits) -{ - size_t size; - -#if CHUNK_MAP_SIZE_SHIFT > 0 - size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; -#elif CHUNK_MAP_SIZE_SHIFT == 0 - size = mapbits & CHUNK_MAP_SIZE_MASK; -#else - size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; -#endif - - return (size); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); - return (arena_mapbits_size_decode(mapbits)); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_large_size_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == - (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); - return (arena_mapbits_size_decode(mapbits)); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_small_runind_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == - CHUNK_MAP_ALLOCATED); - return (mapbits >> CHUNK_MAP_RUNIND_SHIFT); -} - -JEMALLOC_ALWAYS_INLINE szind_t -arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - szind_t binind; - - mapbits = arena_mapbits_get(chunk, pageind); - binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; - assert(binind < NBINS || binind == BININD_INVALID); - return (binind); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & - (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); - return (mapbits & CHUNK_MAP_DIRTY); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & - (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); - return (mapbits & CHUNK_MAP_UNZEROED); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_decommitted_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits & - (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); - return (mapbits & CHUNK_MAP_DECOMMITTED); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - return (mapbits & CHUNK_MAP_LARGE); -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind) -{ - size_t mapbits; - - mapbits = arena_mapbits_get(chunk, pageind); - return (mapbits & CHUNK_MAP_ALLOCATED); -} - -JEMALLOC_ALWAYS_INLINE void -arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) -{ - - *mapbitsp = mapbits; -} - -JEMALLOC_ALWAYS_INLINE size_t -arena_mapbits_size_encode(size_t size) -{ - size_t mapbits; - -#if CHUNK_MAP_SIZE_SHIFT > 0 - mapbits = size << CHUNK_MAP_SIZE_SHIFT; -#elif CHUNK_MAP_SIZE_SHIFT == 0 - mapbits = size; -#else - mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; -#endif - - assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0); - return (mapbits); -} - -JEMALLOC_ALWAYS_INLINE void -arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, - size_t flags) -{ - size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); - - assert((size & PAGE_MASK) == 0); - assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); - assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & - (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); - arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | - CHUNK_MAP_BININD_INVALID | flags); -} - -JEMALLOC_ALWAYS_INLINE void -arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, - size_t size) -{ - size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); - size_t mapbits = arena_mapbitsp_read(mapbitsp); - - assert((size & PAGE_MASK) == 0); - assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); - arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | - (mapbits & ~CHUNK_MAP_SIZE_MASK)); -} - -JEMALLOC_ALWAYS_INLINE void -arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, size_t flags) -{ - size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); - - assert((flags & CHUNK_MAP_UNZEROED) == flags); - arena_mapbitsp_write(mapbitsp, flags); -} - -JEMALLOC_ALWAYS_INLINE void -arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, - size_t flags) -{ - size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); - - assert((size & PAGE_MASK) == 0); - assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); - assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & - (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); - arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | - CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE | - CHUNK_MAP_ALLOCATED); -} - -JEMALLOC_ALWAYS_INLINE void -arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, - szind_t binind) -{ - size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); - size_t mapbits = arena_mapbitsp_read(mapbitsp); - - assert(binind <= BININD_INVALID); - assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS + - large_pad); - arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | - (binind << CHUNK_MAP_BININD_SHIFT)); -} - -JEMALLOC_ALWAYS_INLINE void -arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, - szind_t binind, size_t flags) -{ - size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind); - - assert(binind < BININD_INVALID); - assert(pageind - runind >= map_bias); - assert((flags & CHUNK_MAP_UNZEROED) == flags); - arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) | - (binind << CHUNK_MAP_BININD_SHIFT) | flags | CHUNK_MAP_ALLOCATED); -} - JEMALLOC_INLINE void arena_metadata_allocated_add(arena_t *arena, size_t size) { @@ -1022,54 +454,6 @@ arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) # endif /* JEMALLOC_ARENA_INLINE_A */ # ifdef JEMALLOC_ARENA_INLINE_B -JEMALLOC_ALWAYS_INLINE szind_t -arena_ptr_small_binind_get(tsdn_t *tsdn, const void *ptr, size_t mapbits) -{ - szind_t binind; - - binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; - - if (config_debug) { - const extent_t *extent; - arena_chunk_t *chunk; - arena_t *arena; - size_t pageind; - size_t actual_mapbits; - size_t rpages_ind; - const arena_run_t *run; - arena_bin_t *bin; - szind_t run_binind, actual_binind; - const arena_bin_info_t *bin_info; - const arena_chunk_map_misc_t *miscelm; - const void *rpages; - - assert(binind != BININD_INVALID); - assert(binind < NBINS); - extent = iealloc(tsdn, ptr); - chunk = (arena_chunk_t *)extent_base_get(extent); - arena = extent_arena_get(extent); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - actual_mapbits = arena_mapbits_get(chunk, pageind); - assert(mapbits == actual_mapbits); - assert(arena_mapbits_large_get(chunk, pageind) == 0); - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, - pageind); - miscelm = arena_miscelm_get_const(chunk, rpages_ind); - run = &miscelm->run; - run_binind = run->binind; - bin = &arena->bins[run_binind]; - actual_binind = (szind_t)(bin - arena->bins); - assert(run_binind == actual_binind); - bin_info = &arena_bin_info[actual_binind]; - rpages = arena_miscelm_to_rpages(extent, miscelm); - assert(((uintptr_t)ptr - (uintptr_t)rpages) % bin_info->reg_size - == 0); - } - - return (binind); -} - JEMALLOC_INLINE szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin) { @@ -1081,27 +465,13 @@ arena_bin_index(arena_t *arena, arena_bin_t *bin) JEMALLOC_INLINE prof_tctx_t * arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr) { - prof_tctx_t *ret; cassert(config_prof); assert(ptr != NULL); - if (likely(extent_slab_get(extent))) { - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - size_t mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) - ret = (prof_tctx_t *)(uintptr_t)1U; - else { - arena_chunk_map_misc_t *elm = - arena_miscelm_get_mutable(chunk, pageind); - ret = atomic_read_p(&elm->prof_tctx_pun); - } - } else - ret = huge_prof_tctx_get(tsdn, extent); - - return (ret); + if (unlikely(!extent_slab_get(extent))) + return (huge_prof_tctx_get(tsdn, extent)); + return ((prof_tctx_t *)(uintptr_t)1U); } JEMALLOC_INLINE void @@ -1112,61 +482,20 @@ arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, cassert(config_prof); assert(ptr != NULL); - if (likely(extent_slab_get(extent))) { - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - - if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx > - (uintptr_t)1U)) { - arena_chunk_map_misc_t *elm; - - assert(arena_mapbits_large_get(chunk, pageind) != 0); - - elm = arena_miscelm_get_mutable(chunk, pageind); - atomic_write_p(&elm->prof_tctx_pun, tctx); - } else { - /* - * tctx must always be initialized for large runs. - * Assert that the surrounding conditional logic is - * equivalent to checking whether ptr refers to a large - * run. - */ - assert(arena_mapbits_large_get(chunk, pageind) == 0); - } - } else + if (unlikely(!extent_slab_get(extent))) huge_prof_tctx_set(tsdn, extent, tctx); } JEMALLOC_INLINE void arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, - size_t usize, const void *old_ptr, prof_tctx_t *old_tctx) + prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); + assert(!extent_slab_get(extent)); - if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr && - (uintptr_t)old_tctx > (uintptr_t)1U))) { - if (likely(extent_slab_get(extent))) { - arena_chunk_t *chunk = - (arena_chunk_t *)extent_base_get(extent); - size_t pageind; - arena_chunk_map_misc_t *elm; - - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> - LG_PAGE; - assert(arena_mapbits_allocated_get(chunk, pageind) != - 0); - assert(arena_mapbits_large_get(chunk, pageind) != 0); - - elm = arena_miscelm_get_mutable(chunk, pageind); - atomic_write_p(&elm->prof_tctx_pun, - (prof_tctx_t *)(uintptr_t)1U); - } else - huge_prof_tctx_reset(tsdn, extent); - } + huge_prof_tctx_reset(tsdn, extent); } JEMALLOC_ALWAYS_INLINE void @@ -1231,20 +560,9 @@ arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr) assert(ptr != NULL); - if (likely(extent_slab_get(extent))) { - const arena_chunk_t *chunk = - (const arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - szind_t binind; - - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - binind = arena_mapbits_binind_get(chunk, pageind); - /* Small allocation. */ - assert(arena_mapbits_large_get(chunk, pageind) != 0 || - arena_ptr_small_binind_get(tsdn, ptr, - arena_mapbits_get(chunk, pageind)) == binind); - ret = index2size(binind); - } else + if (likely(extent_slab_get(extent))) + ret = index2size(extent_slab_data_get_const(extent)->binind); + else ret = huge_salloc(tsdn, extent); return (ret); @@ -1260,19 +578,13 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache, if (likely(extent_slab_get(extent))) { /* Small allocation. */ - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - size_t mapbits = arena_mapbits_get(chunk, pageind); - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - assert((mapbits & CHUNK_MAP_LARGE) == 0); if (likely(tcache != NULL)) { - szind_t binind = arena_ptr_small_binind_get(tsdn, ptr, - mapbits); + szind_t binind = extent_slab_data_get(extent)->binind; tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, binind, slow_path); } else { arena_dalloc_small(tsdn, extent_arena_get(extent), - chunk, extent, ptr, pageind); + extent, ptr); } } else { size_t usize = extent_usize_get(extent); @@ -1282,8 +594,8 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache, arena_dalloc_promoted(tsdn, extent, ptr, tcache, slow_path); } else { - tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, ptr, - usize, slow_path); + tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, + ptr, usize, slow_path); } } else huge_dalloc(tsdn, extent); @@ -1302,15 +614,12 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size, /* Small allocation. */ if (likely(tcache != NULL)) { szind_t binind = size2index(size); + assert(binind == extent_slab_data_get(extent)->binind); tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, binind, slow_path); } else { - arena_chunk_t *chunk = - (arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)ptr - - (uintptr_t)chunk) >> LG_PAGE; arena_dalloc_small(tsdn, extent_arena_get(extent), - chunk, extent, ptr, pageind); + extent, ptr); } } else { if (likely(tcache != NULL) && size <= tcache_maxclass) { diff --git a/include/jemalloc/internal/bitmap.h b/include/jemalloc/internal/bitmap.h index 0d456e2d..c2e34554 100644 --- a/include/jemalloc/internal/bitmap.h +++ b/include/jemalloc/internal/bitmap.h @@ -2,7 +2,7 @@ #ifdef JEMALLOC_H_TYPES /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ -#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS +#define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS #define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) typedef struct bitmap_level_s bitmap_level_t; diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 4e1e97ea..bfe61811 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -27,9 +27,6 @@ struct extent_s { /* True if extent is active (in use). */ bool e_active; - /* True if extent is dirty (touched). */ - bool e_dirty; - /* * The zeroed flag is used by chunk recycling code to track whether * memory is zero-filled. @@ -50,21 +47,27 @@ struct extent_s { */ bool e_slab; - /* Profile counters, used for huge objects. */ union { - void *e_prof_tctx_pun; - prof_tctx_t *e_prof_tctx; + /* Small region slab metadata. */ + arena_slab_data_t e_slab_data; + + /* Profile counters, used for huge objects. */ + union { + void *e_prof_tctx_pun; + prof_tctx_t *e_prof_tctx; + }; }; - /* Linkage for arena's runs_dirty and chunks_cache rings. */ - arena_runs_dirty_link_t rd; - qr(extent_t) cc_link; + /* + * Linkage for arena's extents_dirty and arena_bin_t's slabs_full rings. + */ + qr(extent_t) qr_link; union { /* Linkage for per size class address-ordered heaps. */ phn(extent_t) ph_link; - /* Linkage for arena's achunks, huge, and node_cache lists. */ + /* Linkage for arena's huge and extent_cache lists. */ ql_elm(extent_t) ql_link; }; }; @@ -102,11 +105,12 @@ void *extent_before_get(const extent_t *extent); void *extent_last_get(const extent_t *extent); void *extent_past_get(const extent_t *extent); bool extent_active_get(const extent_t *extent); -bool extent_dirty_get(const extent_t *extent); bool extent_retained_get(const extent_t *extent); bool extent_zeroed_get(const extent_t *extent); bool extent_committed_get(const extent_t *extent); bool extent_slab_get(const extent_t *extent); +arena_slab_data_t *extent_slab_data_get(extent_t *extent); +const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent); prof_tctx_t *extent_prof_tctx_get(const extent_t *extent); void extent_arena_set(extent_t *extent, arena_t *arena); void extent_addr_set(extent_t *extent, void *addr); @@ -114,17 +118,15 @@ void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment); void extent_size_set(extent_t *extent, size_t size); void extent_usize_set(extent_t *extent, size_t usize); void extent_active_set(extent_t *extent, bool active); -void extent_dirty_set(extent_t *extent, bool dirty); void extent_zeroed_set(extent_t *extent, bool zeroed); void extent_committed_set(extent_t *extent, bool committed); void extent_slab_set(extent_t *extent, bool slab); void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx); void extent_init(extent_t *extent, arena_t *arena, void *addr, - size_t size, size_t usize, bool active, bool dirty, bool zeroed, - bool committed, bool slab); -void extent_dirty_insert(extent_t *extent, - arena_runs_dirty_link_t *runs_dirty, extent_t *chunks_dirty); -void extent_dirty_remove(extent_t *extent); + size_t size, size_t usize, bool active, bool zeroed, bool committed, + bool slab); +void extent_ring_insert(extent_t *sentinel, extent_t *extent); +void extent_ring_remove(extent_t *extent); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) @@ -197,18 +199,11 @@ extent_active_get(const extent_t *extent) return (extent->e_active); } -JEMALLOC_INLINE bool -extent_dirty_get(const extent_t *extent) -{ - - return (extent->e_dirty); -} - JEMALLOC_INLINE bool extent_retained_get(const extent_t *extent) { - return (qr_next(&extent->rd, rd_link) == &extent->rd); + return (qr_next(extent, qr_link) == extent); } JEMALLOC_INLINE bool @@ -232,6 +227,22 @@ extent_slab_get(const extent_t *extent) return (extent->e_slab); } +JEMALLOC_INLINE arena_slab_data_t * +extent_slab_data_get(extent_t *extent) +{ + + assert(extent->e_slab); + return (&extent->e_slab_data); +} + +JEMALLOC_INLINE const arena_slab_data_t * +extent_slab_data_get_const(const extent_t *extent) +{ + + assert(extent->e_slab); + return (&extent->e_slab_data); +} + JEMALLOC_INLINE prof_tctx_t * extent_prof_tctx_get(const extent_t *extent) { @@ -296,13 +307,6 @@ extent_active_set(extent_t *extent, bool active) extent->e_active = active; } -JEMALLOC_INLINE void -extent_dirty_set(extent_t *extent, bool dirty) -{ - - extent->e_dirty = dirty; -} - JEMALLOC_INLINE void extent_zeroed_set(extent_t *extent, bool zeroed) { @@ -333,8 +337,7 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) JEMALLOC_INLINE void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, - size_t usize, bool active, bool dirty, bool zeroed, bool committed, - bool slab) + size_t usize, bool active, bool zeroed, bool committed, bool slab) { assert(addr == PAGE_ADDR2BASE(addr) || !slab); @@ -344,31 +347,26 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, extent_size_set(extent, size); extent_usize_set(extent, usize); extent_active_set(extent, active); - extent_dirty_set(extent, dirty); extent_zeroed_set(extent, zeroed); extent_committed_set(extent, committed); extent_slab_set(extent, slab); if (config_prof) extent_prof_tctx_set(extent, NULL); - qr_new(&extent->rd, rd_link); - qr_new(extent, cc_link); + qr_new(extent, qr_link); } JEMALLOC_INLINE void -extent_dirty_insert(extent_t *extent, - arena_runs_dirty_link_t *runs_dirty, extent_t *chunks_dirty) +extent_ring_insert(extent_t *sentinel, extent_t *extent) { - qr_meld(runs_dirty, &extent->rd, rd_link); - qr_meld(chunks_dirty, extent, cc_link); + qr_meld(sentinel, extent, qr_link); } JEMALLOC_INLINE void -extent_dirty_remove(extent_t *extent) +extent_ring_remove(extent_t *extent) { - qr_remove(&extent->rd, rd_link); - qr_remove(extent, cc_link); + qr_remove(extent, qr_link); } #endif diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 5f94d2c2..676c2431 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -5,8 +5,6 @@ arena_alloc_junk_small arena_basic_stats_merge arena_bin_index arena_bin_info -arena_bitselm_get_const -arena_bitselm_get_mutable arena_boot arena_choose arena_choose_hard @@ -43,38 +41,11 @@ arena_lg_dirty_mult_get arena_lg_dirty_mult_set arena_malloc arena_malloc_hard -arena_mapbits_allocated_get -arena_mapbits_binind_get -arena_mapbits_decommitted_get -arena_mapbits_dirty_get -arena_mapbits_get -arena_mapbits_internal_set -arena_mapbits_large_binind_set -arena_mapbits_large_get -arena_mapbits_large_set -arena_mapbits_large_size_get -arena_mapbits_size_decode -arena_mapbits_size_encode -arena_mapbits_small_runind_get -arena_mapbits_small_set -arena_mapbits_unallocated_set -arena_mapbits_unallocated_size_get -arena_mapbits_unallocated_size_set -arena_mapbits_unzeroed_get -arena_mapbitsp_get_const -arena_mapbitsp_get_mutable -arena_mapbitsp_read -arena_mapbitsp_write -arena_maxrun arena_maybe_purge arena_metadata_allocated_add arena_metadata_allocated_get arena_metadata_allocated_sub arena_migrate -arena_miscelm_get_const -arena_miscelm_get_mutable -arena_miscelm_to_pageind -arena_miscelm_to_rpages arena_new arena_nthreads_dec arena_nthreads_get @@ -93,14 +64,11 @@ arena_prof_promote arena_prof_tctx_get arena_prof_tctx_reset arena_prof_tctx_set -arena_ptr_small_binind_get arena_purge arena_ralloc arena_ralloc_junk_large arena_ralloc_no_move -arena_rd_to_miscelm arena_reset -arena_run_to_miscelm arena_salloc arena_sdalloc arena_stats_merge @@ -213,22 +181,23 @@ extent_before_get extent_committed_get extent_committed_set extent_dalloc -extent_dirty_get -extent_dirty_insert -extent_dirty_remove -extent_dirty_set extent_init extent_last_get extent_past_get extent_prof_tctx_get extent_prof_tctx_set extent_retained_get +extent_ring_insert +extent_ring_remove extent_size_get extent_size_set extent_size_quantize_ceil extent_size_quantize_floor +extent_slab_data_get +extent_slab_data_get_const extent_slab_get extent_slab_set +extent_slab_data_get extent_usize_get extent_zeroed_get extent_zeroed_set @@ -309,8 +278,6 @@ malloc_tsd_no_cleanup malloc_vcprintf malloc_vsnprintf malloc_write -map_bias -map_misc_offset mb_write narenas_auto narenas_tdata_cleanup @@ -451,8 +418,6 @@ rtree_subtree_read rtree_subtree_read_hard rtree_subtree_tryread rtree_write -run_quantize_ceil -run_quantize_floor s2u s2u_compute s2u_lookup diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 7da20ad0..8fdc27f6 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -335,8 +335,8 @@ prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, void prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, - size_t usize, const void *old_ptr, prof_tctx_t *tctx); -bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, + prof_tctx_t *tctx); +bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, prof_tdata_t **tdata_out); prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update); @@ -344,7 +344,8 @@ void prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated, - const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx); + extent_t *old_extent, const void *old_ptr, size_t old_usize, + prof_tctx_t *old_tctx); void prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr, size_t usize); #endif @@ -421,14 +422,14 @@ prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize, } JEMALLOC_ALWAYS_INLINE void -prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize, - const void *old_ptr, prof_tctx_t *old_tctx) +prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, + prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); - arena_prof_tctx_reset(tsdn, extent, ptr, usize, old_ptr, old_tctx); + arena_prof_tctx_reset(tsdn, extent, ptr, tctx); } JEMALLOC_ALWAYS_INLINE bool @@ -501,10 +502,10 @@ prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize, JEMALLOC_ALWAYS_INLINE void prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize, - prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr, - size_t old_usize, prof_tctx_t *old_tctx) + prof_tctx_t *tctx, bool prof_active, bool updated, extent_t *old_extent, + const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx) { - bool sampled, old_sampled; + bool sampled, old_sampled, moved; cassert(config_prof); assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); @@ -523,19 +524,30 @@ prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize, } } + /* + * The following code must differentiate among eight possible cases, + * based on three boolean conditions. + */ sampled = ((uintptr_t)tctx > (uintptr_t)1U); old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U); + moved = (ptr != old_ptr); + + /* + * The following block must only execute if this is a non-moving + * reallocation, because for moving reallocation the old allocation will + * be deallocated via a separate call. + */ + if (unlikely(old_sampled) && !moved) + prof_free_sampled_object(tsd, old_usize, old_tctx); if (unlikely(sampled)) { prof_malloc_sample_object(tsd_tsdn(tsd), extent, ptr, usize, tctx); - } else { - prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, usize, old_ptr, - old_tctx); - } - - if (unlikely(old_sampled)) - prof_free_sampled_object(tsd, old_usize, old_tctx); + } else if (moved) { + prof_tctx_set(tsd_tsdn(tsd), extent, ptr, usize, + (prof_tctx_t *)(uintptr_t)1U); + } else if (unlikely(old_sampled)) + prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, tctx); } JEMALLOC_ALWAYS_INLINE void diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 440953ad..b73064d1 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -50,7 +50,7 @@ reg_size_compute() { reg_size=$((${grp} + ${delta}*${ndelta})) } -run_size() { +slab_size() { lg_p=$1 lg_grp=$2 lg_delta=$3 @@ -59,22 +59,22 @@ run_size() { pow2 ${lg_p}; p=${pow2_result} reg_size_compute ${lg_grp} ${lg_delta} ${ndelta} - # Compute smallest run size that is an integer multiple of reg_size. - try_run_size=${p} - try_nregs=$((${try_run_size} / ${reg_size})) + # Compute smallest slab size that is an integer multiple of reg_size. + try_slab_size=${p} + try_nregs=$((${try_slab_size} / ${reg_size})) perfect=0 while [ ${perfect} -eq 0 ] ; do - perfect_run_size=${try_run_size} + perfect_slab_size=${try_slab_size} perfect_nregs=${try_nregs} - try_run_size=$((${try_run_size} + ${p})) - try_nregs=$((${try_run_size} / ${reg_size})) - if [ ${perfect_run_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then + try_slab_size=$((${try_slab_size} + ${p})) + try_nregs=$((${try_slab_size} / ${reg_size})) + if [ ${perfect_slab_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then perfect=1 fi done - run_size_pgs=$((${perfect_run_size} / ${p})) + slab_size_pgs=$((${perfect_slab_size} / ${p})) } size_class() { @@ -117,7 +117,7 @@ size_class() { if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then bin="yes" - run_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${run_size_pgs} + slab_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${slab_size_pgs} else bin="no" pgs=0 @@ -278,7 +278,7 @@ cat < b_miscelm) - (a_miscelm < b_miscelm)); -} - -/* Generate pairing heap functions. */ -ph_gen(static UNUSED, arena_run_heap_, arena_run_heap_t, arena_chunk_map_misc_t, - ph_link, arena_run_addr_comp) - -#ifdef JEMALLOC_JET -#undef run_quantize_floor -#define run_quantize_floor JEMALLOC_N(n_run_quantize_floor) -#endif -static size_t -run_quantize_floor(size_t size) -{ - size_t ret; - pszind_t pind; - - assert(size > 0); - assert(size <= HUGE_MAXCLASS); - assert((size & PAGE_MASK) == 0); - - assert(size != 0); - assert(size == PAGE_CEILING(size)); - - pind = psz2ind(size - large_pad + 1); - if (pind == 0) { - /* - * Avoid underflow. This short-circuit would also do the right - * thing for all sizes in the range for which there are - * PAGE-spaced size classes, but it's simplest to just handle - * the one case that would cause erroneous results. - */ - return (size); - } - ret = pind2sz(pind - 1) + large_pad; - assert(ret <= size); - return (ret); -} -#ifdef JEMALLOC_JET -#undef run_quantize_floor -#define run_quantize_floor JEMALLOC_N(run_quantize_floor) -run_quantize_t *run_quantize_floor = JEMALLOC_N(n_run_quantize_floor); -#endif - -#ifdef JEMALLOC_JET -#undef run_quantize_ceil -#define run_quantize_ceil JEMALLOC_N(n_run_quantize_ceil) -#endif -static size_t -run_quantize_ceil(size_t size) -{ - size_t ret; - - assert(size > 0); - assert(size <= HUGE_MAXCLASS); - assert((size & PAGE_MASK) == 0); - - ret = run_quantize_floor(size); - if (ret < size) { - /* - * Skip a quantization that may have an adequately large run, - * because under-sized runs may be mixed in. This only happens - * when an unusual size is requested, i.e. for aligned - * allocation, and is just one of several places where linear - * search would potentially find sufficiently aligned available - * memory somewhere lower. - */ - ret = pind2sz(psz2ind(ret - large_pad + 1)) + large_pad; - } - return (ret); -} -#ifdef JEMALLOC_JET -#undef run_quantize_ceil -#define run_quantize_ceil JEMALLOC_N(run_quantize_ceil) -run_quantize_t *run_quantize_ceil = JEMALLOC_N(n_run_quantize_ceil); -#endif - -static void -arena_avail_insert(arena_t *arena, extent_t *extent, size_t pageind, - size_t npages) -{ - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( - extent, arena_miscelm_get_const(chunk, pageind)))); - assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> - LG_PAGE)); - arena_run_heap_insert(&arena->runs_avail[pind], - arena_miscelm_get_mutable(chunk, pageind)); -} - -static void -arena_avail_remove(arena_t *arena, extent_t *extent, size_t pageind, - size_t npages) -{ - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get( - extent, arena_miscelm_get_const(chunk, pageind)))); - assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> - LG_PAGE)); - arena_run_heap_remove(&arena->runs_avail[pind], - arena_miscelm_get_mutable(chunk, pageind)); -} - -static void -arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, - size_t npages) -{ - arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, - pageind); - - assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> - LG_PAGE)); - assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); - assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == - CHUNK_MAP_DIRTY); - - qr_new(&miscelm->rd, rd_link); - qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link); - arena->ndirty += npages; -} - -static void -arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, - size_t npages) -{ - arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk, - pageind); - - assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> - LG_PAGE)); - assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); - assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == - CHUNK_MAP_DIRTY); - - qr_remove(&miscelm->rd, rd_link); - assert(arena->ndirty >= npages); - arena->ndirty -= npages; -} - static size_t arena_chunk_dirty_npages(const extent_t *extent) { @@ -269,8 +105,7 @@ arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache) { if (cache) { - extent_dirty_insert(extent, &arena->runs_dirty, - &arena->chunks_cache); + extent_ring_insert(&arena->extents_dirty, extent); arena->ndirty += arena_chunk_dirty_npages(extent); } } @@ -280,54 +115,49 @@ arena_chunk_cache_maybe_remove(arena_t *arena, extent_t *extent, bool dirty) { if (dirty) { - extent_dirty_remove(extent); + extent_ring_remove(extent); assert(arena->ndirty >= arena_chunk_dirty_npages(extent)); arena->ndirty -= arena_chunk_dirty_npages(extent); } } JEMALLOC_INLINE_C void * -arena_run_reg_alloc(tsdn_t *tsdn, arena_run_t *run, +arena_slab_reg_alloc(tsdn_t *tsdn, extent_t *slab, const arena_bin_info_t *bin_info) { void *ret; - extent_t *extent; + arena_slab_data_t *slab_data = extent_slab_data_get(slab); size_t regind; - arena_chunk_map_misc_t *miscelm; - void *rpages; - assert(run->nfree > 0); - assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info)); + assert(slab_data->nfree > 0); + assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); - extent = iealloc(tsdn, run); - regind = (unsigned)bitmap_sfu(run->bitmap, &bin_info->bitmap_info); - miscelm = arena_run_to_miscelm(extent, run); - rpages = arena_miscelm_to_rpages(extent, miscelm); - ret = (void *)((uintptr_t)rpages + (uintptr_t)(bin_info->reg_size * - regind)); - run->nfree--; + regind = (unsigned)bitmap_sfu(slab_data->bitmap, + &bin_info->bitmap_info); + ret = (void *)((uintptr_t)extent_addr_get(slab) + + (uintptr_t)(bin_info->reg_size * regind)); + slab_data->nfree--; return (ret); } JEMALLOC_INLINE_C size_t -arena_run_regind(extent_t *extent, arena_run_t *run, - const arena_bin_info_t *bin_info, const void *ptr) +arena_slab_regind(extent_t *slab, const arena_bin_info_t *bin_info, + const void *ptr) { size_t diff, interval, shift, regind; - arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(extent, run); - void *rpages = arena_miscelm_to_rpages(extent, miscelm); - /* - * Freeing a pointer lower than region zero can cause assertion - * failure. - */ - assert((uintptr_t)ptr >= (uintptr_t)rpages); + /* Freeing a pointer outside the slab can cause assertion failure. */ + assert((uintptr_t)ptr >= (uintptr_t)extent_addr_get(slab)); + assert((uintptr_t)ptr < (uintptr_t)extent_past_get(slab)); + /* Freeing an interior pointer can cause assertion failure. */ + assert(((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)) % + (uintptr_t)bin_info->reg_size == 0); /* * Avoid doing division with a variable divisor if possible. Using * actual division here can reduce allocator throughput by over 20%! */ - diff = (size_t)((uintptr_t)ptr - (uintptr_t)rpages); + diff = (size_t)((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)); /* Rescale (factor powers of 2 out of the numerator and denominator). */ interval = bin_info->reg_size; @@ -353,7 +183,7 @@ arena_run_regind(extent_t *extent, arena_run_t *run, * divide by 0, and 1 and 2 are both powers of two, which are * handled above. */ -#define SIZE_INV_SHIFT ((sizeof(size_t) << 3) - LG_RUN_MAXREGS) +#define SIZE_INV_SHIFT ((sizeof(size_t) << 3) - LG_SLAB_MAXREGS) #define SIZE_INV(s) (((ZU(1) << SIZE_INV_SHIFT) / (s)) + 1) static const size_t interval_invs[] = { SIZE_INV(3), @@ -382,48 +212,19 @@ arena_run_regind(extent_t *extent, arena_run_t *run, } JEMALLOC_INLINE_C void -arena_run_reg_dalloc(tsdn_t *tsdn, arena_run_t *run, extent_t *extent, - void *ptr) +arena_slab_reg_dalloc(tsdn_t *tsdn, extent_t *slab, + arena_slab_data_t *slab_data, void *ptr) { - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - size_t mapbits = arena_mapbits_get(chunk, pageind); - szind_t binind = arena_ptr_small_binind_get(tsdn, ptr, mapbits); + szind_t binind = slab_data->binind; const arena_bin_info_t *bin_info = &arena_bin_info[binind]; - size_t regind = arena_run_regind(extent, run, bin_info, ptr); + size_t regind = arena_slab_regind(slab, bin_info, ptr); - assert(run->nfree < bin_info->nregs); - /* Freeing an interior pointer can cause assertion failure. */ - assert(((uintptr_t)ptr - - (uintptr_t)arena_miscelm_to_rpages(extent, - arena_run_to_miscelm(extent, run))) % (uintptr_t)bin_info->reg_size - == 0); - assert((uintptr_t)ptr >= - (uintptr_t)arena_miscelm_to_rpages(extent, - arena_run_to_miscelm(extent, run))); + assert(slab_data->nfree < bin_info->nregs); /* Freeing an unallocated pointer can cause assertion failure. */ - assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)); + assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind)); - bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind); - run->nfree++; -} - -JEMALLOC_INLINE_C void -arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) -{ - - memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, - (npages << LG_PAGE)); -} - -JEMALLOC_INLINE_C void -arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) -{ - size_t i; - UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); - - for (i = 0; i < PAGE / sizeof(size_t); i++) - assert(p[i] == 0); + bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind); + slab_data->nfree++; } static void @@ -454,373 +255,6 @@ arena_nactive_sub(arena_t *arena, size_t sub_pages) arena->nactive -= sub_pages; } -static void -arena_run_split_remove(arena_t *arena, extent_t *extent, size_t run_ind, - size_t flag_dirty, size_t flag_decommitted, size_t need_pages) -{ - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t total_pages, rem_pages; - - assert(flag_dirty == 0 || flag_decommitted == 0); - - total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> - LG_PAGE; - assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == - flag_dirty); - assert(need_pages <= total_pages); - rem_pages = total_pages - need_pages; - - arena_avail_remove(arena, extent, run_ind, total_pages); - if (flag_dirty != 0) - arena_run_dirty_remove(arena, chunk, run_ind, total_pages); - arena_nactive_add(arena, need_pages); - - /* Keep track of trailing unused pages for later use. */ - if (rem_pages > 0) { - size_t flags = flag_dirty | flag_decommitted; - size_t flag_unzeroed_mask = (flags == 0) ? CHUNK_MAP_UNZEROED : - 0; - - arena_mapbits_unallocated_set(chunk, run_ind+need_pages, - (rem_pages << LG_PAGE), flags | - (arena_mapbits_unzeroed_get(chunk, run_ind+need_pages) & - flag_unzeroed_mask)); - arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, - (rem_pages << LG_PAGE), flags | - (arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1) & - flag_unzeroed_mask)); - if (flag_dirty != 0) { - arena_run_dirty_insert(arena, chunk, run_ind+need_pages, - rem_pages); - } - arena_avail_insert(arena, extent, run_ind+need_pages, - rem_pages); - } -} - -static bool -arena_run_split_large_helper(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - arena_run_t *run, size_t size, bool remove, bool zero) -{ - arena_chunk_t *chunk; - arena_chunk_map_misc_t *miscelm; - size_t flag_dirty, flag_decommitted, run_ind, need_pages; - size_t flag_unzeroed_mask; - - chunk = (arena_chunk_t *)extent_base_get(extent); - miscelm = arena_run_to_miscelm(extent, run); - run_ind = arena_miscelm_to_pageind(extent, miscelm); - flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); - flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); - need_pages = (size >> LG_PAGE); - assert(need_pages > 0); - - if (flag_decommitted != 0 && chunk_commit_wrapper(tsdn, arena, - &arena->chunk_hooks, extent, run_ind << LG_PAGE, size)) - return (true); - - if (remove) { - arena_run_split_remove(arena, extent, run_ind, flag_dirty, - flag_decommitted, need_pages); - } - - if (zero) { - if (flag_decommitted != 0) - ; /* The run is untouched, and therefore zeroed. */ - else if (flag_dirty != 0) { - /* The run is dirty, so all pages must be zeroed. */ - arena_run_zero(chunk, run_ind, need_pages); - } else { - /* - * The run is clean, so some pages may be zeroed (i.e. - * never before touched). - */ - size_t i; - for (i = 0; i < need_pages; i++) { - if (arena_mapbits_unzeroed_get(chunk, run_ind+i) - != 0) - arena_run_zero(chunk, run_ind+i, 1); - else if (config_debug) { - arena_run_page_validate_zeroed(chunk, - run_ind+i); - } - } - } - } - - /* - * Set the last element first, in case the run only contains one page - * (i.e. both statements set the same element). - */ - flag_unzeroed_mask = (flag_dirty | flag_decommitted) == 0 ? - CHUNK_MAP_UNZEROED : 0; - arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty | - (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, - run_ind+need_pages-1))); - arena_mapbits_large_set(chunk, run_ind, size, flag_dirty | - (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk, run_ind))); - return (false); -} - -static bool -arena_run_split_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - arena_run_t *run, size_t size, bool zero) -{ - - return (arena_run_split_large_helper(tsdn, arena, extent, run, size, - true, zero)); -} - -static bool -arena_run_split_small(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - arena_run_t *run, size_t size, szind_t binind) -{ - arena_chunk_t *chunk; - arena_chunk_map_misc_t *miscelm; - size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; - - assert(binind != BININD_INVALID); - - chunk = (arena_chunk_t *)extent_base_get(extent); - miscelm = arena_run_to_miscelm(extent, run); - run_ind = arena_miscelm_to_pageind(extent, miscelm); - flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); - flag_decommitted = arena_mapbits_decommitted_get(chunk, run_ind); - need_pages = (size >> LG_PAGE); - assert(need_pages > 0); - - if (flag_decommitted != 0 && chunk_commit_wrapper(tsdn, arena, - &arena->chunk_hooks, extent, run_ind << LG_PAGE, size)) - return (true); - - arena_run_split_remove(arena, extent, run_ind, flag_dirty, - flag_decommitted, need_pages); - - for (i = 0; i < need_pages; i++) { - size_t flag_unzeroed = arena_mapbits_unzeroed_get(chunk, - run_ind+i); - arena_mapbits_small_set(chunk, run_ind+i, i, binind, - flag_unzeroed); - if (config_debug && flag_dirty == 0 && flag_unzeroed == 0) - arena_run_page_validate_zeroed(chunk, run_ind+i); - } - return (false); -} - -static extent_t * -arena_chunk_init_spare(arena_t *arena) -{ - extent_t *extent; - - assert(arena->spare != NULL); - - extent = arena->spare; - arena->spare = NULL; - - assert(arena_mapbits_allocated_get((arena_chunk_t *) - extent_base_get(extent), map_bias) == 0); - assert(arena_mapbits_allocated_get((arena_chunk_t *) - extent_base_get(extent), chunk_npages-1) == 0); - assert(arena_mapbits_unallocated_size_get((arena_chunk_t *) - extent_base_get(extent), map_bias) == arena_maxrun); - assert(arena_mapbits_unallocated_size_get((arena_chunk_t *) - extent_base_get(extent), chunk_npages-1) == arena_maxrun); - assert(arena_mapbits_dirty_get((arena_chunk_t *) - extent_base_get(extent), map_bias) == - arena_mapbits_dirty_get((arena_chunk_t *)extent_base_get(extent), - chunk_npages-1)); - - return (extent); -} - -static extent_t * -arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, - chunk_hooks_t *chunk_hooks, bool *zero, bool *commit) -{ - extent_t *extent; - - malloc_mutex_unlock(tsdn, &arena->lock); - - extent = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, chunksize, - 0, CACHELINE, zero, commit, true); - if (extent != NULL && !*commit) { - /* Commit header. */ - if (chunk_commit_wrapper(tsdn, arena, chunk_hooks, extent, 0, - map_bias << LG_PAGE)) { - chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, extent); - extent = NULL; - } - } - - malloc_mutex_lock(tsdn, &arena->lock); - - return (extent); -} - -static extent_t * -arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, - bool *commit) -{ - extent_t *extent; - chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; - - extent = arena_chunk_cache_alloc_locked(tsdn, arena, &chunk_hooks, NULL, - chunksize, 0, CACHELINE, zero, true); - if (extent != NULL) - *commit = true; - if (extent == NULL) { - extent = arena_chunk_alloc_internal_hard(tsdn, arena, - &chunk_hooks, zero, commit); - if (extent == NULL) - return (NULL); - } - assert(extent_slab_get(extent)); - - if (config_stats) { - arena->stats.mapped += extent_size_get(extent); - arena->stats.metadata_mapped += (map_bias << LG_PAGE); - } - - return (extent); -} - -static extent_t * -arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena) -{ - extent_t *extent; - bool zero, commit; - size_t flag_unzeroed, flag_decommitted, i; - - assert(arena->spare == NULL); - - zero = false; - commit = false; - extent = arena_chunk_alloc_internal(tsdn, arena, &zero, &commit); - if (extent == NULL) - return (NULL); - - /* - * Initialize the map to contain one maximal free untouched run. Mark - * the pages as zeroed if arena_chunk_alloc_internal() returned a zeroed - * or decommitted chunk. - */ - flag_unzeroed = (zero || !commit) ? 0 : CHUNK_MAP_UNZEROED; - flag_decommitted = commit ? 0 : CHUNK_MAP_DECOMMITTED; - arena_mapbits_unallocated_set((arena_chunk_t *)extent_base_get(extent), - map_bias, arena_maxrun, flag_unzeroed | flag_decommitted); - /* - * There is no need to initialize the internal page map entries unless - * the chunk is not zeroed. - */ - if (!zero) { - for (i = map_bias+1; i < chunk_npages-1; i++) { - arena_mapbits_internal_set((arena_chunk_t *) - extent_base_get(extent), i, flag_unzeroed); - } - } else { - if (config_debug) { - for (i = map_bias+1; i < chunk_npages-1; i++) { - assert(arena_mapbits_unzeroed_get( - (arena_chunk_t *)extent_base_get(extent), i) - == flag_unzeroed); - } - } - } - arena_mapbits_unallocated_set((arena_chunk_t *)extent_base_get(extent), - chunk_npages-1, arena_maxrun, flag_unzeroed); - - return (extent); -} - -static extent_t * -arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena) -{ - extent_t *extent; - - if (arena->spare != NULL) - extent = arena_chunk_init_spare(arena); - else { - extent = arena_chunk_init_hard(tsdn, arena); - if (extent == NULL) - return (NULL); - } - - ql_elm_new(extent, ql_link); - ql_tail_insert(&arena->achunks, extent, ql_link); - arena_avail_insert(arena, extent, map_bias, chunk_npages-map_bias); - - return (extent); -} - -static void -arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, extent_t *extent) -{ - chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; - - extent_committed_set(extent, - (arena_mapbits_decommitted_get((arena_chunk_t *) - extent_base_get(extent), map_bias) == 0)); - if (!extent_committed_get(extent)) { - /* - * Decommit the header. Mark the chunk as decommitted even if - * header decommit fails, since treating a partially committed - * chunk as committed has a high potential for causing later - * access of decommitted memory. - */ - chunk_decommit_wrapper(tsdn, arena, &chunk_hooks, extent, 0, - map_bias << LG_PAGE); - } - - if (config_stats) { - arena->stats.mapped -= extent_size_get(extent); - arena->stats.metadata_mapped -= (map_bias << LG_PAGE); - } - - arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, extent); -} - -static void -arena_spare_discard(tsdn_t *tsdn, arena_t *arena, extent_t *spare) -{ - - assert(arena->spare != spare); - - if (arena_mapbits_dirty_get((arena_chunk_t *)extent_base_get(spare), - map_bias) != 0) { - arena_run_dirty_remove(arena, (arena_chunk_t *) - extent_base_get(spare), map_bias, chunk_npages-map_bias); - } - - arena_chunk_discard(tsdn, arena, spare); -} - -static void -arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) -{ - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - extent_t *spare; - - assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); - assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); - assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == - arena_maxrun); - assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == - arena_maxrun); - assert(arena_mapbits_dirty_get(chunk, map_bias) == - arena_mapbits_dirty_get(chunk, chunk_npages-1)); - assert(arena_mapbits_decommitted_get(chunk, map_bias) == - arena_mapbits_decommitted_get(chunk, chunk_npages-1)); - - /* Remove run from runs_avail, so that the arena does not use it. */ - arena_avail_remove(arena, extent, map_bias, chunk_npages-map_bias); - - ql_remove(&arena->achunks, extent, ql_link); - spare = arena->spare; - arena->spare = extent; - if (spare != NULL) - arena_spare_discard(tsdn, arena, spare); -} - static void arena_huge_malloc_stats_update(arena_t *arena, size_t usize) { @@ -986,77 +420,6 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, malloc_mutex_unlock(tsdn, &arena->lock); } -/* - * Do first-best-fit run selection, i.e. select the lowest run that best fits. - * Run sizes are indexed, so not all candidate runs are necessarily exactly the - * same size. - */ -static arena_run_t * -arena_run_first_best_fit(arena_t *arena, size_t size) -{ - pszind_t pind, i; - - pind = psz2ind(run_quantize_ceil(size)); - - for (i = pind; pind2sz(i) <= arena_maxrun; i++) { - arena_chunk_map_misc_t *miscelm = arena_run_heap_first( - &arena->runs_avail[i]); - if (miscelm != NULL) - return (&miscelm->run); - } - - return (NULL); -} - -static arena_run_t * -arena_run_alloc_small_helper(tsdn_t *tsdn, arena_t *arena, size_t size, - szind_t binind) -{ - arena_run_t *run = arena_run_first_best_fit(arena, size); - if (run != NULL) { - if (arena_run_split_small(tsdn, arena, iealloc(tsdn, run), run, - size, binind)) - run = NULL; - } - return (run); -} - -static arena_run_t * -arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind) -{ - arena_run_t *run; - extent_t *extent; - - assert(size <= arena_maxrun); - assert(size == PAGE_CEILING(size)); - assert(binind != BININD_INVALID); - - /* Search the arena's chunks for the lowest best fit. */ - run = arena_run_alloc_small_helper(tsdn, arena, size, binind); - if (run != NULL) - return (run); - - /* - * No usable runs. Create a new chunk from which to allocate the run. - */ - extent = arena_chunk_alloc(tsdn, arena); - if (extent != NULL) { - run = &arena_miscelm_get_mutable( - (arena_chunk_t *)extent_base_get(extent), map_bias)->run; - if (arena_run_split_small(tsdn, arena, iealloc(tsdn, run), run, - size, binind)) - run = NULL; - return (run); - } - - /* - * arena_chunk_alloc() failed, but another thread may have made - * sufficient memory available while this one dropped arena->lock in - * arena_chunk_alloc(), so search one more time. - */ - return (arena_run_alloc_small_helper(tsdn, arena, size, binind)); -} - static bool arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult) { @@ -1360,120 +723,45 @@ arena_maybe_purge(tsdn_t *tsdn, arena_t *arena) static size_t arena_dirty_count(tsdn_t *tsdn, arena_t *arena) { + extent_t *extent; size_t ndirty = 0; - arena_runs_dirty_link_t *rdelm; - extent_t *chunkselm; - for (rdelm = qr_next(&arena->runs_dirty, rd_link), - chunkselm = qr_next(&arena->chunks_cache, cc_link); - rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { - size_t npages; - - if (rdelm == &chunkselm->rd) { - npages = extent_size_get(chunkselm) >> LG_PAGE; - chunkselm = qr_next(chunkselm, cc_link); - } else { - extent_t *extent = iealloc(tsdn, rdelm); - arena_chunk_t *chunk = - (arena_chunk_t *)extent_base_get(extent); - arena_chunk_map_misc_t *miscelm = - arena_rd_to_miscelm(extent, rdelm); - size_t pageind = arena_miscelm_to_pageind(extent, - miscelm); - assert(arena_mapbits_allocated_get(chunk, pageind) == - 0); - assert(arena_mapbits_large_get(chunk, pageind) == 0); - assert(arena_mapbits_dirty_get(chunk, pageind) != 0); - npages = arena_mapbits_unallocated_size_get(chunk, - pageind) >> LG_PAGE; - } - ndirty += npages; - } + for (extent = qr_next(&arena->extents_dirty, qr_link); extent != + &arena->extents_dirty; extent = qr_next(extent, qr_link)) + ndirty += extent_size_get(extent) >> LG_PAGE; return (ndirty); } static size_t arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - size_t ndirty_limit, arena_runs_dirty_link_t *purge_runs_sentinel, - extent_t *purge_chunks_sentinel) + size_t ndirty_limit, extent_t *purge_extents_sentinel) { - arena_runs_dirty_link_t *rdelm, *rdelm_next; - extent_t *chunkselm; + extent_t *extent, *next; size_t nstashed = 0; - /* Stash runs/chunks according to ndirty_limit. */ - for (rdelm = qr_next(&arena->runs_dirty, rd_link), - chunkselm = qr_next(&arena->chunks_cache, cc_link); - rdelm != &arena->runs_dirty; rdelm = rdelm_next) { + /* Stash extents according to ndirty_limit. */ + for (extent = qr_next(&arena->extents_dirty, qr_link); extent != + &arena->extents_dirty; extent = next) { size_t npages; - rdelm_next = qr_next(rdelm, rd_link); + bool zero; + UNUSED extent_t *textent; - if (rdelm == &chunkselm->rd) { - extent_t *chunkselm_next; - bool zero; - UNUSED extent_t *extent; + npages = extent_size_get(extent) >> LG_PAGE; + if (opt_purge == purge_mode_decay && arena->ndirty - (nstashed + + npages) < ndirty_limit) + break; - npages = extent_size_get(chunkselm) >> LG_PAGE; - if (opt_purge == purge_mode_decay && arena->ndirty - - (nstashed + npages) < ndirty_limit) - break; - - chunkselm_next = qr_next(chunkselm, cc_link); - /* Allocate. */ - zero = false; - extent = arena_chunk_cache_alloc_locked(tsdn, arena, - chunk_hooks, extent_base_get(chunkselm), - extent_size_get(chunkselm), 0, CACHELINE, &zero, - false); - assert(extent == chunkselm); - assert(zero == extent_zeroed_get(chunkselm)); - extent_dirty_insert(chunkselm, purge_runs_sentinel, - purge_chunks_sentinel); - assert(npages == (extent_size_get(chunkselm) >> - LG_PAGE)); - chunkselm = chunkselm_next; - } else { - extent_t *extent = iealloc(tsdn, rdelm); - arena_chunk_map_misc_t *miscelm = - arena_rd_to_miscelm(extent, rdelm); - size_t pageind = arena_miscelm_to_pageind(extent, - miscelm); - arena_run_t *run = &miscelm->run; - size_t run_size = - arena_mapbits_unallocated_size_get((arena_chunk_t *) - extent_base_get(extent), pageind); - - npages = run_size >> LG_PAGE; - if (opt_purge == purge_mode_decay && arena->ndirty - - (nstashed + npages) < ndirty_limit) - break; - - assert(pageind + npages <= chunk_npages); - assert(arena_mapbits_dirty_get((arena_chunk_t *) - extent_base_get(extent), pageind) == - arena_mapbits_dirty_get((arena_chunk_t *) - extent_base_get(extent), pageind+npages-1)); - - /* - * If purging the spare chunk's run, make it available - * prior to allocation. - */ - if (extent == arena->spare) - arena_chunk_alloc(tsdn, arena); - - /* Temporarily allocate the free dirty run. */ - arena_run_split_large(tsdn, arena, extent, run, - run_size, false); - /* Stash. */ - if (false) - qr_new(rdelm, rd_link); /* Redundant. */ - else { - assert(qr_next(rdelm, rd_link) == rdelm); - assert(qr_prev(rdelm, rd_link) == rdelm); - } - qr_meld(purge_runs_sentinel, rdelm, rd_link); - } + next = qr_next(extent, qr_link); + /* Allocate. */ + zero = false; + textent = arena_chunk_cache_alloc_locked(tsdn, arena, + chunk_hooks, extent_base_get(extent), + extent_size_get(extent), 0, CACHELINE, &zero, false); + assert(textent == extent); + assert(zero == extent_zeroed_get(extent)); + extent_ring_remove(extent); + extent_ring_insert(purge_extents_sentinel, extent); nstashed += npages; if (opt_purge == purge_mode_ratio && arena->ndirty - nstashed <= @@ -1486,90 +774,26 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, static size_t arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - arena_runs_dirty_link_t *purge_runs_sentinel, - extent_t *purge_chunks_sentinel) + extent_t *purge_extents_sentinel) { - size_t npurged, nmadvise; - arena_runs_dirty_link_t *rdelm; - extent_t *chunkselm; + UNUSED size_t nmadvise; + size_t npurged; + extent_t *extent, *next; if (config_stats) nmadvise = 0; npurged = 0; - malloc_mutex_unlock(tsdn, &arena->lock); - for (rdelm = qr_next(purge_runs_sentinel, rd_link), - chunkselm = qr_next(purge_chunks_sentinel, cc_link); - rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { - size_t npages; - - if (rdelm == &chunkselm->rd) { - /* - * Don't actually purge the chunk here because 1) - * chunkselm is embedded in the chunk and must remain - * valid, and 2) we deallocate the chunk in - * arena_unstash_purged(), where it is destroyed, - * decommitted, or purged, depending on chunk - * deallocation policy. - */ - size_t size = extent_size_get(chunkselm); - npages = size >> LG_PAGE; - chunkselm = qr_next(chunkselm, cc_link); - } else { - size_t pageind, run_size, flag_unzeroed, flags, i; - bool decommitted; - extent_t *extent = iealloc(tsdn, rdelm); - arena_chunk_t *chunk = - (arena_chunk_t *)extent_base_get(extent); - arena_chunk_map_misc_t *miscelm = - arena_rd_to_miscelm(extent, rdelm); - pageind = arena_miscelm_to_pageind(extent, miscelm); - run_size = arena_mapbits_large_size_get(chunk, pageind); - npages = run_size >> LG_PAGE; - - assert(pageind + npages <= chunk_npages); - assert(!arena_mapbits_decommitted_get(chunk, pageind)); - assert(!arena_mapbits_decommitted_get(chunk, - pageind+npages-1)); - decommitted = !chunk_decommit_wrapper(tsdn, arena, - chunk_hooks, extent, pageind << LG_PAGE, npages << - LG_PAGE); - if (decommitted) { - flag_unzeroed = 0; - flags = CHUNK_MAP_DECOMMITTED; - } else { - flag_unzeroed = chunk_purge_wrapper(tsdn, arena, - chunk_hooks, extent, pageind << LG_PAGE, - run_size) ? CHUNK_MAP_UNZEROED : 0; - flags = flag_unzeroed; - } - arena_mapbits_large_set(chunk, pageind+npages-1, 0, - flags); - arena_mapbits_large_set(chunk, pageind, run_size, - flags); - - /* - * Set the unzeroed flag for internal pages, now that - * chunk_purge_wrapper() has returned whether the pages - * were zeroed as a side effect of purging. This chunk - * map modification is safe even though the arena mutex - * isn't currently owned by this thread, because the run - * is marked as allocated, thus protecting it from being - * modified by any other thread. As long as these - * writes don't perturb the first and last elements' - * CHUNK_MAP_ALLOCATED bits, behavior is well defined. - */ - for (i = 1; i < npages-1; i++) { - arena_mapbits_internal_set(chunk, pageind+i, - flag_unzeroed); - } - } - - npurged += npages; + for (extent = qr_next(purge_extents_sentinel, qr_link); extent != + purge_extents_sentinel; extent = next) { if (config_stats) nmadvise++; + npurged += extent_size_get(extent) >> LG_PAGE; + + next = qr_next(extent, qr_link); + extent_ring_remove(extent); + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, extent); } - malloc_mutex_lock(tsdn, &arena->lock); if (config_stats) { arena->stats.nmadvise += nmadvise; @@ -1579,49 +803,12 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, return (npurged); } -static void -arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, - arena_runs_dirty_link_t *purge_runs_sentinel, - extent_t *purge_chunks_sentinel) -{ - arena_runs_dirty_link_t *rdelm, *rdelm_next; - extent_t *chunkselm; - - /* Deallocate chunks/runs. */ - for (rdelm = qr_next(purge_runs_sentinel, rd_link), - chunkselm = qr_next(purge_chunks_sentinel, cc_link); - rdelm != purge_runs_sentinel; rdelm = rdelm_next) { - rdelm_next = qr_next(rdelm, rd_link); - if (rdelm == &chunkselm->rd) { - extent_t *chunkselm_next = qr_next(chunkselm, cc_link); - extent_dirty_remove(chunkselm); - chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, - chunkselm); - chunkselm = chunkselm_next; - } else { - extent_t *extent = iealloc(tsdn, rdelm); - arena_chunk_t *chunk = - (arena_chunk_t *)extent_base_get(extent); - arena_chunk_map_misc_t *miscelm = - arena_rd_to_miscelm(extent, rdelm); - size_t pageind = arena_miscelm_to_pageind(extent, - miscelm); - bool decommitted = (arena_mapbits_decommitted_get(chunk, - pageind) != 0); - arena_run_t *run = &miscelm->run; - qr_remove(rdelm, rd_link); - arena_run_dalloc(tsdn, arena, extent, run, false, true, - decommitted); - } - } -} - /* * NB: ndirty_limit is interpreted differently depending on opt_purge: - * - purge_mode_ratio: Purge as few dirty run/chunks as possible to reach the + * - purge_mode_ratio: Purge as few dirty extents as possible to reach the * desired state: * (arena->ndirty <= ndirty_limit) - * - purge_mode_decay: Purge as many dirty runs/chunks as possible without + * - purge_mode_decay: Purge as many dirty extents as possible without * violating the invariant: * (arena->ndirty >= ndirty_limit) */ @@ -1630,8 +817,7 @@ arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) { chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena); size_t npurge, npurged; - arena_runs_dirty_link_t purge_runs_sentinel; - extent_t purge_chunks_sentinel; + extent_t purge_extents_sentinel; arena->purging = true; @@ -1646,19 +832,16 @@ arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) assert(opt_purge != purge_mode_ratio || (arena->nactive >> arena->lg_dirty_mult) < arena->ndirty || ndirty_limit == 0); - qr_new(&purge_runs_sentinel, rd_link); - extent_init(&purge_chunks_sentinel, arena, NULL, 0, 0, false, false, - false, false, false); + extent_init(&purge_extents_sentinel, arena, NULL, 0, 0, false, false, + false, false); npurge = arena_stash_dirty(tsdn, arena, &chunk_hooks, ndirty_limit, - &purge_runs_sentinel, &purge_chunks_sentinel); + &purge_extents_sentinel); if (npurge == 0) goto label_return; npurged = arena_purge_stashed(tsdn, arena, &chunk_hooks, - &purge_runs_sentinel, &purge_chunks_sentinel); + &purge_extents_sentinel); assert(npurged == npurge); - arena_unstash_purged(tsdn, arena, &chunk_hooks, &purge_runs_sentinel, - &purge_chunks_sentinel); if (config_stats) arena->stats.npurge++; @@ -1679,6 +862,15 @@ arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) malloc_mutex_unlock(tsdn, &arena->lock); } +static void +arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) +{ + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + + arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); + arena_chunk_cache_dalloc_locked(tsdn, arena, &chunk_hooks, slab); +} + void arena_reset(tsd_t *tsd, arena_t *arena) { @@ -1724,367 +916,225 @@ arena_reset(tsd_t *tsd, arena_t *arena) /* Bins. */ for (i = 0; i < NBINS; i++) { + extent_t *slab, *next; arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - bin->runcur = NULL; - arena_run_heap_new(&bin->runs); + if (bin->slabcur != NULL) { + arena_slab_dalloc(tsd_tsdn(tsd), arena, bin->slabcur); + bin->slabcur = NULL; + } + while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != + NULL) + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + for (slab = qr_next(&bin->slabs_full, qr_link); slab != + &bin->slabs_full; slab = next) { + next = qr_next(slab, qr_link); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + } if (config_stats) { bin->stats.curregs = 0; - bin->stats.curruns = 0; + bin->stats.curslabs = 0; } malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); } - /* - * Re-initialize runs_dirty such that the chunks_cache and runs_dirty - * chains directly correspond. - */ - qr_new(&arena->runs_dirty, rd_link); - for (extent = qr_next(&arena->chunks_cache, cc_link); - extent != &arena->chunks_cache; extent = qr_next(extent, cc_link)) { - qr_new(&extent->rd, rd_link); - qr_meld(&arena->runs_dirty, &extent->rd, rd_link); - } - - /* Arena chunks. */ - for (extent = ql_last(&arena->achunks, ql_link); extent != NULL; extent - = ql_last(&arena->achunks, ql_link)) { - ql_remove(&arena->achunks, extent, ql_link); - arena_chunk_discard(tsd_tsdn(tsd), arena, extent); - } - - /* Spare. */ - if (arena->spare != NULL) { - arena_chunk_discard(tsd_tsdn(tsd), arena, arena->spare); - arena->spare = NULL; - } - assert(!arena->purging); arena->nactive = 0; - for (i = 0; i < sizeof(arena->runs_avail) / sizeof(arena_run_heap_t); - i++) - arena_run_heap_new(&arena->runs_avail[i]); - malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock); } static void -arena_run_coalesce(arena_t *arena, extent_t *extent, size_t *p_size, - size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty, - size_t flag_decommitted) +arena_bin_slabs_nonfull_insert(arena_bin_t *bin, extent_t *slab) { - arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent); - size_t size = *p_size; - size_t run_ind = *p_run_ind; - size_t run_pages = *p_run_pages; - /* Try to coalesce forward. */ - if (run_ind + run_pages < chunk_npages && - arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && - arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty && - arena_mapbits_decommitted_get(chunk, run_ind+run_pages) == - flag_decommitted) { - size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, - run_ind+run_pages); - size_t nrun_pages = nrun_size >> LG_PAGE; - - /* - * Remove successor from runs_avail; the coalesced run is - * inserted later. - */ - assert(arena_mapbits_unallocated_size_get(chunk, - run_ind+run_pages+nrun_pages-1) == nrun_size); - assert(arena_mapbits_dirty_get(chunk, - run_ind+run_pages+nrun_pages-1) == flag_dirty); - assert(arena_mapbits_decommitted_get(chunk, - run_ind+run_pages+nrun_pages-1) == flag_decommitted); - arena_avail_remove(arena, extent, run_ind+run_pages, - nrun_pages); - - /* - * If the successor is dirty, remove it from the set of dirty - * pages. - */ - if (flag_dirty != 0) { - arena_run_dirty_remove(arena, chunk, run_ind+run_pages, - nrun_pages); - } - - size += nrun_size; - run_pages += nrun_pages; - - arena_mapbits_unallocated_size_set(chunk, run_ind, size); - arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, - size); - } - - /* Try to coalesce backward. */ - if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, - run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == - flag_dirty && arena_mapbits_decommitted_get(chunk, run_ind-1) == - flag_decommitted) { - size_t prun_size = arena_mapbits_unallocated_size_get(chunk, - run_ind-1); - size_t prun_pages = prun_size >> LG_PAGE; - - run_ind -= prun_pages; - - /* - * Remove predecessor from runs_avail; the coalesced run is - * inserted later. - */ - assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == - prun_size); - assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); - assert(arena_mapbits_decommitted_get(chunk, run_ind) == - flag_decommitted); - arena_avail_remove(arena, extent, run_ind, prun_pages); - - /* - * If the predecessor is dirty, remove it from the set of dirty - * pages. - */ - if (flag_dirty != 0) { - arena_run_dirty_remove(arena, chunk, run_ind, - prun_pages); - } - - size += prun_size; - run_pages += prun_pages; - - arena_mapbits_unallocated_size_set(chunk, run_ind, size); - arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, - size); - } - - *p_size = size; - *p_run_ind = run_ind; - *p_run_pages = run_pages; -} - -static size_t -arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, - size_t run_ind) -{ - size_t size; - - assert(run_ind >= map_bias); - assert(run_ind < chunk_npages); - - if (arena_mapbits_large_get(chunk, run_ind) != 0) { - size = arena_mapbits_large_size_get(chunk, run_ind); - assert(size == PAGE || arena_mapbits_large_size_get(chunk, - run_ind+(size>>LG_PAGE)-1) == 0); - } else { - const arena_bin_info_t *bin_info = &arena_bin_info[run->binind]; - size = bin_info->run_size; - } - - return (size); + assert(extent_slab_data_get(slab)->nfree > 0); + extent_heap_insert(&bin->slabs_nonfull, slab); } static void -arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - arena_run_t *run, bool dirty, bool cleaned, bool decommitted) +arena_bin_slabs_nonfull_remove(arena_bin_t *bin, extent_t *slab) { - arena_chunk_t *chunk; - arena_chunk_map_misc_t *miscelm; - size_t size, run_ind, run_pages, flag_dirty, flag_decommitted; - chunk = (arena_chunk_t *)extent_base_get(extent); - miscelm = arena_run_to_miscelm(extent, run); - run_ind = arena_miscelm_to_pageind(extent, miscelm); - assert(run_ind >= map_bias); - assert(run_ind < chunk_npages); - size = arena_run_size_get(arena, chunk, run, run_ind); - run_pages = (size >> LG_PAGE); - arena_nactive_sub(arena, run_pages); - - /* - * The run is dirty if the caller claims to have dirtied it, as well as - * if it was already dirty before being allocated and the caller - * doesn't claim to have cleaned it. - */ - assert(arena_mapbits_dirty_get(chunk, run_ind) == - arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); - if (!cleaned && !decommitted && arena_mapbits_dirty_get(chunk, run_ind) - != 0) - dirty = true; - flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; - flag_decommitted = decommitted ? CHUNK_MAP_DECOMMITTED : 0; - - /* Mark pages as unallocated in the chunk map. */ - if (dirty || decommitted) { - size_t flags = flag_dirty | flag_decommitted; - arena_mapbits_unallocated_set(chunk, run_ind, size, flags); - arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, - flags); - } else { - arena_mapbits_unallocated_set(chunk, run_ind, size, - arena_mapbits_unzeroed_get(chunk, run_ind)); - arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, - arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); - } - - arena_run_coalesce(arena, extent, &size, &run_ind, &run_pages, - flag_dirty, flag_decommitted); - - /* Insert into runs_avail, now that coalescing is complete. */ - assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == - arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); - assert(arena_mapbits_dirty_get(chunk, run_ind) == - arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); - assert(arena_mapbits_decommitted_get(chunk, run_ind) == - arena_mapbits_decommitted_get(chunk, run_ind+run_pages-1)); - arena_avail_insert(arena, extent, run_ind, run_pages); - - if (dirty) - arena_run_dirty_insert(arena, chunk, run_ind, run_pages); - - /* Deallocate chunk if it is now completely unused. */ - if (size == arena_maxrun) { - assert(run_ind == map_bias); - assert(run_pages == (arena_maxrun >> LG_PAGE)); - arena_chunk_dalloc(tsdn, arena, extent); - } - - /* - * It is okay to do dirty page processing here even if the chunk was - * deallocated above, since in that case it is the spare. Waiting - * until after possible chunk deallocation to do dirty processing - * allows for an old spare to be fully deallocated, thus decreasing the - * chances of spuriously crossing the dirty page purging threshold. - */ - if (dirty) - arena_maybe_purge(tsdn, arena); + extent_heap_remove(&bin->slabs_nonfull, slab); } -static void -arena_bin_runs_insert(arena_bin_t *bin, extent_t *extent, arena_run_t *run) +static extent_t * +arena_bin_slabs_nonfull_tryget(arena_bin_t *bin) { - arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(extent, run); - - arena_run_heap_insert(&bin->runs, miscelm); -} - -static arena_run_t * -arena_bin_nonfull_run_tryget(arena_bin_t *bin) -{ - arena_chunk_map_misc_t *miscelm; - - miscelm = arena_run_heap_remove_first(&bin->runs); - if (miscelm == NULL) + extent_t *slab = extent_heap_remove_first(&bin->slabs_nonfull); + if (slab == NULL) return (NULL); if (config_stats) - bin->stats.reruns++; - - return (&miscelm->run); + bin->stats.reslabs++; + return (slab); } -static arena_run_t * -arena_bin_nonfull_run_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) +static void +arena_bin_slabs_full_insert(arena_bin_t *bin, extent_t *slab) { - arena_run_t *run; - szind_t binind; + + assert(extent_slab_data_get(slab)->nfree == 0); + extent_ring_insert(&bin->slabs_full, slab); +} + +static void +arena_bin_slabs_full_remove(extent_t *slab) +{ + + extent_ring_remove(slab); +} + +static extent_t * +arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, + const arena_bin_info_t *bin_info) +{ + extent_t *slab; + bool zero, commit; + + zero = false; + commit = true; + malloc_mutex_unlock(tsdn, &arena->lock); + slab = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, + bin_info->slab_size, 0, PAGE, &zero, &commit, true); + malloc_mutex_lock(tsdn, &arena->lock); + + return (slab); +} + +static extent_t * +arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, + const arena_bin_info_t *bin_info) +{ + extent_t *slab; + arena_slab_data_t *slab_data; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; + bool zero; + + zero = false; + slab = arena_chunk_cache_alloc_locked(tsdn, arena, &chunk_hooks, NULL, + bin_info->slab_size, 0, PAGE, &zero, true); + if (slab == NULL) { + slab = arena_slab_alloc_hard(tsdn, arena, &chunk_hooks, + bin_info); + if (slab == NULL) + return (NULL); + } + assert(extent_slab_get(slab)); + + arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); + + /* Initialize slab internals. */ + slab_data = extent_slab_data_get(slab); + slab_data->binind = binind; + slab_data->nfree = bin_info->nregs; + bitmap_init(slab_data->bitmap, &bin_info->bitmap_info); + + if (config_stats) + arena->stats.mapped += extent_size_get(slab); + + return (slab); +} + +static extent_t * +arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, + szind_t binind) +{ + extent_t *slab; const arena_bin_info_t *bin_info; - /* Look for a usable run. */ - run = arena_bin_nonfull_run_tryget(bin); - if (run != NULL) - return (run); - /* No existing runs have any space available. */ + /* Look for a usable slab. */ + slab = arena_bin_slabs_nonfull_tryget(bin); + if (slab != NULL) + return (slab); + /* No existing slabs have any space available. */ - binind = arena_bin_index(arena, bin); bin_info = &arena_bin_info[binind]; - /* Allocate a new run. */ + /* Allocate a new slab. */ malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ malloc_mutex_lock(tsdn, &arena->lock); - run = arena_run_alloc_small(tsdn, arena, bin_info->run_size, binind); - if (run != NULL) { - /* Initialize run internals. */ - run->binind = binind; - run->nfree = bin_info->nregs; - bitmap_init(run->bitmap, &bin_info->bitmap_info); - } + slab = arena_slab_alloc(tsdn, arena, binind, bin_info); malloc_mutex_unlock(tsdn, &arena->lock); /********************************/ malloc_mutex_lock(tsdn, &bin->lock); - if (run != NULL) { + if (slab != NULL) { if (config_stats) { - bin->stats.nruns++; - bin->stats.curruns++; + bin->stats.nslabs++; + bin->stats.curslabs++; } - return (run); + return (slab); } /* - * arena_run_alloc_small() failed, but another thread may have made + * arena_slab_alloc() failed, but another thread may have made * sufficient memory available while this one dropped bin->lock above, * so search one more time. */ - run = arena_bin_nonfull_run_tryget(bin); - if (run != NULL) - return (run); + slab = arena_bin_slabs_nonfull_tryget(bin); + if (slab != NULL) + return (slab); return (NULL); } -/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ +/* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */ static void * -arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin) +arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, + szind_t binind) { - szind_t binind; const arena_bin_info_t *bin_info; - arena_run_t *run; + extent_t *slab; + - binind = arena_bin_index(arena, bin); bin_info = &arena_bin_info[binind]; - bin->runcur = NULL; - run = arena_bin_nonfull_run_get(tsdn, arena, bin); - if (bin->runcur != NULL && bin->runcur->nfree > 0) { + if (bin->slabcur != NULL) { + arena_bin_slabs_full_insert(bin, bin->slabcur); + bin->slabcur = NULL; + } + slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind); + if (bin->slabcur != NULL) { /* - * Another thread updated runcur while this one ran without the - * bin lock in arena_bin_nonfull_run_get(). + * Another thread updated slabcur while this one ran without the + * bin lock in arena_bin_nonfull_slab_get(). */ - void *ret; - assert(bin->runcur->nfree > 0); - ret = arena_run_reg_alloc(tsdn, bin->runcur, bin_info); - if (run != NULL) { - extent_t *extent; - arena_chunk_t *chunk; - - /* - * arena_run_alloc_small() may have allocated run, or - * it may have pulled run from the bin's run tree. - * Therefore it is unsafe to make any assumptions about - * how run has previously been used, and - * arena_bin_lower_run() must be called, as if a region - * were just deallocated from the run. - */ - extent = iealloc(tsdn, run); - chunk = (arena_chunk_t *)extent_base_get(extent); - if (run->nfree == bin_info->nregs) { - arena_dalloc_bin_run(tsdn, arena, chunk, extent, - run, bin); - } else { - arena_bin_lower_run(tsdn, arena, extent, run, - bin); + if (extent_slab_data_get(bin->slabcur)->nfree > 0) { + void *ret = arena_slab_reg_alloc(tsdn, bin->slabcur, + bin_info); + if (slab != NULL) { + /* + * arena_slab_alloc() may have allocated slab, + * or it may have been pulled from + * slabs_nonfull. Therefore it is unsafe to + * make any assumptions about how slab has + * previously been used, and + * arena_bin_lower_slab() must be called, as if + * a region were just deallocated from the slab. + */ + if (extent_slab_data_get(slab)->nfree == + bin_info->nregs) { + arena_dalloc_bin_slab(tsdn, arena, slab, + bin); + } else { + arena_bin_lower_slab(tsdn, arena, slab, + bin); + } } + return (ret); } - return (ret); + + arena_bin_slabs_full_insert(bin, bin->slabcur); + bin->slabcur = NULL; } - if (run == NULL) + if (slab == NULL) return (NULL); + bin->slabcur = slab; - bin->runcur = run; + assert(extent_slab_data_get(bin->slabcur)->nfree > 0); - assert(bin->runcur->nfree > 0); - - return (arena_run_reg_alloc(tsdn, bin->runcur, bin_info)); + return (arena_slab_reg_alloc(tsdn, slab, bin_info)); } void @@ -2102,13 +1152,14 @@ arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_bin_t *tbin, malloc_mutex_lock(tsdn, &bin->lock); for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> tbin->lg_fill_div); i < nfill; i++) { - arena_run_t *run; + extent_t *slab; void *ptr; - if ((run = bin->runcur) != NULL && run->nfree > 0) { - ptr = arena_run_reg_alloc(tsdn, run, + if ((slab = bin->slabcur) != NULL && + extent_slab_data_get(slab)->nfree > 0) { + ptr = arena_slab_reg_alloc(tsdn, slab, &arena_bin_info[binind]); } else - ptr = arena_bin_malloc_hard(tsdn, arena, bin); + ptr = arena_bin_malloc_hard(tsdn, arena, bin, binind); if (ptr == NULL) { /* * OOM. tbin->avail isn't yet filled down to its first @@ -2171,17 +1222,18 @@ arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) void *ret; arena_bin_t *bin; size_t usize; - arena_run_t *run; + extent_t *slab; assert(binind < NBINS); bin = &arena->bins[binind]; usize = index2size(binind); malloc_mutex_lock(tsdn, &bin->lock); - if ((run = bin->runcur) != NULL && run->nfree > 0) - ret = arena_run_reg_alloc(tsdn, run, &arena_bin_info[binind]); + if ((slab = bin->slabcur) != NULL && extent_slab_data_get(slab)->nfree > + 0) + ret = arena_slab_reg_alloc(tsdn, slab, &arena_bin_info[binind]); else - ret = arena_bin_malloc_hard(tsdn, arena, bin); + ret = arena_bin_malloc_hard(tsdn, arena, bin, binind); if (ret == NULL) { malloc_mutex_unlock(tsdn, &bin->lock); @@ -2242,7 +1294,7 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE && (usize & PAGE_MASK) == 0))) { - /* Small; alignment doesn't require special run placement. */ + /* Small; alignment doesn't require special slab placement. */ ret = arena_malloc(tsdn, arena, usize, size2index(usize), zero, tcache, true); } else { @@ -2315,97 +1367,92 @@ arena_dalloc_promoted(tsdn_t *tsdn, extent_t *extent, void *ptr, } static void -arena_dissociate_bin_run(extent_t *extent, arena_run_t *run, arena_bin_t *bin) +arena_dissociate_bin_slab(extent_t *slab, arena_bin_t *bin) { - /* Dissociate run from bin. */ - if (run == bin->runcur) - bin->runcur = NULL; + /* Dissociate slab from bin. */ + if (slab == bin->slabcur) + bin->slabcur = NULL; else { - szind_t binind = arena_bin_index(extent_arena_get(extent), bin); + szind_t binind = extent_slab_data_get(slab)->binind; const arena_bin_info_t *bin_info = &arena_bin_info[binind]; /* * The following block's conditional is necessary because if the - * run only contains one region, then it never gets inserted - * into the non-full runs tree. + * slab only contains one region, then it never gets inserted + * into the non-full slabs heap. */ - if (bin_info->nregs != 1) { - arena_chunk_map_misc_t *miscelm = - arena_run_to_miscelm(extent, run); - - arena_run_heap_remove(&bin->runs, miscelm); - } + if (bin_info->nregs == 1) + arena_bin_slabs_full_remove(slab); + else + arena_bin_slabs_nonfull_remove(bin, slab); } } static void -arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, - extent_t *extent, arena_run_t *run, arena_bin_t *bin) +arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, + arena_bin_t *bin) { - assert(run != bin->runcur); + assert(slab != bin->slabcur); malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ malloc_mutex_lock(tsdn, &arena->lock); - arena_run_dalloc(tsdn, arena, extent, run, true, false, false); + arena_slab_dalloc(tsdn, arena, slab); malloc_mutex_unlock(tsdn, &arena->lock); /****************************/ malloc_mutex_lock(tsdn, &bin->lock); if (config_stats) - bin->stats.curruns--; + bin->stats.curslabs--; } static void -arena_bin_lower_run(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - arena_run_t *run, arena_bin_t *bin) +arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, + arena_bin_t *bin) { + assert(extent_slab_data_get(slab)->nfree > 0); + /* - * Make sure that if bin->runcur is non-NULL, it refers to the lowest - * non-full run. It is okay to NULL runcur out rather than proactively - * keeping it pointing at the lowest non-full run. + * Make sure that if bin->slabcur is non-NULL, it refers to the lowest + * non-full slab. It is okay to NULL slabcur out rather than + * proactively keeping it pointing at the lowest non-full slab. */ - if ((uintptr_t)run < (uintptr_t)bin->runcur) { - /* Switch runcur. */ - if (bin->runcur->nfree > 0) { - arena_bin_runs_insert(bin, iealloc(tsdn, bin->runcur), - bin->runcur); - } - bin->runcur = run; + if (bin->slabcur != NULL && (uintptr_t)extent_addr_get(slab) < + (uintptr_t)extent_addr_get(bin->slabcur)) { + /* Switch slabcur. */ + if (extent_slab_data_get(bin->slabcur)->nfree > 0) + arena_bin_slabs_nonfull_insert(bin, bin->slabcur); + else + arena_bin_slabs_full_insert(bin, bin->slabcur); + bin->slabcur = slab; if (config_stats) - bin->stats.reruns++; + bin->stats.reslabs++; } else - arena_bin_runs_insert(bin, extent, run); + arena_bin_slabs_nonfull_insert(bin, slab); } static void -arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, - extent_t *extent, void *ptr, arena_chunk_map_bits_t *bitselm, bool junked) +arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, + void *ptr, bool junked) { - size_t pageind, rpages_ind; - arena_run_t *run; - arena_bin_t *bin; - const arena_bin_info_t *bin_info; - szind_t binind; - - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); - run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; - binind = run->binind; - bin = &arena->bins[binind]; - bin_info = &arena_bin_info[binind]; + arena_slab_data_t *slab_data = extent_slab_data_get(slab); + szind_t binind = slab_data->binind; + arena_bin_t *bin = &arena->bins[binind]; + const arena_bin_info_t *bin_info = &arena_bin_info[binind]; if (!junked && config_fill && unlikely(opt_junk_free)) arena_dalloc_junk_small(ptr, bin_info); - arena_run_reg_dalloc(tsdn, run, extent, ptr); - if (run->nfree == bin_info->nregs) { - arena_dissociate_bin_run(extent, run, bin); - arena_dalloc_bin_run(tsdn, arena, chunk, extent, run, bin); - } else if (run->nfree == 1 && run != bin->runcur) - arena_bin_lower_run(tsdn, arena, extent, run, bin); + arena_slab_reg_dalloc(tsdn, slab, slab_data, ptr); + if (slab_data->nfree == bin_info->nregs) { + arena_dissociate_bin_slab(slab, bin); + arena_dalloc_bin_slab(tsdn, arena, slab, bin); + } else if (slab_data->nfree == 1 && slab != bin->slabcur) { + arena_bin_slabs_full_remove(slab); + arena_bin_lower_slab(tsdn, arena, slab, bin); + } if (config_stats) { bin->stats.ndalloc++; @@ -2414,45 +1461,28 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, } void -arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, - arena_chunk_t *chunk, extent_t *extent, void *ptr, - arena_chunk_map_bits_t *bitselm) +arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, extent_t *extent, + void *ptr) { - arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm, - true); + arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, true); } static void -arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, - extent_t *extent, void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm) +arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { - arena_run_t *run; - arena_bin_t *bin; - size_t rpages_ind; + arena_bin_t *bin = &arena->bins[extent_slab_data_get(extent)->binind]; - rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); - run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run; - bin = &arena->bins[run->binind]; malloc_mutex_lock(tsdn, &bin->lock); - arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm, - false); + arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, false); malloc_mutex_unlock(tsdn, &bin->lock); } void -arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, - extent_t *extent, void *ptr, size_t pageind) +arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { - arena_chunk_map_bits_t *bitselm; - if (config_debug) { - /* arena_ptr_small_binind_get() does extra sanity checking. */ - assert(arena_ptr_small_binind_get(tsdn, ptr, - arena_mapbits_get(chunk, pageind)) != BININD_INVALID); - } - bitselm = arena_bitselm_get_mutable(chunk, pageind); - arena_dalloc_bin(tsdn, arena, chunk, extent, ptr, pageind, bitselm); + arena_dalloc_bin(tsdn, arena, extent, ptr); arena_decay_tick(tsdn, arena); } @@ -2682,9 +1712,9 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, bstats[i].nfills += bin->stats.nfills; bstats[i].nflushes += bin->stats.nflushes; } - bstats[i].nruns += bin->stats.nruns; - bstats[i].reruns += bin->stats.reruns; - bstats[i].curruns += bin->stats.curruns; + bstats[i].nslabs += bin->stats.nslabs; + bstats[i].reslabs += bin->stats.reslabs; + bstats[i].curslabs += bin->stats.curslabs; malloc_mutex_unlock(tsdn, &bin->lock); } } @@ -2745,17 +1775,13 @@ arena_new(tsdn_t *tsdn, unsigned ind) arena->dss_prec = chunk_dss_prec_get(tsdn); - ql_new(&arena->achunks); - - arena->spare = NULL; - arena->lg_dirty_mult = arena_lg_dirty_mult_default_get(); arena->purging = false; arena->nactive = 0; arena->ndirty = 0; - qr_new(&arena->runs_dirty, rd_link); - qr_new(&arena->chunks_cache, cc_link); + extent_init(&arena->extents_dirty, arena, NULL, 0, 0, false, false, + false, false); if (opt_purge == purge_mode_decay) arena_decay_init(arena, arena_decay_time_default_get()); @@ -2786,52 +1812,23 @@ arena_new(tsdn_t *tsdn, unsigned ind) if (malloc_mutex_init(&bin->lock, "arena_bin", WITNESS_RANK_ARENA_BIN)) return (NULL); - bin->runcur = NULL; - arena_run_heap_new(&bin->runs); + bin->slabcur = NULL; + extent_heap_new(&bin->slabs_nonfull); + extent_init(&bin->slabs_full, arena, NULL, 0, 0, false, false, + false, false); if (config_stats) memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } - for (i = 0; i < NPSIZES; i++) - arena_run_heap_new(&arena->runs_avail[i]); - return (arena); } void arena_boot(void) { - unsigned i; arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); arena_decay_time_default_set(opt_decay_time); - - /* - * Compute the header size such that it is large enough to contain the - * page map. The page map is biased to omit entries for the header - * itself, so some iteration is necessary to compute the map bias. - * - * 1) Compute safe header_size and map_bias values that include enough - * space for an unbiased page map. - * 2) Refine map_bias based on (1) to omit the header pages in the page - * map. The resulting map_bias may be one too small. - * 3) Refine map_bias based on (2). The result will be >= the result - * from (2), and will always be correct. - */ - map_bias = 0; - for (i = 0; i < 3; i++) { - size_t header_size = offsetof(arena_chunk_t, map_bits) + - ((sizeof(arena_chunk_map_bits_t) + - sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); - map_bias = (header_size + PAGE_MASK) >> LG_PAGE; - } - assert(map_bias > 0); - - map_misc_offset = offsetof(arena_chunk_t, map_bits) + - sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); - - arena_maxrun = chunksize - (map_bias << LG_PAGE); - assert(arena_maxrun > 0); } void diff --git a/src/base.c b/src/base.c index 134018a8..3807422c 100644 --- a/src/base.c +++ b/src/base.c @@ -74,8 +74,7 @@ base_chunk_alloc(tsdn_t *tsdn, size_t minsize) base_resident += PAGE_CEILING(nsize); } } - extent_init(extent, NULL, addr, csize, 0, true, false, true, true, - false); + extent_init(extent, NULL, addr, csize, 0, true, true, true, false); return (extent); } diff --git a/src/chunk.c b/src/chunk.c index 8c4f741f..e2e9de03 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -558,8 +558,7 @@ chunk_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, extent_dalloc(tsdn, arena, extent); return (NULL); } - extent_init(extent, arena, addr, size, usize, true, false, zero, commit, - slab); + extent_init(extent, arena, addr, size, usize, true, zero, commit, slab); if (pad != 0) extent_addr_randomize(tsdn, extent, alignment); if (chunk_register(tsdn, extent)) { @@ -828,8 +827,8 @@ chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent_init(&lead, arena, extent_addr_get(extent), size_a, usize_a, extent_active_get(extent), - extent_dirty_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent), extent_slab_get(extent)); + extent_zeroed_get(extent), extent_committed_get(extent), + extent_slab_get(extent)); if (extent_rtree_acquire(tsdn, &lead, false, true, &lead_elm_a, &lead_elm_b)) @@ -838,8 +837,8 @@ chunk_split_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + size_a), size_b, usize_b, extent_active_get(extent), - extent_dirty_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent), extent_slab_get(extent)); + extent_zeroed_get(extent), extent_committed_get(extent), + extent_slab_get(extent)); if (extent_rtree_acquire(tsdn, trail, false, true, &trail_elm_a, &trail_elm_b)) goto label_error_c; diff --git a/src/chunk_dss.c b/src/chunk_dss.c index e92fda72..f890a5cd 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -121,7 +121,7 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, pad_size = (uintptr_t)ret - (uintptr_t)pad_addr; if (pad_size != 0) { extent_init(pad, arena, pad_addr, pad_size, - pad_size, false, true, false, true, false); + pad_size, false, false, true, false); } dss_next = (void *)((uintptr_t)ret + size); if ((uintptr_t)ret < (uintptr_t)dss_max || diff --git a/src/ctl.c b/src/ctl.c index 26bc1750..34c7e1bd 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -124,7 +124,7 @@ CTL_PROTO(arena_i_chunk_hooks) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) -CTL_PROTO(arenas_bin_i_run_size) +CTL_PROTO(arenas_bin_i_slab_size) INDEX_PROTO(arenas_bin_i) CTL_PROTO(arenas_hchunk_i_size) INDEX_PROTO(arenas_hchunk_i) @@ -160,9 +160,9 @@ CTL_PROTO(stats_arenas_i_bins_j_nrequests) CTL_PROTO(stats_arenas_i_bins_j_curregs) CTL_PROTO(stats_arenas_i_bins_j_nfills) CTL_PROTO(stats_arenas_i_bins_j_nflushes) -CTL_PROTO(stats_arenas_i_bins_j_nruns) -CTL_PROTO(stats_arenas_i_bins_j_nreruns) -CTL_PROTO(stats_arenas_i_bins_j_curruns) +CTL_PROTO(stats_arenas_i_bins_j_nslabs) +CTL_PROTO(stats_arenas_i_bins_j_nreslabs) +CTL_PROTO(stats_arenas_i_bins_j_curslabs) INDEX_PROTO(stats_arenas_i_bins_j) CTL_PROTO(stats_arenas_i_hchunks_j_nmalloc) CTL_PROTO(stats_arenas_i_hchunks_j_ndalloc) @@ -300,7 +300,7 @@ static const ctl_indexed_node_t arena_node[] = { static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("run_size"), CTL(arenas_bin_i_run_size)} + {NAME("slab_size"), CTL(arenas_bin_i_slab_size)} }; static const ctl_named_node_t super_arenas_bin_i_node[] = { {NAME(""), CHILD(named, arenas_bin_i)} @@ -373,9 +373,9 @@ static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, - {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, - {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, - {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} + {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, + {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, + {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)} }; static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { {NAME(""), CHILD(named, stats_arenas_i_bins_j)} @@ -549,9 +549,10 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->bstats[i].nflushes += astats->bstats[i].nflushes; } - sstats->bstats[i].nruns += astats->bstats[i].nruns; - sstats->bstats[i].reruns += astats->bstats[i].reruns; - sstats->bstats[i].curruns += astats->bstats[i].curruns; + sstats->bstats[i].nslabs += astats->bstats[i].nslabs; + sstats->bstats[i].reslabs += astats->bstats[i].reslabs; + sstats->bstats[i].curslabs += + astats->bstats[i].curslabs; } for (i = 0; i < NSIZES - NBINS; i++) { @@ -1801,7 +1802,7 @@ CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned) CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) -CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_slab_size, arena_bin_info[mib[2]].slab_size, size_t) static const ctl_named_node_t * arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { @@ -2032,12 +2033,12 @@ CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nfills, ctl_stats.arenas[mib[2]].bstats[mib[4]].nfills, uint64_t) CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nflushes, ctl_stats.arenas[mib[2]].bstats[mib[4]].nflushes, uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].nruns, uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].reruns, uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs, + ctl_stats.arenas[mib[2]].bstats[mib[4]].nslabs, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs, + ctl_stats.arenas[mib[2]].bstats[mib[4]].reslabs, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, + ctl_stats.arenas[mib[2]].bstats[mib[4]].curslabs, size_t) static const ctl_named_node_t * stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, diff --git a/src/huge.c b/src/huge.c index 5375b59f..5f758140 100644 --- a/src/huge.c +++ b/src/huge.c @@ -153,8 +153,8 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, * Zero the trailing bytes of the original allocation's * last page, since they are in an indeterminate state. * There will always be trailing bytes, because ptr's - * offset from the beginning of the run is a multiple of - * CACHELINE in [0 .. PAGE). + * offset from the beginning of the extent is a multiple + * of CACHELINE in [0 .. PAGE). */ void *zbase = (void *) ((uintptr_t)extent_addr_get(extent) + oldusize); diff --git a/src/jemalloc.c b/src/jemalloc.c index 9f8bd01e..429667f6 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1707,28 +1707,30 @@ irealloc_prof_sample(tsd_t *tsd, extent_t *extent, void *old_ptr, } JEMALLOC_ALWAYS_INLINE_C void * -irealloc_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize, +irealloc_prof(tsd_t *tsd, extent_t *old_extent, void *old_ptr, size_t old_usize, size_t usize) { void *p; - extent_t *e; + extent_t *extent; bool prof_active; prof_tctx_t *old_tctx, *tctx; prof_active = prof_active_get_unlocked(); - old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr); + old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_extent, old_ptr); tctx = prof_alloc_prep(tsd, usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irealloc_prof_sample(tsd, extent, old_ptr, old_usize, usize, - tctx); - } else - p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false); + p = irealloc_prof_sample(tsd, old_extent, old_ptr, old_usize, + usize, tctx); + } else { + p = iralloc(tsd, old_extent, old_ptr, old_usize, usize, 0, + false); + } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); return (NULL); } - e = (p == old_ptr) ? extent : iealloc(tsd_tsdn(tsd), p); - prof_realloc(tsd, e, p, usize, tctx, prof_active, true, + extent = (p == old_ptr) ? old_extent : iealloc(tsd_tsdn(tsd), p); + prof_realloc(tsd, extent, p, usize, tctx, prof_active, true, old_extent, old_ptr, old_usize, old_tctx); return (p); @@ -2146,24 +2148,24 @@ irallocx_prof_sample(tsdn_t *tsdn, extent_t *extent, void *old_ptr, } JEMALLOC_ALWAYS_INLINE_C void * -irallocx_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize, +irallocx_prof(tsd_t *tsd, extent_t *old_extent, void *old_ptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache, arena_t *arena) { void *p; - extent_t *e; + extent_t *extent; bool prof_active; prof_tctx_t *old_tctx, *tctx; prof_active = prof_active_get_unlocked(); - old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr); + old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_extent, old_ptr); tctx = prof_alloc_prep(tsd, *usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irallocx_prof_sample(tsd_tsdn(tsd), extent, old_ptr, + p = irallocx_prof_sample(tsd_tsdn(tsd), old_extent, old_ptr, old_usize, *usize, alignment, zero, tcache, arena, tctx); } else { - p = iralloct(tsd_tsdn(tsd), extent, old_ptr, old_usize, size, - alignment, zero, tcache, arena); + p = iralloct(tsd_tsdn(tsd), old_extent, old_ptr, old_usize, + size, alignment, zero, tcache, arena); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); @@ -2179,12 +2181,12 @@ irallocx_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize, * be the same as the current usize because of in-place large * reallocation. Therefore, query the actual value of usize. */ - e = extent; - *usize = isalloc(tsd_tsdn(tsd), e, p); + extent = old_extent; + *usize = isalloc(tsd_tsdn(tsd), extent, p); } else - e = iealloc(tsd_tsdn(tsd), p); - prof_realloc(tsd, e, p, *usize, tctx, prof_active, true, old_ptr, - old_usize, old_tctx); + extent = iealloc(tsd_tsdn(tsd), p); + prof_realloc(tsd, extent, p, *usize, tctx, prof_active, true, + old_extent, old_ptr, old_usize, old_tctx); return (p); } @@ -2338,8 +2340,8 @@ ixallocx_prof(tsd_t *tsd, extent_t *extent, void *ptr, size_t old_usize, prof_alloc_rollback(tsd, tctx, false); return (usize); } - prof_realloc(tsd, extent, ptr, usize, tctx, prof_active, false, ptr, - old_usize, old_tctx); + prof_realloc(tsd, extent, ptr, usize, tctx, prof_active, false, extent, + ptr, old_usize, old_tctx); return (usize); } diff --git a/src/stats.c b/src/stats.c index 4dc48d5b..599e377d 100644 --- a/src/stats.c +++ b/src/stats.c @@ -58,29 +58,29 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, if (config_tcache) { malloc_cprintf(write_cb, cbopaque, "bins: size ind allocated nmalloc" - " ndalloc nrequests curregs curruns regs" - " pgs util nfills nflushes newruns" - " reruns\n"); + " ndalloc nrequests curregs curslabs regs" + " pgs util nfills nflushes newslabs" + " reslabs\n"); } else { malloc_cprintf(write_cb, cbopaque, "bins: size ind allocated nmalloc" - " ndalloc nrequests curregs curruns regs" - " pgs util newruns reruns\n"); + " ndalloc nrequests curregs curslabs regs" + " pgs util newslabs reslabs\n"); } CTL_GET("arenas.nbins", &nbins, unsigned); for (j = 0, in_gap = false; j < nbins; j++) { - uint64_t nruns; + uint64_t nslabs; - CTL_M2_M4_GET("stats.arenas.0.bins.0.nruns", i, j, &nruns, + CTL_M2_M4_GET("stats.arenas.0.bins.0.nslabs", i, j, &nslabs, uint64_t); - if (nruns == 0) + if (nslabs == 0) in_gap = true; else { - size_t reg_size, run_size, curregs, availregs, milli; - size_t curruns; + size_t reg_size, slab_size, curregs, availregs, milli; + size_t curslabs; uint32_t nregs; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; - uint64_t reruns; + uint64_t reslabs; char util[6]; /* "x.yyy". */ if (in_gap) { @@ -90,7 +90,7 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } CTL_M2_GET("arenas.bin.0.size", j, ®_size, size_t); CTL_M2_GET("arenas.bin.0.nregs", j, &nregs, uint32_t); - CTL_M2_GET("arenas.bin.0.run_size", j, &run_size, + CTL_M2_GET("arenas.bin.0.slab_size", j, &slab_size, size_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.nmalloc", i, j, &nmalloc, uint64_t); @@ -106,12 +106,12 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_M2_M4_GET("stats.arenas.0.bins.0.nflushes", i, j, &nflushes, uint64_t); } - CTL_M2_M4_GET("stats.arenas.0.bins.0.nreruns", i, j, - &reruns, uint64_t); - CTL_M2_M4_GET("stats.arenas.0.bins.0.curruns", i, j, - &curruns, size_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nreslabs", i, j, + &reslabs, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.curslabs", i, j, + &curslabs, size_t); - availregs = nregs * curruns; + availregs = nregs * curslabs; milli = (availregs != 0) ? (1000 * curregs) / availregs : 1000; assert(milli <= 1000); @@ -134,9 +134,9 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, " %12zu %4u %3zu %-5s %12"FMTu64 " %12"FMTu64" %12"FMTu64" %12"FMTu64"\n", reg_size, j, curregs * reg_size, nmalloc, - ndalloc, nrequests, curregs, curruns, nregs, - run_size / page, util, nfills, nflushes, - nruns, reruns); + ndalloc, nrequests, curregs, curslabs, + nregs, slab_size / page, util, nfills, + nflushes, nslabs, reslabs); } else { malloc_cprintf(write_cb, cbopaque, "%20zu %3u %12zu %12"FMTu64 @@ -144,8 +144,9 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, " %12zu %4u %3zu %-5s %12"FMTu64 " %12"FMTu64"\n", reg_size, j, curregs * reg_size, nmalloc, - ndalloc, nrequests, curregs, curruns, nregs, - run_size / page, util, nruns, reruns); + ndalloc, nrequests, curregs, curslabs, + nregs, slab_size / page, util, nslabs, + reslabs); } } } diff --git a/src/tcache.c b/src/tcache.c index 41074d34..02015227 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -127,14 +127,8 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, extent = iealloc(tsd_tsdn(tsd), ptr); if (extent_arena_get(extent) == bin_arena) { - arena_chunk_t *chunk = - (arena_chunk_t *)extent_base_get(extent); - size_t pageind = ((uintptr_t)ptr - - (uintptr_t)chunk) >> LG_PAGE; - arena_chunk_map_bits_t *bitselm = - arena_bitselm_get_mutable(chunk, pageind); arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), - bin_arena, chunk, extent, ptr, bitselm); + bin_arena, extent, ptr); } else { /* * This object was allocated via a different diff --git a/test/unit/extent_quantize.c b/test/unit/extent_quantize.c index 98c9fde4..a165aece 100644 --- a/test/unit/extent_quantize.c +++ b/test/unit/extent_quantize.c @@ -16,7 +16,7 @@ TEST_BEGIN(test_small_extent_size) assert_d_eq(mallctl("arenas.nbins", &nbins, &sz, NULL, 0), 0, "Unexpected mallctl failure"); - assert_d_eq(mallctlnametomib("arenas.bin.0.run_size", mib, &miblen), 0, + assert_d_eq(mallctlnametomib("arenas.bin.0.slab_size", mib, &miblen), 0, "Unexpected mallctlnametomib failure"); for (i = 0; i < nbins; i++) { mib[2] = i; @@ -71,12 +71,12 @@ TEST_BEGIN(test_huge_extent_size) ceil = extent_size_quantize_ceil(extent_size); assert_zu_eq(extent_size, floor, - "Large run quantization should be a no-op for precise " - "size (lextent_size=%zu, extent_size=%zu)", lextent_size, + "Extent quantization should be a no-op for precise size " + "(lextent_size=%zu, extent_size=%zu)", lextent_size, extent_size); assert_zu_eq(extent_size, ceil, - "Large run quantization should be a no-op for precise " - "size (lextent_size=%zu, extent_size=%zu)", lextent_size, + "Extent quantization should be a no-op for precise size " + "(lextent_size=%zu, extent_size=%zu)", lextent_size, extent_size); if (i > 0) { diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 9ba730a6..872aeaa0 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -615,7 +615,8 @@ TEST_BEGIN(test_arenas_bin_constants) TEST_ARENAS_BIN_CONSTANT(size_t, size, arena_bin_info[0].reg_size); TEST_ARENAS_BIN_CONSTANT(uint32_t, nregs, arena_bin_info[0].nregs); - TEST_ARENAS_BIN_CONSTANT(size_t, run_size, arena_bin_info[0].run_size); + TEST_ARENAS_BIN_CONSTANT(size_t, slab_size, + arena_bin_info[0].slab_size); #undef TEST_ARENAS_BIN_CONSTANT } diff --git a/test/unit/stats.c b/test/unit/stats.c index b0e318a5..f524c005 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -229,9 +229,9 @@ TEST_BEGIN(test_stats_arenas_bins) { unsigned arena; void *p; - size_t sz, curruns, curregs; + size_t sz, curslabs, curregs; uint64_t epoch, nmalloc, ndalloc, nrequests, nfills, nflushes; - uint64_t nruns, nreruns; + uint64_t nslabs, nreslabs; int expected = config_stats ? 0 : ENOENT; arena = 0; @@ -266,12 +266,12 @@ TEST_BEGIN(test_stats_arenas_bins) NULL, 0), config_tcache ? expected : ENOENT, "Unexpected mallctl() result"); - assert_d_eq(mallctl("stats.arenas.0.bins.0.nruns", &nruns, &sz, + assert_d_eq(mallctl("stats.arenas.0.bins.0.nslabs", &nslabs, &sz, NULL, 0), expected, "Unexpected mallctl() result"); - assert_d_eq(mallctl("stats.arenas.0.bins.0.nreruns", &nreruns, &sz, + assert_d_eq(mallctl("stats.arenas.0.bins.0.nreslabs", &nreslabs, &sz, NULL, 0), expected, "Unexpected mallctl() result"); sz = sizeof(size_t); - assert_d_eq(mallctl("stats.arenas.0.bins.0.curruns", &curruns, &sz, + assert_d_eq(mallctl("stats.arenas.0.bins.0.curslabs", &curslabs, &sz, NULL, 0), expected, "Unexpected mallctl() result"); if (config_stats) { @@ -289,10 +289,10 @@ TEST_BEGIN(test_stats_arenas_bins) assert_u64_gt(nflushes, 0, "At least one flush should have occurred"); } - assert_u64_gt(nruns, 0, - "At least one run should have been allocated"); - assert_zu_gt(curruns, 0, - "At least one run should be currently allocated"); + assert_u64_gt(nslabs, 0, + "At least one slab should have been allocated"); + assert_zu_gt(curslabs, 0, + "At least one slab should be currently allocated"); } dallocx(p, 0);