diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index ecc82304..d0af91bf 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -13,22 +13,17 @@ extern ssize_t opt_decay_time; extern const arena_bin_info_t arena_bin_info[NBINS]; -extent_t *arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, - size_t alignment, bool *zero); void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); -void arena_extent_cache_maybe_insert(tsdn_t *tsdn, arena_t *arena, - extent_t *extent, bool cache); -void arena_extent_cache_maybe_remove(tsdn_t *tsdn, arena_t *arena, - extent_t *extent, bool cache); #ifdef JEMALLOC_JET size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr); #endif extent_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero); -void arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, +void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, extent_t *extent, bool locked); +void arena_extent_dalloc_large_finish(tsdn_t *tsdn, arena_t *arena, + extent_t *extent); void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t oldsize); void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h index c1c20731..8629446d 100644 --- a/include/jemalloc/internal/arena_structs_b.h +++ b/include/jemalloc/internal/arena_structs_b.h @@ -66,8 +66,8 @@ struct arena_decay_s { /* * Number of dirty pages at beginning of current epoch. During epoch * advancement we use the delta between arena->decay.ndirty and - * arena->ndirty to determine how many dirty pages, if any, were - * generated. + * extents_npages_get(&arena->extents_cached) to determine how many + * dirty pages, if any, were generated. */ size_t nunpurged; /* @@ -98,8 +98,8 @@ struct arena_bin_s { */ extent_heap_t slabs_nonfull; - /* Ring sentinel used to track full slabs. */ - extent_t slabs_full; + /* List used to track full slabs. */ + extent_list_t slabs_full; /* Bin statistics. */ malloc_bin_stats_t stats; @@ -107,84 +107,97 @@ struct arena_bin_s { struct arena_s { /* - * Number of threads currently assigned to this arena, synchronized via - * atomic operations. Each thread has two distinct assignments, one for - * application-serving allocation, and the other for internal metadata - * allocation. Internal metadata must not be allocated from arenas - * explicitly created via the arenas.create mallctl, because the - * arena..reset mallctl indiscriminately discards all allocations for - * the affected arena. + * Number of threads currently assigned to this arena. Each thread has + * two distinct assignments, one for application-serving allocation, and + * the other for internal metadata allocation. Internal metadata must + * not be allocated from arenas explicitly created via the arenas.create + * mallctl, because the arena..reset mallctl indiscriminately + * discards all allocations for the affected arena. * * 0: Application allocation. * 1: Internal metadata allocation. + * + * Synchronization: atomic. */ unsigned nthreads[2]; /* - * There are three classes of arena operations from a locking - * perspective: - * 1) Thread assignment (modifies nthreads) is synchronized via atomics. - * 2) Bin-related operations are protected by bin locks. - * 3) Extent-related operations are protected by this mutex. + * Synchronizes various arena operations, as indicated in field-specific + * comments. */ malloc_mutex_t lock; + /* Synchronization: lock. */ arena_stats_t stats; /* * List of tcaches for extant threads associated with this arena. * Stats from these are merged incrementally, and at exit if * opt_stats_print is enabled. + * + * Synchronization: lock. */ ql_head(tcache_t) tcache_ql; + /* Synchronization: lock. */ uint64_t prof_accumbytes; /* * PRNG state for cache index randomization of large allocation base * pointers. + * + * Synchronization: atomic. */ size_t offset_state; - /* Extent serial number generator state. */ + /* + * Extent serial number generator state. + * + * Synchronization: atomic. + */ size_t extent_sn_next; + /* Synchronization: lock. */ dss_prec_t dss_prec; - /* True if a thread is currently executing arena_purge_to_limit(). */ - bool purging; + /* + * 1/0 (true/false) if a thread is currently executing + * arena_purge_to_limit(). + * + * Synchronization: atomic. + */ + unsigned purging; - /* Number of pages in active extents. */ + /* + * Number of pages in active extents. + * + * Synchronization: atomic. + */ size_t nactive; /* - * Current count of pages within unused extents that are potentially - * dirty, and for which pages_purge_*() has not been called. By - * tracking this, we can institute a limit on how much dirty unused - * memory is mapped for each arena. + * Decay-based purging state. + * + * Synchronization: lock. */ - size_t ndirty; - - /* Decay-based purging state. */ arena_decay_t decay; - /* Extant large allocations. */ - ql_head(extent_t) large; + /* + * Extant large allocations. + * + * Synchronization: large_mtx. + */ + extent_list_t large; /* Synchronizes all large allocation/update/deallocation. */ malloc_mutex_t large_mtx; /* - * Heaps of extents that were previously allocated. These are used when - * allocating extents, in an attempt to re-use address space. + * Collections of extents that were previously allocated. These are + * used when allocating extents, in an attempt to re-use address space. + * + * Synchronization: internal. */ - extent_heap_t extents_cached[NPSIZES+1]; - extent_heap_t extents_retained[NPSIZES+1]; - /* - * Ring sentinel used to track unused dirty memory. Dirty memory is - * managed as an LRU of cached extents. - */ - extent_t extents_dirty; - /* Protects extents_{cached,retained,dirty}. */ - malloc_mutex_t extents_mtx; + extents_t extents_cached; + extents_t extents_retained; /* * Next extent size class in a growing series to use when satisfying a @@ -192,17 +205,31 @@ struct arena_s { * the number of disjoint virtual memory ranges so that extent merging * can be effective even if multiple arenas' extent allocation requests * are highly interleaved. + * + * Synchronization: atomic. */ pszind_t extent_grow_next; - /* Cache of extent structures that were allocated via base_alloc(). */ - ql_head(extent_t) extent_cache; - malloc_mutex_t extent_cache_mtx; + /* + * Freelist of extent structures that were allocated via base_alloc(). + * + * Synchronization: extent_freelist_mtx. + */ + extent_list_t extent_freelist; + malloc_mutex_t extent_freelist_mtx; - /* bins is used to store heaps of free regions. */ + /* + * bins is used to store heaps of free regions. + * + * Synchronization: internal. + */ arena_bin_t bins[NBINS]; - /* Base allocator, from which arena metadata are allocated. */ + /* + * Base allocator, from which arena metadata are allocated. + * + * Synchronization: internal. + */ base_t *base; }; diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h index 59f3c7ca..a3556118 100644 --- a/include/jemalloc/internal/extent_externs.h +++ b/include/jemalloc/internal/extent_externs.h @@ -21,9 +21,13 @@ size_t extent_size_quantize_ceil(size_t size); ph_proto(, extent_heap_, extent_heap_t, extent_t) -extent_t *extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, - size_t alignment, bool *zero, bool *commit, bool slab); +bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state); +extent_state_t extents_state_get(const extents_t *extents); +size_t extents_npages_get(extents_t *extents); +extent_t *extents_evict(tsdn_t *tsdn, extents_t *extents, size_t npages_min); +void extents_prefork(tsdn_t *tsdn, extents_t *extents); +void extents_postfork_parent(tsdn_t *tsdn, extents_t *extents); +void extents_postfork_child(tsdn_t *tsdn, extents_t *extents); extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab); diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h index 379dd290..473aad71 100644 --- a/include/jemalloc/internal/extent_inlines.h +++ b/include/jemalloc/internal/extent_inlines.h @@ -12,8 +12,7 @@ void *extent_before_get(const extent_t *extent); void *extent_last_get(const extent_t *extent); void *extent_past_get(const extent_t *extent); size_t extent_sn_get(const extent_t *extent); -bool extent_active_get(const extent_t *extent); -bool extent_retained_get(const extent_t *extent); +extent_state_t extent_state_get(const extent_t *extent); bool extent_zeroed_get(const extent_t *extent); bool extent_committed_get(const extent_t *extent); bool extent_slab_get(const extent_t *extent); @@ -26,16 +25,19 @@ void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment); void extent_size_set(extent_t *extent, size_t size); void extent_usize_set(extent_t *extent, size_t usize); void extent_sn_set(extent_t *extent, size_t sn); -void extent_active_set(extent_t *extent, bool active); +void extent_state_set(extent_t *extent, extent_state_t state); void extent_zeroed_set(extent_t *extent, bool zeroed); void extent_committed_set(extent_t *extent, bool committed); void extent_slab_set(extent_t *extent, bool slab); void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx); void extent_init(extent_t *extent, arena_t *arena, void *addr, - size_t size, size_t usize, size_t sn, bool active, bool zeroed, + size_t size, size_t usize, size_t sn, extent_state_t state, bool zeroed, bool committed, bool slab); -void extent_ring_insert(extent_t *sentinel, extent_t *extent); -void extent_ring_remove(extent_t *extent); +void extent_list_init(extent_list_t *list); +extent_t *extent_list_first(const extent_list_t *list); +extent_t *extent_list_last(const extent_list_t *list); +void extent_list_append(extent_list_t *list, extent_t *extent); +void extent_list_remove(extent_list_t *list, extent_t *extent); int extent_sn_comp(const extent_t *a, const extent_t *b); int extent_ad_comp(const extent_t *a, const extent_t *b); int extent_snad_comp(const extent_t *a, const extent_t *b); @@ -103,14 +105,9 @@ extent_sn_get(const extent_t *extent) { return extent->e_sn; } -JEMALLOC_INLINE bool -extent_active_get(const extent_t *extent) { - return extent->e_active; -} - -JEMALLOC_INLINE bool -extent_retained_get(const extent_t *extent) { - return (qr_next(extent, qr_link) == extent); +JEMALLOC_INLINE extent_state_t +extent_state_get(const extent_t *extent) { + return extent->e_state; } JEMALLOC_INLINE bool @@ -191,8 +188,8 @@ extent_sn_set(extent_t *extent, size_t sn) { } JEMALLOC_INLINE void -extent_active_set(extent_t *extent, bool active) { - extent->e_active = active; +extent_state_set(extent_t *extent, extent_state_t state) { + extent->e_state = state; } JEMALLOC_INLINE void @@ -217,7 +214,7 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { JEMALLOC_INLINE void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, - size_t usize, size_t sn, bool active, bool zeroed, bool committed, + size_t usize, size_t sn, extent_state_t state, bool zeroed, bool committed, bool slab) { assert(addr == PAGE_ADDR2BASE(addr) || !slab); @@ -226,24 +223,39 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, extent_size_set(extent, size); extent_usize_set(extent, usize); extent_sn_set(extent, sn); - extent_active_set(extent, active); + extent_state_set(extent, state); extent_zeroed_set(extent, zeroed); extent_committed_set(extent, committed); extent_slab_set(extent, slab); if (config_prof) { extent_prof_tctx_set(extent, NULL); } - qr_new(extent, qr_link); + ql_elm_new(extent, ql_link); } JEMALLOC_INLINE void -extent_ring_insert(extent_t *sentinel, extent_t *extent) { - qr_meld(sentinel, extent, extent_t, qr_link); +extent_list_init(extent_list_t *list) { + ql_new(list); +} + +JEMALLOC_INLINE extent_t * +extent_list_first(const extent_list_t *list) { + return ql_first(list); +} + +JEMALLOC_INLINE extent_t * +extent_list_last(const extent_list_t *list) { + return ql_last(list, ql_link); } JEMALLOC_INLINE void -extent_ring_remove(extent_t *extent) { - qr_remove(extent, qr_link); +extent_list_append(extent_list_t *list, extent_t *extent) { + ql_tail_insert(list, extent, ql_link); +} + +JEMALLOC_INLINE void +extent_list_remove(extent_list_t *list, extent_t *extent) { + ql_remove(list, extent, ql_link); } JEMALLOC_INLINE int diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h index de31317c..33ca4ac7 100644 --- a/include/jemalloc/internal/extent_structs.h +++ b/include/jemalloc/internal/extent_structs.h @@ -1,6 +1,12 @@ #ifndef JEMALLOC_INTERNAL_EXTENT_STRUCTS_H #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H +typedef enum { + extent_state_active = 0, + extent_state_dirty = 1, + extent_state_retained = 2 +} extent_state_t; + /* Extent (span of pages). Use accessor functions for e_* fields. */ struct extent_s { /* Arena from which this extent came, if any. */ @@ -32,8 +38,8 @@ struct extent_s { */ size_t e_sn; - /* True if extent is active (in use). */ - bool e_active; + /* Extent state. */ + extent_state_t e_state; /* * The zeroed flag is used by extent recycling code to track whether @@ -67,18 +73,48 @@ struct extent_s { }; /* - * Linkage for arena's extents_dirty and arena_bin_t's slabs_full rings. + * List linkage, used by a variety of lists: + * - arena_bin_t's slabs_full + * - extents_t's LRU + * - stashed dirty extents + * - arena's large allocations + * - arena's extent structure freelist */ - qr(extent_t) qr_link; + ql_elm(extent_t) ql_link; - union { - /* Linkage for per size class sn/address-ordered heaps. */ - phn(extent_t) ph_link; - - /* Linkage for arena's large and extent_cache lists. */ - ql_elm(extent_t) ql_link; - }; + /* Linkage for per size class sn/address-ordered heaps. */ + phn(extent_t) ph_link; }; +typedef ql_head(extent_t) extent_list_t; typedef ph(extent_t) extent_heap_t; +/* Quantized collection of extents, with built-in LRU queue. */ +struct extents_s { + malloc_mutex_t mtx; + + /* + * Quantized per size class heaps of extents. + * + * Synchronization: mtx. + */ + extent_heap_t heaps[NPSIZES+1]; + + /* + * LRU of all extents in heaps. + * + * Synchronization: mtx. + */ + extent_list_t lru; + + /* + * Page sum for all extents in heaps. + * + * Synchronization: atomic. + */ + size_t npages; + + /* All stored extents must be in the same state. */ + extent_state_t state; +}; + #endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */ diff --git a/include/jemalloc/internal/extent_types.h b/include/jemalloc/internal/extent_types.h index 53db1c36..b6905ce1 100644 --- a/include/jemalloc/internal/extent_types.h +++ b/include/jemalloc/internal/extent_types.h @@ -2,6 +2,7 @@ #define JEMALLOC_INTERNAL_EXTENT_TYPES_H typedef struct extent_s extent_t; +typedef struct extents_s extents_t; #define EXTENT_HOOKS_INITIALIZER NULL diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 33fd2fac..bace9c46 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -979,6 +979,7 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_ind_get(arena) < narenas_auto); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); if (config_stats && is_internal && likely(ret != NULL)) { @@ -1004,6 +1005,7 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_ind_get(arena) < narenas_auto); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); @@ -1042,7 +1044,7 @@ ivsalloc(tsdn_t *tsdn, const void *ptr) { if (extent == NULL) { return 0; } - assert(extent_active_get(extent)); + assert(extent_state_get(extent) == extent_state_active); /* Only slab members should be looked up via interior pointers. */ assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); @@ -1056,6 +1058,7 @@ idalloctm(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache, assert(!is_internal || tcache == NULL); assert(!is_internal || arena_ind_get(iaalloc(tsdn, ptr)) < narenas_auto); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); if (config_stats && is_internal) { arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, extent, ptr)); @@ -1073,6 +1076,7 @@ idalloc(tsd_t *tsd, extent_t *extent, void *ptr) { JEMALLOC_ALWAYS_INLINE void isdalloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size, tcache_t *tcache, bool slow_path) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); arena_sdalloc(tsdn, extent, ptr, size, tcache, slow_path); } @@ -1080,6 +1084,7 @@ JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); void *p; size_t usize, copysize; @@ -1117,6 +1122,7 @@ iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { assert(ptr != NULL); assert(size != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { @@ -1144,6 +1150,7 @@ ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero) { assert(ptr != NULL); assert(size != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { diff --git a/include/jemalloc/internal/large_externs.h b/include/jemalloc/internal/large_externs.h index f0a03399..66aa755c 100644 --- a/include/jemalloc/internal/large_externs.h +++ b/include/jemalloc/internal/large_externs.h @@ -17,7 +17,8 @@ extern large_dalloc_maybe_junk_t *large_dalloc_maybe_junk; void large_dalloc_junk(void *ptr, size_t usize); void large_dalloc_maybe_junk(void *ptr, size_t usize); #endif -void large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent); +void large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent); +void large_dalloc_finish(tsdn_t *tsdn, extent_t *extent); void large_dalloc(tsdn_t *tsdn, extent_t *extent); size_t large_salloc(tsdn_t *tsdn, const extent_t *extent); prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 36bcda24..d1166b20 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -25,11 +25,9 @@ arena_destroy arena_dss_prec_get arena_dss_prec_set arena_extent_alloc_large -arena_extent_cache_alloc arena_extent_cache_dalloc -arena_extent_cache_maybe_insert -arena_extent_cache_maybe_remove -arena_extent_dalloc_large +arena_extent_dalloc_large_finish +arena_extent_dalloc_large_prep arena_extent_ralloc_large_expand arena_extent_ralloc_large_shrink arena_extent_sn_next @@ -141,15 +139,12 @@ ctl_postfork_parent ctl_prefork decay_ticker_get dss_prec_names -extent_active_get -extent_active_set extent_ad_comp extent_addr_get extent_addr_randomize extent_addr_set extent_alloc extent_alloc_cache -extent_alloc_cache_locked extent_alloc_dss extent_alloc_mmap extent_alloc_wrapper @@ -184,6 +179,10 @@ extent_hooks_set extent_in_dss extent_init extent_last_get +extent_list_append +extent_list_first +extent_list_last +extent_list_remove extent_lookup extent_merge_wrapper extent_past_get @@ -191,9 +190,6 @@ extent_prof_tctx_get extent_prof_tctx_set extent_purge_forced_wrapper extent_purge_lazy_wrapper -extent_retained_get -extent_ring_insert -extent_ring_remove extent_size_get extent_size_quantize_ceil extent_size_quantize_floor @@ -207,11 +203,20 @@ extent_sn_get extent_sn_set extent_snad_comp extent_split_wrapper +extent_state_get +extent_state_set extent_usize_get extent_usize_set extent_zeroed_get extent_zeroed_set +extents_evict +extents_init +extents_npages_get +extents_prefork +extents_postfork_child +extents_postfork_parent extents_rtree +extents_state_get ffs_llu ffs_lu ffs_u @@ -255,9 +260,10 @@ jemalloc_postfork_child jemalloc_postfork_parent jemalloc_prefork large_dalloc +large_dalloc_finish large_dalloc_junk -large_dalloc_junked_locked large_dalloc_maybe_junk +large_dalloc_prep_junked_locked large_malloc large_palloc large_prof_tctx_get diff --git a/include/jemalloc/internal/stats_structs.h b/include/jemalloc/internal/stats_structs.h index 32ef6118..5cdb0cd9 100644 --- a/include/jemalloc/internal/stats_structs.h +++ b/include/jemalloc/internal/stats_structs.h @@ -70,9 +70,14 @@ struct malloc_large_stats_s { size_t curlextents; }; +/* + * Arena stats. Note that fields marked "derived" are not directly maintained + * within the arena code; rather their values are derived during stats merge + * requests. + */ struct arena_stats_s { - /* Number of bytes currently mapped. */ - size_t mapped; + /* Number of bytes currently mapped, excluding retained memory. */ + size_t mapped; /* Derived. */ /* * Number of bytes currently retained as a side effect of munmap() being @@ -80,7 +85,7 @@ struct arena_stats_s { * always decommitted or purged), but they are excluded from the mapped * statistic (above). */ - size_t retained; + size_t retained; /* Derived. */ /* * Total number of purge sweeps, total number of madvise calls made, @@ -91,9 +96,9 @@ struct arena_stats_s { uint64_t nmadvise; uint64_t purged; - size_t base; + size_t base; /* Derived. */ size_t internal; /* Protected via atomic_*_zu(). */ - size_t resident; + size_t resident; /* Derived. */ size_t allocated_large; uint64_t nmalloc_large; @@ -101,7 +106,7 @@ struct arena_stats_s { uint64_t nrequests_large; /* Number of bytes cached in tcache associated with this arena. */ - size_t tcache_bytes; + size_t tcache_bytes; /* Derived. */ /* One element for each large size class. */ malloc_large_stats_t lstats[NSIZES - NBINS]; diff --git a/include/jemalloc/internal/witness_types.h b/include/jemalloc/internal/witness_types.h index dfcf1621..29299168 100644 --- a/include/jemalloc/internal/witness_types.h +++ b/include/jemalloc/internal/witness_types.h @@ -26,9 +26,17 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *, #define WITNESS_RANK_PROF_TDATA 7U #define WITNESS_RANK_PROF_GCTX 8U +/* + * Used as an argument to witness_depth_to_rank() in order to validate depth + * excluding non-core locks with lower ranks. Since the rank argument to + * witness_depth_to_rank() is inclusive rather than exclusive, this definition + * can have the same value as the minimally ranked core lock. + */ +#define WITNESS_RANK_CORE 9U + #define WITNESS_RANK_ARENA 9U -#define WITNESS_RANK_ARENA_EXTENTS 10U -#define WITNESS_RANK_ARENA_EXTENT_CACHE 11U +#define WITNESS_RANK_EXTENTS 10U +#define WITNESS_RANK_EXTENT_FREELIST 11U #define WITNESS_RANK_RTREE_ELM 12U #define WITNESS_RANK_RTREE 13U diff --git a/src/arena.c b/src/arena.c index b0da9a03..5905306c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -37,75 +37,13 @@ static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, /******************************************************************************/ -static size_t -arena_extent_dirty_npages(const extent_t *extent) { - return (extent_size_get(extent) >> LG_PAGE); -} - -static extent_t * -arena_extent_cache_alloc_locked(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, - size_t alignment, bool *zero, bool slab) { - bool commit = true; - - malloc_mutex_assert_owner(tsdn, &arena->lock); - - return extent_alloc_cache(tsdn, arena, r_extent_hooks, new_addr, usize, - pad, alignment, zero, &commit, slab); -} - -extent_t * -arena_extent_cache_alloc(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, - size_t alignment, bool *zero) { - extent_t *extent; - - malloc_mutex_lock(tsdn, &arena->lock); - extent = arena_extent_cache_alloc_locked(tsdn, arena, r_extent_hooks, - new_addr, size, 0, alignment, zero, false); - malloc_mutex_unlock(tsdn, &arena->lock); - - return extent; -} - -static void -arena_extent_cache_dalloc_locked(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_t *extent) { - malloc_mutex_assert_owner(tsdn, &arena->lock); - - extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent); - arena_maybe_purge(tsdn, arena); -} - void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { - malloc_mutex_lock(tsdn, &arena->lock); - arena_extent_cache_dalloc_locked(tsdn, arena, r_extent_hooks, extent); - malloc_mutex_unlock(tsdn, &arena->lock); -} + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); -void -arena_extent_cache_maybe_insert(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - bool cache) { - malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); - - if (cache) { - extent_ring_insert(&arena->extents_dirty, extent); - arena->ndirty += arena_extent_dirty_npages(extent); - } -} - -void -arena_extent_cache_maybe_remove(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - bool dirty) { - malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); - - if (dirty) { - extent_ring_remove(extent); - assert(arena->ndirty >= arena_extent_dirty_npages(extent)); - arena->ndirty -= arena_extent_dirty_npages(extent); - } + extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent); + arena_purge(tsdn, arena, false); } JEMALLOC_INLINE_C void * @@ -180,13 +118,13 @@ arena_slab_reg_dalloc(tsdn_t *tsdn, extent_t *slab, static void arena_nactive_add(arena_t *arena, size_t add_pages) { - arena->nactive += add_pages; + atomic_add_zu(&arena->nactive, add_pages); } static void arena_nactive_sub(arena_t *arena, size_t sub_pages) { - assert(arena->nactive >= sub_pages); - arena->nactive -= sub_pages; + assert(atomic_read_zu(&arena->nactive) >= sub_pages); + atomic_sub_zu(&arena->nactive, sub_pages); } static void @@ -269,6 +207,8 @@ arena_extent_alloc_large_hard(tsdn_t *tsdn, arena_t *arena, extent_t *extent; bool commit = true; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + extent = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, usize, large_pad, alignment, zero, &commit, false); if (extent == NULL) { @@ -291,6 +231,8 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, extent_t *extent; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + malloc_mutex_lock(tsdn, &arena->lock); /* Optimistically update stats. */ @@ -300,9 +242,11 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, } arena_nactive_add(arena, (usize + large_pad) >> LG_PAGE); - extent = arena_extent_cache_alloc_locked(tsdn, arena, &extent_hooks, - NULL, usize, large_pad, alignment, zero, false); malloc_mutex_unlock(tsdn, &arena->lock); + + bool commit = true; + extent = extent_alloc_cache(tsdn, arena, &extent_hooks, NULL, usize, + large_pad, alignment, zero, &commit, false); if (extent == NULL) { extent = arena_extent_alloc_large_hard(tsdn, arena, &extent_hooks, usize, alignment, zero); @@ -312,10 +256,8 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, } void -arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent, +arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, extent_t *extent, bool locked) { - extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; - if (!locked) { malloc_mutex_lock(tsdn, &arena->lock); } else { @@ -326,12 +268,17 @@ arena_extent_dalloc_large(tsdn_t *tsdn, arena_t *arena, extent_t *extent, extent_usize_get(extent)); arena->stats.mapped -= extent_size_get(extent); } - arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE); - - arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, extent); if (!locked) { malloc_mutex_unlock(tsdn, &arena->lock); } + arena_nactive_sub(arena, extent_size_get(extent) >> LG_PAGE); +} + +void +arena_extent_dalloc_large_finish(tsdn_t *tsdn, arena_t *arena, + extent_t *extent) { + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; + extent_dalloc_cache(tsdn, arena, &extent_hooks, extent); } void @@ -414,8 +361,9 @@ arena_decay_backlog_npages_limit(const arena_t *arena) { static void arena_decay_backlog_update_last(arena_t *arena) { - size_t ndirty_delta = (arena->ndirty > arena->decay.nunpurged) ? - arena->ndirty - arena->decay.nunpurged : 0; + size_t ndirty = extents_npages_get(&arena->extents_cached); + size_t ndirty_delta = (ndirty > arena->decay.nunpurged) ? ndirty - + arena->decay.nunpurged : 0; arena->decay.backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta; } @@ -468,10 +416,15 @@ static void arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena) { size_t ndirty_limit = arena_decay_backlog_npages_limit(arena); - if (arena->ndirty > ndirty_limit) { + if (extents_npages_get(&arena->extents_cached) > ndirty_limit) { arena_purge_to_limit(tsdn, arena, ndirty_limit); } - arena->decay.nunpurged = arena->ndirty; + /* + * There may be concurrent ndirty fluctuation between the purge above + * and the nunpurged update below, but this is inconsequential to decay + * machinery correctness. + */ + arena->decay.nunpurged = extents_npages_get(&arena->extents_cached); } static void @@ -492,7 +445,7 @@ arena_decay_init(arena_t *arena, ssize_t decay_time) { nstime_update(&arena->decay.epoch); arena->decay.jitter_state = (uint64_t)(uintptr_t)arena; arena_decay_deadline_init(arena); - arena->decay.nunpurged = arena->ndirty; + arena->decay.nunpurged = extents_npages_get(&arena->extents_cached); memset(arena->decay.backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t)); } @@ -540,9 +493,9 @@ arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) { return false; } -static void -arena_maybe_purge_helper(tsdn_t *tsdn, arena_t *arena) { - nstime_t time; +void +arena_maybe_purge(tsdn_t *tsdn, arena_t *arena) { + malloc_mutex_assert_owner(tsdn, &arena->lock); /* Purge all or nothing if the option is disabled. */ if (arena->decay.time <= 0) { @@ -552,6 +505,7 @@ arena_maybe_purge_helper(tsdn_t *tsdn, arena_t *arena) { return; } + nstime_t time; nstime_init(&time, 0); nstime_update(&time); if (unlikely(!nstime_monotonic() && nstime_compare(&arena->decay.epoch, @@ -583,95 +537,40 @@ arena_maybe_purge_helper(tsdn_t *tsdn, arena_t *arena) { } } -void -arena_maybe_purge(tsdn_t *tsdn, arena_t *arena) { - malloc_mutex_assert_owner(tsdn, &arena->lock); - - /* Don't recursively purge. */ - if (arena->purging) { - return; - } - - arena_maybe_purge_helper(tsdn, arena); -} - -static size_t -arena_dirty_count(tsdn_t *tsdn, arena_t *arena) { - extent_t *extent; - size_t ndirty = 0; - - malloc_mutex_lock(tsdn, &arena->extents_mtx); - - for (extent = qr_next(&arena->extents_dirty, qr_link); extent != - &arena->extents_dirty; extent = qr_next(extent, qr_link)) { - ndirty += extent_size_get(extent) >> LG_PAGE; - } - - malloc_mutex_unlock(tsdn, &arena->extents_mtx); - - return ndirty; -} - static size_t arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - size_t ndirty_limit, extent_t *purge_extents_sentinel) { - extent_t *extent, *next; - size_t nstashed = 0; - - malloc_mutex_lock(tsdn, &arena->extents_mtx); + size_t ndirty_limit, extent_list_t *purge_extents) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); /* Stash extents according to ndirty_limit. */ - for (extent = qr_next(&arena->extents_dirty, qr_link); extent != - &arena->extents_dirty; extent = next) { - size_t npages; - bool zero, commit; - UNUSED extent_t *textent; - - npages = extent_size_get(extent) >> LG_PAGE; - if (arena->ndirty - (nstashed + npages) < ndirty_limit) { - break; - } - - next = qr_next(extent, qr_link); - /* Allocate. */ - zero = false; - commit = false; - textent = extent_alloc_cache_locked(tsdn, arena, r_extent_hooks, - extent_base_get(extent), extent_size_get(extent), 0, PAGE, - &zero, &commit, false); - assert(textent == extent); - assert(zero == extent_zeroed_get(extent)); - extent_ring_remove(extent); - extent_ring_insert(purge_extents_sentinel, extent); - - nstashed += npages; + size_t nstashed = 0; + for (extent_t *extent = extents_evict(tsdn, &arena->extents_cached, + ndirty_limit); extent != NULL; extent = extents_evict(tsdn, + &arena->extents_cached, ndirty_limit)) { + extent_list_append(purge_extents, extent); + nstashed += extent_size_get(extent) >> LG_PAGE; } - - malloc_mutex_unlock(tsdn, &arena->extents_mtx); return nstashed; } static size_t arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_t *purge_extents_sentinel) { + extent_hooks_t **r_extent_hooks, extent_list_t *purge_extents) { UNUSED size_t nmadvise; size_t npurged; - extent_t *extent, *next; if (config_stats) { nmadvise = 0; } npurged = 0; - for (extent = qr_next(purge_extents_sentinel, qr_link); extent != - purge_extents_sentinel; extent = next) { + for (extent_t *extent = extent_list_first(purge_extents); extent != + NULL; extent = extent_list_first(purge_extents)) { if (config_stats) { nmadvise++; } npurged += extent_size_get(extent) >> LG_PAGE; - - next = qr_next(extent, qr_link); - extent_ring_remove(extent); + extent_list_remove(purge_extents, extent); extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, extent); } @@ -684,43 +583,44 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, } /* - * ndirty_limit: Purge as many dirty extents as possible without violating the - * invariant: (arena->ndirty >= ndirty_limit) + * ndirty_limit: Purge as many dirty extents as possible without violating the + * invariant: (extents_npages_get(&arena->extents_cached) >= ndirty_limit) */ static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1); + malloc_mutex_assert_owner(tsdn, &arena->lock); + + if (atomic_cas_u(&arena->purging, 0, 1)) { + return; + } + extent_hooks_t *extent_hooks = extent_hooks_get(arena); size_t npurge, npurged; - extent_t purge_extents_sentinel; + extent_list_t purge_extents; - arena->purging = true; + extent_list_init(&purge_extents); - /* - * Calls to arena_dirty_count() are disabled even for debug builds - * because overhead grows nonlinearly as memory usage increases. - */ - if (false && config_debug) { - size_t ndirty = arena_dirty_count(tsdn, arena); - assert(ndirty == arena->ndirty); - } - extent_init(&purge_extents_sentinel, arena, NULL, 0, 0, 0, false, false, - false, false); + malloc_mutex_unlock(tsdn, &arena->lock); npurge = arena_stash_dirty(tsdn, arena, &extent_hooks, ndirty_limit, - &purge_extents_sentinel); + &purge_extents); if (npurge == 0) { + malloc_mutex_lock(tsdn, &arena->lock); goto label_return; } npurged = arena_purge_stashed(tsdn, arena, &extent_hooks, - &purge_extents_sentinel); + &purge_extents); assert(npurged == npurge); + malloc_mutex_lock(tsdn, &arena->lock); + if (config_stats) { arena->stats.npurge++; } label_return: - arena->purging = false; + atomic_write_u(&arena->purging, 0); } void @@ -737,9 +637,14 @@ arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) { static void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; + size_t npages = extent_size_get(slab) >> LG_PAGE; - arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); - arena_extent_cache_dalloc_locked(tsdn, arena, &extent_hooks, slab); + extent_dalloc_cache(tsdn, arena, &extent_hooks, slab); + + arena_nactive_sub(arena, npages); + malloc_mutex_lock(tsdn, &arena->lock); + arena_maybe_purge(tsdn, arena); + malloc_mutex_unlock(tsdn, &arena->lock); } static void @@ -768,19 +673,16 @@ arena_bin_slabs_nonfull_tryget(arena_bin_t *bin) { static void arena_bin_slabs_full_insert(arena_bin_t *bin, extent_t *slab) { assert(extent_slab_data_get(slab)->nfree == 0); - extent_ring_insert(&bin->slabs_full, slab); + extent_list_append(&bin->slabs_full, slab); } static void -arena_bin_slabs_full_remove(extent_t *slab) { - extent_ring_remove(slab); +arena_bin_slabs_full_remove(arena_bin_t *bin, extent_t *slab) { + extent_list_remove(&bin->slabs_full, slab); } void arena_reset(tsd_t *tsd, arena_t *arena) { - unsigned i; - extent_t *extent; - /* * Locking in this function is unintuitive. The caller guarantees that * no concurrent operations are happening in this arena, but there are @@ -797,8 +699,9 @@ arena_reset(tsd_t *tsd, arena_t *arena) { /* Large allocations. */ malloc_mutex_lock(tsd_tsdn(tsd), &arena->large_mtx); - for (extent = ql_last(&arena->large, ql_link); extent != NULL; extent = - ql_last(&arena->large, ql_link)) { + + for (extent_t *extent = extent_list_first(&arena->large); extent != + NULL; extent = extent_list_first(&arena->large)) { void *ptr = extent_base_get(extent); size_t usize; @@ -819,10 +722,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) { } malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); - malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock); - /* Bins. */ - for (i = 0; i < NBINS; i++) { + for (unsigned i = 0; i < NBINS; i++) { extent_t *slab; arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); @@ -839,10 +740,9 @@ arena_reset(tsd_t *tsd, arena_t *arena) { arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); } - for (slab = qr_next(&bin->slabs_full, qr_link); slab != - &bin->slabs_full; slab = qr_next(&bin->slabs_full, - qr_link)) { - arena_bin_slabs_full_remove(slab); + for (slab = extent_list_first(&bin->slabs_full); slab != NULL; + slab = extent_list_first(&bin->slabs_full)) { + arena_bin_slabs_full_remove(bin, slab); malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); @@ -854,17 +754,12 @@ arena_reset(tsd_t *tsd, arena_t *arena) { malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); } - assert(!arena->purging); - arena->nactive = 0; - - malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock); + assert(atomic_read_u(&arena->purging) == 0); + atomic_write_zu(&arena->nactive, 0); } static void arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) { - extent_hooks_t *extent_hooks = extent_hooks_get(arena); - size_t i; - /* * Iterate over the retained extents and blindly attempt to deallocate * them. This gives the extent allocator underlying the extent hooks an @@ -876,15 +771,11 @@ arena_destroy_retained(tsdn_t *tsdn, arena_t *arena) { * dss for arenas to be destroyed), or provide custom extent hooks that * either unmap retained extents or track them for later use. */ - for (i = 0; i < sizeof(arena->extents_retained)/sizeof(extent_heap_t); - i++) { - extent_heap_t *extents = &arena->extents_retained[i]; - extent_t *extent; - - while ((extent = extent_heap_remove_first(extents)) != NULL) { - extent_dalloc_wrapper_try(tsdn, arena, &extent_hooks, - extent); - } + extent_hooks_t *extent_hooks = extent_hooks_get(arena); + for (extent_t *extent = extents_evict(tsdn, &arena->extents_retained, + 0); extent != NULL; extent = extents_evict(tsdn, + &arena->extents_retained, 0)) { + extent_dalloc_wrapper_try(tsdn, arena, &extent_hooks, extent); } } @@ -899,7 +790,7 @@ arena_destroy(tsd_t *tsd, arena_t *arena) { * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached * extents, so only retained extents may remain. */ - assert(arena->ndirty == 0); + assert(extents_npages_get(&arena->extents_cached) == 0); /* Attempt to deallocate retained memory. */ arena_destroy_retained(tsd_tsdn(tsd), arena); @@ -929,12 +820,12 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, extent_t *slab; bool zero, commit; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + zero = false; commit = true; - malloc_mutex_unlock(tsdn, &arena->lock); slab = extent_alloc_wrapper(tsdn, arena, r_extent_hooks, NULL, bin_info->slab_size, 0, PAGE, &zero, &commit, true); - malloc_mutex_lock(tsdn, &arena->lock); return slab; } @@ -942,13 +833,13 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, static extent_t * arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, const arena_bin_info_t *bin_info) { - extent_t *slab; - arena_slab_data_t *slab_data; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; bool zero = false; - - slab = arena_extent_cache_alloc_locked(tsdn, arena, &extent_hooks, NULL, - bin_info->slab_size, 0, PAGE, &zero, true); + bool commit = true; + extent_t *slab = extent_alloc_cache(tsdn, arena, &extent_hooks, NULL, + bin_info->slab_size, 0, PAGE, &zero, &commit, true); if (slab == NULL) { slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks, bin_info); @@ -958,10 +849,12 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, } assert(extent_slab_get(slab)); + malloc_mutex_lock(tsdn, &arena->lock); + arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); /* Initialize slab internals. */ - slab_data = extent_slab_data_get(slab); + arena_slab_data_t *slab_data = extent_slab_data_get(slab); slab_data->binind = binind; slab_data->nfree = bin_info->nregs; bitmap_init(slab_data->bitmap, &bin_info->bitmap_info); @@ -969,6 +862,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, if (config_stats) { arena->stats.mapped += extent_size_get(slab); } + malloc_mutex_unlock(tsdn, &arena->lock); return slab; } @@ -991,9 +885,7 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, /* Allocate a new slab. */ malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ - malloc_mutex_lock(tsdn, &arena->lock); slab = arena_slab_alloc(tsdn, arena, binind, bin_info); - malloc_mutex_unlock(tsdn, &arena->lock); /********************************/ malloc_mutex_lock(tsdn, &bin->lock); if (slab != NULL) { @@ -1317,7 +1209,7 @@ arena_dissociate_bin_slab(extent_t *slab, arena_bin_t *bin) { * into the non-full slabs heap. */ if (bin_info->nregs == 1) { - arena_bin_slabs_full_remove(slab); + arena_bin_slabs_full_remove(bin, slab); } else { arena_bin_slabs_nonfull_remove(bin, slab); } @@ -1331,9 +1223,7 @@ arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ - malloc_mutex_lock(tsdn, &arena->lock); arena_slab_dalloc(tsdn, arena, slab); - malloc_mutex_unlock(tsdn, &arena->lock); /****************************/ malloc_mutex_lock(tsdn, &bin->lock); if (config_stats) { @@ -1385,7 +1275,7 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, arena_dissociate_bin_slab(slab, bin); arena_dalloc_bin_slab(tsdn, arena, slab, bin); } else if (slab_data->nfree == 1 && slab != bin->slabcur) { - arena_bin_slabs_full_remove(slab); + arena_bin_slabs_full_remove(bin, slab); arena_bin_lower_slab(tsdn, arena, slab, bin); } @@ -1554,8 +1444,8 @@ arena_basic_stats_merge_locked(arena_t *arena, unsigned *nthreads, *nthreads += arena_nthreads_get(arena, false); *dss = dss_prec_names[arena->dss_prec]; *decay_time = arena->decay.time; - *nactive += arena->nactive; - *ndirty += arena->ndirty; + *nactive += atomic_read_zu(&arena->nactive); + *ndirty += extents_npages_get(&arena->extents_cached); } void @@ -1585,14 +1475,15 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, &base_mapped); astats->mapped += base_mapped + arena->stats.mapped; - astats->retained += arena->stats.retained; + astats->retained += (extents_npages_get(&arena->extents_retained) << + LG_PAGE); astats->npurge += arena->stats.npurge; astats->nmadvise += arena->stats.nmadvise; astats->purged += arena->stats.purged; astats->base += base_allocated; astats->internal += arena_internal_get(arena); - astats->resident += base_resident + (((arena->nactive + arena->ndirty) - << LG_PAGE)); + astats->resident += base_resident + (((atomic_read_zu(&arena->nactive) + + extents_npages_get(&arena->extents_cached)) << LG_PAGE)); astats->allocated_large += arena->stats.allocated_large; astats->nmalloc_large += arena->stats.nmalloc_large; astats->ndalloc_large += arena->stats.ndalloc_large; @@ -1709,28 +1600,22 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { arena->dss_prec = extent_dss_prec_get(); - arena->purging = false; - arena->nactive = 0; - arena->ndirty = 0; + atomic_write_u(&arena->purging, 0); + atomic_write_zu(&arena->nactive, 0); arena_decay_init(arena, arena_decay_time_default_get()); - ql_new(&arena->large); + extent_list_init(&arena->large); if (malloc_mutex_init(&arena->large_mtx, "arena_large", WITNESS_RANK_ARENA_LARGE)) { goto label_error; } - for (i = 0; i < NPSIZES+1; i++) { - extent_heap_new(&arena->extents_cached[i]); - extent_heap_new(&arena->extents_retained[i]); + if (extents_init(tsdn, &arena->extents_cached, extent_state_dirty)) { + goto label_error; } - - extent_init(&arena->extents_dirty, arena, NULL, 0, 0, 0, false, false, - false, false); - - if (malloc_mutex_init(&arena->extents_mtx, "arena_extents", - WITNESS_RANK_ARENA_EXTENTS)) { + if (extents_init(tsdn, &arena->extents_retained, + extent_state_retained)) { goto label_error; } @@ -1738,9 +1623,9 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { arena->extent_grow_next = psz2ind(HUGEPAGE); } - ql_new(&arena->extent_cache); - if (malloc_mutex_init(&arena->extent_cache_mtx, "arena_extent_cache", - WITNESS_RANK_ARENA_EXTENT_CACHE)) { + extent_list_init(&arena->extent_freelist); + if (malloc_mutex_init(&arena->extent_freelist_mtx, "extent_freelist", + WITNESS_RANK_EXTENT_FREELIST)) { goto label_error; } @@ -1753,8 +1638,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } bin->slabcur = NULL; extent_heap_new(&bin->slabs_nonfull); - extent_init(&bin->slabs_full, arena, NULL, 0, 0, 0, false, - false, false, false); + extent_list_init(&bin->slabs_full); if (config_stats) { memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } @@ -1782,12 +1666,13 @@ arena_prefork0(tsdn_t *tsdn, arena_t *arena) { void arena_prefork1(tsdn_t *tsdn, arena_t *arena) { - malloc_mutex_prefork(tsdn, &arena->extents_mtx); + extents_prefork(tsdn, &arena->extents_cached); + extents_prefork(tsdn, &arena->extents_retained); } void arena_prefork2(tsdn_t *tsdn, arena_t *arena) { - malloc_mutex_prefork(tsdn, &arena->extent_cache_mtx); + malloc_mutex_prefork(tsdn, &arena->extent_freelist_mtx); } void @@ -1810,8 +1695,9 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock); } base_postfork_parent(tsdn, arena->base); - malloc_mutex_postfork_parent(tsdn, &arena->extent_cache_mtx); - malloc_mutex_postfork_parent(tsdn, &arena->extents_mtx); + malloc_mutex_postfork_parent(tsdn, &arena->extent_freelist_mtx); + extents_postfork_parent(tsdn, &arena->extents_cached); + extents_postfork_parent(tsdn, &arena->extents_retained); malloc_mutex_postfork_parent(tsdn, &arena->lock); } @@ -1824,7 +1710,8 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock); } base_postfork_child(tsdn, arena->base); - malloc_mutex_postfork_child(tsdn, &arena->extent_cache_mtx); - malloc_mutex_postfork_child(tsdn, &arena->extents_mtx); + malloc_mutex_postfork_child(tsdn, &arena->extent_freelist_mtx); + extents_postfork_child(tsdn, &arena->extents_cached); + extents_postfork_child(tsdn, &arena->extents_retained); malloc_mutex_postfork_child(tsdn, &arena->lock); } diff --git a/src/base.c b/src/base.c index 9fb1f14f..e7712a64 100644 --- a/src/base.c +++ b/src/base.c @@ -87,7 +87,8 @@ base_extent_init(size_t *extent_sn_next, extent_t *extent, void *addr, sn = *extent_sn_next; (*extent_sn_next)++; - extent_init(extent, NULL, addr, size, 0, sn, true, true, true, false); + extent_init(extent, NULL, addr, size, 0, sn, extent_state_active, true, + true, false); } static void * @@ -104,7 +105,7 @@ base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size, assert(extent_size_get(extent) >= *gap_size + size); extent_init(extent, NULL, (void *)((uintptr_t)extent_addr_get(extent) + *gap_size + size), extent_size_get(extent) - *gap_size - size, 0, - extent_sn_get(extent), true, true, true, false); + extent_sn_get(extent), extent_state_active, true, true, false); return ret; } diff --git a/src/extent.c b/src/extent.c index e2af2b50..293b96e5 100644 --- a/src/extent.c +++ b/src/extent.c @@ -68,9 +68,9 @@ static size_t highpages; * definition. */ -static void extent_record(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES+1], - bool cache, extent_t *extent); +static void extent_deregister(tsdn_t *tsdn, extent_t *extent); +static void extent_record(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent); /******************************************************************************/ @@ -78,24 +78,26 @@ extent_t * extent_alloc(tsdn_t *tsdn, arena_t *arena) { extent_t *extent; - malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); - extent = ql_last(&arena->extent_cache, ql_link); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + malloc_mutex_lock(tsdn, &arena->extent_freelist_mtx); + extent = extent_list_last(&arena->extent_freelist); if (extent == NULL) { - malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); - return base_alloc(tsdn, arena->base, sizeof(extent_t), - QUANTUM); + malloc_mutex_unlock(tsdn, &arena->extent_freelist_mtx); + return base_alloc(tsdn, arena->base, sizeof(extent_t), QUANTUM); } - ql_tail_remove(&arena->extent_cache, extent_t, ql_link); - malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); + extent_list_remove(&arena->extent_freelist, extent); + malloc_mutex_unlock(tsdn, &arena->extent_freelist_mtx); return extent; } void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { - malloc_mutex_lock(tsdn, &arena->extent_cache_mtx); - ql_elm_new(extent, ql_link); - ql_tail_insert(&arena->extent_cache, extent, ql_link); - malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + malloc_mutex_lock(tsdn, &arena->extent_freelist_mtx); + extent_list_append(&arena->extent_freelist, extent); + malloc_mutex_unlock(tsdn, &arena->extent_freelist_mtx); } extent_hooks_t * @@ -188,26 +190,174 @@ extent_size_quantize_t *extent_size_quantize_ceil = /* Generate pairing heap functions. */ ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp) -static void -extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], - extent_t *extent) { - size_t psz = extent_size_quantize_floor(extent_size_get(extent)); - pszind_t pind = psz2ind(psz); +bool +extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state) { + if (malloc_mutex_init(&extents->mtx, "extents", WITNESS_RANK_EXTENTS)) { + return true; + } + for (unsigned i = 0; i < NPSIZES+1; i++) { + extent_heap_new(&extents->heaps[i]); + } + extent_list_init(&extents->lru); + extents->npages = 0; + extents->state = state; + return false; +} - malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); +extent_state_t +extents_state_get(const extents_t *extents) { + return extents->state; +} - extent_heap_insert(&extent_heaps[pind], extent); +size_t +extents_npages_get(extents_t *extents) { + return atomic_read_zu(&extents->npages); } static void -extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], - extent_t *extent) { - size_t psz = extent_size_quantize_floor(extent_size_get(extent)); +extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { + malloc_mutex_assert_owner(tsdn, &extents->mtx); + assert(extent_state_get(extent) == extents->state); + + size_t size = extent_size_get(extent); + size_t psz = extent_size_quantize_floor(size); pszind_t pind = psz2ind(psz); + extent_heap_insert(&extents->heaps[pind], extent); + extent_list_append(&extents->lru, extent); + size_t npages = size >> LG_PAGE; + atomic_add_zu(&extents->npages, npages); +} - malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); +static void +extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { + malloc_mutex_assert_owner(tsdn, &extents->mtx); + assert(extent_state_get(extent) == extents->state); - extent_heap_remove(&extent_heaps[pind], extent); + size_t size = extent_size_get(extent); + size_t psz = extent_size_quantize_floor(size); + pszind_t pind = psz2ind(psz); + extent_heap_remove(&extents->heaps[pind], extent); + extent_list_remove(&extents->lru, extent); + size_t npages = size >> LG_PAGE; + assert(atomic_read_zu(&extents->npages) >= npages); + atomic_sub_zu(&extents->npages, size >> LG_PAGE); +} + +/* + * Do first-best-fit extent selection, i.e. select the oldest/lowest extent that + * best fits. + */ +static extent_t * +extents_first_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, + size_t size) { + malloc_mutex_assert_owner(tsdn, &extents->mtx); + + pszind_t pind = psz2ind(extent_size_quantize_ceil(size)); + for (pszind_t i = pind; i < NPSIZES+1; i++) { + extent_t *extent = extent_heap_first(&extents->heaps[i]); + if (extent != NULL) { + return extent; + } + } + + return NULL; +} + +extent_t * +extents_evict(tsdn_t *tsdn, extents_t *extents, size_t npages_min) { + malloc_mutex_lock(tsdn, &extents->mtx); + + /* Get the LRU extent, if any. */ + extent_t *extent = extent_list_first(&extents->lru); + if (extent == NULL) { + goto label_return; + } + /* Check the eviction limit. */ + size_t npages = extent_size_get(extent) >> LG_PAGE; + if (atomic_read_zu(&extents->npages) - npages < npages_min) { + extent = NULL; + goto label_return; + } + extents_remove_locked(tsdn, extents, extent); + + /* + * Either mark the extent active or deregister it to protect against + * concurrent operations. + */ + switch (extents_state_get(extents)) { + case extent_state_dirty: + extent_state_set(extent, extent_state_active); + break; + case extent_state_retained: + extent_deregister(tsdn, extent); + break; + default: + not_reached(); + } + +label_return: + malloc_mutex_unlock(tsdn, &extents->mtx); + return extent; +} + +static void +extents_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, + extents_t *extents, extent_t *extent) { + /* + * Leak extent after making sure its pages have already been purged, so + * that this is only a virtual memory leak. + */ + if (extents_state_get(extents) == extent_state_dirty) { + if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks, + extent, 0, extent_size_get(extent))) { + extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks, + extent, 0, extent_size_get(extent)); + } + } + extent_dalloc(tsdn, arena, extent); +} + +void +extents_prefork(tsdn_t *tsdn, extents_t *extents) { + malloc_mutex_prefork(tsdn, &extents->mtx); +} + +void +extents_postfork_parent(tsdn_t *tsdn, extents_t *extents) { + malloc_mutex_postfork_parent(tsdn, &extents->mtx); +} + +void +extents_postfork_child(tsdn_t *tsdn, extents_t *extents) { + malloc_mutex_postfork_child(tsdn, &extents->mtx); +} + +static void +extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, + extent_t *extent) { + assert(extent_arena_get(extent) == arena); + assert(extent_state_get(extent) == extent_state_active); + + extent_state_set(extent, extents_state_get(extents)); + extents_insert_locked(tsdn, extents, extent); +} + +static void +extent_deactivate(tsdn_t *tsdn, arena_t *arena, extents_t *extents, + extent_t *extent) { + malloc_mutex_lock(tsdn, &extents->mtx); + extent_deactivate_locked(tsdn, arena, extents, extent); + malloc_mutex_unlock(tsdn, &extents->mtx); +} + +static void +extent_activate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, + extent_t *extent) { + assert(extent_arena_get(extent) == arena); + assert(extent_state_get(extent) == extents_state_get(extents)); + + extents_remove_locked(tsdn, extents, extent); + extent_state_set(extent, extent_state_active); } static bool @@ -269,10 +419,12 @@ extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, } static void -extent_gprof_add(tsdn_t *tsdn, const extent_t *extent) { +extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) { cassert(config_prof); + /* prof_gdump() requirement. */ + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - if (opt_prof && extent_active_get(extent)) { + if (opt_prof && extent_state_get(extent) == extent_state_active) { size_t nadd = extent_size_get(extent) >> LG_PAGE; size_t cur = atomic_add_zu(&curpages, nadd); size_t high = atomic_read_zu(&highpages); @@ -290,10 +442,10 @@ extent_gprof_add(tsdn_t *tsdn, const extent_t *extent) { } static void -extent_gprof_sub(tsdn_t *tsdn, const extent_t *extent) { +extent_gdump_sub(tsdn_t *tsdn, const extent_t *extent) { cassert(config_prof); - if (opt_prof && extent_active_get(extent)) { + if (opt_prof && extent_state_get(extent) == extent_state_active) { size_t nsub = extent_size_get(extent) >> LG_PAGE; assert(atomic_read_zu(&curpages) >= nsub); atomic_sub_zu(&curpages, nsub); @@ -317,7 +469,7 @@ extent_register(tsdn_t *tsdn, const extent_t *extent) { extent_rtree_release(tsdn, elm_a, elm_b); if (config_prof) { - extent_gprof_add(tsdn, extent); + extent_gdump_add(tsdn, extent); } return false; @@ -359,68 +511,21 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent) { extent_rtree_release(tsdn, elm_a, elm_b); if (config_prof) { - extent_gprof_sub(tsdn, extent); + extent_gdump_sub(tsdn, extent); } } -/* - * Do first-best-fit extent selection, i.e. select the oldest/lowest extent that - * best fits. - */ -static extent_t * -extent_first_best_fit(tsdn_t *tsdn, arena_t *arena, - extent_heap_t extent_heaps[NPSIZES+1], size_t size) { - pszind_t pind, i; - - malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); - - pind = psz2ind(extent_size_quantize_ceil(size)); - for (i = pind; i < NPSIZES+1; i++) { - extent_t *extent = extent_heap_first(&extent_heaps[i]); - if (extent != NULL) { - return extent; - } - } - - return NULL; -} - -static void -extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - bool cache, extent_t *extent) { - /* - * Leak extent after making sure its pages have already been purged, so - * that this is only a virtual memory leak. - */ - if (cache) { - if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks, - extent, 0, extent_size_get(extent))) { - extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks, - extent, 0, extent_size_get(extent)); - } - } - extent_dalloc(tsdn, arena, extent); -} - static extent_t * -extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_heap_t extent_heaps[NPSIZES+1], bool locked, bool cache, - void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, - bool *commit, bool slab) { - extent_t *extent; - rtree_ctx_t rtree_ctx_fallback; - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - size_t size, alloc_size, leadsize, trailsize; - +extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + bool locked, void *new_addr, size_t usize, size_t pad, size_t alignment, + bool *zero, bool *commit) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, locked ? 1 : 0); if (locked) { - malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); + malloc_mutex_assert_owner(tsdn, &extents->mtx); } - assert(new_addr == NULL || !slab); - assert(pad == 0 || !slab); assert(alignment > 0); if (config_debug && new_addr != NULL) { - extent_t *prev; - /* * Non-NULL new_addr has two use cases: * @@ -435,21 +540,19 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, assert(PAGE_ADDR2BASE(new_addr) == new_addr); assert(pad == 0); assert(alignment <= PAGE); - prev = extent_lookup(tsdn, (void *)((uintptr_t)new_addr - PAGE), - false); - assert(prev == NULL || extent_past_get(prev) == new_addr); } - size = usize + pad; - alloc_size = size + PAGE_CEILING(alignment) - PAGE; + size_t size = usize + pad; + size_t alloc_size = size + PAGE_CEILING(alignment) - PAGE; /* Beware size_t wrap-around. */ if (alloc_size < usize) { return NULL; } if (!locked) { - malloc_mutex_lock(tsdn, &arena->extents_mtx); + malloc_mutex_lock(tsdn, &extents->mtx); } extent_hooks_assure_initialized(arena, r_extent_hooks); + extent_t *extent; if (new_addr != NULL) { rtree_elm_t *elm; @@ -462,8 +565,8 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, assert(extent_base_get(extent) == new_addr); if (extent_arena_get(extent) != arena || extent_size_get(extent) < size || - extent_active_get(extent) || - extent_retained_get(extent) == cache) { + extent_state_get(extent) != + extents_state_get(extents)) { extent = NULL; } } @@ -472,23 +575,21 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent = NULL; } } else { - extent = extent_first_best_fit(tsdn, arena, extent_heaps, + extent = extents_first_best_fit_locked(tsdn, arena, extents, alloc_size); } if (extent == NULL) { if (!locked) { - malloc_mutex_unlock(tsdn, &arena->extents_mtx); + malloc_mutex_unlock(tsdn, &extents->mtx); } return NULL; } - extent_heaps_remove(tsdn, extent_heaps, extent); - arena_extent_cache_maybe_remove(tsdn, arena, extent, cache); - leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent), - PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent); - assert(new_addr == NULL || leadsize == 0); - assert(extent_size_get(extent) >= leadsize + size); - trailsize = extent_size_get(extent) - leadsize - size; + extent_activate_locked(tsdn, arena, extents, extent); + if (!locked) { + malloc_mutex_unlock(tsdn, &extents->mtx); + } + if (extent_zeroed_get(extent)) { *zero = true; } @@ -496,6 +597,21 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, *commit = true; } + return extent; +} + +static extent_t * +extent_recycle_split(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + void *new_addr, size_t usize, size_t pad, size_t alignment, + extent_t *extent) { + size_t size = usize + pad; + size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent), + PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent); + assert(new_addr == NULL || leadsize == 0); + assert(extent_size_get(extent) >= leadsize + size); + size_t trailsize = extent_size_get(extent) - leadsize - size; + /* Split the lead. */ if (leadsize != 0) { extent_t *lead = extent; @@ -504,14 +620,11 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, trailsize); if (extent == NULL) { extent_deregister(tsdn, lead); - extent_leak(tsdn, arena, r_extent_hooks, cache, lead); - if (!locked) { - malloc_mutex_unlock(tsdn, &arena->extents_mtx); - } + extents_leak(tsdn, arena, r_extent_hooks, extents, + lead); return NULL; } - extent_heaps_insert(tsdn, extent_heaps, lead); - arena_extent_cache_maybe_insert(tsdn, arena, lead, cache); + extent_deactivate(tsdn, arena, extents, lead); } /* Split the trail. */ @@ -520,15 +633,11 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, r_extent_hooks, extent, size, usize, trailsize, trailsize); if (trail == NULL) { extent_deregister(tsdn, extent); - extent_leak(tsdn, arena, r_extent_hooks, cache, + extents_leak(tsdn, arena, r_extent_hooks, extents, extent); - if (!locked) { - malloc_mutex_unlock(tsdn, &arena->extents_mtx); - } return NULL; } - extent_heaps_insert(tsdn, extent_heaps, trail); - arena_extent_cache_maybe_insert(tsdn, arena, trail, cache); + extent_deactivate(tsdn, arena, extents, trail); } else if (leadsize == 0) { /* * Splitting causes usize to be set as a side effect, but no @@ -537,14 +646,38 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_usize_set(extent, usize); } + return extent; +} + +static extent_t * +extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, + extents_t *extents, void *new_addr, size_t usize, size_t pad, + size_t alignment, bool *zero, bool *commit, bool slab) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + assert(new_addr == NULL || !slab); + assert(pad == 0 || !slab); + + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + extent_t *extent = extent_recycle_extract(tsdn, arena, r_extent_hooks, + rtree_ctx, extents, false, new_addr, usize, pad, alignment, zero, + commit); + if (extent == NULL) { + return NULL; + } + + extent = extent_recycle_split(tsdn, arena, r_extent_hooks, rtree_ctx, + extents, new_addr, usize, pad, alignment, extent); + if (extent == NULL) { + return NULL; + } + if (*commit && !extent_committed_get(extent)) { if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent, 0, extent_size_get(extent))) { - if (!locked) { - malloc_mutex_unlock(tsdn, &arena->extents_mtx); - } - extent_record(tsdn, arena, r_extent_hooks, extent_heaps, - cache, extent); + extent_record(tsdn, arena, r_extent_hooks, extents, + extent); return NULL; } extent_zeroed_set(extent, true); @@ -553,16 +686,12 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (pad != 0) { extent_addr_randomize(tsdn, extent, alignment); } - extent_active_set(extent, true); + assert(extent_state_get(extent) == extent_state_active); if (slab) { extent_slab_set(extent, slab); extent_interior_register(tsdn, rtree_ctx, extent); } - if (!locked) { - malloc_mutex_unlock(tsdn, &arena->extents_mtx); - } - if (*zero) { if (!extent_zeroed_get(extent)) { memset(extent_addr_get(extent), 0, @@ -616,37 +745,17 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, return NULL; } -static extent_t * -extent_alloc_cache_impl(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, bool locked, void *new_addr, size_t usize, - size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) { - extent_t *extent; - - assert(usize + pad != 0); - assert(alignment != 0); - - extent = extent_recycle(tsdn, arena, r_extent_hooks, - arena->extents_cached, locked, true, new_addr, usize, pad, - alignment, zero, commit, slab); - return extent; -} - -extent_t * -extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, - size_t alignment, bool *zero, bool *commit, bool slab) { - malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); - - return extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, true, - new_addr, usize, pad, alignment, zero, commit, slab); -} - extent_t * extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) { - return extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, false, - new_addr, usize, pad, alignment, zero, commit, slab); + assert(usize + pad != 0); + assert(alignment != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + return extent_recycle(tsdn, arena, r_extent_hooks, + &arena->extents_cached, new_addr, usize, pad, alignment, zero, + commit, slab); } static void * @@ -679,16 +788,6 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, alignment, zero, commit); } -static void -extent_retain(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_t *extent) { - if (config_stats) { - arena->stats.retained += extent_size_get(extent); - } - extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained, - false, extent); -} - /* * If virtual memory is retained, create increasingly larger extents from which * to split requested extents in order to limit the total number of disjoint @@ -728,16 +827,17 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE, &zeroed, &committed, arena->dss_prec); extent_init(extent, arena, ptr, alloc_size, alloc_size, - arena_extent_sn_next(arena), false, zeroed, committed, false); + arena_extent_sn_next(arena), extent_state_retained, zeroed, + committed, false); if (ptr == NULL || extent_register(tsdn, extent)) { extent_dalloc(tsdn, arena, extent); return NULL; } /* - * Set the extent as active *after registration so that no gprof-related + * Set the extent as active *after registration so that no gdump-related * accounting occurs during registration. */ - extent_active_set(extent, true); + extent_state_set(extent, extent_state_active); leadsize = ALIGNMENT_CEILING((uintptr_t)ptr, PAGE_CEILING(alignment)) - (uintptr_t)ptr; @@ -758,10 +858,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, leadsize, leadsize, size + trailsize, usize + trailsize); if (extent == NULL) { extent_deregister(tsdn, lead); - extent_leak(tsdn, arena, r_extent_hooks, false, lead); + extents_leak(tsdn, arena, r_extent_hooks, false, lead); return NULL; } - extent_retain(tsdn, arena, r_extent_hooks, lead); + extent_record(tsdn, arena, r_extent_hooks, + &arena->extents_retained, lead); } /* Split the trail. */ @@ -770,10 +871,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, r_extent_hooks, extent, size, usize, trailsize, trailsize); if (trail == NULL) { extent_deregister(tsdn, extent); - extent_leak(tsdn, arena, r_extent_hooks, false, extent); + extents_leak(tsdn, arena, r_extent_hooks, + &arena->extents_retained, extent); return NULL; } - extent_retain(tsdn, arena, r_extent_hooks, trail); + extent_record(tsdn, arena, r_extent_hooks, + &arena->extents_retained, trail); } else if (leadsize == 0) { /* * Splitting causes usize to be set as a side effect, but no @@ -785,15 +888,16 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, if (*commit && !extent_committed_get(extent)) { if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent, 0, extent_size_get(extent))) { - extent_retain(tsdn, arena, r_extent_hooks, extent); + extent_record(tsdn, arena, r_extent_hooks, + &arena->extents_retained, extent); return NULL; } extent_zeroed_set(extent, true); } if (config_prof) { - /* Adjust gprof stats now that extent is final size. */ - extent_gprof_add(tsdn, extent); + /* Adjust gdump stats now that extent is final size. */ + extent_gdump_add(tsdn, extent); } if (pad != 0) { extent_addr_randomize(tsdn, extent, alignment); @@ -837,15 +941,11 @@ extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, assert(alignment != 0); extent = extent_recycle(tsdn, arena, r_extent_hooks, - arena->extents_retained, false, false, new_addr, usize, pad, - alignment, zero, commit, slab); + &arena->extents_retained, new_addr, usize, pad, alignment, zero, + commit, slab); if (extent != NULL) { - if (config_stats) { - size_t size = usize + pad; - arena->stats.retained -= size; - } if (config_prof) { - extent_gprof_add(tsdn, extent); + extent_gdump_add(tsdn, extent); } } if (!config_munmap && extent == NULL) { @@ -882,12 +982,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, return NULL; } extent_init(extent, arena, addr, size, usize, - arena_extent_sn_next(arena), true, zero, commit, slab); + arena_extent_sn_next(arena), extent_state_active, zero, commit, + slab); if (pad != 0) { extent_addr_randomize(tsdn, extent, alignment); } if (extent_register(tsdn, extent)) { - extent_leak(tsdn, arena, r_extent_hooks, false, extent); + extents_leak(tsdn, arena, r_extent_hooks, + &arena->extents_retained, extent); return NULL; } @@ -898,12 +1000,12 @@ extent_t * extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) { - extent_t *extent; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); - extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, new_addr, - usize, pad, alignment, zero, commit, slab); + extent_t *extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, + new_addr, usize, pad, alignment, zero, commit, slab); if (extent == NULL) { extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks, new_addr, usize, pad, alignment, zero, commit, slab); @@ -917,96 +1019,103 @@ extent_can_coalesce(const extent_t *a, const extent_t *b) { if (extent_arena_get(a) != extent_arena_get(b)) { return false; } - if (extent_active_get(a) != extent_active_get(b)) { + if (extent_state_get(a) != extent_state_get(b)) { return false; } if (extent_committed_get(a) != extent_committed_get(b)) { return false; } - if (extent_retained_get(a) != extent_retained_get(b)) { - return false; - } return true; } -static void +static bool extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, - extent_heap_t extent_heaps[NPSIZES+1], bool cache) { + extents_t *extents) { if (!extent_can_coalesce(a, b)) { - return; + return true; + } + assert(extent_arena_get(a) == arena); + assert(extent_arena_get(b) == arena); + + extent_activate_locked(tsdn, arena, extents, a); + extent_activate_locked(tsdn, arena, extents, b); + + malloc_mutex_unlock(tsdn, &extents->mtx); + bool err = extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b); + malloc_mutex_lock(tsdn, &extents->mtx); + extent_deactivate_locked(tsdn, arena, extents, a); + if (err) { + extent_deactivate_locked(tsdn, arena, extents, b); + return true; } - extent_heaps_remove(tsdn, extent_heaps, a); - extent_heaps_remove(tsdn, extent_heaps, b); - - arena_extent_cache_maybe_remove(tsdn, extent_arena_get(a), a, cache); - arena_extent_cache_maybe_remove(tsdn, extent_arena_get(b), b, cache); - - if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) { - extent_heaps_insert(tsdn, extent_heaps, a); - extent_heaps_insert(tsdn, extent_heaps, b); - arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a, - cache); - arena_extent_cache_maybe_insert(tsdn, extent_arena_get(b), b, - cache); - return; - } - - extent_heaps_insert(tsdn, extent_heaps, a); - arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a, cache); + return false; } static void extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_heap_t extent_heaps[NPSIZES+1], bool cache, extent_t *extent) { + extents_t *extents, extent_t *extent) { extent_t *prev, *next; rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - assert(!cache || !extent_zeroed_get(extent)); + assert(extents_state_get(extents) != extent_state_dirty || + !extent_zeroed_get(extent)); - malloc_mutex_lock(tsdn, &arena->extents_mtx); + malloc_mutex_lock(tsdn, &extents->mtx); extent_hooks_assure_initialized(arena, r_extent_hooks); extent_usize_set(extent, 0); - extent_active_set(extent, false); - extent_zeroed_set(extent, !cache && extent_zeroed_get(extent)); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); } assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent); - extent_heaps_insert(tsdn, extent_heaps, extent); - arena_extent_cache_maybe_insert(tsdn, arena, extent, cache); + extent_deactivate_locked(tsdn, arena, extents, extent); - /* Try to coalesce forward. */ - next = rtree_read(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)extent_past_get(extent), false); - if (next != NULL) { - extent_try_coalesce(tsdn, arena, r_extent_hooks, extent, next, - extent_heaps, cache); - } + /* + * Continue attempting to coalesce until failure, to protect against + * races with other threads that are thwarted by this one. + */ + bool coalesced; + do { + coalesced = false; - /* Try to coalesce backward. */ - prev = rtree_read(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)extent_before_get(extent), false); - if (prev != NULL) { - extent_try_coalesce(tsdn, arena, r_extent_hooks, prev, extent, - extent_heaps, cache); - } + /* Try to coalesce forward. */ + next = rtree_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)extent_past_get(extent), false); + if (next != NULL) { + coalesced = !extent_try_coalesce(tsdn, arena, + r_extent_hooks, extent, next, extents); + } - malloc_mutex_unlock(tsdn, &arena->extents_mtx); + /* Try to coalesce backward. */ + prev = rtree_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)extent_before_get(extent), false); + if (prev != NULL) { + coalesced = !extent_try_coalesce(tsdn, arena, + r_extent_hooks, prev, extent, extents); + if (coalesced) { + extent = prev; + } + } + } while (coalesced); + + malloc_mutex_unlock(tsdn, &extents->mtx); } void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + if (extent_register(tsdn, extent)) { - extent_leak(tsdn, arena, &extent_hooks, false, extent); + extents_leak(tsdn, arena, &extent_hooks, + &arena->extents_retained, extent); return; } extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); @@ -1017,11 +1126,12 @@ extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); extent_addr_set(extent, extent_base_get(extent)); extent_zeroed_set(extent, false); - extent_record(tsdn, arena, r_extent_hooks, arena->extents_cached, true, + extent_record(tsdn, arena, r_extent_hooks, &arena->extents_cached, extent); } @@ -1048,15 +1158,12 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); extent_addr_set(extent, extent_base_get(extent)); extent_hooks_assure_initialized(arena, r_extent_hooks); - /* - * Try to deallocate. Deregister first to avoid a race with other - * allocating threads, and reregister if deallocation fails. - */ - extent_deregister(tsdn, extent); + /* Try to deallocate. */ if (*r_extent_hooks == &extent_hooks_default) { /* Call directly to propagate tsdn. */ err = extent_dalloc_default_impl(extent_base_get(extent), @@ -1078,14 +1185,20 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { - bool zeroed; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + /* + * Deregister first to avoid a race with other allocating threads, and + * reregister if deallocation fails. + */ + extent_deregister(tsdn, extent); if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) { return; } extent_reregister(tsdn, extent); /* Try to decommit; purge if that fails. */ + bool zeroed; if (!extent_committed_get(extent)) { zeroed = true; } else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, @@ -1106,15 +1219,12 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, } extent_zeroed_set(extent, zeroed); - if (config_stats) { - arena->stats.retained += extent_size_get(extent); - } if (config_prof) { - extent_gprof_sub(tsdn, extent); + extent_gdump_sub(tsdn, extent); } - extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained, - false, extent); + extent_record(tsdn, arena, r_extent_hooks, &arena->extents_retained, + extent); } static bool @@ -1130,10 +1240,10 @@ bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - bool err; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); - err = ((*r_extent_hooks)->commit == NULL || + bool err = ((*r_extent_hooks)->commit == NULL || (*r_extent_hooks)->commit(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena))); extent_committed_set(extent, extent_committed_get(extent) || !err); @@ -1153,11 +1263,11 @@ bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - bool err; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); - err = ((*r_extent_hooks)->decommit == NULL || + bool err = ((*r_extent_hooks)->decommit == NULL || (*r_extent_hooks)->decommit(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena))); @@ -1184,6 +1294,8 @@ bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + extent_hooks_assure_initialized(arena, r_extent_hooks); return ((*r_extent_hooks)->purge_lazy == NULL || (*r_extent_hooks)->purge_lazy(*r_extent_hooks, @@ -1210,6 +1322,8 @@ bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + extent_hooks_assure_initialized(arena, r_extent_hooks); return ((*r_extent_hooks)->purge_forced == NULL || (*r_extent_hooks)->purge_forced(*r_extent_hooks, @@ -1231,13 +1345,14 @@ extent_t * extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, size_t usize_a, size_t size_b, size_t usize_b) { + assert(extent_size_get(extent) == size_a + size_b); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + extent_t *trail; rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_elm_t *lead_elm_a, *lead_elm_b, *trail_elm_a, *trail_elm_b; - assert(extent_size_get(extent) == size_a + size_b); - extent_hooks_assure_initialized(arena, r_extent_hooks); if ((*r_extent_hooks)->split == NULL) { @@ -1253,7 +1368,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t lead; extent_init(&lead, arena, extent_addr_get(extent), size_a, - usize_a, extent_sn_get(extent), extent_active_get(extent), + usize_a, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), extent_committed_get(extent), extent_slab_get(extent)); @@ -1265,7 +1380,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + size_a), size_b, usize_b, extent_sn_get(extent), - extent_active_get(extent), extent_zeroed_get(extent), + extent_state_get(extent), extent_zeroed_get(extent), extent_committed_get(extent), extent_slab_get(extent)); if (extent_rtree_acquire(tsdn, rtree_ctx, trail, false, true, &trail_elm_a, &trail_elm_b)) { @@ -1323,10 +1438,7 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) { - bool err; - rtree_ctx_t rtree_ctx_fallback; - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); @@ -1334,6 +1446,7 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, return true; } + bool err; if (*r_extent_hooks == &extent_hooks_default) { /* Call directly to propagate tsdn. */ err = extent_merge_default_impl(extent_base_get(a), @@ -1354,6 +1467,9 @@ extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, * owned, so the following code uses decomposed helper functions rather * than extent_{,de}register() to do things in the right order. */ + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + rtree_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b; extent_rtree_acquire(tsdn, rtree_ctx, a, true, false, &a_elm_a, &a_elm_b); extent_rtree_acquire(tsdn, rtree_ctx, b, true, false, &b_elm_a, diff --git a/src/extent_dss.c b/src/extent_dss.c index ed4140e7..a3cfab26 100644 --- a/src/extent_dss.c +++ b/src/extent_dss.c @@ -143,7 +143,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, if (gap_size != 0) { extent_init(gap, arena, gap_addr, gap_size, gap_size, arena_extent_sn_next(arena), - false, false, true, false); + extent_state_active, false, true, false); } dss_next = (void *)((uintptr_t)ret + size); if ((uintptr_t)ret < (uintptr_t)max_cur || @@ -180,7 +180,8 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, extent_t extent; extent_init(&extent, arena, ret, size, - size, 0, true, false, true, false); + size, 0, extent_state_active, false, + true, false); if (extent_purge_forced_wrapper(tsdn, arena, &extent_hooks, &extent, 0, size)) { diff --git a/src/large.c b/src/large.c index 6458d81a..bfe2f714 100644 --- a/src/large.c +++ b/src/large.c @@ -40,8 +40,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, /* Insert extent into large. */ malloc_mutex_lock(tsdn, &arena->large_mtx); - ql_elm_new(extent, ql_link); - ql_tail_insert(&arena->large, extent, ql_link); + extent_list_append(&arena->large, extent); malloc_mutex_unlock(tsdn, &arena->large_mtx); if (config_prof && arena_prof_accum(tsdn, arena, usize)) { prof_idump(tsdn); @@ -138,19 +137,19 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool zero) { arena_t *arena = extent_arena_get(extent); size_t oldusize = extent_usize_get(extent); - bool is_zeroed_trail = false; extent_hooks_t *extent_hooks = extent_hooks_get(arena); size_t trailsize = usize - extent_usize_get(extent); - extent_t *trail; if (extent_hooks->merge == NULL) { return true; } - if ((trail = arena_extent_cache_alloc(tsdn, arena, &extent_hooks, - extent_past_get(extent), trailsize, CACHELINE, &is_zeroed_trail)) == - NULL) { - bool commit = true; + bool is_zeroed_trail = false; + bool commit = true; + extent_t *trail; + if ((trail = extent_alloc_cache(tsdn, arena, &extent_hooks, + extent_past_get(extent), trailsize, 0, CACHELINE, &is_zeroed_trail, + &commit, false)) == NULL) { if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, extent_past_get(extent), trailsize, 0, CACHELINE, &is_zeroed_trail, &commit, false)) == NULL) { @@ -291,32 +290,39 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, * independent of these considerations. */ static void -large_dalloc_impl(tsdn_t *tsdn, extent_t *extent, bool junked_locked) { - arena_t *arena; - - arena = extent_arena_get(extent); +large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, + bool junked_locked) { malloc_mutex_lock(tsdn, &arena->large_mtx); - ql_remove(&arena->large, extent, ql_link); + extent_list_remove(&arena->large, extent); malloc_mutex_unlock(tsdn, &arena->large_mtx); if (!junked_locked) { large_dalloc_maybe_junk(extent_addr_get(extent), extent_usize_get(extent)); } - arena_extent_dalloc_large(tsdn, arena, extent, junked_locked); + arena_extent_dalloc_large_prep(tsdn, arena, extent, junked_locked); +} - if (!junked_locked) { - arena_decay_tick(tsdn, arena); - } +static void +large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { + arena_extent_dalloc_large_finish(tsdn, arena, extent); } void -large_dalloc_junked_locked(tsdn_t *tsdn, extent_t *extent) { - large_dalloc_impl(tsdn, extent, true); +large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent) { + large_dalloc_prep_impl(tsdn, extent_arena_get(extent), extent, true); +} + +void +large_dalloc_finish(tsdn_t *tsdn, extent_t *extent) { + large_dalloc_finish_impl(tsdn, extent_arena_get(extent), extent); } void large_dalloc(tsdn_t *tsdn, extent_t *extent) { - large_dalloc_impl(tsdn, extent, false); + arena_t *arena = extent_arena_get(extent); + large_dalloc_prep_impl(tsdn, arena, extent, false); + large_dalloc_finish_impl(tsdn, arena, extent); + arena_decay_tick(tsdn, arena); } size_t diff --git a/src/tcache.c b/src/tcache.c index 76277f06..94c45707 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -170,17 +170,15 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, unsigned rem, tcache_t *tcache) { - arena_t *arena; - void *ptr; - unsigned i, nflush, ndeferred; bool merged_stats = false; assert(binind < nhbins); assert(rem <= tbin->ncached); - arena = arena_choose(tsd, NULL); + arena_t *arena = arena_choose(tsd, NULL); assert(arena != NULL); - for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { + unsigned nflush = tbin->ncached - rem; + while (nflush > 0) { /* Lock the arena associated with the first object. */ extent_t *extent = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1)); arena_t *locked_arena = extent_arena_get(extent); @@ -189,7 +187,17 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, if (config_prof) { idump = false; } + malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->lock); + for (unsigned i = 0; i < nflush; i++) { + void *ptr = *(tbin->avail - 1 - i); + assert(ptr != NULL); + extent = iealloc(tsd_tsdn(tsd), ptr); + if (extent_arena_get(extent) == locked_arena) { + large_dalloc_prep_junked_locked(tsd_tsdn(tsd), + extent); + } + } if ((config_prof || config_stats) && locked_arena == arena) { if (config_prof) { idump = arena_prof_accum_locked(arena, @@ -205,14 +213,15 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, tbin->tstats.nrequests = 0; } } - ndeferred = 0; - for (i = 0; i < nflush; i++) { - ptr = *(tbin->avail - 1 - i); + malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->lock); + + unsigned ndeferred = 0; + for (unsigned i = 0; i < nflush; i++) { + void *ptr = *(tbin->avail - 1 - i); assert(ptr != NULL); extent = iealloc(tsd_tsdn(tsd), ptr); if (extent_arena_get(extent) == locked_arena) { - large_dalloc_junked_locked(tsd_tsdn(tsd), - extent); + large_dalloc_finish(tsd_tsdn(tsd), extent); } else { /* * This object was allocated via a different @@ -224,12 +233,12 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, ndeferred++; } } - malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->lock); if (config_prof && idump) { prof_idump(tsd_tsdn(tsd)); } arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush - ndeferred); + nflush = ndeferred; } if (config_stats && !merged_stats) { /* diff --git a/test/unit/arena_reset.c b/test/unit/arena_reset.c index d2a9bb4f..24c7f526 100644 --- a/test/unit/arena_reset.c +++ b/test/unit/arena_reset.c @@ -63,7 +63,7 @@ vsalloc(tsdn_t *tsdn, const void *ptr) { if (extent == NULL) { return 0; } - if (!extent_active_get(extent)) { + if (extent_state_get(extent) != extent_state_active) { return 0; } diff --git a/test/unit/slab.c b/test/unit/slab.c index d3b45e80..1f2a260c 100644 --- a/test/unit/slab.c +++ b/test/unit/slab.c @@ -8,8 +8,8 @@ TEST_BEGIN(test_arena_slab_regind) { extent_t slab; const arena_bin_info_t *bin_info = &arena_bin_info[binind]; extent_init(&slab, NULL, mallocx(bin_info->slab_size, - MALLOCX_LG_ALIGN(LG_PAGE)), bin_info->slab_size, 0, 0, true, - false, true, true); + MALLOCX_LG_ALIGN(LG_PAGE)), bin_info->slab_size, 0, 0, + extent_state_active, false, true, true); assert_ptr_not_null(extent_addr_get(&slab), "Unexpected malloc() failure"); for (regind = 0; regind < bin_info->nregs; regind++) {