diff --git a/Makefile.in b/Makefile.in index 04ce288a..4fb852da 100644 --- a/Makefile.in +++ b/Makefile.in @@ -442,8 +442,8 @@ ifeq ($(enable_prof), 1) $(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) endif check_integration_decay: tests_integration check_integration_dir - $(MALLOC_CONF)="decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) - $(MALLOC_CONF)="decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) + $(MALLOC_CONF)="dirty_decay_time:-1,muzzy_decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) + $(MALLOC_CONF)="dirty_decay_time:0,muzzy_decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) check_integration: tests_integration check_integration_dir $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) stress: tests_stress stress_dir diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 937879a8..7faa474d 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -944,24 +944,54 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", percpu. - + - opt.decay_time + opt.dirty_decay_time (ssize_t) r- Approximate time in seconds from the creation of a set of unused dirty pages until an equivalent set of unused dirty pages is - purged and/or reused. The pages are incrementally purged according to a - sigmoidal decay curve that starts and ends with zero purge rate. A - decay time of 0 causes all unused dirty pages to be purged immediately - upon creation. A decay time of -1 disables purging. The default decay - time is 10 seconds. See arenas.decay_time + purged (i.e. converted to muzzy via e.g. + madvise(...MADV_FREE) + if supported by the operating system, or converted to clean otherwise) + and/or reused. Dirty pages are defined as previously having been + potentially written to by the application, and therefore consuming + physical memory, yet having no current use. The pages are incrementally + purged according to a sigmoidal decay curve that starts and ends with + zero purge rate. A decay time of 0 causes all unused dirty pages to be + purged immediately upon creation. A decay time of -1 disables purging. + The default decay time is 10 seconds. See arenas.dirty_decay_time and arena.<i>.decay_time - for related dynamic control options. - + linkend="arena.i.muzzy_decay_time">arena.<i>.muzzy_decay_time + for related dynamic control options. See opt.muzzy_decay_time + for a description of muzzy pages. + + + + + opt.muzzy_decay_time + (ssize_t) + r- + + Approximate time in seconds from the creation of a set + of unused muzzy pages until an equivalent set of unused muzzy pages is + purged (i.e. converted to clean) and/or reused. Muzzy pages are defined + as previously having been unused dirty pages that were subsequently + purged in a manner that left them subject to the reclamation whims of + the operating system (e.g. + madvise(...MADV_FREE)), + and therefore in an indeterminate state. The pages are incrementally + purged according to a sigmoidal decay curve that starts and ends with + zero purge rate. A decay time of 0 causes all unused muzzy pages to be + purged immediately upon creation. A decay time of -1 disables purging. + The default decay time is 10 seconds. See arenas.muzzy_decay_time + and arena.<i>.muzzy_decay_time + for related dynamic control options. @@ -1460,6 +1490,22 @@ malloc_conf = "xmalloc:true";]]> initialized (always true). + + + arena.<i>.decay + (void) + -- + + Trigger decay-based purging of unused dirty/muzzy pages + for arena <i>, or for all arenas if <i> equals + MALLCTL_ARENAS_ALL. The proportion of unused + dirty/muzzy pages to be purged depends on the current time; see opt.dirty_decay_time + and opt.muzy_decay_time + for details. + + arena.<i>.purge @@ -1471,20 +1517,6 @@ malloc_conf = "xmalloc:true";]]> - - - arena.<i>.decay - (void) - -- - - Trigger decay-based purging of unused dirty pages for - arena <i>, or for all arenas if <i> equals - MALLCTL_ARENAS_ALL. The proportion of unused dirty - pages to be purged depends on the current time; see opt.decay_time for - details. - - arena.<i>.reset @@ -1532,9 +1564,9 @@ malloc_conf = "xmalloc:true";]]> settings. - + - arena.<i>.decay_time + arena.<i>.dirty_decay_time (ssize_t) rw @@ -1544,8 +1576,24 @@ malloc_conf = "xmalloc:true";]]> set, all currently unused dirty pages are considered to have fully decayed, which causes immediate purging of all unused dirty pages unless the decay time is set to -1 (i.e. purging disabled). See opt.decay_time for - additional information. + linkend="opt.dirty_decay_time">opt.dirty_decay_time + for additional information. + + + + + arena.<i>.muzzy_decay_time + (ssize_t) + rw + + Current per-arena approximate time in seconds from the + creation of a set of unused muzzy pages until an equivalent set of + unused muzzy pages is purged and/or reused. Each time this interface is + set, all currently unused muzzy pages are considered to have fully + decayed, which causes immediate purging of all unused muzzy pages unless + the decay time is set to -1 (i.e. purging disabled). See opt.muzzy_decay_time + for additional information. @@ -1584,7 +1632,7 @@ struct extent_hooks_s { mapped committed memory, in the simplest case followed by deallocation. However, there are performance and platform reasons to retain extents for later reuse. Cleanup attempts cascade from deallocation to decommit - to lazy purging to forced purging, which gives the extent management + to forced purging to lazy purging, which gives the extent management functions opportunities to reject the most permanent cleanup operations in favor of less permanent (and often less costly) operations. All operations except allocation can be universally opted out of by setting @@ -1707,12 +1755,14 @@ struct extent_hooks_s { addr and size at offset bytes, extending for length on behalf of arena - arena_ind. A lazy extent purge function can - delay purging indefinitely and leave the pages within the purged virtual - memory range in an indeterminite state, whereas a forced extent purge - function immediately purges, and the pages within the virtual memory - range will be zero-filled the next time they are accessed. If the - function returns true, this indicates failure to purge. + arena_ind. A lazy extent purge function (e.g. + implemented via + madvise(...MADV_FREE)) + can delay purging indefinitely and leave the pages within the purged + virtual memory range in an indeterminite state, whereas a forced extent + purge function immediately purges, and the pages within the virtual + memory range will be zero-filled the next time they are accessed. If + the function returns true, this indicates failure to purge. typedef bool (extent_split_t) @@ -1769,19 +1819,34 @@ struct extent_hooks_s { Current limit on number of arenas. - + - arenas.decay_time + arenas.dirty_decay_time (ssize_t) rw Current default per-arena approximate time in seconds from the creation of a set of unused dirty pages until an equivalent set of unused dirty pages is purged and/or reused, used to initialize arena.<i>.decay_time + linkend="arena.i.dirty_decay_time">arena.<i>.dirty_decay_time during arena creation. See opt.decay_time for - additional information. + linkend="opt.dirty_decay_time">opt.dirty_decay_time + for additional information. + + + + + arenas.muzzy_decay_time + (ssize_t) + rw + + Current default per-arena approximate time in seconds + from the creation of a set of unused muzzy pages until an equivalent set + of unused muzzy pages is purged and/or reused, used to initialize arena.<i>.muzzy_decay_time + during arena creation. See opt.muzzy_decay_time + for additional information. @@ -2014,7 +2079,9 @@ struct extent_hooks_s { equal to stats.allocated. This does not include - stats.arenas.<i>.pdirty, nor pages + stats.arenas.<i>.pdirty, + + stats.arenas.<i>.pmuzzy, nor pages entirely devoted to allocator metadata. @@ -2099,16 +2166,29 @@ struct extent_hooks_s { - + - stats.arenas.<i>.decay_time + stats.arenas.<i>.dirty_decay_time (ssize_t) r- Approximate time in seconds from the creation of a set of unused dirty pages until an equivalent set of unused dirty pages is purged and/or reused. See opt.decay_time + linkend="opt.dirty_decay_time">opt.dirty_decay_time + for details. + + + + + stats.arenas.<i>.muzzy_decay_time + (ssize_t) + r- + + Approximate time in seconds from the creation of a set + of unused muzzy pages until an equivalent set of unused muzzy pages is + purged and/or reused. See opt.muzzy_decay_time for details. @@ -2138,10 +2218,22 @@ struct extent_hooks_s { r- Number of pages within unused extents that are - potentially dirty, and for which - madvise(... - MADV_DONTNEED) or - similar has not been called. + potentially dirty, and for which madvise() or + similar has not been called. See opt.dirty_decay_time + for a description of dirty pages. + + + + + stats.arenas.<i>.pmuzzy + (size_t) + r- + + Number of pages within unused extents that are muzzy. + See opt.muzzy_decay_time + for a description of muzzy pages. @@ -2207,9 +2299,9 @@ struct extent_hooks_s { size. - + - stats.arenas.<i>.npurge + stats.arenas.<i>.dirty_npurge (uint64_t) r- [] @@ -2218,26 +2310,57 @@ struct extent_hooks_s { - + - stats.arenas.<i>.nmadvise + stats.arenas.<i>.dirty_nmadvise (uint64_t) r- [] - Number of madvise(... - MADV_DONTNEED) or - similar calls made to purge dirty pages. + Number of madvise() or similar + calls made to purge dirty pages. - + - stats.arenas.<i>.purged + stats.arenas.<i>.dirty_purged (uint64_t) r- [] - Number of pages purged. + Number of dirty pages purged. + + + + + stats.arenas.<i>.muzzy_npurge + (uint64_t) + r- + [] + + Number of muzzy page purge sweeps performed. + + + + + + stats.arenas.<i>.muzzy_nmadvise + (uint64_t) + r- + [] + + Number of madvise() or similar + calls made to purge muzzy pages. + + + + + stats.arenas.<i>.muzzy_purged + (uint64_t) + r- + [] + + Number of muzzy pages purged. diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index 349bae99..9603d74f 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -9,7 +9,8 @@ static const size_t large_pad = #endif ; -extern ssize_t opt_decay_time; +extern ssize_t opt_dirty_decay_time; +extern ssize_t opt_muzzy_decay_time; extern const arena_bin_info_t arena_bin_info[NBINS]; @@ -22,13 +23,13 @@ void arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, void arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size); void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, - unsigned *nthreads, const char **dss, ssize_t *decay_time, size_t *nactive, - size_t *ndirty); + unsigned *nthreads, const char **dss, ssize_t *dirty_decay_time, + ssize_t *muzzy_decay_time, size_t *nactive, size_t *ndirty, size_t *nmuzzy); void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, - const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats); -void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, + const char **dss, ssize_t *dirty_decay_time, ssize_t *muzzy_decay_time, + size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, + malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats); +void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); #ifdef JEMALLOC_JET size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr); @@ -41,9 +42,13 @@ void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t oldsize); void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t oldsize); -ssize_t arena_decay_time_get(arena_t *arena); -bool arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time); -void arena_purge(tsdn_t *tsdn, arena_t *arena, bool all); +ssize_t arena_dirty_decay_time_get(arena_t *arena); +bool arena_dirty_decay_time_set(tsdn_t *tsdn, arena_t *arena, + ssize_t decay_time); +ssize_t arena_muzzy_decay_time_get(arena_t *arena); +bool arena_muzzy_decay_time_set(tsdn_t *tsdn, arena_t *arena, + ssize_t decay_time); +void arena_decay(tsdn_t *tsdn, arena_t *arena, bool all); void arena_reset(tsd_t *tsd, arena_t *arena); void arena_destroy(tsd_t *tsd, arena_t *arena); void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, @@ -74,8 +79,10 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); -ssize_t arena_decay_time_default_get(void); -bool arena_decay_time_default_set(ssize_t decay_time); +ssize_t arena_dirty_decay_time_default_get(void); +bool arena_dirty_decay_time_default_set(ssize_t decay_time); +ssize_t arena_muzzy_decay_time_default_get(void); +bool arena_muzzy_decay_time_default_set(ssize_t decay_time); unsigned arena_nthreads_get(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 275866a4..b718451b 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -75,13 +75,14 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { return; } if (unlikely(ticker_ticks(decay_ticker, nticks))) { - arena_purge(tsdn, arena, false); + arena_decay(tsdn, arena, false); } } JEMALLOC_ALWAYS_INLINE void arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { - malloc_mutex_assert_not_owner(tsdn, &arena->decay.mtx); + malloc_mutex_assert_not_owner(tsdn, &arena->decay_dirty.mtx); + malloc_mutex_assert_not_owner(tsdn, &arena->decay_muzzy.mtx); arena_decay_ticks(tsdn, arena, 1); } diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h index 612b4e7d..a5191d16 100644 --- a/include/jemalloc/internal/arena_structs_b.h +++ b/include/jemalloc/internal/arena_structs_b.h @@ -48,10 +48,8 @@ struct arena_decay_s { * Approximate time in seconds from the creation of a set of unused * dirty pages until an equivalent set of unused dirty pages is purged * and/or reused. - * - * Synchronization: atomic. */ - ssize_t time; + atomic_zd_t time; /* time / SMOOTHSTEP_NSTEPS. */ nstime_t interval; /* @@ -73,10 +71,10 @@ struct arena_decay_s { */ nstime_t deadline; /* - * Number of dirty pages at beginning of current epoch. During epoch - * advancement we use the delta between arena->decay.ndirty and - * extents_npages_get(&arena->extents_cached) to determine how many - * dirty pages, if any, were generated. + * Number of unpurged pages at beginning of current epoch. During epoch + * advancement we use the delta between arena->decay_*.nunpurged and + * extents_npages_get(&arena->extents_*) to determine how many dirty + * pages, if any, were generated. */ size_t nunpurged; /* @@ -86,6 +84,14 @@ struct arena_decay_s { * relative to epoch. */ size_t backlog[SMOOTHSTEP_NSTEPS]; + + /* + * Pointer to associated stats. These stats are embedded directly in + * the arena's stats due to how stats structures are shared between the + * arena and ctl code. + * + * Synchronization: Same as associated arena's stats field. */ + decay_stats_t *stats; }; struct arena_bin_s { @@ -194,15 +200,18 @@ struct arena_s { * * Synchronization: internal. */ - extents_t extents_cached; + extents_t extents_dirty; + extents_t extents_muzzy; extents_t extents_retained; /* - * Decay-based purging state. + * Decay-based purging state, responsible for scheduling extent state + * transitions. * * Synchronization: internal. */ - arena_decay_t decay; + arena_decay_t decay_dirty; /* dirty --> muzzy */ + arena_decay_t decay_muzzy; /* muzzy --> retained */ /* * Next extent size class in a growing series to use when satisfying a diff --git a/include/jemalloc/internal/arena_types.h b/include/jemalloc/internal/arena_types.h index 067c9ee9..ba53c408 100644 --- a/include/jemalloc/internal/arena_types.h +++ b/include/jemalloc/internal/arena_types.h @@ -7,8 +7,9 @@ #define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN) #define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) -/* Default decay time in seconds. */ -#define DECAY_TIME_DEFAULT 10 +/* Default decay times in seconds. */ +#define DIRTY_DECAY_TIME_DEFAULT 10 +#define MUZZY_DECAY_TIME_DEFAULT 10 /* Number of event ticks between time checks. */ #define DECAY_NTICKS_PER_UPDATE 1000 diff --git a/include/jemalloc/internal/ctl_structs.h b/include/jemalloc/internal/ctl_structs.h index 18806a59..4df43d90 100644 --- a/include/jemalloc/internal/ctl_structs.h +++ b/include/jemalloc/internal/ctl_structs.h @@ -51,9 +51,11 @@ struct ctl_arena_s { /* Basic stats, supported even if !config_stats. */ unsigned nthreads; const char *dss; - ssize_t decay_time; + ssize_t dirty_decay_time; + ssize_t muzzy_decay_time; size_t pactive; size_t pdirty; + size_t pmuzzy; /* NULL if !config_stats. */ ctl_arena_stats_t *astats; diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h index ef2467e1..e8f632f8 100644 --- a/include/jemalloc/internal/extent_externs.h +++ b/include/jemalloc/internal/extent_externs.h @@ -21,20 +21,21 @@ bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, bool delay_coalesce); extent_state_t extents_state_get(const extents_t *extents); size_t extents_npages_get(extents_t *extents); +extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, + size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, + bool slab); +void extents_dalloc(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent); extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_min); void extents_prefork(tsdn_t *tsdn, extents_t *extents); void extents_postfork_parent(tsdn_t *tsdn, extents_t *extents); void extents_postfork_child(tsdn_t *tsdn, extents_t *extents); -extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, - size_t alignment, bool *zero, bool *commit, bool slab); extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab); void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent); -void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_t *extent); bool extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h index c14aef86..001b7c13 100644 --- a/include/jemalloc/internal/extent_structs.h +++ b/include/jemalloc/internal/extent_structs.h @@ -4,7 +4,8 @@ typedef enum { extent_state_active = 0, extent_state_dirty = 1, - extent_state_retained = 2 + extent_state_muzzy = 2, + extent_state_retained = 3 } extent_state_t; /* Extent (span of pages). Use accessor functions for e_* fields. */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index c0211e58..5ca72818 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -15,21 +15,26 @@ arena_dalloc_bin_junked_locked arena_dalloc_junk_small arena_dalloc_promoted arena_dalloc_small +arena_decay arena_decay_tick arena_decay_ticks -arena_decay_time_default_get -arena_decay_time_default_set -arena_decay_time_get -arena_decay_time_set +arena_dirty_decay_time_default_get +arena_dirty_decay_time_default_set +arena_dirty_decay_time_get +arena_dirty_decay_time_set +arena_muzzy_decay_time_default_get +arena_muzzy_decay_time_default_set +arena_muzzy_decay_time_get +arena_muzzy_decay_time_set arena_destroy arena_dss_prec_get arena_dss_prec_set arena_extent_alloc_large -arena_extent_cache_dalloc arena_extent_dalloc_large_prep arena_extent_ralloc_large_expand arena_extent_ralloc_large_shrink arena_extent_sn_next +arena_extents_dirty_dalloc arena_get arena_ichoose arena_ind_get @@ -59,7 +64,6 @@ arena_prof_promote arena_prof_tctx_get arena_prof_tctx_reset arena_prof_tctx_set -arena_purge arena_ralloc arena_ralloc_no_move arena_reset @@ -138,7 +142,6 @@ extent_commit_wrapper extent_committed_get extent_committed_set extent_dalloc -extent_dalloc_cache extent_dalloc_gap extent_dalloc_mmap extent_dalloc_wrapper @@ -192,6 +195,8 @@ extent_usize_get extent_usize_set extent_zeroed_get extent_zeroed_set +extents_alloc +extents_dalloc extents_evict extents_init extents_npages_get @@ -299,7 +304,8 @@ nstime_sec nstime_subtract nstime_update opt_abort -opt_decay_time +opt_dirty_decay_time +opt_muzzy_decay_time opt_dss opt_junk opt_junk_alloc diff --git a/include/jemalloc/internal/stats_structs.h b/include/jemalloc/internal/stats_structs.h index 4e9c898a..ffcb3c18 100644 --- a/include/jemalloc/internal/stats_structs.h +++ b/include/jemalloc/internal/stats_structs.h @@ -77,6 +77,15 @@ struct malloc_large_stats_s { size_t curlextents; /* Derived. */ }; +struct decay_stats_s { + /* Total number of purge sweeps. */ + arena_stats_u64_t npurge; + /* Total number of madvise calls made. */ + arena_stats_u64_t nmadvise; + /* Total number of pages purged. */ + arena_stats_u64_t purged; +}; + /* * Arena stats. Note that fields marked "derived" are not directly maintained * within the arena code; rather their values are derived during stats merge @@ -84,7 +93,7 @@ struct malloc_large_stats_s { */ struct arena_stats_s { #ifndef JEMALLOC_ATOMIC_U64 - malloc_mutex_t mtx; + malloc_mutex_t mtx; #endif /* Number of bytes currently mapped, excluding retained memory. */ @@ -98,14 +107,8 @@ struct arena_stats_s { */ atomic_zu_t retained; /* Derived. */ - /* - * Total number of purge sweeps, total number of madvise calls made, - * and total pages purged in order to keep dirty unused memory under - * control. - */ - arena_stats_u64_t npurge; - arena_stats_u64_t nmadvise; - arena_stats_u64_t purged; + decay_stats_t decay_dirty; + decay_stats_t decay_muzzy; atomic_zu_t base; /* Derived. */ atomic_zu_t internal; diff --git a/include/jemalloc/internal/stats_types.h b/include/jemalloc/internal/stats_types.h index f202b231..48483388 100644 --- a/include/jemalloc/internal/stats_types.h +++ b/include/jemalloc/internal/stats_types.h @@ -4,6 +4,7 @@ typedef struct tcache_bin_stats_s tcache_bin_stats_t; typedef struct malloc_bin_stats_s malloc_bin_stats_t; typedef struct malloc_large_stats_s malloc_large_stats_t; +typedef struct decay_stats_s decay_stats_t; typedef struct arena_stats_s arena_stats_t; #endif /* JEMALLOC_INTERNAL_STATS_TYPES_H */ diff --git a/src/arena.c b/src/arena.c index c253760b..d861fff6 100644 --- a/src/arena.c +++ b/src/arena.c @@ -13,8 +13,10 @@ const char *percpu_arena_mode_names[] = { const char *opt_percpu_arena = OPT_PERCPU_ARENA_DEFAULT; percpu_arena_mode_t percpu_arena_mode = PERCPU_ARENA_MODE_DEFAULT; -ssize_t opt_decay_time = DECAY_TIME_DEFAULT; -static ssize_t decay_time_default; +ssize_t opt_dirty_decay_time = DIRTY_DECAY_TIME_DEFAULT; +ssize_t opt_muzzy_decay_time = MUZZY_DECAY_TIME_DEFAULT; +static ssize_t dirty_decay_time_default; +static ssize_t muzzy_decay_time_default; const arena_bin_info_t arena_bin_info[NBINS] = { #define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \ @@ -37,12 +39,13 @@ const arena_bin_info_t arena_bin_info[NBINS] = { * definition. */ -static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, - arena_decay_t *decay, extents_t *extents, size_t ndirty_limit); -static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, - extent_t *slab, arena_bin_t *bin); -static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, - extent_t *slab, arena_bin_t *bin); +static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, + arena_decay_t *decay, extents_t *extents, bool all, size_t npages_limit); +static void arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool all); +static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, + arena_bin_t *bin); +static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, + arena_bin_t *bin); /******************************************************************************/ @@ -50,7 +53,7 @@ static bool arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { if (config_debug) { for (size_t i = 0; i < sizeof(arena_stats_t); i++) { - assert(((char *)arena_stats)[0] == 0); + assert(((char *)arena_stats)[i] == 0); } } #ifndef JEMALLOC_ATOMIC_U64 @@ -187,23 +190,27 @@ arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) { void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, - const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty) { + const char **dss, ssize_t *dirty_decay_time, ssize_t *muzzy_decay_time, + size_t *nactive, size_t *ndirty, + size_t *nmuzzy) { *nthreads += arena_nthreads_get(arena, false); *dss = dss_prec_names[arena_dss_prec_get(arena)]; - *decay_time = arena_decay_time_get(arena); + *dirty_decay_time = arena_dirty_decay_time_get(arena); + *muzzy_decay_time = arena_muzzy_decay_time_get(arena); *nactive += atomic_read_zu(&arena->nactive); - *ndirty += extents_npages_get(&arena->extents_cached); + *ndirty += extents_npages_get(&arena->extents_dirty); + *nmuzzy += extents_npages_get(&arena->extents_muzzy); } void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, - const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats) { + const char **dss, ssize_t *dirty_decay_time, ssize_t *muzzy_decay_time, + size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, + malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats) { cassert(config_stats); - arena_basic_stats_merge(tsdn, arena, nthreads, dss, decay_time, - nactive, ndirty); + arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_time, + muzzy_decay_time, nactive, ndirty, nmuzzy); size_t base_allocated, base_resident, base_mapped; base_stats_get(tsdn, arena->base, &base_allocated, &base_resident, @@ -215,17 +222,33 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, + arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped)); arena_stats_accum_zu(&astats->retained, extents_npages_get(&arena->extents_retained) << LG_PAGE); - arena_stats_accum_u64(&astats->npurge, arena_stats_read_u64(tsdn, - &arena->stats, &arena->stats.npurge)); - arena_stats_accum_u64(&astats->nmadvise, arena_stats_read_u64(tsdn, - &arena->stats, &arena->stats.nmadvise)); - arena_stats_accum_u64(&astats->purged, arena_stats_read_u64(tsdn, - &arena->stats, &arena->stats.purged)); + + arena_stats_accum_u64(&astats->decay_dirty.npurge, + arena_stats_read_u64(tsdn, &arena->stats, + &arena->stats.decay_dirty.npurge)); + arena_stats_accum_u64(&astats->decay_dirty.nmadvise, + arena_stats_read_u64(tsdn, &arena->stats, + &arena->stats.decay_dirty.nmadvise)); + arena_stats_accum_u64(&astats->decay_dirty.purged, + arena_stats_read_u64(tsdn, &arena->stats, + &arena->stats.decay_dirty.purged)); + + arena_stats_accum_u64(&astats->decay_muzzy.npurge, + arena_stats_read_u64(tsdn, &arena->stats, + &arena->stats.decay_muzzy.npurge)); + arena_stats_accum_u64(&astats->decay_muzzy.nmadvise, + arena_stats_read_u64(tsdn, &arena->stats, + &arena->stats.decay_muzzy.nmadvise)); + arena_stats_accum_u64(&astats->decay_muzzy.purged, + arena_stats_read_u64(tsdn, &arena->stats, + &arena->stats.decay_muzzy.purged)); + arena_stats_accum_zu(&astats->base, base_allocated); arena_stats_accum_zu(&astats->internal, arena_internal_get(arena)); - arena_stats_accum_zu(&astats->resident, base_resident - + (((atomic_read_zu(&arena->nactive) - + extents_npages_get(&arena->extents_cached)) << LG_PAGE))); + arena_stats_accum_zu(&astats->resident, base_resident + + (((atomic_read_zu(&arena->nactive) + + extents_npages_get(&arena->extents_dirty) + + extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); for (szind_t i = 0; i < NSIZES - NBINS; i++) { uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, @@ -292,13 +315,14 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, } void -arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, +arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent); - if (arena_decay_time_get(arena) == 0) { - arena_purge(tsdn, arena, true); + extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty, + extent); + if (arena_dirty_decay_time_get(arena) == 0) { + arena_decay_dirty(tsdn, arena, true); } } @@ -432,8 +456,14 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t mapped_add; bool commit = true; - extent = extent_alloc_cache(tsdn, arena, &extent_hooks, NULL, usize, - large_pad, alignment, zero, &commit, false); + extent = extents_alloc(tsdn, arena, &extent_hooks, + &arena->extents_dirty, NULL, usize, large_pad, alignment, zero, + &commit, false); + if (extent == NULL) { + extent = extents_alloc(tsdn, arena, &extent_hooks, + &arena->extents_muzzy, NULL, usize, large_pad, alignment, + zero, &commit, false); + } size_t size = usize + large_pad; if (extent == NULL) { extent = extent_alloc_wrapper(tsdn, arena, &extent_hooks, NULL, @@ -507,12 +537,12 @@ arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, static ssize_t arena_decay_time_read(arena_decay_t *decay) { - return atomic_read_zd(&decay->time); + return atomic_load_zd(&decay->time, ATOMIC_RELAXED); } static void arena_decay_time_write(arena_decay_t *decay, ssize_t decay_time) { - atomic_write_zd(&decay->time, decay_time); + atomic_store_zd(&decay->time, decay_time, ATOMIC_RELAXED); } static void @@ -621,10 +651,11 @@ arena_decay_epoch_advance_helper(arena_decay_t *decay, extents_t *extents, static void arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, extents_t *extents) { - size_t ndirty_limit = arena_decay_backlog_npages_limit(decay); + size_t npages_limit = arena_decay_backlog_npages_limit(decay); - if (extents_npages_get(extents) > ndirty_limit) { - arena_purge_to_limit(tsdn, arena, decay, extents, ndirty_limit); + if (extents_npages_get(extents) > npages_limit) { + arena_decay_to_limit(tsdn, arena, decay, extents, false, + npages_limit); } /* * There may be concurrent ndirty fluctuation between the purge above @@ -659,12 +690,22 @@ arena_decay_reinit(arena_decay_t *decay, extents_t *extents, } static bool -arena_decay_init(arena_decay_t *decay, extents_t *extents, ssize_t decay_time) { +arena_decay_init(arena_decay_t *decay, extents_t *extents, ssize_t decay_time, + decay_stats_t *stats) { + if (config_debug) { + for (size_t i = 0; i < sizeof(arena_decay_t); i++) { + assert(((char *)decay)[i] == 0); + } + } if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY)) { return true; } decay->purging = false; arena_decay_reinit(decay, extents, decay_time); + /* Memory is zeroed, so there is no need to clear stats. */ + if (config_stats) { + decay->stats = stats; + } return false; } @@ -680,7 +721,7 @@ arena_decay_time_valid(ssize_t decay_time) { } static void -arena_maybe_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, +arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, extents_t *extents) { malloc_mutex_assert_owner(tsdn, &decay->mtx); @@ -688,7 +729,8 @@ arena_maybe_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, ssize_t decay_time = arena_decay_time_read(decay); if (decay_time <= 0) { if (decay_time == 0) { - arena_purge_to_limit(tsdn, arena, decay, extents, 0); + arena_decay_to_limit(tsdn, arena, decay, extents, false, + 0); } return; } @@ -725,18 +767,29 @@ arena_maybe_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, } } -ssize_t -arena_decay_time_get(arena_t *arena) { - return arena_decay_time_read(&arena->decay); +static ssize_t +arena_decay_time_get(arena_decay_t *decay) { + return arena_decay_time_read(decay); } -bool -arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) { +ssize_t +arena_dirty_decay_time_get(arena_t *arena) { + return arena_decay_time_get(&arena->decay_dirty); +} + +ssize_t +arena_muzzy_decay_time_get(arena_t *arena) { + return arena_decay_time_get(&arena->decay_muzzy); +} + +static bool +arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, + extents_t *extents, ssize_t decay_time) { if (!arena_decay_time_valid(decay_time)) { return true; } - malloc_mutex_lock(tsdn, &arena->decay.mtx); + malloc_mutex_lock(tsdn, &decay->mtx); /* * Restart decay backlog from scratch, which may cause many dirty pages * to be immediately purged. It would conceptually be possible to map @@ -745,58 +798,100 @@ arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) { * infrequent, either between the {-1, 0, >0} states, or a one-time * arbitrary change during initial arena configuration. */ - arena_decay_reinit(&arena->decay, &arena->extents_cached, decay_time); - arena_maybe_purge(tsdn, arena, &arena->decay, &arena->extents_cached); - malloc_mutex_unlock(tsdn, &arena->decay.mtx); + arena_decay_reinit(decay, extents, decay_time); + arena_maybe_decay(tsdn, arena, decay, extents); + malloc_mutex_unlock(tsdn, &decay->mtx); return false; } +bool +arena_dirty_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) { + return arena_decay_time_set(tsdn, arena, &arena->decay_dirty, + &arena->extents_dirty, decay_time); +} + +bool +arena_muzzy_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) { + return arena_decay_time_set(tsdn, arena, &arena->decay_muzzy, + &arena->extents_muzzy, decay_time); +} + static size_t -arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extents_t *extents, size_t ndirty_limit, extent_list_t *purge_extents) { +arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit, + extent_list_t *decay_extents) { witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - /* Stash extents according to ndirty_limit. */ + /* Stash extents according to npages_limit. */ size_t nstashed = 0; extent_t *extent; while ((extent = extents_evict(tsdn, arena, r_extent_hooks, extents, - ndirty_limit)) != NULL) { - extent_list_append(purge_extents, extent); + npages_limit)) != NULL) { + extent_list_append(decay_extents, extent); nstashed += extent_size_get(extent) >> LG_PAGE; } return nstashed; } static size_t -arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_list_t *purge_extents) { - UNUSED size_t nmadvise; +arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents, + bool all, extent_list_t *decay_extents) { + UNUSED size_t nmadvise, nunmapped; size_t npurged; if (config_stats) { nmadvise = 0; + nunmapped = 0; } npurged = 0; - for (extent_t *extent = extent_list_first(purge_extents); extent != - NULL; extent = extent_list_first(purge_extents)) { + ssize_t muzzy_decay_time = arena_muzzy_decay_time_get(arena); + for (extent_t *extent = extent_list_first(decay_extents); extent != + NULL; extent = extent_list_first(decay_extents)) { if (config_stats) { nmadvise++; } - npurged += extent_size_get(extent) >> LG_PAGE; - extent_list_remove(purge_extents, extent); - extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, extent); + size_t npages = extent_size_get(extent) >> LG_PAGE; + npurged += npages; + extent_list_remove(decay_extents, extent); + switch (extents_state_get(extents)) { + case extent_state_active: + not_reached(); + case extent_state_dirty: + if (!all && muzzy_decay_time != 0 && + !extent_purge_lazy_wrapper(tsdn, arena, + r_extent_hooks, extent, 0, + extent_size_get(extent))) { + extents_dalloc(tsdn, arena, r_extent_hooks, + &arena->extents_muzzy, extent); + break; + } + /* Fall through. */ + case extent_state_muzzy: + extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, + extent); + if (config_stats) { + nunmapped += npages; + } + break; + case extent_state_retained: + default: + not_reached(); + } } if (config_stats) { arena_stats_lock(tsdn, &arena->stats); - arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.nmadvise, - nmadvise); - arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.purged, + arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->npurge, + 1); + arena_stats_add_u64(tsdn, &arena->stats, + &decay->stats->nmadvise, nmadvise); + arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->purged, npurged); arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, - npurged << LG_PAGE); + nunmapped); arena_stats_unlock(tsdn, &arena->stats); } @@ -804,12 +899,12 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, } /* - * ndirty_limit: Purge as many dirty extents as possible without violating the - * invariant: (extents_npages_get(extents) >= ndirty_limit) + * npages_limit: Decay as many dirty extents as possible without violating the + * invariant: (extents_npages_get(extents) >= npages_limit) */ static void -arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - extents_t *extents, size_t ndirty_limit) { +arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, + extents_t *extents, bool all, size_t npages_limit) { witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1); malloc_mutex_assert_owner(tsdn, &decay->mtx); @@ -817,49 +912,53 @@ arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, return; } decay->purging = true; - - extent_hooks_t *extent_hooks = extent_hooks_get(arena); - size_t npurge, npurged; - extent_list_t purge_extents; - - extent_list_init(&purge_extents); - malloc_mutex_unlock(tsdn, &decay->mtx); - npurge = arena_stash_dirty(tsdn, arena, &extent_hooks, extents, - ndirty_limit, &purge_extents); - if (npurge == 0) { - malloc_mutex_lock(tsdn, &decay->mtx); - goto label_return; + extent_hooks_t *extent_hooks = extent_hooks_get(arena); + + extent_list_t decay_extents; + extent_list_init(&decay_extents); + + size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents, + npages_limit, &decay_extents); + if (npurge != 0) { + UNUSED size_t npurged = arena_decay_stashed(tsdn, arena, + &extent_hooks, decay, extents, all, &decay_extents); + assert(npurged == npurge); } - npurged = arena_purge_stashed(tsdn, arena, &extent_hooks, - &purge_extents); - assert(npurged == npurge); malloc_mutex_lock(tsdn, &decay->mtx); - - if (config_stats) { - arena_stats_lock(tsdn, &arena->stats); - arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.npurge, - 1); - arena_stats_unlock(tsdn, &arena->stats); - } - -label_return: decay->purging = false; } -void -arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) { - malloc_mutex_lock(tsdn, &arena->decay.mtx); +static void +arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, + extents_t *extents, bool all) { + malloc_mutex_lock(tsdn, &decay->mtx); if (all) { - arena_purge_to_limit(tsdn, arena, &arena->decay, - &arena->extents_cached, 0); + arena_decay_to_limit(tsdn, arena, decay, extents, all, 0); } else { - arena_maybe_purge(tsdn, arena, &arena->decay, - &arena->extents_cached); + arena_maybe_decay(tsdn, arena, decay, extents); } - malloc_mutex_unlock(tsdn, &arena->decay.mtx); + malloc_mutex_unlock(tsdn, &decay->mtx); +} + +static void +arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool all) { + arena_decay_impl(tsdn, arena, &arena->decay_dirty, + &arena->extents_dirty, all); +} + +static void +arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool all) { + arena_decay_impl(tsdn, arena, &arena->decay_muzzy, + &arena->extents_muzzy, all); +} + +void +arena_decay(tsdn_t *tsdn, arena_t *arena, bool all) { + arena_decay_dirty(tsdn, arena, all); + arena_decay_muzzy(tsdn, arena, all); } static void @@ -867,7 +966,7 @@ arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; - arena_extent_cache_dalloc(tsdn, arena, &extent_hooks, slab); + arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, slab); } static void @@ -1008,7 +1107,7 @@ arena_destroy(tsd_t *tsd, arena_t *arena) { * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached * extents, so only retained extents may remain. */ - assert(extents_npages_get(&arena->extents_cached) == 0); + assert(extents_npages_get(&arena->extents_dirty) == 0); /* Attempt to deallocate retained memory. */ arena_destroy_retained(tsd_tsdn(tsd), arena); @@ -1061,8 +1160,14 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; bool zero = false; bool commit = true; - extent_t *slab = extent_alloc_cache(tsdn, arena, &extent_hooks, NULL, - bin_info->slab_size, 0, PAGE, &zero, &commit, true); + extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks, + &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, &zero, + &commit, true); + if (slab == NULL) { + slab = extents_alloc(tsdn, arena, &extent_hooks, + &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE, + &zero, &commit, true); + } if (slab == NULL) { slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks, bin_info); @@ -1622,16 +1727,32 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) { } ssize_t -arena_decay_time_default_get(void) { - return (ssize_t)atomic_read_zu((size_t *)&decay_time_default); +arena_dirty_decay_time_default_get(void) { + return (ssize_t)atomic_read_zu((size_t *)&dirty_decay_time_default); } bool -arena_decay_time_default_set(ssize_t decay_time) { +arena_dirty_decay_time_default_set(ssize_t decay_time) { if (!arena_decay_time_valid(decay_time)) { return true; } - atomic_write_zu((size_t *)&decay_time_default, (size_t)decay_time); + atomic_write_zu((size_t *)&dirty_decay_time_default, + (size_t)decay_time); + return false; +} + +ssize_t +arena_muzzy_decay_time_default_get(void) { + return (ssize_t)atomic_read_zu((size_t *)&muzzy_decay_time_default); +} + +bool +arena_muzzy_decay_time_default_set(ssize_t decay_time) { + if (!arena_decay_time_valid(decay_time)) { + return true; + } + atomic_write_zu((size_t *)&muzzy_decay_time_default, + (size_t)decay_time); return false; } @@ -1723,28 +1844,40 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } /* - * Delay coalescing for cached extents despite the disruptive effect on + * Delay coalescing for dirty extents despite the disruptive effect on * memory layout for best-fit extent allocation, since cached extents * are likely to be reused soon after deallocation, and the cost of * merging/splitting extents is non-trivial. */ - if (extents_init(tsdn, &arena->extents_cached, extent_state_dirty, + if (extents_init(tsdn, &arena->extents_dirty, extent_state_dirty, true)) { goto label_error; } + /* + * Coalesce muzzy extents immediately, because operations on them are in + * the critical path much less often than for dirty extents. + */ + if (extents_init(tsdn, &arena->extents_muzzy, extent_state_muzzy, + false)) { + goto label_error; + } /* * Coalesce retained extents immediately, in part because they will * never be evicted (and therefore there's no opportunity for delayed * coalescing), but also because operations on retained extents are not * in the critical path. */ - if (extents_init(tsdn, &arena->extents_retained, - extent_state_retained, false)) { + if (extents_init(tsdn, &arena->extents_retained, extent_state_retained, + false)) { goto label_error; } - if (arena_decay_init(&arena->decay, &arena->extents_cached, - arena_decay_time_default_get())) { + if (arena_decay_init(&arena->decay_dirty, &arena->extents_dirty, + arena_dirty_decay_time_default_get(), &arena->stats.decay_dirty)) { + goto label_error; + } + if (arena_decay_init(&arena->decay_muzzy, &arena->extents_muzzy, + arena_muzzy_decay_time_default_get(), &arena->stats.decay_muzzy)) { goto label_error; } @@ -1785,12 +1918,14 @@ label_error: void arena_boot(void) { - arena_decay_time_default_set(opt_decay_time); + arena_dirty_decay_time_default_set(opt_dirty_decay_time); + arena_muzzy_decay_time_default_set(opt_muzzy_decay_time); } void arena_prefork0(tsdn_t *tsdn, arena_t *arena) { - malloc_mutex_prefork(tsdn, &arena->decay.mtx); + malloc_mutex_prefork(tsdn, &arena->decay_dirty.mtx); + malloc_mutex_prefork(tsdn, &arena->decay_muzzy.mtx); } void @@ -1802,7 +1937,8 @@ arena_prefork1(tsdn_t *tsdn, arena_t *arena) { void arena_prefork2(tsdn_t *tsdn, arena_t *arena) { - extents_prefork(tsdn, &arena->extents_cached); + extents_prefork(tsdn, &arena->extents_dirty); + extents_prefork(tsdn, &arena->extents_muzzy); extents_prefork(tsdn, &arena->extents_retained); } @@ -1838,9 +1974,11 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); base_postfork_parent(tsdn, arena->base); malloc_mutex_postfork_parent(tsdn, &arena->extent_freelist_mtx); - extents_postfork_parent(tsdn, &arena->extents_cached); + extents_postfork_parent(tsdn, &arena->extents_dirty); + extents_postfork_parent(tsdn, &arena->extents_muzzy); extents_postfork_parent(tsdn, &arena->extents_retained); - malloc_mutex_postfork_parent(tsdn, &arena->decay.mtx); + malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx); + malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx); if (config_stats && config_tcache) { malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx); } @@ -1856,9 +1994,11 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_postfork_child(tsdn, &arena->large_mtx); base_postfork_child(tsdn, arena->base); malloc_mutex_postfork_child(tsdn, &arena->extent_freelist_mtx); - extents_postfork_child(tsdn, &arena->extents_cached); + extents_postfork_child(tsdn, &arena->extents_dirty); + extents_postfork_child(tsdn, &arena->extents_muzzy); extents_postfork_child(tsdn, &arena->extents_retained); - malloc_mutex_postfork_child(tsdn, &arena->decay.mtx); + malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx); + malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx); if (config_stats && config_tcache) { malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx); } diff --git a/src/ctl.c b/src/ctl.c index 70721584..ee69be6d 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -73,7 +73,8 @@ CTL_PROTO(opt_abort) CTL_PROTO(opt_dss) CTL_PROTO(opt_narenas) CTL_PROTO(opt_percpu_arena) -CTL_PROTO(opt_decay_time) +CTL_PROTO(opt_dirty_decay_time) +CTL_PROTO(opt_muzzy_decay_time) CTL_PROTO(opt_stats_print) CTL_PROTO(opt_junk) CTL_PROTO(opt_zero) @@ -95,12 +96,13 @@ CTL_PROTO(tcache_create) CTL_PROTO(tcache_flush) CTL_PROTO(tcache_destroy) CTL_PROTO(arena_i_initialized) -CTL_PROTO(arena_i_purge) CTL_PROTO(arena_i_decay) +CTL_PROTO(arena_i_purge) CTL_PROTO(arena_i_reset) CTL_PROTO(arena_i_destroy) CTL_PROTO(arena_i_dss) -CTL_PROTO(arena_i_decay_time) +CTL_PROTO(arena_i_dirty_decay_time) +CTL_PROTO(arena_i_muzzy_decay_time) CTL_PROTO(arena_i_extent_hooks) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) @@ -110,7 +112,8 @@ INDEX_PROTO(arenas_bin_i) CTL_PROTO(arenas_lextent_i_size) INDEX_PROTO(arenas_lextent_i) CTL_PROTO(arenas_narenas) -CTL_PROTO(arenas_decay_time) +CTL_PROTO(arenas_dirty_decay_time) +CTL_PROTO(arenas_muzzy_decay_time) CTL_PROTO(arenas_quantum) CTL_PROTO(arenas_page) CTL_PROTO(arenas_tcache_max) @@ -150,14 +153,19 @@ CTL_PROTO(stats_arenas_i_lextents_j_curlextents) INDEX_PROTO(stats_arenas_i_lextents_j) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_dss) -CTL_PROTO(stats_arenas_i_decay_time) +CTL_PROTO(stats_arenas_i_dirty_decay_time) +CTL_PROTO(stats_arenas_i_muzzy_decay_time) CTL_PROTO(stats_arenas_i_pactive) CTL_PROTO(stats_arenas_i_pdirty) +CTL_PROTO(stats_arenas_i_pmuzzy) CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_retained) -CTL_PROTO(stats_arenas_i_npurge) -CTL_PROTO(stats_arenas_i_nmadvise) -CTL_PROTO(stats_arenas_i_purged) +CTL_PROTO(stats_arenas_i_dirty_npurge) +CTL_PROTO(stats_arenas_i_dirty_nmadvise) +CTL_PROTO(stats_arenas_i_dirty_purged) +CTL_PROTO(stats_arenas_i_muzzy_npurge) +CTL_PROTO(stats_arenas_i_muzzy_nmadvise) +CTL_PROTO(stats_arenas_i_muzzy_purged) CTL_PROTO(stats_arenas_i_base) CTL_PROTO(stats_arenas_i_internal) CTL_PROTO(stats_arenas_i_tcache_bytes) @@ -231,7 +239,8 @@ static const ctl_named_node_t opt_node[] = { {NAME("dss"), CTL(opt_dss)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("percpu_arena"), CTL(opt_percpu_arena)}, - {NAME("decay_time"), CTL(opt_decay_time)}, + {NAME("dirty_decay_time"), CTL(opt_dirty_decay_time)}, + {NAME("muzzy_decay_time"), CTL(opt_muzzy_decay_time)}, {NAME("stats_print"), CTL(opt_stats_print)}, {NAME("junk"), CTL(opt_junk)}, {NAME("zero"), CTL(opt_zero)}, @@ -259,12 +268,13 @@ static const ctl_named_node_t tcache_node[] = { static const ctl_named_node_t arena_i_node[] = { {NAME("initialized"), CTL(arena_i_initialized)}, - {NAME("purge"), CTL(arena_i_purge)}, {NAME("decay"), CTL(arena_i_decay)}, + {NAME("purge"), CTL(arena_i_purge)}, {NAME("reset"), CTL(arena_i_reset)}, {NAME("destroy"), CTL(arena_i_destroy)}, {NAME("dss"), CTL(arena_i_dss)}, - {NAME("decay_time"), CTL(arena_i_decay_time)}, + {NAME("dirty_decay_time"), CTL(arena_i_dirty_decay_time)}, + {NAME("muzzy_decay_time"), CTL(arena_i_muzzy_decay_time)}, {NAME("extent_hooks"), CTL(arena_i_extent_hooks)} }; static const ctl_named_node_t super_arena_i_node[] = { @@ -301,7 +311,8 @@ static const ctl_indexed_node_t arenas_lextent_node[] = { static const ctl_named_node_t arenas_node[] = { {NAME("narenas"), CTL(arenas_narenas)}, - {NAME("decay_time"), CTL(arenas_decay_time)}, + {NAME("dirty_decay_time"), CTL(arenas_dirty_decay_time)}, + {NAME("muzzy_decay_time"), CTL(arenas_muzzy_decay_time)}, {NAME("quantum"), CTL(arenas_quantum)}, {NAME("page"), CTL(arenas_page)}, {NAME("tcache_max"), CTL(arenas_tcache_max)}, @@ -373,14 +384,19 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, {NAME("dss"), CTL(stats_arenas_i_dss)}, - {NAME("decay_time"), CTL(stats_arenas_i_decay_time)}, + {NAME("dirty_decay_time"), CTL(stats_arenas_i_dirty_decay_time)}, + {NAME("muzzy_decay_time"), CTL(stats_arenas_i_muzzy_decay_time)}, {NAME("pactive"), CTL(stats_arenas_i_pactive)}, {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, + {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("retained"), CTL(stats_arenas_i_retained)}, - {NAME("npurge"), CTL(stats_arenas_i_npurge)}, - {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, - {NAME("purged"), CTL(stats_arenas_i_purged)}, + {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, + {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, + {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, + {NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)}, + {NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)}, + {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, {NAME("base"), CTL(stats_arenas_i_base)}, {NAME("internal"), CTL(stats_arenas_i_internal)}, {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, @@ -554,9 +570,11 @@ static void ctl_arena_clear(ctl_arena_t *ctl_arena) { ctl_arena->nthreads = 0; ctl_arena->dss = dss_prec_names[dss_prec_limit]; - ctl_arena->decay_time = -1; + ctl_arena->dirty_decay_time = -1; + ctl_arena->muzzy_decay_time = -1; ctl_arena->pactive = 0; ctl_arena->pdirty = 0; + ctl_arena->pmuzzy = 0; if (config_stats) { memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t)); ctl_arena->astats->allocated_small = 0; @@ -576,8 +594,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { if (config_stats) { arena_stats_merge(tsdn, arena, &ctl_arena->nthreads, - &ctl_arena->dss, &ctl_arena->decay_time, - &ctl_arena->pactive, &ctl_arena->pdirty, + &ctl_arena->dss, &ctl_arena->dirty_decay_time, + &ctl_arena->muzzy_decay_time, &ctl_arena->pactive, + &ctl_arena->pdirty, &ctl_arena->pmuzzy, &ctl_arena->astats->astats, ctl_arena->astats->bstats, ctl_arena->astats->lstats); @@ -594,8 +613,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { } } else { arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, - &ctl_arena->dss, &ctl_arena->decay_time, - &ctl_arena->pactive, &ctl_arena->pdirty); + &ctl_arena->dss, &ctl_arena->dirty_decay_time, + &ctl_arena->muzzy_decay_time, &ctl_arena->pactive, + &ctl_arena->pdirty, &ctl_arena->pmuzzy); } } @@ -608,10 +628,12 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, ctl_sdarena->nthreads += ctl_arena->nthreads; ctl_sdarena->pactive += ctl_arena->pactive; ctl_sdarena->pdirty += ctl_arena->pdirty; + ctl_sdarena->pmuzzy += ctl_arena->pmuzzy; } else { assert(ctl_arena->nthreads == 0); assert(ctl_arena->pactive == 0); assert(ctl_arena->pdirty == 0); + assert(ctl_arena->pmuzzy == 0); } if (config_stats) { @@ -624,12 +646,20 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, accum_atomic_zu(&sdstats->astats.retained, &astats->astats.retained); } - accum_arena_stats_u64(&sdstats->astats.npurge, - &astats->astats.npurge); - accum_arena_stats_u64(&sdstats->astats.nmadvise, - &astats->astats.nmadvise); - accum_arena_stats_u64(&sdstats->astats.purged, - &astats->astats.purged); + + accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, + &astats->astats.decay_dirty.npurge); + accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise, + &astats->astats.decay_dirty.nmadvise); + accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged, + &astats->astats.decay_dirty.purged); + + accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge, + &astats->astats.decay_muzzy.npurge); + accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise, + &astats->astats.decay_muzzy.nmadvise); + accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, + &astats->astats.decay_muzzy.purged); if (!destroyed) { accum_atomic_zu(&sdstats->astats.base, @@ -1340,7 +1370,8 @@ CTL_RO_NL_GEN(opt_abort, opt_abort, bool) CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) CTL_RO_NL_GEN(opt_percpu_arena, opt_percpu_arena, const char *) -CTL_RO_NL_GEN(opt_decay_time, opt_decay_time, ssize_t) +CTL_RO_NL_GEN(opt_dirty_decay_time, opt_dirty_decay_time, ssize_t) +CTL_RO_NL_GEN(opt_muzzy_decay_time, opt_muzzy_decay_time, ssize_t) CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) @@ -1630,7 +1661,7 @@ label_return: } static void -arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all) { +arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { malloc_mutex_lock(tsdn, &ctl_mtx); { unsigned narenas = ctl_arenas->narenas; @@ -1655,7 +1686,7 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all) { for (i = 0; i < narenas; i++) { if (tarenas[i] != NULL) { - arena_purge(tsdn, tarenas[i], all); + arena_decay(tsdn, tarenas[i], all); } } } else { @@ -1669,28 +1700,12 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all) { malloc_mutex_unlock(tsdn, &ctl_mtx); if (tarena != NULL) { - arena_purge(tsdn, tarena, all); + arena_decay(tsdn, tarena, all); } } } } -static int -arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { - int ret; - unsigned arena_ind; - - READONLY(); - WRITEONLY(); - MIB_UNSIGNED(arena_ind, 1); - arena_i_purge(tsd_tsdn(tsd), arena_ind, true); - - ret = 0; -label_return: - return ret; -} - static int arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { @@ -1700,7 +1715,23 @@ arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, READONLY(); WRITEONLY(); MIB_UNSIGNED(arena_ind, 1); - arena_i_purge(tsd_tsdn(tsd), arena_ind, false); + arena_i_decay(tsd_tsdn(tsd), arena_ind, false); + + ret = 0; +label_return: + return ret; +} + +static int +arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; + unsigned arena_ind; + + READONLY(); + WRITEONLY(); + MIB_UNSIGNED(arena_ind, 1); + arena_i_decay(tsd_tsdn(tsd), arena_ind, true); ret = 0; label_return: @@ -1773,7 +1804,7 @@ arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, /* Merge stats after resetting and purging arena. */ arena_reset(tsd, arena); - arena_purge(tsd_tsdn(tsd), arena, true); + arena_decay(tsd_tsdn(tsd), arena, true); ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED); ctl_darena->initialized = true; ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true); @@ -1852,8 +1883,8 @@ label_return: } static int -arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arena_i_decay_time_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { int ret; unsigned arena_ind; arena_t *arena; @@ -1866,7 +1897,8 @@ arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } if (oldp != NULL && oldlenp != NULL) { - size_t oldval = arena_decay_time_get(arena); + size_t oldval = dirty ? arena_dirty_decay_time_get(arena) : + arena_muzzy_decay_time_get(arena); READ(oldval, ssize_t); } if (newp != NULL) { @@ -1874,7 +1906,9 @@ arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, ret = EINVAL; goto label_return; } - if (arena_decay_time_set(tsd_tsdn(tsd), arena, + if (dirty ? arena_dirty_decay_time_set(tsd_tsdn(tsd), arena, + *(ssize_t *)newp) : + arena_muzzy_decay_time_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp)) { ret = EFAULT; goto label_return; @@ -1886,6 +1920,20 @@ label_return: return ret; } +static int +arena_i_dirty_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + return arena_i_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp, + newp, newlen, true); +} + +static int +arena_i_muzzy_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + return arena_i_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp, + newp, newlen, false); +} + static int arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { @@ -1967,12 +2015,13 @@ label_return: } static int -arenas_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_decay_time_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { int ret; if (oldp != NULL && oldlenp != NULL) { - size_t oldval = arena_decay_time_default_get(); + size_t oldval = (dirty ? arena_dirty_decay_time_default_get() : + arena_muzzy_decay_time_default_get()); READ(oldval, ssize_t); } if (newp != NULL) { @@ -1980,7 +2029,8 @@ arenas_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, ret = EINVAL; goto label_return; } - if (arena_decay_time_default_set(*(ssize_t *)newp)) { + if (dirty ? arena_dirty_decay_time_default_set(*(ssize_t *)newp) + : arena_muzzy_decay_time_default_set(*(ssize_t *)newp)) { ret = EFAULT; goto label_return; } @@ -1991,6 +2041,20 @@ label_return: return ret; } +static int +arenas_dirty_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + return arenas_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, + newlen, true); +} + +static int +arenas_muzzy_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + return arenas_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp, + newlen, false); +} + CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) @@ -2182,23 +2246,41 @@ CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t) CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t) CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *) -CTL_RO_GEN(stats_arenas_i_decay_time, arenas_i(mib[2])->decay_time, +CTL_RO_GEN(stats_arenas_i_dirty_decay_time, arenas_i(mib[2])->dirty_decay_time, + ssize_t) +CTL_RO_GEN(stats_arenas_i_muzzy_decay_time, arenas_i(mib[2])->muzzy_decay_time, ssize_t) CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned) CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t) CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t) +CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, atomic_load_zu(&arenas_i(mib[2])->astats->astats.mapped, ATOMIC_RELAXED), size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_retained, atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), size_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_npurge, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.npurge), uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.nmadvise), uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_purged, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.purged), uint64_t) + +CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, + arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.npurge), + uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, + arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, + arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.purged), + uint64_t) + +CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, + arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), + uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, + arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, + arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.purged), + uint64_t) + CTL_RO_CGEN(config_stats, stats_arenas_i_base, atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED), size_t) @@ -2268,8 +2350,8 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc, arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, - arena_stats_read_u64( - &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t) + arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) diff --git a/src/extent.c b/src/extent.c index c44ecb89..c690b98e 100644 --- a/src/extent.c +++ b/src/extent.c @@ -69,6 +69,10 @@ static size_t highpages; */ static void extent_deregister(tsdn_t *tsdn, extent_t *extent); +static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, + size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, + bool slab); static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, extent_t *extent, bool *coalesced); @@ -293,6 +297,31 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, return false; } +extent_t * +extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, + extents_t *extents, void *new_addr, size_t usize, size_t pad, + size_t alignment, bool *zero, bool *commit, bool slab) { + assert(usize + pad != 0); + assert(alignment != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + return extent_recycle(tsdn, arena, r_extent_hooks, extents, new_addr, + usize, pad, alignment, zero, commit, slab); +} + +void +extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, + extents_t *extents, extent_t *extent) { + assert(extent_base_get(extent) != NULL); + assert(extent_size_get(extent) != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + extent_addr_set(extent, extent_base_get(extent)); + extent_zeroed_set(extent, false); + + extent_record(tsdn, arena, r_extent_hooks, extents, extent); +} + extent_t * extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_min) { @@ -340,7 +369,10 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, * concurrent operations. */ switch (extents_state_get(extents)) { + case extent_state_active: + not_reached(); case extent_state_dirty: + case extent_state_muzzy: extent_state_set(extent, extent_state_active); break; case extent_state_retained: @@ -813,19 +845,6 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, return NULL; } -extent_t * -extent_alloc_cache(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, - size_t alignment, bool *zero, bool *commit, bool slab) { - assert(usize + pad != 0); - assert(alignment != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - - return extent_recycle(tsdn, arena, r_extent_hooks, - &arena->extents_cached, new_addr, usize, pad, alignment, zero, - commit, slab); -} - static void * extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { @@ -1206,7 +1225,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - assert(extents_state_get(extents) != extent_state_dirty || + assert((extents_state_get(extents) != extent_state_dirty && + extents_state_get(extents) != extent_state_muzzy) || !extent_zeroed_get(extent)); malloc_mutex_lock(tsdn, &extents->mtx); @@ -1244,20 +1264,6 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); } -void -extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_t *extent) { - assert(extent_base_get(extent) != NULL); - assert(extent_size_get(extent) != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - - extent_addr_set(extent, extent_base_get(extent)); - extent_zeroed_set(extent, false); - - extent_record(tsdn, arena, r_extent_hooks, &arena->extents_cached, - extent); -} - static bool extent_dalloc_default_impl(void *addr, size_t size) { if (!have_dss || !extent_in_dss(addr)) { @@ -1327,16 +1333,17 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, } else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, 0, extent_size_get(extent))) { zeroed = true; - } else if ((*r_extent_hooks)->purge_lazy != NULL && - !(*r_extent_hooks)->purge_lazy(*r_extent_hooks, - extent_base_get(extent), extent_size_get(extent), 0, - extent_size_get(extent), arena_ind_get(arena))) { - zeroed = false; } else if ((*r_extent_hooks)->purge_forced != NULL && !(*r_extent_hooks)->purge_forced(*r_extent_hooks, extent_base_get(extent), extent_size_get(extent), 0, extent_size_get(extent), arena_ind_get(arena))) { zeroed = true; + } else if (extent_state_get(extent) == extent_state_muzzy || + ((*r_extent_hooks)->purge_lazy != NULL && + !(*r_extent_hooks)->purge_lazy(*r_extent_hooks, + extent_base_get(extent), extent_size_get(extent), 0, + extent_size_get(extent), arena_ind_get(arena)))) { + zeroed = false; } else { zeroed = false; } diff --git a/src/jemalloc.c b/src/jemalloc.c index ce84b3cf..c1b05dbb 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1064,8 +1064,10 @@ malloc_conf_init(void) { } CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, UINT_MAX, yes, no, false) - CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1, - NSTIME_SEC_MAX); + CONF_HANDLE_SSIZE_T(opt_dirty_decay_time, + "dirty_decay_time", -1, NSTIME_SEC_MAX); + CONF_HANDLE_SSIZE_T(opt_muzzy_decay_time, + "muzzy_decay_time", -1, NSTIME_SEC_MAX); CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) if (config_fill) { if (CONF_MATCH("junk")) { diff --git a/src/large.c b/src/large.c index 5145f418..c578995c 100644 --- a/src/large.c +++ b/src/large.c @@ -125,7 +125,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { extent_usize_get(trail)); } - arena_extent_cache_dalloc(tsdn, arena, &extent_hooks, trail); + arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, trail); } arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize); @@ -158,9 +158,16 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool commit = true; extent_t *trail; bool new_mapping; - if ((trail = extent_alloc_cache(tsdn, arena, &extent_hooks, - extent_past_get(extent), trailsize, 0, CACHELINE, &is_zeroed_trail, - &commit, false)) == NULL) { + if ((trail = extents_alloc(tsdn, arena, &extent_hooks, + &arena->extents_dirty, extent_past_get(extent), trailsize, 0, + CACHELINE, &is_zeroed_trail, &commit, false)) != NULL + || (trail = extents_alloc(tsdn, arena, &extent_hooks, + &arena->extents_muzzy, extent_past_get(extent), trailsize, 0, + CACHELINE, &is_zeroed_trail, &commit, false)) != NULL) { + if (config_stats) { + new_mapping = false; + } + } else { if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, extent_past_get(extent), trailsize, 0, CACHELINE, &is_zeroed_trail, &commit, false)) == NULL) { @@ -169,10 +176,6 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, if (config_stats) { new_mapping = true; } - } else { - if (config_stats) { - new_mapping = false; - } } if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) { @@ -327,7 +330,7 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, static void large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; - arena_extent_cache_dalloc(tsdn, arena, &extent_hooks, extent); + arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, extent); } void diff --git a/src/stats.c b/src/stats.c index 776fb862..58b9a04f 100644 --- a/src/stats.c +++ b/src/stats.c @@ -259,10 +259,11 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, bool json, unsigned i, bool bins, bool large) { unsigned nthreads; const char *dss; - ssize_t decay_time; - size_t page, pactive, pdirty, mapped, retained; + ssize_t dirty_decay_time, muzzy_decay_time; + size_t page, pactive, pdirty, pmuzzy, mapped, retained; size_t base, internal, resident; - uint64_t npurge, nmadvise, purged; + uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; + uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; size_t small_allocated; uint64_t small_nmalloc, small_ndalloc, small_nrequests; size_t large_allocated; @@ -289,39 +290,70 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, "dss allocation precedence: %s\n", dss); } - CTL_M2_GET("stats.arenas.0.decay_time", i, &decay_time, ssize_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"decay_time\": %zd,\n", decay_time); - } else { - if (decay_time >= 0) { - malloc_cprintf(write_cb, cbopaque, "decay time: %zd\n", - decay_time); - } else { - malloc_cprintf(write_cb, cbopaque, "decay time: N/A\n"); - } - } - + CTL_M2_GET("stats.arenas.0.dirty_decay_time", i, &dirty_decay_time, + ssize_t); + CTL_M2_GET("stats.arenas.0.muzzy_decay_time", i, &muzzy_decay_time, + ssize_t); CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t); CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t); - CTL_M2_GET("stats.arenas.0.npurge", i, &npurge, uint64_t); - CTL_M2_GET("stats.arenas.0.nmadvise", i, &nmadvise, uint64_t); - CTL_M2_GET("stats.arenas.0.purged", i, &purged, uint64_t); + CTL_M2_GET("stats.arenas.0.pmuzzy", i, &pmuzzy, size_t); + CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &dirty_npurge, uint64_t); + CTL_M2_GET("stats.arenas.0.dirty_nmadvise", i, &dirty_nmadvise, + uint64_t); + CTL_M2_GET("stats.arenas.0.dirty_purged", i, &dirty_purged, uint64_t); + CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &muzzy_npurge, uint64_t); + CTL_M2_GET("stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise, + uint64_t); + CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &muzzy_purged, uint64_t); if (json) { + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\"dirty_decay_time\": %zd,\n", dirty_decay_time); + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\"muzzy_decay_time\": %zd,\n", muzzy_decay_time); malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"pactive\": %zu,\n", pactive); malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"pdirty\": %zu,\n", pdirty); malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"npurge\": %"FMTu64",\n", npurge); + "\t\t\t\t\"pmuzzy\": %zu,\n", pmuzzy); malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"nmadvise\": %"FMTu64",\n", nmadvise); + "\t\t\t\t\"dirty_npurge\": %"FMTu64",\n", dirty_npurge); malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"purged\": %"FMTu64",\n", purged); + "\t\t\t\t\"dirty_nmadvise\": %"FMTu64",\n", dirty_nmadvise); + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\"dirty_purged\": %"FMTu64",\n", dirty_purged); + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\"muzzy_npurge\": %"FMTu64",\n", muzzy_npurge); + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\"muzzy_nmadvise\": %"FMTu64",\n", muzzy_nmadvise); + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\"muzzy_purged\": %"FMTu64",\n", muzzy_purged); } else { malloc_cprintf(write_cb, cbopaque, - "purging: dirty: %zu, sweeps: %"FMTu64", madvises: %"FMTu64 - ", purged: %"FMTu64"\n", pdirty, npurge, nmadvise, purged); + "decaying: time npages sweeps madvises" + " purged\n"); + if (dirty_decay_time >= 0) { + malloc_cprintf(write_cb, cbopaque, + " dirty: %5zd %12zu %12"FMTu64" %12"FMTu64" %12" + FMTu64"\n", dirty_decay_time, pdirty, dirty_npurge, + dirty_nmadvise, dirty_purged); + } else { + malloc_cprintf(write_cb, cbopaque, + " dirty: N/A %12zu %12"FMTu64" %12"FMTu64" %12" + FMTu64"\n", pdirty, dirty_npurge, dirty_nmadvise, + dirty_purged); + } + if (muzzy_decay_time >= 0) { + malloc_cprintf(write_cb, cbopaque, + " muzzy: %5zd %12zu %12"FMTu64" %12"FMTu64" %12" + FMTu64"\n", muzzy_decay_time, pmuzzy, muzzy_npurge, + muzzy_nmadvise, muzzy_purged); + } else { + malloc_cprintf(write_cb, cbopaque, + " muzzy: N/A %12zu %12"FMTu64" %12"FMTu64" %12" + FMTu64"\n", pmuzzy, muzzy_npurge, muzzy_nmadvise, + muzzy_purged); + } } CTL_M2_GET("stats.arenas.0.small.allocated", i, &small_allocated, @@ -622,7 +654,10 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, OPT_WRITE_CHAR_P(dss, ",") OPT_WRITE_UNSIGNED(narenas, ",") OPT_WRITE_CHAR_P(percpu_arena, ",") - OPT_WRITE_SSIZE_T_MUTABLE(decay_time, arenas.decay_time, ",") + OPT_WRITE_SSIZE_T_MUTABLE(dirty_decay_time, arenas.dirty_decay_time, + ",") + OPT_WRITE_SSIZE_T_MUTABLE(muzzy_decay_time, arenas.muzzy_decay_time, + ",") OPT_WRITE_CHAR_P(junk, ",") OPT_WRITE_BOOL(zero, ",") OPT_WRITE_BOOL(utrace, ",") @@ -670,16 +705,26 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv); } - CTL_GET("arenas.decay_time", &ssv, ssize_t); + CTL_GET("arenas.dirty_decay_time", &ssv, ssize_t); if (json) { malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"decay_time\": %zd,\n", ssv); + "\t\t\t\"dirty_decay_time\": %zd,\n", ssv); } else { malloc_cprintf(write_cb, cbopaque, "Unused dirty page decay time: %zd%s\n", ssv, (ssv < 0) ? " (no decay)" : ""); } + CTL_GET("arenas.muzzy_decay_time", &ssv, ssize_t); + if (json) { + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\"muzzy_decay_time\": %zd,\n", ssv); + } else { + malloc_cprintf(write_cb, cbopaque, + "Unused muzzy page decay time: %zd%s\n", ssv, (ssv < 0) ? + " (no decay)" : ""); + } + CTL_GET("arenas.quantum", &sv, size_t); if (json) { malloc_cprintf(write_cb, cbopaque, diff --git a/test/unit/decay.c b/test/unit/decay.c index eb4df9d7..471a558c 100644 --- a/test/unit/decay.c +++ b/test/unit/decay.c @@ -22,18 +22,28 @@ nstime_update_mock(nstime_t *time) { } static unsigned -do_arena_create(ssize_t decay_time) { +do_arena_create(ssize_t dirty_decay_time, ssize_t muzzy_decay_time) { unsigned arena_ind; size_t sz = sizeof(unsigned); assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); size_t mib[3]; size_t miblen = sizeof(mib)/sizeof(size_t); - assert_d_eq(mallctlnametomib("arena.0.decay_time", mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); + + assert_d_eq(mallctlnametomib("arena.0.dirty_decay_time", mib, &miblen), + 0, "Unexpected mallctlnametomib() failure"); mib[1] = (size_t)arena_ind; - assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&decay_time, - sizeof(decay_time)), 0, "Unexpected mallctlbymib() failure"); + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, + (void *)&dirty_decay_time, + sizeof(dirty_decay_time)), 0, "Unexpected mallctlbymib() failure"); + + assert_d_eq(mallctlnametomib("arena.0.muzzy_decay_time", mib, &miblen), + 0, "Unexpected mallctlnametomib() failure"); + mib[1] = (size_t)arena_ind; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, + (void *)&muzzy_decay_time, + sizeof(muzzy_decay_time)), 0, "Unexpected mallctlbymib() failure"); + return arena_ind; } @@ -78,11 +88,10 @@ do_decay(unsigned arena_ind) { } static uint64_t -get_arena_npurge(unsigned arena_ind) { - do_epoch(); +get_arena_npurge_impl(const char *mibname, unsigned arena_ind) { size_t mib[4]; size_t miblen = sizeof(mib)/sizeof(size_t); - assert_d_eq(mallctlnametomib("stats.arenas.0.npurge", mib, &miblen), 0, + assert_d_eq(mallctlnametomib(mibname, mib, &miblen), 0, "Unexpected mallctlnametomib() failure"); mib[2] = (size_t)arena_ind; uint64_t npurge = 0; @@ -92,6 +101,25 @@ get_arena_npurge(unsigned arena_ind) { return npurge; } +static uint64_t +get_arena_dirty_npurge(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind); +} + +static uint64_t +get_arena_muzzy_npurge(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind); +} + +static uint64_t +get_arena_npurge(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) + + get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind); +} + static size_t get_arena_pdirty(unsigned arena_ind) { do_epoch(); @@ -107,6 +135,21 @@ get_arena_pdirty(unsigned arena_ind) { return pdirty; } +static size_t +get_arena_pmuzzy(unsigned arena_ind) { + do_epoch(); + size_t mib[4]; + size_t miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("stats.arenas.0.pmuzzy", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[2] = (size_t)arena_ind; + size_t pmuzzy; + size_t sz = sizeof(pmuzzy); + assert_d_eq(mallctlbymib(mib, miblen, (void *)&pmuzzy, &sz, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); + return pmuzzy; +} + static void * do_mallocx(size_t size, int flags) { void *p = mallocx(size, flags); @@ -133,7 +176,7 @@ TEST_BEGIN(test_decay_ticks) { int err; /* Set up a manually managed arena for test. */ - arena_ind = do_arena_create(0); + arena_ind = do_arena_create(0, 0); /* Migrate to the new arena, and get the ticker. */ unsigned old_arena_ind; @@ -317,19 +360,66 @@ TEST_BEGIN(test_decay_ticks) { } TEST_END -TEST_BEGIN(test_decay_ticker) { -#define NPS 1024 +static void +decay_ticker_helper(unsigned arena_ind, int flags, bool dirty, ssize_t dt, + uint64_t dirty_npurge0, uint64_t muzzy_npurge0, bool terminate_asap) { #define NINTERVALS 101 - ssize_t dt = opt_decay_time; - unsigned arena_ind = do_arena_create(dt); + nstime_t time, update_interval, decay_time, deadline; + + nstime_init(&time, 0); + nstime_update(&time); + + nstime_init2(&decay_time, dt, 0); + nstime_copy(&deadline, &time); + nstime_add(&deadline, &decay_time); + + nstime_init2(&update_interval, dt, 0); + nstime_idivide(&update_interval, NINTERVALS); + + /* + * Keep q's slab from being deallocated during the looping below. If a + * cached slab were to repeatedly come and go during looping, it could + * prevent the decay backlog ever becoming empty. + */ + void *p = do_mallocx(1, flags); + uint64_t dirty_npurge1, muzzy_npurge1; + do { + for (unsigned i = 0; i < DECAY_NTICKS_PER_UPDATE / 2; + i++) { + void *q = do_mallocx(1, flags); + dallocx(q, flags); + } + dirty_npurge1 = get_arena_dirty_npurge(arena_ind); + muzzy_npurge1 = get_arena_muzzy_npurge(arena_ind); + + nstime_add(&time_mock, &update_interval); + nstime_update(&time); + } while (nstime_compare(&time, &deadline) <= 0 && ((dirty_npurge1 == + dirty_npurge0 && muzzy_npurge1 == muzzy_npurge0) || + !terminate_asap)); + dallocx(p, flags); + + if (config_stats) { + assert_u64_gt(dirty_npurge1 + muzzy_npurge1, dirty_npurge0 + + muzzy_npurge0, "Expected purging to occur"); + } +#undef NINTERVALS +} + +TEST_BEGIN(test_decay_ticker) { +#define NPS 2048 + ssize_t ddt = opt_dirty_decay_time; + ssize_t mdt = opt_muzzy_decay_time; + unsigned arena_ind = do_arena_create(ddt, mdt); int flags = (MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE); void *ps[NPS]; size_t large; /* - * Allocate a bunch of large objects, pause the clock, deallocate the - * objects, restore the clock, then [md]allocx() in a tight loop while - * advancing time rapidly to verify the ticker triggers purging. + * Allocate a bunch of large objects, pause the clock, deallocate every + * other object (to fragment virtual memory), restore the clock, then + * [md]allocx() in a tight loop while advancing time rapidly to verify + * the ticker triggers purging. */ if (config_tcache) { @@ -346,7 +436,8 @@ TEST_BEGIN(test_decay_ticker) { } do_purge(arena_ind); - uint64_t npurge0 = get_arena_npurge(arena_ind); + uint64_t dirty_npurge0 = get_arena_dirty_npurge(arena_ind); + uint64_t muzzy_npurge0 = get_arena_muzzy_npurge(arena_ind); for (unsigned i = 0; i < NPS; i++) { ps[i] = do_mallocx(large, flags); @@ -362,7 +453,7 @@ TEST_BEGIN(test_decay_ticker) { nstime_monotonic = nstime_monotonic_mock; nstime_update = nstime_update_mock; - for (unsigned i = 0; i < NPS; i++) { + for (unsigned i = 0; i < NPS; i += 2) { dallocx(ps[i], flags); unsigned nupdates0 = nupdates_mock; do_decay(arena_ind); @@ -370,51 +461,16 @@ TEST_BEGIN(test_decay_ticker) { "Expected nstime_update() to be called"); } - nstime_t time, update_interval, decay_time, deadline; + decay_ticker_helper(arena_ind, flags, true, ddt, dirty_npurge0, + muzzy_npurge0, true); + decay_ticker_helper(arena_ind, flags, false, ddt+mdt, dirty_npurge0, + muzzy_npurge0, false); - nstime_init(&time, 0); - nstime_update(&time); - - nstime_init2(&decay_time, dt, 0); - nstime_copy(&deadline, &time); - nstime_add(&deadline, &decay_time); - - nstime_init2(&update_interval, dt, 0); - nstime_idivide(&update_interval, NINTERVALS); - - nstime_init2(&decay_time, dt, 0); - nstime_copy(&deadline, &time); - nstime_add(&deadline, &decay_time); - - /* - * Keep q's slab from being deallocated during the looping below. If - * a cached slab were to repeatedly come and go during looping, it could - * prevent the decay backlog ever becoming empty. - */ - void *p = do_mallocx(1, flags); - uint64_t npurge1; - do { - for (unsigned i = 0; i < DECAY_NTICKS_PER_UPDATE / 2; i++) { - void *q = do_mallocx(1, flags); - dallocx(q, flags); - } - npurge1 = get_arena_npurge(arena_ind); - - nstime_add(&time_mock, &update_interval); - nstime_update(&time); - } while (nstime_compare(&time, &deadline) <= 0 && npurge1 == npurge0); - dallocx(p, flags); + do_arena_destroy(arena_ind); nstime_monotonic = nstime_monotonic_orig; nstime_update = nstime_update_orig; - - if (config_stats) { - assert_u64_gt(npurge1, npurge0, "Expected purging to occur"); - } - - do_arena_destroy(arena_ind); #undef NPS -#undef NINTERVALS } TEST_END @@ -435,8 +491,7 @@ TEST_BEGIN(test_decay_nonmonotonic) { "Unexpected mallctl failure"); do_epoch(); sz = sizeof(uint64_t); - assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge0, &sz, - NULL, 0), config_stats ? 0 : ENOENT, "Unexpected mallctl result"); + npurge0 = get_arena_npurge(0); nupdates_mock = 0; nstime_init(&time_mock, 0); @@ -464,8 +519,7 @@ TEST_BEGIN(test_decay_nonmonotonic) { do_epoch(); sz = sizeof(uint64_t); - assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge1, &sz, - NULL, 0), config_stats ? 0 : ENOENT, "Unexpected mallctl result"); + npurge1 = get_arena_npurge(0); if (config_stats) { assert_u64_eq(npurge0, npurge1, "Unexpected purging occurred"); @@ -478,24 +532,28 @@ TEST_BEGIN(test_decay_nonmonotonic) { TEST_END TEST_BEGIN(test_decay_now) { - unsigned arena_ind = do_arena_create(0); + unsigned arena_ind = do_arena_create(0, 0); assert_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); + assert_zu_eq(get_arena_pmuzzy(arena_ind), 0, "Unexpected muzzy pages"); size_t sizes[] = {16, PAGE<<2, HUGEPAGE<<2}; - /* Verify that dirty pages never linger after deallocation. */ + /* Verify that dirty/muzzy pages never linger after deallocation. */ for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) { size_t size = sizes[i]; generate_dirty(arena_ind, size); assert_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); + assert_zu_eq(get_arena_pmuzzy(arena_ind), 0, + "Unexpected muzzy pages"); } do_arena_destroy(arena_ind); } TEST_END TEST_BEGIN(test_decay_never) { - unsigned arena_ind = do_arena_create(-1); + unsigned arena_ind = do_arena_create(-1, -1); int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; assert_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); + assert_zu_eq(get_arena_pmuzzy(arena_ind), 0, "Unexpected muzzy pages"); size_t sizes[] = {16, PAGE<<2, HUGEPAGE<<2}; void *ptrs[sizeof(sizes)/sizeof(size_t)]; for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) { @@ -503,12 +561,16 @@ TEST_BEGIN(test_decay_never) { } /* Verify that each deallocation generates additional dirty pages. */ size_t pdirty_prev = get_arena_pdirty(arena_ind); + size_t pmuzzy_prev = get_arena_pmuzzy(arena_ind); assert_zu_eq(pdirty_prev, 0, "Unexpected dirty pages"); + assert_zu_eq(pmuzzy_prev, 0, "Unexpected muzzy pages"); for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) { dallocx(ptrs[i], flags); size_t pdirty = get_arena_pdirty(arena_ind); + size_t pmuzzy = get_arena_pmuzzy(arena_ind); assert_zu_gt(pdirty, pdirty_prev, "Expected dirty pages to increase."); + assert_zu_eq(pmuzzy, 0, "Unexpected muzzy pages"); pdirty_prev = pdirty; } do_arena_destroy(arena_ind); diff --git a/test/unit/decay.sh b/test/unit/decay.sh index 284af815..0df17884 100644 --- a/test/unit/decay.sh +++ b/test/unit/decay.sh @@ -1,6 +1,6 @@ #!/bin/sh -export MALLOC_CONF="decay_time:1" +export MALLOC_CONF="dirty_decay_time:1,muzzy_decay_time:1" if [ "x${enable_tcache}" = "x1" ] ; then export MALLOC_CONF="${MALLOC_CONF},lg_tcache_max:0" fi diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 1aedbe8a..4241063e 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -161,7 +161,8 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(const char *, dss, always); TEST_MALLCTL_OPT(unsigned, narenas, always); TEST_MALLCTL_OPT(const char *, percpu_arena, always); - TEST_MALLCTL_OPT(ssize_t, decay_time, always); + TEST_MALLCTL_OPT(ssize_t, dirty_decay_time, always); + TEST_MALLCTL_OPT(ssize_t, muzzy_decay_time, always); TEST_MALLCTL_OPT(bool, stats_print, always); TEST_MALLCTL_OPT(const char *, junk, fill); TEST_MALLCTL_OPT(bool, zero, fill); @@ -401,32 +402,68 @@ TEST_BEGIN(test_arena_i_initialized) { } TEST_END -TEST_BEGIN(test_arena_i_decay_time) { - ssize_t decay_time, orig_decay_time, prev_decay_time; +TEST_BEGIN(test_arena_i_dirty_decay_time) { + ssize_t dirty_decay_time, orig_dirty_decay_time, prev_dirty_decay_time; size_t sz = sizeof(ssize_t); - assert_d_eq(mallctl("arena.0.decay_time", (void *)&orig_decay_time, &sz, - NULL, 0), 0, "Unexpected mallctl() failure"); - - decay_time = -2; - assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL, - (void *)&decay_time, sizeof(ssize_t)), EFAULT, - "Unexpected mallctl() success"); - - decay_time = 0x7fffffff; - assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL, - (void *)&decay_time, sizeof(ssize_t)), 0, + assert_d_eq(mallctl("arena.0.dirty_decay_time", + (void *)&orig_dirty_decay_time, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); - for (prev_decay_time = decay_time, decay_time = -1; - decay_time < 20; prev_decay_time = decay_time, decay_time++) { - ssize_t old_decay_time; + dirty_decay_time = -2; + assert_d_eq(mallctl("arena.0.dirty_decay_time", NULL, NULL, + (void *)&dirty_decay_time, sizeof(ssize_t)), EFAULT, + "Unexpected mallctl() success"); - assert_d_eq(mallctl("arena.0.decay_time", (void *)&old_decay_time, - &sz, (void *)&decay_time, sizeof(ssize_t)), 0, + dirty_decay_time = 0x7fffffff; + assert_d_eq(mallctl("arena.0.dirty_decay_time", NULL, NULL, + (void *)&dirty_decay_time, sizeof(ssize_t)), 0, + "Unexpected mallctl() failure"); + + for (prev_dirty_decay_time = dirty_decay_time, dirty_decay_time = -1; + dirty_decay_time < 20; prev_dirty_decay_time = dirty_decay_time, + dirty_decay_time++) { + ssize_t old_dirty_decay_time; + + assert_d_eq(mallctl("arena.0.dirty_decay_time", + (void *)&old_dirty_decay_time, &sz, + (void *)&dirty_decay_time, sizeof(ssize_t)), 0, "Unexpected mallctl() failure"); - assert_zd_eq(old_decay_time, prev_decay_time, - "Unexpected old arena.0.decay_time"); + assert_zd_eq(old_dirty_decay_time, prev_dirty_decay_time, + "Unexpected old arena.0.dirty_decay_time"); + } +} +TEST_END + +TEST_BEGIN(test_arena_i_muzzy_decay_time) { + ssize_t muzzy_decay_time, orig_muzzy_decay_time, prev_muzzy_decay_time; + size_t sz = sizeof(ssize_t); + + assert_d_eq(mallctl("arena.0.muzzy_decay_time", + (void *)&orig_muzzy_decay_time, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + + muzzy_decay_time = -2; + assert_d_eq(mallctl("arena.0.muzzy_decay_time", NULL, NULL, + (void *)&muzzy_decay_time, sizeof(ssize_t)), EFAULT, + "Unexpected mallctl() success"); + + muzzy_decay_time = 0x7fffffff; + assert_d_eq(mallctl("arena.0.muzzy_decay_time", NULL, NULL, + (void *)&muzzy_decay_time, sizeof(ssize_t)), 0, + "Unexpected mallctl() failure"); + + for (prev_muzzy_decay_time = muzzy_decay_time, muzzy_decay_time = -1; + muzzy_decay_time < 20; prev_muzzy_decay_time = muzzy_decay_time, + muzzy_decay_time++) { + ssize_t old_muzzy_decay_time; + + assert_d_eq(mallctl("arena.0.muzzy_decay_time", + (void *)&old_muzzy_decay_time, &sz, + (void *)&muzzy_decay_time, sizeof(ssize_t)), 0, + "Unexpected mallctl() failure"); + assert_zd_eq(old_muzzy_decay_time, prev_muzzy_decay_time, + "Unexpected old arena.0.muzzy_decay_time"); } } TEST_END @@ -522,32 +559,68 @@ TEST_BEGIN(test_arena_i_dss) { } TEST_END -TEST_BEGIN(test_arenas_decay_time) { - ssize_t decay_time, orig_decay_time, prev_decay_time; +TEST_BEGIN(test_arenas_dirty_decay_time) { + ssize_t dirty_decay_time, orig_dirty_decay_time, prev_dirty_decay_time; size_t sz = sizeof(ssize_t); - assert_d_eq(mallctl("arenas.decay_time", (void *)&orig_decay_time, &sz, - NULL, 0), 0, "Unexpected mallctl() failure"); + assert_d_eq(mallctl("arenas.dirty_decay_time", + (void *)&orig_dirty_decay_time, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); - decay_time = -2; - assert_d_eq(mallctl("arenas.decay_time", NULL, NULL, - (void *)&decay_time, sizeof(ssize_t)), EFAULT, + dirty_decay_time = -2; + assert_d_eq(mallctl("arenas.dirty_decay_time", NULL, NULL, + (void *)&dirty_decay_time, sizeof(ssize_t)), EFAULT, "Unexpected mallctl() success"); - decay_time = 0x7fffffff; - assert_d_eq(mallctl("arenas.decay_time", NULL, NULL, - (void *)&decay_time, sizeof(ssize_t)), 0, + dirty_decay_time = 0x7fffffff; + assert_d_eq(mallctl("arenas.dirty_decay_time", NULL, NULL, + (void *)&dirty_decay_time, sizeof(ssize_t)), 0, "Expected mallctl() failure"); - for (prev_decay_time = decay_time, decay_time = -1; - decay_time < 20; prev_decay_time = decay_time, decay_time++) { - ssize_t old_decay_time; + for (prev_dirty_decay_time = dirty_decay_time, dirty_decay_time = -1; + dirty_decay_time < 20; prev_dirty_decay_time = dirty_decay_time, + dirty_decay_time++) { + ssize_t old_dirty_decay_time; - assert_d_eq(mallctl("arenas.decay_time", - (void *)&old_decay_time, &sz, (void *)&decay_time, - sizeof(ssize_t)), 0, "Unexpected mallctl() failure"); - assert_zd_eq(old_decay_time, prev_decay_time, - "Unexpected old arenas.decay_time"); + assert_d_eq(mallctl("arenas.dirty_decay_time", + (void *)&old_dirty_decay_time, &sz, + (void *)&dirty_decay_time, sizeof(ssize_t)), 0, + "Unexpected mallctl() failure"); + assert_zd_eq(old_dirty_decay_time, prev_dirty_decay_time, + "Unexpected old arenas.dirty_decay_time"); + } +} +TEST_END + +TEST_BEGIN(test_arenas_muzzy_decay_time) { + ssize_t muzzy_decay_time, orig_muzzy_decay_time, prev_muzzy_decay_time; + size_t sz = sizeof(ssize_t); + + assert_d_eq(mallctl("arenas.muzzy_decay_time", + (void *)&orig_muzzy_decay_time, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + + muzzy_decay_time = -2; + assert_d_eq(mallctl("arenas.muzzy_decay_time", NULL, NULL, + (void *)&muzzy_decay_time, sizeof(ssize_t)), EFAULT, + "Unexpected mallctl() success"); + + muzzy_decay_time = 0x7fffffff; + assert_d_eq(mallctl("arenas.muzzy_decay_time", NULL, NULL, + (void *)&muzzy_decay_time, sizeof(ssize_t)), 0, + "Expected mallctl() failure"); + + for (prev_muzzy_decay_time = muzzy_decay_time, muzzy_decay_time = -1; + muzzy_decay_time < 20; prev_muzzy_decay_time = muzzy_decay_time, + muzzy_decay_time++) { + ssize_t old_muzzy_decay_time; + + assert_d_eq(mallctl("arenas.muzzy_decay_time", + (void *)&old_muzzy_decay_time, &sz, + (void *)&muzzy_decay_time, sizeof(ssize_t)), 0, + "Unexpected mallctl() failure"); + assert_zd_eq(old_muzzy_decay_time, prev_muzzy_decay_time, + "Unexpected old arenas.muzzy_decay_time"); } } TEST_END @@ -630,7 +703,8 @@ TEST_BEGIN(test_stats_arenas) { TEST_STATS_ARENAS(unsigned, nthreads); TEST_STATS_ARENAS(const char *, dss); - TEST_STATS_ARENAS(ssize_t, decay_time); + TEST_STATS_ARENAS(ssize_t, dirty_decay_time); + TEST_STATS_ARENAS(ssize_t, muzzy_decay_time); TEST_STATS_ARENAS(size_t, pactive); TEST_STATS_ARENAS(size_t, pdirty); @@ -653,11 +727,13 @@ main(void) { test_tcache, test_thread_arena, test_arena_i_initialized, - test_arena_i_decay_time, + test_arena_i_dirty_decay_time, + test_arena_i_muzzy_decay_time, test_arena_i_purge, test_arena_i_decay, test_arena_i_dss, - test_arenas_decay_time, + test_arenas_dirty_decay_time, + test_arenas_muzzy_decay_time, test_arenas_constants, test_arenas_bin_constants, test_arenas_lextent_constants, diff --git a/test/unit/pack.sh b/test/unit/pack.sh index de12e553..76757ac3 100644 --- a/test/unit/pack.sh +++ b/test/unit/pack.sh @@ -1,4 +1,4 @@ #!/bin/sh # Immediately purge to minimize fragmentation. -export MALLOC_CONF="decay_time:-1" +export MALLOC_CONF="dirty_decay_time:0,muzzy_decay_time:0" diff --git a/test/unit/stats.c b/test/unit/stats.c index c458d3f9..f8c6b104 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -71,7 +71,8 @@ TEST_BEGIN(test_stats_arenas_summary) { size_t sz; int expected = config_stats ? 0 : ENOENT; size_t mapped; - uint64_t npurge, nmadvise, purged; + uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; + uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; little = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); assert_ptr_not_null(little, "Unexpected mallocx() failure"); @@ -92,19 +93,34 @@ TEST_BEGIN(test_stats_arenas_summary) { sz = sizeof(size_t); assert_d_eq(mallctl("stats.arenas.0.mapped", (void *)&mapped, &sz, NULL, 0), expected, "Unexepected mallctl() result"); + sz = sizeof(uint64_t); - assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge, &sz, NULL, - 0), expected, "Unexepected mallctl() result"); - assert_d_eq(mallctl("stats.arenas.0.nmadvise", (void *)&nmadvise, &sz, - NULL, 0), expected, "Unexepected mallctl() result"); - assert_d_eq(mallctl("stats.arenas.0.purged", (void *)&purged, &sz, NULL, - 0), expected, "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.dirty_npurge", + (void *)&dirty_npurge, &sz, NULL, 0), expected, + "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.dirty_nmadvise", + (void *)&dirty_nmadvise, &sz, NULL, 0), expected, + "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.dirty_purged", + (void *)&dirty_purged, &sz, NULL, 0), expected, + "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.muzzy_npurge", + (void *)&muzzy_npurge, &sz, NULL, 0), expected, + "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.muzzy_nmadvise", + (void *)&muzzy_nmadvise, &sz, NULL, 0), expected, + "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.muzzy_purged", + (void *)&muzzy_purged, &sz, NULL, 0), expected, + "Unexepected mallctl() result"); if (config_stats) { - assert_u64_gt(npurge, 0, + assert_u64_gt(dirty_npurge + muzzy_npurge, 0, "At least one purge should have occurred"); - assert_u64_le(nmadvise, purged, - "nmadvise should be no greater than purged"); + assert_u64_le(dirty_nmadvise, dirty_purged, + "dirty_nmadvise should be no greater than dirty_purged"); + assert_u64_le(muzzy_nmadvise, muzzy_purged, + "muzzy_nmadvise should be no greater than muzzy_purged"); } } TEST_END