project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement two-phase decay-based purging.

Browse Source 
Split decay-based purging into two phases, the first of which uses lazy
purging to convert dirty pages to "muzzy", and the second of which uses
forced purging, decommit, or unmapping to convert pages to clean or
destroy them altogether.  Not all operating systems support lazy
purging, yet the application may provide extent hooks that implement
lazy purging, so care must be taken to dynamically omit the first phase
when necessary.

The mallctl interfaces change as follows:
- opt.decay_time --> opt.{dirty,muzzy}_decay_time
- arena.<i>.decay_time --> arena.<i>.{dirty,muzzy}_decay_time
- arenas.decay_time --> arenas.{dirty,muzzy}_decay_time
- stats.arenas.<i>.pdirty --> stats.arenas.<i>.p{dirty,muzzy}
- stats.arenas.<i>.{npurge,nmadvise,purged} -->
  stats.arenas.<i>.{dirty,muzzy}_{npurge,nmadvise,purged}

This resolves #521.
This commit is contained in:
Jason Evans 2017-03-08 22:42:57 -08:00
parent 38a5bfc816
commit 64e458f5cd
23 changed files with 1078 additions and 490 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile.in
View File

@ -442,8 +442,8 @@ ifeq ($(enable_prof), 1)
$(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) $(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
endif endif
check_integration_decay: tests_integration check_integration_dir check_integration_decay: tests_integration check_integration_dir
$(MALLOC_CONF)="decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) $(MALLOC_CONF)="dirty_decay_time:-1,muzzy_decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
$(MALLOC_CONF)="decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) $(MALLOC_CONF)="dirty_decay_time:0,muzzy_decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
check_integration: tests_integration check_integration_dir check_integration: tests_integration check_integration_dir
$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%) $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%)
stress: tests_stress stress_dir stress: tests_stress stress_dir

241
doc/jemalloc.xml.in
View File

@ -944,24 +944,54 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
<quote>percpu</quote>. </para></listitem> <quote>percpu</quote>. </para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="opt.decay_time"> <varlistentry id="opt.dirty_decay_time">
<term> <term>
<mallctl>opt.decay_time</mallctl> <mallctl>opt.dirty_decay_time</mallctl>
(<type>ssize_t</type>) (<type>ssize_t</type>)
<literal>r-</literal> <literal>r-</literal>
</term> </term>
<listitem><para>Approximate time in seconds from the creation of a set <listitem><para>Approximate time in seconds from the creation of a set
of unused dirty pages until an equivalent set of unused dirty pages is of unused dirty pages until an equivalent set of unused dirty pages is
purged and/or reused. The pages are incrementally purged according to a purged (i.e. converted to muzzy via e.g.
sigmoidal decay curve that starts and ends with zero purge rate. A <function>madvise(<parameter>...</parameter><parameter><constant>MADV_FREE</constant></parameter>)</function>
decay time of 0 causes all unused dirty pages to be purged immediately if supported by the operating system, or converted to clean otherwise)
upon creation. A decay time of -1 disables purging. The default decay and/or reused. Dirty pages are defined as previously having been
time is 10 seconds. See <link potentially written to by the application, and therefore consuming
linkend="arenas.decay_time"><mallctl>arenas.decay_time</mallctl></link> physical memory, yet having no current use. The pages are incrementally
purged according to a sigmoidal decay curve that starts and ends with
zero purge rate. A decay time of 0 causes all unused dirty pages to be
purged immediately upon creation. A decay time of -1 disables purging.
The default decay time is 10 seconds. See <link
linkend="arenas.dirty_decay_time"><mallctl>arenas.dirty_decay_time</mallctl></link>
and <link and <link
linkend="arena.i.decay_time"><mallctl>arena.&lt;i&gt;.decay_time</mallctl></link> linkend="arena.i.muzzy_decay_time"><mallctl>arena.&lt;i&gt;.muzzy_decay_time</mallctl></link>
for related dynamic control options. for related dynamic control options. See <link
</para></listitem> linkend="opt.muzzy_decay_time"><mallctl>opt.muzzy_decay_time</mallctl></link>
for a description of muzzy pages.</para></listitem>
</varlistentry>
<varlistentry id="opt.muzzy_decay_time">
<term>
<mallctl>opt.muzzy_decay_time</mallctl>
(<type>ssize_t</type>)
<literal>r-</literal>
</term>
<listitem><para>Approximate time in seconds from the creation of a set
of unused muzzy pages until an equivalent set of unused muzzy pages is
purged (i.e. converted to clean) and/or reused. Muzzy pages are defined
as previously having been unused dirty pages that were subsequently
purged in a manner that left them subject to the reclamation whims of
the operating system (e.g.
<function>madvise(<parameter>...</parameter><parameter><constant>MADV_FREE</constant></parameter>)</function>),
and therefore in an indeterminate state. The pages are incrementally
purged according to a sigmoidal decay curve that starts and ends with
zero purge rate. A decay time of 0 causes all unused muzzy pages to be
purged immediately upon creation. A decay time of -1 disables purging.
The default decay time is 10 seconds. See <link
linkend="arenas.muzzy_decay_time"><mallctl>arenas.muzzy_decay_time</mallctl></link>
and <link
linkend="arena.i.muzzy_decay_time"><mallctl>arena.&lt;i&gt;.muzzy_decay_time</mallctl></link>
for related dynamic control options.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="opt.stats_print"> <varlistentry id="opt.stats_print">
@ -1460,6 +1490,22 @@ malloc_conf = "xmalloc:true";]]></programlisting>
initialized (always true).</para></listitem> initialized (always true).</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arena.i.decay">
<term>
<mallctl>arena.&lt;i&gt;.decay</mallctl>
(<type>void</type>)
<literal>--</literal>
</term>
<listitem><para>Trigger decay-based purging of unused dirty/muzzy pages
for arena &lt;i&gt;, or for all arenas if &lt;i&gt; equals
<constant>MALLCTL_ARENAS_ALL</constant>. The proportion of unused
dirty/muzzy pages to be purged depends on the current time; see <link
linkend="opt.dirty_decay_time"><mallctl>opt.dirty_decay_time</mallctl></link>
and <link
linkend="opt.muzzy_decay_time"><mallctl>opt.muzy_decay_time</mallctl></link>
for details.</para></listitem>
</varlistentry>
<varlistentry id="arena.i.purge"> <varlistentry id="arena.i.purge">
<term> <term>
<mallctl>arena.&lt;i&gt;.purge</mallctl> <mallctl>arena.&lt;i&gt;.purge</mallctl>
@ -1471,20 +1517,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
</para></listitem> </para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arena.i.decay">
<term>
<mallctl>arena.&lt;i&gt;.decay</mallctl>
(<type>void</type>)
<literal>--</literal>
</term>
<listitem><para>Trigger decay-based purging of unused dirty pages for
arena &lt;i&gt;, or for all arenas if &lt;i&gt; equals
<constant>MALLCTL_ARENAS_ALL</constant>. The proportion of unused dirty
pages to be purged depends on the current time; see <link
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> for
details.</para></listitem>
</varlistentry>
<varlistentry id="arena.i.reset"> <varlistentry id="arena.i.reset">
<term> <term>
<mallctl>arena.&lt;i&gt;.reset</mallctl> <mallctl>arena.&lt;i&gt;.reset</mallctl>
@ -1532,9 +1564,9 @@ malloc_conf = "xmalloc:true";]]></programlisting>
settings.</para></listitem> settings.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arena.i.decay_time"> <varlistentry id="arena.i.dirty_decay_time">
<term> <term>
<mallctl>arena.&lt;i&gt;.decay_time</mallctl> <mallctl>arena.&lt;i&gt;.dirty_decay_time</mallctl>
(<type>ssize_t</type>) (<type>ssize_t</type>)
<literal>rw</literal> <literal>rw</literal>
</term> </term>
@ -1544,8 +1576,24 @@ malloc_conf = "xmalloc:true";]]></programlisting>
set, all currently unused dirty pages are considered to have fully set, all currently unused dirty pages are considered to have fully
decayed, which causes immediate purging of all unused dirty pages unless decayed, which causes immediate purging of all unused dirty pages unless
the decay time is set to -1 (i.e. purging disabled). See <link the decay time is set to -1 (i.e. purging disabled). See <link
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> for linkend="opt.dirty_decay_time"><mallctl>opt.dirty_decay_time</mallctl></link>
additional information.</para></listitem> for additional information.</para></listitem>
</varlistentry>
<varlistentry id="arena.i.muzzy_decay_time">
<term>
<mallctl>arena.&lt;i&gt;.muzzy_decay_time</mallctl>
(<type>ssize_t</type>)
<literal>rw</literal>
</term>
<listitem><para>Current per-arena approximate time in seconds from the
creation of a set of unused muzzy pages until an equivalent set of
unused muzzy pages is purged and/or reused. Each time this interface is
set, all currently unused muzzy pages are considered to have fully
decayed, which causes immediate purging of all unused muzzy pages unless
the decay time is set to -1 (i.e. purging disabled). See <link
linkend="opt.muzzy_decay_time"><mallctl>opt.muzzy_decay_time</mallctl></link>
for additional information.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arena.i.extent_hooks"> <varlistentry id="arena.i.extent_hooks">
@ -1584,7 +1632,7 @@ struct extent_hooks_s {
mapped committed memory, in the simplest case followed by deallocation. mapped committed memory, in the simplest case followed by deallocation.
However, there are performance and platform reasons to retain extents However, there are performance and platform reasons to retain extents
for later reuse. Cleanup attempts cascade from deallocation to decommit for later reuse. Cleanup attempts cascade from deallocation to decommit
to lazy purging to forced purging, which gives the extent management to forced purging to lazy purging, which gives the extent management
functions opportunities to reject the most permanent cleanup operations functions opportunities to reject the most permanent cleanup operations
in favor of less permanent (and often less costly) operations. All in favor of less permanent (and often less costly) operations. All
operations except allocation can be universally opted out of by setting operations except allocation can be universally opted out of by setting
@ -1707,12 +1755,14 @@ struct extent_hooks_s {
<parameter>addr</parameter> and <parameter>size</parameter> at <parameter>addr</parameter> and <parameter>size</parameter> at
<parameter>offset</parameter> bytes, extending for <parameter>offset</parameter> bytes, extending for
<parameter>length</parameter> on behalf of arena <parameter>length</parameter> on behalf of arena
<parameter>arena_ind</parameter>. A lazy extent purge function can <parameter>arena_ind</parameter>. A lazy extent purge function (e.g.
delay purging indefinitely and leave the pages within the purged virtual implemented via
memory range in an indeterminite state, whereas a forced extent purge <function>madvise(<parameter>...</parameter><parameter><constant>MADV_FREE</constant></parameter>)</function>)
function immediately purges, and the pages within the virtual memory can delay purging indefinitely and leave the pages within the purged
range will be zero-filled the next time they are accessed. If the virtual memory range in an indeterminite state, whereas a forced extent
function returns true, this indicates failure to purge.</para> purge function immediately purges, and the pages within the virtual
memory range will be zero-filled the next time they are accessed. If
the function returns true, this indicates failure to purge.</para>
<funcsynopsis><funcprototype> <funcsynopsis><funcprototype>
<funcdef>typedef bool <function>(extent_split_t)</function></funcdef> <funcdef>typedef bool <function>(extent_split_t)</function></funcdef>
@ -1769,19 +1819,34 @@ struct extent_hooks_s {
<listitem><para>Current limit on number of arenas.</para></listitem> <listitem><para>Current limit on number of arenas.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arenas.decay_time"> <varlistentry id="arenas.dirty_decay_time">
<term> <term>
<mallctl>arenas.decay_time</mallctl> <mallctl>arenas.dirty_decay_time</mallctl>
(<type>ssize_t</type>) (<type>ssize_t</type>)
<literal>rw</literal> <literal>rw</literal>
</term> </term>
<listitem><para>Current default per-arena approximate time in seconds <listitem><para>Current default per-arena approximate time in seconds
from the creation of a set of unused dirty pages until an equivalent set from the creation of a set of unused dirty pages until an equivalent set
of unused dirty pages is purged and/or reused, used to initialize <link of unused dirty pages is purged and/or reused, used to initialize <link
linkend="arena.i.decay_time"><mallctl>arena.&lt;i&gt;.decay_time</mallctl></link> linkend="arena.i.dirty_decay_time"><mallctl>arena.&lt;i&gt;.dirty_decay_time</mallctl></link>
during arena creation. See <link during arena creation. See <link
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> for linkend="opt.dirty_decay_time"><mallctl>opt.dirty_decay_time</mallctl></link>
additional information.</para></listitem> for additional information.</para></listitem>
</varlistentry>
<varlistentry id="arenas.muzzy_decay_time">
<term>
<mallctl>arenas.muzzy_decay_time</mallctl>
(<type>ssize_t</type>)
<literal>rw</literal>
</term>
<listitem><para>Current default per-arena approximate time in seconds
from the creation of a set of unused muzzy pages until an equivalent set
of unused muzzy pages is purged and/or reused, used to initialize <link
linkend="arena.i.muzzy_decay_time"><mallctl>arena.&lt;i&gt;.muzzy_decay_time</mallctl></link>
during arena creation. See <link
linkend="opt.muzzy_decay_time"><mallctl>opt.muzzy_decay_time</mallctl></link>
for additional information.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="arenas.quantum"> <varlistentry id="arenas.quantum">
@ -2014,7 +2079,9 @@ struct extent_hooks_s {
equal to <link equal to <link
linkend="stats.allocated"><mallctl>stats.allocated</mallctl></link>. linkend="stats.allocated"><mallctl>stats.allocated</mallctl></link>.
This does not include <link linkend="stats.arenas.i.pdirty"> This does not include <link linkend="stats.arenas.i.pdirty">
<mallctl>stats.arenas.&lt;i&gt;.pdirty</mallctl></link>, nor pages <mallctl>stats.arenas.&lt;i&gt;.pdirty</mallctl></link>,
<link linkend="stats.arenas.i.pmuzzy">
<mallctl>stats.arenas.&lt;i&gt;.pmuzzy</mallctl></link>, nor pages
entirely devoted to allocator metadata.</para></listitem> entirely devoted to allocator metadata.</para></listitem>
</varlistentry> </varlistentry>
@ -2099,16 +2166,29 @@ struct extent_hooks_s {
</para></listitem> </para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.decay_time"> <varlistentry id="stats.arenas.i.dirty_decay_time">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.decay_time</mallctl> <mallctl>stats.arenas.&lt;i&gt;.dirty_decay_time</mallctl>
(<type>ssize_t</type>) (<type>ssize_t</type>)
<literal>r-</literal> <literal>r-</literal>
</term> </term>
<listitem><para>Approximate time in seconds from the creation of a set <listitem><para>Approximate time in seconds from the creation of a set
of unused dirty pages until an equivalent set of unused dirty pages is of unused dirty pages until an equivalent set of unused dirty pages is
purged and/or reused. See <link purged and/or reused. See <link
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> linkend="opt.dirty_decay_time"><mallctl>opt.dirty_decay_time</mallctl></link>
for details.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.muzzy_decay_time">
<term>
<mallctl>stats.arenas.&lt;i&gt;.muzzy_decay_time</mallctl>
(<type>ssize_t</type>)
<literal>r-</literal>
</term>
<listitem><para>Approximate time in seconds from the creation of a set
of unused muzzy pages until an equivalent set of unused muzzy pages is
purged and/or reused. See <link
linkend="opt.muzzy_decay_time"><mallctl>opt.muzzy_decay_time</mallctl></link>
for details.</para></listitem> for details.</para></listitem>
</varlistentry> </varlistentry>
@ -2138,10 +2218,22 @@ struct extent_hooks_s {
<literal>r-</literal> <literal>r-</literal>
</term> </term>
<listitem><para>Number of pages within unused extents that are <listitem><para>Number of pages within unused extents that are
potentially dirty, and for which potentially dirty, and for which <function>madvise()</function> or
<function>madvise(<parameter>...</parameter> similar has not been called. See <link
<parameter><constant>MADV_DONTNEED</constant></parameter>)</function> or linkend="opt.dirty_decay_time"><mallctl>opt.dirty_decay_time</mallctl></link>
similar has not been called.</para></listitem> for a description of dirty pages.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.pmuzzy">
<term>
<mallctl>stats.arenas.&lt;i&gt;.pmuzzy</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
</term>
<listitem><para>Number of pages within unused extents that are muzzy.
See <link
linkend="opt.muzzy_decay_time"><mallctl>opt.muzzy_decay_time</mallctl></link>
for a description of muzzy pages.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.mapped"> <varlistentry id="stats.arenas.i.mapped">
@ -2207,9 +2299,9 @@ struct extent_hooks_s {
size.</para></listitem> size.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.npurge"> <varlistentry id="stats.arenas.i.dirty_npurge">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.npurge</mallctl> <mallctl>stats.arenas.&lt;i&gt;.dirty_npurge</mallctl>
(<type>uint64_t</type>) (<type>uint64_t</type>)
<literal>r-</literal> <literal>r-</literal>
[<option>--enable-stats</option>] [<option>--enable-stats</option>]
@ -2218,26 +2310,57 @@ struct extent_hooks_s {
</para></listitem> </para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.nmadvise"> <varlistentry id="stats.arenas.i.dirty_nmadvise">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.nmadvise</mallctl> <mallctl>stats.arenas.&lt;i&gt;.dirty_nmadvise</mallctl>
(<type>uint64_t</type>) (<type>uint64_t</type>)
<literal>r-</literal> <literal>r-</literal>
[<option>--enable-stats</option>] [<option>--enable-stats</option>]
</term> </term>
<listitem><para>Number of <function>madvise(<parameter>...</parameter> <listitem><para>Number of <function>madvise()</function> or similar
<parameter><constant>MADV_DONTNEED</constant></parameter>)</function> or calls made to purge dirty pages.</para></listitem>
similar calls made to purge dirty pages.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.purged"> <varlistentry id="stats.arenas.i.dirty_purged">
<term> <term>
<mallctl>stats.arenas.&lt;i&gt;.purged</mallctl> <mallctl>stats.arenas.&lt;i&gt;.dirty_purged</mallctl>
(<type>uint64_t</type>) (<type>uint64_t</type>)
<literal>r-</literal> <literal>r-</literal>
[<option>--enable-stats</option>] [<option>--enable-stats</option>]
</term> </term>
<listitem><para>Number of pages purged.</para></listitem> <listitem><para>Number of dirty pages purged.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.muzzy_npurge">
<term>
<mallctl>stats.arenas.&lt;i&gt;.muzzy_npurge</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Number of muzzy page purge sweeps performed.
</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.muzzy_nmadvise">
<term>
<mallctl>stats.arenas.&lt;i&gt;.muzzy_nmadvise</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Number of <function>madvise()</function> or similar
calls made to purge muzzy pages.</para></listitem>
</varlistentry>
<varlistentry id="stats.arenas.i.muzzy_purged">
<term>
<mallctl>stats.arenas.&lt;i&gt;.muzzy_purged</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option>]
</term>
<listitem><para>Number of muzzy pages purged.</para></listitem>
</varlistentry> </varlistentry>
<varlistentry id="stats.arenas.i.small.allocated"> <varlistentry id="stats.arenas.i.small.allocated">

31
include/jemalloc/internal/arena_externs.h
View File

@ -9,7 +9,8 @@ static const size_t large_pad =
#endif #endif
; ;
extern ssize_t opt_decay_time; extern ssize_t opt_dirty_decay_time;
extern ssize_t opt_muzzy_decay_time;
extern const arena_bin_info_t arena_bin_info[NBINS]; extern const arena_bin_info_t arena_bin_info[NBINS];
@ -22,13 +23,13 @@ void arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
void arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, void arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
size_t size); size_t size);
void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena,
unsigned *nthreads, const char **dss, ssize_t *decay_time, size_t *nactive, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_time,
size_t *ndirty); ssize_t *muzzy_decay_time, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty, const char **dss, ssize_t *dirty_decay_time, ssize_t *muzzy_decay_time,
arena_stats_t *astats, malloc_bin_stats_t *bstats, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
malloc_large_stats_t *lstats); malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats);
void arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent); extent_hooks_t **r_extent_hooks, extent_t *extent);
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr); size_t arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr);
@ -41,9 +42,13 @@ void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize); extent_t *extent, size_t oldsize);
void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena,
extent_t *extent, size_t oldsize); extent_t *extent, size_t oldsize);
ssize_t arena_decay_time_get(arena_t *arena); ssize_t arena_dirty_decay_time_get(arena_t *arena);
bool arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time); bool arena_dirty_decay_time_set(tsdn_t *tsdn, arena_t *arena,
void arena_purge(tsdn_t *tsdn, arena_t *arena, bool all); ssize_t decay_time);
ssize_t arena_muzzy_decay_time_get(arena_t *arena);
bool arena_muzzy_decay_time_set(tsdn_t *tsdn, arena_t *arena,
ssize_t decay_time);
void arena_decay(tsdn_t *tsdn, arena_t *arena, bool all);
void arena_reset(tsd_t *tsd, arena_t *arena); void arena_reset(tsd_t *tsd, arena_t *arena);
void arena_destroy(tsd_t *tsd, arena_t *arena); void arena_destroy(tsd_t *tsd, arena_t *arena);
void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena,
@ -74,8 +79,10 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache); size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache);
dss_prec_t arena_dss_prec_get(arena_t *arena); dss_prec_t arena_dss_prec_get(arena_t *arena);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
ssize_t arena_decay_time_default_get(void); ssize_t arena_dirty_decay_time_default_get(void);
bool arena_decay_time_default_set(ssize_t decay_time); bool arena_dirty_decay_time_default_set(ssize_t decay_time);
ssize_t arena_muzzy_decay_time_default_get(void);
bool arena_muzzy_decay_time_default_set(ssize_t decay_time);
unsigned arena_nthreads_get(arena_t *arena, bool internal); unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal);

5
include/jemalloc/internal/arena_inlines_b.h
View File

@ -75,13 +75,14 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
return; return;
} }
if (unlikely(ticker_ticks(decay_ticker, nticks))) { if (unlikely(ticker_ticks(decay_ticker, nticks))) {
arena_purge(tsdn, arena, false); arena_decay(tsdn, arena, false);
} }
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_assert_not_owner(tsdn, &arena->decay.mtx); malloc_mutex_assert_not_owner(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_assert_not_owner(tsdn, &arena->decay_muzzy.mtx);
arena_decay_ticks(tsdn, arena, 1); arena_decay_ticks(tsdn, arena, 1);
} }

29
include/jemalloc/internal/arena_structs_b.h
View File

@ -48,10 +48,8 @@ struct arena_decay_s {
* Approximate time in seconds from the creation of a set of unused * Approximate time in seconds from the creation of a set of unused
* dirty pages until an equivalent set of unused dirty pages is purged * dirty pages until an equivalent set of unused dirty pages is purged
* and/or reused. * and/or reused.
*
* Synchronization: atomic.
*/ */
ssize_t time; atomic_zd_t time;
/* time / SMOOTHSTEP_NSTEPS. */ /* time / SMOOTHSTEP_NSTEPS. */
nstime_t interval; nstime_t interval;
/* /*
@ -73,10 +71,10 @@ struct arena_decay_s {
*/ */
nstime_t deadline; nstime_t deadline;
/* /*
* Number of dirty pages at beginning of current epoch. During epoch * Number of unpurged pages at beginning of current epoch. During epoch
* advancement we use the delta between arena->decay.ndirty and * advancement we use the delta between arena->decay_*.nunpurged and
* extents_npages_get(&arena->extents_cached) to determine how many * extents_npages_get(&arena->extents_*) to determine how many dirty
* dirty pages, if any, were generated. * pages, if any, were generated.
*/ */
size_t nunpurged; size_t nunpurged;
/* /*
@ -86,6 +84,14 @@ struct arena_decay_s {
* relative to epoch. * relative to epoch.
*/ */
size_t backlog[SMOOTHSTEP_NSTEPS]; size_t backlog[SMOOTHSTEP_NSTEPS];
/*
* Pointer to associated stats. These stats are embedded directly in
* the arena's stats due to how stats structures are shared between the
* arena and ctl code.
*
* Synchronization: Same as associated arena's stats field. */
decay_stats_t *stats;
}; };
struct arena_bin_s { struct arena_bin_s {
@ -194,15 +200,18 @@ struct arena_s {
* *
* Synchronization: internal. * Synchronization: internal.
*/ */
extents_t extents_cached; extents_t extents_dirty;
extents_t extents_muzzy;
extents_t extents_retained; extents_t extents_retained;
/* /*
* Decay-based purging state. * Decay-based purging state, responsible for scheduling extent state
* transitions.
* *
* Synchronization: internal. * Synchronization: internal.
*/ */
arena_decay_t decay; arena_decay_t decay_dirty; /* dirty --> muzzy */
arena_decay_t decay_muzzy; /* muzzy --> retained */
/* /*
* Next extent size class in a growing series to use when satisfying a * Next extent size class in a growing series to use when satisfying a

5
include/jemalloc/internal/arena_types.h
View File

@ -7,8 +7,9 @@
#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN) #define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN)
#define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) #define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS)
/* Default decay time in seconds. */ /* Default decay times in seconds. */
#define DECAY_TIME_DEFAULT 10 #define DIRTY_DECAY_TIME_DEFAULT 10
#define MUZZY_DECAY_TIME_DEFAULT 10
/* Number of event ticks between time checks. */ /* Number of event ticks between time checks. */
#define DECAY_NTICKS_PER_UPDATE 1000 #define DECAY_NTICKS_PER_UPDATE 1000

4
include/jemalloc/internal/ctl_structs.h
View File

@ -51,9 +51,11 @@ struct ctl_arena_s {
/* Basic stats, supported even if !config_stats. */ /* Basic stats, supported even if !config_stats. */
unsigned nthreads; unsigned nthreads;
const char *dss; const char *dss;
ssize_t decay_time; ssize_t dirty_decay_time;
ssize_t muzzy_decay_time;
size_t pactive; size_t pactive;
size_t pdirty; size_t pdirty;
size_t pmuzzy;
/* NULL if !config_stats. */ /* NULL if !config_stats. */
ctl_arena_stats_t *astats; ctl_arena_stats_t *astats;

11
include/jemalloc/internal/extent_externs.h
View File

@ -21,20 +21,21 @@ bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
bool delay_coalesce); bool delay_coalesce);
extent_state_t extents_state_get(const extents_t *extents); extent_state_t extents_state_get(const extents_t *extents);
size_t extents_npages_get(extents_t *extents); size_t extents_npages_get(extents_t *extents);
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr,
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
bool slab);
void extents_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent);
extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena, extent_t *extents_evict(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_min); extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_min);
void extents_prefork(tsdn_t *tsdn, extents_t *extents); void extents_prefork(tsdn_t *tsdn, extents_t *extents);
void extents_postfork_parent(tsdn_t *tsdn, extents_t *extents); void extents_postfork_parent(tsdn_t *tsdn, extents_t *extents);
void extents_postfork_child(tsdn_t *tsdn, extents_t *extents); void extents_postfork_child(tsdn_t *tsdn, extents_t *extents);
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab);
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad, extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab); size_t alignment, bool *zero, bool *commit, bool slab);
void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent); void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
bool extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, bool extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent); extent_hooks_t **r_extent_hooks, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,

3
include/jemalloc/internal/extent_structs.h
View File

@ -4,7 +4,8 @@
typedef enum { typedef enum {
extent_state_active = 0, extent_state_active = 0,
extent_state_dirty = 1, extent_state_dirty = 1,
extent_state_retained = 2 extent_state_muzzy = 2,
extent_state_retained = 3
} extent_state_t; } extent_state_t;
/* Extent (span of pages). Use accessor functions for e_* fields. */ /* Extent (span of pages). Use accessor functions for e_* fields. */

22
include/jemalloc/internal/private_symbols.txt
View File

@ -15,21 +15,26 @@ arena_dalloc_bin_junked_locked
arena_dalloc_junk_small arena_dalloc_junk_small
arena_dalloc_promoted arena_dalloc_promoted
arena_dalloc_small arena_dalloc_small
arena_decay
arena_decay_tick arena_decay_tick
arena_decay_ticks arena_decay_ticks
arena_decay_time_default_get arena_dirty_decay_time_default_get
arena_decay_time_default_set arena_dirty_decay_time_default_set
arena_decay_time_get arena_dirty_decay_time_get
arena_decay_time_set arena_dirty_decay_time_set
arena_muzzy_decay_time_default_get
arena_muzzy_decay_time_default_set
arena_muzzy_decay_time_get
arena_muzzy_decay_time_set
arena_destroy arena_destroy
arena_dss_prec_get arena_dss_prec_get
arena_dss_prec_set arena_dss_prec_set
arena_extent_alloc_large arena_extent_alloc_large
arena_extent_cache_dalloc
arena_extent_dalloc_large_prep arena_extent_dalloc_large_prep
arena_extent_ralloc_large_expand arena_extent_ralloc_large_expand
arena_extent_ralloc_large_shrink arena_extent_ralloc_large_shrink
arena_extent_sn_next arena_extent_sn_next
arena_extents_dirty_dalloc
arena_get arena_get
arena_ichoose arena_ichoose
arena_ind_get arena_ind_get
@ -59,7 +64,6 @@ arena_prof_promote
arena_prof_tctx_get arena_prof_tctx_get
arena_prof_tctx_reset arena_prof_tctx_reset
arena_prof_tctx_set arena_prof_tctx_set
arena_purge
arena_ralloc arena_ralloc
arena_ralloc_no_move arena_ralloc_no_move
arena_reset arena_reset
@ -138,7 +142,6 @@ extent_commit_wrapper
extent_committed_get extent_committed_get
extent_committed_set extent_committed_set
extent_dalloc extent_dalloc
extent_dalloc_cache
extent_dalloc_gap extent_dalloc_gap
extent_dalloc_mmap extent_dalloc_mmap
extent_dalloc_wrapper extent_dalloc_wrapper
@ -192,6 +195,8 @@ extent_usize_get
extent_usize_set extent_usize_set
extent_zeroed_get extent_zeroed_get
extent_zeroed_set extent_zeroed_set
extents_alloc
extents_dalloc
extents_evict extents_evict
extents_init extents_init
extents_npages_get extents_npages_get
@ -299,7 +304,8 @@ nstime_sec
nstime_subtract nstime_subtract
nstime_update nstime_update
opt_abort opt_abort
opt_decay_time opt_dirty_decay_time
opt_muzzy_decay_time
opt_dss opt_dss
opt_junk opt_junk
opt_junk_alloc opt_junk_alloc

21
include/jemalloc/internal/stats_structs.h
View File

@ -77,6 +77,15 @@ struct malloc_large_stats_s {
size_t curlextents; /* Derived. */ size_t curlextents; /* Derived. */
}; };
struct decay_stats_s {
/* Total number of purge sweeps. */
arena_stats_u64_t npurge;
/* Total number of madvise calls made. */
arena_stats_u64_t nmadvise;
/* Total number of pages purged. */
arena_stats_u64_t purged;
};
/* /*
* Arena stats. Note that fields marked "derived" are not directly maintained * Arena stats. Note that fields marked "derived" are not directly maintained
* within the arena code; rather their values are derived during stats merge * within the arena code; rather their values are derived during stats merge
@ -84,7 +93,7 @@ struct malloc_large_stats_s {
*/ */
struct arena_stats_s { struct arena_stats_s {
#ifndef JEMALLOC_ATOMIC_U64 #ifndef JEMALLOC_ATOMIC_U64
malloc_mutex_t mtx; malloc_mutex_t mtx;
#endif #endif
/* Number of bytes currently mapped, excluding retained memory. */ /* Number of bytes currently mapped, excluding retained memory. */
@ -98,14 +107,8 @@ struct arena_stats_s {
*/ */
atomic_zu_t retained; /* Derived. */ atomic_zu_t retained; /* Derived. */
/* decay_stats_t decay_dirty;
* Total number of purge sweeps, total number of madvise calls made, decay_stats_t decay_muzzy;
* and total pages purged in order to keep dirty unused memory under
* control.
*/
arena_stats_u64_t npurge;
arena_stats_u64_t nmadvise;
arena_stats_u64_t purged;
atomic_zu_t base; /* Derived. */ atomic_zu_t base; /* Derived. */
atomic_zu_t internal; atomic_zu_t internal;

1
include/jemalloc/internal/stats_types.h
View File

@ -4,6 +4,7 @@
typedef struct tcache_bin_stats_s tcache_bin_stats_t; typedef struct tcache_bin_stats_s tcache_bin_stats_t;
typedef struct malloc_bin_stats_s malloc_bin_stats_t; typedef struct malloc_bin_stats_s malloc_bin_stats_t;
typedef struct malloc_large_stats_s malloc_large_stats_t; typedef struct malloc_large_stats_s malloc_large_stats_t;
typedef struct decay_stats_s decay_stats_t;
typedef struct arena_stats_s arena_stats_t; typedef struct arena_stats_s arena_stats_t;
#endif /* JEMALLOC_INTERNAL_STATS_TYPES_H */ #endif /* JEMALLOC_INTERNAL_STATS_TYPES_H */

386
src/arena.c
View File

@ -13,8 +13,10 @@ const char *percpu_arena_mode_names[] = {
const char *opt_percpu_arena = OPT_PERCPU_ARENA_DEFAULT; const char *opt_percpu_arena = OPT_PERCPU_ARENA_DEFAULT;
percpu_arena_mode_t percpu_arena_mode = PERCPU_ARENA_MODE_DEFAULT; percpu_arena_mode_t percpu_arena_mode = PERCPU_ARENA_MODE_DEFAULT;
ssize_t opt_decay_time = DECAY_TIME_DEFAULT; ssize_t opt_dirty_decay_time = DIRTY_DECAY_TIME_DEFAULT;
static ssize_t decay_time_default; ssize_t opt_muzzy_decay_time = MUZZY_DECAY_TIME_DEFAULT;
static ssize_t dirty_decay_time_default;
static ssize_t muzzy_decay_time_default;
const arena_bin_info_t arena_bin_info[NBINS] = { const arena_bin_info_t arena_bin_info[NBINS] = {
#define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \ #define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \
@ -37,12 +39,13 @@ const arena_bin_info_t arena_bin_info[NBINS] = {
* definition. * definition.
*/ */
static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena,
arena_decay_t *decay, extents_t *extents, size_t ndirty_limit); arena_decay_t *decay, extents_t *extents, bool all, size_t npages_limit);
static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, static void arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool all);
extent_t *slab, arena_bin_t *bin); static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin);
extent_t *slab, arena_bin_t *bin); static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
arena_bin_t *bin);
/******************************************************************************/ /******************************************************************************/
@ -50,7 +53,7 @@ static bool
arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) {
if (config_debug) { if (config_debug) {
for (size_t i = 0; i < sizeof(arena_stats_t); i++) { for (size_t i = 0; i < sizeof(arena_stats_t); i++) {
assert(((char *)arena_stats)[0] == 0); assert(((char *)arena_stats)[i] == 0);
} }
} }
#ifndef JEMALLOC_ATOMIC_U64 #ifndef JEMALLOC_ATOMIC_U64
@ -187,23 +190,27 @@ arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) {
void void
arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty) { const char **dss, ssize_t *dirty_decay_time, ssize_t *muzzy_decay_time,
size_t *nactive, size_t *ndirty,
size_t *nmuzzy) {
*nthreads += arena_nthreads_get(arena, false); *nthreads += arena_nthreads_get(arena, false);
*dss = dss_prec_names[arena_dss_prec_get(arena)]; *dss = dss_prec_names[arena_dss_prec_get(arena)];
*decay_time = arena_decay_time_get(arena); *dirty_decay_time = arena_dirty_decay_time_get(arena);
*muzzy_decay_time = arena_muzzy_decay_time_get(arena);
*nactive += atomic_read_zu(&arena->nactive); *nactive += atomic_read_zu(&arena->nactive);
*ndirty += extents_npages_get(&arena->extents_cached); *ndirty += extents_npages_get(&arena->extents_dirty);
*nmuzzy += extents_npages_get(&arena->extents_muzzy);
} }
void void
arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty, const char **dss, ssize_t *dirty_decay_time, ssize_t *muzzy_decay_time,
arena_stats_t *astats, malloc_bin_stats_t *bstats, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
malloc_large_stats_t *lstats) { malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats) {
cassert(config_stats); cassert(config_stats);
arena_basic_stats_merge(tsdn, arena, nthreads, dss, decay_time, arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_time,
nactive, ndirty); muzzy_decay_time, nactive, ndirty, nmuzzy);
size_t base_allocated, base_resident, base_mapped; size_t base_allocated, base_resident, base_mapped;
base_stats_get(tsdn, arena->base, &base_allocated, &base_resident, base_stats_get(tsdn, arena->base, &base_allocated, &base_resident,
@ -215,17 +222,33 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
+ arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped)); + arena_stats_read_zu(tsdn, &arena->stats, &arena->stats.mapped));
arena_stats_accum_zu(&astats->retained, arena_stats_accum_zu(&astats->retained,
extents_npages_get(&arena->extents_retained) << LG_PAGE); extents_npages_get(&arena->extents_retained) << LG_PAGE);
arena_stats_accum_u64(&astats->npurge, arena_stats_read_u64(tsdn,
&arena->stats, &arena->stats.npurge)); arena_stats_accum_u64(&astats->decay_dirty.npurge,
arena_stats_accum_u64(&astats->nmadvise, arena_stats_read_u64(tsdn, arena_stats_read_u64(tsdn, &arena->stats,
&arena->stats, &arena->stats.nmadvise)); &arena->stats.decay_dirty.npurge));
arena_stats_accum_u64(&astats->purged, arena_stats_read_u64(tsdn, arena_stats_accum_u64(&astats->decay_dirty.nmadvise,
&arena->stats, &arena->stats.purged)); arena_stats_read_u64(tsdn, &arena->stats,
&arena->stats.decay_dirty.nmadvise));
arena_stats_accum_u64(&astats->decay_dirty.purged,
arena_stats_read_u64(tsdn, &arena->stats,
&arena->stats.decay_dirty.purged));
arena_stats_accum_u64(&astats->decay_muzzy.npurge,
arena_stats_read_u64(tsdn, &arena->stats,
&arena->stats.decay_muzzy.npurge));
arena_stats_accum_u64(&astats->decay_muzzy.nmadvise,
arena_stats_read_u64(tsdn, &arena->stats,
&arena->stats.decay_muzzy.nmadvise));
arena_stats_accum_u64(&astats->decay_muzzy.purged,
arena_stats_read_u64(tsdn, &arena->stats,
&arena->stats.decay_muzzy.purged));
arena_stats_accum_zu(&astats->base, base_allocated); arena_stats_accum_zu(&astats->base, base_allocated);
arena_stats_accum_zu(&astats->internal, arena_internal_get(arena)); arena_stats_accum_zu(&astats->internal, arena_internal_get(arena));
arena_stats_accum_zu(&astats->resident, base_resident arena_stats_accum_zu(&astats->resident, base_resident +
+ (((atomic_read_zu(&arena->nactive) (((atomic_read_zu(&arena->nactive) +
+ extents_npages_get(&arena->extents_cached)) << LG_PAGE))); extents_npages_get(&arena->extents_dirty) +
extents_npages_get(&arena->extents_muzzy)) << LG_PAGE)));
for (szind_t i = 0; i < NSIZES - NBINS; i++) { for (szind_t i = 0; i < NSIZES - NBINS; i++) {
uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats,
@ -292,13 +315,14 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
} }
void void
arena_extent_cache_dalloc(tsdn_t *tsdn, arena_t *arena, arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent) { extent_hooks_t **r_extent_hooks, extent_t *extent) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
extent_dalloc_cache(tsdn, arena, r_extent_hooks, extent); extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty,
if (arena_decay_time_get(arena) == 0) { extent);
arena_purge(tsdn, arena, true); if (arena_dirty_decay_time_get(arena) == 0) {
arena_decay_dirty(tsdn, arena, true);
} }
} }
@ -432,8 +456,14 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t mapped_add; size_t mapped_add;
bool commit = true; bool commit = true;
extent = extent_alloc_cache(tsdn, arena, &extent_hooks, NULL, usize, extent = extents_alloc(tsdn, arena, &extent_hooks,
large_pad, alignment, zero, &commit, false); &arena->extents_dirty, NULL, usize, large_pad, alignment, zero,
&commit, false);
if (extent == NULL) {
extent = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_muzzy, NULL, usize, large_pad, alignment,
zero, &commit, false);
}
size_t size = usize + large_pad; size_t size = usize + large_pad;
if (extent == NULL) { if (extent == NULL) {
extent = extent_alloc_wrapper(tsdn, arena, &extent_hooks, NULL, extent = extent_alloc_wrapper(tsdn, arena, &extent_hooks, NULL,
@ -507,12 +537,12 @@ arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
static ssize_t static ssize_t
arena_decay_time_read(arena_decay_t *decay) { arena_decay_time_read(arena_decay_t *decay) {
return atomic_read_zd(&decay->time); return atomic_load_zd(&decay->time, ATOMIC_RELAXED);
} }
static void static void
arena_decay_time_write(arena_decay_t *decay, ssize_t decay_time) { arena_decay_time_write(arena_decay_t *decay, ssize_t decay_time) {
atomic_write_zd(&decay->time, decay_time); atomic_store_zd(&decay->time, decay_time, ATOMIC_RELAXED);
} }
static void static void
@ -621,10 +651,11 @@ arena_decay_epoch_advance_helper(arena_decay_t *decay, extents_t *extents,
static void static void
arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena,
arena_decay_t *decay, extents_t *extents) { arena_decay_t *decay, extents_t *extents) {
size_t ndirty_limit = arena_decay_backlog_npages_limit(decay); size_t npages_limit = arena_decay_backlog_npages_limit(decay);
if (extents_npages_get(extents) > ndirty_limit) { if (extents_npages_get(extents) > npages_limit) {
arena_purge_to_limit(tsdn, arena, decay, extents, ndirty_limit); arena_decay_to_limit(tsdn, arena, decay, extents, false,
npages_limit);
} }
/* /*
* There may be concurrent ndirty fluctuation between the purge above * There may be concurrent ndirty fluctuation between the purge above
@ -659,12 +690,22 @@ arena_decay_reinit(arena_decay_t *decay, extents_t *extents,
} }
static bool static bool
arena_decay_init(arena_decay_t *decay, extents_t *extents, ssize_t decay_time) { arena_decay_init(arena_decay_t *decay, extents_t *extents, ssize_t decay_time,
decay_stats_t *stats) {
if (config_debug) {
for (size_t i = 0; i < sizeof(arena_decay_t); i++) {
assert(((char *)decay)[i] == 0);
}
}
if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY)) { if (malloc_mutex_init(&decay->mtx, "decay", WITNESS_RANK_DECAY)) {
return true; return true;
} }
decay->purging = false; decay->purging = false;
arena_decay_reinit(decay, extents, decay_time); arena_decay_reinit(decay, extents, decay_time);
/* Memory is zeroed, so there is no need to clear stats. */
if (config_stats) {
decay->stats = stats;
}
return false; return false;
} }
@ -680,7 +721,7 @@ arena_decay_time_valid(ssize_t decay_time) {
} }
static void static void
arena_maybe_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
extents_t *extents) { extents_t *extents) {
malloc_mutex_assert_owner(tsdn, &decay->mtx); malloc_mutex_assert_owner(tsdn, &decay->mtx);
@ -688,7 +729,8 @@ arena_maybe_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
ssize_t decay_time = arena_decay_time_read(decay); ssize_t decay_time = arena_decay_time_read(decay);
if (decay_time <= 0) { if (decay_time <= 0) {
if (decay_time == 0) { if (decay_time == 0) {
arena_purge_to_limit(tsdn, arena, decay, extents, 0); arena_decay_to_limit(tsdn, arena, decay, extents, false,
0);
} }
return; return;
} }
@ -725,18 +767,29 @@ arena_maybe_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
} }
} }
ssize_t static ssize_t
arena_decay_time_get(arena_t *arena) { arena_decay_time_get(arena_decay_t *decay) {
return arena_decay_time_read(&arena->decay); return arena_decay_time_read(decay);
} }
bool ssize_t
arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) { arena_dirty_decay_time_get(arena_t *arena) {
return arena_decay_time_get(&arena->decay_dirty);
}
ssize_t
arena_muzzy_decay_time_get(arena_t *arena) {
return arena_decay_time_get(&arena->decay_muzzy);
}
static bool
arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
extents_t *extents, ssize_t decay_time) {
if (!arena_decay_time_valid(decay_time)) { if (!arena_decay_time_valid(decay_time)) {
return true; return true;
} }
malloc_mutex_lock(tsdn, &arena->decay.mtx); malloc_mutex_lock(tsdn, &decay->mtx);
/* /*
* Restart decay backlog from scratch, which may cause many dirty pages * Restart decay backlog from scratch, which may cause many dirty pages
* to be immediately purged. It would conceptually be possible to map * to be immediately purged. It would conceptually be possible to map
@ -745,58 +798,100 @@ arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) {
* infrequent, either between the {-1, 0, >0} states, or a one-time * infrequent, either between the {-1, 0, >0} states, or a one-time
* arbitrary change during initial arena configuration. * arbitrary change during initial arena configuration.
*/ */
arena_decay_reinit(&arena->decay, &arena->extents_cached, decay_time); arena_decay_reinit(decay, extents, decay_time);
arena_maybe_purge(tsdn, arena, &arena->decay, &arena->extents_cached); arena_maybe_decay(tsdn, arena, decay, extents);
malloc_mutex_unlock(tsdn, &arena->decay.mtx); malloc_mutex_unlock(tsdn, &decay->mtx);
return false; return false;
} }
bool
arena_dirty_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) {
return arena_decay_time_set(tsdn, arena, &arena->decay_dirty,
&arena->extents_dirty, decay_time);
}
bool
arena_muzzy_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time) {
return arena_decay_time_set(tsdn, arena, &arena->decay_muzzy,
&arena->extents_muzzy, decay_time);
}
static size_t static size_t
arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, arena_stash_decayed(tsdn_t *tsdn, arena_t *arena,
extents_t *extents, size_t ndirty_limit, extent_list_t *purge_extents) { extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit,
extent_list_t *decay_extents) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
/* Stash extents according to ndirty_limit. */ /* Stash extents according to npages_limit. */
size_t nstashed = 0; size_t nstashed = 0;
extent_t *extent; extent_t *extent;
while ((extent = extents_evict(tsdn, arena, r_extent_hooks, extents, while ((extent = extents_evict(tsdn, arena, r_extent_hooks, extents,
ndirty_limit)) != NULL) { npages_limit)) != NULL) {
extent_list_append(purge_extents, extent); extent_list_append(decay_extents, extent);
nstashed += extent_size_get(extent) >> LG_PAGE; nstashed += extent_size_get(extent) >> LG_PAGE;
} }
return nstashed; return nstashed;
} }
static size_t static size_t
arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, arena_decay_stashed(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_list_t *purge_extents) { extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents,
UNUSED size_t nmadvise; bool all, extent_list_t *decay_extents) {
UNUSED size_t nmadvise, nunmapped;
size_t npurged; size_t npurged;
if (config_stats) { if (config_stats) {
nmadvise = 0; nmadvise = 0;
nunmapped = 0;
} }
npurged = 0; npurged = 0;
for (extent_t *extent = extent_list_first(purge_extents); extent != ssize_t muzzy_decay_time = arena_muzzy_decay_time_get(arena);
NULL; extent = extent_list_first(purge_extents)) { for (extent_t *extent = extent_list_first(decay_extents); extent !=
NULL; extent = extent_list_first(decay_extents)) {
if (config_stats) { if (config_stats) {
nmadvise++; nmadvise++;
} }
npurged += extent_size_get(extent) >> LG_PAGE; size_t npages = extent_size_get(extent) >> LG_PAGE;
extent_list_remove(purge_extents, extent); npurged += npages;
extent_dalloc_wrapper(tsdn, arena, r_extent_hooks, extent); extent_list_remove(decay_extents, extent);
switch (extents_state_get(extents)) {
case extent_state_active:
not_reached();
case extent_state_dirty:
if (!all && muzzy_decay_time != 0 &&
!extent_purge_lazy_wrapper(tsdn, arena,
r_extent_hooks, extent, 0,
extent_size_get(extent))) {
extents_dalloc(tsdn, arena, r_extent_hooks,
&arena->extents_muzzy, extent);
break;
}
/* Fall through. */
case extent_state_muzzy:
extent_dalloc_wrapper(tsdn, arena, r_extent_hooks,
extent);
if (config_stats) {
nunmapped += npages;
}
break;
case extent_state_retained:
default:
not_reached();
}
} }
if (config_stats) { if (config_stats) {
arena_stats_lock(tsdn, &arena->stats); arena_stats_lock(tsdn, &arena->stats);
arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.nmadvise, arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->npurge,
nmadvise); 1);
arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.purged, arena_stats_add_u64(tsdn, &arena->stats,
&decay->stats->nmadvise, nmadvise);
arena_stats_add_u64(tsdn, &arena->stats, &decay->stats->purged,
npurged); npurged);
arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped,
npurged << LG_PAGE); nunmapped);
arena_stats_unlock(tsdn, &arena->stats); arena_stats_unlock(tsdn, &arena->stats);
} }
@ -804,12 +899,12 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena,
} }
/* /*
* ndirty_limit: Purge as many dirty extents as possible without violating the * npages_limit: Decay as many dirty extents as possible without violating the
* invariant: (extents_npages_get(extents) >= ndirty_limit) * invariant: (extents_npages_get(extents) >= npages_limit)
*/ */
static void static void
arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
extents_t *extents, size_t ndirty_limit) { extents_t *extents, bool all, size_t npages_limit) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1); witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1);
malloc_mutex_assert_owner(tsdn, &decay->mtx); malloc_mutex_assert_owner(tsdn, &decay->mtx);
@ -817,49 +912,53 @@ arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
return; return;
} }
decay->purging = true; decay->purging = true;
extent_hooks_t *extent_hooks = extent_hooks_get(arena);
size_t npurge, npurged;
extent_list_t purge_extents;
extent_list_init(&purge_extents);
malloc_mutex_unlock(tsdn, &decay->mtx); malloc_mutex_unlock(tsdn, &decay->mtx);
npurge = arena_stash_dirty(tsdn, arena, &extent_hooks, extents, extent_hooks_t *extent_hooks = extent_hooks_get(arena);
ndirty_limit, &purge_extents);
if (npurge == 0) { extent_list_t decay_extents;
malloc_mutex_lock(tsdn, &decay->mtx); extent_list_init(&decay_extents);
goto label_return;
size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents,
npages_limit, &decay_extents);
if (npurge != 0) {
UNUSED size_t npurged = arena_decay_stashed(tsdn, arena,
&extent_hooks, decay, extents, all, &decay_extents);
assert(npurged == npurge);
} }
npurged = arena_purge_stashed(tsdn, arena, &extent_hooks,
&purge_extents);
assert(npurged == npurge);
malloc_mutex_lock(tsdn, &decay->mtx); malloc_mutex_lock(tsdn, &decay->mtx);
if (config_stats) {
arena_stats_lock(tsdn, &arena->stats);
arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.npurge,
1);
arena_stats_unlock(tsdn, &arena->stats);
}
label_return:
decay->purging = false; decay->purging = false;
} }
void static void
arena_purge(tsdn_t *tsdn, arena_t *arena, bool all) { arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay,
malloc_mutex_lock(tsdn, &arena->decay.mtx); extents_t *extents, bool all) {
malloc_mutex_lock(tsdn, &decay->mtx);
if (all) { if (all) {
arena_purge_to_limit(tsdn, arena, &arena->decay, arena_decay_to_limit(tsdn, arena, decay, extents, all, 0);
&arena->extents_cached, 0);
} else { } else {
arena_maybe_purge(tsdn, arena, &arena->decay, arena_maybe_decay(tsdn, arena, decay, extents);
&arena->extents_cached);
} }
malloc_mutex_unlock(tsdn, &arena->decay.mtx); malloc_mutex_unlock(tsdn, &decay->mtx);
}
static void
arena_decay_dirty(tsdn_t *tsdn, arena_t *arena, bool all) {
arena_decay_impl(tsdn, arena, &arena->decay_dirty,
&arena->extents_dirty, all);
}
static void
arena_decay_muzzy(tsdn_t *tsdn, arena_t *arena, bool all) {
arena_decay_impl(tsdn, arena, &arena->decay_muzzy,
&arena->extents_muzzy, all);
}
void
arena_decay(tsdn_t *tsdn, arena_t *arena, bool all) {
arena_decay_dirty(tsdn, arena, all);
arena_decay_muzzy(tsdn, arena, all);
} }
static void static void
@ -867,7 +966,7 @@ arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) {
arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE); arena_nactive_sub(arena, extent_size_get(slab) >> LG_PAGE);
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
arena_extent_cache_dalloc(tsdn, arena, &extent_hooks, slab); arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, slab);
} }
static void static void
@ -1008,7 +1107,7 @@ arena_destroy(tsd_t *tsd, arena_t *arena) {
* Furthermore, the caller (arena_i_destroy_ctl()) purged all cached * Furthermore, the caller (arena_i_destroy_ctl()) purged all cached
* extents, so only retained extents may remain. * extents, so only retained extents may remain.
*/ */
assert(extents_npages_get(&arena->extents_cached) == 0); assert(extents_npages_get(&arena->extents_dirty) == 0);
/* Attempt to deallocate retained memory. */ /* Attempt to deallocate retained memory. */
arena_destroy_retained(tsd_tsdn(tsd), arena); arena_destroy_retained(tsd_tsdn(tsd), arena);
@ -1061,8 +1160,14 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
bool zero = false; bool zero = false;
bool commit = true; bool commit = true;
extent_t *slab = extent_alloc_cache(tsdn, arena, &extent_hooks, NULL, extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks,
bin_info->slab_size, 0, PAGE, &zero, &commit, true); &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, &zero,
&commit, true);
if (slab == NULL) {
slab = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE,
&zero, &commit, true);
}
if (slab == NULL) { if (slab == NULL) {
slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks, slab = arena_slab_alloc_hard(tsdn, arena, &extent_hooks,
bin_info); bin_info);
@ -1622,16 +1727,32 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) {
} }
ssize_t ssize_t
arena_decay_time_default_get(void) { arena_dirty_decay_time_default_get(void) {
return (ssize_t)atomic_read_zu((size_t *)&decay_time_default); return (ssize_t)atomic_read_zu((size_t *)&dirty_decay_time_default);
} }
bool bool
arena_decay_time_default_set(ssize_t decay_time) { arena_dirty_decay_time_default_set(ssize_t decay_time) {
if (!arena_decay_time_valid(decay_time)) { if (!arena_decay_time_valid(decay_time)) {
return true; return true;
} }
atomic_write_zu((size_t *)&decay_time_default, (size_t)decay_time); atomic_write_zu((size_t *)&dirty_decay_time_default,
(size_t)decay_time);
return false;
}
ssize_t
arena_muzzy_decay_time_default_get(void) {
return (ssize_t)atomic_read_zu((size_t *)&muzzy_decay_time_default);
}
bool
arena_muzzy_decay_time_default_set(ssize_t decay_time) {
if (!arena_decay_time_valid(decay_time)) {
return true;
}
atomic_write_zu((size_t *)&muzzy_decay_time_default,
(size_t)decay_time);
return false; return false;
} }
@ -1723,28 +1844,40 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
} }
/* /*
* Delay coalescing for cached extents despite the disruptive effect on * Delay coalescing for dirty extents despite the disruptive effect on
* memory layout for best-fit extent allocation, since cached extents * memory layout for best-fit extent allocation, since cached extents
* are likely to be reused soon after deallocation, and the cost of * are likely to be reused soon after deallocation, and the cost of
* merging/splitting extents is non-trivial. * merging/splitting extents is non-trivial.
*/ */
if (extents_init(tsdn, &arena->extents_cached, extent_state_dirty, if (extents_init(tsdn, &arena->extents_dirty, extent_state_dirty,
true)) { true)) {
goto label_error; goto label_error;
} }
/*
* Coalesce muzzy extents immediately, because operations on them are in
* the critical path much less often than for dirty extents.
*/
if (extents_init(tsdn, &arena->extents_muzzy, extent_state_muzzy,
false)) {
goto label_error;
}
/* /*
* Coalesce retained extents immediately, in part because they will * Coalesce retained extents immediately, in part because they will
* never be evicted (and therefore there's no opportunity for delayed * never be evicted (and therefore there's no opportunity for delayed
* coalescing), but also because operations on retained extents are not * coalescing), but also because operations on retained extents are not
* in the critical path. * in the critical path.
*/ */
if (extents_init(tsdn, &arena->extents_retained, if (extents_init(tsdn, &arena->extents_retained, extent_state_retained,
extent_state_retained, false)) { false)) {
goto label_error; goto label_error;
} }
if (arena_decay_init(&arena->decay, &arena->extents_cached, if (arena_decay_init(&arena->decay_dirty, &arena->extents_dirty,
arena_decay_time_default_get())) { arena_dirty_decay_time_default_get(), &arena->stats.decay_dirty)) {
goto label_error;
}
if (arena_decay_init(&arena->decay_muzzy, &arena->extents_muzzy,
arena_muzzy_decay_time_default_get(), &arena->stats.decay_muzzy)) {
goto label_error; goto label_error;
} }
@ -1785,12 +1918,14 @@ label_error:
void void
arena_boot(void) { arena_boot(void) {
arena_decay_time_default_set(opt_decay_time); arena_dirty_decay_time_default_set(opt_dirty_decay_time);
arena_muzzy_decay_time_default_set(opt_muzzy_decay_time);
} }
void void
arena_prefork0(tsdn_t *tsdn, arena_t *arena) { arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_prefork(tsdn, &arena->decay.mtx); malloc_mutex_prefork(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_prefork(tsdn, &arena->decay_muzzy.mtx);
} }
void void
@ -1802,7 +1937,8 @@ arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
void void
arena_prefork2(tsdn_t *tsdn, arena_t *arena) { arena_prefork2(tsdn_t *tsdn, arena_t *arena) {
extents_prefork(tsdn, &arena->extents_cached); extents_prefork(tsdn, &arena->extents_dirty);
extents_prefork(tsdn, &arena->extents_muzzy);
extents_prefork(tsdn, &arena->extents_retained); extents_prefork(tsdn, &arena->extents_retained);
} }
@ -1838,9 +1974,11 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
base_postfork_parent(tsdn, arena->base); base_postfork_parent(tsdn, arena->base);
malloc_mutex_postfork_parent(tsdn, &arena->extent_freelist_mtx); malloc_mutex_postfork_parent(tsdn, &arena->extent_freelist_mtx);
extents_postfork_parent(tsdn, &arena->extents_cached); extents_postfork_parent(tsdn, &arena->extents_dirty);
extents_postfork_parent(tsdn, &arena->extents_muzzy);
extents_postfork_parent(tsdn, &arena->extents_retained); extents_postfork_parent(tsdn, &arena->extents_retained);
malloc_mutex_postfork_parent(tsdn, &arena->decay.mtx); malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx);
if (config_stats && config_tcache) { if (config_stats && config_tcache) {
malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx); malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
} }
@ -1856,9 +1994,11 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_postfork_child(tsdn, &arena->large_mtx); malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
base_postfork_child(tsdn, arena->base); base_postfork_child(tsdn, arena->base);
malloc_mutex_postfork_child(tsdn, &arena->extent_freelist_mtx); malloc_mutex_postfork_child(tsdn, &arena->extent_freelist_mtx);
extents_postfork_child(tsdn, &arena->extents_cached); extents_postfork_child(tsdn, &arena->extents_dirty);
extents_postfork_child(tsdn, &arena->extents_muzzy);
extents_postfork_child(tsdn, &arena->extents_retained); extents_postfork_child(tsdn, &arena->extents_retained);
malloc_mutex_postfork_child(tsdn, &arena->decay.mtx); malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx);
if (config_stats && config_tcache) { if (config_stats && config_tcache) {
malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx); malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
} }

214
src/ctl.c
View File

@ -73,7 +73,8 @@ CTL_PROTO(opt_abort)
CTL_PROTO(opt_dss) CTL_PROTO(opt_dss)
CTL_PROTO(opt_narenas) CTL_PROTO(opt_narenas)
CTL_PROTO(opt_percpu_arena) CTL_PROTO(opt_percpu_arena)
CTL_PROTO(opt_decay_time) CTL_PROTO(opt_dirty_decay_time)
CTL_PROTO(opt_muzzy_decay_time)
CTL_PROTO(opt_stats_print) CTL_PROTO(opt_stats_print)
CTL_PROTO(opt_junk) CTL_PROTO(opt_junk)
CTL_PROTO(opt_zero) CTL_PROTO(opt_zero)
@ -95,12 +96,13 @@ CTL_PROTO(tcache_create)
CTL_PROTO(tcache_flush) CTL_PROTO(tcache_flush)
CTL_PROTO(tcache_destroy) CTL_PROTO(tcache_destroy)
CTL_PROTO(arena_i_initialized) CTL_PROTO(arena_i_initialized)
CTL_PROTO(arena_i_purge)
CTL_PROTO(arena_i_decay) CTL_PROTO(arena_i_decay)
CTL_PROTO(arena_i_purge)
CTL_PROTO(arena_i_reset) CTL_PROTO(arena_i_reset)
CTL_PROTO(arena_i_destroy) CTL_PROTO(arena_i_destroy)
CTL_PROTO(arena_i_dss) CTL_PROTO(arena_i_dss)
CTL_PROTO(arena_i_decay_time) CTL_PROTO(arena_i_dirty_decay_time)
CTL_PROTO(arena_i_muzzy_decay_time)
CTL_PROTO(arena_i_extent_hooks) CTL_PROTO(arena_i_extent_hooks)
INDEX_PROTO(arena_i) INDEX_PROTO(arena_i)
CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_size)
@ -110,7 +112,8 @@ INDEX_PROTO(arenas_bin_i)
CTL_PROTO(arenas_lextent_i_size) CTL_PROTO(arenas_lextent_i_size)
INDEX_PROTO(arenas_lextent_i) INDEX_PROTO(arenas_lextent_i)
CTL_PROTO(arenas_narenas) CTL_PROTO(arenas_narenas)
CTL_PROTO(arenas_decay_time) CTL_PROTO(arenas_dirty_decay_time)
CTL_PROTO(arenas_muzzy_decay_time)
CTL_PROTO(arenas_quantum) CTL_PROTO(arenas_quantum)
CTL_PROTO(arenas_page) CTL_PROTO(arenas_page)
CTL_PROTO(arenas_tcache_max) CTL_PROTO(arenas_tcache_max)
@ -150,14 +153,19 @@ CTL_PROTO(stats_arenas_i_lextents_j_curlextents)
INDEX_PROTO(stats_arenas_i_lextents_j) INDEX_PROTO(stats_arenas_i_lextents_j)
CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_nthreads)
CTL_PROTO(stats_arenas_i_dss) CTL_PROTO(stats_arenas_i_dss)
CTL_PROTO(stats_arenas_i_decay_time) CTL_PROTO(stats_arenas_i_dirty_decay_time)
CTL_PROTO(stats_arenas_i_muzzy_decay_time)
CTL_PROTO(stats_arenas_i_pactive) CTL_PROTO(stats_arenas_i_pactive)
CTL_PROTO(stats_arenas_i_pdirty) CTL_PROTO(stats_arenas_i_pdirty)
CTL_PROTO(stats_arenas_i_pmuzzy)
CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_mapped)
CTL_PROTO(stats_arenas_i_retained) CTL_PROTO(stats_arenas_i_retained)
CTL_PROTO(stats_arenas_i_npurge) CTL_PROTO(stats_arenas_i_dirty_npurge)
CTL_PROTO(stats_arenas_i_nmadvise) CTL_PROTO(stats_arenas_i_dirty_nmadvise)
CTL_PROTO(stats_arenas_i_purged) CTL_PROTO(stats_arenas_i_dirty_purged)
CTL_PROTO(stats_arenas_i_muzzy_npurge)
CTL_PROTO(stats_arenas_i_muzzy_nmadvise)
CTL_PROTO(stats_arenas_i_muzzy_purged)
CTL_PROTO(stats_arenas_i_base) CTL_PROTO(stats_arenas_i_base)
CTL_PROTO(stats_arenas_i_internal) CTL_PROTO(stats_arenas_i_internal)
CTL_PROTO(stats_arenas_i_tcache_bytes) CTL_PROTO(stats_arenas_i_tcache_bytes)
@ -231,7 +239,8 @@ static const ctl_named_node_t opt_node[] = {
{NAME("dss"), CTL(opt_dss)}, {NAME("dss"), CTL(opt_dss)},
{NAME("narenas"), CTL(opt_narenas)}, {NAME("narenas"), CTL(opt_narenas)},
{NAME("percpu_arena"), CTL(opt_percpu_arena)}, {NAME("percpu_arena"), CTL(opt_percpu_arena)},
{NAME("decay_time"), CTL(opt_decay_time)}, {NAME("dirty_decay_time"), CTL(opt_dirty_decay_time)},
{NAME("muzzy_decay_time"), CTL(opt_muzzy_decay_time)},
{NAME("stats_print"), CTL(opt_stats_print)}, {NAME("stats_print"), CTL(opt_stats_print)},
{NAME("junk"), CTL(opt_junk)}, {NAME("junk"), CTL(opt_junk)},
{NAME("zero"), CTL(opt_zero)}, {NAME("zero"), CTL(opt_zero)},
@ -259,12 +268,13 @@ static const ctl_named_node_t tcache_node[] = {
static const ctl_named_node_t arena_i_node[] = { static const ctl_named_node_t arena_i_node[] = {
{NAME("initialized"), CTL(arena_i_initialized)}, {NAME("initialized"), CTL(arena_i_initialized)},
{NAME("purge"), CTL(arena_i_purge)},
{NAME("decay"), CTL(arena_i_decay)}, {NAME("decay"), CTL(arena_i_decay)},
{NAME("purge"), CTL(arena_i_purge)},
{NAME("reset"), CTL(arena_i_reset)}, {NAME("reset"), CTL(arena_i_reset)},
{NAME("destroy"), CTL(arena_i_destroy)}, {NAME("destroy"), CTL(arena_i_destroy)},
{NAME("dss"), CTL(arena_i_dss)}, {NAME("dss"), CTL(arena_i_dss)},
{NAME("decay_time"), CTL(arena_i_decay_time)}, {NAME("dirty_decay_time"), CTL(arena_i_dirty_decay_time)},
{NAME("muzzy_decay_time"), CTL(arena_i_muzzy_decay_time)},
{NAME("extent_hooks"), CTL(arena_i_extent_hooks)} {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}
}; };
static const ctl_named_node_t super_arena_i_node[] = { static const ctl_named_node_t super_arena_i_node[] = {
@ -301,7 +311,8 @@ static const ctl_indexed_node_t arenas_lextent_node[] = {
static const ctl_named_node_t arenas_node[] = { static const ctl_named_node_t arenas_node[] = {
{NAME("narenas"), CTL(arenas_narenas)}, {NAME("narenas"), CTL(arenas_narenas)},
{NAME("decay_time"), CTL(arenas_decay_time)}, {NAME("dirty_decay_time"), CTL(arenas_dirty_decay_time)},
{NAME("muzzy_decay_time"), CTL(arenas_muzzy_decay_time)},
{NAME("quantum"), CTL(arenas_quantum)}, {NAME("quantum"), CTL(arenas_quantum)},
{NAME("page"), CTL(arenas_page)}, {NAME("page"), CTL(arenas_page)},
{NAME("tcache_max"), CTL(arenas_tcache_max)}, {NAME("tcache_max"), CTL(arenas_tcache_max)},
@ -373,14 +384,19 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
static const ctl_named_node_t stats_arenas_i_node[] = { static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, {NAME("nthreads"), CTL(stats_arenas_i_nthreads)},
{NAME("dss"), CTL(stats_arenas_i_dss)}, {NAME("dss"), CTL(stats_arenas_i_dss)},
{NAME("decay_time"), CTL(stats_arenas_i_decay_time)}, {NAME("dirty_decay_time"), CTL(stats_arenas_i_dirty_decay_time)},
{NAME("muzzy_decay_time"), CTL(stats_arenas_i_muzzy_decay_time)},
{NAME("pactive"), CTL(stats_arenas_i_pactive)}, {NAME("pactive"), CTL(stats_arenas_i_pactive)},
{NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, {NAME("pdirty"), CTL(stats_arenas_i_pdirty)},
{NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)},
{NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)},
{NAME("retained"), CTL(stats_arenas_i_retained)}, {NAME("retained"), CTL(stats_arenas_i_retained)},
{NAME("npurge"), CTL(stats_arenas_i_npurge)}, {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)},
{NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)},
{NAME("purged"), CTL(stats_arenas_i_purged)}, {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)},
{NAME("muzzy_npurge"), CTL(stats_arenas_i_muzzy_npurge)},
{NAME("muzzy_nmadvise"), CTL(stats_arenas_i_muzzy_nmadvise)},
{NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)},
{NAME("base"), CTL(stats_arenas_i_base)}, {NAME("base"), CTL(stats_arenas_i_base)},
{NAME("internal"), CTL(stats_arenas_i_internal)}, {NAME("internal"), CTL(stats_arenas_i_internal)},
{NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)},
@ -554,9 +570,11 @@ static void
ctl_arena_clear(ctl_arena_t *ctl_arena) { ctl_arena_clear(ctl_arena_t *ctl_arena) {
ctl_arena->nthreads = 0; ctl_arena->nthreads = 0;
ctl_arena->dss = dss_prec_names[dss_prec_limit]; ctl_arena->dss = dss_prec_names[dss_prec_limit];
ctl_arena->decay_time = -1; ctl_arena->dirty_decay_time = -1;
ctl_arena->muzzy_decay_time = -1;
ctl_arena->pactive = 0; ctl_arena->pactive = 0;
ctl_arena->pdirty = 0; ctl_arena->pdirty = 0;
ctl_arena->pmuzzy = 0;
if (config_stats) { if (config_stats) {
memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t)); memset(&ctl_arena->astats->astats, 0, sizeof(arena_stats_t));
ctl_arena->astats->allocated_small = 0; ctl_arena->astats->allocated_small = 0;
@ -576,8 +594,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
if (config_stats) { if (config_stats) {
arena_stats_merge(tsdn, arena, &ctl_arena->nthreads, arena_stats_merge(tsdn, arena, &ctl_arena->nthreads,
&ctl_arena->dss, &ctl_arena->decay_time, &ctl_arena->dss, &ctl_arena->dirty_decay_time,
&ctl_arena->pactive, &ctl_arena->pdirty, &ctl_arena->muzzy_decay_time, &ctl_arena->pactive,
&ctl_arena->pdirty, &ctl_arena->pmuzzy,
&ctl_arena->astats->astats, ctl_arena->astats->bstats, &ctl_arena->astats->astats, ctl_arena->astats->bstats,
ctl_arena->astats->lstats); ctl_arena->astats->lstats);
@ -594,8 +613,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
} }
} else { } else {
arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads,
&ctl_arena->dss, &ctl_arena->decay_time, &ctl_arena->dss, &ctl_arena->dirty_decay_time,
&ctl_arena->pactive, &ctl_arena->pdirty); &ctl_arena->muzzy_decay_time, &ctl_arena->pactive,
&ctl_arena->pdirty, &ctl_arena->pmuzzy);
} }
} }
@ -608,10 +628,12 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
ctl_sdarena->nthreads += ctl_arena->nthreads; ctl_sdarena->nthreads += ctl_arena->nthreads;
ctl_sdarena->pactive += ctl_arena->pactive; ctl_sdarena->pactive += ctl_arena->pactive;
ctl_sdarena->pdirty += ctl_arena->pdirty; ctl_sdarena->pdirty += ctl_arena->pdirty;
ctl_sdarena->pmuzzy += ctl_arena->pmuzzy;
} else { } else {
assert(ctl_arena->nthreads == 0); assert(ctl_arena->nthreads == 0);
assert(ctl_arena->pactive == 0); assert(ctl_arena->pactive == 0);
assert(ctl_arena->pdirty == 0); assert(ctl_arena->pdirty == 0);
assert(ctl_arena->pmuzzy == 0);
} }
if (config_stats) { if (config_stats) {
@ -624,12 +646,20 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena,
accum_atomic_zu(&sdstats->astats.retained, accum_atomic_zu(&sdstats->astats.retained,
&astats->astats.retained); &astats->astats.retained);
} }
accum_arena_stats_u64(&sdstats->astats.npurge,
&astats->astats.npurge); accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge,
accum_arena_stats_u64(&sdstats->astats.nmadvise, &astats->astats.decay_dirty.npurge);
&astats->astats.nmadvise); accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise,
accum_arena_stats_u64(&sdstats->astats.purged, &astats->astats.decay_dirty.nmadvise);
&astats->astats.purged); accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged,
&astats->astats.decay_dirty.purged);
accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge,
&astats->astats.decay_muzzy.npurge);
accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise,
&astats->astats.decay_muzzy.nmadvise);
accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged,
&astats->astats.decay_muzzy.purged);
if (!destroyed) { if (!destroyed) {
accum_atomic_zu(&sdstats->astats.base, accum_atomic_zu(&sdstats->astats.base,
@ -1340,7 +1370,8 @@ CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned)
CTL_RO_NL_GEN(opt_percpu_arena, opt_percpu_arena, const char *) CTL_RO_NL_GEN(opt_percpu_arena, opt_percpu_arena, const char *)
CTL_RO_NL_GEN(opt_decay_time, opt_decay_time, ssize_t) CTL_RO_NL_GEN(opt_dirty_decay_time, opt_dirty_decay_time, ssize_t)
CTL_RO_NL_GEN(opt_muzzy_decay_time, opt_muzzy_decay_time, ssize_t)
CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *)
CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
@ -1630,7 +1661,7 @@ label_return:
} }
static void static void
arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all) { arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) {
malloc_mutex_lock(tsdn, &ctl_mtx); malloc_mutex_lock(tsdn, &ctl_mtx);
{ {
unsigned narenas = ctl_arenas->narenas; unsigned narenas = ctl_arenas->narenas;
@ -1655,7 +1686,7 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all) {
for (i = 0; i < narenas; i++) { for (i = 0; i < narenas; i++) {
if (tarenas[i] != NULL) { if (tarenas[i] != NULL) {
arena_purge(tsdn, tarenas[i], all); arena_decay(tsdn, tarenas[i], all);
} }
} }
} else { } else {
@ -1669,28 +1700,12 @@ arena_i_purge(tsdn_t *tsdn, unsigned arena_ind, bool all) {
malloc_mutex_unlock(tsdn, &ctl_mtx); malloc_mutex_unlock(tsdn, &ctl_mtx);
if (tarena != NULL) { if (tarena != NULL) {
arena_purge(tsdn, tarena, all); arena_decay(tsdn, tarena, all);
} }
} }
} }
} }
static int
arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
READONLY();
WRITEONLY();
MIB_UNSIGNED(arena_ind, 1);
arena_i_purge(tsd_tsdn(tsd), arena_ind, true);
ret = 0;
label_return:
return ret;
}
static int static int
arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen) { size_t *oldlenp, void *newp, size_t newlen) {
@ -1700,7 +1715,23 @@ arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
READONLY(); READONLY();
WRITEONLY(); WRITEONLY();
MIB_UNSIGNED(arena_ind, 1); MIB_UNSIGNED(arena_ind, 1);
arena_i_purge(tsd_tsdn(tsd), arena_ind, false); arena_i_decay(tsd_tsdn(tsd), arena_ind, false);
ret = 0;
label_return:
return ret;
}
static int
arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen) {
int ret;
unsigned arena_ind;
READONLY();
WRITEONLY();
MIB_UNSIGNED(arena_ind, 1);
arena_i_decay(tsd_tsdn(tsd), arena_ind, true);
ret = 0; ret = 0;
label_return: label_return:
@ -1773,7 +1804,7 @@ arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
/* Merge stats after resetting and purging arena. */ /* Merge stats after resetting and purging arena. */
arena_reset(tsd, arena); arena_reset(tsd, arena);
arena_purge(tsd_tsdn(tsd), arena, true); arena_decay(tsd_tsdn(tsd), arena, true);
ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED); ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED);
ctl_darena->initialized = true; ctl_darena->initialized = true;
ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true); ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true);
@ -1852,8 +1883,8 @@ label_return:
} }
static int static int
arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, arena_i_decay_time_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
size_t *oldlenp, void *newp, size_t newlen) { void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
int ret; int ret;
unsigned arena_ind; unsigned arena_ind;
arena_t *arena; arena_t *arena;
@ -1866,7 +1897,8 @@ arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
} }
if (oldp != NULL && oldlenp != NULL) { if (oldp != NULL && oldlenp != NULL) {
size_t oldval = arena_decay_time_get(arena); size_t oldval = dirty ? arena_dirty_decay_time_get(arena) :
arena_muzzy_decay_time_get(arena);
READ(oldval, ssize_t); READ(oldval, ssize_t);
} }
if (newp != NULL) { if (newp != NULL) {
@ -1874,7 +1906,9 @@ arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = EINVAL; ret = EINVAL;
goto label_return; goto label_return;
} }
if (arena_decay_time_set(tsd_tsdn(tsd), arena, if (dirty ? arena_dirty_decay_time_set(tsd_tsdn(tsd), arena,
*(ssize_t *)newp) :
arena_muzzy_decay_time_set(tsd_tsdn(tsd), arena,
*(ssize_t *)newp)) { *(ssize_t *)newp)) {
ret = EFAULT; ret = EFAULT;
goto label_return; goto label_return;
@ -1886,6 +1920,20 @@ label_return:
return ret; return ret;
} }
static int
arena_i_dirty_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
return arena_i_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp,
newp, newlen, true);
}
static int
arena_i_muzzy_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
return arena_i_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp,
newp, newlen, false);
}
static int static int
arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) { void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
@ -1967,12 +2015,13 @@ label_return:
} }
static int static int
arenas_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, arenas_decay_time_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
size_t *oldlenp, void *newp, size_t newlen) { void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
int ret; int ret;
if (oldp != NULL && oldlenp != NULL) { if (oldp != NULL && oldlenp != NULL) {
size_t oldval = arena_decay_time_default_get(); size_t oldval = (dirty ? arena_dirty_decay_time_default_get() :
arena_muzzy_decay_time_default_get());
READ(oldval, ssize_t); READ(oldval, ssize_t);
} }
if (newp != NULL) { if (newp != NULL) {
@ -1980,7 +2029,8 @@ arenas_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = EINVAL; ret = EINVAL;
goto label_return; goto label_return;
} }
if (arena_decay_time_default_set(*(ssize_t *)newp)) { if (dirty ? arena_dirty_decay_time_default_set(*(ssize_t *)newp)
: arena_muzzy_decay_time_default_set(*(ssize_t *)newp)) {
ret = EFAULT; ret = EFAULT;
goto label_return; goto label_return;
} }
@ -1991,6 +2041,20 @@ label_return:
return ret; return ret;
} }
static int
arenas_dirty_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
return arenas_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
newlen, true);
}
static int
arenas_muzzy_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
return arenas_decay_time_ctl_impl(tsd, mib, miblen, oldp, oldlenp, newp,
newlen, false);
}
CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t)
@ -2182,23 +2246,41 @@ CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t) CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *) CTL_RO_GEN(stats_arenas_i_dss, arenas_i(mib[2])->dss, const char *)
CTL_RO_GEN(stats_arenas_i_decay_time, arenas_i(mib[2])->decay_time, CTL_RO_GEN(stats_arenas_i_dirty_decay_time, arenas_i(mib[2])->dirty_decay_time,
ssize_t)
CTL_RO_GEN(stats_arenas_i_muzzy_decay_time, arenas_i(mib[2])->muzzy_decay_time,
ssize_t) ssize_t)
CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned) CTL_RO_GEN(stats_arenas_i_nthreads, arenas_i(mib[2])->nthreads, unsigned)
CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t) CTL_RO_GEN(stats_arenas_i_pactive, arenas_i(mib[2])->pactive, size_t)
CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t) CTL_RO_GEN(stats_arenas_i_pdirty, arenas_i(mib[2])->pdirty, size_t)
CTL_RO_GEN(stats_arenas_i_pmuzzy, arenas_i(mib[2])->pmuzzy, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
atomic_load_zu(&arenas_i(mib[2])->astats->astats.mapped, ATOMIC_RELAXED), atomic_load_zu(&arenas_i(mib[2])->astats->astats.mapped, ATOMIC_RELAXED),
size_t) size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_retained, CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED),
size_t) size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_npurge,
arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.npurge), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge,
CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.npurge),
arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.nmadvise), uint64_t) uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_purged, CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise,
arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.purged), uint64_t) arena_stats_read_u64(
&arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged,
arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.purged),
uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge,
arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.npurge),
uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise,
arena_stats_read_u64(
&arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged,
arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.purged),
uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_base, CTL_RO_CGEN(config_stats, stats_arenas_i_base,
atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED), atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED),
size_t) size_t)
@ -2268,8 +2350,8 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc,
arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc),
uint64_t) uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests,
arena_stats_read_u64( arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].nrequests),
&arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t) uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents,
arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t)

73
src/extent.c
View File

@ -69,6 +69,10 @@ static size_t highpages;
*/ */
static void extent_deregister(tsdn_t *tsdn, extent_t *extent); static void extent_deregister(tsdn_t *tsdn, extent_t *extent);
static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr,
size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit,
bool slab);
static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents,
extent_t *extent, bool *coalesced); extent_t *extent, bool *coalesced);
@ -293,6 +297,31 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena,
return false; return false;
} }
extent_t *
extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extents_t *extents, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab) {
assert(usize + pad != 0);
assert(alignment != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
return extent_recycle(tsdn, arena, r_extent_hooks, extents, new_addr,
usize, pad, alignment, zero, commit, slab);
}
void
extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extents_t *extents, extent_t *extent) {
assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false);
extent_record(tsdn, arena, r_extent_hooks, extents, extent);
}
extent_t * extent_t *
extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
extents_t *extents, size_t npages_min) { extents_t *extents, size_t npages_min) {
@ -340,7 +369,10 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
* concurrent operations. * concurrent operations.
*/ */
switch (extents_state_get(extents)) { switch (extents_state_get(extents)) {
case extent_state_active:
not_reached();
case extent_state_dirty: case extent_state_dirty:
case extent_state_muzzy:
extent_state_set(extent, extent_state_active); extent_state_set(extent, extent_state_active);
break; break;
case extent_state_retained: case extent_state_retained:
@ -813,19 +845,6 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
return NULL; return NULL;
} }
extent_t *
extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab) {
assert(usize + pad != 0);
assert(alignment != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
return extent_recycle(tsdn, arena, r_extent_hooks,
&arena->extents_cached, new_addr, usize, pad, alignment, zero,
commit, slab);
}
static void * static void *
extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit) { size_t size, size_t alignment, bool *zero, bool *commit) {
@ -1206,7 +1225,8 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
assert(extents_state_get(extents) != extent_state_dirty || assert((extents_state_get(extents) != extent_state_dirty &&
extents_state_get(extents) != extent_state_muzzy) ||
!extent_zeroed_get(extent)); !extent_zeroed_get(extent));
malloc_mutex_lock(tsdn, &extents->mtx); malloc_mutex_lock(tsdn, &extents->mtx);
@ -1244,20 +1264,6 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent);
} }
void
extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent) {
assert(extent_base_get(extent) != NULL);
assert(extent_size_get(extent) != 0);
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
extent_addr_set(extent, extent_base_get(extent));
extent_zeroed_set(extent, false);
extent_record(tsdn, arena, r_extent_hooks, &arena->extents_cached,
extent);
}
static bool static bool
extent_dalloc_default_impl(void *addr, size_t size) { extent_dalloc_default_impl(void *addr, size_t size) {
if (!have_dss || !extent_in_dss(addr)) { if (!have_dss || !extent_in_dss(addr)) {
@ -1327,16 +1333,17 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
} else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent, } else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
0, extent_size_get(extent))) { 0, extent_size_get(extent))) {
zeroed = true; zeroed = true;
} else if ((*r_extent_hooks)->purge_lazy != NULL &&
!(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena_ind_get(arena))) {
zeroed = false;
} else if ((*r_extent_hooks)->purge_forced != NULL && } else if ((*r_extent_hooks)->purge_forced != NULL &&
!(*r_extent_hooks)->purge_forced(*r_extent_hooks, !(*r_extent_hooks)->purge_forced(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0, extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena_ind_get(arena))) { extent_size_get(extent), arena_ind_get(arena))) {
zeroed = true; zeroed = true;
} else if (extent_state_get(extent) == extent_state_muzzy ||
((*r_extent_hooks)->purge_lazy != NULL &&
!(*r_extent_hooks)->purge_lazy(*r_extent_hooks,
extent_base_get(extent), extent_size_get(extent), 0,
extent_size_get(extent), arena_ind_get(arena)))) {
zeroed = false;
} else { } else {
zeroed = false; zeroed = false;
} }

6
src/jemalloc.c
View File

@ -1064,8 +1064,10 @@ malloc_conf_init(void) {
} }
CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
UINT_MAX, yes, no, false) UINT_MAX, yes, no, false)
CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1, CONF_HANDLE_SSIZE_T(opt_dirty_decay_time,
NSTIME_SEC_MAX); "dirty_decay_time", -1, NSTIME_SEC_MAX);
CONF_HANDLE_SSIZE_T(opt_muzzy_decay_time,
"muzzy_decay_time", -1, NSTIME_SEC_MAX);
CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true)
if (config_fill) { if (config_fill) {
if (CONF_MATCH("junk")) { if (CONF_MATCH("junk")) {

21
src/large.c
View File

@ -125,7 +125,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {
extent_usize_get(trail)); extent_usize_get(trail));
} }
arena_extent_cache_dalloc(tsdn, arena, &extent_hooks, trail); arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, trail);
} }
arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize); arena_extent_ralloc_large_shrink(tsdn, arena, extent, oldusize);
@ -158,9 +158,16 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
bool commit = true; bool commit = true;
extent_t *trail; extent_t *trail;
bool new_mapping; bool new_mapping;
if ((trail = extent_alloc_cache(tsdn, arena, &extent_hooks, if ((trail = extents_alloc(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, 0, CACHELINE, &is_zeroed_trail, &arena->extents_dirty, extent_past_get(extent), trailsize, 0,
&commit, false)) == NULL) { CACHELINE, &is_zeroed_trail, &commit, false)) != NULL
|| (trail = extents_alloc(tsdn, arena, &extent_hooks,
&arena->extents_muzzy, extent_past_get(extent), trailsize, 0,
CACHELINE, &is_zeroed_trail, &commit, false)) != NULL) {
if (config_stats) {
new_mapping = false;
}
} else {
if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks,
extent_past_get(extent), trailsize, 0, CACHELINE, extent_past_get(extent), trailsize, 0, CACHELINE,
&is_zeroed_trail, &commit, false)) == NULL) { &is_zeroed_trail, &commit, false)) == NULL) {
@ -169,10 +176,6 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
if (config_stats) { if (config_stats) {
new_mapping = true; new_mapping = true;
} }
} else {
if (config_stats) {
new_mapping = false;
}
} }
if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) { if (extent_merge_wrapper(tsdn, arena, &extent_hooks, extent, trail)) {
@ -327,7 +330,7 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
static void static void
large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER;
arena_extent_cache_dalloc(tsdn, arena, &extent_hooks, extent); arena_extents_dirty_dalloc(tsdn, arena, &extent_hooks, extent);
} }
void void

99
src/stats.c
View File

@ -259,10 +259,11 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, unsigned i, bool bins, bool large) { bool json, unsigned i, bool bins, bool large) {
unsigned nthreads; unsigned nthreads;
const char *dss; const char *dss;
ssize_t decay_time; ssize_t dirty_decay_time, muzzy_decay_time;
size_t page, pactive, pdirty, mapped, retained; size_t page, pactive, pdirty, pmuzzy, mapped, retained;
size_t base, internal, resident; size_t base, internal, resident;
uint64_t npurge, nmadvise, purged; uint64_t dirty_npurge, dirty_nmadvise, dirty_purged;
uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged;
size_t small_allocated; size_t small_allocated;
uint64_t small_nmalloc, small_ndalloc, small_nrequests; uint64_t small_nmalloc, small_ndalloc, small_nrequests;
size_t large_allocated; size_t large_allocated;
@ -289,39 +290,70 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
"dss allocation precedence: %s\n", dss); "dss allocation precedence: %s\n", dss);
} }
CTL_M2_GET("stats.arenas.0.decay_time", i, &decay_time, ssize_t); CTL_M2_GET("stats.arenas.0.dirty_decay_time", i, &dirty_decay_time,
if (json) { ssize_t);
malloc_cprintf(write_cb, cbopaque, CTL_M2_GET("stats.arenas.0.muzzy_decay_time", i, &muzzy_decay_time,
"\t\t\t\t\"decay_time\": %zd,\n", decay_time); ssize_t);
} else {
if (decay_time >= 0) {
malloc_cprintf(write_cb, cbopaque, "decay time: %zd\n",
decay_time);
} else {
malloc_cprintf(write_cb, cbopaque, "decay time: N/A\n");
}
}
CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t); CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t);
CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t); CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t);
CTL_M2_GET("stats.arenas.0.npurge", i, &npurge, uint64_t); CTL_M2_GET("stats.arenas.0.pmuzzy", i, &pmuzzy, size_t);
CTL_M2_GET("stats.arenas.0.nmadvise", i, &nmadvise, uint64_t); CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &dirty_npurge, uint64_t);
CTL_M2_GET("stats.arenas.0.purged", i, &purged, uint64_t); CTL_M2_GET("stats.arenas.0.dirty_nmadvise", i, &dirty_nmadvise,
uint64_t);
CTL_M2_GET("stats.arenas.0.dirty_purged", i, &dirty_purged, uint64_t);
CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &muzzy_npurge, uint64_t);
CTL_M2_GET("stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise,
uint64_t);
CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &muzzy_purged, uint64_t);
if (json) { if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"dirty_decay_time\": %zd,\n", dirty_decay_time);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"muzzy_decay_time\": %zd,\n", muzzy_decay_time);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"pactive\": %zu,\n", pactive); "\t\t\t\t\"pactive\": %zu,\n", pactive);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"pdirty\": %zu,\n", pdirty); "\t\t\t\t\"pdirty\": %zu,\n", pdirty);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"npurge\": %"FMTu64",\n", npurge); "\t\t\t\t\"pmuzzy\": %zu,\n", pmuzzy);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"nmadvise\": %"FMTu64",\n", nmadvise); "\t\t\t\t\"dirty_npurge\": %"FMTu64",\n", dirty_npurge);
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"purged\": %"FMTu64",\n", purged); "\t\t\t\t\"dirty_nmadvise\": %"FMTu64",\n", dirty_nmadvise);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"dirty_purged\": %"FMTu64",\n", dirty_purged);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"muzzy_npurge\": %"FMTu64",\n", muzzy_npurge);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"muzzy_nmadvise\": %"FMTu64",\n", muzzy_nmadvise);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"muzzy_purged\": %"FMTu64",\n", muzzy_purged);
} else { } else {
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"purging: dirty: %zu, sweeps: %"FMTu64", madvises: %"FMTu64 "decaying: time npages sweeps madvises"
", purged: %"FMTu64"\n", pdirty, npurge, nmadvise, purged); " purged\n");
if (dirty_decay_time >= 0) {
malloc_cprintf(write_cb, cbopaque,
" dirty: %5zd %12zu %12"FMTu64" %12"FMTu64" %12"
FMTu64"\n", dirty_decay_time, pdirty, dirty_npurge,
dirty_nmadvise, dirty_purged);
} else {
malloc_cprintf(write_cb, cbopaque,
" dirty: N/A %12zu %12"FMTu64" %12"FMTu64" %12"
FMTu64"\n", pdirty, dirty_npurge, dirty_nmadvise,
dirty_purged);
}
if (muzzy_decay_time >= 0) {
malloc_cprintf(write_cb, cbopaque,
" muzzy: %5zd %12zu %12"FMTu64" %12"FMTu64" %12"
FMTu64"\n", muzzy_decay_time, pmuzzy, muzzy_npurge,
muzzy_nmadvise, muzzy_purged);
} else {
malloc_cprintf(write_cb, cbopaque,
" muzzy: N/A %12zu %12"FMTu64" %12"FMTu64" %12"
FMTu64"\n", pmuzzy, muzzy_npurge, muzzy_nmadvise,
muzzy_purged);
}
} }
CTL_M2_GET("stats.arenas.0.small.allocated", i, &small_allocated, CTL_M2_GET("stats.arenas.0.small.allocated", i, &small_allocated,
@ -622,7 +654,10 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
OPT_WRITE_CHAR_P(dss, ",") OPT_WRITE_CHAR_P(dss, ",")
OPT_WRITE_UNSIGNED(narenas, ",") OPT_WRITE_UNSIGNED(narenas, ",")
OPT_WRITE_CHAR_P(percpu_arena, ",") OPT_WRITE_CHAR_P(percpu_arena, ",")
OPT_WRITE_SSIZE_T_MUTABLE(decay_time, arenas.decay_time, ",") OPT_WRITE_SSIZE_T_MUTABLE(dirty_decay_time, arenas.dirty_decay_time,
",")
OPT_WRITE_SSIZE_T_MUTABLE(muzzy_decay_time, arenas.muzzy_decay_time,
",")
OPT_WRITE_CHAR_P(junk, ",") OPT_WRITE_CHAR_P(junk, ",")
OPT_WRITE_BOOL(zero, ",") OPT_WRITE_BOOL(zero, ",")
OPT_WRITE_BOOL(utrace, ",") OPT_WRITE_BOOL(utrace, ",")
@ -670,16 +705,26 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv); malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv);
} }
CTL_GET("arenas.decay_time", &ssv, ssize_t); CTL_GET("arenas.dirty_decay_time", &ssv, ssize_t);
if (json) { if (json) {
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"decay_time\": %zd,\n", ssv); "\t\t\t\"dirty_decay_time\": %zd,\n", ssv);
} else { } else {
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,
"Unused dirty page decay time: %zd%s\n", ssv, (ssv < 0) ? "Unused dirty page decay time: %zd%s\n", ssv, (ssv < 0) ?
" (no decay)" : ""); " (no decay)" : "");
} }
CTL_GET("arenas.muzzy_decay_time", &ssv, ssize_t);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"muzzy_decay_time\": %zd,\n", ssv);
} else {
malloc_cprintf(write_cb, cbopaque,
"Unused muzzy page decay time: %zd%s\n", ssv, (ssv < 0) ?
" (no decay)" : "");
}
CTL_GET("arenas.quantum", &sv, size_t); CTL_GET("arenas.quantum", &sv, size_t);
if (json) { if (json) {
malloc_cprintf(write_cb, cbopaque, malloc_cprintf(write_cb, cbopaque,

192
test/unit/decay.c
View File

@ -22,18 +22,28 @@ nstime_update_mock(nstime_t *time) {
} }
static unsigned static unsigned
do_arena_create(ssize_t decay_time) { do_arena_create(ssize_t dirty_decay_time, ssize_t muzzy_decay_time) {
unsigned arena_ind; unsigned arena_ind;
size_t sz = sizeof(unsigned); size_t sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
0, "Unexpected mallctl() failure"); 0, "Unexpected mallctl() failure");
size_t mib[3]; size_t mib[3];
size_t miblen = sizeof(mib)/sizeof(size_t); size_t miblen = sizeof(mib)/sizeof(size_t);
assert_d_eq(mallctlnametomib("arena.0.decay_time", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure"); assert_d_eq(mallctlnametomib("arena.0.dirty_decay_time", mib, &miblen),
0, "Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind; mib[1] = (size_t)arena_ind;
assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&decay_time, assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL,
sizeof(decay_time)), 0, "Unexpected mallctlbymib() failure"); (void *)&dirty_decay_time,
sizeof(dirty_decay_time)), 0, "Unexpected mallctlbymib() failure");
assert_d_eq(mallctlnametomib("arena.0.muzzy_decay_time", mib, &miblen),
0, "Unexpected mallctlnametomib() failure");
mib[1] = (size_t)arena_ind;
assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL,
(void *)&muzzy_decay_time,
sizeof(muzzy_decay_time)), 0, "Unexpected mallctlbymib() failure");
return arena_ind; return arena_ind;
} }
@ -78,11 +88,10 @@ do_decay(unsigned arena_ind) {
} }
static uint64_t static uint64_t
get_arena_npurge(unsigned arena_ind) { get_arena_npurge_impl(const char *mibname, unsigned arena_ind) {
do_epoch();
size_t mib[4]; size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t); size_t miblen = sizeof(mib)/sizeof(size_t);
assert_d_eq(mallctlnametomib("stats.arenas.0.npurge", mib, &miblen), 0, assert_d_eq(mallctlnametomib(mibname, mib, &miblen), 0,
"Unexpected mallctlnametomib() failure"); "Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind; mib[2] = (size_t)arena_ind;
uint64_t npurge = 0; uint64_t npurge = 0;
@ -92,6 +101,25 @@ get_arena_npurge(unsigned arena_ind) {
return npurge; return npurge;
} }
static uint64_t
get_arena_dirty_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind);
}
static uint64_t
get_arena_muzzy_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static uint64_t
get_arena_npurge(unsigned arena_ind) {
do_epoch();
return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind) +
get_arena_npurge_impl("stats.arenas.0.muzzy_npurge", arena_ind);
}
static size_t static size_t
get_arena_pdirty(unsigned arena_ind) { get_arena_pdirty(unsigned arena_ind) {
do_epoch(); do_epoch();
@ -107,6 +135,21 @@ get_arena_pdirty(unsigned arena_ind) {
return pdirty; return pdirty;
} }
static size_t
get_arena_pmuzzy(unsigned arena_ind) {
do_epoch();
size_t mib[4];
size_t miblen = sizeof(mib)/sizeof(size_t);
assert_d_eq(mallctlnametomib("stats.arenas.0.pmuzzy", mib, &miblen), 0,
"Unexpected mallctlnametomib() failure");
mib[2] = (size_t)arena_ind;
size_t pmuzzy;
size_t sz = sizeof(pmuzzy);
assert_d_eq(mallctlbymib(mib, miblen, (void *)&pmuzzy, &sz, NULL, 0), 0,
"Unexpected mallctlbymib() failure");
return pmuzzy;
}
static void * static void *
do_mallocx(size_t size, int flags) { do_mallocx(size_t size, int flags) {
void *p = mallocx(size, flags); void *p = mallocx(size, flags);
@ -133,7 +176,7 @@ TEST_BEGIN(test_decay_ticks) {
int err; int err;
/* Set up a manually managed arena for test. */ /* Set up a manually managed arena for test. */
arena_ind = do_arena_create(0); arena_ind = do_arena_create(0, 0);
/* Migrate to the new arena, and get the ticker. */ /* Migrate to the new arena, and get the ticker. */
unsigned old_arena_ind; unsigned old_arena_ind;
@ -317,19 +360,66 @@ TEST_BEGIN(test_decay_ticks) {
} }
TEST_END TEST_END
TEST_BEGIN(test_decay_ticker) { static void
#define NPS 1024 decay_ticker_helper(unsigned arena_ind, int flags, bool dirty, ssize_t dt,
uint64_t dirty_npurge0, uint64_t muzzy_npurge0, bool terminate_asap) {
#define NINTERVALS 101 #define NINTERVALS 101
ssize_t dt = opt_decay_time; nstime_t time, update_interval, decay_time, deadline;
unsigned arena_ind = do_arena_create(dt);
nstime_init(&time, 0);
nstime_update(&time);
nstime_init2(&decay_time, dt, 0);
nstime_copy(&deadline, &time);
nstime_add(&deadline, &decay_time);
nstime_init2(&update_interval, dt, 0);
nstime_idivide(&update_interval, NINTERVALS);
/*
* Keep q's slab from being deallocated during the looping below. If a
* cached slab were to repeatedly come and go during looping, it could
* prevent the decay backlog ever becoming empty.
*/
void *p = do_mallocx(1, flags);
uint64_t dirty_npurge1, muzzy_npurge1;
do {
for (unsigned i = 0; i < DECAY_NTICKS_PER_UPDATE / 2;
i++) {
void *q = do_mallocx(1, flags);
dallocx(q, flags);
}
dirty_npurge1 = get_arena_dirty_npurge(arena_ind);
muzzy_npurge1 = get_arena_muzzy_npurge(arena_ind);
nstime_add(&time_mock, &update_interval);
nstime_update(&time);
} while (nstime_compare(&time, &deadline) <= 0 && ((dirty_npurge1 ==
dirty_npurge0 && muzzy_npurge1 == muzzy_npurge0) ||
!terminate_asap));
dallocx(p, flags);
if (config_stats) {
assert_u64_gt(dirty_npurge1 + muzzy_npurge1, dirty_npurge0 +
muzzy_npurge0, "Expected purging to occur");
}
#undef NINTERVALS
}
TEST_BEGIN(test_decay_ticker) {
#define NPS 2048
ssize_t ddt = opt_dirty_decay_time;
ssize_t mdt = opt_muzzy_decay_time;
unsigned arena_ind = do_arena_create(ddt, mdt);
int flags = (MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE); int flags = (MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE);
void *ps[NPS]; void *ps[NPS];
size_t large; size_t large;
/* /*
* Allocate a bunch of large objects, pause the clock, deallocate the * Allocate a bunch of large objects, pause the clock, deallocate every
* objects, restore the clock, then [md]allocx() in a tight loop while * other object (to fragment virtual memory), restore the clock, then
* advancing time rapidly to verify the ticker triggers purging. * [md]allocx() in a tight loop while advancing time rapidly to verify
* the ticker triggers purging.
*/ */
if (config_tcache) { if (config_tcache) {
@ -346,7 +436,8 @@ TEST_BEGIN(test_decay_ticker) {
} }
do_purge(arena_ind); do_purge(arena_ind);
uint64_t npurge0 = get_arena_npurge(arena_ind); uint64_t dirty_npurge0 = get_arena_dirty_npurge(arena_ind);
uint64_t muzzy_npurge0 = get_arena_muzzy_npurge(arena_ind);
for (unsigned i = 0; i < NPS; i++) { for (unsigned i = 0; i < NPS; i++) {
ps[i] = do_mallocx(large, flags); ps[i] = do_mallocx(large, flags);
@ -362,7 +453,7 @@ TEST_BEGIN(test_decay_ticker) {
nstime_monotonic = nstime_monotonic_mock; nstime_monotonic = nstime_monotonic_mock;
nstime_update = nstime_update_mock; nstime_update = nstime_update_mock;
for (unsigned i = 0; i < NPS; i++) { for (unsigned i = 0; i < NPS; i += 2) {
dallocx(ps[i], flags); dallocx(ps[i], flags);
unsigned nupdates0 = nupdates_mock; unsigned nupdates0 = nupdates_mock;
do_decay(arena_ind); do_decay(arena_ind);
@ -370,51 +461,16 @@ TEST_BEGIN(test_decay_ticker) {
"Expected nstime_update() to be called"); "Expected nstime_update() to be called");
} }
nstime_t time, update_interval, decay_time, deadline; decay_ticker_helper(arena_ind, flags, true, ddt, dirty_npurge0,
muzzy_npurge0, true);
decay_ticker_helper(arena_ind, flags, false, ddt+mdt, dirty_npurge0,
muzzy_npurge0, false);
nstime_init(&time, 0); do_arena_destroy(arena_ind);
nstime_update(&time);
nstime_init2(&decay_time, dt, 0);
nstime_copy(&deadline, &time);
nstime_add(&deadline, &decay_time);
nstime_init2(&update_interval, dt, 0);
nstime_idivide(&update_interval, NINTERVALS);
nstime_init2(&decay_time, dt, 0);
nstime_copy(&deadline, &time);
nstime_add(&deadline, &decay_time);
/*
* Keep q's slab from being deallocated during the looping below. If
* a cached slab were to repeatedly come and go during looping, it could
* prevent the decay backlog ever becoming empty.
*/
void *p = do_mallocx(1, flags);
uint64_t npurge1;
do {
for (unsigned i = 0; i < DECAY_NTICKS_PER_UPDATE / 2; i++) {
void *q = do_mallocx(1, flags);
dallocx(q, flags);
}
npurge1 = get_arena_npurge(arena_ind);
nstime_add(&time_mock, &update_interval);
nstime_update(&time);
} while (nstime_compare(&time, &deadline) <= 0 && npurge1 == npurge0);
dallocx(p, flags);
nstime_monotonic = nstime_monotonic_orig; nstime_monotonic = nstime_monotonic_orig;
nstime_update = nstime_update_orig; nstime_update = nstime_update_orig;
if (config_stats) {
assert_u64_gt(npurge1, npurge0, "Expected purging to occur");
}
do_arena_destroy(arena_ind);
#undef NPS #undef NPS
#undef NINTERVALS
} }
TEST_END TEST_END
@ -435,8 +491,7 @@ TEST_BEGIN(test_decay_nonmonotonic) {
"Unexpected mallctl failure"); "Unexpected mallctl failure");
do_epoch(); do_epoch();
sz = sizeof(uint64_t); sz = sizeof(uint64_t);
assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge0, &sz, npurge0 = get_arena_npurge(0);
NULL, 0), config_stats ? 0 : ENOENT, "Unexpected mallctl result");
nupdates_mock = 0; nupdates_mock = 0;
nstime_init(&time_mock, 0); nstime_init(&time_mock, 0);
@ -464,8 +519,7 @@ TEST_BEGIN(test_decay_nonmonotonic) {
do_epoch(); do_epoch();
sz = sizeof(uint64_t); sz = sizeof(uint64_t);
assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge1, &sz, npurge1 = get_arena_npurge(0);
NULL, 0), config_stats ? 0 : ENOENT, "Unexpected mallctl result");
if (config_stats) { if (config_stats) {
assert_u64_eq(npurge0, npurge1, "Unexpected purging occurred"); assert_u64_eq(npurge0, npurge1, "Unexpected purging occurred");
@ -478,24 +532,28 @@ TEST_BEGIN(test_decay_nonmonotonic) {
TEST_END TEST_END
TEST_BEGIN(test_decay_now) { TEST_BEGIN(test_decay_now) {
unsigned arena_ind = do_arena_create(0); unsigned arena_ind = do_arena_create(0, 0);
assert_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); assert_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages");
assert_zu_eq(get_arena_pmuzzy(arena_ind), 0, "Unexpected muzzy pages");
size_t sizes[] = {16, PAGE<<2, HUGEPAGE<<2}; size_t sizes[] = {16, PAGE<<2, HUGEPAGE<<2};
/* Verify that dirty pages never linger after deallocation. */ /* Verify that dirty/muzzy pages never linger after deallocation. */
for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) { for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) {
size_t size = sizes[i]; size_t size = sizes[i];
generate_dirty(arena_ind, size); generate_dirty(arena_ind, size);
assert_zu_eq(get_arena_pdirty(arena_ind), 0, assert_zu_eq(get_arena_pdirty(arena_ind), 0,
"Unexpected dirty pages"); "Unexpected dirty pages");
assert_zu_eq(get_arena_pmuzzy(arena_ind), 0,
"Unexpected muzzy pages");
} }
do_arena_destroy(arena_ind); do_arena_destroy(arena_ind);
} }
TEST_END TEST_END
TEST_BEGIN(test_decay_never) { TEST_BEGIN(test_decay_never) {
unsigned arena_ind = do_arena_create(-1); unsigned arena_ind = do_arena_create(-1, -1);
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
assert_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages"); assert_zu_eq(get_arena_pdirty(arena_ind), 0, "Unexpected dirty pages");
assert_zu_eq(get_arena_pmuzzy(arena_ind), 0, "Unexpected muzzy pages");
size_t sizes[] = {16, PAGE<<2, HUGEPAGE<<2}; size_t sizes[] = {16, PAGE<<2, HUGEPAGE<<2};
void *ptrs[sizeof(sizes)/sizeof(size_t)]; void *ptrs[sizeof(sizes)/sizeof(size_t)];
for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) { for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) {
@ -503,12 +561,16 @@ TEST_BEGIN(test_decay_never) {
} }
/* Verify that each deallocation generates additional dirty pages. */ /* Verify that each deallocation generates additional dirty pages. */
size_t pdirty_prev = get_arena_pdirty(arena_ind); size_t pdirty_prev = get_arena_pdirty(arena_ind);
size_t pmuzzy_prev = get_arena_pmuzzy(arena_ind);
assert_zu_eq(pdirty_prev, 0, "Unexpected dirty pages"); assert_zu_eq(pdirty_prev, 0, "Unexpected dirty pages");
assert_zu_eq(pmuzzy_prev, 0, "Unexpected muzzy pages");
for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) { for (unsigned i = 0; i < sizeof(sizes)/sizeof(size_t); i++) {
dallocx(ptrs[i], flags); dallocx(ptrs[i], flags);
size_t pdirty = get_arena_pdirty(arena_ind); size_t pdirty = get_arena_pdirty(arena_ind);
size_t pmuzzy = get_arena_pmuzzy(arena_ind);
assert_zu_gt(pdirty, pdirty_prev, assert_zu_gt(pdirty, pdirty_prev,
"Expected dirty pages to increase."); "Expected dirty pages to increase.");
assert_zu_eq(pmuzzy, 0, "Unexpected muzzy pages");
pdirty_prev = pdirty; pdirty_prev = pdirty;
} }
do_arena_destroy(arena_ind); do_arena_destroy(arena_ind);

2
test/unit/decay.sh
View File

@ -1,6 +1,6 @@
#!/bin/sh #!/bin/sh
export MALLOC_CONF="decay_time:1" export MALLOC_CONF="dirty_decay_time:1,muzzy_decay_time:1"
if [ "x${enable_tcache}" = "x1" ] ; then if [ "x${enable_tcache}" = "x1" ] ; then
export MALLOC_CONF="${MALLOC_CONF},lg_tcache_max:0" export MALLOC_CONF="${MALLOC_CONF},lg_tcache_max:0"
fi fi

160
test/unit/mallctl.c
View File

@ -161,7 +161,8 @@ TEST_BEGIN(test_mallctl_opt) {
TEST_MALLCTL_OPT(const char *, dss, always); TEST_MALLCTL_OPT(const char *, dss, always);
TEST_MALLCTL_OPT(unsigned, narenas, always); TEST_MALLCTL_OPT(unsigned, narenas, always);
TEST_MALLCTL_OPT(const char *, percpu_arena, always); TEST_MALLCTL_OPT(const char *, percpu_arena, always);
TEST_MALLCTL_OPT(ssize_t, decay_time, always); TEST_MALLCTL_OPT(ssize_t, dirty_decay_time, always);
TEST_MALLCTL_OPT(ssize_t, muzzy_decay_time, always);
TEST_MALLCTL_OPT(bool, stats_print, always); TEST_MALLCTL_OPT(bool, stats_print, always);
TEST_MALLCTL_OPT(const char *, junk, fill); TEST_MALLCTL_OPT(const char *, junk, fill);
TEST_MALLCTL_OPT(bool, zero, fill); TEST_MALLCTL_OPT(bool, zero, fill);
@ -401,32 +402,68 @@ TEST_BEGIN(test_arena_i_initialized) {
} }
TEST_END TEST_END
TEST_BEGIN(test_arena_i_decay_time) { TEST_BEGIN(test_arena_i_dirty_decay_time) {
ssize_t decay_time, orig_decay_time, prev_decay_time; ssize_t dirty_decay_time, orig_dirty_decay_time, prev_dirty_decay_time;
size_t sz = sizeof(ssize_t); size_t sz = sizeof(ssize_t);
assert_d_eq(mallctl("arena.0.decay_time", (void *)&orig_decay_time, &sz, assert_d_eq(mallctl("arena.0.dirty_decay_time",
NULL, 0), 0, "Unexpected mallctl() failure"); (void *)&orig_dirty_decay_time, &sz, NULL, 0), 0,
decay_time = -2;
assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL,
(void *)&decay_time, sizeof(ssize_t)), EFAULT,
"Unexpected mallctl() success");
decay_time = 0x7fffffff;
assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL,
(void *)&decay_time, sizeof(ssize_t)), 0,
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
for (prev_decay_time = decay_time, decay_time = -1; dirty_decay_time = -2;
decay_time < 20; prev_decay_time = decay_time, decay_time++) { assert_d_eq(mallctl("arena.0.dirty_decay_time", NULL, NULL,
ssize_t old_decay_time; (void *)&dirty_decay_time, sizeof(ssize_t)), EFAULT,
"Unexpected mallctl() success");
assert_d_eq(mallctl("arena.0.decay_time", (void *)&old_decay_time, dirty_decay_time = 0x7fffffff;
&sz, (void *)&decay_time, sizeof(ssize_t)), 0, assert_d_eq(mallctl("arena.0.dirty_decay_time", NULL, NULL,
(void *)&dirty_decay_time, sizeof(ssize_t)), 0,
"Unexpected mallctl() failure");
for (prev_dirty_decay_time = dirty_decay_time, dirty_decay_time = -1;
dirty_decay_time < 20; prev_dirty_decay_time = dirty_decay_time,
dirty_decay_time++) {
ssize_t old_dirty_decay_time;
assert_d_eq(mallctl("arena.0.dirty_decay_time",
(void *)&old_dirty_decay_time, &sz,
(void *)&dirty_decay_time, sizeof(ssize_t)), 0,
"Unexpected mallctl() failure"); "Unexpected mallctl() failure");
assert_zd_eq(old_decay_time, prev_decay_time, assert_zd_eq(old_dirty_decay_time, prev_dirty_decay_time,
"Unexpected old arena.0.decay_time"); "Unexpected old arena.0.dirty_decay_time");
}
}
TEST_END
TEST_BEGIN(test_arena_i_muzzy_decay_time) {
ssize_t muzzy_decay_time, orig_muzzy_decay_time, prev_muzzy_decay_time;
size_t sz = sizeof(ssize_t);
assert_d_eq(mallctl("arena.0.muzzy_decay_time",
(void *)&orig_muzzy_decay_time, &sz, NULL, 0), 0,
"Unexpected mallctl() failure");
muzzy_decay_time = -2;
assert_d_eq(mallctl("arena.0.muzzy_decay_time", NULL, NULL,
(void *)&muzzy_decay_time, sizeof(ssize_t)), EFAULT,
"Unexpected mallctl() success");
muzzy_decay_time = 0x7fffffff;
assert_d_eq(mallctl("arena.0.muzzy_decay_time", NULL, NULL,
(void *)&muzzy_decay_time, sizeof(ssize_t)), 0,
"Unexpected mallctl() failure");
for (prev_muzzy_decay_time = muzzy_decay_time, muzzy_decay_time = -1;
muzzy_decay_time < 20; prev_muzzy_decay_time = muzzy_decay_time,
muzzy_decay_time++) {
ssize_t old_muzzy_decay_time;
assert_d_eq(mallctl("arena.0.muzzy_decay_time",
(void *)&old_muzzy_decay_time, &sz,
(void *)&muzzy_decay_time, sizeof(ssize_t)), 0,
"Unexpected mallctl() failure");
assert_zd_eq(old_muzzy_decay_time, prev_muzzy_decay_time,
"Unexpected old arena.0.muzzy_decay_time");
} }
} }
TEST_END TEST_END
@ -522,32 +559,68 @@ TEST_BEGIN(test_arena_i_dss) {
} }
TEST_END TEST_END
TEST_BEGIN(test_arenas_decay_time) { TEST_BEGIN(test_arenas_dirty_decay_time) {
ssize_t decay_time, orig_decay_time, prev_decay_time; ssize_t dirty_decay_time, orig_dirty_decay_time, prev_dirty_decay_time;
size_t sz = sizeof(ssize_t); size_t sz = sizeof(ssize_t);
assert_d_eq(mallctl("arenas.decay_time", (void *)&orig_decay_time, &sz, assert_d_eq(mallctl("arenas.dirty_decay_time",
NULL, 0), 0, "Unexpected mallctl() failure"); (void *)&orig_dirty_decay_time, &sz, NULL, 0), 0,
"Unexpected mallctl() failure");
decay_time = -2; dirty_decay_time = -2;
assert_d_eq(mallctl("arenas.decay_time", NULL, NULL, assert_d_eq(mallctl("arenas.dirty_decay_time", NULL, NULL,
(void *)&decay_time, sizeof(ssize_t)), EFAULT, (void *)&dirty_decay_time, sizeof(ssize_t)), EFAULT,
"Unexpected mallctl() success"); "Unexpected mallctl() success");
decay_time = 0x7fffffff; dirty_decay_time = 0x7fffffff;
assert_d_eq(mallctl("arenas.decay_time", NULL, NULL, assert_d_eq(mallctl("arenas.dirty_decay_time", NULL, NULL,
(void *)&decay_time, sizeof(ssize_t)), 0, (void *)&dirty_decay_time, sizeof(ssize_t)), 0,
"Expected mallctl() failure"); "Expected mallctl() failure");
for (prev_decay_time = decay_time, decay_time = -1; for (prev_dirty_decay_time = dirty_decay_time, dirty_decay_time = -1;
decay_time < 20; prev_decay_time = decay_time, decay_time++) { dirty_decay_time < 20; prev_dirty_decay_time = dirty_decay_time,
ssize_t old_decay_time; dirty_decay_time++) {
ssize_t old_dirty_decay_time;
assert_d_eq(mallctl("arenas.decay_time", assert_d_eq(mallctl("arenas.dirty_decay_time",
(void *)&old_decay_time, &sz, (void *)&decay_time, (void *)&old_dirty_decay_time, &sz,
sizeof(ssize_t)), 0, "Unexpected mallctl() failure"); (void *)&dirty_decay_time, sizeof(ssize_t)), 0,
assert_zd_eq(old_decay_time, prev_decay_time, "Unexpected mallctl() failure");
"Unexpected old arenas.decay_time"); assert_zd_eq(old_dirty_decay_time, prev_dirty_decay_time,
"Unexpected old arenas.dirty_decay_time");
}
}
TEST_END
TEST_BEGIN(test_arenas_muzzy_decay_time) {
ssize_t muzzy_decay_time, orig_muzzy_decay_time, prev_muzzy_decay_time;
size_t sz = sizeof(ssize_t);
assert_d_eq(mallctl("arenas.muzzy_decay_time",
(void *)&orig_muzzy_decay_time, &sz, NULL, 0), 0,
"Unexpected mallctl() failure");
muzzy_decay_time = -2;
assert_d_eq(mallctl("arenas.muzzy_decay_time", NULL, NULL,
(void *)&muzzy_decay_time, sizeof(ssize_t)), EFAULT,
"Unexpected mallctl() success");
muzzy_decay_time = 0x7fffffff;
assert_d_eq(mallctl("arenas.muzzy_decay_time", NULL, NULL,
(void *)&muzzy_decay_time, sizeof(ssize_t)), 0,
"Expected mallctl() failure");
for (prev_muzzy_decay_time = muzzy_decay_time, muzzy_decay_time = -1;
muzzy_decay_time < 20; prev_muzzy_decay_time = muzzy_decay_time,
muzzy_decay_time++) {
ssize_t old_muzzy_decay_time;
assert_d_eq(mallctl("arenas.muzzy_decay_time",
(void *)&old_muzzy_decay_time, &sz,
(void *)&muzzy_decay_time, sizeof(ssize_t)), 0,
"Unexpected mallctl() failure");
assert_zd_eq(old_muzzy_decay_time, prev_muzzy_decay_time,
"Unexpected old arenas.muzzy_decay_time");
} }
} }
TEST_END TEST_END
@ -630,7 +703,8 @@ TEST_BEGIN(test_stats_arenas) {
TEST_STATS_ARENAS(unsigned, nthreads); TEST_STATS_ARENAS(unsigned, nthreads);
TEST_STATS_ARENAS(const char *, dss); TEST_STATS_ARENAS(const char *, dss);
TEST_STATS_ARENAS(ssize_t, decay_time); TEST_STATS_ARENAS(ssize_t, dirty_decay_time);
TEST_STATS_ARENAS(ssize_t, muzzy_decay_time);
TEST_STATS_ARENAS(size_t, pactive); TEST_STATS_ARENAS(size_t, pactive);
TEST_STATS_ARENAS(size_t, pdirty); TEST_STATS_ARENAS(size_t, pdirty);
@ -653,11 +727,13 @@ main(void) {
test_tcache, test_tcache,
test_thread_arena, test_thread_arena,
test_arena_i_initialized, test_arena_i_initialized,
test_arena_i_decay_time, test_arena_i_dirty_decay_time,
test_arena_i_muzzy_decay_time,
test_arena_i_purge, test_arena_i_purge,
test_arena_i_decay, test_arena_i_decay,
test_arena_i_dss, test_arena_i_dss,
test_arenas_decay_time, test_arenas_dirty_decay_time,
test_arenas_muzzy_decay_time,
test_arenas_constants, test_arenas_constants,
test_arenas_bin_constants, test_arenas_bin_constants,
test_arenas_lextent_constants, test_arenas_lextent_constants,

2
test/unit/pack.sh
View File

@ -1,4 +1,4 @@
#!/bin/sh #!/bin/sh
# Immediately purge to minimize fragmentation. # Immediately purge to minimize fragmentation.
export MALLOC_CONF="decay_time:-1" export MALLOC_CONF="dirty_decay_time:0,muzzy_decay_time:0"

36
test/unit/stats.c
View File

@ -71,7 +71,8 @@ TEST_BEGIN(test_stats_arenas_summary) {
size_t sz; size_t sz;
int expected = config_stats ? 0 : ENOENT; int expected = config_stats ? 0 : ENOENT;
size_t mapped; size_t mapped;
uint64_t npurge, nmadvise, purged; uint64_t dirty_npurge, dirty_nmadvise, dirty_purged;
uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged;
little = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); little = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0));
assert_ptr_not_null(little, "Unexpected mallocx() failure"); assert_ptr_not_null(little, "Unexpected mallocx() failure");
@ -92,19 +93,34 @@ TEST_BEGIN(test_stats_arenas_summary) {
sz = sizeof(size_t); sz = sizeof(size_t);
assert_d_eq(mallctl("stats.arenas.0.mapped", (void *)&mapped, &sz, NULL, assert_d_eq(mallctl("stats.arenas.0.mapped", (void *)&mapped, &sz, NULL,
0), expected, "Unexepected mallctl() result"); 0), expected, "Unexepected mallctl() result");
sz = sizeof(uint64_t); sz = sizeof(uint64_t);
assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge, &sz, NULL, assert_d_eq(mallctl("stats.arenas.0.dirty_npurge",
0), expected, "Unexepected mallctl() result"); (void *)&dirty_npurge, &sz, NULL, 0), expected,
assert_d_eq(mallctl("stats.arenas.0.nmadvise", (void *)&nmadvise, &sz, "Unexepected mallctl() result");
NULL, 0), expected, "Unexepected mallctl() result"); assert_d_eq(mallctl("stats.arenas.0.dirty_nmadvise",
assert_d_eq(mallctl("stats.arenas.0.purged", (void *)&purged, &sz, NULL, (void *)&dirty_nmadvise, &sz, NULL, 0), expected,
0), expected, "Unexepected mallctl() result"); "Unexepected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.dirty_purged",
(void *)&dirty_purged, &sz, NULL, 0), expected,
"Unexepected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.muzzy_npurge",
(void *)&muzzy_npurge, &sz, NULL, 0), expected,
"Unexepected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.muzzy_nmadvise",
(void *)&muzzy_nmadvise, &sz, NULL, 0), expected,
"Unexepected mallctl() result");
assert_d_eq(mallctl("stats.arenas.0.muzzy_purged",
(void *)&muzzy_purged, &sz, NULL, 0), expected,
"Unexepected mallctl() result");
if (config_stats) { if (config_stats) {
assert_u64_gt(npurge, 0, assert_u64_gt(dirty_npurge + muzzy_npurge, 0,
"At least one purge should have occurred"); "At least one purge should have occurred");
assert_u64_le(nmadvise, purged, assert_u64_le(dirty_nmadvise, dirty_purged,
"nmadvise should be no greater than purged"); "dirty_nmadvise should be no greater than dirty_purged");
assert_u64_le(muzzy_nmadvise, muzzy_purged,
"muzzy_nmadvise should be no greater than muzzy_purged");
} }
} }
TEST_END TEST_END