Implement decay-based unused dirty page purging.
This is an alternative to the existing ratio-based unused dirty page purging, and is intended to eventually become the sole purging mechanism. Add mallctls: - opt.purge - opt.decay_time - arena.<i>.decay - arena.<i>.decay_time - arenas.decay_time - stats.arenas.<i>.decay_time This resolves #325.
This commit is contained in:
Jason Evans
parent
8e82af1166
commit
243f7a0508
11
Makefile.in
11
Makefile.in
@ -121,6 +121,7 @@ C_UTIL_INTEGRATION_SRCS := $(srcroot)src/time.c $(srcroot)src/util.c
|
||||
TESTS_UNIT := $(srcroot)test/unit/atomic.c \
|
||||
$(srcroot)test/unit/bitmap.c \
|
||||
$(srcroot)test/unit/ckh.c \
|
||||
$(srcroot)test/unit/decay.c \
|
||||
$(srcroot)test/unit/hash.c \
|
||||
$(srcroot)test/unit/junk.c \
|
||||
$(srcroot)test/unit/junk_alloc.c \
|
||||
@ -354,18 +355,22 @@ stress_dir:
|
||||
check_dir: check_unit_dir check_integration_dir
|
||||
|
||||
check_unit: tests_unit check_unit_dir
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="purge:ratio" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%)
|
||||
check_integration_prof: tests_integration check_integration_dir
|
||||
ifeq ($(enable_prof), 1)
|
||||
$(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
endif
|
||||
check_integration_decay: tests_integration check_integration_dir
|
||||
$(MALLOC_CONF)="purge:decay,decay_time:-1" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="purge:decay,decay_time:0" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
$(MALLOC_CONF)="purge:decay" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
check_integration: tests_integration check_integration_dir
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
stress: tests_stress stress_dir
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%)
|
||||
check: tests check_dir check_integration_prof
|
||||
$(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
|
||||
check: check_unit check_integration check_integration_decay check_integration_prof
|
||||
|
||||
ifeq ($(enable_code_coverage), 1)
|
||||
coverage_unit: check_unit
|
||||
|
||||
@ -949,6 +949,20 @@ for (i = 0; i < nbins; i++) {
|
||||
number of CPUs, or one if there is a single CPU.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.purge">
|
||||
<term>
|
||||
<mallctl>opt.purge</mallctl>
|
||||
(<type>const char *</type>)
|
||||
<literal>r-</literal>
|
||||
</term>
|
||||
<listitem><para>Purge mode is “ratio” (default) or
|
||||
“decay”. See <link
|
||||
linkend="opt.lg_dirty_mult"><mallctl>opt.lg_dirty_mult</mallctl></link>
|
||||
for details of the ratio mode. See <link
|
||||
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> for
|
||||
details of the decay mode.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.lg_dirty_mult">
|
||||
<term>
|
||||
<mallctl>opt.lg_dirty_mult</mallctl>
|
||||
@ -971,6 +985,26 @@ for (i = 0; i < nbins; i++) {
|
||||
for related dynamic control options.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.decay_time">
|
||||
<term>
|
||||
<mallctl>opt.decay_time</mallctl>
|
||||
(<type>ssize_t</type>)
|
||||
<literal>r-</literal>
|
||||
</term>
|
||||
<listitem><para>Approximate time in seconds from the creation of a set
|
||||
of unused dirty pages until an equivalent set of unused dirty pages is
|
||||
purged and/or reused. The pages are incrementally purged according to a
|
||||
sigmoidal decay curve that starts and ends with zero purge rate. A
|
||||
decay time of 0 causes all unused dirty pages to be purged immediately
|
||||
upon creation. A decay time of -1 disables purging. The default decay
|
||||
time is 10 seconds. See <link
|
||||
linkend="arenas.decay_time"><mallctl>arenas.decay_time</mallctl></link>
|
||||
and <link
|
||||
linkend="arena.i.decay_time"><mallctl>arena.<i>.decay_time</mallctl></link>
|
||||
for related dynamic control options.
|
||||
</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="opt.stats_print">
|
||||
<term>
|
||||
<mallctl>opt.stats_print</mallctl>
|
||||
@ -1501,12 +1535,27 @@ malloc_conf = "xmalloc:true";]]></programlisting>
|
||||
(<type>void</type>)
|
||||
<literal>--</literal>
|
||||
</term>
|
||||
<listitem><para>Purge unused dirty pages for arena <i>, or for
|
||||
<listitem><para>Purge all unused dirty pages for arena <i>, or for
|
||||
all arenas if <i> equals <link
|
||||
linkend="arenas.narenas"><mallctl>arenas.narenas</mallctl></link>.
|
||||
</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="arena.i.decay">
|
||||
<term>
|
||||
<mallctl>arena.<i>.decay</mallctl>
|
||||
(<type>void</type>)
|
||||
<literal>--</literal>
|
||||
</term>
|
||||
<listitem><para>Trigger decay-based purging of unused dirty pages for
|
||||
arena <i>, or for all arenas if <i> equals <link
|
||||
linkend="arenas.narenas"><mallctl>arenas.narenas</mallctl></link>.
|
||||
The proportion of unused dirty pages to be purged depends on the current
|
||||
time; see <link
|
||||
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> for
|
||||
details.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="arena.i.dss">
|
||||
<term>
|
||||
<mallctl>arena.<i>.dss</mallctl>
|
||||
@ -1535,6 +1584,22 @@ malloc_conf = "xmalloc:true";]]></programlisting>
|
||||
for additional information.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="arena.i.decay_time">
|
||||
<term>
|
||||
<mallctl>arena.<i>.decay_time</mallctl>
|
||||
(<type>ssize_t</type>)
|
||||
<literal>rw</literal>
|
||||
</term>
|
||||
<listitem><para>Current per-arena approximate time in seconds from the
|
||||
creation of a set of unused dirty pages until an equivalent set of
|
||||
unused dirty pages is purged and/or reused. Each time this interface is
|
||||
set, all currently unused dirty pages are considered to have fully
|
||||
decayed, which causes immediate purging of all unused dirty pages unless
|
||||
the decay time is set to -1 (i.e. purging disabled). See <link
|
||||
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> for
|
||||
additional information.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="arena.i.chunk_hooks">
|
||||
<term>
|
||||
<mallctl>arena.<i>.chunk_hooks</mallctl>
|
||||
@ -1769,6 +1834,21 @@ typedef struct {
|
||||
for additional information.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="arenas.decay_time">
|
||||
<term>
|
||||
<mallctl>arenas.decay_time</mallctl>
|
||||
(<type>ssize_t</type>)
|
||||
<literal>rw</literal>
|
||||
</term>
|
||||
<listitem><para>Current default per-arena approximate time in seconds
|
||||
from the creation of a set of unused dirty pages until an equivalent set
|
||||
of unused dirty pages is purged and/or reused, used to initialize <link
|
||||
linkend="arena.i.decay_time"><mallctl>arena.<i>.decay_time</mallctl></link>
|
||||
during arena creation. See <link
|
||||
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link> for
|
||||
additional information.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="arenas.quantum">
|
||||
<term>
|
||||
<mallctl>arenas.quantum</mallctl>
|
||||
@ -2113,6 +2193,19 @@ typedef struct {
|
||||
for details.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="stats.arenas.i.decay_time">
|
||||
<term>
|
||||
<mallctl>stats.arenas.<i>.decay_time</mallctl>
|
||||
(<type>ssize_t</type>)
|
||||
<literal>r-</literal>
|
||||
</term>
|
||||
<listitem><para>Approximate time in seconds from the creation of a set
|
||||
of unused dirty pages until an equivalent set of unused dirty pages is
|
||||
purged and/or reused. See <link
|
||||
linkend="opt.decay_time"><mallctl>opt.decay_time</mallctl></link>
|
||||
for details.</para></listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry id="stats.arenas.i.nthreads">
|
||||
<term>
|
||||
<mallctl>stats.arenas.<i>.nthreads</mallctl>
|
||||
|
||||
@ -23,6 +23,18 @@
|
||||
*/
|
||||
#define LG_DIRTY_MULT_DEFAULT 3
|
||||
|
||||
typedef enum {
|
||||
purge_mode_ratio = 0,
|
||||
purge_mode_decay = 1,
|
||||
|
||||
purge_mode_limit = 2
|
||||
} purge_mode_t;
|
||||
#define PURGE_DEFAULT purge_mode_ratio
|
||||
/* Default decay time in seconds. */
|
||||
#define DECAY_TIME_DEFAULT 10
|
||||
/* Number of event ticks between time checks. */
|
||||
#define DECAY_NTICKS_PER_UPDATE 1000
|
||||
|
||||
typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t;
|
||||
typedef struct arena_run_s arena_run_t;
|
||||
typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t;
|
||||
@ -325,7 +337,7 @@ struct arena_s {
|
||||
/* Minimum ratio (log base 2) of nactive:ndirty. */
|
||||
ssize_t lg_dirty_mult;
|
||||
|
||||
/* True if a thread is currently executing arena_purge(). */
|
||||
/* True if a thread is currently executing arena_purge_to_limit(). */
|
||||
bool purging;
|
||||
|
||||
/* Number of pages in active runs and huge regions. */
|
||||
@ -376,6 +388,53 @@ struct arena_s {
|
||||
arena_runs_dirty_link_t runs_dirty;
|
||||
extent_node_t chunks_cache;
|
||||
|
||||
/*
|
||||
* Approximate time in seconds from the creation of a set of unused
|
||||
* dirty pages until an equivalent set of unused dirty pages is purged
|
||||
* and/or reused.
|
||||
*/
|
||||
ssize_t decay_time;
|
||||
/* decay_time / SMOOTHSTEP_NSTEPS. */
|
||||
struct timespec decay_interval;
|
||||
/*
|
||||
* Time at which the current decay interval logically started. We do
|
||||
* not actually advance to a new epoch until sometime after it starts
|
||||
* because of scheduling and computation delays, and it is even possible
|
||||
* to completely skip epochs. In all cases, during epoch advancement we
|
||||
* merge all relevant activity into the most recently recorded epoch.
|
||||
*/
|
||||
struct timespec decay_epoch;
|
||||
/* decay_deadline randomness generator. */
|
||||
uint64_t decay_jitter_state;
|
||||
/*
|
||||
* Deadline for current epoch. This is the sum of decay_interval and
|
||||
* per epoch jitter which is a uniform random variable in
|
||||
* [0..decay_interval). Epochs always advance by precise multiples of
|
||||
* decay_interval, but we randomize the deadline to reduce the
|
||||
* likelihood of arenas purging in lockstep.
|
||||
*/
|
||||
struct timespec decay_deadline;
|
||||
/*
|
||||
* Number of dirty pages at beginning of current epoch. During epoch
|
||||
* advancement we use the delta between decay_ndirty and ndirty to
|
||||
* determine how many dirty pages, if any, were generated, and record
|
||||
* the result in decay_backlog.
|
||||
*/
|
||||
size_t decay_ndirty;
|
||||
/*
|
||||
* Memoized result of arena_decay_backlog_npages_limit() corresponding
|
||||
* to the current contents of decay_backlog, i.e. the limit on how many
|
||||
* pages are allowed to exist for the decay epochs.
|
||||
*/
|
||||
size_t decay_backlog_npages_limit;
|
||||
/*
|
||||
* Trailing log of how many unused dirty pages were generated during
|
||||
* each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last
|
||||
* element is the most recent epoch. Corresponding epoch times are
|
||||
* relative to decay_epoch.
|
||||
*/
|
||||
size_t decay_backlog[SMOOTHSTEP_NSTEPS];
|
||||
|
||||
/* Extant huge allocations. */
|
||||
ql_head(extent_node_t) huge;
|
||||
/* Synchronizes all huge allocation/update/deallocation. */
|
||||
@ -408,6 +467,7 @@ struct arena_s {
|
||||
/* Used in conjunction with tsd for fast arena-related context lookup. */
|
||||
struct arena_tdata_s {
|
||||
arena_t *arena;
|
||||
ticker_t decay_ticker;
|
||||
};
|
||||
#endif /* JEMALLOC_ARENA_STRUCTS_B */
|
||||
|
||||
@ -423,7 +483,10 @@ static const size_t large_pad =
|
||||
#endif
|
||||
;
|
||||
|
||||
extern purge_mode_t opt_purge;
|
||||
extern const char *purge_mode_names[];
|
||||
extern ssize_t opt_lg_dirty_mult;
|
||||
extern ssize_t opt_decay_time;
|
||||
|
||||
extern arena_bin_info_t arena_bin_info[NBINS];
|
||||
|
||||
@ -451,9 +514,11 @@ bool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk,
|
||||
size_t oldsize, size_t usize, bool *zero);
|
||||
ssize_t arena_lg_dirty_mult_get(arena_t *arena);
|
||||
bool arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult);
|
||||
ssize_t arena_decay_time_get(arena_t *arena);
|
||||
bool arena_decay_time_set(arena_t *arena, ssize_t decay_time);
|
||||
void arena_maybe_purge(arena_t *arena);
|
||||
void arena_purge_all(arena_t *arena);
|
||||
void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
|
||||
void arena_purge(arena_t *arena, bool all);
|
||||
void arena_tcache_fill_small(tsd_t *tsd, arena_t *arena, tcache_bin_t *tbin,
|
||||
szind_t binind, uint64_t prof_accumbytes);
|
||||
void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info,
|
||||
bool zero);
|
||||
@ -467,7 +532,7 @@ extern arena_dalloc_junk_small_t *arena_dalloc_junk_small;
|
||||
void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info);
|
||||
#endif
|
||||
void arena_quarantine_junk_small(void *ptr, size_t usize);
|
||||
void *arena_malloc_large(arena_t *arena, size_t size,
|
||||
void *arena_malloc_large(tsd_t *tsd, arena_t *arena, size_t size,
|
||||
szind_t ind, bool zero);
|
||||
void *arena_malloc_hard(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind,
|
||||
bool zero, tcache_t *tcache);
|
||||
@ -478,8 +543,8 @@ void arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr, arena_chunk_map_bits_t *bitselm);
|
||||
void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
size_t pageind, arena_chunk_map_bits_t *bitselm);
|
||||
void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
size_t pageind);
|
||||
void arena_dalloc_small(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr, size_t pageind);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (arena_dalloc_junk_large_t)(void *, size_t);
|
||||
extern arena_dalloc_junk_large_t *arena_dalloc_junk_large;
|
||||
@ -488,12 +553,13 @@ void arena_dalloc_junk_large(void *ptr, size_t usize);
|
||||
#endif
|
||||
void arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr);
|
||||
void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
|
||||
void arena_dalloc_large(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t);
|
||||
extern arena_ralloc_junk_large_t *arena_ralloc_junk_large;
|
||||
#endif
|
||||
bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
|
||||
bool arena_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
||||
size_t extra, bool zero);
|
||||
void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
|
||||
size_t size, size_t alignment, bool zero, tcache_t *tcache);
|
||||
@ -501,9 +567,11 @@ dss_prec_t arena_dss_prec_get(arena_t *arena);
|
||||
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
|
||||
ssize_t arena_lg_dirty_mult_default_get(void);
|
||||
bool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult);
|
||||
ssize_t arena_decay_time_default_get(void);
|
||||
bool arena_decay_time_default_set(ssize_t decay_time);
|
||||
void arena_stats_merge(arena_t *arena, const char **dss,
|
||||
ssize_t *lg_dirty_mult, size_t *nactive, size_t *ndirty,
|
||||
arena_stats_t *astats, malloc_bin_stats_t *bstats,
|
||||
ssize_t *lg_dirty_mult, ssize_t *decay_time, size_t *nactive,
|
||||
size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats,
|
||||
malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats);
|
||||
arena_t *arena_new(unsigned ind);
|
||||
bool arena_boot(void);
|
||||
@ -566,6 +634,8 @@ prof_tctx_t *arena_prof_tctx_get(const void *ptr);
|
||||
void arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx);
|
||||
void arena_prof_tctx_reset(const void *ptr, size_t usize,
|
||||
const void *old_ptr, prof_tctx_t *old_tctx);
|
||||
void arena_decay_ticks(tsd_t *tsd, arena_t *arena, unsigned nticks);
|
||||
void arena_decay_tick(tsd_t *tsd, arena_t *arena);
|
||||
void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind,
|
||||
bool zero, tcache_t *tcache, bool slow_path);
|
||||
arena_t *arena_aalloc(const void *ptr);
|
||||
@ -1165,6 +1235,27 @@ arena_prof_tctx_reset(const void *ptr, size_t usize, const void *old_ptr,
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
arena_decay_ticks(tsd_t *tsd, arena_t *arena, unsigned nticks)
|
||||
{
|
||||
ticker_t *decay_ticker;
|
||||
|
||||
if (unlikely(tsd == NULL))
|
||||
return;
|
||||
decay_ticker = decay_ticker_get(tsd, arena->ind);
|
||||
if (unlikely(decay_ticker == NULL))
|
||||
return;
|
||||
if (unlikely(ticker_ticks(decay_ticker, nticks)))
|
||||
arena_purge(arena, false);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
arena_decay_tick(tsd_t *tsd, arena_t *arena)
|
||||
{
|
||||
|
||||
arena_decay_ticks(tsd, arena, 1);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind, bool zero,
|
||||
tcache_t *tcache, bool slow_path)
|
||||
@ -1271,7 +1362,7 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
|
||||
tcache_dalloc_small(tsd, tcache, ptr, binind,
|
||||
slow_path);
|
||||
} else {
|
||||
arena_dalloc_small(extent_node_arena_get(
|
||||
arena_dalloc_small(tsd, extent_node_arena_get(
|
||||
&chunk->node), chunk, ptr, pageind);
|
||||
}
|
||||
} else {
|
||||
@ -1286,7 +1377,7 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
|
||||
tcache_dalloc_large(tsd, tcache, ptr, size -
|
||||
large_pad, slow_path);
|
||||
} else {
|
||||
arena_dalloc_large(extent_node_arena_get(
|
||||
arena_dalloc_large(tsd, extent_node_arena_get(
|
||||
&chunk->node), chunk, ptr);
|
||||
}
|
||||
}
|
||||
@ -1326,7 +1417,7 @@ arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
|
||||
} else {
|
||||
size_t pageind = ((uintptr_t)ptr -
|
||||
(uintptr_t)chunk) >> LG_PAGE;
|
||||
arena_dalloc_small(extent_node_arena_get(
|
||||
arena_dalloc_small(tsd, extent_node_arena_get(
|
||||
&chunk->node), chunk, ptr, pageind);
|
||||
}
|
||||
} else {
|
||||
@ -1337,7 +1428,7 @@ arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache)
|
||||
tcache_dalloc_large(tsd, tcache, ptr, size,
|
||||
true);
|
||||
} else {
|
||||
arena_dalloc_large(extent_node_arena_get(
|
||||
arena_dalloc_large(tsd, extent_node_arena_get(
|
||||
&chunk->node), chunk, ptr);
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,6 +35,7 @@ struct ctl_arena_stats_s {
|
||||
unsigned nthreads;
|
||||
const char *dss;
|
||||
ssize_t lg_dirty_mult;
|
||||
ssize_t decay_time;
|
||||
size_t pactive;
|
||||
size_t pdirty;
|
||||
arena_stats_t astats;
|
||||
|
||||
@ -13,8 +13,8 @@ void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero,
|
||||
tcache_t *tcache);
|
||||
void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
|
||||
bool zero, tcache_t *tcache);
|
||||
bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min,
|
||||
size_t usize_max, bool zero);
|
||||
bool huge_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize,
|
||||
size_t usize_min, size_t usize_max, bool zero);
|
||||
void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
|
||||
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
|
||||
#ifdef JEMALLOC_JET
|
||||
|
||||
@ -545,6 +545,7 @@ arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind,
|
||||
bool refresh_if_missing);
|
||||
arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing,
|
||||
bool refresh_if_missing);
|
||||
ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
||||
@ -833,6 +834,17 @@ arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing,
|
||||
|
||||
return (tdata->arena);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE ticker_t *
|
||||
decay_ticker_get(tsd_t *tsd, unsigned ind)
|
||||
{
|
||||
arena_tdata_t *tdata;
|
||||
|
||||
tdata = arena_tdata_get(tsd, ind, true);
|
||||
if (unlikely(tdata == NULL))
|
||||
return (NULL);
|
||||
return (&tdata->decay_ticker);
|
||||
}
|
||||
#endif
|
||||
|
||||
#include "jemalloc/internal/bitmap.h"
|
||||
@ -883,8 +895,8 @@ void *iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
||||
size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
|
||||
void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
||||
size_t alignment, bool zero);
|
||||
bool ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
size_t alignment, bool zero);
|
||||
bool ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
||||
size_t extra, size_t alignment, bool zero);
|
||||
#endif
|
||||
|
||||
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
|
||||
@ -1150,8 +1162,8 @@ iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment,
|
||||
bool zero)
|
||||
ixalloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
size_t alignment, bool zero)
|
||||
{
|
||||
|
||||
assert(ptr != NULL);
|
||||
@ -1163,7 +1175,7 @@ ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment,
|
||||
return (true);
|
||||
}
|
||||
|
||||
return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero));
|
||||
return (arena_ralloc_no_move(tsd, ptr, oldsize, size, extra, zero));
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@ -25,6 +25,12 @@ arena_dalloc_junk_small
|
||||
arena_dalloc_large
|
||||
arena_dalloc_large_junked_locked
|
||||
arena_dalloc_small
|
||||
arena_decay_time_default_get
|
||||
arena_decay_time_default_set
|
||||
arena_decay_time_get
|
||||
arena_decay_time_set
|
||||
arena_decay_tick
|
||||
arena_decay_ticks
|
||||
arena_dss_prec_get
|
||||
arena_dss_prec_set
|
||||
arena_get
|
||||
@ -83,7 +89,7 @@ arena_prof_tctx_get
|
||||
arena_prof_tctx_reset
|
||||
arena_prof_tctx_set
|
||||
arena_ptr_small_binind_get
|
||||
arena_purge_all
|
||||
arena_purge
|
||||
arena_quarantine_junk_small
|
||||
arena_ralloc
|
||||
arena_ralloc_junk_large
|
||||
@ -185,6 +191,7 @@ ctl_nametomib
|
||||
ctl_postfork_child
|
||||
ctl_postfork_parent
|
||||
ctl_prefork
|
||||
decay_ticker_get
|
||||
dss_prec_names
|
||||
extent_node_achunk_get
|
||||
extent_node_achunk_set
|
||||
@ -318,6 +325,7 @@ narenas_total_get
|
||||
ncpus
|
||||
nhbins
|
||||
opt_abort
|
||||
opt_decay_time
|
||||
opt_dss
|
||||
opt_junk
|
||||
opt_junk_alloc
|
||||
@ -336,6 +344,7 @@ opt_prof_gdump
|
||||
opt_prof_leak
|
||||
opt_prof_prefix
|
||||
opt_prof_thread_active_init
|
||||
opt_purge
|
||||
opt_quarantine
|
||||
opt_redzone
|
||||
opt_stats_print
|
||||
@ -397,6 +406,7 @@ prof_thread_active_init_set
|
||||
prof_thread_active_set
|
||||
prof_thread_name_get
|
||||
prof_thread_name_set
|
||||
purge_mode_names
|
||||
quarantine
|
||||
quarantine_alloc_hook
|
||||
quarantine_alloc_hook_work
|
||||
|
||||
@ -361,7 +361,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
|
||||
|
||||
usize = index2size(binind);
|
||||
assert(usize <= tcache_maxclass);
|
||||
ret = arena_malloc_large(arena, usize, binind, zero);
|
||||
ret = arena_malloc_large(tsd, arena, usize, binind, zero);
|
||||
if (ret == NULL)
|
||||
return (NULL);
|
||||
} else {
|
||||
|
||||
@ -26,7 +26,12 @@ void time_imultiply(struct timespec *time, uint64_t multiplier);
|
||||
void time_idivide(struct timespec *time, uint64_t divisor);
|
||||
uint64_t time_divide(const struct timespec *time,
|
||||
const struct timespec *divisor);
|
||||
#ifdef JEMALLOC_JET
|
||||
typedef bool (time_update_t)(struct timespec *);
|
||||
extern time_update_t *time_update;
|
||||
#else
|
||||
bool time_update(struct timespec *time);
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_H_EXTERNS */
|
||||
/******************************************************************************/
|
||||
|
||||
323
src/arena.c
323
src/arena.c
@ -4,8 +4,17 @@
|
||||
/******************************************************************************/
|
||||
/* Data. */
|
||||
|
||||
purge_mode_t opt_purge = PURGE_DEFAULT;
|
||||
const char *purge_mode_names[] = {
|
||||
"ratio",
|
||||
"decay",
|
||||
"N/A"
|
||||
};
|
||||
ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
|
||||
static ssize_t lg_dirty_mult_default;
|
||||
ssize_t opt_decay_time = DECAY_TIME_DEFAULT;
|
||||
static ssize_t decay_time_default;
|
||||
|
||||
arena_bin_info_t arena_bin_info[NBINS];
|
||||
|
||||
size_t map_bias;
|
||||
@ -1205,10 +1214,193 @@ arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult)
|
||||
return (false);
|
||||
}
|
||||
|
||||
static void
|
||||
arena_decay_deadline_init(arena_t *arena)
|
||||
{
|
||||
|
||||
assert(opt_purge == purge_mode_decay);
|
||||
|
||||
/*
|
||||
* Generate a new deadline that is uniformly random within the next
|
||||
* epoch after the current one.
|
||||
*/
|
||||
time_copy(&arena->decay_deadline, &arena->decay_epoch);
|
||||
time_add(&arena->decay_deadline, &arena->decay_interval);
|
||||
if (arena->decay_time > 0) {
|
||||
uint64_t decay_interval_ns, r;
|
||||
struct timespec jitter;
|
||||
|
||||
decay_interval_ns = time_sec(&arena->decay_interval) *
|
||||
1000000000 + time_nsec(&arena->decay_interval);
|
||||
r = prng_range(&arena->decay_jitter_state, decay_interval_ns);
|
||||
time_init(&jitter, r / 1000000000, r % 1000000000);
|
||||
time_add(&arena->decay_deadline, &jitter);
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
arena_decay_deadline_reached(const arena_t *arena, const struct timespec *time)
|
||||
{
|
||||
|
||||
assert(opt_purge == purge_mode_decay);
|
||||
|
||||
return (time_compare(&arena->decay_deadline, time) <= 0);
|
||||
}
|
||||
|
||||
static size_t
|
||||
arena_decay_backlog_npages_limit(const arena_t *arena)
|
||||
{
|
||||
static const uint64_t h_steps[] = {
|
||||
#define STEP(step, h, x, y) \
|
||||
h,
|
||||
SMOOTHSTEP
|
||||
#undef STEP
|
||||
};
|
||||
uint64_t sum;
|
||||
size_t npages_limit_backlog;
|
||||
unsigned i;
|
||||
|
||||
assert(opt_purge == purge_mode_decay);
|
||||
|
||||
/*
|
||||
* For each element of decay_backlog, multiply by the corresponding
|
||||
* fixed-point smoothstep decay factor. Sum the products, then divide
|
||||
* to round down to the nearest whole number of pages.
|
||||
*/
|
||||
sum = 0;
|
||||
for (i = 0; i < SMOOTHSTEP_NSTEPS; i++)
|
||||
sum += arena->decay_backlog[i] * h_steps[i];
|
||||
npages_limit_backlog = (sum >> SMOOTHSTEP_BFP);
|
||||
|
||||
return (npages_limit_backlog);
|
||||
}
|
||||
|
||||
static void
|
||||
arena_decay_epoch_advance(arena_t *arena, const struct timespec *time)
|
||||
{
|
||||
uint64_t nadvance;
|
||||
struct timespec delta;
|
||||
size_t ndirty_delta;
|
||||
|
||||
assert(opt_purge == purge_mode_decay);
|
||||
assert(arena_decay_deadline_reached(arena, time));
|
||||
|
||||
time_copy(&delta, time);
|
||||
time_subtract(&delta, &arena->decay_epoch);
|
||||
nadvance = time_divide(&delta, &arena->decay_interval);
|
||||
assert(nadvance > 0);
|
||||
|
||||
/* Add nadvance decay intervals to epoch. */
|
||||
time_copy(&delta, &arena->decay_interval);
|
||||
time_imultiply(&delta, nadvance);
|
||||
time_add(&arena->decay_epoch, &delta);
|
||||
|
||||
/* Set a new deadline. */
|
||||
arena_decay_deadline_init(arena);
|
||||
|
||||
/* Update the backlog. */
|
||||
if (nadvance >= SMOOTHSTEP_NSTEPS) {
|
||||
memset(arena->decay_backlog, 0, (SMOOTHSTEP_NSTEPS-1) *
|
||||
sizeof(size_t));
|
||||
} else {
|
||||
memmove(arena->decay_backlog, &arena->decay_backlog[nadvance],
|
||||
(SMOOTHSTEP_NSTEPS - nadvance) * sizeof(size_t));
|
||||
if (nadvance > 1) {
|
||||
memset(&arena->decay_backlog[SMOOTHSTEP_NSTEPS -
|
||||
nadvance], 0, (nadvance-1) * sizeof(size_t));
|
||||
}
|
||||
}
|
||||
ndirty_delta = (arena->ndirty > arena->decay_ndirty) ? arena->ndirty -
|
||||
arena->decay_ndirty : 0;
|
||||
arena->decay_ndirty = arena->ndirty;
|
||||
arena->decay_backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta;
|
||||
arena->decay_backlog_npages_limit =
|
||||
arena_decay_backlog_npages_limit(arena);
|
||||
}
|
||||
|
||||
static size_t
|
||||
arena_decay_npages_limit(arena_t *arena)
|
||||
{
|
||||
size_t npages_limit;
|
||||
|
||||
assert(opt_purge == purge_mode_decay);
|
||||
|
||||
npages_limit = arena->decay_backlog_npages_limit;
|
||||
|
||||
/* Add in any dirty pages created during the current epoch. */
|
||||
if (arena->ndirty > arena->decay_ndirty)
|
||||
npages_limit += arena->ndirty - arena->decay_ndirty;
|
||||
|
||||
return (npages_limit);
|
||||
}
|
||||
|
||||
static void
|
||||
arena_decay_init(arena_t *arena, ssize_t decay_time)
|
||||
{
|
||||
|
||||
arena->decay_time = decay_time;
|
||||
if (decay_time > 0) {
|
||||
time_init(&arena->decay_interval, decay_time, 0);
|
||||
time_idivide(&arena->decay_interval, SMOOTHSTEP_NSTEPS);
|
||||
}
|
||||
|
||||
time_init(&arena->decay_epoch, 0, 0);
|
||||
time_update(&arena->decay_epoch);
|
||||
arena->decay_jitter_state = (uint64_t)(uintptr_t)arena;
|
||||
arena_decay_deadline_init(arena);
|
||||
arena->decay_ndirty = arena->ndirty;
|
||||
arena->decay_backlog_npages_limit = 0;
|
||||
memset(arena->decay_backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t));
|
||||
}
|
||||
|
||||
static bool
|
||||
arena_decay_time_valid(ssize_t decay_time)
|
||||
{
|
||||
|
||||
return (decay_time >= -1 && decay_time <= TIME_SEC_MAX);
|
||||
}
|
||||
|
||||
ssize_t
|
||||
arena_decay_time_get(arena_t *arena)
|
||||
{
|
||||
ssize_t decay_time;
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
decay_time = arena->decay_time;
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
|
||||
return (decay_time);
|
||||
}
|
||||
|
||||
bool
|
||||
arena_decay_time_set(arena_t *arena, ssize_t decay_time)
|
||||
{
|
||||
|
||||
if (!arena_decay_time_valid(decay_time))
|
||||
return (true);
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
/*
|
||||
* Restart decay backlog from scratch, which may cause many dirty pages
|
||||
* to be immediately purged. It would conceptually be possible to map
|
||||
* the old backlog onto the new backlog, but there is no justification
|
||||
* for such complexity since decay_time changes are intended to be
|
||||
* infrequent, either between the {-1, 0, >0} states, or a one-time
|
||||
* arbitrary change during initial arena configuration.
|
||||
*/
|
||||
arena_decay_init(arena, decay_time);
|
||||
arena_maybe_purge(arena);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
|
||||
return (false);
|
||||
}
|
||||
|
||||
static void
|
||||
arena_maybe_purge_ratio(arena_t *arena)
|
||||
{
|
||||
|
||||
assert(opt_purge == purge_mode_ratio);
|
||||
|
||||
/* Don't purge if the option is disabled. */
|
||||
if (arena->lg_dirty_mult < 0)
|
||||
return;
|
||||
@ -1231,6 +1423,41 @@ arena_maybe_purge_ratio(arena_t *arena)
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
arena_maybe_purge_decay(arena_t *arena)
|
||||
{
|
||||
struct timespec time;
|
||||
size_t ndirty_limit;
|
||||
|
||||
assert(opt_purge == purge_mode_decay);
|
||||
|
||||
/* Purge all or nothing if the option is disabled. */
|
||||
if (arena->decay_time <= 0) {
|
||||
if (arena->decay_time == 0)
|
||||
arena_purge_to_limit(arena, 0);
|
||||
return;
|
||||
}
|
||||
|
||||
time_copy(&time, &arena->decay_epoch);
|
||||
if (unlikely(time_update(&time))) {
|
||||
/* Time went backwards. Force an epoch advance. */
|
||||
time_copy(&time, &arena->decay_deadline);
|
||||
}
|
||||
|
||||
if (arena_decay_deadline_reached(arena, &time))
|
||||
arena_decay_epoch_advance(arena, &time);
|
||||
|
||||
ndirty_limit = arena_decay_npages_limit(arena);
|
||||
|
||||
/*
|
||||
* Don't try to purge unless the number of purgeable pages exceeds the
|
||||
* current limit.
|
||||
*/
|
||||
if (arena->ndirty <= ndirty_limit)
|
||||
return;
|
||||
arena_purge_to_limit(arena, ndirty_limit);
|
||||
}
|
||||
|
||||
void
|
||||
arena_maybe_purge(arena_t *arena)
|
||||
{
|
||||
@ -1239,7 +1466,10 @@ arena_maybe_purge(arena_t *arena)
|
||||
if (arena->purging)
|
||||
return;
|
||||
|
||||
if (opt_purge == purge_mode_ratio)
|
||||
arena_maybe_purge_ratio(arena);
|
||||
else
|
||||
arena_maybe_purge_decay(arena);
|
||||
}
|
||||
|
||||
static size_t
|
||||
@ -1298,6 +1528,9 @@ arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks,
|
||||
UNUSED void *chunk;
|
||||
|
||||
npages = extent_node_size_get(chunkselm) >> LG_PAGE;
|
||||
if (opt_purge == purge_mode_decay && arena->ndirty -
|
||||
(nstashed + npages) < ndirty_limit)
|
||||
break;
|
||||
|
||||
chunkselm_next = qr_next(chunkselm, cc_link);
|
||||
/*
|
||||
@ -1327,6 +1560,9 @@ arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks,
|
||||
arena_mapbits_unallocated_size_get(chunk, pageind);
|
||||
|
||||
npages = run_size >> LG_PAGE;
|
||||
if (opt_purge == purge_mode_decay && arena->ndirty -
|
||||
(nstashed + npages) < ndirty_limit)
|
||||
break;
|
||||
|
||||
assert(pageind + npages <= chunk_npages);
|
||||
assert(arena_mapbits_dirty_get(chunk, pageind) ==
|
||||
@ -1352,7 +1588,8 @@ arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks,
|
||||
}
|
||||
|
||||
nstashed += npages;
|
||||
if (arena->ndirty - nstashed <= ndirty_limit)
|
||||
if (opt_purge == purge_mode_ratio && arena->ndirty - nstashed <=
|
||||
ndirty_limit)
|
||||
break;
|
||||
}
|
||||
|
||||
@ -1492,6 +1729,15 @@ arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks,
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* NB: ndirty_limit is interpreted differently depending on opt_purge:
|
||||
* - purge_mode_ratio: Purge as few dirty run/chunks as possible to reach the
|
||||
* desired state:
|
||||
* (arena->ndirty <= ndirty_limit)
|
||||
* - purge_mode_decay: Purge as many dirty runs/chunks as possible without
|
||||
* violating the invariant:
|
||||
* (arena->ndirty >= ndirty_limit)
|
||||
*/
|
||||
static void
|
||||
arena_purge_to_limit(arena_t *arena, size_t ndirty_limit)
|
||||
{
|
||||
@ -1510,8 +1756,8 @@ arena_purge_to_limit(arena_t *arena, size_t ndirty_limit)
|
||||
size_t ndirty = arena_dirty_count(arena);
|
||||
assert(ndirty == arena->ndirty);
|
||||
}
|
||||
assert((arena->nactive >> arena->lg_dirty_mult) < arena->ndirty ||
|
||||
ndirty_limit == 0);
|
||||
assert(opt_purge != purge_mode_ratio || (arena->nactive >>
|
||||
arena->lg_dirty_mult) < arena->ndirty || ndirty_limit == 0);
|
||||
|
||||
qr_new(&purge_runs_sentinel, rd_link);
|
||||
extent_node_dirty_linkage_init(&purge_chunks_sentinel);
|
||||
@ -1534,11 +1780,14 @@ label_return:
|
||||
}
|
||||
|
||||
void
|
||||
arena_purge_all(arena_t *arena)
|
||||
arena_purge(arena_t *arena, bool all)
|
||||
{
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
if (all)
|
||||
arena_purge_to_limit(arena, 0);
|
||||
else
|
||||
arena_maybe_purge(arena);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
}
|
||||
|
||||
@ -1960,8 +2209,8 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
|
||||
}
|
||||
|
||||
void
|
||||
arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind,
|
||||
uint64_t prof_accumbytes)
|
||||
arena_tcache_fill_small(tsd_t *tsd, arena_t *arena, tcache_bin_t *tbin,
|
||||
szind_t binind, uint64_t prof_accumbytes)
|
||||
{
|
||||
unsigned i, nfill;
|
||||
arena_bin_t *bin;
|
||||
@ -2008,6 +2257,7 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind,
|
||||
}
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
tbin->ncached = i;
|
||||
arena_decay_tick(tsd, arena);
|
||||
}
|
||||
|
||||
void
|
||||
@ -2118,7 +2368,8 @@ arena_quarantine_junk_small(void *ptr, size_t usize)
|
||||
}
|
||||
|
||||
static void *
|
||||
arena_malloc_small(arena_t *arena, size_t size, szind_t binind, bool zero)
|
||||
arena_malloc_small(tsd_t *tsd, arena_t *arena, size_t size, szind_t binind,
|
||||
bool zero)
|
||||
{
|
||||
void *ret;
|
||||
arena_bin_t *bin;
|
||||
@ -2166,11 +2417,13 @@ arena_malloc_small(arena_t *arena, size_t size, szind_t binind, bool zero)
|
||||
memset(ret, 0, size);
|
||||
}
|
||||
|
||||
arena_decay_tick(tsd, arena);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
void *
|
||||
arena_malloc_large(arena_t *arena, size_t size, szind_t binind, bool zero)
|
||||
arena_malloc_large(tsd_t *tsd, arena_t *arena, size_t size, szind_t binind,
|
||||
bool zero)
|
||||
{
|
||||
void *ret;
|
||||
size_t usize;
|
||||
@ -2227,6 +2480,7 @@ arena_malloc_large(arena_t *arena, size_t size, szind_t binind, bool zero)
|
||||
}
|
||||
}
|
||||
|
||||
arena_decay_tick(tsd, arena);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -2240,9 +2494,9 @@ arena_malloc_hard(tsd_t *tsd, arena_t *arena, size_t size, szind_t ind,
|
||||
return (NULL);
|
||||
|
||||
if (likely(size <= SMALL_MAXCLASS))
|
||||
return (arena_malloc_small(arena, size, ind, zero));
|
||||
return (arena_malloc_small(tsd, arena, size, ind, zero));
|
||||
if (likely(size <= large_maxclass))
|
||||
return (arena_malloc_large(arena, size, ind, zero));
|
||||
return (arena_malloc_large(tsd, arena, size, ind, zero));
|
||||
return (huge_malloc(tsd, arena, size, zero, tcache));
|
||||
}
|
||||
|
||||
@ -2329,6 +2583,7 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
|
||||
else if (unlikely(opt_zero))
|
||||
memset(ret, 0, usize);
|
||||
}
|
||||
arena_decay_tick(tsd, arena);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -2515,7 +2770,7 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
}
|
||||
|
||||
void
|
||||
arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
arena_dalloc_small(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
size_t pageind)
|
||||
{
|
||||
arena_chunk_map_bits_t *bitselm;
|
||||
@ -2527,6 +2782,7 @@ arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
}
|
||||
bitselm = arena_bitselm_get(chunk, pageind);
|
||||
arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm);
|
||||
arena_decay_tick(tsd, arena);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
@ -2583,12 +2839,13 @@ arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk,
|
||||
}
|
||||
|
||||
void
|
||||
arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
|
||||
arena_dalloc_large(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, void *ptr)
|
||||
{
|
||||
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
arena_dalloc_large_locked_impl(arena, chunk, ptr, false);
|
||||
malloc_mutex_unlock(&arena->lock);
|
||||
arena_decay_tick(tsd, arena);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -2789,14 +3046,16 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min,
|
||||
}
|
||||
|
||||
bool
|
||||
arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
bool zero)
|
||||
arena_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
|
||||
size_t extra, bool zero)
|
||||
{
|
||||
size_t usize_min, usize_max;
|
||||
|
||||
usize_min = s2u(size);
|
||||
usize_max = s2u(size + extra);
|
||||
if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) {
|
||||
arena_chunk_t *chunk;
|
||||
|
||||
/*
|
||||
* Avoid moving the allocation if the size class can be left the
|
||||
* same.
|
||||
@ -2816,10 +3075,12 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
|
||||
return (true);
|
||||
}
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
arena_decay_tick(tsd, extent_node_arena_get(&chunk->node));
|
||||
return (false);
|
||||
} else {
|
||||
return (huge_ralloc_no_move(ptr, oldsize, usize_min, usize_max,
|
||||
zero));
|
||||
return (huge_ralloc_no_move(tsd, ptr, oldsize, usize_min,
|
||||
usize_max, zero));
|
||||
}
|
||||
}
|
||||
|
||||
@ -2852,7 +3113,7 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
|
||||
size_t copysize;
|
||||
|
||||
/* Try to avoid moving the allocation. */
|
||||
if (!arena_ralloc_no_move(ptr, oldsize, usize, 0, zero))
|
||||
if (!arena_ralloc_no_move(tsd, ptr, oldsize, usize, 0, zero))
|
||||
return (ptr);
|
||||
|
||||
/*
|
||||
@ -2915,15 +3176,36 @@ bool
|
||||
arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult)
|
||||
{
|
||||
|
||||
if (opt_purge != purge_mode_ratio)
|
||||
return (true);
|
||||
if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
|
||||
return (true);
|
||||
atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult);
|
||||
return (false);
|
||||
}
|
||||
|
||||
ssize_t
|
||||
arena_decay_time_default_get(void)
|
||||
{
|
||||
|
||||
return ((ssize_t)atomic_read_z((size_t *)&decay_time_default));
|
||||
}
|
||||
|
||||
bool
|
||||
arena_decay_time_default_set(ssize_t decay_time)
|
||||
{
|
||||
|
||||
if (opt_purge != purge_mode_decay)
|
||||
return (true);
|
||||
if (!arena_decay_time_valid(decay_time))
|
||||
return (true);
|
||||
atomic_write_z((size_t *)&decay_time_default, (size_t)decay_time);
|
||||
return (false);
|
||||
}
|
||||
|
||||
void
|
||||
arena_stats_merge(arena_t *arena, const char **dss, ssize_t *lg_dirty_mult,
|
||||
size_t *nactive, size_t *ndirty, arena_stats_t *astats,
|
||||
ssize_t *decay_time, size_t *nactive, size_t *ndirty, arena_stats_t *astats,
|
||||
malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats,
|
||||
malloc_huge_stats_t *hstats)
|
||||
{
|
||||
@ -2932,6 +3214,7 @@ arena_stats_merge(arena_t *arena, const char **dss, ssize_t *lg_dirty_mult,
|
||||
malloc_mutex_lock(&arena->lock);
|
||||
*dss = dss_prec_names[arena->dss_prec];
|
||||
*lg_dirty_mult = arena->lg_dirty_mult;
|
||||
*decay_time = arena->decay_time;
|
||||
*nactive += arena->nactive;
|
||||
*ndirty += arena->ndirty;
|
||||
|
||||
@ -3050,6 +3333,9 @@ arena_new(unsigned ind)
|
||||
qr_new(&arena->runs_dirty, rd_link);
|
||||
qr_new(&arena->chunks_cache, cc_link);
|
||||
|
||||
if (opt_purge == purge_mode_decay)
|
||||
arena_decay_init(arena, arena_decay_time_default_get());
|
||||
|
||||
ql_new(&arena->huge);
|
||||
if (malloc_mutex_init(&arena->huge_mtx))
|
||||
return (NULL);
|
||||
@ -3227,6 +3513,7 @@ arena_boot(void)
|
||||
unsigned i;
|
||||
|
||||
arena_lg_dirty_mult_default_set(opt_lg_dirty_mult);
|
||||
arena_decay_time_default_set(opt_decay_time);
|
||||
|
||||
/*
|
||||
* Compute the header size such that it is large enough to contain the
|
||||
|
||||
148
src/ctl.c
148
src/ctl.c
@ -92,7 +92,9 @@ CTL_PROTO(opt_abort)
|
||||
CTL_PROTO(opt_dss)
|
||||
CTL_PROTO(opt_lg_chunk)
|
||||
CTL_PROTO(opt_narenas)
|
||||
CTL_PROTO(opt_purge)
|
||||
CTL_PROTO(opt_lg_dirty_mult)
|
||||
CTL_PROTO(opt_decay_time)
|
||||
CTL_PROTO(opt_stats_print)
|
||||
CTL_PROTO(opt_junk)
|
||||
CTL_PROTO(opt_zero)
|
||||
@ -115,10 +117,12 @@ CTL_PROTO(opt_prof_accum)
|
||||
CTL_PROTO(tcache_create)
|
||||
CTL_PROTO(tcache_flush)
|
||||
CTL_PROTO(tcache_destroy)
|
||||
static void arena_i_purge(unsigned arena_ind, bool all);
|
||||
CTL_PROTO(arena_i_purge)
|
||||
static void arena_i_purge(unsigned arena_ind);
|
||||
CTL_PROTO(arena_i_decay)
|
||||
CTL_PROTO(arena_i_dss)
|
||||
CTL_PROTO(arena_i_lg_dirty_mult)
|
||||
CTL_PROTO(arena_i_decay_time)
|
||||
CTL_PROTO(arena_i_chunk_hooks)
|
||||
INDEX_PROTO(arena_i)
|
||||
CTL_PROTO(arenas_bin_i_size)
|
||||
@ -132,6 +136,7 @@ INDEX_PROTO(arenas_hchunk_i)
|
||||
CTL_PROTO(arenas_narenas)
|
||||
CTL_PROTO(arenas_initialized)
|
||||
CTL_PROTO(arenas_lg_dirty_mult)
|
||||
CTL_PROTO(arenas_decay_time)
|
||||
CTL_PROTO(arenas_quantum)
|
||||
CTL_PROTO(arenas_page)
|
||||
CTL_PROTO(arenas_tcache_max)
|
||||
@ -182,6 +187,7 @@ INDEX_PROTO(stats_arenas_i_hchunks_j)
|
||||
CTL_PROTO(stats_arenas_i_nthreads)
|
||||
CTL_PROTO(stats_arenas_i_dss)
|
||||
CTL_PROTO(stats_arenas_i_lg_dirty_mult)
|
||||
CTL_PROTO(stats_arenas_i_decay_time)
|
||||
CTL_PROTO(stats_arenas_i_pactive)
|
||||
CTL_PROTO(stats_arenas_i_pdirty)
|
||||
CTL_PROTO(stats_arenas_i_mapped)
|
||||
@ -260,7 +266,9 @@ static const ctl_named_node_t opt_node[] = {
|
||||
{NAME("dss"), CTL(opt_dss)},
|
||||
{NAME("lg_chunk"), CTL(opt_lg_chunk)},
|
||||
{NAME("narenas"), CTL(opt_narenas)},
|
||||
{NAME("purge"), CTL(opt_purge)},
|
||||
{NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)},
|
||||
{NAME("decay_time"), CTL(opt_decay_time)},
|
||||
{NAME("stats_print"), CTL(opt_stats_print)},
|
||||
{NAME("junk"), CTL(opt_junk)},
|
||||
{NAME("zero"), CTL(opt_zero)},
|
||||
@ -290,8 +298,10 @@ static const ctl_named_node_t tcache_node[] = {
|
||||
|
||||
static const ctl_named_node_t arena_i_node[] = {
|
||||
{NAME("purge"), CTL(arena_i_purge)},
|
||||
{NAME("decay"), CTL(arena_i_decay)},
|
||||
{NAME("dss"), CTL(arena_i_dss)},
|
||||
{NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)},
|
||||
{NAME("decay_time"), CTL(arena_i_decay_time)},
|
||||
{NAME("chunk_hooks"), CTL(arena_i_chunk_hooks)}
|
||||
};
|
||||
static const ctl_named_node_t super_arena_i_node[] = {
|
||||
@ -341,6 +351,7 @@ static const ctl_named_node_t arenas_node[] = {
|
||||
{NAME("narenas"), CTL(arenas_narenas)},
|
||||
{NAME("initialized"), CTL(arenas_initialized)},
|
||||
{NAME("lg_dirty_mult"), CTL(arenas_lg_dirty_mult)},
|
||||
{NAME("decay_time"), CTL(arenas_decay_time)},
|
||||
{NAME("quantum"), CTL(arenas_quantum)},
|
||||
{NAME("page"), CTL(arenas_page)},
|
||||
{NAME("tcache_max"), CTL(arenas_tcache_max)},
|
||||
@ -441,6 +452,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
|
||||
{NAME("nthreads"), CTL(stats_arenas_i_nthreads)},
|
||||
{NAME("dss"), CTL(stats_arenas_i_dss)},
|
||||
{NAME("lg_dirty_mult"), CTL(stats_arenas_i_lg_dirty_mult)},
|
||||
{NAME("decay_time"), CTL(stats_arenas_i_decay_time)},
|
||||
{NAME("pactive"), CTL(stats_arenas_i_pactive)},
|
||||
{NAME("pdirty"), CTL(stats_arenas_i_pdirty)},
|
||||
{NAME("mapped"), CTL(stats_arenas_i_mapped)},
|
||||
@ -523,6 +535,7 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
|
||||
|
||||
astats->dss = dss_prec_names[dss_prec_limit];
|
||||
astats->lg_dirty_mult = -1;
|
||||
astats->decay_time = -1;
|
||||
astats->pactive = 0;
|
||||
astats->pdirty = 0;
|
||||
if (config_stats) {
|
||||
@ -545,8 +558,8 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
|
||||
unsigned i;
|
||||
|
||||
arena_stats_merge(arena, &cstats->dss, &cstats->lg_dirty_mult,
|
||||
&cstats->pactive, &cstats->pdirty, &cstats->astats, cstats->bstats,
|
||||
cstats->lstats, cstats->hstats);
|
||||
&cstats->decay_time, &cstats->pactive, &cstats->pdirty,
|
||||
&cstats->astats, cstats->bstats, cstats->lstats, cstats->hstats);
|
||||
|
||||
for (i = 0; i < NBINS; i++) {
|
||||
cstats->allocated_small += cstats->bstats[i].curregs *
|
||||
@ -1265,7 +1278,9 @@ CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
|
||||
CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
|
||||
CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t)
|
||||
CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t)
|
||||
CTL_RO_NL_GEN(opt_purge, purge_mode_names[opt_purge], const char *)
|
||||
CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t)
|
||||
CTL_RO_NL_GEN(opt_decay_time, opt_decay_time, ssize_t)
|
||||
CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool)
|
||||
CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *)
|
||||
CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t)
|
||||
@ -1539,34 +1554,52 @@ label_return:
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
/* ctl_mutex must be held during execution of this function. */
|
||||
static void
|
||||
arena_i_purge(unsigned arena_ind)
|
||||
arena_i_purge(unsigned arena_ind, bool all)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
|
||||
malloc_mutex_lock(&ctl_mtx);
|
||||
{
|
||||
tsd_t *tsd = tsd_fetch();
|
||||
unsigned narenas = ctl_stats.narenas;
|
||||
|
||||
if (arena_ind == narenas) {
|
||||
unsigned i;
|
||||
bool refreshed;
|
||||
VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
|
||||
VARIABLE_ARRAY(arena_t *, tarenas, narenas);
|
||||
|
||||
tsd = tsd_fetch();
|
||||
for (i = 0, refreshed = false; i < ctl_stats.narenas; i++) {
|
||||
for (i = 0, refreshed = false; i < narenas; i++) {
|
||||
tarenas[i] = arena_get(tsd, i, false, false);
|
||||
if (tarenas[i] == NULL && !refreshed) {
|
||||
tarenas[i] = arena_get(tsd, i, false, true);
|
||||
tarenas[i] = arena_get(tsd, i, false,
|
||||
true);
|
||||
refreshed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (arena_ind == ctl_stats.narenas) {
|
||||
unsigned i;
|
||||
for (i = 0; i < ctl_stats.narenas; i++) {
|
||||
/*
|
||||
* No further need to hold ctl_mtx, since narenas and
|
||||
* tarenas contain everything needed below.
|
||||
*/
|
||||
malloc_mutex_unlock(&ctl_mtx);
|
||||
|
||||
for (i = 0; i < narenas; i++) {
|
||||
if (tarenas[i] != NULL)
|
||||
arena_purge_all(tarenas[i]);
|
||||
arena_purge(tarenas[i], all);
|
||||
}
|
||||
} else {
|
||||
assert(arena_ind < ctl_stats.narenas);
|
||||
if (tarenas[arena_ind] != NULL)
|
||||
arena_purge_all(tarenas[arena_ind]);
|
||||
arena_t *tarena;
|
||||
|
||||
assert(arena_ind < narenas);
|
||||
|
||||
tarena = arena_get(tsd, arena_ind, false, true);
|
||||
|
||||
/* No further need to hold ctl_mtx. */
|
||||
malloc_mutex_unlock(&ctl_mtx);
|
||||
|
||||
if (tarena != NULL)
|
||||
arena_purge(tarena, all);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1578,9 +1611,22 @@ arena_i_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
|
||||
|
||||
READONLY();
|
||||
WRITEONLY();
|
||||
malloc_mutex_lock(&ctl_mtx);
|
||||
arena_i_purge(mib[1]);
|
||||
malloc_mutex_unlock(&ctl_mtx);
|
||||
arena_i_purge(mib[1], true);
|
||||
|
||||
ret = 0;
|
||||
label_return:
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static int
|
||||
arena_i_decay_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
|
||||
void *newp, size_t newlen)
|
||||
{
|
||||
int ret;
|
||||
|
||||
READONLY();
|
||||
WRITEONLY();
|
||||
arena_i_purge(mib[1], false);
|
||||
|
||||
ret = 0;
|
||||
label_return:
|
||||
@ -1677,6 +1723,40 @@ label_return:
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static int
|
||||
arena_i_decay_time_ctl(const size_t *mib, size_t miblen, void *oldp,
|
||||
size_t *oldlenp, void *newp, size_t newlen)
|
||||
{
|
||||
int ret;
|
||||
unsigned arena_ind = mib[1];
|
||||
arena_t *arena;
|
||||
|
||||
arena = arena_get(tsd_fetch(), arena_ind, false, true);
|
||||
if (arena == NULL) {
|
||||
ret = EFAULT;
|
||||
goto label_return;
|
||||
}
|
||||
|
||||
if (oldp != NULL && oldlenp != NULL) {
|
||||
size_t oldval = arena_decay_time_get(arena);
|
||||
READ(oldval, ssize_t);
|
||||
}
|
||||
if (newp != NULL) {
|
||||
if (newlen != sizeof(ssize_t)) {
|
||||
ret = EINVAL;
|
||||
goto label_return;
|
||||
}
|
||||
if (arena_decay_time_set(arena, *(ssize_t *)newp)) {
|
||||
ret = EFAULT;
|
||||
goto label_return;
|
||||
}
|
||||
}
|
||||
|
||||
ret = 0;
|
||||
label_return:
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static int
|
||||
arena_i_chunk_hooks_ctl(const size_t *mib, size_t miblen, void *oldp,
|
||||
size_t *oldlenp, void *newp, size_t newlen)
|
||||
@ -1801,6 +1881,32 @@ label_return:
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static int
|
||||
arenas_decay_time_ctl(const size_t *mib, size_t miblen, void *oldp,
|
||||
size_t *oldlenp, void *newp, size_t newlen)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (oldp != NULL && oldlenp != NULL) {
|
||||
size_t oldval = arena_decay_time_default_get();
|
||||
READ(oldval, ssize_t);
|
||||
}
|
||||
if (newp != NULL) {
|
||||
if (newlen != sizeof(ssize_t)) {
|
||||
ret = EINVAL;
|
||||
goto label_return;
|
||||
}
|
||||
if (arena_decay_time_default_set(*(ssize_t *)newp)) {
|
||||
ret = EFAULT;
|
||||
goto label_return;
|
||||
}
|
||||
}
|
||||
|
||||
ret = 0;
|
||||
label_return:
|
||||
return (ret);
|
||||
}
|
||||
|
||||
CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
|
||||
CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
|
||||
CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t)
|
||||
@ -2002,6 +2108,8 @@ CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t)
|
||||
CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *)
|
||||
CTL_RO_GEN(stats_arenas_i_lg_dirty_mult, ctl_stats.arenas[mib[2]].lg_dirty_mult,
|
||||
ssize_t)
|
||||
CTL_RO_GEN(stats_arenas_i_decay_time, ctl_stats.arenas[mib[2]].decay_time,
|
||||
ssize_t)
|
||||
CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned)
|
||||
CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t)
|
||||
CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t)
|
||||
|
||||
25
src/huge.c
25
src/huge.c
@ -99,6 +99,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
|
||||
} else if (config_fill && unlikely(opt_junk_alloc))
|
||||
memset(ret, 0xa5, size);
|
||||
|
||||
arena_decay_tick(tsd, arena);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
@ -280,7 +281,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) {
|
||||
}
|
||||
|
||||
bool
|
||||
huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min,
|
||||
huge_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize_min,
|
||||
size_t usize_max, bool zero)
|
||||
{
|
||||
|
||||
@ -292,14 +293,19 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min,
|
||||
|
||||
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
|
||||
/* Attempt to expand the allocation in-place. */
|
||||
if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max, zero))
|
||||
if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max,
|
||||
zero)) {
|
||||
arena_decay_tick(tsd, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
/* Try again, this time with usize_min. */
|
||||
if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
|
||||
CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr,
|
||||
oldsize, usize_min, zero))
|
||||
oldsize, usize_min, zero)) {
|
||||
arena_decay_tick(tsd, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Avoid moving the allocation if the existing chunk size accommodates
|
||||
@ -309,12 +315,17 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min,
|
||||
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
|
||||
huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max,
|
||||
zero);
|
||||
arena_decay_tick(tsd, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
|
||||
/* Attempt to shrink the allocation in-place. */
|
||||
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max))
|
||||
return (huge_ralloc_no_move_shrink(ptr, oldsize, usize_max));
|
||||
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
|
||||
if (!huge_ralloc_no_move_shrink(ptr, oldsize, usize_max)) {
|
||||
arena_decay_tick(tsd, huge_aalloc(ptr));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
return (true);
|
||||
}
|
||||
|
||||
@ -336,7 +347,7 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize,
|
||||
size_t copysize;
|
||||
|
||||
/* Try to avoid moving the allocation. */
|
||||
if (!huge_ralloc_no_move(ptr, oldsize, usize, usize, zero))
|
||||
if (!huge_ralloc_no_move(tsd, ptr, oldsize, usize, usize, zero))
|
||||
return (ptr);
|
||||
|
||||
/*
|
||||
@ -373,6 +384,8 @@ huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
|
||||
arena_chunk_dalloc_huge(extent_node_arena_get(node),
|
||||
extent_node_addr_get(node), extent_node_size_get(node));
|
||||
idalloctm(tsd, node, tcache, true, true);
|
||||
|
||||
arena_decay_tick(tsd, arena);
|
||||
}
|
||||
|
||||
arena_t *
|
||||
|
||||
@ -577,6 +577,17 @@ arena_tdata_get_hard(tsd_t *tsd, unsigned ind)
|
||||
* (narenas_tdata - narenas_actual));
|
||||
}
|
||||
|
||||
/* Copy/initialize tickers. */
|
||||
for (i = 0; i < narenas_actual; i++) {
|
||||
if (i < narenas_tdata_old) {
|
||||
ticker_copy(&arenas_tdata[i].decay_ticker,
|
||||
&arenas_tdata_old[i].decay_ticker);
|
||||
} else {
|
||||
ticker_init(&arenas_tdata[i].decay_ticker,
|
||||
DECAY_NTICKS_PER_UPDATE);
|
||||
}
|
||||
}
|
||||
|
||||
/* Read the refreshed tdata array. */
|
||||
tdata = &arenas_tdata[ind];
|
||||
label_return:
|
||||
@ -1120,8 +1131,27 @@ malloc_conf_init(void)
|
||||
}
|
||||
CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
|
||||
SIZE_T_MAX, false)
|
||||
if (strncmp("purge", k, klen) == 0) {
|
||||
int i;
|
||||
bool match = false;
|
||||
for (i = 0; i < purge_mode_limit; i++) {
|
||||
if (strncmp(purge_mode_names[i], v,
|
||||
vlen) == 0) {
|
||||
opt_purge = (purge_mode_t)i;
|
||||
match = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!match) {
|
||||
malloc_conf_error("Invalid conf value",
|
||||
k, klen, v, vlen);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
|
||||
-1, (sizeof(size_t) << 3) - 1)
|
||||
CONF_HANDLE_SSIZE_T(opt_decay_time, "decay_time", -1,
|
||||
TIME_SEC_MAX);
|
||||
CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true)
|
||||
if (config_fill) {
|
||||
if (CONF_MATCH("junk")) {
|
||||
@ -2344,12 +2374,12 @@ label_oom:
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE_C size_t
|
||||
ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
|
||||
size_t alignment, bool zero)
|
||||
ixallocx_helper(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
|
||||
size_t extra, size_t alignment, bool zero)
|
||||
{
|
||||
size_t usize;
|
||||
|
||||
if (ixalloc(ptr, old_usize, size, extra, alignment, zero))
|
||||
if (ixalloc(tsd, ptr, old_usize, size, extra, alignment, zero))
|
||||
return (old_usize);
|
||||
usize = isalloc(ptr, config_prof);
|
||||
|
||||
@ -2357,14 +2387,15 @@ ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
|
||||
}
|
||||
|
||||
static size_t
|
||||
ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
|
||||
size_t alignment, bool zero, prof_tctx_t *tctx)
|
||||
ixallocx_prof_sample(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
|
||||
size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx)
|
||||
{
|
||||
size_t usize;
|
||||
|
||||
if (tctx == NULL)
|
||||
return (old_usize);
|
||||
usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero);
|
||||
usize = ixallocx_helper(tsd, ptr, old_usize, size, extra, alignment,
|
||||
zero);
|
||||
|
||||
return (usize);
|
||||
}
|
||||
@ -2390,11 +2421,11 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
|
||||
assert(usize_max != 0);
|
||||
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
|
||||
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
|
||||
usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
|
||||
usize = ixallocx_prof_sample(tsd, ptr, old_usize, size, extra,
|
||||
alignment, zero, tctx);
|
||||
} else {
|
||||
usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
|
||||
zero);
|
||||
usize = ixallocx_helper(tsd, ptr, old_usize, size, extra,
|
||||
alignment, zero);
|
||||
}
|
||||
if (usize == old_usize) {
|
||||
prof_alloc_rollback(tsd, tctx, false);
|
||||
@ -2441,8 +2472,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
|
||||
usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
|
||||
alignment, zero);
|
||||
} else {
|
||||
usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
|
||||
zero);
|
||||
usize = ixallocx_helper(tsd, ptr, old_usize, size, extra,
|
||||
alignment, zero);
|
||||
}
|
||||
if (unlikely(usize == old_usize))
|
||||
goto label_not_resized;
|
||||
|
||||
40
src/stats.c
40
src/stats.c
@ -258,7 +258,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
{
|
||||
unsigned nthreads;
|
||||
const char *dss;
|
||||
ssize_t lg_dirty_mult;
|
||||
ssize_t lg_dirty_mult, decay_time;
|
||||
size_t page, pactive, pdirty, mapped;
|
||||
size_t metadata_mapped, metadata_allocated;
|
||||
uint64_t npurge, nmadvise, purged;
|
||||
@ -278,6 +278,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n",
|
||||
dss);
|
||||
CTL_M2_GET("stats.arenas.0.lg_dirty_mult", i, &lg_dirty_mult, ssize_t);
|
||||
if (opt_purge == purge_mode_ratio) {
|
||||
if (lg_dirty_mult >= 0) {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"min active:dirty page ratio: %u:1\n",
|
||||
@ -286,15 +287,23 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"min active:dirty page ratio: N/A\n");
|
||||
}
|
||||
}
|
||||
CTL_M2_GET("stats.arenas.0.decay_time", i, &decay_time, ssize_t);
|
||||
if (opt_purge == purge_mode_decay) {
|
||||
if (decay_time >= 0) {
|
||||
malloc_cprintf(write_cb, cbopaque, "decay time: %zd\n",
|
||||
decay_time);
|
||||
} else
|
||||
malloc_cprintf(write_cb, cbopaque, "decay time: N/A\n");
|
||||
}
|
||||
CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t);
|
||||
CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t);
|
||||
CTL_M2_GET("stats.arenas.0.npurge", i, &npurge, uint64_t);
|
||||
CTL_M2_GET("stats.arenas.0.nmadvise", i, &nmadvise, uint64_t);
|
||||
CTL_M2_GET("stats.arenas.0.purged", i, &purged, uint64_t);
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"dirty pages: %zu:%zu active:dirty, %"FMTu64" sweep%s, %"FMTu64
|
||||
" madvise%s, %"FMTu64" purged\n", pactive, pdirty, npurge, npurge ==
|
||||
1 ? "" : "s", nmadvise, nmadvise == 1 ? "" : "s", purged);
|
||||
"purging: dirty: %zu, sweeps: %"FMTu64", madvises: %"FMTu64", "
|
||||
"purged: %"FMTu64"\n", pdirty, npurge, nmadvise, purged);
|
||||
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
" allocated nmalloc ndalloc"
|
||||
@ -486,7 +495,13 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
OPT_WRITE_SIZE_T(lg_chunk)
|
||||
OPT_WRITE_CHAR_P(dss)
|
||||
OPT_WRITE_SIZE_T(narenas)
|
||||
OPT_WRITE_SSIZE_T_MUTABLE(lg_dirty_mult, arenas.lg_dirty_mult)
|
||||
OPT_WRITE_CHAR_P(purge)
|
||||
if (opt_purge == purge_mode_ratio) {
|
||||
OPT_WRITE_SSIZE_T_MUTABLE(lg_dirty_mult,
|
||||
arenas.lg_dirty_mult)
|
||||
}
|
||||
if (opt_purge == purge_mode_decay)
|
||||
OPT_WRITE_SSIZE_T_MUTABLE(decay_time, arenas.decay_time)
|
||||
OPT_WRITE_BOOL(stats_print)
|
||||
OPT_WRITE_CHAR_P(junk)
|
||||
OPT_WRITE_SIZE_T(quarantine)
|
||||
@ -531,13 +546,22 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
|
||||
malloc_cprintf(write_cb, cbopaque, "Page size: %zu\n", sv);
|
||||
|
||||
CTL_GET("arenas.lg_dirty_mult", &ssv, ssize_t);
|
||||
if (opt_purge == purge_mode_ratio) {
|
||||
if (ssv >= 0) {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"Min active:dirty page ratio per arena: %u:1\n",
|
||||
(1U << ssv));
|
||||
"Min active:dirty page ratio per arena: "
|
||||
"%u:1\n", (1U << ssv));
|
||||
} else {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"Min active:dirty page ratio per arena: N/A\n");
|
||||
"Min active:dirty page ratio per arena: "
|
||||
"N/A\n");
|
||||
}
|
||||
}
|
||||
CTL_GET("arenas.decay_time", &ssv, ssize_t);
|
||||
if (opt_purge == purge_mode_decay) {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
"Unused dirty page decay time: %zd%s\n",
|
||||
ssv, (ssv < 0) ? " (no decay)" : "");
|
||||
}
|
||||
if (je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0) == 0) {
|
||||
malloc_cprintf(write_cb, cbopaque,
|
||||
|
||||
@ -75,7 +75,7 @@ tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
|
||||
{
|
||||
void *ret;
|
||||
|
||||
arena_tcache_fill_small(arena, tbin, binind, config_prof ?
|
||||
arena_tcache_fill_small(tsd, arena, tbin, binind, config_prof ?
|
||||
tcache->prof_accumbytes : 0);
|
||||
if (config_prof)
|
||||
tcache->prof_accumbytes = 0;
|
||||
@ -143,6 +143,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
|
||||
}
|
||||
}
|
||||
malloc_mutex_unlock(&bin->lock);
|
||||
arena_decay_ticks(tsd, bin_arena, nflush - ndeferred);
|
||||
}
|
||||
if (config_stats && !merged_stats) {
|
||||
/*
|
||||
@ -226,6 +227,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
||||
malloc_mutex_unlock(&locked_arena->lock);
|
||||
if (config_prof && idump)
|
||||
prof_idump();
|
||||
arena_decay_ticks(tsd, locked_arena, nflush - ndeferred);
|
||||
}
|
||||
if (config_stats && !merged_stats) {
|
||||
/*
|
||||
|
||||
@ -147,6 +147,10 @@ time_divide(const struct timespec *time, const struct timespec *divisor)
|
||||
return (t / d);
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef time_update
|
||||
#define time_update JEMALLOC_N(time_update_impl)
|
||||
#endif
|
||||
bool
|
||||
time_update(struct timespec *time)
|
||||
{
|
||||
@ -184,3 +188,8 @@ time_update(struct timespec *time)
|
||||
assert(time_valid(time));
|
||||
return (false);
|
||||
}
|
||||
#ifdef JEMALLOC_JET
|
||||
#undef time_update
|
||||
#define time_update JEMALLOC_N(time_update)
|
||||
time_update_t *time_update = JEMALLOC_N(time_update_impl);
|
||||
#endif
|
||||
|
||||
370
test/unit/decay.c
Normal file
370
test/unit/decay.c
Normal file
@ -0,0 +1,370 @@
|
||||
#include "test/jemalloc_test.h"
|
||||
|
||||
const char *malloc_conf = "purge:decay,decay_time:1";
|
||||
|
||||
static time_update_t *time_update_orig;
|
||||
|
||||
static unsigned nupdates_mock;
|
||||
static struct timespec time_mock;
|
||||
static bool nonmonotonic_mock;
|
||||
|
||||
static bool
|
||||
time_update_mock(struct timespec *time)
|
||||
{
|
||||
|
||||
nupdates_mock++;
|
||||
if (!nonmonotonic_mock)
|
||||
time_copy(time, &time_mock);
|
||||
return (nonmonotonic_mock);
|
||||
}
|
||||
|
||||
TEST_BEGIN(test_decay_ticks)
|
||||
{
|
||||
ticker_t *decay_ticker;
|
||||
unsigned tick0, tick1;
|
||||
size_t sz, huge0, large0;
|
||||
void *p;
|
||||
unsigned tcache_ind;
|
||||
|
||||
test_skip_if(opt_purge != purge_mode_decay);
|
||||
|
||||
decay_ticker = decay_ticker_get(tsd_fetch(), 0);
|
||||
assert_ptr_not_null(decay_ticker,
|
||||
"Unexpected failure getting decay ticker");
|
||||
|
||||
sz = sizeof(size_t);
|
||||
assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
assert_d_eq(mallctl("arenas.lrun.0.size", &large0, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
|
||||
/* malloc(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = malloc(huge0);
|
||||
assert_ptr_not_null(p, "Unexpected malloc() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during malloc()");
|
||||
/* free(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
free(p);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during free()");
|
||||
|
||||
/* calloc(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = calloc(1, huge0);
|
||||
assert_ptr_not_null(p, "Unexpected calloc() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during calloc()");
|
||||
free(p);
|
||||
|
||||
/* posix_memalign(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
assert_d_eq(posix_memalign(&p, sizeof(size_t), huge0), 0,
|
||||
"Unexpected posix_memalign() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during posix_memalign()");
|
||||
free(p);
|
||||
|
||||
/* aligned_alloc(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = aligned_alloc(sizeof(size_t), huge0);
|
||||
assert_ptr_not_null(p, "Unexpected aligned_alloc() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during aligned_alloc()");
|
||||
free(p);
|
||||
|
||||
/* realloc(). */
|
||||
/* Allocate. */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = realloc(NULL, huge0);
|
||||
assert_ptr_not_null(p, "Unexpected realloc() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
||||
/* Reallocate. */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = realloc(p, huge0);
|
||||
assert_ptr_not_null(p, "Unexpected realloc() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
||||
/* Deallocate. */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
realloc(p, 0);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0, "Expected ticker to tick during realloc()");
|
||||
|
||||
/* Huge mallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = mallocx(huge0, 0);
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during huge mallocx()");
|
||||
/* Huge rallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = rallocx(p, huge0, 0);
|
||||
assert_ptr_not_null(p, "Unexpected rallocx() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during huge rallocx()");
|
||||
/* Huge xallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
xallocx(p, huge0, 0, 0);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during huge xallocx()");
|
||||
/* Huge dallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
dallocx(p, 0);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during huge dallocx()");
|
||||
/* Huge sdallocx(). */
|
||||
p = mallocx(huge0, 0);
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
sdallocx(p, huge0, 0);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during huge sdallocx()");
|
||||
|
||||
/* Large mallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = mallocx(large0, MALLOCX_TCACHE_NONE);
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during large mallocx()");
|
||||
/* Large rallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = rallocx(p, large0, MALLOCX_TCACHE_NONE);
|
||||
assert_ptr_not_null(p, "Unexpected rallocx() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during large rallocx()");
|
||||
/* Large xallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
xallocx(p, large0, 0, MALLOCX_TCACHE_NONE);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during large xallocx()");
|
||||
/* Large dallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
dallocx(p, MALLOCX_TCACHE_NONE);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during large dallocx()");
|
||||
/* Large sdallocx(). */
|
||||
p = mallocx(large0, MALLOCX_TCACHE_NONE);
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
sdallocx(p, large0, MALLOCX_TCACHE_NONE);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during large sdallocx()");
|
||||
|
||||
/* Small mallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = mallocx(1, MALLOCX_TCACHE_NONE);
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during small mallocx()");
|
||||
/* Small rallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = rallocx(p, 1, MALLOCX_TCACHE_NONE);
|
||||
assert_ptr_not_null(p, "Unexpected rallocx() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during small rallocx()");
|
||||
/* Small xallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
xallocx(p, 1, 0, MALLOCX_TCACHE_NONE);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during small xallocx()");
|
||||
/* Small dallocx(). */
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
dallocx(p, MALLOCX_TCACHE_NONE);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during small dallocx()");
|
||||
/* Small sdallocx(). */
|
||||
p = mallocx(1, MALLOCX_TCACHE_NONE);
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
sdallocx(p, 1, MALLOCX_TCACHE_NONE);
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during small sdallocx()");
|
||||
|
||||
/* tcache fill. */
|
||||
sz = sizeof(unsigned);
|
||||
assert_d_eq(mallctl("tcache.create", &tcache_ind, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
p = mallocx(1, MALLOCX_TCACHE(tcache_ind));
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during tcache fill");
|
||||
/* tcache flush. */
|
||||
dallocx(p, MALLOCX_TCACHE(tcache_ind));
|
||||
tick0 = ticker_read(decay_ticker);
|
||||
assert_d_eq(mallctl("tcache.flush", NULL, NULL, &tcache_ind,
|
||||
sizeof(unsigned)), 0, "Unexpected mallctl failure");
|
||||
tick1 = ticker_read(decay_ticker);
|
||||
assert_u32_ne(tick1, tick0,
|
||||
"Expected ticker to tick during tcache flush");
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_decay_ticker)
|
||||
{
|
||||
#define NPS 1024
|
||||
int flags = (MALLOCX_ARENA(0) | MALLOCX_TCACHE_NONE);
|
||||
void *ps[NPS];
|
||||
uint64_t epoch, npurge0, npurge1;
|
||||
size_t sz, tcache_max, large;
|
||||
unsigned i, nupdates0;
|
||||
struct timespec time, decay_time, deadline;
|
||||
|
||||
test_skip_if(opt_purge != purge_mode_decay);
|
||||
|
||||
/*
|
||||
* Allocate a bunch of large objects, pause the clock, deallocate the
|
||||
* objects, restore the clock, then [md]allocx() in a tight loop to
|
||||
* verify the ticker triggers purging.
|
||||
*/
|
||||
|
||||
sz = sizeof(size_t);
|
||||
assert_d_eq(mallctl("arenas.tcache_max", &tcache_max, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
large = nallocx(tcache_max + 1, flags);
|
||||
|
||||
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(uint64_t)), 0,
|
||||
"Unexpected mallctl failure");
|
||||
sz = sizeof(uint64_t);
|
||||
assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge0, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
|
||||
for (i = 0; i < NPS; i++) {
|
||||
ps[i] = mallocx(large, flags);
|
||||
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
|
||||
}
|
||||
|
||||
nupdates_mock = 0;
|
||||
time_init(&time_mock, 0, 0);
|
||||
time_update(&time_mock);
|
||||
nonmonotonic_mock = false;
|
||||
|
||||
time_update_orig = time_update;
|
||||
time_update = time_update_mock;
|
||||
|
||||
for (i = 0; i < NPS; i++) {
|
||||
dallocx(ps[i], flags);
|
||||
nupdates0 = nupdates_mock;
|
||||
assert_d_eq(mallctl("arena.0.decay", NULL, NULL, NULL, 0), 0,
|
||||
"Unexpected arena.0.decay failure");
|
||||
assert_u_gt(nupdates_mock, nupdates0,
|
||||
"Expected time_update() to be called");
|
||||
}
|
||||
|
||||
time_update = time_update_orig;
|
||||
|
||||
time_init(&time, 0, 0);
|
||||
time_update(&time);
|
||||
time_init(&decay_time, opt_decay_time, 0);
|
||||
time_copy(&deadline, &time);
|
||||
time_add(&deadline, &decay_time);
|
||||
do {
|
||||
for (i = 0; i < DECAY_NTICKS_PER_UPDATE / 2; i++) {
|
||||
void *p = mallocx(1, flags);
|
||||
assert_ptr_not_null(p, "Unexpected mallocx() failure");
|
||||
dallocx(p, flags);
|
||||
}
|
||||
assert_d_eq(mallctl("epoch", NULL, NULL, &epoch,
|
||||
sizeof(uint64_t)), 0, "Unexpected mallctl failure");
|
||||
sz = sizeof(uint64_t);
|
||||
assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge1, &sz,
|
||||
NULL, 0), 0, "Unexpected mallctl failure");
|
||||
|
||||
time_update(&time);
|
||||
} while (time_compare(&time, &deadline) <= 0 && npurge1 == npurge0);
|
||||
|
||||
assert_u64_gt(npurge1, npurge0, "Expected purging to occur");
|
||||
#undef NPS
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_decay_nonmonotonic)
|
||||
{
|
||||
#define NPS (SMOOTHSTEP_NSTEPS + 1)
|
||||
int flags = (MALLOCX_ARENA(0) | MALLOCX_TCACHE_NONE);
|
||||
void *ps[NPS];
|
||||
uint64_t epoch, npurge0, npurge1;
|
||||
size_t sz, large0;
|
||||
unsigned i, nupdates0;
|
||||
|
||||
test_skip_if(opt_purge != purge_mode_decay);
|
||||
|
||||
sz = sizeof(size_t);
|
||||
assert_d_eq(mallctl("arenas.lrun.0.size", &large0, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
|
||||
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(uint64_t)), 0,
|
||||
"Unexpected mallctl failure");
|
||||
sz = sizeof(uint64_t);
|
||||
assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge0, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
|
||||
nupdates_mock = 0;
|
||||
time_init(&time_mock, 0, 0);
|
||||
time_update(&time_mock);
|
||||
nonmonotonic_mock = true;
|
||||
|
||||
time_update_orig = time_update;
|
||||
time_update = time_update_mock;
|
||||
|
||||
for (i = 0; i < NPS; i++) {
|
||||
ps[i] = mallocx(large0, flags);
|
||||
assert_ptr_not_null(ps[i], "Unexpected mallocx() failure");
|
||||
}
|
||||
|
||||
for (i = 0; i < NPS; i++) {
|
||||
dallocx(ps[i], flags);
|
||||
nupdates0 = nupdates_mock;
|
||||
assert_d_eq(mallctl("arena.0.decay", NULL, NULL, NULL, 0), 0,
|
||||
"Unexpected arena.0.decay failure");
|
||||
assert_u_gt(nupdates_mock, nupdates0,
|
||||
"Expected time_update() to be called");
|
||||
}
|
||||
|
||||
assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(uint64_t)), 0,
|
||||
"Unexpected mallctl failure");
|
||||
sz = sizeof(uint64_t);
|
||||
assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge1, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
|
||||
assert_u64_gt(npurge1, npurge0, "Expected purging to occur");
|
||||
|
||||
time_update = time_update_orig;
|
||||
#undef NPS
|
||||
}
|
||||
TEST_END
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
|
||||
return (test(
|
||||
test_decay_ticks,
|
||||
test_decay_ticker,
|
||||
test_decay_nonmonotonic));
|
||||
}
|
||||
@ -164,7 +164,9 @@ TEST_BEGIN(test_mallctl_opt)
|
||||
TEST_MALLCTL_OPT(size_t, lg_chunk, always);
|
||||
TEST_MALLCTL_OPT(const char *, dss, always);
|
||||
TEST_MALLCTL_OPT(size_t, narenas, always);
|
||||
TEST_MALLCTL_OPT(const char *, purge, always);
|
||||
TEST_MALLCTL_OPT(ssize_t, lg_dirty_mult, always);
|
||||
TEST_MALLCTL_OPT(ssize_t, decay_time, always);
|
||||
TEST_MALLCTL_OPT(bool, stats_print, always);
|
||||
TEST_MALLCTL_OPT(const char *, junk, fill);
|
||||
TEST_MALLCTL_OPT(size_t, quarantine, fill);
|
||||
@ -355,6 +357,8 @@ TEST_BEGIN(test_arena_i_lg_dirty_mult)
|
||||
ssize_t lg_dirty_mult, orig_lg_dirty_mult, prev_lg_dirty_mult;
|
||||
size_t sz = sizeof(ssize_t);
|
||||
|
||||
test_skip_if(opt_purge != purge_mode_ratio);
|
||||
|
||||
assert_d_eq(mallctl("arena.0.lg_dirty_mult", &orig_lg_dirty_mult, &sz,
|
||||
NULL, 0), 0, "Unexpected mallctl() failure");
|
||||
|
||||
@ -382,6 +386,39 @@ TEST_BEGIN(test_arena_i_lg_dirty_mult)
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_arena_i_decay_time)
|
||||
{
|
||||
ssize_t decay_time, orig_decay_time, prev_decay_time;
|
||||
size_t sz = sizeof(ssize_t);
|
||||
|
||||
test_skip_if(opt_purge != purge_mode_decay);
|
||||
|
||||
assert_d_eq(mallctl("arena.0.decay_time", &orig_decay_time, &sz,
|
||||
NULL, 0), 0, "Unexpected mallctl() failure");
|
||||
|
||||
decay_time = -2;
|
||||
assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL,
|
||||
&decay_time, sizeof(ssize_t)), EFAULT,
|
||||
"Unexpected mallctl() success");
|
||||
|
||||
decay_time = TIME_SEC_MAX;
|
||||
assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL,
|
||||
&decay_time, sizeof(ssize_t)), 0,
|
||||
"Unexpected mallctl() failure");
|
||||
|
||||
for (prev_decay_time = decay_time, decay_time = -1;
|
||||
decay_time < 20; prev_decay_time = decay_time, decay_time++) {
|
||||
ssize_t old_decay_time;
|
||||
|
||||
assert_d_eq(mallctl("arena.0.decay_time", &old_decay_time,
|
||||
&sz, &decay_time, sizeof(ssize_t)), 0,
|
||||
"Unexpected mallctl() failure");
|
||||
assert_zd_eq(old_decay_time, prev_decay_time,
|
||||
"Unexpected old arena.0.decay_time");
|
||||
}
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_arena_i_purge)
|
||||
{
|
||||
unsigned narenas;
|
||||
@ -402,6 +439,26 @@ TEST_BEGIN(test_arena_i_purge)
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_arena_i_decay)
|
||||
{
|
||||
unsigned narenas;
|
||||
size_t sz = sizeof(unsigned);
|
||||
size_t mib[3];
|
||||
size_t miblen = 3;
|
||||
|
||||
assert_d_eq(mallctl("arena.0.decay", NULL, NULL, NULL, 0), 0,
|
||||
"Unexpected mallctl() failure");
|
||||
|
||||
assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl() failure");
|
||||
assert_d_eq(mallctlnametomib("arena.0.decay", mib, &miblen), 0,
|
||||
"Unexpected mallctlnametomib() failure");
|
||||
mib[1] = narenas;
|
||||
assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
|
||||
"Unexpected mallctlbymib() failure");
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_arena_i_dss)
|
||||
{
|
||||
const char *dss_prec_old, *dss_prec_new;
|
||||
@ -466,6 +523,8 @@ TEST_BEGIN(test_arenas_lg_dirty_mult)
|
||||
ssize_t lg_dirty_mult, orig_lg_dirty_mult, prev_lg_dirty_mult;
|
||||
size_t sz = sizeof(ssize_t);
|
||||
|
||||
test_skip_if(opt_purge != purge_mode_ratio);
|
||||
|
||||
assert_d_eq(mallctl("arenas.lg_dirty_mult", &orig_lg_dirty_mult, &sz,
|
||||
NULL, 0), 0, "Unexpected mallctl() failure");
|
||||
|
||||
@ -493,6 +552,39 @@ TEST_BEGIN(test_arenas_lg_dirty_mult)
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_arenas_decay_time)
|
||||
{
|
||||
ssize_t decay_time, orig_decay_time, prev_decay_time;
|
||||
size_t sz = sizeof(ssize_t);
|
||||
|
||||
test_skip_if(opt_purge != purge_mode_decay);
|
||||
|
||||
assert_d_eq(mallctl("arenas.decay_time", &orig_decay_time, &sz,
|
||||
NULL, 0), 0, "Unexpected mallctl() failure");
|
||||
|
||||
decay_time = -2;
|
||||
assert_d_eq(mallctl("arenas.decay_time", NULL, NULL,
|
||||
&decay_time, sizeof(ssize_t)), EFAULT,
|
||||
"Unexpected mallctl() success");
|
||||
|
||||
decay_time = TIME_SEC_MAX;
|
||||
assert_d_eq(mallctl("arenas.decay_time", NULL, NULL,
|
||||
&decay_time, sizeof(ssize_t)), 0,
|
||||
"Expected mallctl() failure");
|
||||
|
||||
for (prev_decay_time = decay_time, decay_time = -1;
|
||||
decay_time < 20; prev_decay_time = decay_time, decay_time++) {
|
||||
ssize_t old_decay_time;
|
||||
|
||||
assert_d_eq(mallctl("arenas.decay_time", &old_decay_time,
|
||||
&sz, &decay_time, sizeof(ssize_t)), 0,
|
||||
"Unexpected mallctl() failure");
|
||||
assert_zd_eq(old_decay_time, prev_decay_time,
|
||||
"Unexpected old arenas.decay_time");
|
||||
}
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_arenas_constants)
|
||||
{
|
||||
|
||||
@ -621,10 +713,13 @@ main(void)
|
||||
test_tcache,
|
||||
test_thread_arena,
|
||||
test_arena_i_lg_dirty_mult,
|
||||
test_arena_i_decay_time,
|
||||
test_arena_i_purge,
|
||||
test_arena_i_decay,
|
||||
test_arena_i_dss,
|
||||
test_arenas_initialized,
|
||||
test_arenas_lg_dirty_mult,
|
||||
test_arenas_decay_time,
|
||||
test_arenas_constants,
|
||||
test_arenas_bin_constants,
|
||||
test_arenas_lrun_constants,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user