project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove --disable-tcache.

Browse Source 
Simplify configuration by removing the --disable-tcache option, but
replace the testing for that configuration with
--with-malloc-conf=tcache:false.

Fix the thread.arena and thread.tcache.flush mallctls to work correctly
if tcache is disabled.

This partially resolves #580.
This commit is contained in:
Jason Evans 2017-04-20 17:21:37 -07:00
parent 5aa46f027d
commit 4403c9ab44
25 changed files with 188 additions and 412 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.travis.yml
View File

@ -21,7 +21,7 @@ matrix:
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
@ -31,7 +31,7 @@ matrix:
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: osx
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
@ -45,7 +45,7 @@ matrix:
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
@ -65,7 +65,7 @@ matrix:
packages:
- gcc-multilib
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
addons:
apt:
packages:
@ -75,13 +75,13 @@ matrix:
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
- os: linux
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
before_script:

5
INSTALL
View File

@ -153,11 +153,6 @@ any of the following arguments (not a definitive list) to 'configure':
Statically link against the specified libunwind.a rather than dynamically
linking with -lunwind.
--disable-tcache
Disable thread-specific caches for small objects. Objects are cached and
released in bulk, thus reducing the total number of mutex operations. See
the "opt.tcache" option for usage details.
--disable-munmap
Disable virtual memory deallocation via munmap(2); instead keep track of
the virtual memory for later use. munmap() is disabled by default (i.e.

17
configure.ac
View File

@ -1137,22 +1137,6 @@ if test "x$enable_prof" = "x1" ; then
fi
AC_SUBST([enable_prof])
dnl Enable thread-specific caching by default.
AC_ARG_ENABLE([tcache],
[AS_HELP_STRING([--disable-tcache], [Disable per thread caches])],
[if test "x$enable_tcache" = "xno" ; then
enable_tcache="0"
else
enable_tcache="1"
fi
],
[enable_tcache="1"]
)
if test "x$enable_tcache" = "x1" ; then
AC_DEFINE([JEMALLOC_TCACHE], [ ])
fi
AC_SUBST([enable_tcache])
dnl Indicate whether adjacent virtual memory mappings automatically coalesce
dnl (and fragment on demand).
if test "x${maps_coalesce}" = "x1" ; then
@ -2181,7 +2165,6 @@ AC_MSG_RESULT([prof : ${enable_prof}])
AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}])
AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}])
AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}])
AC_MSG_RESULT([tcache : ${enable_tcache}])
AC_MSG_RESULT([fill : ${enable_fill}])
AC_MSG_RESULT([utrace : ${enable_utrace}])
AC_MSG_RESULT([xmalloc : ${enable_xmalloc}])

34
doc/jemalloc.xml.in
View File

@ -510,13 +510,12 @@ for (i = 0; i < nbins; i++) {
sense to reduce the number of arenas if an application does not make much
use of the allocation functions.</para>
<para>In addition to multiple arenas, unless
<option>--disable-tcache</option> is specified during configuration, this
allocator supports thread-specific caching, in order to make it possible to
completely avoid synchronization for most allocation requests. Such caching
allows very fast allocation in the common case, but it increases memory
usage and fragmentation, since a bounded number of objects can remain
allocated in each thread cache.</para>
<para>In addition to multiple arenas, this allocator supports
thread-specific caching, in order to make it possible to completely avoid
synchronization for most allocation requests. Such caching allows very fast
allocation in the common case, but it increases memory usage and
fragmentation, since a bounded number of objects can remain allocated in
each thread cache.</para>
<para>Memory is conceptually broken into extents. Extents are always
aligned to multiples of the page size. This alignment makes it possible to
@ -839,16 +838,6 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
build configuration.</para></listitem>
</varlistentry>
<varlistentry id="config.tcache">
<term>
<mallctl>config.tcache</mallctl>
(<type>bool</type>)
<literal>r-</literal>
</term>
<listitem><para><option>--disable-tcache</option> was not specified
during build configuration.</para></listitem>
</varlistentry>
<varlistentry id="config.tls">
<term>
<mallctl>config.tls</mallctl>
@ -1095,7 +1084,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<mallctl>opt.tcache</mallctl>
(<type>bool</type>)
<literal>r-</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Thread-specific caching (tcache) enabled/disabled. When
there are multiple threads, each thread uses a tcache for objects up to
@ -1112,7 +1100,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<mallctl>opt.lg_tcache_max</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Maximum size class (log base 2) to cache in the
thread-specific cache (tcache). At a minimum, all small size classes
@ -1370,7 +1357,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<mallctl>thread.tcache.enabled</mallctl>
(<type>bool</type>)
<literal>rw</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Enable/disable calling thread's tcache. The tcache is
implicitly flushed as a side effect of becoming
@ -1384,7 +1370,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<mallctl>thread.tcache.flush</mallctl>
(<type>void</type>)
<literal>--</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Flush calling thread's thread-specific cache (tcache).
This interface releases all cached objects and internal data structures
@ -1440,7 +1425,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<mallctl>tcache.create</mallctl>
(<type>unsigned</type>)
<literal>r-</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Create an explicit thread-specific cache (tcache) and
return an identifier that can be passed to the <link
@ -1457,7 +1441,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<mallctl>tcache.flush</mallctl>
(<type>unsigned</type>)
<literal>-w</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Flush the specified thread-specific cache (tcache). The
same considerations apply to this interface as to <link
@ -1471,7 +1454,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<mallctl>tcache.destroy</mallctl>
(<type>unsigned</type>)
<literal>-w</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Flush the specified thread-specific cache (tcache) and
make the identifier available for use during a future tcache creation.
@ -1873,7 +1855,6 @@ struct extent_hooks_s {
<mallctl>arenas.tcache_max</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Maximum thread-cached size class.</para></listitem>
</varlistentry>
@ -1892,7 +1873,6 @@ struct extent_hooks_s {
<mallctl>arenas.nhbins</mallctl>
(<type>unsigned</type>)
<literal>r-</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Total number of thread cache bin size
classes.</para></listitem>
@ -2575,7 +2555,6 @@ struct extent_hooks_s {
<mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nfills</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option> <option>--enable-tcache</option>]
</term>
<listitem><para>Cumulative number of tcache fills.</para></listitem>
</varlistentry>
@ -2585,7 +2564,6 @@ struct extent_hooks_s {
<mallctl>stats.arenas.&lt;i&gt;.bins.&lt;j&gt;.nflushes</mallctl>
(<type>uint64_t</type>)
<literal>r-</literal>
[<option>--enable-stats</option> <option>--enable-tcache</option>]
</term>
<listitem><para>Cumulative number of tcache flushes.</para></listitem>
</varlistentry>

2
include/jemalloc/internal/arena_inlines_a.h
View File

@ -58,7 +58,7 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
/* Set new arena/tcache associations. */
arena_migrate(tsd, oldind, newind);
tcache_t *tcache = tcache_get(tsd);
if (config_tcache && tcache) {
if (tcache != NULL) {
tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
newarena);
}

7
include/jemalloc/internal/jemalloc_internal_defs.h.in
View File

@ -154,13 +154,6 @@
/* Use gcc intrinsics for profile backtracing if defined. */
#undef JEMALLOC_PROF_GCC
/*
* JEMALLOC_TCACHE enables a thread-specific caching layer for small objects.
* This makes it possible to allocate/deallocate objects without any locking
* when the cache is in the steady state.
*/
#undef JEMALLOC_TCACHE
/*
* JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage
* segment (DSS).

19
include/jemalloc/internal/jemalloc_internal_inlines_a.h
View File

@ -323,7 +323,8 @@ malloc_getcpu(void) {
JEMALLOC_ALWAYS_INLINE unsigned
percpu_arena_choose(void) {
unsigned arena_ind;
assert(have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled));
assert(have_percpu_arena && (percpu_arena_mode !=
percpu_arena_disabled));
malloc_cpuid_t cpuid = malloc_getcpu();
assert(cpuid >= 0);
@ -420,19 +421,16 @@ tcache_large_bin_get(tcache_t *tcache, szind_t binind) {
JEMALLOC_ALWAYS_INLINE bool
tcache_available(tsd_t *tsd) {
cassert(config_tcache);
/*
* Thread specific auto tcache might be unavailable if: 1) during tcache
* initialization, or 2) disabled through thread.tcache.enabled mallctl
* or config options. This check covers all cases.
*/
if (likely(tsd_tcache_enabled_get(tsd) == true)) {
/* Associated arena == null implies tcache init in progress. */
if (tsd_tcachep_get(tsd)->arena != NULL) {
assert(tcache_small_bin_get(tsd_tcachep_get(tsd),
0)->avail != NULL);
}
if (likely(tsd_tcache_enabled_get(tsd))) {
/* Associated arena == NULL implies tcache init in progress. */
assert(tsd_tcachep_get(tsd)->arena == NULL ||
tcache_small_bin_get(tsd_tcachep_get(tsd), 0)->avail !=
NULL);
return true;
}
@ -441,9 +439,6 @@ tcache_available(tsd_t *tsd) {
JEMALLOC_ALWAYS_INLINE tcache_t *
tcache_get(tsd_t *tsd) {
if (!config_tcache) {
return NULL;
}
if (!tcache_available(tsd)) {
return NULL;
}

2
include/jemalloc/internal/jemalloc_internal_inlines_b.h
View File

@ -24,7 +24,7 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
if (unlikely(ret == NULL)) {
ret = arena_choose_hard(tsd, internal);
assert(ret);
if (config_tcache && tcache_available(tsd)) {
if (tcache_available(tsd)) {
tcache_t *tcache = tcache_get(tsd);
if (tcache->arena != NULL) {
/* See comments in tcache_data_init().*/

7
include/jemalloc/internal/jemalloc_preamble.h.in
View File

@ -111,13 +111,6 @@ static const bool config_stats =
false
#endif
;
static const bool config_tcache =
#ifdef JEMALLOC_TCACHE
true
#else
false
#endif
;
static const bool config_tls =
#ifdef JEMALLOC_TLS
true

5
include/jemalloc/internal/tcache_inlines.h
View File

@ -6,7 +6,6 @@
#ifndef JEMALLOC_ENABLE_INLINE
void tcache_event(tsd_t *tsd, tcache_t *tcache);
void tcache_flush(void);
bool tcache_enabled_get(tsd_t *tsd);
tcache_t *tcache_get(tsd_t *tsd);
void tcache_enabled_set(tsd_t *tsd, bool enabled);
@ -25,15 +24,11 @@ tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
JEMALLOC_INLINE bool
tcache_enabled_get(tsd_t *tsd) {
cassert(config_tcache);
return tsd_tcache_enabled_get(tsd);
}
JEMALLOC_INLINE void
tcache_enabled_set(tsd_t *tsd, bool enabled) {
cassert(config_tcache);
bool was_enabled = tsd_tcache_enabled_get(tsd);
if (!was_enabled && enabled) {

9
include/jemalloc/internal/tcache_structs.h
View File

@ -40,23 +40,14 @@ struct tcache_s {
* element of tbins is initialized to point to the proper offset within
* this array.
*/
#ifdef JEMALLOC_TCACHE
tcache_bin_t tbins_small[NBINS];
#else
tcache_bin_t tbins_small[0];
#endif
/* Data accessed less often below. */
ql_elm(tcache_t) link; /* Used for aggregating stats. */
arena_t *arena; /* Associated arena. */
szind_t next_gc_bin; /* Next bin to GC. */
#ifdef JEMALLOC_TCACHE
/* For small bins, fill (ncached_max >> lg_fill_div). */
uint8_t lg_fill_div[NBINS];
tcache_bin_t tbins_large[NSIZES-NBINS];
#else
uint8_t lg_fill_div[0];
tcache_bin_t tbins_large[0];
#endif
};
/* Linkage for list of available (previously used) explicit tcache IDs. */

2
scripts/gen_run_tests.py
View File

@ -18,7 +18,7 @@ possible_config_opts = [
'--enable-debug',
'--enable-prof',
'--disable-stats',
'--disable-tcache',
'--with-malloc-conf=tcache:false',
]
print 'set -e'

15
scripts/gen_travis.py
View File

@ -24,11 +24,11 @@ script:
# The 'default' configuration is gcc, on linux, with no compiler or configure
# flags. We also test with clang, -m32, --enable-debug, --enable-prof,
# --disable-stats, and --disable-tcache. To avoid abusing travis though, we
# don't test all 2**7 = 128 possible combinations of these; instead, we only
# test combinations of up to 2 'unusual' settings, under the hope that bugs
# involving interactions of such settings are rare.
# things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29
# --disable-stats, and --with-malloc-conf=tcache:false. To avoid abusing
# travis though, we don't test all 2**7 = 128 possible combinations of these;
# instead, we only test combinations of up to 2 'unusual' settings, under the
# hope that bugs involving interactions of such settings are rare.
# Things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29
MAX_UNUSUAL_OPTIONS = 2
os_default = 'linux'
@ -40,7 +40,10 @@ compilers_unusual = 'CC=clang CXX=clang++'
compiler_flag_unusuals = ['-m32']
configure_flag_unusuals = [
'--enable-debug', '--enable-prof', '--disable-stats', '--disable-tcache',
'--enable-debug',
'--enable-prof',
'--disable-stats',
'--with-malloc-conf=tcache:false',
]
all_unusuals = (

56
src/arena.c
View File

@ -283,31 +283,27 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
arena_stats_unlock(tsdn, &arena->stats);
if (config_tcache) {
tcache_bin_t *tbin;
tcache_t *tcache;
/* tcache_bytes counts currently cached bytes. */
atomic_store_zu(&astats->tcache_bytes, 0, ATOMIC_RELAXED);
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
ql_foreach(tcache, &arena->tcache_ql, link) {
szind_t i = 0;
for (; i < NBINS; i++) {
tbin = tcache_small_bin_get(tcache, i);
arena_stats_accum_zu(&astats->tcache_bytes,
tbin->ncached * index2size(i));
}
for (; i < nhbins; i++) {
tbin = tcache_large_bin_get(tcache, i);
arena_stats_accum_zu(&astats->tcache_bytes,
tbin->ncached * index2size(i));
}
/* tcache_bytes counts currently cached bytes. */
atomic_store_zu(&astats->tcache_bytes, 0, ATOMIC_RELAXED);
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
tcache_t *tcache;
ql_foreach(tcache, &arena->tcache_ql, link) {
szind_t i = 0;
for (; i < NBINS; i++) {
tcache_bin_t *tbin = tcache_small_bin_get(tcache, i);
arena_stats_accum_zu(&astats->tcache_bytes,
tbin->ncached * index2size(i));
}
for (; i < nhbins; i++) {
tcache_bin_t *tbin = tcache_large_bin_get(tcache, i);
arena_stats_accum_zu(&astats->tcache_bytes,
tbin->ncached * index2size(i));
}
malloc_mutex_prof_read(tsdn,
&astats->mutex_prof_data[arena_prof_mutex_tcache_list],
&arena->tcache_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
}
malloc_mutex_prof_read(tsdn,
&astats->mutex_prof_data[arena_prof_mutex_tcache_list],
&arena->tcache_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \
malloc_mutex_lock(tsdn, &arena->mtx); \
@ -342,10 +338,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
bstats[i].ndalloc += bin->stats.ndalloc;
bstats[i].nrequests += bin->stats.nrequests;
bstats[i].curregs += bin->stats.curregs;
if (config_tcache) {
bstats[i].nfills += bin->stats.nfills;
bstats[i].nflushes += bin->stats.nflushes;
}
bstats[i].nfills += bin->stats.nfills;
bstats[i].nflushes += bin->stats.nflushes;
bstats[i].nslabs += bin->stats.nslabs;
bstats[i].reslabs += bin->stats.reslabs;
bstats[i].curslabs += bin->stats.curslabs;
@ -1867,9 +1861,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
if (arena_stats_init(tsdn, &arena->stats)) {
goto label_error;
}
}
if (config_stats && config_tcache) {
ql_new(&arena->tcache_ql);
if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql",
WITNESS_RANK_TCACHE_QL)) {
@ -2007,7 +1999,7 @@ arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
void
arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
if (config_stats && config_tcache) {
if (config_stats) {
malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx);
}
}
@ -2056,7 +2048,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
extents_postfork_parent(tsdn, &arena->extents_retained);
malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx);
if (config_stats && config_tcache) {
if (config_stats) {
malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
}
}
@ -2076,7 +2068,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
extents_postfork_child(tsdn, &arena->extents_retained);
malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx);
if (config_stats && config_tcache) {
if (config_stats) {
malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
}
}

66
src/ctl.c
View File

@ -70,7 +70,6 @@ CTL_PROTO(config_prof)
CTL_PROTO(config_prof_libgcc)
CTL_PROTO(config_prof_libunwind)
CTL_PROTO(config_stats)
CTL_PROTO(config_tcache)
CTL_PROTO(config_tls)
CTL_PROTO(config_utrace)
CTL_PROTO(config_xmalloc)
@ -255,7 +254,6 @@ static const ctl_named_node_t config_node[] = {
{NAME("prof_libgcc"), CTL(config_prof_libgcc)},
{NAME("prof_libunwind"), CTL(config_prof_libunwind)},
{NAME("stats"), CTL(config_stats)},
{NAME("tcache"), CTL(config_tcache)},
{NAME("tls"), CTL(config_tls)},
{NAME("utrace"), CTL(config_utrace)},
{NAME("xmalloc"), CTL(config_xmalloc)}
@ -777,10 +775,8 @@ ARENA_PROF_MUTEXES
accum_arena_stats_u64(&sdstats->astats.nrequests_large,
&astats->astats.nrequests_large);
if (config_tcache) {
accum_atomic_zu(&sdstats->astats.tcache_bytes,
&astats->astats.tcache_bytes);
}
accum_atomic_zu(&sdstats->astats.tcache_bytes,
&astats->astats.tcache_bytes);
for (i = 0; i < NBINS; i++) {
sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
@ -793,12 +789,9 @@ ARENA_PROF_MUTEXES
} else {
assert(astats->bstats[i].curregs == 0);
}
if (config_tcache) {
sdstats->bstats[i].nfills +=
astats->bstats[i].nfills;
sdstats->bstats[i].nflushes +=
astats->bstats[i].nflushes;
}
sdstats->bstats[i].nfills += astats->bstats[i].nfills;
sdstats->bstats[i].nflushes +=
astats->bstats[i].nflushes;
sdstats->bstats[i].nslabs += astats->bstats[i].nslabs;
sdstats->bstats[i].reslabs += astats->bstats[i].reslabs;
if (!destroyed) {
@ -1457,7 +1450,6 @@ CTL_RO_CONFIG_GEN(config_prof, bool)
CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
CTL_RO_CONFIG_GEN(config_stats, bool)
CTL_RO_CONFIG_GEN(config_tcache, bool)
CTL_RO_CONFIG_GEN(config_tls, bool)
CTL_RO_CONFIG_GEN(config_utrace, bool)
CTL_RO_CONFIG_GEN(config_xmalloc, bool)
@ -1475,8 +1467,8 @@ CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *)
CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool)
CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool)
CTL_RO_NL_GEN(opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool)
@ -1536,12 +1528,9 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
}
/* Set new arena/tcache associations. */
arena_migrate(tsd, oldind, newind);
if (config_tcache) {
tcache_t *tcache = tsd_tcachep_get(tsd);
if (tcache != NULL) {
tcache_arena_reassociate(tsd_tsdn(tsd), tcache,
newarena);
}
if (tcache_available(tsd)) {
tcache_arena_reassociate(tsd_tsdn(tsd),
tsd_tcachep_get(tsd), newarena);
}
}
@ -1565,10 +1554,6 @@ thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
int ret;
bool oldval;
if (!config_tcache) {
return ENOENT;
}
oldval = tcache_enabled_get(tsd);
if (newp != NULL) {
if (newlen != sizeof(bool)) {
@ -1589,8 +1574,9 @@ thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
int ret;
if (!config_tcache) {
return ENOENT;
if (!tcache_available(tsd)) {
ret = EFAULT;
goto label_return;
}
READONLY();
@ -1670,10 +1656,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
int ret;
unsigned tcache_ind;
if (!config_tcache) {
return ENOENT;
}
READONLY();
if (tcaches_create(tsd, &tcache_ind)) {
ret = EFAULT;
@ -1692,10 +1674,6 @@ tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
int ret;
unsigned tcache_ind;
if (!config_tcache) {
return ENOENT;
}
WRITEONLY();
tcache_ind = UINT_MAX;
WRITE(tcache_ind, unsigned);
@ -1716,10 +1694,6 @@ tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
int ret;
unsigned tcache_ind;
if (!config_tcache) {
return ENOENT;
}
WRITEONLY();
tcache_ind = UINT_MAX;
WRITE(tcache_ind, unsigned);
@ -2150,9 +2124,9 @@ arenas_muzzy_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t)
CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t)
CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned)
CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned)
CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned)
CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_slab_size, arena_bin_info[mib[2]].slab_size, size_t)
@ -2380,7 +2354,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_base,
CTL_RO_CGEN(config_stats, stats_arenas_i_internal,
atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED),
size_t)
CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_tcache_bytes,
CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes,
atomic_load_zu(&arenas_i(mib[2])->astats->astats.tcache_bytes,
ATOMIC_RELAXED), size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_resident,
@ -2480,9 +2454,7 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
MUTEX_PROF_RESET(arena->extents_retained.mtx);
MUTEX_PROF_RESET(arena->decay_dirty.mtx);
MUTEX_PROF_RESET(arena->decay_muzzy.mtx);
if (config_tcache) {
MUTEX_PROF_RESET(arena->tcache_ql_mtx);
}
MUTEX_PROF_RESET(arena->tcache_ql_mtx);
MUTEX_PROF_RESET(arena->base->mtx);
for (szind_t i = 0; i < NBINS; i++) {
@ -2502,9 +2474,9 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests,
arenas_i(mib[2])->astats->bstats[mib[4]].nrequests, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs,
arenas_i(mib[2])->astats->bstats[mib[4]].curregs, size_t)
CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nfills,
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills,
arenas_i(mib[2])->astats->bstats[mib[4]].nfills, uint64_t)
CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nflushes,
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes,
arenas_i(mib[2])->astats->bstats[mib[4]].nflushes, uint64_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs,
arenas_i(mib[2])->astats->bstats[mib[4]].nslabs, uint64_t)

13
src/jemalloc.c
View File

@ -682,7 +682,7 @@ arenas_tdata_cleanup(tsd_t *tsd) {
static void
stats_print_atexit(void) {
if (config_tcache && config_stats) {
if (config_stats) {
tsdn_t *tsdn;
unsigned narenas, i;
@ -1106,12 +1106,9 @@ malloc_conf_init(void) {
if (config_xmalloc) {
CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true)
}
if (config_tcache) {
CONF_HANDLE_BOOL(opt_tcache, "tcache", true)
CONF_HANDLE_SSIZE_T(opt_lg_tcache_max,
"lg_tcache_max", -1,
(sizeof(size_t) << 3) - 1)
}
CONF_HANDLE_BOOL(opt_tcache, "tcache", true)
CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max",
-1, (sizeof(size_t) << 3) - 1)
if (strncmp("percpu_arena", k, klen) == 0) {
int i;
bool match = false;
@ -1236,7 +1233,7 @@ malloc_init_hard_a0_locked() {
prof_boot1();
}
arena_boot();
if (config_tcache && tcache_boot(TSDN_NULL)) {
if (tcache_boot(TSDN_NULL)) {
return true;
}
if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS)) {

114
src/stats.c
View File

@ -128,20 +128,11 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
} else {
char *mutex_counters = " n_lock_ops n_waiting"
" n_spin_acq max_wait_ns\n";
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
"bins: size ind allocated nmalloc"
" ndalloc nrequests curregs"
" curslabs regs pgs util nfills"
" nflushes newslabs reslabs%s",
mutex ? mutex_counters : "\n");
} else {
malloc_cprintf(write_cb, cbopaque,
"bins: size ind allocated nmalloc"
" ndalloc nrequests curregs"
" curslabs regs pgs util newslabs"
" reslabs%s", mutex ? mutex_counters : "\n");
}
malloc_cprintf(write_cb, cbopaque,
"bins: size ind allocated nmalloc"
" ndalloc nrequests curregs curslabs regs"
" pgs util nfills nflushes newslabs"
" reslabs%s", mutex ? mutex_counters : "\n");
}
for (j = 0, in_gap = false; j < nbins; j++) {
uint64_t nslabs;
@ -173,12 +164,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
size_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.nrequests", i, j,
&nrequests, uint64_t);
if (config_tcache) {
CTL_M2_M4_GET("stats.arenas.0.bins.0.nfills", i, j,
&nfills, uint64_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.nflushes", i, j,
&nflushes, uint64_t);
}
CTL_M2_M4_GET("stats.arenas.0.bins.0.nfills", i, j, &nfills,
uint64_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.nflushes", i, j, &nflushes,
uint64_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.nreslabs", i, j, &nreslabs,
uint64_t);
CTL_M2_M4_GET("stats.arenas.0.bins.0.curslabs", i, j, &curslabs,
@ -190,23 +179,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
"\t\t\t\t\t\t\"nmalloc\": %"FMTu64",\n"
"\t\t\t\t\t\t\"ndalloc\": %"FMTu64",\n"
"\t\t\t\t\t\t\"curregs\": %zu,\n"
"\t\t\t\t\t\t\"nrequests\": %"FMTu64",\n",
nmalloc,
ndalloc,
curregs,
nrequests);
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t\t\"nfills\": %"FMTu64",\n"
"\t\t\t\t\t\t\"nflushes\": %"FMTu64",\n",
nfills,
nflushes);
}
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\t\t\"nrequests\": %"FMTu64",\n"
"\t\t\t\t\t\t\"nfills\": %"FMTu64",\n"
"\t\t\t\t\t\t\"nflushes\": %"FMTu64",\n"
"\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n"
"\t\t\t\t\t\t\"curslabs\": %zu%s\n",
nreslabs, curslabs, mutex ? "," : "");
nmalloc, ndalloc, curregs, nrequests, nfills,
nflushes, nreslabs, curslabs, mutex ? "," : "");
if (mutex) {
uint64_t mutex_stats[num_mutex_prof_counters];
read_arena_bin_mutex_stats(i, j, mutex_stats);
@ -260,27 +239,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
}
}
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
"%20zu %3u %12zu %12"FMTu64
" %12"FMTu64" %12"FMTu64" %12zu"
" %12zu %4u %3zu %-5s %12"FMTu64
" %12"FMTu64" %12"FMTu64" %12"FMTu64,
reg_size, j, curregs * reg_size, nmalloc,
ndalloc, nrequests, curregs, curslabs,
nregs, slab_size / page, util, nfills,
nflushes, nslabs, nreslabs);
} else {
malloc_cprintf(write_cb, cbopaque,
"%20zu %3u %12zu %12"FMTu64
" %12"FMTu64" %12"FMTu64" %12zu"
" %12zu %4u %3zu %-5s %12"FMTu64
" %12"FMTu64,
reg_size, j, curregs * reg_size, nmalloc,
ndalloc, nrequests, curregs, curslabs,
nregs, slab_size / page, util, nslabs,
nreslabs);
}
malloc_cprintf(write_cb, cbopaque, "%20zu %3u %12zu %12"
FMTu64" %12"FMTu64" %12"FMTu64" %12zu %12zu %4u"
" %3zu %-5s %12"FMTu64" %12"FMTu64" %12"FMTu64
" %12"FMTu64, reg_size, j, curregs * reg_size,
nmalloc, ndalloc, nrequests, curregs, curslabs,
nregs, slab_size / page, util, nfills, nflushes,
nslabs, nreslabs);
if (mutex) {
malloc_cprintf(write_cb, cbopaque,
" %12"FMTu64" %12"FMTu64" %12"FMTu64
@ -423,14 +388,7 @@ stats_arena_mutexes_print(void (*write_cb)(void *, const char *),
malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n");
arena_prof_mutex_ind_t i, last_mutex;
last_mutex = num_arena_prof_mutexes - 1;
if (!config_tcache) {
last_mutex--;
}
for (i = 0; i < num_arena_prof_mutexes; i++) {
if (!config_tcache &&
i == arena_prof_mutex_tcache_list) {
continue;
}
mutex_stats_output_json(write_cb, cbopaque,
arena_mutex_names[i], mutex_stats[i],
"\t\t\t\t\t", (i == last_mutex));
@ -440,10 +398,6 @@ stats_arena_mutexes_print(void (*write_cb)(void *, const char *),
} else {
arena_prof_mutex_ind_t i;
for (i = 0; i < num_arena_prof_mutexes; i++) {
if (!config_tcache &&
i == arena_prof_mutex_tcache_list) {
continue;
}
mutex_stats_output(write_cb, cbopaque,
arena_mutex_names[i], mutex_stats[i], i == 0);
}
@ -659,16 +613,13 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
"internal: %12zu\n", internal);
}
if (config_tcache) {
CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &tcache_bytes,
size_t);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"tcache\": %zu,\n", tcache_bytes);
} else {
malloc_cprintf(write_cb, cbopaque,
"tcache: %12zu\n", tcache_bytes);
}
CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &tcache_bytes, size_t);
if (json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"tcache\": %zu,\n", tcache_bytes);
} else {
malloc_cprintf(write_cb, cbopaque,
"tcache: %12zu\n", tcache_bytes);
}
CTL_M2_GET("stats.arenas.0.resident", i, &resident, size_t);
@ -761,7 +712,6 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
CONFIG_WRITE_BOOL_JSON(prof_libgcc, ",")
CONFIG_WRITE_BOOL_JSON(prof_libunwind, ",")
CONFIG_WRITE_BOOL_JSON(stats, ",")
CONFIG_WRITE_BOOL_JSON(tcache, ",")
CONFIG_WRITE_BOOL_JSON(tls, ",")
CONFIG_WRITE_BOOL_JSON(utrace, ",")
CONFIG_WRITE_BOOL_JSON(xmalloc, "")
@ -959,11 +909,9 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"nbins\": %u,\n", nbins);
if (config_tcache) {
CTL_GET("arenas.nhbins", &uv, unsigned);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"nhbins\": %u,\n", uv);
}
CTL_GET("arenas.nhbins", &uv, unsigned);
malloc_cprintf(write_cb, cbopaque, "\t\t\t\"nhbins\": %u,\n",
uv);
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"bin\": [\n");

44
src/tcache.c
View File

@ -7,13 +7,7 @@
/******************************************************************************/
/* Data. */
bool opt_tcache =
#ifdef JEMALLOC_TCACHE
true
#else
false
#endif
;
bool opt_tcache = true;
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
tcache_bin_info_t *tcache_bin_info;
@ -93,7 +87,7 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
tcache_bin_t *tbin, szind_t binind, bool *tcache_success) {
void *ret;
assert(tcache->arena);
assert(tcache->arena != NULL);
arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind,
config_prof ? tcache->prof_accumbytes : 0);
if (config_prof) {
@ -304,7 +298,7 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
static void
tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
arena_t *arena = tcache->arena;
assert(arena);
assert(arena != NULL);
if (config_stats) {
/* Unlink from list of extant tcaches. */
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
@ -383,10 +377,6 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
/* Initialize auto tcache (embedded in TSD). */
bool
tsd_tcache_data_init(tsd_t *tsd) {
if (!config_tcache) {
return false;
}
tcache_t *tcache = &tsd->tcache;
assert(tcache_small_bin_get(tcache, 0)->avail == NULL);
size_t size = stack_nelms * sizeof(void *);
@ -458,9 +448,9 @@ tcache_create_explicit(tsd_t *tsd) {
static void
tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
unsigned i;
assert(tcache->arena != NULL);
for (i = 0; i < NBINS; i++) {
for (unsigned i = 0; i < NBINS; i++) {
tcache_bin_t *tbin = tcache_small_bin_get(tcache, i);
tcache_bin_flush_small(tsd, tcache, tbin, i, 0);
@ -468,7 +458,7 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
assert(tbin->tstats.nrequests == 0);
}
}
for (; i < nhbins; i++) {
for (unsigned i = NBINS; i < nhbins; i++) {
tcache_bin_t *tbin = tcache_large_bin_get(tcache, i);
tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
@ -477,20 +467,17 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
}
}
arena_t *arena = tcache->arena;
if (config_prof && arena && tcache->prof_accumbytes > 0 &&
arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes)) {
if (config_prof && tcache->prof_accumbytes > 0 &&
arena_prof_accum(tsd_tsdn(tsd), tcache->arena,
tcache->prof_accumbytes)) {
prof_idump(tsd_tsdn(tsd));
}
}
void
tcache_flush(void) {
tsd_t *tsd;
cassert(config_tcache);
tsd = tsd_fetch();
tsd_t *tsd = tsd_fetch();
assert(tcache_available(tsd));
tcache_flush_cache(tsd, tsd_tcachep_get(tsd));
}
@ -514,10 +501,6 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
/* For auto tcache (embedded in TSD) only. */
void
tcache_cleanup(tsd_t *tsd) {
if (!config_tcache) {
return;
}
tcache_t *tcache = tsd_tcachep_get(tsd);
if (!tcache_available(tsd)) {
assert(tsd_tcache_enabled_get(tsd) == false);
@ -660,10 +643,6 @@ tcaches_destroy(tsd_t *tsd, unsigned ind) {
bool
tcache_boot(tsdn_t *tsdn) {
cassert(config_tcache);
unsigned i;
/* If necessary, clamp opt_lg_tcache_max. */
if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) <
SMALL_MAXCLASS) {
@ -685,6 +664,7 @@ tcache_boot(tsdn_t *tsdn) {
return true;
}
stack_nelms = 0;
unsigned i;
for (i = 0; i < NBINS; i++) {
if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
tcache_bin_info[i].ncached_max =

25
test/integration/thread_tcache_enabled.c
View File

@ -1,29 +1,11 @@
#include "test/jemalloc_test.h"
static const bool config_tcache =
#ifdef JEMALLOC_TCACHE
true
#else
false
#endif
;
void *
thd_start(void *arg) {
int err;
size_t sz;
bool e0, e1;
sz = sizeof(bool);
if ((err = mallctl("thread.tcache.enabled", (void *)&e0, &sz, NULL,
0))) {
if (err == ENOENT) {
assert_false(config_tcache,
"ENOENT should only be returned if tcache is "
"disabled");
}
goto label_ENOENT;
}
size_t sz = sizeof(bool);
assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz, NULL,
0), 0, "Unexpected mallctl failure");
if (e0) {
e1 = false;
@ -78,7 +60,6 @@ thd_start(void *arg) {
free(malloc(1));
return NULL;
label_ENOENT:
test_skip("\"thread.tcache.enabled\" mallctl not available");
return NULL;
}

1
test/test.sh.in
View File

@ -43,7 +43,6 @@ for t in $@; do
# per test shell script to ignore the @JEMALLOC_CPREFIX@ detail).
$(enable_fill=@enable_fill@ \
enable_prof=@enable_prof@ \
enable_tcache=@enable_tcache@ \
. @srcroot@${t}.sh && \
export_malloc_conf && \
${t}@exe@ @abs_srcroot@ @abs_objroot@)

101
test/unit/decay.c
View File

@ -174,16 +174,15 @@ TEST_BEGIN(test_decay_ticks) {
assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large0, &sz, NULL,
0), 0, "Unexpected mallctl failure");
int err;
/* Set up a manually managed arena for test. */
arena_ind = do_arena_create(0, 0);
/* Migrate to the new arena, and get the ticker. */
unsigned old_arena_ind;
size_t sz_arena_ind = sizeof(old_arena_ind);
err = mallctl("thread.arena", (void *)&old_arena_ind, &sz_arena_ind,
(void *)&arena_ind, sizeof(arena_ind));
assert_d_eq(err, 0, "Unexpected mallctl() failure");
assert_d_eq(mallctl("thread.arena", (void *)&old_arena_ind,
&sz_arena_ind, (void *)&arena_ind, sizeof(arena_ind)), 0,
"Unexpected mallctl() failure");
decay_ticker = decay_ticker_get(tsd_fetch(), arena_ind);
assert_ptr_not_null(decay_ticker,
"Unexpected failure getting decay ticker");
@ -310,51 +309,48 @@ TEST_BEGIN(test_decay_ticks) {
* Test tcache fill/flush interactions for large and small size classes,
* using an explicit tcache.
*/
if (config_tcache) {
unsigned tcache_ind, i;
size_t tcache_sizes[2];
tcache_sizes[0] = large0;
tcache_sizes[1] = 1;
unsigned tcache_ind, i;
size_t tcache_sizes[2];
tcache_sizes[0] = large0;
tcache_sizes[1] = 1;
size_t tcache_max, sz_tcache_max;
sz_tcache_max = sizeof(tcache_max);
err = mallctl("arenas.tcache_max", (void *)&tcache_max,
&sz_tcache_max, NULL, 0);
assert_d_eq(err, 0, "Unexpected mallctl() failure");
size_t tcache_max, sz_tcache_max;
sz_tcache_max = sizeof(tcache_max);
assert_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max,
&sz_tcache_max, NULL, 0), 0, "Unexpected mallctl() failure");
sz = sizeof(unsigned);
assert_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz,
NULL, 0), 0, "Unexpected mallctl failure");
sz = sizeof(unsigned);
assert_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz,
NULL, 0), 0, "Unexpected mallctl failure");
for (i = 0; i < sizeof(tcache_sizes) / sizeof(size_t); i++) {
sz = tcache_sizes[i];
for (i = 0; i < sizeof(tcache_sizes) / sizeof(size_t); i++) {
sz = tcache_sizes[i];
/* tcache fill. */
tick0 = ticker_read(decay_ticker);
p = mallocx(sz, MALLOCX_TCACHE(tcache_ind));
assert_ptr_not_null(p, "Unexpected mallocx() failure");
tick1 = ticker_read(decay_ticker);
/* tcache fill. */
tick0 = ticker_read(decay_ticker);
p = mallocx(sz, MALLOCX_TCACHE(tcache_ind));
assert_ptr_not_null(p, "Unexpected mallocx() failure");
tick1 = ticker_read(decay_ticker);
assert_u32_ne(tick1, tick0,
"Expected ticker to tick during tcache fill "
"(sz=%zu)", sz);
/* tcache flush. */
dallocx(p, MALLOCX_TCACHE(tcache_ind));
tick0 = ticker_read(decay_ticker);
assert_d_eq(mallctl("tcache.flush", NULL, NULL,
(void *)&tcache_ind, sizeof(unsigned)), 0,
"Unexpected mallctl failure");
tick1 = ticker_read(decay_ticker);
/* Will only tick if it's in tcache. */
if (sz <= tcache_max) {
assert_u32_ne(tick1, tick0,
"Expected ticker to tick during tcache fill "
"(sz=%zu)", sz);
/* tcache flush. */
dallocx(p, MALLOCX_TCACHE(tcache_ind));
tick0 = ticker_read(decay_ticker);
assert_d_eq(mallctl("tcache.flush", NULL, NULL,
(void *)&tcache_ind, sizeof(unsigned)), 0,
"Unexpected mallctl failure");
tick1 = ticker_read(decay_ticker);
/* Will only tick if it's in tcache. */
if (sz <= tcache_max) {
assert_u32_ne(tick1, tick0,
"Expected ticker to tick during tcache "
"flush (sz=%zu)", sz);
} else {
assert_u32_eq(tick1, tick0,
"Unexpected ticker tick during tcache "
"flush (sz=%zu)", sz);
}
"Expected ticker to tick during tcache "
"flush (sz=%zu)", sz);
} else {
assert_u32_eq(tick1, tick0,
"Unexpected ticker tick during tcache "
"flush (sz=%zu)", sz);
}
}
}
@ -422,18 +418,11 @@ TEST_BEGIN(test_decay_ticker) {
* the ticker triggers purging.
*/
if (config_tcache) {
size_t tcache_max;
size_t sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max,
&sz, NULL, 0), 0, "Unexpected mallctl failure");
large = nallocx(tcache_max + 1, flags);
} else {
size_t sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.lextent.0.size", &large, &sz, NULL,
0), 0, "Unexpected mallctl failure");
}
size_t tcache_max;
size_t sz = sizeof(size_t);
assert_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max, &sz, NULL,
0), 0, "Unexpected mallctl failure");
large = nallocx(tcache_max + 1, flags);
do_purge(arena_ind);
uint64_t dirty_npurge0 = get_arena_dirty_npurge(arena_ind);

5
test/unit/decay.sh
View File

@ -1,6 +1,3 @@
#!/bin/sh
export MALLOC_CONF="dirty_decay_time:1,muzzy_decay_time:1"
if [ "x${enable_tcache}" = "x1" ] ; then
export MALLOC_CONF="${MALLOC_CONF},lg_tcache_max:0"
fi
export MALLOC_CONF="dirty_decay_time:1,muzzy_decay_time:1,lg_tcache_max:0"

9
test/unit/mallctl.c
View File

@ -136,7 +136,6 @@ TEST_BEGIN(test_mallctl_config) {
TEST_MALLCTL_CONFIG(prof_libgcc, bool);
TEST_MALLCTL_CONFIG(prof_libunwind, bool);
TEST_MALLCTL_CONFIG(stats, bool);
TEST_MALLCTL_CONFIG(tcache, bool);
TEST_MALLCTL_CONFIG(tls, bool);
TEST_MALLCTL_CONFIG(utrace, bool);
TEST_MALLCTL_CONFIG(xmalloc, bool);
@ -170,8 +169,8 @@ TEST_BEGIN(test_mallctl_opt) {
TEST_MALLCTL_OPT(bool, zero, fill);
TEST_MALLCTL_OPT(bool, utrace, utrace);
TEST_MALLCTL_OPT(bool, xmalloc, xmalloc);
TEST_MALLCTL_OPT(bool, tcache, tcache);
TEST_MALLCTL_OPT(size_t, lg_tcache_max, tcache);
TEST_MALLCTL_OPT(bool, tcache, always);
TEST_MALLCTL_OPT(size_t, lg_tcache_max, always);
TEST_MALLCTL_OPT(bool, prof, prof);
TEST_MALLCTL_OPT(const char *, prof_prefix, prof);
TEST_MALLCTL_OPT(bool, prof_active, prof);
@ -213,8 +212,6 @@ TEST_END
TEST_BEGIN(test_tcache_none) {
void *p0, *q, *p1;
test_skip_if(!config_tcache);
/* Allocate p and q. */
p0 = mallocx(42, 0);
assert_ptr_not_null(p0, "Unexpected mallocx() failure");
@ -243,8 +240,6 @@ TEST_BEGIN(test_tcache) {
unsigned i;
size_t sz, psz, qsz;
test_skip_if(!config_tcache);
psz = 42;
qsz = nallocx(psz, 0) + 1;

8
test/unit/prof_idump.sh
View File

@ -1,12 +1,8 @@
#!/bin/sh
export MALLOC_CONF="tcache:false"
if [ "x${enable_prof}" = "x1" ] ; then
export MALLOC_CONF="prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0,lg_prof_interval:0"
if [ "x${enable_tcache}" = "x1" ] ; then
export MALLOC_CONF="${MALLOC_CONF},tcache:false"
fi
elif [ "x${enable_tcache}" = "x1" ] ; then
export MALLOC_CONF="tcache:false"
export MALLOC_CONF="${MALLOC_CONF},prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0,lg_prof_interval:0"
fi

20
test/unit/stats.c
View File

@ -83,7 +83,7 @@ TEST_BEGIN(test_stats_arenas_summary) {
dallocx(large, 0);
assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
config_tcache ? 0 : ENOENT, "Unexpected mallctl() result");
opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
"Unexpected mallctl() failure");
@ -150,7 +150,7 @@ TEST_BEGIN(test_stats_arenas_small) {
assert_ptr_not_null(p, "Unexpected mallocx() failure");
assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
config_tcache ? 0 : ENOENT, "Unexpected mallctl() result");
opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
0, "Unexpected mallctl() failure");
@ -230,6 +230,10 @@ TEST_BEGIN(test_stats_arenas_bins) {
uint64_t nslabs, nreslabs;
int expected = config_stats ? 0 : ENOENT;
/* Make sure allocation below isn't satisfied by tcache. */
assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
unsigned arena_ind, old_arena_ind;
sz = sizeof(unsigned);
assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
@ -243,7 +247,7 @@ TEST_BEGIN(test_stats_arenas_bins) {
assert_ptr_not_null(p, "Unexpected malloc() failure");
assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
config_tcache ? 0 : ENOENT, "Unexpected mallctl() result");
opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result");
assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
0, "Unexpected mallctl() failure");
@ -266,11 +270,11 @@ TEST_BEGIN(test_stats_arenas_bins) {
sz = sizeof(uint64_t);
gen_mallctl_str(cmd, "nfills", arena_ind);
assert_d_eq(mallctl(cmd, (void *)&nfills, &sz, NULL, 0),
config_tcache ? expected : ENOENT, "Unexpected mallctl() result");
assert_d_eq(mallctl(cmd, (void *)&nfills, &sz, NULL, 0), expected,
"Unexpected mallctl() result");
gen_mallctl_str(cmd, "nflushes", arena_ind);
assert_d_eq(mallctl(cmd, (void *)&nflushes, &sz, NULL, 0),
config_tcache ? expected : ENOENT, "Unexpected mallctl() result");
assert_d_eq(mallctl(cmd, (void *)&nflushes, &sz, NULL, 0), expected,
"Unexpected mallctl() result");
gen_mallctl_str(cmd, "nslabs", arena_ind);
assert_d_eq(mallctl(cmd, (void *)&nslabs, &sz, NULL, 0), expected,
@ -292,7 +296,7 @@ TEST_BEGIN(test_stats_arenas_bins) {
"nrequests should be greater than zero");
assert_zu_gt(curregs, 0,
"allocated should be greater than zero");
if (config_tcache) {
if (opt_tcache) {
assert_u64_gt(nfills, 0,
"At least one fill should have occurred");
assert_u64_gt(nflushes, 0,