Add the "thread.tcache.enabled" mallctl.

This commit is contained in:
Jason Evans 2012-03-26 18:54:44 -07:00
parent fd4fcefa00
commit d4be8b7b6e
7 changed files with 271 additions and 36 deletions

View File

@ -66,7 +66,8 @@ DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3)
DOCS := $(DOCS_HTML) $(DOCS_MAN3)
CTESTS := @srcroot@test/aligned_alloc.c @srcroot@test/allocated.c \
@srcroot@test/bitmap.c @srcroot@test/mremap.c \
@srcroot@test/posix_memalign.c @srcroot@test/thread_arena.c
@srcroot@test/posix_memalign.c @srcroot@test/thread_arena.c \
@srcroot@test/thread_tcache_enabled.c
ifeq (@enable_experimental@, 1)
CTESTS += @srcroot@test/allocm.c @srcroot@test/rallocm.c
endif

View File

@ -1103,6 +1103,20 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<function>mallctl*<parameter/></function> calls.</para></listitem>
</varlistentry>
<varlistentry>
<term>
<mallctl>thread.tcache.enabled</mallctl>
(<type>bool</type>)
<literal>rw</literal>
[<option>--enable-tcache</option>]
</term>
<listitem><para>Enable/disable calling thread's tcache. The tcache is
implicitly flushed as a side effect of becoming
disabled (see <link
lenkend="thread.tcache.flush"><mallctl>thread.tcache.flush</mallctl></link>).
</para></listitem>
</varlistentry>
<varlistentry>
<term>
<mallctl>thread.tcache.flush</mallctl>

View File

@ -5,6 +5,16 @@ typedef struct tcache_bin_info_s tcache_bin_info_t;
typedef struct tcache_bin_s tcache_bin_t;
typedef struct tcache_s tcache_t;
/*
* tcache pointers close to NULL are used to encode state information that is
* used for two purposes: preventing thread caching on a per thread basis and
* cleaning up during thread shutdown.
*/
#define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1)
#define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2)
#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
/*
* Absolute maximum number of cache slots for each small bin in the thread
* cache. This is an additional constraint beyond that imposed as: twice the
@ -35,6 +45,12 @@ typedef struct tcache_s tcache_t;
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
typedef enum {
tcache_enabled_false = 0, /* Enable cast to/from bool. */
tcache_enabled_true = 1,
tcache_enabled_default = 2
} tcache_enabled_t;
/*
* Read-only information associated with each element of tcache_t's tbins array
* is stored separately, mainly to reduce memory usage.
@ -105,9 +121,13 @@ bool tcache_boot1(void);
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
void tcache_event(tcache_t *tcache);
void tcache_flush(void);
bool tcache_enabled_get(void);
tcache_t *tcache_get(void);
void tcache_enabled_set(bool enabled);
void *tcache_alloc_easy(tcache_bin_t *tbin);
void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero);
@ -120,6 +140,69 @@ void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
malloc_tsd_externs(tcache, tcache_t *)
malloc_tsd_funcs(JEMALLOC_INLINE, tcache, tcache_t *, NULL,
tcache_thread_cleanup)
/* Per thread flag that allows thread caches to be disabled. */
malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
malloc_tsd_funcs(JEMALLOC_INLINE, tcache_enabled, tcache_enabled_t,
tcache_enabled_default, malloc_tsd_no_cleanup)
JEMALLOC_INLINE void
tcache_flush(void)
{
tcache_t *tcache;
cassert(config_tcache);
tcache = *tcache_tsd_get();
if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
return;
tcache_destroy(tcache);
tcache = NULL;
tcache_tsd_set(&tcache);
}
JEMALLOC_INLINE bool
tcache_enabled_get(void)
{
tcache_enabled_t tcache_enabled;
cassert(config_tcache);
tcache_enabled = *tcache_enabled_tsd_get();
if (tcache_enabled == tcache_enabled_default) {
tcache_enabled = (tcache_enabled_t)opt_tcache;
tcache_enabled_tsd_set(&tcache_enabled);
}
return ((bool)tcache_enabled);
}
JEMALLOC_INLINE void
tcache_enabled_set(bool enabled)
{
tcache_enabled_t tcache_enabled;
tcache_t *tcache;
cassert(config_tcache);
tcache_enabled = (tcache_enabled_t)enabled;
tcache_enabled_tsd_set(&tcache_enabled);
tcache = *tcache_tsd_get();
if (enabled) {
if (tcache == TCACHE_STATE_DISABLED) {
tcache = NULL;
tcache_tsd_set(&tcache);
}
} else /* disabled */ {
if (tcache > TCACHE_STATE_MAX) {
tcache_destroy(tcache);
tcache = NULL;
}
if (tcache == NULL) {
tcache = TCACHE_STATE_DISABLED;
tcache_tsd_set(&tcache);
}
}
}
JEMALLOC_INLINE tcache_t *
tcache_get(void)
@ -128,29 +211,32 @@ tcache_get(void)
if (config_tcache == false)
return (NULL);
if (config_lazy_lock && (isthreaded & opt_tcache) == false)
return (NULL);
else if (opt_tcache == false)
if (config_lazy_lock && isthreaded == false)
return (NULL);
tcache = *tcache_tsd_get();
if ((uintptr_t)tcache <= (uintptr_t)2) {
if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) {
if (tcache == TCACHE_STATE_DISABLED)
return (NULL);
if (tcache == NULL) {
tcache = tcache_create(choose_arena());
if (tcache == NULL)
if (tcache_enabled_get() == false) {
tcache_enabled_set(false); /* Memoize. */
return (NULL);
} else {
if (tcache == (void *)(uintptr_t)1) {
/*
* Make a note that an allocator function was
* called after the tcache_thread_cleanup() was
* called.
*/
tcache = (tcache_t *)(uintptr_t)2;
tcache_tsd_set(&tcache);
}
return (tcache_create(choose_arena()));
}
if (tcache == TCACHE_STATE_PURGATORY) {
/*
* Make a note that an allocator function was called
* after tcache_thread_cleanup() was called.
*/
tcache = TCACHE_STATE_REINCARNATED;
tcache_tsd_set(&tcache);
return (NULL);
}
if (tcache == TCACHE_STATE_REINCARNATED)
return (NULL);
not_reached();
}
return (tcache);

View File

@ -39,6 +39,7 @@ static int ctl_lookup(const char *name, ctl_node_t const **nodesp,
CTL_PROTO(version)
CTL_PROTO(epoch)
CTL_PROTO(thread_tcache_enabled)
CTL_PROTO(thread_tcache_flush)
CTL_PROTO(thread_arena)
CTL_PROTO(thread_allocated)
@ -151,6 +152,7 @@ CTL_PROTO(stats_mapped)
#define INDEX(i) false, {.indexed = {i##_index}}, NULL
static const ctl_node_t tcache_node[] = {
{NAME("enabled"), CTL(thread_tcache_enabled)},
{NAME("flush"), CTL(thread_tcache_flush)}
};
@ -966,25 +968,43 @@ RETURN:
return (ret);
}
static int
thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
bool oldval;
if (config_tcache == false)
return (ENOENT);
oldval = tcache_enabled_get();
if (newp != NULL) {
if (newlen != sizeof(bool)) {
ret = EINVAL;
goto RETURN;
}
tcache_enabled_set(*(bool *)newp);
}
READ(oldval, bool);
RETURN:
ret = 0;
return (ret);
}
static int
thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
tcache_t *tcache;
if (config_tcache == false)
return (ENOENT);
VOID();
if ((tcache = *tcache_tsd_get()) == NULL) {
ret = 0;
goto RETURN;
}
tcache_destroy(tcache);
tcache = NULL;
tcache_tsd_set(&tcache);
tcache_flush();
ret = 0;
RETURN:

View File

@ -5,6 +5,7 @@
/* Data. */
malloc_tsd_data(, tcache, tcache_t *, NULL)
malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default)
bool opt_tcache = true;
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
@ -328,25 +329,27 @@ tcache_thread_cleanup(void *arg)
{
tcache_t *tcache = *(tcache_t **)arg;
if (tcache == (void *)(uintptr_t)1) {
if (tcache == TCACHE_STATE_DISABLED) {
/* Do nothing. */
} else if (tcache == TCACHE_STATE_REINCARNATED) {
/*
* Another destructor called an allocator function after this
* destructor was called. Reset tcache to 1 in order to
* receive another callback.
*/
tcache = TCACHE_STATE_PURGATORY;
tcache_tsd_set(&tcache);
} else if (tcache == TCACHE_STATE_PURGATORY) {
/*
* The previous time this destructor was called, we set the key
* to 1 so that other destructors wouldn't cause re-creation of
* the tcache. This time, do nothing, so that the destructor
* will not be called again.
*/
} else if (tcache == (void *)(uintptr_t)2) {
/*
* Another destructor called an allocator function after this
* destructor was called. Reset tcache to 1 in order to
* receive another callback.
*/
tcache = (tcache_t *)(uintptr_t)1;
tcache_tsd_set(&tcache);
} else if (tcache != NULL) {
assert(tcache != (void *)(uintptr_t)1);
assert(tcache != TCACHE_STATE_PURGATORY);
tcache_destroy(tcache);
tcache = (tcache_t *)(uintptr_t)1;
tcache = TCACHE_STATE_PURGATORY;
tcache_tsd_set(&tcache);
}
}
@ -428,7 +431,7 @@ tcache_boot1(void)
{
if (opt_tcache) {
if (tcache_tsd_boot())
if (tcache_tsd_boot() || tcache_enabled_tsd_boot())
return (true);
}

View File

@ -0,0 +1,109 @@
#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <pthread.h>
#include <assert.h>
#include <errno.h>
#define JEMALLOC_MANGLE
#include "jemalloc_test.h"
void *
thread_start(void *arg)
{
int err;
size_t sz;
bool e0, e1;
sz = sizeof(bool);
if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) {
if (err == ENOENT) {
#ifdef JEMALLOC_TCACHE
assert(false);
#endif
}
goto RETURN;
}
if (e0) {
e1 = false;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz)
== 0);
assert(e0);
}
e1 = true;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0 == false);
e1 = true;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0);
e1 = false;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0);
e1 = false;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0 == false);
free(malloc(1));
e1 = true;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0 == false);
free(malloc(1));
e1 = true;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0);
free(malloc(1));
e1 = false;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0);
free(malloc(1));
e1 = false;
assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
assert(e0 == false);
free(malloc(1));
RETURN:
return (NULL);
}
int
main(void)
{
int ret = 0;
pthread_t thread;
fprintf(stderr, "Test begin\n");
thread_start(NULL);
if (pthread_create(&thread, NULL, thread_start, NULL)
!= 0) {
fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
ret = 1;
goto RETURN;
}
pthread_join(thread, (void *)&ret);
thread_start(NULL);
if (pthread_create(&thread, NULL, thread_start, NULL)
!= 0) {
fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
ret = 1;
goto RETURN;
}
pthread_join(thread, (void *)&ret);
thread_start(NULL);
RETURN:
fprintf(stderr, "Test end\n");
return (ret);
}

View File

@ -0,0 +1,2 @@
Test begin
Test end