diff --git a/Makefile.in b/Makefile.in
index 494ac9a6..821c0634 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -66,7 +66,8 @@ DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3)
DOCS := $(DOCS_HTML) $(DOCS_MAN3)
CTESTS := @srcroot@test/aligned_alloc.c @srcroot@test/allocated.c \
@srcroot@test/bitmap.c @srcroot@test/mremap.c \
- @srcroot@test/posix_memalign.c @srcroot@test/thread_arena.c
+ @srcroot@test/posix_memalign.c @srcroot@test/thread_arena.c \
+ @srcroot@test/thread_tcache_enabled.c
ifeq (@enable_experimental@, 1)
CTESTS += @srcroot@test/allocm.c @srcroot@test/rallocm.c
endif
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index 3cbc851f..0b468b04 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -1103,6 +1103,20 @@ malloc_conf = "xmalloc:true";]]>
mallctl* calls.
+
+
+ thread.tcache.enabled
+ (bool)
+ rw
+ []
+
+ Enable/disable calling thread's tcache. The tcache is
+ implicitly flushed as a side effect of becoming
+ disabled (see thread.tcache.flush).
+
+
+
thread.tcache.flush
diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h
index 30e63a50..0d999f24 100644
--- a/include/jemalloc/internal/tcache.h
+++ b/include/jemalloc/internal/tcache.h
@@ -5,6 +5,16 @@ typedef struct tcache_bin_info_s tcache_bin_info_t;
typedef struct tcache_bin_s tcache_bin_t;
typedef struct tcache_s tcache_t;
+/*
+ * tcache pointers close to NULL are used to encode state information that is
+ * used for two purposes: preventing thread caching on a per thread basis and
+ * cleaning up during thread shutdown.
+ */
+#define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1)
+#define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2)
+#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3)
+#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY
+
/*
* Absolute maximum number of cache slots for each small bin in the thread
* cache. This is an additional constraint beyond that imposed as: twice the
@@ -35,6 +45,12 @@ typedef struct tcache_s tcache_t;
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
+typedef enum {
+ tcache_enabled_false = 0, /* Enable cast to/from bool. */
+ tcache_enabled_true = 1,
+ tcache_enabled_default = 2
+} tcache_enabled_t;
+
/*
* Read-only information associated with each element of tcache_t's tbins array
* is stored separately, mainly to reduce memory usage.
@@ -105,9 +121,13 @@ bool tcache_boot1(void);
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
+malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
void tcache_event(tcache_t *tcache);
+void tcache_flush(void);
+bool tcache_enabled_get(void);
tcache_t *tcache_get(void);
+void tcache_enabled_set(bool enabled);
void *tcache_alloc_easy(tcache_bin_t *tbin);
void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero);
@@ -120,6 +140,69 @@ void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
malloc_tsd_externs(tcache, tcache_t *)
malloc_tsd_funcs(JEMALLOC_INLINE, tcache, tcache_t *, NULL,
tcache_thread_cleanup)
+/* Per thread flag that allows thread caches to be disabled. */
+malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
+malloc_tsd_funcs(JEMALLOC_INLINE, tcache_enabled, tcache_enabled_t,
+ tcache_enabled_default, malloc_tsd_no_cleanup)
+
+JEMALLOC_INLINE void
+tcache_flush(void)
+{
+ tcache_t *tcache;
+
+ cassert(config_tcache);
+
+ tcache = *tcache_tsd_get();
+ if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
+ return;
+ tcache_destroy(tcache);
+ tcache = NULL;
+ tcache_tsd_set(&tcache);
+}
+
+JEMALLOC_INLINE bool
+tcache_enabled_get(void)
+{
+ tcache_enabled_t tcache_enabled;
+
+ cassert(config_tcache);
+
+ tcache_enabled = *tcache_enabled_tsd_get();
+ if (tcache_enabled == tcache_enabled_default) {
+ tcache_enabled = (tcache_enabled_t)opt_tcache;
+ tcache_enabled_tsd_set(&tcache_enabled);
+ }
+
+ return ((bool)tcache_enabled);
+}
+
+JEMALLOC_INLINE void
+tcache_enabled_set(bool enabled)
+{
+ tcache_enabled_t tcache_enabled;
+ tcache_t *tcache;
+
+ cassert(config_tcache);
+
+ tcache_enabled = (tcache_enabled_t)enabled;
+ tcache_enabled_tsd_set(&tcache_enabled);
+ tcache = *tcache_tsd_get();
+ if (enabled) {
+ if (tcache == TCACHE_STATE_DISABLED) {
+ tcache = NULL;
+ tcache_tsd_set(&tcache);
+ }
+ } else /* disabled */ {
+ if (tcache > TCACHE_STATE_MAX) {
+ tcache_destroy(tcache);
+ tcache = NULL;
+ }
+ if (tcache == NULL) {
+ tcache = TCACHE_STATE_DISABLED;
+ tcache_tsd_set(&tcache);
+ }
+ }
+}
JEMALLOC_INLINE tcache_t *
tcache_get(void)
@@ -128,29 +211,32 @@ tcache_get(void)
if (config_tcache == false)
return (NULL);
- if (config_lazy_lock && (isthreaded & opt_tcache) == false)
- return (NULL);
- else if (opt_tcache == false)
+ if (config_lazy_lock && isthreaded == false)
return (NULL);
tcache = *tcache_tsd_get();
- if ((uintptr_t)tcache <= (uintptr_t)2) {
+ if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) {
+ if (tcache == TCACHE_STATE_DISABLED)
+ return (NULL);
if (tcache == NULL) {
- tcache = tcache_create(choose_arena());
- if (tcache == NULL)
+ if (tcache_enabled_get() == false) {
+ tcache_enabled_set(false); /* Memoize. */
return (NULL);
- } else {
- if (tcache == (void *)(uintptr_t)1) {
- /*
- * Make a note that an allocator function was
- * called after the tcache_thread_cleanup() was
- * called.
- */
- tcache = (tcache_t *)(uintptr_t)2;
- tcache_tsd_set(&tcache);
}
+ return (tcache_create(choose_arena()));
+ }
+ if (tcache == TCACHE_STATE_PURGATORY) {
+ /*
+ * Make a note that an allocator function was called
+ * after tcache_thread_cleanup() was called.
+ */
+ tcache = TCACHE_STATE_REINCARNATED;
+ tcache_tsd_set(&tcache);
return (NULL);
}
+ if (tcache == TCACHE_STATE_REINCARNATED)
+ return (NULL);
+ not_reached();
}
return (tcache);
diff --git a/src/ctl.c b/src/ctl.c
index 943c2925..08011616 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -39,6 +39,7 @@ static int ctl_lookup(const char *name, ctl_node_t const **nodesp,
CTL_PROTO(version)
CTL_PROTO(epoch)
+CTL_PROTO(thread_tcache_enabled)
CTL_PROTO(thread_tcache_flush)
CTL_PROTO(thread_arena)
CTL_PROTO(thread_allocated)
@@ -151,6 +152,7 @@ CTL_PROTO(stats_mapped)
#define INDEX(i) false, {.indexed = {i##_index}}, NULL
static const ctl_node_t tcache_node[] = {
+ {NAME("enabled"), CTL(thread_tcache_enabled)},
{NAME("flush"), CTL(thread_tcache_flush)}
};
@@ -966,25 +968,43 @@ RETURN:
return (ret);
}
+static int
+thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
+{
+ int ret;
+ bool oldval;
+
+ if (config_tcache == false)
+ return (ENOENT);
+
+ oldval = tcache_enabled_get();
+ if (newp != NULL) {
+ if (newlen != sizeof(bool)) {
+ ret = EINVAL;
+ goto RETURN;
+ }
+ tcache_enabled_set(*(bool *)newp);
+ }
+ READ(oldval, bool);
+
+RETURN:
+ ret = 0;
+ return (ret);
+}
+
static int
thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
- tcache_t *tcache;
if (config_tcache == false)
return (ENOENT);
VOID();
- if ((tcache = *tcache_tsd_get()) == NULL) {
- ret = 0;
- goto RETURN;
- }
- tcache_destroy(tcache);
- tcache = NULL;
- tcache_tsd_set(&tcache);
+ tcache_flush();
ret = 0;
RETURN:
diff --git a/src/tcache.c b/src/tcache.c
index 3442406d..bc911a6e 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -5,6 +5,7 @@
/* Data. */
malloc_tsd_data(, tcache, tcache_t *, NULL)
+malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default)
bool opt_tcache = true;
ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
@@ -328,25 +329,27 @@ tcache_thread_cleanup(void *arg)
{
tcache_t *tcache = *(tcache_t **)arg;
- if (tcache == (void *)(uintptr_t)1) {
+ if (tcache == TCACHE_STATE_DISABLED) {
+ /* Do nothing. */
+ } else if (tcache == TCACHE_STATE_REINCARNATED) {
+ /*
+ * Another destructor called an allocator function after this
+ * destructor was called. Reset tcache to 1 in order to
+ * receive another callback.
+ */
+ tcache = TCACHE_STATE_PURGATORY;
+ tcache_tsd_set(&tcache);
+ } else if (tcache == TCACHE_STATE_PURGATORY) {
/*
* The previous time this destructor was called, we set the key
* to 1 so that other destructors wouldn't cause re-creation of
* the tcache. This time, do nothing, so that the destructor
* will not be called again.
*/
- } else if (tcache == (void *)(uintptr_t)2) {
- /*
- * Another destructor called an allocator function after this
- * destructor was called. Reset tcache to 1 in order to
- * receive another callback.
- */
- tcache = (tcache_t *)(uintptr_t)1;
- tcache_tsd_set(&tcache);
} else if (tcache != NULL) {
- assert(tcache != (void *)(uintptr_t)1);
+ assert(tcache != TCACHE_STATE_PURGATORY);
tcache_destroy(tcache);
- tcache = (tcache_t *)(uintptr_t)1;
+ tcache = TCACHE_STATE_PURGATORY;
tcache_tsd_set(&tcache);
}
}
@@ -428,7 +431,7 @@ tcache_boot1(void)
{
if (opt_tcache) {
- if (tcache_tsd_boot())
+ if (tcache_tsd_boot() || tcache_enabled_tsd_boot())
return (true);
}
diff --git a/test/thread_tcache_enabled.c b/test/thread_tcache_enabled.c
new file mode 100644
index 00000000..46540385
--- /dev/null
+++ b/test/thread_tcache_enabled.c
@@ -0,0 +1,109 @@
+#include
+#include
+#include
+#include
+#include
+#include
+
+#define JEMALLOC_MANGLE
+#include "jemalloc_test.h"
+
+void *
+thread_start(void *arg)
+{
+ int err;
+ size_t sz;
+ bool e0, e1;
+
+ sz = sizeof(bool);
+ if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) {
+ if (err == ENOENT) {
+#ifdef JEMALLOC_TCACHE
+ assert(false);
+#endif
+ }
+ goto RETURN;
+ }
+
+ if (e0) {
+ e1 = false;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz)
+ == 0);
+ assert(e0);
+ }
+
+ e1 = true;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0 == false);
+
+ e1 = true;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0);
+
+ e1 = false;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0);
+
+ e1 = false;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0 == false);
+
+ free(malloc(1));
+ e1 = true;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0 == false);
+
+ free(malloc(1));
+ e1 = true;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0);
+
+ free(malloc(1));
+ e1 = false;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0);
+
+ free(malloc(1));
+ e1 = false;
+ assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+ assert(e0 == false);
+
+ free(malloc(1));
+RETURN:
+ return (NULL);
+}
+
+int
+main(void)
+{
+ int ret = 0;
+ pthread_t thread;
+
+ fprintf(stderr, "Test begin\n");
+
+ thread_start(NULL);
+
+ if (pthread_create(&thread, NULL, thread_start, NULL)
+ != 0) {
+ fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
+ ret = 1;
+ goto RETURN;
+ }
+ pthread_join(thread, (void *)&ret);
+
+ thread_start(NULL);
+
+ if (pthread_create(&thread, NULL, thread_start, NULL)
+ != 0) {
+ fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
+ ret = 1;
+ goto RETURN;
+ }
+ pthread_join(thread, (void *)&ret);
+
+ thread_start(NULL);
+
+RETURN:
+ fprintf(stderr, "Test end\n");
+ return (ret);
+}
diff --git a/test/thread_tcache_enabled.exp b/test/thread_tcache_enabled.exp
new file mode 100644
index 00000000..369a88dd
--- /dev/null
+++ b/test/thread_tcache_enabled.exp
@@ -0,0 +1,2 @@
+Test begin
+Test end