Add the "thread.tcache.enabled" mallctl.

Makefile.in

@@ -66,7 +66,8 @@ DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3)
 DOCS := $(DOCS_HTML) $(DOCS_MAN3)
 CTESTS := @srcroot@test/aligned_alloc.c @srcroot@test/allocated.c \
 	@srcroot@test/bitmap.c @srcroot@test/mremap.c \
-	@srcroot@test/posix_memalign.c @srcroot@test/thread_arena.c
+	@srcroot@test/posix_memalign.c @srcroot@test/thread_arena.c \
+	@srcroot@test/thread_tcache_enabled.c
 ifeq (@enable_experimental@, 1)
 CTESTS += @srcroot@test/allocm.c @srcroot@test/rallocm.c
 endif

doc/jemalloc.xml.in

@@ -1103,6 +1103,20 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <function>mallctl*<parameter/></function> calls.</para></listitem>
       </varlistentry>
 
+      <varlistentry>
+        <term>
+          <mallctl>thread.tcache.enabled</mallctl>
+          (<type>bool</type>)
+          <literal>rw</literal>
+          [<option>--enable-tcache</option>]
+        </term>
+        <listitem><para>Enable/disable calling thread's tcache.  The tcache is
+        implicitly flushed as a side effect of becoming
+        disabled (see <link
+        lenkend="thread.tcache.flush"><mallctl>thread.tcache.flush</mallctl></link>).
+        </para></listitem>
+      </varlistentry>
+
       <varlistentry>
         <term>
           <mallctl>thread.tcache.flush</mallctl>

include/jemalloc/internal/tcache.h

@@ -5,6 +5,16 @@ typedef struct tcache_bin_info_s tcache_bin_info_t;
 typedef struct tcache_bin_s tcache_bin_t;
 typedef struct tcache_s tcache_t;
 
+/*
+ * tcache pointers close to NULL are used to encode state information that is
+ * used for two purposes: preventing thread caching on a per thread basis and
+ * cleaning up during thread shutdown.
+ */
+#define	TCACHE_STATE_DISABLED		((tcache_t *)(uintptr_t)1)
+#define	TCACHE_STATE_REINCARNATED	((tcache_t *)(uintptr_t)2)
+#define	TCACHE_STATE_PURGATORY		((tcache_t *)(uintptr_t)3)
+#define	TCACHE_STATE_MAX		TCACHE_STATE_PURGATORY
+
 /*
  * Absolute maximum number of cache slots for each small bin in the thread
  * cache.  This is an additional constraint beyond that imposed as: twice the
@@ -35,6 +45,12 @@ typedef struct tcache_s tcache_t;
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+typedef enum {
+	tcache_enabled_false   = 0, /* Enable cast to/from bool. */
+	tcache_enabled_true    = 1,
+	tcache_enabled_default = 2
+} tcache_enabled_t;
+
 /*
  * Read-only information associated with each element of tcache_t's tbins array
  * is stored separately, mainly to reduce memory usage.
@@ -105,9 +121,13 @@ bool	tcache_boot1(void);
 
 #ifndef JEMALLOC_ENABLE_INLINE
 malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *)
+malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t)
 
 void	tcache_event(tcache_t *tcache);
+void	tcache_flush(void);
+bool	tcache_enabled_get(void);
 tcache_t *tcache_get(void);
+void	tcache_enabled_set(bool enabled);
 void	*tcache_alloc_easy(tcache_bin_t *tbin);
 void	*tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
 void	*tcache_alloc_large(tcache_t *tcache, size_t size, bool zero);
@@ -120,6 +140,69 @@ void	tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
 malloc_tsd_externs(tcache, tcache_t *)
 malloc_tsd_funcs(JEMALLOC_INLINE, tcache, tcache_t *, NULL,
     tcache_thread_cleanup)
+/* Per thread flag that allows thread caches to be disabled. */
+malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
+malloc_tsd_funcs(JEMALLOC_INLINE, tcache_enabled, tcache_enabled_t,
+    tcache_enabled_default, malloc_tsd_no_cleanup)
+
+JEMALLOC_INLINE void
+tcache_flush(void)
+{
+	tcache_t *tcache;
+
+	cassert(config_tcache);
+
+	tcache = *tcache_tsd_get();
+	if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)
+		return;
+	tcache_destroy(tcache);
+	tcache = NULL;
+	tcache_tsd_set(&tcache);
+}
+
+JEMALLOC_INLINE bool
+tcache_enabled_get(void)
+{
+	tcache_enabled_t tcache_enabled;
+
+	cassert(config_tcache);
+
+	tcache_enabled = *tcache_enabled_tsd_get();
+	if (tcache_enabled == tcache_enabled_default) {
+		tcache_enabled = (tcache_enabled_t)opt_tcache;
+		tcache_enabled_tsd_set(&tcache_enabled);
+	}
+
+	return ((bool)tcache_enabled);
+}
+
+JEMALLOC_INLINE void
+tcache_enabled_set(bool enabled)
+{
+	tcache_enabled_t tcache_enabled;
+	tcache_t *tcache;
+
+	cassert(config_tcache);
+
+	tcache_enabled = (tcache_enabled_t)enabled;
+	tcache_enabled_tsd_set(&tcache_enabled);
+	tcache = *tcache_tsd_get();
+	if (enabled) {
+		if (tcache == TCACHE_STATE_DISABLED) {
+			tcache = NULL;
+			tcache_tsd_set(&tcache);
+		}
+	} else /* disabled */ {
+		if (tcache > TCACHE_STATE_MAX) {
+			tcache_destroy(tcache);
+			tcache = NULL;
+		}
+		if (tcache == NULL) {
+			tcache = TCACHE_STATE_DISABLED;
+			tcache_tsd_set(&tcache);
+		}
+	}
+}
 
 JEMALLOC_INLINE tcache_t *
 tcache_get(void)
@@ -128,29 +211,32 @@ tcache_get(void)
 
 	if (config_tcache == false)
 		return (NULL);
-	if (config_lazy_lock && (isthreaded & opt_tcache) == false)
+	if (config_lazy_lock && isthreaded == false)
-		return (NULL);
-	else if (opt_tcache == false)
 		return (NULL);
 
 	tcache = *tcache_tsd_get();
-	if ((uintptr_t)tcache <= (uintptr_t)2) {
+	if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) {
+		if (tcache == TCACHE_STATE_DISABLED)
+			return (NULL);
 		if (tcache == NULL) {
-			tcache = tcache_create(choose_arena());
+			if (tcache_enabled_get() == false) {
-			if (tcache == NULL)
+				tcache_enabled_set(false); /* Memoize. */
 				return (NULL);
-		} else {
+			}
-			if (tcache == (void *)(uintptr_t)1) {
+			return (tcache_create(choose_arena()));
+		}
+		if (tcache == TCACHE_STATE_PURGATORY) {
 			/*
-				 * Make a note that an allocator function was
+			 * Make a note that an allocator function was called
-				 * called after the tcache_thread_cleanup() was
+			 * after tcache_thread_cleanup() was called.
-				 * called.
 			 */
-				tcache = (tcache_t *)(uintptr_t)2;
+			tcache = TCACHE_STATE_REINCARNATED;
 			tcache_tsd_set(&tcache);
-			}
 			return (NULL);
 		}
+		if (tcache == TCACHE_STATE_REINCARNATED)
+			return (NULL);
+		not_reached();
 	}
 
 	return (tcache);

src/ctl.c

@@ -39,6 +39,7 @@ static int	ctl_lookup(const char *name, ctl_node_t const **nodesp,
 
 CTL_PROTO(version)
 CTL_PROTO(epoch)
+CTL_PROTO(thread_tcache_enabled)
 CTL_PROTO(thread_tcache_flush)
 CTL_PROTO(thread_arena)
 CTL_PROTO(thread_allocated)
@@ -151,6 +152,7 @@ CTL_PROTO(stats_mapped)
 #define	INDEX(i)	false,	{.indexed = {i##_index}},		NULL
 
 static const ctl_node_t	tcache_node[] = {
+	{NAME("enabled"),	CTL(thread_tcache_enabled)},
 	{NAME("flush"),		CTL(thread_tcache_flush)}
 };
 
@@ -966,25 +968,43 @@ RETURN:
 	return (ret);
 }
 
+static int
+thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
+    size_t *oldlenp, void *newp, size_t newlen)
+{
+	int ret;
+	bool oldval;
+
+	if (config_tcache == false)
+		return (ENOENT);
+
+	oldval = tcache_enabled_get();
+	if (newp != NULL) {
+		if (newlen != sizeof(bool)) {
+			ret = EINVAL;
+			goto RETURN;
+		}
+		tcache_enabled_set(*(bool *)newp);
+	}
+	READ(oldval, bool);
+
+RETURN:
+	ret = 0;
+	return (ret);
+}
+
 static int
 thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
     size_t *oldlenp, void *newp, size_t newlen)
 {
 	int ret;
-	tcache_t *tcache;
 
 	if (config_tcache == false)
 		return (ENOENT);
 
 	VOID();
 
-	if ((tcache = *tcache_tsd_get()) == NULL) {
+	tcache_flush();
-		ret = 0;
-		goto RETURN;
-	}
-	tcache_destroy(tcache);
-	tcache = NULL;
-	tcache_tsd_set(&tcache);
 
 	ret = 0;
 RETURN:

src/tcache.c

@@ -5,6 +5,7 @@
 /* Data. */
 
 malloc_tsd_data(, tcache, tcache_t *, NULL)
+malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default)
 
 bool	opt_tcache = true;
 ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
@@ -328,25 +329,27 @@ tcache_thread_cleanup(void *arg)
 {
 	tcache_t *tcache = *(tcache_t **)arg;
 
-	if (tcache == (void *)(uintptr_t)1) {
+	if (tcache == TCACHE_STATE_DISABLED) {
+		/* Do nothing. */
+	} else if (tcache == TCACHE_STATE_REINCARNATED) {
+		/*
+		 * Another destructor called an allocator function after this
+		 * destructor was called.  Reset tcache to 1 in order to
+		 * receive another callback.
+		 */
+		tcache = TCACHE_STATE_PURGATORY;
+		tcache_tsd_set(&tcache);
+	} else if (tcache == TCACHE_STATE_PURGATORY) {
 		/*
 		 * The previous time this destructor was called, we set the key
 		 * to 1 so that other destructors wouldn't cause re-creation of
 		 * the tcache.  This time, do nothing, so that the destructor
 		 * will not be called again.
 		 */
-	} else if (tcache == (void *)(uintptr_t)2) {
-		/*
-		 * Another destructor called an allocator function after this
-		 * destructor was called.  Reset tcache to 1 in order to
-		 * receive another callback.
-		 */
-		tcache = (tcache_t *)(uintptr_t)1;
-		tcache_tsd_set(&tcache);
 	} else if (tcache != NULL) {
-		assert(tcache != (void *)(uintptr_t)1);
+		assert(tcache != TCACHE_STATE_PURGATORY);
 		tcache_destroy(tcache);
-		tcache = (tcache_t *)(uintptr_t)1;
+		tcache = TCACHE_STATE_PURGATORY;
 		tcache_tsd_set(&tcache);
 	}
 }
@@ -428,7 +431,7 @@ tcache_boot1(void)
 {
 
 	if (opt_tcache) {
-		if (tcache_tsd_boot())
+		if (tcache_tsd_boot() || tcache_enabled_tsd_boot())
 			return (true);
 	}
 

test/thread_tcache_enabled.c

@@ -0,0 +1,109 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <pthread.h>
+#include <assert.h>
+#include <errno.h>
+
+#define	JEMALLOC_MANGLE
+#include "jemalloc_test.h"
+
+void *
+thread_start(void *arg)
+{
+	int err;
+	size_t sz;
+	bool e0, e1;
+
+	sz = sizeof(bool);
+	if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) {
+		if (err == ENOENT) {
+#ifdef JEMALLOC_TCACHE
+			assert(false);
+#endif
+		}
+		goto RETURN;
+	}
+
+	if (e0) {
+		e1 = false;
+		assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz)
+		    == 0);
+		assert(e0);
+	}
+
+	e1 = true;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0 == false);
+
+	e1 = true;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0);
+
+	e1 = false;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0);
+
+	e1 = false;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0 == false);
+
+	free(malloc(1));
+	e1 = true;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0 == false);
+
+	free(malloc(1));
+	e1 = true;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0);
+
+	free(malloc(1));
+	e1 = false;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0);
+
+	free(malloc(1));
+	e1 = false;
+	assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0);
+	assert(e0 == false);
+
+	free(malloc(1));
+RETURN:
+	return (NULL);
+}
+
+int
+main(void)
+{
+	int ret = 0;
+	pthread_t thread;
+
+	fprintf(stderr, "Test begin\n");
+
+	thread_start(NULL);
+
+	if (pthread_create(&thread, NULL, thread_start, NULL)
+	    != 0) {
+		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
+		ret = 1;
+		goto RETURN;
+	}
+	pthread_join(thread, (void *)&ret);
+
+	thread_start(NULL);
+
+	if (pthread_create(&thread, NULL, thread_start, NULL)
+	    != 0) {
+		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
+		ret = 1;
+		goto RETURN;
+	}
+	pthread_join(thread, (void *)&ret);
+
+	thread_start(NULL);
+
+RETURN:
+	fprintf(stderr, "Test end\n");
+	return (ret);
+}

test/thread_tcache_enabled.exp

@@ -0,0 +1,2 @@
+Test begin
+Test end