Implement deallocation events.

Make the event module to accept two event types, and pass around the event
context.  Use bytes-based events to trigger tcache GC on deallocation, and get
rid of the tcache ticker.

include/jemalloc/internal/tcache_inlines.h

@@ -5,7 +5,6 @@
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/ticker.h"
 #include "jemalloc/internal/util.h"
 
 static inline bool
@@ -27,17 +26,6 @@ tcache_enabled_set(tsd_t *tsd, bool enabled) {
 	tsd_slow_update(tsd);
 }
 
-JEMALLOC_ALWAYS_INLINE void
-tcache_event(tsd_t *tsd, tcache_t *tcache) {
-	if (TCACHE_GC_INCR == 0) {
-		return;
-	}
-
-	if (unlikely(ticker_tick(&tcache->gc_ticker))) {
-		tcache_event_hard(tsd, tcache);
-	}
-}
-
 JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
     size_t size, szind_t binind, bool zero, bool slow_path) {
@@ -171,8 +159,6 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
 		bool ret = cache_bin_dalloc_easy(bin, ptr);
 		assert(ret);
 	}
-
-	tcache_event(tsd, tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -195,8 +181,6 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
 		bool ret = cache_bin_dalloc_easy(bin, ptr);
 		assert(ret);
 	}
-
-	tcache_event(tsd, tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE tcache_t *

include/jemalloc/internal/tcache_structs.h

@@ -16,9 +16,6 @@ struct tcache_s {
 	 * together at the start of this struct.
 	 */
 
-	/* Drives incremental GC. */
-	ticker_t	gc_ticker;
-
 	/*
 	 * The pointer stacks associated with bins follow as a contiguous array.
 	 * During tcache initialization, the avail pointer in each element of

include/jemalloc/internal/thread_event.h

@@ -4,42 +4,51 @@
 #include "jemalloc/internal/tsd.h"
 
 /*
- * Maximum threshold on thread_allocated_next_event_fast, so that there is no
+ * Maximum threshold on thread_(de)allocated_next_event_fast, so that there is
- * need to check overflow in malloc fast path. (The allocation size in malloc
+ * no need to check overflow in malloc fast path. (The allocation size in malloc
  * fast path never exceeds SC_LOOKUP_MAXCLASS.)
  */
-#define THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX				\
+#define THREAD_NEXT_EVENT_FAST_MAX				\
     (UINT64_MAX - SC_LOOKUP_MAXCLASS + 1U)
 
 /*
  * The max interval helps make sure that malloc stays on the fast path in the
- * common case, i.e. thread_allocated < thread_allocated_next_event_fast.
+ * common case, i.e. thread_allocated < thread_allocated_next_event_fast.  When
- * When thread_allocated is within an event's distance to
+ * thread_allocated is within an event's distance to THREAD_NEXT_EVENT_FAST_MAX
- * THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX above, thread_allocated_next_event_fast
+ * above, thread_allocated_next_event_fast is wrapped around and we fall back to
- * is wrapped around and we fall back to the medium-fast path. The max interval
+ * the medium-fast path. The max interval makes sure that we're not staying on
- * makes sure that we're not staying on the fallback case for too long, even if
+ * the fallback case for too long, even if there's no active event or if all
- * there's no active event or if all active events have long wait times.
+ * active events have long wait times.
  */
 #define THREAD_EVENT_MAX_INTERVAL ((uint64_t)(4U << 20))
 
+typedef struct event_ctx_s {
+	bool is_alloc;
+	uint64_t *current;
+	uint64_t *last_event;
+	uint64_t *next_event;
+	uint64_t *next_event_fast;
+} event_ctx_t;
+
 void thread_event_assert_invariants_debug(tsd_t *tsd);
-void thread_event_trigger(tsd_t *tsd, bool delay_event);
+void thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event);
-void thread_event_rollback(tsd_t *tsd, size_t diff);
+void thread_alloc_event_rollback(tsd_t *tsd, size_t diff);
-void thread_event_update(tsd_t *tsd);
+void thread_event_update(tsd_t *tsd, bool alloc_event);
 void thread_event_boot();
 void thread_event_recompute_fast_threshold(tsd_t *tsd);
 void tsd_thread_event_init(tsd_t *tsd);
 
 /*
  * List of all events, in the following format:
- *  E(event,		(condition))
+ *  E(event,		(condition), is_alloc_event)
  */
 #define ITERATE_OVER_ALL_EVENTS						\
-    E(tcache_gc,	(TCACHE_GC_INCR_BYTES > 0))			\
+    E(tcache_gc,	(TCACHE_GC_INCR_BYTES > 0), true)		\
-    E(prof_sample,	(config_prof && opt_prof))	    		\
+    E(prof_sample,	(config_prof && opt_prof), true)	    	\
-    E(stats_interval,	(opt_stats_interval >= 0))
+    E(stats_interval,	(opt_stats_interval >= 0), true)	    	\
+    E(tcache_gc_dalloc,	(TCACHE_GC_INCR_BYTES > 0), false)
 
-#define E(event, condition)						\
+#define E(event, condition_unused, is_alloc_event_unused)		\
     C(event##_event_wait)
 
 /* List of all thread event counters. */
@@ -83,9 +92,9 @@ ITERATE_OVER_ALL_COUNTERS
 #undef E
 
 /*
- * Two malloc fastpath getters -- use the unsafe getters since tsd may be
+ * The malloc and free fastpath getters -- use the unsafe getters since tsd may
- * non-nominal, in which case the fast_threshold will be set to 0.  This allows
+ * be non-nominal, in which case the fast_threshold will be set to 0.  This
- * checking for events and tsd non-nominal in a single branch.
+ * allows checking for events and tsd non-nominal in a single branch.
  *
  * Note that these can only be used on the fastpath.
  */
@@ -97,42 +106,83 @@ thread_allocated_malloc_fastpath(tsd_t *tsd) {
 JEMALLOC_ALWAYS_INLINE uint64_t
 thread_allocated_next_event_malloc_fastpath(tsd_t *tsd) {
 	uint64_t v = *tsd_thread_allocated_next_event_fastp_get_unsafe(tsd);
-	assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
+	assert(v <= THREAD_NEXT_EVENT_FAST_MAX);
 	return v;
 }
 
+JEMALLOC_ALWAYS_INLINE void
+thread_event_free_fastpath_ctx(tsd_t *tsd, uint64_t *deallocated,
+    uint64_t *threshold, bool size_hint) {
+	if (!size_hint) {
+		*deallocated = tsd_thread_deallocated_get(tsd);
+		*threshold = tsd_thread_deallocated_next_event_fast_get(tsd);
+	} else {
+		/* Unsafe getters since this may happen before tsd_init. */
+		*deallocated = *tsd_thread_deallocatedp_get_unsafe(tsd);
+		*threshold =
+		    *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd);
+	}
+	assert(*threshold <= THREAD_NEXT_EVENT_FAST_MAX);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+event_ctx_is_alloc(event_ctx_t *ctx) {
+	return ctx->is_alloc;
+}
+
+JEMALLOC_ALWAYS_INLINE uint64_t
+event_ctx_current_bytes_get(event_ctx_t *ctx) {
+	return *ctx->current;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+event_ctx_current_bytes_set(event_ctx_t *ctx, uint64_t v) {
+	*ctx->current = v;
+}
+
+JEMALLOC_ALWAYS_INLINE uint64_t
+event_ctx_last_event_get(event_ctx_t *ctx) {
+	return *ctx->last_event;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+event_ctx_last_event_set(event_ctx_t *ctx, uint64_t v) {
+	*ctx->last_event = v;
+}
+
 /* Below 3 for next_event_fast. */
 JEMALLOC_ALWAYS_INLINE uint64_t
-thread_allocated_next_event_fast_get(tsd_t *tsd) {
+event_ctx_next_event_fast_get(event_ctx_t *ctx) {
-	uint64_t v = tsd_thread_allocated_next_event_fast_get(tsd);
+	uint64_t v = *ctx->next_event_fast;
-	assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
+	assert(v <= THREAD_NEXT_EVENT_FAST_MAX);
 	return v;
 }
 
 JEMALLOC_ALWAYS_INLINE void
-thread_allocated_next_event_fast_set(tsd_t *tsd, uint64_t v) {
+event_ctx_next_event_fast_set(event_ctx_t *ctx, uint64_t v) {
-	assert(v <= THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
+	assert(v <= THREAD_NEXT_EVENT_FAST_MAX);
-	*tsd_thread_allocated_next_event_fastp_get(tsd) = v;
+	*ctx->next_event_fast = v;
 }
 
 JEMALLOC_ALWAYS_INLINE void
-thread_allocated_next_event_fast_set_non_nominal(tsd_t *tsd) {
+thread_next_event_fast_set_non_nominal(tsd_t *tsd) {
 	/*
-	 * Set the fast threshold to zero when tsd is non-nominal.  Use the
+	 * Set the fast thresholds to zero when tsd is non-nominal.  Use the
 	 * unsafe getter as this may get called during tsd init and clean up.
 	 */
 	*tsd_thread_allocated_next_event_fastp_get_unsafe(tsd) = 0;
+	*tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) = 0;
 }
 
 /* For next_event.  Setter also updates the fast threshold. */
 JEMALLOC_ALWAYS_INLINE uint64_t
-thread_allocated_next_event_get(tsd_t *tsd) {
+event_ctx_next_event_get(event_ctx_t *ctx) {
-	return tsd_thread_allocated_next_event_get(tsd);
+	return *ctx->next_event;
 }
 
 JEMALLOC_ALWAYS_INLINE void
-thread_allocated_next_event_set(tsd_t *tsd, uint64_t v) {
+event_ctx_next_event_set(tsd_t *tsd, event_ctx_t *ctx, uint64_t v) {
-	*tsd_thread_allocated_next_eventp_get(tsd) = v;
+	*ctx->next_event = v;
 	thread_event_recompute_fast_threshold(tsd);
 }
 
@@ -145,8 +195,8 @@ thread_allocated_next_event_set(tsd_t *tsd, uint64_t v) {
  *     at the end will restore the invariants),
  * (b) thread_##event##_event_update() (the thread_event_update() call at the
  *     end will restore the invariants), or
- * (c) thread_event_rollback() if the rollback falls below the last_event (the
+ * (c) thread_alloc_event_rollback() if the rollback falls below the last_event
- *     thread_event_update() call at the end will restore the invariants).
+ *     (the thread_event_update() call at the end will restore the invariants).
  */
 JEMALLOC_ALWAYS_INLINE void
 thread_event_assert_invariants(tsd_t *tsd) {
@@ -156,22 +206,52 @@ thread_event_assert_invariants(tsd_t *tsd) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-thread_event(tsd_t *tsd, size_t usize) {
+event_ctx_get(tsd_t *tsd, event_ctx_t *ctx, bool is_alloc) {
-	thread_event_assert_invariants(tsd);
+	ctx->is_alloc = is_alloc;
-
+	if (is_alloc) {
-	uint64_t thread_allocated_before = thread_allocated_get(tsd);
+		ctx->current = tsd_thread_allocatedp_get(tsd);
-	thread_allocated_set(tsd, thread_allocated_before + usize);
+		ctx->last_event = tsd_thread_allocated_last_eventp_get(tsd);
-
+		ctx->next_event = tsd_thread_allocated_next_eventp_get(tsd);
-	/* The subtraction is intentionally susceptible to underflow. */
+		ctx->next_event_fast =
-	if (likely(usize < thread_allocated_next_event_get(tsd) -
+		    tsd_thread_allocated_next_event_fastp_get(tsd);
-	    thread_allocated_before)) {
-		thread_event_assert_invariants(tsd);
 	} else {
-		thread_event_trigger(tsd, false);
+		ctx->current = tsd_thread_deallocatedp_get(tsd);
+		ctx->last_event = tsd_thread_deallocated_last_eventp_get(tsd);
+		ctx->next_event = tsd_thread_deallocated_next_eventp_get(tsd);
+		ctx->next_event_fast =
+		    tsd_thread_deallocated_next_event_fastp_get(tsd);
 	}
 }
 
-#define E(event, condition)						\
+JEMALLOC_ALWAYS_INLINE void
+thread_event_advance(tsd_t *tsd, size_t usize, bool is_alloc) {
+	thread_event_assert_invariants(tsd);
+
+	event_ctx_t ctx;
+	event_ctx_get(tsd, &ctx, is_alloc);
+
+	uint64_t bytes_before = event_ctx_current_bytes_get(&ctx);
+	event_ctx_current_bytes_set(&ctx, bytes_before + usize);
+
+	/* The subtraction is intentionally susceptible to underflow. */
+	if (likely(usize < event_ctx_next_event_get(&ctx) - bytes_before)) {
+		thread_event_assert_invariants(tsd);
+	} else {
+		thread_event_trigger(tsd, &ctx, false);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+thread_dalloc_event(tsd_t *tsd, size_t usize) {
+	thread_event_advance(tsd, usize, false);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+thread_alloc_event(tsd_t *tsd, size_t usize) {
+	thread_event_advance(tsd, usize, true);
+}
+
+#define E(event, condition, is_alloc)					\
 JEMALLOC_ALWAYS_INLINE void						\
 thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) {	\
 	thread_event_assert_invariants(tsd);				\
@@ -188,7 +268,7 @@ thread_##event##_event_update(tsd_t *tsd, uint64_t event_wait) {	\
 		event_wait = THREAD_EVENT_MAX_START_WAIT;		\
 	}								\
 	event##_event_wait_set(tsd, event_wait);			\
-	thread_event_update(tsd);					\
+	thread_event_update(tsd, is_alloc);				\
 }
 
 ITERATE_OVER_ALL_EVENTS

include/jemalloc/internal/tsd.h

@@ -81,10 +81,14 @@ typedef void (*test_callback_t)(int *);
     O(thread_allocated,		uint64_t,		uint64_t)	\
     O(thread_allocated_next_event_fast,	uint64_t,	uint64_t)	\
     O(thread_deallocated,	uint64_t,		uint64_t)	\
+    O(thread_deallocated_next_event_fast, uint64_t,	uint64_t)	\
     O(rtree_ctx,		rtree_ctx_t,		rtree_ctx_t)	\
     O(thread_allocated_last_event,	uint64_t,	uint64_t)	\
     O(thread_allocated_next_event,	uint64_t,	uint64_t)	\
+    O(thread_deallocated_last_event,	uint64_t,	uint64_t)	\
+    O(thread_deallocated_next_event,	uint64_t,	uint64_t)	\
     O(tcache_gc_event_wait,	uint64_t,		uint64_t)	\
+    O(tcache_gc_dalloc_event_wait,	uint64_t,	uint64_t)	\
     O(prof_sample_event_wait,	uint64_t,		uint64_t)	\
     O(prof_sample_last_event,	uint64_t,		uint64_t)	\
     O(stats_interval_event_wait,	uint64_t,	uint64_t)	\
@@ -114,10 +118,14 @@ typedef void (*test_callback_t)(int *);
     /* thread_allocated */	0,					\
     /* thread_allocated_next_event_fast */ 0, 				\
     /* thread_deallocated */	0,					\
+    /* thread_deallocated_next_event_fast */	0,			\
     /* rtree_ctx */		RTREE_CTX_ZERO_INITIALIZER,		\
     /* thread_allocated_last_event */	0,				\
     /* thread_allocated_next_event */	THREAD_EVENT_MIN_START_WAIT,	\
+    /* thread_deallocated_last_event */	0,				\
+    /* thread_deallocated_next_event */	THREAD_EVENT_MIN_START_WAIT,	\
     /* tcache_gc_event_wait */		THREAD_EVENT_MIN_START_WAIT,	\
+    /* tcache_gc_dalloc_event_wait */	THREAD_EVENT_MIN_START_WAIT,	\
     /* prof_sample_event_wait */	THREAD_EVENT_MIN_START_WAIT,	\
     /* prof_sample_last_event */	0,				\
     /* stats_interval_event_wait */	THREAD_EVENT_MIN_START_WAIT,	\

src/jemalloc.c

@@ -2154,7 +2154,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
 		dopts->arena_ind = 0;
 	}
 
-	thread_event(tsd, usize);
+	thread_alloc_event(tsd, usize);
 
 	/*
 	 * If dopts->alignment > 0, then ind is still 0, but usize was computed
@@ -2181,7 +2181,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
 		}
 
 		if (unlikely(allocation == NULL)) {
-			thread_event_rollback(tsd, usize);
+			thread_alloc_event_rollback(tsd, usize);
 			prof_alloc_rollback(tsd, tctx, true);
 			goto label_oom;
 		}
@@ -2191,7 +2191,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
 		allocation = imalloc_no_sample(sopts, dopts, tsd, size, usize,
 		    ind);
 		if (unlikely(allocation == NULL)) {
-			thread_event_rollback(tsd, usize);
+			thread_alloc_event_rollback(tsd, usize);
 			goto label_oom;
 		}
 	}
@@ -2575,7 +2575,6 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
 	if (config_prof && opt_prof) {
 		prof_free(tsd, ptr, usize, &alloc_ctx);
 	}
-	*tsd_thread_deallocatedp_get(tsd) += usize;
 
 	if (likely(!slow_path)) {
 		idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
@@ -2584,6 +2583,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) {
 		idalloctm(tsd_tsdn(tsd), ptr, tcache, &alloc_ctx, false,
 		    true);
 	}
+	thread_dalloc_event(tsd, usize);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -2645,14 +2645,12 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
 	if (config_prof && opt_prof) {
 		prof_free(tsd, ptr, usize, ctx);
 	}
-
-	*tsd_thread_deallocatedp_get(tsd) += usize;
-
 	if (likely(!slow_path)) {
 		isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false);
 	} else {
 		isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true);
 	}
+	thread_dalloc_event(tsd, usize);
 }
 
 JEMALLOC_NOINLINE
@@ -2694,12 +2692,12 @@ free_default(void *ptr) {
 JEMALLOC_ALWAYS_INLINE
 bool free_fastpath(void *ptr, size_t size, bool size_hint) {
 	tsd_t *tsd = tsd_get(false);
-	if (unlikely(!tsd || !tsd_fast(tsd))) {
-		return false;
-	}
 
 	szind_t szind;
 	if (!size_hint) {
+		if (unlikely(!tsd || !tsd_fast(tsd))) {
+			return false;
+		}
 		bool slab;
 		rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd);
 		bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd),
@@ -2711,6 +2709,15 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
 		}
 		assert(szind != SC_NSIZES);
 	} else {
+		/*
+		 * The size hinted fastpath does not involve rtree lookup, thus
+		 * can tolerate an uninitialized tsd.  This allows the tsd_fast
+		 * check to be folded into the branch testing fast_threshold
+		 * (set to 0 when !tsd_fast).
+		 */
+		if (unlikely(!tsd)) {
+			return false;
+		}
 		/*
 		 * Check for both sizes that are too large, and for sampled
 		 * objects.  Sampled objects are always page-aligned.  The
@@ -2722,19 +2729,26 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
 		}
 		szind = sz_size2index_lookup(size);
 	}
+	uint64_t deallocated, threshold;
+	thread_event_free_fastpath_ctx(tsd, &deallocated, &threshold, size_hint);
 
-	tcache_t *tcache = tsd_tcachep_get(tsd);
+	size_t usize = sz_index2size(szind);
-	if (unlikely(ticker_trytick(&tcache->gc_ticker))) {
+	uint64_t deallocated_after = deallocated + usize;
+	/*
+	 * Check for events and tsd non-nominal (fast_threshold will be set to
+	 * 0) in a single branch.
+	 */
+	if (unlikely(deallocated_after >= threshold)) {
 		return false;
 	}
 
+	tcache_t *tcache = tsd_tcachep_get(tsd);
 	cache_bin_t *bin = tcache_small_bin_get(tcache, szind);
 	if (!cache_bin_dalloc_easy(bin, ptr)) {
 		return false;
 	}
 
-	size_t usize = sz_index2size(szind);
+	*tsd_thread_deallocatedp_get(tsd) = deallocated_after;
-	*tsd_thread_deallocatedp_get(tsd) += usize;
 
 	return true;
 }
@@ -3144,11 +3158,11 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
 		if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
 			goto label_oom;
 		}
-		thread_event(tsd, usize);
+		thread_alloc_event(tsd, usize);
 		p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
 		    zero, tcache, arena, &alloc_ctx, &hook_args);
 		if (unlikely(p == NULL)) {
-			thread_event_rollback(tsd, usize);
+			thread_alloc_event_rollback(tsd, usize);
 			goto label_oom;
 		}
 	} else {
@@ -3158,11 +3172,10 @@ do_rallocx(void *ptr, size_t size, int flags, bool is_realloc) {
 			goto label_oom;
 		}
 		usize = isalloc(tsd_tsdn(tsd), p);
-		thread_event(tsd, usize);
+		thread_alloc_event(tsd, usize);
 	}
 	assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
-
+	thread_dalloc_event(tsd, old_usize);
-	*tsd_thread_deallocatedp_get(tsd) += old_usize;
 
 	UTRACE(ptr, size, p);
 	check_entry_exit_locking(tsd_tsdn(tsd));
@@ -3337,7 +3350,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
 			usize_max = SC_LARGE_MAXCLASS;
 		}
 	}
-	thread_event(tsd, usize_max);
+	thread_alloc_event(tsd, usize_max);
 	bool prof_active = prof_active_get_unlocked();
 	prof_tctx_t *tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
 
@@ -3350,7 +3363,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
 		    extra, alignment, zero);
 	}
 	if (usize <= usize_max) {
-		thread_event_rollback(tsd, usize_max - usize);
+		thread_alloc_event_rollback(tsd, usize_max - usize);
 	} else {
 		/*
 		 * For downsizing request, usize_max can be less than usize.
@@ -3359,7 +3372,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
 		 * to xallocx(), the entire usize will be rolled back if it's
 		 * equal to the old usize.
 		 */
-		thread_event(tsd, usize - usize_max);
+		thread_alloc_event(tsd, usize - usize_max);
 	}
 
 	/*
@@ -3438,7 +3451,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
 	} else {
 		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
 		    extra, alignment, zero);
-		thread_event(tsd, usize);
+		thread_alloc_event(tsd, usize);
 	}
 
 	/*
@@ -3448,12 +3461,10 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) {
 	assert(iealloc(tsd_tsdn(tsd), ptr) == old_edata);
 
 	if (unlikely(usize == old_usize)) {
-		thread_event_rollback(tsd, usize);
+		thread_alloc_event_rollback(tsd, usize);
 		goto label_not_resized;
 	}
-
+	thread_dalloc_event(tsd, old_usize);
-	*tsd_thread_deallocatedp_get(tsd) += old_usize;
-
 label_not_resized:
 	if (unlikely(!tsd_fast(tsd))) {
 		uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags};

src/tcache.c

@@ -437,8 +437,6 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
 	tcache->next_gc_bin = 0;
 	tcache->arena = NULL;
 
-	ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
-
 	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
 	memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS);
 	memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS));

src/thread_event.c

@@ -12,14 +12,14 @@
 static bool thread_event_active = false;
 
 /* TSD event init function signatures. */
-#define E(event, condition)						\
+#define E(event, condition_unused, is_alloc_event_unused)		\
 static void tsd_thread_##event##_event_init(tsd_t *tsd);
 
 ITERATE_OVER_ALL_EVENTS
 #undef E
 
 /* Event handler function signatures. */
-#define E(event, condition)						\
+#define E(event, condition_unused, is_alloc_event_unused)		\
 static void thread_##event##_event_handler(tsd_t *tsd);
 
 ITERATE_OVER_ALL_EVENTS
@@ -32,6 +32,12 @@ tsd_thread_tcache_gc_event_init(tsd_t *tsd) {
 	thread_tcache_gc_event_update(tsd, TCACHE_GC_INCR_BYTES);
 }
 
+static void
+tsd_thread_tcache_gc_dalloc_event_init(tsd_t *tsd) {
+	assert(TCACHE_GC_INCR_BYTES > 0);
+	thread_tcache_gc_dalloc_event_update(tsd, TCACHE_GC_INCR_BYTES);
+}
+
 static void
 tsd_thread_prof_sample_event_init(tsd_t *tsd) {
 	assert(config_prof && opt_prof);
@@ -46,17 +52,30 @@ tsd_thread_stats_interval_event_init(tsd_t *tsd) {
 }
 
 /* Handler functions. */
+
 static void
-thread_tcache_gc_event_handler(tsd_t *tsd) {
+tcache_gc_event(tsd_t *tsd) {
 	assert(TCACHE_GC_INCR_BYTES > 0);
-	assert(tcache_gc_event_wait_get(tsd) == 0U);
-	tsd_thread_tcache_gc_event_init(tsd);
 	tcache_t *tcache = tcache_get(tsd);
 	if (tcache != NULL) {
 		tcache_event_hard(tsd, tcache);
 	}
 }
 
+static void
+thread_tcache_gc_event_handler(tsd_t *tsd) {
+	assert(tcache_gc_event_wait_get(tsd) == 0U);
+	tsd_thread_tcache_gc_event_init(tsd);
+	tcache_gc_event(tsd);
+}
+
+static void
+thread_tcache_gc_dalloc_event_handler(tsd_t *tsd) {
+	assert(tcache_gc_dalloc_event_wait_get(tsd) == 0U);
+	tsd_thread_tcache_gc_dalloc_event_init(tsd);
+	tcache_gc_event(tsd);
+}
+
 static void
 thread_prof_sample_event_handler(tsd_t *tsd) {
 	assert(config_prof && opt_prof);
@@ -96,12 +115,12 @@ thread_stats_interval_event_handler(tsd_t *tsd) {
 /* Per event facilities done. */
 
 static uint64_t
-thread_allocated_next_event_compute(tsd_t *tsd) {
+thread_next_event_compute(tsd_t *tsd, bool is_alloc) {
 	uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
 	bool no_event_on = true;
 
-#define E(event, condition)						\
+#define E(event, condition, alloc_event)				\
-	if (condition) {						\
+	if (is_alloc == alloc_event && condition) {			\
 		no_event_on = false;					\
 		uint64_t event_wait =					\
 		    event##_event_wait_get(tsd);			\
@@ -119,15 +138,15 @@ thread_allocated_next_event_compute(tsd_t *tsd) {
 	return wait;
 }
 
-void
+static void
-thread_event_assert_invariants_debug(tsd_t *tsd) {
+thread_event_assert_invariants_impl(tsd_t *tsd, event_ctx_t *ctx) {
-	uint64_t thread_allocated = thread_allocated_get(tsd);
+	uint64_t current_bytes = event_ctx_current_bytes_get(ctx);
-	uint64_t last_event = thread_allocated_last_event_get(tsd);
+	uint64_t last_event = event_ctx_last_event_get(ctx);
-	uint64_t next_event = thread_allocated_next_event_get(tsd);
+	uint64_t next_event = event_ctx_next_event_get(ctx);
-	uint64_t next_event_fast = thread_allocated_next_event_fast_get(tsd);
+	uint64_t next_event_fast = event_ctx_next_event_fast_get(ctx);
 
 	assert(last_event != next_event);
-	if (next_event > THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX ||
+	if (next_event > THREAD_NEXT_EVENT_FAST_MAX ||
 	    !tsd_fast(tsd)) {
 		assert(next_event_fast == 0U);
 	} else {
@@ -138,10 +157,9 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
 	uint64_t interval = next_event - last_event;
 
 	/* The subtraction is intentionally susceptible to underflow. */
-	assert(thread_allocated - last_event < interval);
+	assert(current_bytes - last_event < interval);
-
+	uint64_t min_wait = thread_next_event_compute(tsd,
-	uint64_t min_wait = thread_allocated_next_event_compute(tsd);
+	    event_ctx_is_alloc(ctx));
-
 	/*
 	 * next_event should have been pushed up only except when no event is
 	 * on and the TSD is just initialized.  The last_event == 0U guard
@@ -153,6 +171,16 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
 	    (interval < min_wait && interval == THREAD_EVENT_MAX_INTERVAL));
 }
 
+void
+thread_event_assert_invariants_debug(tsd_t *tsd) {
+	event_ctx_t ctx;
+	event_ctx_get(tsd, &ctx, true);
+	thread_event_assert_invariants_impl(tsd, &ctx);
+
+	event_ctx_get(tsd, &ctx, false);
+	thread_event_assert_invariants_impl(tsd, &ctx);
+}
+
 /*
  * Synchronization around the fast threshold in tsd --
  * There are two threads to consider in the synchronization here:
@@ -200,39 +228,50 @@ thread_event_assert_invariants_debug(tsd_t *tsd) {
  * of the owner thread's next_event_fast, but that's always safe (it just sends
  * it down the slow path earlier).
  */
+static void
+event_ctx_next_event_fast_update(event_ctx_t *ctx) {
+	uint64_t next_event = event_ctx_next_event_get(ctx);
+	uint64_t next_event_fast = (next_event <=
+	    THREAD_NEXT_EVENT_FAST_MAX) ? next_event : 0U;
+	event_ctx_next_event_fast_set(ctx, next_event_fast);
+}
+
 void
 thread_event_recompute_fast_threshold(tsd_t *tsd) {
 	if (tsd_state_get(tsd) != tsd_state_nominal) {
 		/* Check first because this is also called on purgatory. */
-		thread_allocated_next_event_fast_set_non_nominal(tsd);
+		thread_next_event_fast_set_non_nominal(tsd);
 		return;
 	}
-	uint64_t next_event = thread_allocated_next_event_get(tsd);
+
-	uint64_t next_event_fast = (next_event <=
+	event_ctx_t ctx;
-	    THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX) ? next_event : 0U;
+	event_ctx_get(tsd, &ctx, true);
-	thread_allocated_next_event_fast_set(tsd, next_event_fast);
+	event_ctx_next_event_fast_update(&ctx);
+	event_ctx_get(tsd, &ctx, false);
+	event_ctx_next_event_fast_update(&ctx);
 
 	atomic_fence(ATOMIC_SEQ_CST);
 	if (tsd_state_get(tsd) != tsd_state_nominal) {
-		thread_allocated_next_event_fast_set_non_nominal(tsd);
+		thread_next_event_fast_set_non_nominal(tsd);
 	}
 }
 
 static void
-thread_event_adjust_thresholds_helper(tsd_t *tsd, uint64_t wait) {
+thread_event_adjust_thresholds_helper(tsd_t *tsd, event_ctx_t *ctx,
+    uint64_t wait) {
 	assert(wait <= THREAD_EVENT_MAX_START_WAIT);
-	uint64_t next_event = thread_allocated_last_event_get(tsd) + (wait <=
+	uint64_t next_event = event_ctx_last_event_get(ctx) + (wait <=
 	    THREAD_EVENT_MAX_INTERVAL ? wait : THREAD_EVENT_MAX_INTERVAL);
-	thread_allocated_next_event_set(tsd, next_event);
+	event_ctx_next_event_set(tsd, ctx, next_event);
 }
 
 static uint64_t
 thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
-    bool allow_event_trigger) {
+    bool is_alloc, bool allow_event_trigger) {
 	uint64_t wait = THREAD_EVENT_MAX_START_WAIT;
 
-#define E(event, condition)						\
+#define E(event, condition, alloc_event)				\
-	if (condition) {						\
+	if (is_alloc == alloc_event && condition) {			\
 		uint64_t event_wait = event##_event_wait_get(tsd);	\
 		assert(event_wait <= THREAD_EVENT_MAX_START_WAIT);	\
 		if (event_wait > accumbytes) {				\
@@ -267,28 +306,30 @@ thread_event_trigger_batch_update(tsd_t *tsd, uint64_t accumbytes,
 }
 
 void
-thread_event_trigger(tsd_t *tsd, bool delay_event) {
+thread_event_trigger(tsd_t *tsd, event_ctx_t *ctx, bool delay_event) {
 	/* usize has already been added to thread_allocated. */
-	uint64_t thread_allocated_after = thread_allocated_get(tsd);
+	uint64_t bytes_after = event_ctx_current_bytes_get(ctx);
 
 	/* The subtraction is intentionally susceptible to underflow. */
-	uint64_t accumbytes = thread_allocated_after -
+	uint64_t accumbytes = bytes_after - event_ctx_last_event_get(ctx);
-	    thread_allocated_last_event_get(tsd);
 
 	/* Make sure that accumbytes cannot overflow uint64_t. */
 	assert(THREAD_EVENT_MAX_INTERVAL <= UINT64_MAX - SC_LARGE_MAXCLASS + 1);
 
-	thread_allocated_last_event_set(tsd, thread_allocated_after);
+	event_ctx_last_event_set(ctx, bytes_after);
 	bool allow_event_trigger = !delay_event && tsd_nominal(tsd) &&
 	    tsd_reentrancy_level_get(tsd) == 0;
+
+	bool is_alloc = ctx->is_alloc;
 	uint64_t wait = thread_event_trigger_batch_update(tsd, accumbytes,
-	    allow_event_trigger);
+	    is_alloc, allow_event_trigger);
-	thread_event_adjust_thresholds_helper(tsd, wait);
+	thread_event_adjust_thresholds_helper(tsd, ctx, wait);
 
 	thread_event_assert_invariants(tsd);
 
-#define E(event, condition)						\
+#define E(event, condition, alloc_event)				\
-	if (condition && event##_event_wait_get(tsd) == 0U) {		\
+	if (is_alloc == alloc_event && condition &&			\
+	    event##_event_wait_get(tsd) == 0U) {			\
 		assert(allow_event_trigger);				\
 		thread_##event##_event_handler(tsd);			\
 	}
@@ -300,19 +341,23 @@ thread_event_trigger(tsd_t *tsd, bool delay_event) {
 }
 
 void
-thread_event_rollback(tsd_t *tsd, size_t diff) {
+thread_alloc_event_rollback(tsd_t *tsd, size_t diff) {
 	thread_event_assert_invariants(tsd);
 
 	if (diff == 0U) {
 		return;
 	}
 
-	uint64_t thread_allocated = thread_allocated_get(tsd);
+	/* Rollback happens only on alloc events. */
+	event_ctx_t ctx;
+	event_ctx_get(tsd, &ctx, true);
+
+	uint64_t thread_allocated = event_ctx_current_bytes_get(&ctx);
 	/* The subtraction is intentionally susceptible to underflow. */
 	uint64_t thread_allocated_rollback = thread_allocated - diff;
-	thread_allocated_set(tsd, thread_allocated_rollback);
+	event_ctx_current_bytes_set(&ctx, thread_allocated_rollback);
 
-	uint64_t last_event = thread_allocated_last_event_get(tsd);
+	uint64_t last_event = event_ctx_last_event_get(&ctx);
 	/* Both subtractions are intentionally susceptible to underflow. */
 	if (thread_allocated_rollback - last_event <=
 	    thread_allocated - last_event) {
@@ -320,14 +365,14 @@ thread_event_rollback(tsd_t *tsd, size_t diff) {
 		return;
 	}
 
-	thread_allocated_last_event_set(tsd, thread_allocated_rollback);
+	event_ctx_last_event_set(&ctx, thread_allocated_rollback);
 
 	/* The subtraction is intentionally susceptible to underflow. */
 	uint64_t wait_diff = last_event - thread_allocated_rollback;
 	assert(wait_diff <= diff);
 
-#define E(event, condition)						\
+#define E(event, condition, alloc_event)				\
-	if (condition) {						\
+	if (alloc_event == true && condition) {				\
 		uint64_t event_wait = event##_event_wait_get(tsd);	\
 		assert(event_wait <= THREAD_EVENT_MAX_START_WAIT);	\
 		if (event_wait > 0U) {					\
@@ -347,27 +392,29 @@ thread_event_rollback(tsd_t *tsd, size_t diff) {
 	ITERATE_OVER_ALL_EVENTS
 #undef E
 
-	thread_event_update(tsd);
+	thread_event_update(tsd, true);
 }
 
 void
-thread_event_update(tsd_t *tsd) {
+thread_event_update(tsd_t *tsd, bool is_alloc) {
-	uint64_t wait = thread_allocated_next_event_compute(tsd);
+	event_ctx_t ctx;
-	thread_event_adjust_thresholds_helper(tsd, wait);
+	event_ctx_get(tsd, &ctx, is_alloc);
 
-	uint64_t last_event = thread_allocated_last_event_get(tsd);
+	uint64_t wait = thread_next_event_compute(tsd, is_alloc);
+	thread_event_adjust_thresholds_helper(tsd, &ctx, wait);
 
+	uint64_t last_event = event_ctx_last_event_get(&ctx);
 	/* Both subtractions are intentionally susceptible to underflow. */
-	if (thread_allocated_get(tsd) - last_event >=
+	if (event_ctx_current_bytes_get(&ctx) - last_event >=
-	    thread_allocated_next_event_get(tsd) - last_event) {
+	    event_ctx_next_event_get(&ctx) - last_event) {
-		thread_event_trigger(tsd, true);
+		thread_event_trigger(tsd, &ctx, true);
 	} else {
 		thread_event_assert_invariants(tsd);
 	}
 }
 
 void thread_event_boot() {
-#define E(event, condition)						\
+#define E(event, condition, ignored)					\
 	if (condition) {						\
 		thread_event_active = true;				\
 	}
@@ -377,7 +424,7 @@ void thread_event_boot() {
 }
 
 void tsd_thread_event_init(tsd_t *tsd) {
-#define E(event, condition)						\
+#define E(event, condition, is_alloc_event_unused)			\
 	if (condition) {						\
 		tsd_thread_##event##_event_init(tsd);			\
 	}

src/tsd.c

@@ -119,7 +119,7 @@ tsd_force_recompute(tsdn_t *tsdn) {
 		    tsd_state_nominal_recompute, ATOMIC_RELAXED);
 		/* See comments in thread_event_recompute_fast_threshold(). */
 		atomic_fence(ATOMIC_SEQ_CST);
-		thread_allocated_next_event_fast_set_non_nominal(remote_tsd);
+		thread_next_event_fast_set_non_nominal(remote_tsd);
 	}
 	malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock);
 }

test/unit/thread_event.c

@@ -2,14 +2,18 @@
 
 TEST_BEGIN(test_next_event_fast_roll_back) {
 	tsd_t *tsd = tsd_fetch();
-	thread_allocated_last_event_set(tsd, 0);
+	event_ctx_t ctx;
-	thread_allocated_set(tsd,
+	event_ctx_get(tsd, &ctx, true);
-	    THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX - 8U);
+
-	thread_allocated_next_event_set(tsd,
+	event_ctx_last_event_set(&ctx, 0);
-	    THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
+	event_ctx_current_bytes_set(&ctx,
-#define E(event, condition)						\
+	    THREAD_NEXT_EVENT_FAST_MAX - 8U);
-	event##_event_wait_set(tsd,					\
+	event_ctx_next_event_set(tsd, &ctx,
-	    THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX);
+	    THREAD_NEXT_EVENT_FAST_MAX);
+#define E(event, condition, is_alloc)					\
+	if (is_alloc && condition) {					\
+		event##_event_wait_set(tsd, THREAD_NEXT_EVENT_FAST_MAX);\
+	}
 	ITERATE_OVER_ALL_EVENTS
 #undef E
 	void *p = malloc(16U);
@@ -20,14 +24,20 @@ TEST_END
 
 TEST_BEGIN(test_next_event_fast_resume) {
 	tsd_t *tsd = tsd_fetch();
-	thread_allocated_last_event_set(tsd, 0);
+
-	thread_allocated_set(tsd,
+	event_ctx_t ctx;
-	    THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 8U);
+	event_ctx_get(tsd, &ctx, true);
-	thread_allocated_next_event_set(tsd,
+
-	    THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U);
+	event_ctx_last_event_set(&ctx, 0);
-#define E(event, condition)						\
+	event_ctx_current_bytes_set(&ctx,
-	event##_event_wait_set(tsd,					\
+	    THREAD_NEXT_EVENT_FAST_MAX + 8U);
-	    THREAD_ALLOCATED_NEXT_EVENT_FAST_MAX + 16U);
+	event_ctx_next_event_set(tsd, &ctx,
+	    THREAD_NEXT_EVENT_FAST_MAX + 16U);
+#define E(event, condition, is_alloc)					\
+	if (is_alloc && condition) {					\
+		event##_event_wait_set(tsd,				\
+		    THREAD_NEXT_EVENT_FAST_MAX + 16U);			\
+	}
 	ITERATE_OVER_ALL_EVENTS
 #undef E
 	void *p = malloc(SC_LOOKUP_MAXCLASS);
@@ -42,7 +52,7 @@ TEST_BEGIN(test_event_rollback) {
 	size_t count = 10;
 	uint64_t thread_allocated = thread_allocated_get(tsd);
 	while (count-- != 0) {
-		thread_event_rollback(tsd, diff);
+		thread_alloc_event_rollback(tsd, diff);
 		uint64_t thread_allocated_after = thread_allocated_get(tsd);
 		assert_u64_eq(thread_allocated - thread_allocated_after, diff,
 		    "thread event counters are not properly rolled back");