Last-N profiling mode

include/jemalloc/internal/arena_inlines_b.h

@@ -37,12 +37,12 @@ arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
 
 JEMALLOC_ALWAYS_INLINE void
 arena_prof_info_get(tsd_t *tsd, const void *ptr, alloc_ctx_t *alloc_ctx,
-    prof_info_t *prof_info) {
+    prof_info_t *prof_info, bool reset_recent) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 	assert(prof_info != NULL);
 
-	const edata_t *edata;
+	edata_t *edata = NULL;
 	bool is_slab;
 
 	/* Static check. */
@@ -55,10 +55,14 @@ arena_prof_info_get(tsd_t *tsd, const void *ptr, alloc_ctx_t *alloc_ctx,
 
 	if (unlikely(!is_slab)) {
 		/* edata must have been initialized at this point. */
-		large_prof_info_get(edata, prof_info);
+		assert(edata != NULL);
+		large_prof_info_get(tsd, edata, prof_info, reset_recent);
 	} else {
-		memset(prof_info, 0, sizeof(prof_info_t));
 		prof_info->alloc_tctx = (prof_tctx_t *)(uintptr_t)1U;
+		/*
+		 * No need to set other fields in prof_info; they will never be
+		 * accessed if (uintptr_t)alloc_tctx == (uintptr_t)1U.
+		 */
 	}
 }
 
@@ -92,11 +96,9 @@ arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-arena_prof_info_set(tsd_t *tsd, const void *ptr, prof_tctx_t *tctx) {
+arena_prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx) {
 	cassert(config_prof);
-	assert(ptr != NULL);
 
-	edata_t *edata = iealloc(tsd_tsdn(tsd), ptr);
 	assert(!edata_slab_get(edata));
 	large_prof_info_set(edata, tctx);
 }

include/jemalloc/internal/edata.h

@@ -25,6 +25,20 @@ enum extent_head_state_e {
 };
 typedef enum extent_head_state_e extent_head_state_t;
 
+struct e_prof_info_s {
+	/* Time when this was allocated. */
+	nstime_t	e_prof_alloc_time;
+	/* Points to a prof_tctx_t. */
+	atomic_p_t	e_prof_tctx;
+	/*
+	 * Points to a prof_recent_t for the allocation; NULL
+	 * means the recent allocation record no longer exists.
+	 * Protected by prof_recent_alloc_mtx.
+	 */
+	atomic_p_t	e_prof_recent_alloc;
+};
+typedef struct e_prof_info_s e_prof_info_t;
+
 /* Extent (span of pages).  Use accessor functions for e_* fields. */
 typedef struct edata_s edata_t;
 typedef ql_head(edata_t) edata_list_t;
@@ -186,12 +200,7 @@ struct edata_s {
 		slab_data_t	e_slab_data;
 
 		/* Profiling data, used for large objects. */
-		struct {
-			/* Time when this was allocated. */
-			nstime_t		e_alloc_time;
-			/* Points to a prof_tctx_t. */
-			atomic_p_t		e_prof_tctx;
-		};
+		e_prof_info_t	e_prof_info;
 	};
 };
 
@@ -333,12 +342,21 @@ edata_slab_data_get_const(const edata_t *edata) {
 	return &edata->e_slab_data;
 }
 
-static inline void
-edata_prof_info_get(const edata_t *edata, prof_info_t *prof_info) {
-	assert(prof_info != NULL);
-	prof_info->alloc_tctx = (prof_tctx_t *)atomic_load_p(
-	    &edata->e_prof_tctx, ATOMIC_ACQUIRE);
-	prof_info->alloc_time = edata->e_alloc_time;
+static inline prof_tctx_t *
+edata_prof_tctx_get(const edata_t *edata) {
+	return (prof_tctx_t *)atomic_load_p(&edata->e_prof_info.e_prof_tctx,
+	    ATOMIC_ACQUIRE);
+}
+
+static inline const nstime_t *
+edata_prof_alloc_time_get(const edata_t *edata) {
+	return &edata->e_prof_info.e_prof_alloc_time;
+}
+
+static inline prof_recent_t *
+edata_prof_recent_alloc_get_dont_call_directly(const edata_t *edata) {
+	return (prof_recent_t *)atomic_load_p(
+	    &edata->e_prof_info.e_prof_recent_alloc, ATOMIC_RELAXED);
 }
 
 static inline void
@@ -457,12 +475,19 @@ edata_slab_set(edata_t *edata, bool slab) {
 
 static inline void
 edata_prof_tctx_set(edata_t *edata, prof_tctx_t *tctx) {
-	atomic_store_p(&edata->e_prof_tctx, tctx, ATOMIC_RELEASE);
+	atomic_store_p(&edata->e_prof_info.e_prof_tctx, tctx, ATOMIC_RELEASE);
 }
 
 static inline void
 edata_prof_alloc_time_set(edata_t *edata, nstime_t *t) {
-	nstime_copy(&edata->e_alloc_time, t);
+	nstime_copy(&edata->e_prof_info.e_prof_alloc_time, t);
+}
+
+static inline void
+edata_prof_recent_alloc_set_dont_call_directly(edata_t *edata,
+    prof_recent_t *recent_alloc) {
+	atomic_store_p(&edata->e_prof_info.e_prof_recent_alloc, recent_alloc,
+	    ATOMIC_RELAXED);
 }
 
 static inline bool

include/jemalloc/internal/large_externs.h

@@ -22,7 +22,8 @@ void large_dalloc_prep_junked_locked(tsdn_t *tsdn, edata_t *edata);
 void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);
 void large_dalloc(tsdn_t *tsdn, edata_t *edata);
 size_t large_salloc(tsdn_t *tsdn, const edata_t *edata);
-void large_prof_info_get(const edata_t *edata, prof_info_t *prof_info);
+void large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info,
+    bool reset_recent);
 void large_prof_tctx_reset(edata_t *edata);
 void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx);
 

include/jemalloc/internal/nstime.h

@@ -9,6 +9,8 @@ typedef struct {
 	uint64_t ns;
 } nstime_t;
 
+static const nstime_t zero = NSTIME_ZERO_INITIALIZER;
+
 void nstime_init(nstime_t *time, uint64_t ns);
 void nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
 uint64_t nstime_ns(const nstime_t *time);
@@ -35,8 +37,14 @@ bool nstime_init_update(nstime_t *time);
 
 JEMALLOC_ALWAYS_INLINE void
 nstime_init_zero(nstime_t *time) {
-	static const nstime_t zero = NSTIME_ZERO_INITIALIZER;
 	nstime_copy(time, &zero);
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+nstime_equals_zero(nstime_t *time) {
+	int diff = nstime_compare(time, &zero);
+	assert(diff >= 0);
+	return diff == 0;
+}
+
 #endif /* JEMALLOC_INTERNAL_NSTIME_H */

include/jemalloc/internal/prof_externs.h

@@ -24,6 +24,10 @@ extern char opt_prof_prefix[
 #endif
     1];
 
+/* For recording recent allocations */
+extern ssize_t opt_prof_recent_alloc_max;
+extern malloc_mutex_t prof_recent_alloc_mtx;
+
 /* Accessed via prof_active_[gs]et{_unlocked,}(). */
 extern bool prof_active;
 
@@ -99,4 +103,9 @@ void prof_sample_threshold_update(tsd_t *tsd);
 bool prof_log_start(tsdn_t *tsdn, const char *filename);
 bool prof_log_stop(tsdn_t *tsdn);
 
+ssize_t prof_recent_alloc_max_ctl_read();
+ssize_t prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max);
+void prof_recent_alloc_dump(tsd_t *tsd, void (*write_cb)(void *, const char *),
+    void *cbopaque);
+
 #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */

include/jemalloc/internal/prof_inlines_b.h

@@ -46,7 +46,17 @@ prof_info_get(tsd_t *tsd, const void *ptr, alloc_ctx_t *alloc_ctx,
 	assert(ptr != NULL);
 	assert(prof_info != NULL);
 
-	arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info);
+	arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, false);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+prof_info_get_and_reset_recent(tsd_t *tsd, const void *ptr,
+    alloc_ctx_t *alloc_ctx, prof_info_t *prof_info) {
+	cassert(config_prof);
+	assert(ptr != NULL);
+	assert(prof_info != NULL);
+
+	arena_prof_info_get(tsd, ptr, alloc_ctx, prof_info, true);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -66,12 +76,12 @@ prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_info_set(tsd_t *tsd, const void *ptr, prof_tctx_t *tctx) {
+prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx) {
 	cassert(config_prof);
-	assert(ptr != NULL);
+	assert(edata != NULL);
 	assert((uintptr_t)tctx > (uintptr_t)1U);
 
-	arena_prof_info_set(tsd, ptr, tctx);
+	arena_prof_info_set(tsd, edata, tctx);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -190,7 +200,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
 JEMALLOC_ALWAYS_INLINE void
 prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) {
 	prof_info_t prof_info;
-	prof_info_get(tsd, ptr, alloc_ctx, &prof_info);
+	prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);
 
 	cassert(config_prof);
 	assert(usize == isalloc(tsd_tsdn(tsd), ptr));

include/jemalloc/internal/prof_recent.h

@@ -0,0 +1,16 @@
+#ifndef JEMALLOC_INTERNAL_PROF_RECENT_EXTERNS_H
+#define JEMALLOC_INTERNAL_PROF_RECENT_EXTERNS_H
+
+bool prof_recent_alloc_prepare(tsd_t *tsd, prof_tctx_t *tctx);
+void prof_recent_alloc(tsd_t *tsd, edata_t *edata, size_t usize);
+void prof_recent_alloc_reset(tsd_t *tsd, edata_t *edata);
+bool prof_recent_init();
+void edata_prof_recent_alloc_init(edata_t *edata);
+#ifdef JEMALLOC_JET
+prof_recent_t *prof_recent_alloc_begin(tsd_t *tsd);
+prof_recent_t *prof_recent_alloc_end(tsd_t *tsd);
+prof_recent_t *prof_recent_alloc_next(tsd_t *tsd, prof_recent_t *node);
+prof_recent_t *edata_prof_recent_alloc_get(tsd_t *tsd, const edata_t *edata);
+#endif
+
+#endif /* JEMALLOC_INTERNAL_PROF_RECENT_EXTERNS_H */

include/jemalloc/internal/prof_structs.h

@@ -2,6 +2,7 @@
 #define JEMALLOC_INTERNAL_PROF_STRUCTS_H
 
 #include "jemalloc/internal/ckh.h"
+#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/prng.h"
 #include "jemalloc/internal/rb.h"
@@ -55,6 +56,12 @@ struct prof_tctx_s {
 	uint64_t		thr_uid;
 	uint64_t		thr_discrim;
 
+	/*
+	 * Reference count of how many times this tctx object is referenced in
+	 * recent allocation / deallocation records, protected by tdata->lock.
+	 */
+	uint64_t		recent_count;
+
 	/* Profiling counters, protected by tdata->lock. */
 	prof_cnt_t		cnts;
 
@@ -97,10 +104,10 @@ struct prof_tctx_s {
 typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
 
 struct prof_info_s {
-	/* Points to the prof_tctx_t corresponding to the allocation. */
-	prof_tctx_t		*alloc_tctx;
 	/* Time when the allocation was made. */
 	nstime_t		alloc_time;
+	/* Points to the prof_tctx_t corresponding to the allocation. */
+	prof_tctx_t		*alloc_tctx;
 };
 
 struct prof_gctx_s {
@@ -201,4 +208,15 @@ struct prof_tdata_s {
 };
 typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
 
+struct prof_recent_s {
+	nstime_t alloc_time;
+	nstime_t dalloc_time;
+
+	prof_recent_t *next;
+	size_t usize;
+	prof_tctx_t *alloc_tctx;
+	edata_t *alloc_edata; /* NULL means allocation has been freed. */
+	prof_tctx_t *dalloc_tctx;
+};
+
 #endif /* JEMALLOC_INTERNAL_PROF_STRUCTS_H */

include/jemalloc/internal/prof_types.h

@@ -8,6 +8,7 @@ typedef struct prof_tctx_s prof_tctx_t;
 typedef struct prof_info_s prof_info_t;
 typedef struct prof_gctx_s prof_gctx_t;
 typedef struct prof_tdata_s prof_tdata_t;
+typedef struct prof_recent_s prof_recent_t;
 
 /* Option defaults. */
 #ifdef JEMALLOC_PROF
@@ -53,4 +54,7 @@ typedef struct prof_tdata_s prof_tdata_t;
 #define PROF_DUMP_FILENAME_LEN 1
 #endif
 
+/* Default number of recent allocations to record. */
+#define PROF_RECENT_ALLOC_MAX_DEFAULT 0
+
 #endif /* JEMALLOC_INTERNAL_PROF_TYPES_H */

include/jemalloc/internal/witness.h

@@ -61,6 +61,7 @@
 #define WITNESS_RANK_PROF_GDUMP		WITNESS_RANK_LEAF
 #define WITNESS_RANK_PROF_NEXT_THR_UID	WITNESS_RANK_LEAF
 #define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT	WITNESS_RANK_LEAF
+#define WITNESS_RANK_PROF_RECENT_ALLOC	WITNESS_RANK_LEAF
 
 /******************************************************************************/
 /* PER-WITNESS DATA */