Restructure profiling

Develop new data structure and code logic for holding profiling
related information stored in the extent that may be needed after the
extent is released, which in particular is the case for the
reallocation code path (e.g. in `rallocx()` and `xallocx()`).  The
data structure is a generalization of `prof_tctx_t`: we previously
only copy out the `prof_tctx` before the extent is released, but we
may be in need of additional fields. Currently the only additional
field is the allocation time field, but there may be more fields in
the future.

The restructuring also resolved a bug: `prof_realloc()` mistakenly
passed the new `ptr` to `prof_free_sampled_object()`, but passing in
the `old_ptr` would crash because it's already been released.  Now
the essential profiling information is collectively copied out early
and safely passed to `prof_free_sampled_object()` after the extent is
released.

include/jemalloc/internal/arena_inlines_b.h

@@ -40,23 +40,31 @@ arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
 	return arena_choose(tsd, NULL);
 }
 
-JEMALLOC_ALWAYS_INLINE prof_tctx_t *
-arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_info_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx,
+    prof_info_t *prof_info) {
 	cassert(config_prof);
 	assert(ptr != NULL);
+	assert(prof_info != NULL);
+
+	const extent_t *extent;
+	bool is_slab;
 
 	/* Static check. */
 	if (alloc_ctx == NULL) {
-		const extent_t *extent = iealloc(tsdn, ptr);
-		if (unlikely(!extent_slab_get(extent))) {
-			return large_prof_tctx_get(tsdn, extent);
-		}
-	} else {
-		if (unlikely(!alloc_ctx->slab)) {
-			return large_prof_tctx_get(tsdn, iealloc(tsdn, ptr));
-		}
+		extent = iealloc(tsdn, ptr);
+		is_slab = extent_slab_get(extent);
+	} else if (!unlikely(is_slab = alloc_ctx->slab)) {
+		extent = iealloc(tsdn, ptr);
+	}
+
+	if (unlikely(!is_slab)) {
+		/* extent must have been initialized at this point. */
+		large_prof_info_get(tsdn, extent, prof_info);
+	} else {
+		memset(prof_info, 0, sizeof(prof_info_t));
+		prof_info->prof_tctx = (prof_tctx_t *)(uintptr_t)1U;
 	}
-	return (prof_tctx_t *)(uintptr_t)1U;
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -89,20 +97,6 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
 	large_prof_tctx_reset(tsdn, extent);
 }
 
-JEMALLOC_ALWAYS_INLINE nstime_t
-arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-
-	extent_t *extent = iealloc(tsdn, ptr);
-	/*
-	 * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're
-	 * sure we have a sampled allocation.
-	 */
-	assert(!extent_slab_get(extent));
-	return large_prof_alloc_time_get(extent);
-}
-
 JEMALLOC_ALWAYS_INLINE void
 arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, nstime_t t) {
 	cassert(config_prof);

include/jemalloc/internal/extent.h

@@ -333,15 +333,12 @@ extent_slab_data_get_const(const extent_t *extent) {
 	return &extent->e_slab_data;
 }
 
-static inline prof_tctx_t *
-extent_prof_tctx_get(const extent_t *extent) {
-	return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx,
-	    ATOMIC_ACQUIRE);
-}
-
-static inline nstime_t
-extent_prof_alloc_time_get(const extent_t *extent) {
-	return extent->e_alloc_time;
+static inline void
+extent_prof_info_get(const extent_t *extent, prof_info_t *prof_info) {
+	assert(prof_info != NULL);
+	prof_info->prof_tctx = (prof_tctx_t *)atomic_load_p(
+	    &extent->e_prof_tctx, ATOMIC_ACQUIRE);
+	prof_info->alloc_time = extent->e_alloc_time;
 }
 
 static inline void

include/jemalloc/internal/large_externs.h

@@ -22,11 +22,10 @@ void large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent);
 void large_dalloc_finish(tsdn_t *tsdn, extent_t *extent);
 void large_dalloc(tsdn_t *tsdn, extent_t *extent);
 size_t large_salloc(tsdn_t *tsdn, const extent_t *extent);
-prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
+void large_prof_info_get(tsdn_t *tsdn, const extent_t *extent,
+    prof_info_t *prof_info);
 void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
 void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
-
-nstime_t large_prof_alloc_time_get(const extent_t *extent);
 void large_prof_alloc_time_set(extent_t *extent, nstime_t time);
 
 #endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */

include/jemalloc/internal/prof_externs.h

@@ -51,8 +51,7 @@ void prof_idump_rollback_impl(tsdn_t *tsdn, size_t usize);
 void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
 void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
     prof_tctx_t *tctx);
-void prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize,
-    prof_tctx_t *tctx);
+void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
 void bt_init(prof_bt_t *bt, void **vec);
 void prof_backtrace(tsd_t *tsd, prof_bt_t *bt);
 prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
@@ -102,7 +101,7 @@ void prof_postfork_parent(tsdn_t *tsdn);
 void prof_postfork_child(tsdn_t *tsdn);
 void prof_sample_threshold_update(tsd_t *tsd);
 
-void prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx);
+void prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
 bool prof_log_start(tsdn_t *tsdn, const char *filename);
 bool prof_log_stop(tsdn_t *tsdn);
 bool prof_log_init(tsd_t *tsdn);

include/jemalloc/internal/prof_inlines_b.h

@@ -39,12 +39,14 @@ prof_tdata_get(tsd_t *tsd, bool create) {
 	return tdata;
 }
 
-JEMALLOC_ALWAYS_INLINE prof_tctx_t *
-prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) {
+JEMALLOC_ALWAYS_INLINE void
+prof_info_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx,
+    prof_info_t *prof_info) {
 	cassert(config_prof);
 	assert(ptr != NULL);
+	assert(prof_info != NULL);
 
-	return arena_prof_tctx_get(tsdn, ptr, alloc_ctx);
+	arena_prof_info_get(tsdn, ptr, alloc_ctx, prof_info);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -64,14 +66,6 @@ prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
 	arena_prof_tctx_reset(tsdn, ptr, tctx);
 }
 
-JEMALLOC_ALWAYS_INLINE nstime_t
-prof_alloc_time_get(tsdn_t *tsdn, const void *ptr) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-
-	return arena_prof_alloc_time_get(tsdn, ptr);
-}
-
 JEMALLOC_ALWAYS_INLINE void
 prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, nstime_t t) {
 	cassert(config_prof);
@@ -152,7 +146,7 @@ prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx,
 JEMALLOC_ALWAYS_INLINE void
 prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
     bool prof_active, bool updated, const void *old_ptr, size_t old_usize,
-    prof_tctx_t *old_tctx) {
+    prof_info_t *old_prof_info) {
 	bool sampled, old_sampled, moved;
 
 	cassert(config_prof);
@@ -174,7 +168,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
 	}
 
 	sampled = ((uintptr_t)tctx > (uintptr_t)1U);
-	old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
+	old_sampled = ((uintptr_t)old_prof_info->prof_tctx > (uintptr_t)1U);
 	moved = (ptr != old_ptr);
 
 	if (unlikely(sampled)) {
@@ -191,8 +185,9 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
 		 */
 		prof_tctx_reset(tsd_tsdn(tsd), ptr, tctx);
 	} else {
-		assert((uintptr_t)prof_tctx_get(tsd_tsdn(tsd), ptr, NULL) ==
-		    (uintptr_t)1U);
+		prof_info_t prof_info;
+		prof_info_get(tsd_tsdn(tsd), ptr, NULL, &prof_info);
+		assert((uintptr_t)prof_info.prof_tctx == (uintptr_t)1U);
 	}
 
 	/*
@@ -203,19 +198,20 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
 	 * counters.
 	 */
 	if (unlikely(old_sampled)) {
-		prof_free_sampled_object(tsd, ptr, old_usize, old_tctx);
+		prof_free_sampled_object(tsd, old_usize, old_prof_info);
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE void
 prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) {
-	prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr, alloc_ctx);
+	prof_info_t prof_info;
+	prof_info_get(tsd_tsdn(tsd), ptr, alloc_ctx, &prof_info);
 
 	cassert(config_prof);
 	assert(usize == isalloc(tsd_tsdn(tsd), ptr));
 
-	if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) {
-		prof_free_sampled_object(tsd, ptr, usize, tctx);
+	if (unlikely((uintptr_t)prof_info.prof_tctx > (uintptr_t)1U)) {
+		prof_free_sampled_object(tsd, usize, &prof_info);
 	}
 }
 

include/jemalloc/internal/prof_structs.h

@@ -96,6 +96,13 @@ struct prof_tctx_s {
 };
 typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
 
+struct prof_info_s {
+	/* Points to the prof_tctx_t corresponding to the allocation. */
+	prof_tctx_t		*prof_tctx;
+	/* Time when the allocation was made. */
+	nstime_t		alloc_time;
+};
+
 struct prof_gctx_s {
 	/* Protects nlimbo, cnt_summed, and tctxs. */
 	malloc_mutex_t		*lock;

include/jemalloc/internal/prof_types.h

@@ -5,6 +5,7 @@ typedef struct prof_bt_s prof_bt_t;
 typedef struct prof_accum_s prof_accum_t;
 typedef struct prof_cnt_s prof_cnt_t;
 typedef struct prof_tctx_s prof_tctx_t;
+typedef struct prof_info_s prof_info_t;
 typedef struct prof_gctx_s prof_gctx_t;
 typedef struct prof_tdata_s prof_tdata_t;