Reduce statistical heap sampling memory overhead.

If the mean heap sampling interval is larger than one page, simulate
sampled small objects with large objects.  This allows profiling context
pointers to be omitted for small objects.  As a result, the memory
overhead for sampling decreases as the sampling interval is increased.

Fix a compilation error in the profiling code.

jemalloc/include/jemalloc/internal/arena.h

@@ -121,8 +121,10 @@ struct arena_chunk_map_s {
 	 *
 	 * p : run page offset
 	 * s : run size
+	 * c : size class (used only if prof_promote is true)
 	 * x : don't care
 	 * - : 0
+	 * + : 1
 	 * [DZLA] : bit set
 	 * [dzla] : bit unset
 	 *
@@ -142,17 +144,27 @@ struct arena_chunk_map_s {
 	 *     pppppppp pppppppp pppp---- ----d--a
 	 *
 	 *   Large:
-	 *     ssssssss ssssssss ssss---- ----D-la
+	 *     ssssssss ssssssss ssss++++ ++++D-la
 	 *     xxxxxxxx xxxxxxxx xxxx---- ----xxxx
 	 *     -------- -------- -------- ----D-la
+	 *
+	 *   Large (sampled, size <= PAGE_SIZE):
+	 *     ssssssss ssssssss sssscccc ccccD-la
+	 *
+	 *   Large (not sampled, size == PAGE_SIZE):
+	 *     ssssssss ssssssss ssss++++ ++++D-la
 	 */
 	size_t				bits;
-#define	CHUNK_MAP_FLAGS_MASK	((size_t)0x1fU)
-#define	CHUNK_MAP_KEY		((size_t)0x10U)
-#define	CHUNK_MAP_DIRTY		((size_t)0x08U)
-#define	CHUNK_MAP_ZEROED	((size_t)0x04U)
-#define	CHUNK_MAP_LARGE		((size_t)0x02U)
-#define	CHUNK_MAP_ALLOCATED	((size_t)0x01U)
+#ifdef JEMALLOC_PROF
+#define	CHUNK_MAP_CLASS_SHIFT	4
+#define	CHUNK_MAP_CLASS_MASK	((size_t)0xff0U)
+#endif
+#define	CHUNK_MAP_FLAGS_MASK	((size_t)0xfU)
+#define	CHUNK_MAP_DIRTY		((size_t)0x8U)
+#define	CHUNK_MAP_ZEROED	((size_t)0x4U)
+#define	CHUNK_MAP_LARGE		((size_t)0x2U)
+#define	CHUNK_MAP_ALLOCATED	((size_t)0x1U)
+#define	CHUNK_MAP_KEY		CHUNK_MAP_ALLOCATED
 };
 typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
 typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
@@ -421,6 +433,8 @@ void	*arena_palloc(arena_t *arena, size_t alignment, size_t size,
     size_t alloc_size);
 size_t	arena_salloc(const void *ptr);
 #ifdef JEMALLOC_PROF
+void	arena_prof_promoted(const void *ptr, size_t size);
+size_t	arena_salloc_demote(const void *ptr);
 prof_thr_cnt_t	*arena_prof_cnt_get(const void *ptr);
 void	arena_prof_cnt_set(const void *ptr, prof_thr_cnt_t *cnt);
 #endif

jemalloc/include/jemalloc/internal/jemalloc_internal.h.in

@@ -515,7 +515,11 @@ isalloc(const void *ptr)
 		/* Region. */
 		assert(chunk->arena->magic == ARENA_MAGIC);
 
+#ifdef JEMALLOC_PROF
+		ret = arena_salloc_demote(ptr);
+#else
 		ret = arena_salloc(ptr);
+#endif
 	} else
 		ret = huge_salloc(ptr);
 

jemalloc/include/jemalloc/internal/prof.h

@@ -134,6 +134,12 @@ extern bool	opt_prof_leak; /* Dump leak summary at exit. */
  */
 extern uint64_t	prof_interval;
 
+/*
+ * If true, promote small sampled objects to large objects, since small run
+ * headers do not have embedded profile context pointers.
+ */
+extern bool	prof_promote;
+
 bool	prof_init(prof_t *prof, bool master);
 void	prof_destroy(prof_t *prof);
 

jemalloc/include/jemalloc/internal/tcache.h

@@ -256,6 +256,12 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 		if (ret == NULL)
 			return (NULL);
 	} else {
+#ifdef JEMALLOC_PROF
+		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
+		size_t pageind = (unsigned)(((uintptr_t)ret - (uintptr_t)chunk)
+		    >> PAGE_SHIFT);
+		chunk->map[pageind].bits |= CHUNK_MAP_CLASS_MASK;
+#endif
 		if (zero == false) {
 #ifdef JEMALLOC_FILL
 			if (opt_junk)
@@ -289,6 +295,8 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
 	size_t pageind, binind;
 	arena_chunk_map_t *mapelm;
 
+	assert(arena_salloc(ptr) <= small_maxclass);
+
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	arena = chunk->arena;
 	pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT);
@@ -334,6 +342,8 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 	arena_chunk_map_t *mapelm;
 
 	assert((size & PAGE_MASK) == 0);
+	assert(arena_salloc(ptr) > small_maxclass);
+	assert(arena_salloc(ptr) <= tcache_maxclass);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	arena = chunk->arena;