Merge branch 'arena_purge' into dev

jemalloc/configure.ac

@@ -29,7 +29,7 @@ dnl JE_COMPILABLE(label, hcode, mcode, rvar)
 AC_DEFUN([JE_COMPILABLE],
 [
 AC_MSG_CHECKING([whether $1 is compilable])
-AC_RUN_IFELSE([AC_LANG_PROGRAM(
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
 [$2], [$3])],
               AC_MSG_RESULT([yes])
               [$4="yes"],
@@ -454,6 +454,17 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then
   if test "x${enable_prof_libunwind}" = "x1" ; then
     backtrace_method="libunwind"
     AC_DEFINE([JEMALLOC_PROF_LIBUNWIND], [ ])
+    JE_COMPILABLE([libunwind frame cache], [
+#define UNW_LOCAL_ONLY
+#include <libunwind.h>
+], [
+unw_tdep_make_frame_cache(0, 0);
+unw_tdep_free_frame_cache(0);
+unw_tdep_trace(0, 0, 0, 0);
+], [libunwind_cache])
+    if test "x${libunwind_cache}" = "xyes" ; then
+      AC_DEFINE([JEMALLOC_PROF_LIBUNWIND_CACHE], [ ])
+    fi
   fi
 fi
 

jemalloc/include/jemalloc/jemalloc_defs.h.in

@@ -62,6 +62,9 @@
 /* Use libunwind for profile backtracing if defined. */
 #undef JEMALLOC_PROF_LIBUNWIND
 
+/* Use libunwind's backtrace caching for profile backtracing if defined. */
+#undef JEMALLOC_PROF_LIBUNWIND_CACHE
+
 /* Use libgcc for profile backtracing if defined. */
 #undef JEMALLOC_PROF_LIBGCC
 

jemalloc/src/arena.c

@@ -868,9 +868,10 @@ arena_purge(arena_t *arena, bool all)
 	}
 	assert(ndirty == arena->ndirty);
 #endif
-	assert(arena->ndirty > arena->npurgatory);
+	assert(arena->ndirty > arena->npurgatory || all);
 	assert(arena->ndirty > chunk_npages || all);
-	assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all);
+	assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty -
+	    npurgatory) || all);
 
 #ifdef JEMALLOC_STATS
 	arena->stats.npurge++;
@@ -882,8 +883,10 @@ arena_purge(arena_t *arena, bool all)
 	 * multiple threads from racing to reduce ndirty below the threshold.
 	 */
 	npurgatory = arena->ndirty - arena->npurgatory;
-	if (all == false)
+	if (all == false) {
+		assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult);
 		npurgatory -= arena->nactive >> opt_lg_dirty_mult;
+	}
 	arena->npurgatory += npurgatory;
 
 	while (npurgatory > 0) {

jemalloc/src/prof.c

@@ -44,6 +44,12 @@ pthread_key_t	prof_tdata_tsd;
 static ckh_t		bt2ctx;
 static malloc_mutex_t	bt2ctx_mtx;
 
+#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE
+static __thread unw_tdep_cache_t	*libunwind_cache_tls
+    JEMALLOC_ATTR(tls_model("initial-exec"));
+static pthread_key_t	libunwind_cache_tsd;
+#endif
+
 static malloc_mutex_t	prof_dump_seq_mtx;
 static uint64_t		prof_dump_seq;
 static uint64_t		prof_dump_iseq;
@@ -95,6 +101,9 @@ static void	prof_bt_hash(const void *key, unsigned minbits, size_t *hash1,
     size_t *hash2);
 static bool	prof_bt_keycomp(const void *k1, const void *k2);
 static void	prof_tdata_cleanup(void *arg);
+#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE
+static void	libunwind_cache_thread_cleanup(void *arg);
+#endif
 
 /******************************************************************************/
 
@@ -177,6 +186,11 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max)
 	unw_cursor_t cursor;
 	unsigned i;
 	int err;
+#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE
+	unw_tdep_cache_t *cache;
+	int len = nignore + 1 + max;
+	void* vec[len];
+#endif
 
 	assert(bt->len == 0);
 	assert(bt->vec != NULL);
@@ -185,24 +199,53 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max)
 	unw_getcontext(&uc);
 	unw_init_local(&cursor, &uc);
 
-	/* Throw away (nignore+1) stack frames, if that many exist. */
+#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE
-	for (i = 0; i < nignore + 1; i++) {
+	cache = libunwind_cache_tls;
-		err = unw_step(&cursor);
+	if (cache == NULL) {
-		if (err <= 0)
+		cache = unw_tdep_make_frame_cache(imalloc, idalloc);
-			return;
+		if (cache != NULL) {
+			libunwind_cache_tls = cache;
+			pthread_setspecific(libunwind_cache_tsd, cache);
+		}
 	}
+	if (cache != NULL && unw_tdep_trace(&cursor, vec, &len, cache) >= 0) {
+		/*
+		 * The trace cache successfully looked up the backtrace.
+		 * Discard the first (nignore+1) elements when copying the
+		 * result, since there was no way to tell unw_tdep_trace() to
+		 * skip those frames.
+		 */
+		assert(len >= nignore + 1);
+		len -= nignore + 1;
+		if (len > 0) {
+			memcpy(bt->vec, &vec[nignore + 1], sizeof(void *) *
+			    len);
+			bt->len = len;
+		}
+	} else {
+#endif
+		/* Throw away (nignore+1) stack frames, if that many exist. */
+		for (i = 0; i < nignore + 1; i++) {
+			err = unw_step(&cursor);
+			if (err <= 0)
+				return;
+		}
 
-	/*
+		/*
-	 * Iterate over stack frames until there are no more, or until no space
+		 * Iterate over stack frames until there are no more, or until
-	 * remains in bt.
+		 * no space remains in bt.
-	 */
+		 */
-	for (i = 0; i < max; i++) {
+		for (i = 0; i < max; i++) {
-		unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]);
+			unw_get_reg(&cursor, UNW_REG_IP,
-		bt->len++;
+			    (unw_word_t *)&bt->vec[i]);
-		err = unw_step(&cursor);
+			bt->len++;
-		if (err <= 0)
+			err = unw_step(&cursor);
-			break;
+			if (err <= 0)
+				break;
+		}
+#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE
 	}
+#endif
 }
 #endif
 #ifdef JEMALLOC_PROF_LIBGCC
@@ -1156,6 +1199,19 @@ prof_tdata_cleanup(void *arg)
 	}
 }
 
+#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE
+static void
+libunwind_cache_thread_cleanup(void *arg)
+{
+	unw_tdep_cache_t *cache = libunwind_cache_tls;
+
+	if (cache != NULL) {
+		unw_tdep_free_frame_cache(cache);
+		libunwind_cache_tls = NULL;
+	}
+}
+#endif
+
 void
 prof_boot0(void)
 {
@@ -1208,6 +1264,14 @@ prof_boot2(void)
 			    "<jemalloc>: Error in pthread_key_create()\n");
 			abort();
 		}
+#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE
+		if (pthread_key_create(&libunwind_cache_tsd,
+		    libunwind_cache_thread_cleanup) != 0) {
+			malloc_write(
+			    "<jemalloc>: Error in pthread_key_create()\n");
+			abort();
+		}
+#endif
 
 		prof_bt_max = (1U << opt_lg_prof_bt_max);
 		if (malloc_mutex_init(&prof_dump_seq_mtx))