diff --git a/Makefile.in b/Makefile.in
index 40daf115..1cd973d7 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -117,7 +117,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/pages.c \
$(srcroot)src/prng.c \
$(srcroot)src/prof.c \
- $(srcroot)src/prof_data.c \
$(srcroot)src/prof_log.c \
$(srcroot)src/rtree.c \
$(srcroot)src/safety_check.c \
diff --git a/include/jemalloc/internal/prof_externs.h b/include/jemalloc/internal/prof_externs.h
index 8fc45cf7..e94ac3b2 100644
--- a/include/jemalloc/internal/prof_externs.h
+++ b/include/jemalloc/internal/prof_externs.h
@@ -4,11 +4,6 @@
#include "jemalloc/internal/mutex.h"
extern malloc_mutex_t bt2gctx_mtx;
-extern malloc_mutex_t tdatas_mtx;
-extern malloc_mutex_t prof_dump_mtx;
-
-malloc_mutex_t *prof_gctx_mutex_choose(void);
-malloc_mutex_t *prof_tdata_mutex_choose(uint64_t thr_uid);
extern bool opt_prof;
extern bool opt_prof_active;
@@ -115,13 +110,4 @@ bool prof_log_rep_check(void);
void prof_log_dummy_set(bool new_value);
#endif
-/* Functions in prof_data.c only accessed in prof.c */
-bool prof_data_init(tsd_t *tsd);
-bool prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,
- bool leakcheck);
-prof_tdata_t * prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid,
- uint64_t thr_discrim, char *thread_name, bool active);
-void prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
-void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
-
#endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index 387f14be..d93d9099 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -58,7 +58,6 @@
-
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
index 030d8266..7b09d4e6 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
@@ -67,9 +67,6 @@
Source Files
-
- Source Files
-
Source Files
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 1606a3ab..28bd3cd6 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -58,7 +58,6 @@
-
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
index 622b93f1..a66c209b 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
@@ -67,9 +67,6 @@
Source Files
-
- Source Files
-
Source Files
diff --git a/src/prof.c b/src/prof.c
index 79a0ffc8..9d1edb32 100644
--- a/src/prof.c
+++ b/src/prof.c
@@ -3,14 +3,11 @@
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/ckh.h"
+#include "jemalloc/internal/hash.h"
+#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
-/*
- * This file implements the profiling "APIs" needed by other parts of jemalloc,
- * and also manages the relevant "operational" data, mainly options and mutexes;
- * the core profiling data structures are encapsulated in prof_data.c.
- */
-
/******************************************************************************/
#ifdef JEMALLOC_PROF_LIBUNWIND
@@ -91,10 +88,20 @@ static atomic_u_t cum_gctxs; /* Atomic counter. */
*/
static malloc_mutex_t *tdata_locks;
+/*
+ * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
+ * structure that knows about all backtraces currently captured.
+ */
+static ckh_t bt2gctx;
/* Non static to enable profiling. */
malloc_mutex_t bt2gctx_mtx;
-malloc_mutex_t tdatas_mtx;
+/*
+ * Tree of all extant prof_tdata_t structures, regardless of state,
+ * {attached,detached,expired}.
+ */
+static prof_tdata_tree_t tdatas;
+static malloc_mutex_t tdatas_mtx;
static uint64_t next_thr_uid;
static malloc_mutex_t next_thr_uid_mtx;
@@ -105,29 +112,101 @@ static uint64_t prof_dump_iseq;
static uint64_t prof_dump_mseq;
static uint64_t prof_dump_useq;
-malloc_mutex_t prof_dump_mtx;
+/*
+ * This buffer is rather large for stack allocation, so use a single buffer for
+ * all profile dumps.
+ */
+static malloc_mutex_t prof_dump_mtx;
+static char prof_dump_buf[
+ /* Minimize memory bloat for non-prof builds. */
+#ifdef JEMALLOC_PROF
+ PROF_DUMP_BUFSIZE
+#else
+ 1
+#endif
+];
+static size_t prof_dump_buf_end;
+static int prof_dump_fd;
/* Do not dump any profiles until bootstrapping is complete. */
bool prof_booted = false;
/******************************************************************************/
+/*
+ * Function prototypes for static functions that are referenced prior to
+ * definition.
+ */
-static bool
-prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) {
- malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
+static bool prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx);
+static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
+static bool prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
+ bool even_if_attached);
+static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
+ bool even_if_attached);
+static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name);
- if (opt_prof_accum) {
- return false;
+/******************************************************************************/
+/* Red-black trees. */
+
+static int
+prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {
+ uint64_t a_thr_uid = a->thr_uid;
+ uint64_t b_thr_uid = b->thr_uid;
+ int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
+ if (ret == 0) {
+ uint64_t a_thr_discrim = a->thr_discrim;
+ uint64_t b_thr_discrim = b->thr_discrim;
+ ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
+ b_thr_discrim);
+ if (ret == 0) {
+ uint64_t a_tctx_uid = a->tctx_uid;
+ uint64_t b_tctx_uid = b->tctx_uid;
+ ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
+ b_tctx_uid);
+ }
}
- if (tctx->cnts.curobjs != 0) {
- return false;
- }
- if (tctx->prepared) {
- return false;
- }
- return true;
+ return ret;
}
+rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
+ tctx_link, prof_tctx_comp)
+
+static int
+prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {
+ unsigned a_len = a->bt.len;
+ unsigned b_len = b->bt.len;
+ unsigned comp_len = (a_len < b_len) ? a_len : b_len;
+ int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
+ if (ret == 0) {
+ ret = (a_len > b_len) - (a_len < b_len);
+ }
+ return ret;
+}
+
+rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
+ prof_gctx_comp)
+
+static int
+prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {
+ int ret;
+ uint64_t a_uid = a->thr_uid;
+ uint64_t b_uid = b->thr_uid;
+
+ ret = ((a_uid > b_uid) - (a_uid < b_uid));
+ if (ret == 0) {
+ uint64_t a_discrim = a->thr_discrim;
+ uint64_t b_discrim = b->thr_discrim;
+
+ ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
+ }
+ return ret;
+}
+
+rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
+ prof_tdata_comp)
+
+/******************************************************************************/
+
void
prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) {
prof_tdata_t *tdata;
@@ -207,6 +286,45 @@ bt_init(prof_bt_t *bt, void **vec) {
bt->len = 0;
}
+static void
+prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {
+ cassert(config_prof);
+ assert(tdata == prof_tdata_get(tsd, false));
+
+ if (tdata != NULL) {
+ assert(!tdata->enq);
+ tdata->enq = true;
+ }
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
+}
+
+static void
+prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {
+ cassert(config_prof);
+ assert(tdata == prof_tdata_get(tsd, false));
+
+ malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
+
+ if (tdata != NULL) {
+ bool idump, gdump;
+
+ assert(tdata->enq);
+ tdata->enq = false;
+ idump = tdata->enq_idump;
+ tdata->enq_idump = false;
+ gdump = tdata->enq_gdump;
+ tdata->enq_gdump = false;
+
+ if (idump) {
+ prof_idump(tsd_tsdn(tsd));
+ }
+ if (gdump) {
+ prof_gdump(tsd_tsdn(tsd));
+ }
+ }
+}
+
#ifdef JEMALLOC_PROF_LIBUNWIND
void
prof_backtrace(prof_bt_t *bt) {
@@ -429,18 +547,324 @@ prof_backtrace(prof_bt_t *bt) {
}
#endif
-malloc_mutex_t *
+static malloc_mutex_t *
prof_gctx_mutex_choose(void) {
unsigned ngctxs = atomic_fetch_add_u(&cum_gctxs, 1, ATOMIC_RELAXED);
return &gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS];
}
-malloc_mutex_t *
+static malloc_mutex_t *
prof_tdata_mutex_choose(uint64_t thr_uid) {
return &tdata_locks[thr_uid % PROF_NTDATA_LOCKS];
}
+static prof_gctx_t *
+prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {
+ /*
+ * Create a single allocation that has space for vec of length bt->len.
+ */
+ size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
+ prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
+ sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
+ true);
+ if (gctx == NULL) {
+ return NULL;
+ }
+ gctx->lock = prof_gctx_mutex_choose();
+ /*
+ * Set nlimbo to 1, in order to avoid a race condition with
+ * prof_tctx_destroy()/prof_gctx_try_destroy().
+ */
+ gctx->nlimbo = 1;
+ tctx_tree_new(&gctx->tctxs);
+ /* Duplicate bt. */
+ memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
+ gctx->bt.vec = gctx->vec;
+ gctx->bt.len = bt->len;
+ return gctx;
+}
+
+static void
+prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
+ prof_tdata_t *tdata) {
+ cassert(config_prof);
+
+ /*
+ * Check that gctx is still unused by any thread cache before destroying
+ * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
+ * condition with this function, as does prof_tctx_destroy() in order to
+ * avoid a race between the main body of prof_tctx_destroy() and entry
+ * into this function.
+ */
+ prof_enter(tsd, tdata_self);
+ malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
+ assert(gctx->nlimbo != 0);
+ if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
+ /* Remove gctx from bt2gctx. */
+ if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {
+ not_reached();
+ }
+ prof_leave(tsd, tdata_self);
+ /* Destroy gctx. */
+ malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
+ idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);
+ } else {
+ /*
+ * Compensate for increment in prof_tctx_destroy() or
+ * prof_lookup().
+ */
+ gctx->nlimbo--;
+ malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
+ prof_leave(tsd, tdata_self);
+ }
+}
+
+static bool
+prof_tctx_should_destroy(tsdn_t *tsdn, prof_tctx_t *tctx) {
+ malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
+
+ if (opt_prof_accum) {
+ return false;
+ }
+ if (tctx->cnts.curobjs != 0) {
+ return false;
+ }
+ if (tctx->prepared) {
+ return false;
+ }
+ return true;
+}
+
+static bool
+prof_gctx_should_destroy(prof_gctx_t *gctx) {
+ if (opt_prof_accum) {
+ return false;
+ }
+ if (!tctx_tree_empty(&gctx->tctxs)) {
+ return false;
+ }
+ if (gctx->nlimbo != 0) {
+ return false;
+ }
+ return true;
+}
+
+static void
+prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
+ prof_tdata_t *tdata = tctx->tdata;
+ prof_gctx_t *gctx = tctx->gctx;
+ bool destroy_tdata, destroy_tctx, destroy_gctx;
+
+ malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
+
+ assert(tctx->cnts.curobjs == 0);
+ assert(tctx->cnts.curbytes == 0);
+ assert(!opt_prof_accum);
+ assert(tctx->cnts.accumobjs == 0);
+ assert(tctx->cnts.accumbytes == 0);
+
+ ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
+ destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);
+ malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
+
+ malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
+ switch (tctx->state) {
+ case prof_tctx_state_nominal:
+ tctx_tree_remove(&gctx->tctxs, tctx);
+ destroy_tctx = true;
+ if (prof_gctx_should_destroy(gctx)) {
+ /*
+ * Increment gctx->nlimbo in order to keep another
+ * thread from winning the race to destroy gctx while
+ * this one has gctx->lock dropped. Without this, it
+ * would be possible for another thread to:
+ *
+ * 1) Sample an allocation associated with gctx.
+ * 2) Deallocate the sampled object.
+ * 3) Successfully prof_gctx_try_destroy(gctx).
+ *
+ * The result would be that gctx no longer exists by the
+ * time this thread accesses it in
+ * prof_gctx_try_destroy().
+ */
+ gctx->nlimbo++;
+ destroy_gctx = true;
+ } else {
+ destroy_gctx = false;
+ }
+ break;
+ case prof_tctx_state_dumping:
+ /*
+ * A dumping thread needs tctx to remain valid until dumping
+ * has finished. Change state such that the dumping thread will
+ * complete destruction during a late dump iteration phase.
+ */
+ tctx->state = prof_tctx_state_purgatory;
+ destroy_tctx = false;
+ destroy_gctx = false;
+ break;
+ default:
+ not_reached();
+ destroy_tctx = false;
+ destroy_gctx = false;
+ }
+ malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
+ if (destroy_gctx) {
+ prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
+ tdata);
+ }
+
+ malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);
+
+ if (destroy_tdata) {
+ prof_tdata_destroy(tsd, tdata, false);
+ }
+
+ if (destroy_tctx) {
+ idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);
+ }
+}
+
+static bool
+prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
+ void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {
+ union {
+ prof_gctx_t *p;
+ void *v;
+ } gctx, tgctx;
+ union {
+ prof_bt_t *p;
+ void *v;
+ } btkey;
+ bool new_gctx;
+
+ prof_enter(tsd, tdata);
+ if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
+ /* bt has never been seen before. Insert it. */
+ prof_leave(tsd, tdata);
+ tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
+ if (tgctx.v == NULL) {
+ return true;
+ }
+ prof_enter(tsd, tdata);
+ if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
+ gctx.p = tgctx.p;
+ btkey.p = &gctx.p->bt;
+ if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
+ /* OOM. */
+ prof_leave(tsd, tdata);
+ idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,
+ true, true);
+ return true;
+ }
+ new_gctx = true;
+ } else {
+ new_gctx = false;
+ }
+ } else {
+ tgctx.v = NULL;
+ new_gctx = false;
+ }
+
+ if (!new_gctx) {
+ /*
+ * Increment nlimbo, in order to avoid a race condition with
+ * prof_tctx_destroy()/prof_gctx_try_destroy().
+ */
+ malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
+ gctx.p->nlimbo++;
+ malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
+ new_gctx = false;
+
+ if (tgctx.v != NULL) {
+ /* Lost race to insert. */
+ idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,
+ true);
+ }
+ }
+ prof_leave(tsd, tdata);
+
+ *p_btkey = btkey.v;
+ *p_gctx = gctx.p;
+ *p_new_gctx = new_gctx;
+ return false;
+}
+
+prof_tctx_t *
+prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
+ union {
+ prof_tctx_t *p;
+ void *v;
+ } ret;
+ prof_tdata_t *tdata;
+ bool not_found;
+
+ cassert(config_prof);
+
+ tdata = prof_tdata_get(tsd, false);
+ if (tdata == NULL) {
+ return NULL;
+ }
+
+ malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
+ not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
+ if (!not_found) { /* Note double negative! */
+ ret.p->prepared = true;
+ }
+ malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
+ if (not_found) {
+ void *btkey;
+ prof_gctx_t *gctx;
+ bool new_gctx, error;
+
+ /*
+ * This thread's cache lacks bt. Look for it in the global
+ * cache.
+ */
+ if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
+ &new_gctx)) {
+ return NULL;
+ }
+
+ /* Link a prof_tctx_t into gctx for this thread. */
+ ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
+ sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,
+ arena_ichoose(tsd, NULL), true);
+ if (ret.p == NULL) {
+ if (new_gctx) {
+ prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
+ }
+ return NULL;
+ }
+ ret.p->tdata = tdata;
+ ret.p->thr_uid = tdata->thr_uid;
+ ret.p->thr_discrim = tdata->thr_discrim;
+ memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
+ ret.p->gctx = gctx;
+ ret.p->tctx_uid = tdata->tctx_uid_next++;
+ ret.p->prepared = true;
+ ret.p->state = prof_tctx_state_initializing;
+ malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
+ error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
+ malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
+ if (error) {
+ if (new_gctx) {
+ prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
+ }
+ idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);
+ return NULL;
+ }
+ malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
+ ret.p->state = prof_tctx_state_nominal;
+ tctx_tree_insert(&gctx->tctxs, ret.p);
+ gctx->nlimbo--;
+ malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
+ }
+
+ return ret.p;
+}
+
/*
* The bodies of this function and prof_leakcheck() are compiled out unless heap
* profiling is enabled, so that it is possible to compile jemalloc with
@@ -497,6 +921,520 @@ prof_sample_threshold_update(prof_tdata_t *tdata) {
#endif
}
+#ifdef JEMALLOC_JET
+static prof_tdata_t *
+prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
+ void *arg) {
+ size_t *tdata_count = (size_t *)arg;
+
+ (*tdata_count)++;
+
+ return NULL;
+}
+
+size_t
+prof_tdata_count(void) {
+ size_t tdata_count = 0;
+ tsdn_t *tsdn;
+
+ tsdn = tsdn_fetch();
+ malloc_mutex_lock(tsdn, &tdatas_mtx);
+ tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
+ (void *)&tdata_count);
+ malloc_mutex_unlock(tsdn, &tdatas_mtx);
+
+ return tdata_count;
+}
+
+size_t
+prof_bt_count(void) {
+ size_t bt_count;
+ tsd_t *tsd;
+ prof_tdata_t *tdata;
+
+ tsd = tsd_fetch();
+ tdata = prof_tdata_get(tsd, false);
+ if (tdata == NULL) {
+ return 0;
+ }
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
+ bt_count = ckh_count(&bt2gctx);
+ malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
+
+ return bt_count;
+}
+#endif
+
+static int
+prof_dump_open_impl(bool propagate_err, const char *filename) {
+ int fd;
+
+ fd = creat(filename, 0644);
+ if (fd == -1 && !propagate_err) {
+ malloc_printf(": creat(\"%s\"), 0644) failed\n",
+ filename);
+ if (opt_abort) {
+ abort();
+ }
+ }
+
+ return fd;
+}
+prof_dump_open_t *JET_MUTABLE prof_dump_open = prof_dump_open_impl;
+
+static bool
+prof_dump_flush(bool propagate_err) {
+ bool ret = false;
+ ssize_t err;
+
+ cassert(config_prof);
+
+ err = malloc_write_fd(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
+ if (err == -1) {
+ if (!propagate_err) {
+ malloc_write(": write() failed during heap "
+ "profile flush\n");
+ if (opt_abort) {
+ abort();
+ }
+ }
+ ret = true;
+ }
+ prof_dump_buf_end = 0;
+
+ return ret;
+}
+
+static bool
+prof_dump_close(bool propagate_err) {
+ bool ret;
+
+ assert(prof_dump_fd != -1);
+ ret = prof_dump_flush(propagate_err);
+ close(prof_dump_fd);
+ prof_dump_fd = -1;
+
+ return ret;
+}
+
+static bool
+prof_dump_write(bool propagate_err, const char *s) {
+ size_t i, slen, n;
+
+ cassert(config_prof);
+
+ i = 0;
+ slen = strlen(s);
+ while (i < slen) {
+ /* Flush the buffer if it is full. */
+ if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
+ if (prof_dump_flush(propagate_err) && propagate_err) {
+ return true;
+ }
+ }
+
+ if (prof_dump_buf_end + slen - i <= PROF_DUMP_BUFSIZE) {
+ /* Finish writing. */
+ n = slen - i;
+ } else {
+ /* Write as much of s as will fit. */
+ n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
+ }
+ memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
+ prof_dump_buf_end += n;
+ i += n;
+ }
+
+ return false;
+}
+
+JEMALLOC_FORMAT_PRINTF(2, 3)
+static bool
+prof_dump_printf(bool propagate_err, const char *format, ...) {
+ bool ret;
+ va_list ap;
+ char buf[PROF_PRINTF_BUFSIZE];
+
+ va_start(ap, format);
+ malloc_vsnprintf(buf, sizeof(buf), format, ap);
+ va_end(ap);
+ ret = prof_dump_write(propagate_err, buf);
+
+ return ret;
+}
+
+static void
+prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {
+ malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
+
+ malloc_mutex_lock(tsdn, tctx->gctx->lock);
+
+ switch (tctx->state) {
+ case prof_tctx_state_initializing:
+ malloc_mutex_unlock(tsdn, tctx->gctx->lock);
+ return;
+ case prof_tctx_state_nominal:
+ tctx->state = prof_tctx_state_dumping;
+ malloc_mutex_unlock(tsdn, tctx->gctx->lock);
+
+ memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
+
+ tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
+ tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
+ if (opt_prof_accum) {
+ tdata->cnt_summed.accumobjs +=
+ tctx->dump_cnts.accumobjs;
+ tdata->cnt_summed.accumbytes +=
+ tctx->dump_cnts.accumbytes;
+ }
+ break;
+ case prof_tctx_state_dumping:
+ case prof_tctx_state_purgatory:
+ not_reached();
+ }
+}
+
+static void
+prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {
+ malloc_mutex_assert_owner(tsdn, gctx->lock);
+
+ gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
+ gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
+ if (opt_prof_accum) {
+ gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
+ gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
+ }
+}
+
+static prof_tctx_t *
+prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
+ tsdn_t *tsdn = (tsdn_t *)arg;
+
+ malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
+
+ switch (tctx->state) {
+ case prof_tctx_state_nominal:
+ /* New since dumping started; ignore. */
+ break;
+ case prof_tctx_state_dumping:
+ case prof_tctx_state_purgatory:
+ prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
+ break;
+ default:
+ not_reached();
+ }
+
+ return NULL;
+}
+
+struct prof_tctx_dump_iter_arg_s {
+ tsdn_t *tsdn;
+ bool propagate_err;
+};
+
+static prof_tctx_t *
+prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {
+ struct prof_tctx_dump_iter_arg_s *arg =
+ (struct prof_tctx_dump_iter_arg_s *)opaque;
+
+ malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);
+
+ switch (tctx->state) {
+ case prof_tctx_state_initializing:
+ case prof_tctx_state_nominal:
+ /* Not captured by this dump. */
+ break;
+ case prof_tctx_state_dumping:
+ case prof_tctx_state_purgatory:
+ if (prof_dump_printf(arg->propagate_err,
+ " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
+ "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
+ tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
+ tctx->dump_cnts.accumbytes)) {
+ return tctx;
+ }
+ break;
+ default:
+ not_reached();
+ }
+ return NULL;
+}
+
+static prof_tctx_t *
+prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
+ tsdn_t *tsdn = (tsdn_t *)arg;
+ prof_tctx_t *ret;
+
+ malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
+
+ switch (tctx->state) {
+ case prof_tctx_state_nominal:
+ /* New since dumping started; ignore. */
+ break;
+ case prof_tctx_state_dumping:
+ tctx->state = prof_tctx_state_nominal;
+ break;
+ case prof_tctx_state_purgatory:
+ ret = tctx;
+ goto label_return;
+ default:
+ not_reached();
+ }
+
+ ret = NULL;
+label_return:
+ return ret;
+}
+
+static void
+prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {
+ cassert(config_prof);
+
+ malloc_mutex_lock(tsdn, gctx->lock);
+
+ /*
+ * Increment nlimbo so that gctx won't go away before dump.
+ * Additionally, link gctx into the dump list so that it is included in
+ * prof_dump()'s second pass.
+ */
+ gctx->nlimbo++;
+ gctx_tree_insert(gctxs, gctx);
+
+ memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
+
+ malloc_mutex_unlock(tsdn, gctx->lock);
+}
+
+struct prof_gctx_merge_iter_arg_s {
+ tsdn_t *tsdn;
+ size_t leak_ngctx;
+};
+
+static prof_gctx_t *
+prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
+ struct prof_gctx_merge_iter_arg_s *arg =
+ (struct prof_gctx_merge_iter_arg_s *)opaque;
+
+ malloc_mutex_lock(arg->tsdn, gctx->lock);
+ tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
+ (void *)arg->tsdn);
+ if (gctx->cnt_summed.curobjs != 0) {
+ arg->leak_ngctx++;
+ }
+ malloc_mutex_unlock(arg->tsdn, gctx->lock);
+
+ return NULL;
+}
+
+static void
+prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {
+ prof_tdata_t *tdata = prof_tdata_get(tsd, false);
+ prof_gctx_t *gctx;
+
+ /*
+ * Standard tree iteration won't work here, because as soon as we
+ * decrement gctx->nlimbo and unlock gctx, another thread can
+ * concurrently destroy it, which will corrupt the tree. Therefore,
+ * tear down the tree one node at a time during iteration.
+ */
+ while ((gctx = gctx_tree_first(gctxs)) != NULL) {
+ gctx_tree_remove(gctxs, gctx);
+ malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
+ {
+ prof_tctx_t *next;
+
+ next = NULL;
+ do {
+ prof_tctx_t *to_destroy =
+ tctx_tree_iter(&gctx->tctxs, next,
+ prof_tctx_finish_iter,
+ (void *)tsd_tsdn(tsd));
+ if (to_destroy != NULL) {
+ next = tctx_tree_next(&gctx->tctxs,
+ to_destroy);
+ tctx_tree_remove(&gctx->tctxs,
+ to_destroy);
+ idalloctm(tsd_tsdn(tsd), to_destroy,
+ NULL, NULL, true, true);
+ } else {
+ next = NULL;
+ }
+ } while (next != NULL);
+ }
+ gctx->nlimbo--;
+ if (prof_gctx_should_destroy(gctx)) {
+ gctx->nlimbo++;
+ malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
+ prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
+ } else {
+ malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
+ }
+ }
+}
+
+struct prof_tdata_merge_iter_arg_s {
+ tsdn_t *tsdn;
+ prof_cnt_t cnt_all;
+};
+
+static prof_tdata_t *
+prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
+ void *opaque) {
+ struct prof_tdata_merge_iter_arg_s *arg =
+ (struct prof_tdata_merge_iter_arg_s *)opaque;
+
+ malloc_mutex_lock(arg->tsdn, tdata->lock);
+ if (!tdata->expired) {
+ size_t tabind;
+ union {
+ prof_tctx_t *p;
+ void *v;
+ } tctx;
+
+ tdata->dumping = true;
+ memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
+ for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
+ &tctx.v);) {
+ prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
+ }
+
+ arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;
+ arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;
+ if (opt_prof_accum) {
+ arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;
+ arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;
+ }
+ } else {
+ tdata->dumping = false;
+ }
+ malloc_mutex_unlock(arg->tsdn, tdata->lock);
+
+ return NULL;
+}
+
+static prof_tdata_t *
+prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
+ void *arg) {
+ bool propagate_err = *(bool *)arg;
+
+ if (!tdata->dumping) {
+ return NULL;
+ }
+
+ if (prof_dump_printf(propagate_err,
+ " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
+ tdata->thr_uid, tdata->cnt_summed.curobjs,
+ tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
+ tdata->cnt_summed.accumbytes,
+ (tdata->thread_name != NULL) ? " " : "",
+ (tdata->thread_name != NULL) ? tdata->thread_name : "")) {
+ return tdata;
+ }
+ return NULL;
+}
+
+static bool
+prof_dump_header_impl(tsdn_t *tsdn, bool propagate_err,
+ const prof_cnt_t *cnt_all) {
+ bool ret;
+
+ if (prof_dump_printf(propagate_err,
+ "heap_v2/%"FMTu64"\n"
+ " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
+ ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
+ cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) {
+ return true;
+ }
+
+ malloc_mutex_lock(tsdn, &tdatas_mtx);
+ ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
+ (void *)&propagate_err) != NULL);
+ malloc_mutex_unlock(tsdn, &tdatas_mtx);
+ return ret;
+}
+prof_dump_header_t *JET_MUTABLE prof_dump_header = prof_dump_header_impl;
+
+static bool
+prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx,
+ const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {
+ bool ret;
+ unsigned i;
+ struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg;
+
+ cassert(config_prof);
+ malloc_mutex_assert_owner(tsdn, gctx->lock);
+
+ /* Avoid dumping such gctx's that have no useful data. */
+ if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
+ (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
+ assert(gctx->cnt_summed.curobjs == 0);
+ assert(gctx->cnt_summed.curbytes == 0);
+ assert(gctx->cnt_summed.accumobjs == 0);
+ assert(gctx->cnt_summed.accumbytes == 0);
+ ret = false;
+ goto label_return;
+ }
+
+ if (prof_dump_printf(propagate_err, "@")) {
+ ret = true;
+ goto label_return;
+ }
+ for (i = 0; i < bt->len; i++) {
+ if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
+ (uintptr_t)bt->vec[i])) {
+ ret = true;
+ goto label_return;
+ }
+ }
+
+ if (prof_dump_printf(propagate_err,
+ "\n"
+ " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
+ gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
+ gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
+ ret = true;
+ goto label_return;
+ }
+
+ prof_tctx_dump_iter_arg.tsdn = tsdn;
+ prof_tctx_dump_iter_arg.propagate_err = propagate_err;
+ if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
+ (void *)&prof_tctx_dump_iter_arg) != NULL) {
+ ret = true;
+ goto label_return;
+ }
+
+ ret = false;
+label_return:
+ return ret;
+}
+
+#ifndef _WIN32
+JEMALLOC_FORMAT_PRINTF(1, 2)
+static int
+prof_open_maps(const char *format, ...) {
+ int mfd;
+ va_list ap;
+ char filename[PATH_MAX + 1];
+
+ va_start(ap, format);
+ malloc_vsnprintf(filename, sizeof(filename), format, ap);
+ va_end(ap);
+
+#if defined(O_CLOEXEC)
+ mfd = open(filename, O_RDONLY | O_CLOEXEC);
+#else
+ mfd = open(filename, O_RDONLY);
+ if (mfd != -1) {
+ fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC);
+ }
+#endif
+
+ return mfd;
+}
+#endif
+
int
prof_getpid(void) {
#ifdef _WIN32
@@ -506,6 +1444,291 @@ prof_getpid(void) {
#endif
}
+static bool
+prof_dump_maps(bool propagate_err) {
+ bool ret;
+ int mfd;
+
+ cassert(config_prof);
+#ifdef __FreeBSD__
+ mfd = prof_open_maps("/proc/curproc/map");
+#elif defined(_WIN32)
+ mfd = -1; // Not implemented
+#else
+ {
+ int pid = prof_getpid();
+
+ mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
+ if (mfd == -1) {
+ mfd = prof_open_maps("/proc/%d/maps", pid);
+ }
+ }
+#endif
+ if (mfd != -1) {
+ ssize_t nread;
+
+ if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
+ propagate_err) {
+ ret = true;
+ goto label_return;
+ }
+ nread = 0;
+ do {
+ prof_dump_buf_end += nread;
+ if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
+ /* Make space in prof_dump_buf before read(). */
+ if (prof_dump_flush(propagate_err) &&
+ propagate_err) {
+ ret = true;
+ goto label_return;
+ }
+ }
+ nread = malloc_read_fd(mfd,
+ &prof_dump_buf[prof_dump_buf_end], PROF_DUMP_BUFSIZE
+ - prof_dump_buf_end);
+ } while (nread > 0);
+ } else {
+ ret = true;
+ goto label_return;
+ }
+
+ ret = false;
+label_return:
+ if (mfd != -1) {
+ close(mfd);
+ }
+ return ret;
+}
+
+/*
+ * See prof_sample_threshold_update() comment for why the body of this function
+ * is conditionally compiled.
+ */
+static void
+prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
+ const char *filename) {
+#ifdef JEMALLOC_PROF
+ /*
+ * Scaling is equivalent AdjustSamples() in jeprof, but the result may
+ * differ slightly from what jeprof reports, because here we scale the
+ * summary values, whereas jeprof scales each context individually and
+ * reports the sums of the scaled values.
+ */
+ if (cnt_all->curbytes != 0) {
+ double sample_period = (double)((uint64_t)1 << lg_prof_sample);
+ double ratio = (((double)cnt_all->curbytes) /
+ (double)cnt_all->curobjs) / sample_period;
+ double scale_factor = 1.0 / (1.0 - exp(-ratio));
+ uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
+ * scale_factor);
+ uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
+ scale_factor);
+
+ malloc_printf(": Leak approximation summary: ~%"FMTu64
+ " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
+ curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
+ 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
+ malloc_printf(
+ ": Run jeprof on \"%s\" for leak detail\n",
+ filename);
+ }
+#endif
+}
+
+struct prof_gctx_dump_iter_arg_s {
+ tsdn_t *tsdn;
+ bool propagate_err;
+};
+
+static prof_gctx_t *
+prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
+ prof_gctx_t *ret;
+ struct prof_gctx_dump_iter_arg_s *arg =
+ (struct prof_gctx_dump_iter_arg_s *)opaque;
+
+ malloc_mutex_lock(arg->tsdn, gctx->lock);
+
+ if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt,
+ gctxs)) {
+ ret = gctx;
+ goto label_return;
+ }
+
+ ret = NULL;
+label_return:
+ malloc_mutex_unlock(arg->tsdn, gctx->lock);
+ return ret;
+}
+
+static void
+prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata,
+ struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
+ struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
+ prof_gctx_tree_t *gctxs) {
+ size_t tabind;
+ union {
+ prof_gctx_t *p;
+ void *v;
+ } gctx;
+
+ prof_enter(tsd, tdata);
+
+ /*
+ * Put gctx's in limbo and clear their counters in preparation for
+ * summing.
+ */
+ gctx_tree_new(gctxs);
+ for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) {
+ prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);
+ }
+
+ /*
+ * Iterate over tdatas, and for the non-expired ones snapshot their tctx
+ * stats and merge them into the associated gctx's.
+ */
+ prof_tdata_merge_iter_arg->tsdn = tsd_tsdn(tsd);
+ memset(&prof_tdata_merge_iter_arg->cnt_all, 0, sizeof(prof_cnt_t));
+ malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
+ tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
+ (void *)prof_tdata_merge_iter_arg);
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
+
+ /* Merge tctx stats into gctx's. */
+ prof_gctx_merge_iter_arg->tsdn = tsd_tsdn(tsd);
+ prof_gctx_merge_iter_arg->leak_ngctx = 0;
+ gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter,
+ (void *)prof_gctx_merge_iter_arg);
+
+ prof_leave(tsd, tdata);
+}
+
+static bool
+prof_dump_file(tsd_t *tsd, bool propagate_err, const char *filename,
+ bool leakcheck, prof_tdata_t *tdata,
+ struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
+ struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
+ struct prof_gctx_dump_iter_arg_s *prof_gctx_dump_iter_arg,
+ prof_gctx_tree_t *gctxs) {
+ /* Create dump file. */
+ if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) {
+ return true;
+ }
+
+ /* Dump profile header. */
+ if (prof_dump_header(tsd_tsdn(tsd), propagate_err,
+ &prof_tdata_merge_iter_arg->cnt_all)) {
+ goto label_write_error;
+ }
+
+ /* Dump per gctx profile stats. */
+ prof_gctx_dump_iter_arg->tsdn = tsd_tsdn(tsd);
+ prof_gctx_dump_iter_arg->propagate_err = propagate_err;
+ if (gctx_tree_iter(gctxs, NULL, prof_gctx_dump_iter,
+ (void *)prof_gctx_dump_iter_arg) != NULL) {
+ goto label_write_error;
+ }
+
+ /* Dump /proc//maps if possible. */
+ if (prof_dump_maps(propagate_err)) {
+ goto label_write_error;
+ }
+
+ if (prof_dump_close(propagate_err)) {
+ return true;
+ }
+
+ return false;
+label_write_error:
+ prof_dump_close(propagate_err);
+ return true;
+}
+
+static bool
+prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,
+ bool leakcheck) {
+ cassert(config_prof);
+ assert(tsd_reentrancy_level_get(tsd) == 0);
+
+ prof_tdata_t * tdata = prof_tdata_get(tsd, true);
+ if (tdata == NULL) {
+ return true;
+ }
+
+ pre_reentrancy(tsd, NULL);
+ malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
+
+ prof_gctx_tree_t gctxs;
+ struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
+ struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
+ struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg;
+ prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
+ &prof_gctx_merge_iter_arg, &gctxs);
+ bool err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata,
+ &prof_tdata_merge_iter_arg, &prof_gctx_merge_iter_arg,
+ &prof_gctx_dump_iter_arg, &gctxs);
+ prof_gctx_finish(tsd, &gctxs);
+
+ malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
+ post_reentrancy(tsd);
+
+ if (err) {
+ return true;
+ }
+
+ if (leakcheck) {
+ prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all,
+ prof_gctx_merge_iter_arg.leak_ngctx, filename);
+ }
+ return false;
+}
+
+#ifdef JEMALLOC_JET
+void
+prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes, uint64_t *accumobjs,
+ uint64_t *accumbytes) {
+ tsd_t *tsd;
+ prof_tdata_t *tdata;
+ struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
+ struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
+ prof_gctx_tree_t gctxs;
+
+ tsd = tsd_fetch();
+ tdata = prof_tdata_get(tsd, false);
+ if (tdata == NULL) {
+ if (curobjs != NULL) {
+ *curobjs = 0;
+ }
+ if (curbytes != NULL) {
+ *curbytes = 0;
+ }
+ if (accumobjs != NULL) {
+ *accumobjs = 0;
+ }
+ if (accumbytes != NULL) {
+ *accumbytes = 0;
+ }
+ return;
+ }
+
+ prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
+ &prof_gctx_merge_iter_arg, &gctxs);
+ prof_gctx_finish(tsd, &gctxs);
+
+ if (curobjs != NULL) {
+ *curobjs = prof_tdata_merge_iter_arg.cnt_all.curobjs;
+ }
+ if (curbytes != NULL) {
+ *curbytes = prof_tdata_merge_iter_arg.cnt_all.curbytes;
+ }
+ if (accumobjs != NULL) {
+ *accumobjs = prof_tdata_merge_iter_arg.cnt_all.accumobjs;
+ }
+ if (accumbytes != NULL) {
+ *accumbytes = prof_tdata_merge_iter_arg.cnt_all.accumbytes;
+ }
+}
+#endif
+
#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1)
#define VSEQ_INVALID UINT64_C(0xffffffffffffffff)
static void
@@ -654,6 +1877,28 @@ prof_gdump(tsdn_t *tsdn) {
}
}
+void
+prof_bt_hash(const void *key, size_t r_hash[2]) {
+ prof_bt_t *bt = (prof_bt_t *)key;
+
+ cassert(config_prof);
+
+ hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
+}
+
+bool
+prof_bt_keycomp(const void *k1, const void *k2) {
+ const prof_bt_t *bt1 = (prof_bt_t *)k1;
+ const prof_bt_t *bt2 = (prof_bt_t *)k2;
+
+ cassert(config_prof);
+
+ if (bt1->len != bt2->len) {
+ return false;
+ }
+ return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
+}
+
static uint64_t
prof_thr_uid_alloc(tsdn_t *tsdn) {
uint64_t thr_uid;
@@ -666,33 +1911,124 @@ prof_thr_uid_alloc(tsdn_t *tsdn) {
return thr_uid;
}
+static prof_tdata_t *
+prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
+ char *thread_name, bool active) {
+ prof_tdata_t *tdata;
+
+ cassert(config_prof);
+
+ /* Initialize an empty cache for this thread. */
+ tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
+ sz_size2index(sizeof(prof_tdata_t)), false, NULL, true,
+ arena_get(TSDN_NULL, 0, true), true);
+ if (tdata == NULL) {
+ return NULL;
+ }
+
+ tdata->lock = prof_tdata_mutex_choose(thr_uid);
+ tdata->thr_uid = thr_uid;
+ tdata->thr_discrim = thr_discrim;
+ tdata->thread_name = thread_name;
+ tdata->attached = true;
+ tdata->expired = false;
+ tdata->tctx_uid_next = 0;
+
+ if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
+ prof_bt_keycomp)) {
+ idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
+ return NULL;
+ }
+
+ tdata->prng_state = (uint64_t)(uintptr_t)tdata;
+ prof_sample_threshold_update(tdata);
+
+ tdata->enq = false;
+ tdata->enq_idump = false;
+ tdata->enq_gdump = false;
+
+ tdata->dumping = false;
+ tdata->active = active;
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
+ tdata_tree_insert(&tdatas, tdata);
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
+
+ return tdata;
+}
+
prof_tdata_t *
prof_tdata_init(tsd_t *tsd) {
return prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd_tsdn(tsd)), 0,
NULL, prof_thread_active_init_get(tsd_tsdn(tsd)));
}
-static char *
-prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) {
- char *ret;
- size_t size;
-
- if (thread_name == NULL) {
- return NULL;
+static bool
+prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {
+ if (tdata->attached && !even_if_attached) {
+ return false;
}
-
- size = strlen(thread_name) + 1;
- if (size == 1) {
- return "";
+ if (ckh_count(&tdata->bt2tctx) != 0) {
+ return false;
}
+ return true;
+}
- ret = iallocztm(tsdn, size, sz_size2index(size), false, NULL, true,
- arena_get(TSDN_NULL, 0, true), true);
- if (ret == NULL) {
- return NULL;
+static bool
+prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
+ bool even_if_attached) {
+ malloc_mutex_assert_owner(tsdn, tdata->lock);
+
+ return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);
+}
+
+static void
+prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
+ bool even_if_attached) {
+ malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
+
+ tdata_tree_remove(&tdatas, tdata);
+
+ assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
+
+ if (tdata->thread_name != NULL) {
+ idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
+ true);
+ }
+ ckh_delete(tsd, &tdata->bt2tctx);
+ idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
+}
+
+static void
+prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {
+ malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
+ prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
+}
+
+static void
+prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {
+ bool destroy_tdata;
+
+ malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
+ if (tdata->attached) {
+ destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
+ true);
+ /*
+ * Only detach if !destroy_tdata, because detaching would allow
+ * another thread to win the race to destroy tdata.
+ */
+ if (!destroy_tdata) {
+ tdata->attached = false;
+ }
+ tsd_prof_tdata_set(tsd, NULL);
+ } else {
+ destroy_tdata = false;
+ }
+ malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
+ if (destroy_tdata) {
+ prof_tdata_destroy(tsd, tdata, true);
}
- memcpy(ret, thread_name, size);
- return ret;
}
prof_tdata_t *
@@ -708,6 +2044,58 @@ prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) {
active);
}
+static bool
+prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {
+ bool destroy_tdata;
+
+ malloc_mutex_lock(tsdn, tdata->lock);
+ if (!tdata->expired) {
+ tdata->expired = true;
+ destroy_tdata = tdata->attached ? false :
+ prof_tdata_should_destroy(tsdn, tdata, false);
+ } else {
+ destroy_tdata = false;
+ }
+ malloc_mutex_unlock(tsdn, tdata->lock);
+
+ return destroy_tdata;
+}
+
+static prof_tdata_t *
+prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
+ void *arg) {
+ tsdn_t *tsdn = (tsdn_t *)arg;
+
+ return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
+}
+
+void
+prof_reset(tsd_t *tsd, size_t lg_sample) {
+ prof_tdata_t *next;
+
+ assert(lg_sample < (sizeof(uint64_t) << 3));
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
+ malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
+
+ lg_prof_sample = lg_sample;
+
+ next = NULL;
+ do {
+ prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
+ prof_tdata_reset_iter, (void *)tsd);
+ if (to_destroy != NULL) {
+ next = tdata_tree_next(&tdatas, to_destroy);
+ prof_tdata_destroy_locked(tsd, to_destroy, false);
+ } else {
+ next = NULL;
+ }
+ } while (next != NULL);
+
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
+ malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
+}
+
void
prof_tdata_cleanup(tsd_t *tsd) {
prof_tdata_t *tdata;
@@ -754,6 +2142,29 @@ prof_thread_name_get(tsd_t *tsd) {
return (tdata->thread_name != NULL ? tdata->thread_name : "");
}
+static char *
+prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name) {
+ char *ret;
+ size_t size;
+
+ if (thread_name == NULL) {
+ return NULL;
+ }
+
+ size = strlen(thread_name) + 1;
+ if (size == 1) {
+ return "";
+ }
+
+ ret = iallocztm(tsdn, size, sz_size2index(size), false, NULL, true,
+ arena_get(TSDN_NULL, 0, true), true);
+ if (ret == NULL) {
+ return NULL;
+ }
+ memcpy(ret, thread_name, size);
+ return ret;
+}
+
int
prof_thread_name_set(tsd_t *tsd, const char *thread_name) {
prof_tdata_t *tdata;
@@ -918,15 +2329,16 @@ prof_boot2(tsd_t *tsd) {
return true;
}
- if (prof_data_init(tsd)) {
+ if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
+ prof_bt_keycomp)) {
return true;
}
-
if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx",
WITNESS_RANK_PROF_BT2GCTX, malloc_mutex_rank_exclusive)) {
return true;
}
+ tdata_tree_new(&tdatas);
if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas",
WITNESS_RANK_PROF_TDATAS, malloc_mutex_rank_exclusive)) {
return true;
diff --git a/src/prof_data.c b/src/prof_data.c
deleted file mode 100644
index a4cb749f..00000000
--- a/src/prof_data.c
+++ /dev/null
@@ -1,1440 +0,0 @@
-#define JEMALLOC_PROF_C_
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
-
-#include "jemalloc/internal/assert.h"
-#include "jemalloc/internal/ckh.h"
-#include "jemalloc/internal/hash.h"
-#include "jemalloc/internal/malloc_io.h"
-
-/*
- * This file defines and manages the core profiling data structures.
- *
- * Conceptually, profiling data can be imagined as a table with three columns:
- * thread, stack trace, and current allocation size. (When prof_accum is on,
- * there's one additional column which is the cumulative allocation size.)
- *
- * Implementation wise, each thread maintains a hash recording the stack trace
- * to allocation size correspondences, which are basically the individual rows
- * in the table. In addition, two global "indices" are built to make data
- * aggregation efficient (for dumping): bt2gctx and tdatas, which are basically
- * the "grouped by stack trace" and "grouped by thread" views of the same table,
- * respectively. Note that the allocation size is only aggregated to the two
- * indices at dumping time, so as to optimize for performance.
- */
-
-/******************************************************************************/
-
-/*
- * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data
- * structure that knows about all backtraces currently captured.
- */
-static ckh_t bt2gctx;
-
-/*
- * Tree of all extant prof_tdata_t structures, regardless of state,
- * {attached,detached,expired}.
- */
-static prof_tdata_tree_t tdatas;
-
-/*
- * This buffer is rather large for stack allocation, so use a single buffer for
- * all profile dumps.
- */
-static char prof_dump_buf[
- /* Minimize memory bloat for non-prof builds. */
-#ifdef JEMALLOC_PROF
- PROF_DUMP_BUFSIZE
-#else
- 1
-#endif
-];
-static size_t prof_dump_buf_end;
-static int prof_dump_fd;
-
-/******************************************************************************/
-/* Red-black trees. */
-
-static int
-prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) {
- uint64_t a_thr_uid = a->thr_uid;
- uint64_t b_thr_uid = b->thr_uid;
- int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid);
- if (ret == 0) {
- uint64_t a_thr_discrim = a->thr_discrim;
- uint64_t b_thr_discrim = b->thr_discrim;
- ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim <
- b_thr_discrim);
- if (ret == 0) {
- uint64_t a_tctx_uid = a->tctx_uid;
- uint64_t b_tctx_uid = b->tctx_uid;
- ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid <
- b_tctx_uid);
- }
- }
- return ret;
-}
-
-rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t,
- tctx_link, prof_tctx_comp)
-
-static int
-prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) {
- unsigned a_len = a->bt.len;
- unsigned b_len = b->bt.len;
- unsigned comp_len = (a_len < b_len) ? a_len : b_len;
- int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *));
- if (ret == 0) {
- ret = (a_len > b_len) - (a_len < b_len);
- }
- return ret;
-}
-
-rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link,
- prof_gctx_comp)
-
-static int
-prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) {
- int ret;
- uint64_t a_uid = a->thr_uid;
- uint64_t b_uid = b->thr_uid;
-
- ret = ((a_uid > b_uid) - (a_uid < b_uid));
- if (ret == 0) {
- uint64_t a_discrim = a->thr_discrim;
- uint64_t b_discrim = b->thr_discrim;
-
- ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim));
- }
- return ret;
-}
-
-rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link,
- prof_tdata_comp)
-
-/******************************************************************************/
-
-bool
-prof_data_init(tsd_t *tsd) {
- tdata_tree_new(&tdatas);
- return ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS,
- prof_bt_hash, prof_bt_keycomp);
-}
-
-static void
-prof_enter(tsd_t *tsd, prof_tdata_t *tdata) {
- cassert(config_prof);
- assert(tdata == prof_tdata_get(tsd, false));
-
- if (tdata != NULL) {
- assert(!tdata->enq);
- tdata->enq = true;
- }
-
- malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
-}
-
-static void
-prof_leave(tsd_t *tsd, prof_tdata_t *tdata) {
- cassert(config_prof);
- assert(tdata == prof_tdata_get(tsd, false));
-
- malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
-
- if (tdata != NULL) {
- bool idump, gdump;
-
- assert(tdata->enq);
- tdata->enq = false;
- idump = tdata->enq_idump;
- tdata->enq_idump = false;
- gdump = tdata->enq_gdump;
- tdata->enq_gdump = false;
-
- if (idump) {
- prof_idump(tsd_tsdn(tsd));
- }
- if (gdump) {
- prof_gdump(tsd_tsdn(tsd));
- }
- }
-}
-
-static prof_gctx_t *
-prof_gctx_create(tsdn_t *tsdn, prof_bt_t *bt) {
- /*
- * Create a single allocation that has space for vec of length bt->len.
- */
- size_t size = offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *));
- prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsdn, size,
- sz_size2index(size), false, NULL, true, arena_get(TSDN_NULL, 0, true),
- true);
- if (gctx == NULL) {
- return NULL;
- }
- gctx->lock = prof_gctx_mutex_choose();
- /*
- * Set nlimbo to 1, in order to avoid a race condition with
- * prof_tctx_destroy()/prof_gctx_try_destroy().
- */
- gctx->nlimbo = 1;
- tctx_tree_new(&gctx->tctxs);
- /* Duplicate bt. */
- memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *));
- gctx->bt.vec = gctx->vec;
- gctx->bt.len = bt->len;
- return gctx;
-}
-
-static void
-prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
- prof_tdata_t *tdata) {
- cassert(config_prof);
-
- /*
- * Check that gctx is still unused by any thread cache before destroying
- * it. prof_lookup() increments gctx->nlimbo in order to avoid a race
- * condition with this function, as does prof_tctx_destroy() in order to
- * avoid a race between the main body of prof_tctx_destroy() and entry
- * into this function.
- */
- prof_enter(tsd, tdata_self);
- malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
- assert(gctx->nlimbo != 0);
- if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
- /* Remove gctx from bt2gctx. */
- if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) {
- not_reached();
- }
- prof_leave(tsd, tdata_self);
- /* Destroy gctx. */
- malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
- idalloctm(tsd_tsdn(tsd), gctx, NULL, NULL, true, true);
- } else {
- /*
- * Compensate for increment in prof_tctx_destroy() or
- * prof_lookup().
- */
- gctx->nlimbo--;
- malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
- prof_leave(tsd, tdata_self);
- }
-}
-
-static bool
-prof_gctx_should_destroy(prof_gctx_t *gctx) {
- if (opt_prof_accum) {
- return false;
- }
- if (!tctx_tree_empty(&gctx->tctxs)) {
- return false;
- }
- if (gctx->nlimbo != 0) {
- return false;
- }
- return true;
-}
-
-static bool
-prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
- void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) {
- union {
- prof_gctx_t *p;
- void *v;
- } gctx, tgctx;
- union {
- prof_bt_t *p;
- void *v;
- } btkey;
- bool new_gctx;
-
- prof_enter(tsd, tdata);
- if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
- /* bt has never been seen before. Insert it. */
- prof_leave(tsd, tdata);
- tgctx.p = prof_gctx_create(tsd_tsdn(tsd), bt);
- if (tgctx.v == NULL) {
- return true;
- }
- prof_enter(tsd, tdata);
- if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) {
- gctx.p = tgctx.p;
- btkey.p = &gctx.p->bt;
- if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) {
- /* OOM. */
- prof_leave(tsd, tdata);
- idalloctm(tsd_tsdn(tsd), gctx.v, NULL, NULL,
- true, true);
- return true;
- }
- new_gctx = true;
- } else {
- new_gctx = false;
- }
- } else {
- tgctx.v = NULL;
- new_gctx = false;
- }
-
- if (!new_gctx) {
- /*
- * Increment nlimbo, in order to avoid a race condition with
- * prof_tctx_destroy()/prof_gctx_try_destroy().
- */
- malloc_mutex_lock(tsd_tsdn(tsd), gctx.p->lock);
- gctx.p->nlimbo++;
- malloc_mutex_unlock(tsd_tsdn(tsd), gctx.p->lock);
- new_gctx = false;
-
- if (tgctx.v != NULL) {
- /* Lost race to insert. */
- idalloctm(tsd_tsdn(tsd), tgctx.v, NULL, NULL, true,
- true);
- }
- }
- prof_leave(tsd, tdata);
-
- *p_btkey = btkey.v;
- *p_gctx = gctx.p;
- *p_new_gctx = new_gctx;
- return false;
-}
-
-prof_tctx_t *
-prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
- union {
- prof_tctx_t *p;
- void *v;
- } ret;
- prof_tdata_t *tdata;
- bool not_found;
-
- cassert(config_prof);
-
- tdata = prof_tdata_get(tsd, false);
- if (tdata == NULL) {
- return NULL;
- }
-
- malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
- not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
- if (!not_found) { /* Note double negative! */
- ret.p->prepared = true;
- }
- malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
- if (not_found) {
- void *btkey;
- prof_gctx_t *gctx;
- bool new_gctx, error;
-
- /*
- * This thread's cache lacks bt. Look for it in the global
- * cache.
- */
- if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx,
- &new_gctx)) {
- return NULL;
- }
-
- /* Link a prof_tctx_t into gctx for this thread. */
- ret.v = iallocztm(tsd_tsdn(tsd), sizeof(prof_tctx_t),
- sz_size2index(sizeof(prof_tctx_t)), false, NULL, true,
- arena_ichoose(tsd, NULL), true);
- if (ret.p == NULL) {
- if (new_gctx) {
- prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
- }
- return NULL;
- }
- ret.p->tdata = tdata;
- ret.p->thr_uid = tdata->thr_uid;
- ret.p->thr_discrim = tdata->thr_discrim;
- memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
- ret.p->gctx = gctx;
- ret.p->tctx_uid = tdata->tctx_uid_next++;
- ret.p->prepared = true;
- ret.p->state = prof_tctx_state_initializing;
- malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
- error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
- malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
- if (error) {
- if (new_gctx) {
- prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
- }
- idalloctm(tsd_tsdn(tsd), ret.v, NULL, NULL, true, true);
- return NULL;
- }
- malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
- ret.p->state = prof_tctx_state_nominal;
- tctx_tree_insert(&gctx->tctxs, ret.p);
- gctx->nlimbo--;
- malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
- }
-
- return ret.p;
-}
-
-#ifdef JEMALLOC_JET
-static prof_tdata_t *
-prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
- void *arg) {
- size_t *tdata_count = (size_t *)arg;
-
- (*tdata_count)++;
-
- return NULL;
-}
-
-size_t
-prof_tdata_count(void) {
- size_t tdata_count = 0;
- tsdn_t *tsdn;
-
- tsdn = tsdn_fetch();
- malloc_mutex_lock(tsdn, &tdatas_mtx);
- tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
- (void *)&tdata_count);
- malloc_mutex_unlock(tsdn, &tdatas_mtx);
-
- return tdata_count;
-}
-
-size_t
-prof_bt_count(void) {
- size_t bt_count;
- tsd_t *tsd;
- prof_tdata_t *tdata;
-
- tsd = tsd_fetch();
- tdata = prof_tdata_get(tsd, false);
- if (tdata == NULL) {
- return 0;
- }
-
- malloc_mutex_lock(tsd_tsdn(tsd), &bt2gctx_mtx);
- bt_count = ckh_count(&bt2gctx);
- malloc_mutex_unlock(tsd_tsdn(tsd), &bt2gctx_mtx);
-
- return bt_count;
-}
-#endif
-
-static int
-prof_dump_open_impl(bool propagate_err, const char *filename) {
- int fd;
-
- fd = creat(filename, 0644);
- if (fd == -1 && !propagate_err) {
- malloc_printf(": creat(\"%s\"), 0644) failed\n",
- filename);
- if (opt_abort) {
- abort();
- }
- }
-
- return fd;
-}
-prof_dump_open_t *JET_MUTABLE prof_dump_open = prof_dump_open_impl;
-
-static bool
-prof_dump_flush(bool propagate_err) {
- bool ret = false;
- ssize_t err;
-
- cassert(config_prof);
-
- err = malloc_write_fd(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
- if (err == -1) {
- if (!propagate_err) {
- malloc_write(": write() failed during heap "
- "profile flush\n");
- if (opt_abort) {
- abort();
- }
- }
- ret = true;
- }
- prof_dump_buf_end = 0;
-
- return ret;
-}
-
-static bool
-prof_dump_close(bool propagate_err) {
- bool ret;
-
- assert(prof_dump_fd != -1);
- ret = prof_dump_flush(propagate_err);
- close(prof_dump_fd);
- prof_dump_fd = -1;
-
- return ret;
-}
-
-static bool
-prof_dump_write(bool propagate_err, const char *s) {
- size_t i, slen, n;
-
- cassert(config_prof);
-
- i = 0;
- slen = strlen(s);
- while (i < slen) {
- /* Flush the buffer if it is full. */
- if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
- if (prof_dump_flush(propagate_err) && propagate_err) {
- return true;
- }
- }
-
- if (prof_dump_buf_end + slen - i <= PROF_DUMP_BUFSIZE) {
- /* Finish writing. */
- n = slen - i;
- } else {
- /* Write as much of s as will fit. */
- n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
- }
- memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
- prof_dump_buf_end += n;
- i += n;
- }
-
- return false;
-}
-
-JEMALLOC_FORMAT_PRINTF(2, 3)
-static bool
-prof_dump_printf(bool propagate_err, const char *format, ...) {
- bool ret;
- va_list ap;
- char buf[PROF_PRINTF_BUFSIZE];
-
- va_start(ap, format);
- malloc_vsnprintf(buf, sizeof(buf), format, ap);
- va_end(ap);
- ret = prof_dump_write(propagate_err, buf);
-
- return ret;
-}
-
-static void
-prof_tctx_merge_tdata(tsdn_t *tsdn, prof_tctx_t *tctx, prof_tdata_t *tdata) {
- malloc_mutex_assert_owner(tsdn, tctx->tdata->lock);
-
- malloc_mutex_lock(tsdn, tctx->gctx->lock);
-
- switch (tctx->state) {
- case prof_tctx_state_initializing:
- malloc_mutex_unlock(tsdn, tctx->gctx->lock);
- return;
- case prof_tctx_state_nominal:
- tctx->state = prof_tctx_state_dumping;
- malloc_mutex_unlock(tsdn, tctx->gctx->lock);
-
- memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
-
- tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
- tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
- if (opt_prof_accum) {
- tdata->cnt_summed.accumobjs +=
- tctx->dump_cnts.accumobjs;
- tdata->cnt_summed.accumbytes +=
- tctx->dump_cnts.accumbytes;
- }
- break;
- case prof_tctx_state_dumping:
- case prof_tctx_state_purgatory:
- not_reached();
- }
-}
-
-static void
-prof_tctx_merge_gctx(tsdn_t *tsdn, prof_tctx_t *tctx, prof_gctx_t *gctx) {
- malloc_mutex_assert_owner(tsdn, gctx->lock);
-
- gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
- gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
- if (opt_prof_accum) {
- gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs;
- gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes;
- }
-}
-
-static prof_tctx_t *
-prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
- tsdn_t *tsdn = (tsdn_t *)arg;
-
- malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
-
- switch (tctx->state) {
- case prof_tctx_state_nominal:
- /* New since dumping started; ignore. */
- break;
- case prof_tctx_state_dumping:
- case prof_tctx_state_purgatory:
- prof_tctx_merge_gctx(tsdn, tctx, tctx->gctx);
- break;
- default:
- not_reached();
- }
-
- return NULL;
-}
-
-struct prof_tctx_dump_iter_arg_s {
- tsdn_t *tsdn;
- bool propagate_err;
-};
-
-static prof_tctx_t *
-prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque) {
- struct prof_tctx_dump_iter_arg_s *arg =
- (struct prof_tctx_dump_iter_arg_s *)opaque;
-
- malloc_mutex_assert_owner(arg->tsdn, tctx->gctx->lock);
-
- switch (tctx->state) {
- case prof_tctx_state_initializing:
- case prof_tctx_state_nominal:
- /* Not captured by this dump. */
- break;
- case prof_tctx_state_dumping:
- case prof_tctx_state_purgatory:
- if (prof_dump_printf(arg->propagate_err,
- " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
- "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
- tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
- tctx->dump_cnts.accumbytes)) {
- return tctx;
- }
- break;
- default:
- not_reached();
- }
- return NULL;
-}
-
-static prof_tctx_t *
-prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) {
- tsdn_t *tsdn = (tsdn_t *)arg;
- prof_tctx_t *ret;
-
- malloc_mutex_assert_owner(tsdn, tctx->gctx->lock);
-
- switch (tctx->state) {
- case prof_tctx_state_nominal:
- /* New since dumping started; ignore. */
- break;
- case prof_tctx_state_dumping:
- tctx->state = prof_tctx_state_nominal;
- break;
- case prof_tctx_state_purgatory:
- ret = tctx;
- goto label_return;
- default:
- not_reached();
- }
-
- ret = NULL;
-label_return:
- return ret;
-}
-
-static void
-prof_dump_gctx_prep(tsdn_t *tsdn, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) {
- cassert(config_prof);
-
- malloc_mutex_lock(tsdn, gctx->lock);
-
- /*
- * Increment nlimbo so that gctx won't go away before dump.
- * Additionally, link gctx into the dump list so that it is included in
- * prof_dump()'s second pass.
- */
- gctx->nlimbo++;
- gctx_tree_insert(gctxs, gctx);
-
- memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
-
- malloc_mutex_unlock(tsdn, gctx->lock);
-}
-
-struct prof_gctx_merge_iter_arg_s {
- tsdn_t *tsdn;
- size_t leak_ngctx;
-};
-
-static prof_gctx_t *
-prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
- struct prof_gctx_merge_iter_arg_s *arg =
- (struct prof_gctx_merge_iter_arg_s *)opaque;
-
- malloc_mutex_lock(arg->tsdn, gctx->lock);
- tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
- (void *)arg->tsdn);
- if (gctx->cnt_summed.curobjs != 0) {
- arg->leak_ngctx++;
- }
- malloc_mutex_unlock(arg->tsdn, gctx->lock);
-
- return NULL;
-}
-
-static void
-prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) {
- prof_tdata_t *tdata = prof_tdata_get(tsd, false);
- prof_gctx_t *gctx;
-
- /*
- * Standard tree iteration won't work here, because as soon as we
- * decrement gctx->nlimbo and unlock gctx, another thread can
- * concurrently destroy it, which will corrupt the tree. Therefore,
- * tear down the tree one node at a time during iteration.
- */
- while ((gctx = gctx_tree_first(gctxs)) != NULL) {
- gctx_tree_remove(gctxs, gctx);
- malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
- {
- prof_tctx_t *next;
-
- next = NULL;
- do {
- prof_tctx_t *to_destroy =
- tctx_tree_iter(&gctx->tctxs, next,
- prof_tctx_finish_iter,
- (void *)tsd_tsdn(tsd));
- if (to_destroy != NULL) {
- next = tctx_tree_next(&gctx->tctxs,
- to_destroy);
- tctx_tree_remove(&gctx->tctxs,
- to_destroy);
- idalloctm(tsd_tsdn(tsd), to_destroy,
- NULL, NULL, true, true);
- } else {
- next = NULL;
- }
- } while (next != NULL);
- }
- gctx->nlimbo--;
- if (prof_gctx_should_destroy(gctx)) {
- gctx->nlimbo++;
- malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
- prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
- } else {
- malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
- }
- }
-}
-
-struct prof_tdata_merge_iter_arg_s {
- tsdn_t *tsdn;
- prof_cnt_t cnt_all;
-};
-
-static prof_tdata_t *
-prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
- void *opaque) {
- struct prof_tdata_merge_iter_arg_s *arg =
- (struct prof_tdata_merge_iter_arg_s *)opaque;
-
- malloc_mutex_lock(arg->tsdn, tdata->lock);
- if (!tdata->expired) {
- size_t tabind;
- union {
- prof_tctx_t *p;
- void *v;
- } tctx;
-
- tdata->dumping = true;
- memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
- for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
- &tctx.v);) {
- prof_tctx_merge_tdata(arg->tsdn, tctx.p, tdata);
- }
-
- arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;
- arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;
- if (opt_prof_accum) {
- arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;
- arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;
- }
- } else {
- tdata->dumping = false;
- }
- malloc_mutex_unlock(arg->tsdn, tdata->lock);
-
- return NULL;
-}
-
-static prof_tdata_t *
-prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
- void *arg) {
- bool propagate_err = *(bool *)arg;
-
- if (!tdata->dumping) {
- return NULL;
- }
-
- if (prof_dump_printf(propagate_err,
- " t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]%s%s\n",
- tdata->thr_uid, tdata->cnt_summed.curobjs,
- tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs,
- tdata->cnt_summed.accumbytes,
- (tdata->thread_name != NULL) ? " " : "",
- (tdata->thread_name != NULL) ? tdata->thread_name : "")) {
- return tdata;
- }
- return NULL;
-}
-
-static bool
-prof_dump_header_impl(tsdn_t *tsdn, bool propagate_err,
- const prof_cnt_t *cnt_all) {
- bool ret;
-
- if (prof_dump_printf(propagate_err,
- "heap_v2/%"FMTu64"\n"
- " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
- ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs,
- cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) {
- return true;
- }
-
- malloc_mutex_lock(tsdn, &tdatas_mtx);
- ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
- (void *)&propagate_err) != NULL);
- malloc_mutex_unlock(tsdn, &tdatas_mtx);
- return ret;
-}
-prof_dump_header_t *JET_MUTABLE prof_dump_header = prof_dump_header_impl;
-
-static bool
-prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx,
- const prof_bt_t *bt, prof_gctx_tree_t *gctxs) {
- bool ret;
- unsigned i;
- struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg;
-
- cassert(config_prof);
- malloc_mutex_assert_owner(tsdn, gctx->lock);
-
- /* Avoid dumping such gctx's that have no useful data. */
- if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
- (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) {
- assert(gctx->cnt_summed.curobjs == 0);
- assert(gctx->cnt_summed.curbytes == 0);
- assert(gctx->cnt_summed.accumobjs == 0);
- assert(gctx->cnt_summed.accumbytes == 0);
- ret = false;
- goto label_return;
- }
-
- if (prof_dump_printf(propagate_err, "@")) {
- ret = true;
- goto label_return;
- }
- for (i = 0; i < bt->len; i++) {
- if (prof_dump_printf(propagate_err, " %#"FMTxPTR,
- (uintptr_t)bt->vec[i])) {
- ret = true;
- goto label_return;
- }
- }
-
- if (prof_dump_printf(propagate_err,
- "\n"
- " t*: %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
- gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes,
- gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) {
- ret = true;
- goto label_return;
- }
-
- prof_tctx_dump_iter_arg.tsdn = tsdn;
- prof_tctx_dump_iter_arg.propagate_err = propagate_err;
- if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
- (void *)&prof_tctx_dump_iter_arg) != NULL) {
- ret = true;
- goto label_return;
- }
-
- ret = false;
-label_return:
- return ret;
-}
-
-#ifndef _WIN32
-JEMALLOC_FORMAT_PRINTF(1, 2)
-static int
-prof_open_maps(const char *format, ...) {
- int mfd;
- va_list ap;
- char filename[PATH_MAX + 1];
-
- va_start(ap, format);
- malloc_vsnprintf(filename, sizeof(filename), format, ap);
- va_end(ap);
-
-#if defined(O_CLOEXEC)
- mfd = open(filename, O_RDONLY | O_CLOEXEC);
-#else
- mfd = open(filename, O_RDONLY);
- if (mfd != -1) {
- fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC);
- }
-#endif
-
- return mfd;
-}
-#endif
-
-static bool
-prof_dump_maps(bool propagate_err) {
- bool ret;
- int mfd;
-
- cassert(config_prof);
-#ifdef __FreeBSD__
- mfd = prof_open_maps("/proc/curproc/map");
-#elif defined(_WIN32)
- mfd = -1; // Not implemented
-#else
- {
- int pid = prof_getpid();
-
- mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid);
- if (mfd == -1) {
- mfd = prof_open_maps("/proc/%d/maps", pid);
- }
- }
-#endif
- if (mfd != -1) {
- ssize_t nread;
-
- if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") &&
- propagate_err) {
- ret = true;
- goto label_return;
- }
- nread = 0;
- do {
- prof_dump_buf_end += nread;
- if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
- /* Make space in prof_dump_buf before read(). */
- if (prof_dump_flush(propagate_err) &&
- propagate_err) {
- ret = true;
- goto label_return;
- }
- }
- nread = malloc_read_fd(mfd,
- &prof_dump_buf[prof_dump_buf_end], PROF_DUMP_BUFSIZE
- - prof_dump_buf_end);
- } while (nread > 0);
- } else {
- ret = true;
- goto label_return;
- }
-
- ret = false;
-label_return:
- if (mfd != -1) {
- close(mfd);
- }
- return ret;
-}
-
-/*
- * See prof_sample_threshold_update() comment for why the body of this function
- * is conditionally compiled.
- */
-static void
-prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
- const char *filename) {
-#ifdef JEMALLOC_PROF
- /*
- * Scaling is equivalent AdjustSamples() in jeprof, but the result may
- * differ slightly from what jeprof reports, because here we scale the
- * summary values, whereas jeprof scales each context individually and
- * reports the sums of the scaled values.
- */
- if (cnt_all->curbytes != 0) {
- double sample_period = (double)((uint64_t)1 << lg_prof_sample);
- double ratio = (((double)cnt_all->curbytes) /
- (double)cnt_all->curobjs) / sample_period;
- double scale_factor = 1.0 / (1.0 - exp(-ratio));
- uint64_t curbytes = (uint64_t)round(((double)cnt_all->curbytes)
- * scale_factor);
- uint64_t curobjs = (uint64_t)round(((double)cnt_all->curobjs) *
- scale_factor);
-
- malloc_printf(": Leak approximation summary: ~%"FMTu64
- " byte%s, ~%"FMTu64" object%s, >= %zu context%s\n",
- curbytes, (curbytes != 1) ? "s" : "", curobjs, (curobjs !=
- 1) ? "s" : "", leak_ngctx, (leak_ngctx != 1) ? "s" : "");
- malloc_printf(
- ": Run jeprof on \"%s\" for leak detail\n",
- filename);
- }
-#endif
-}
-
-struct prof_gctx_dump_iter_arg_s {
- tsdn_t *tsdn;
- bool propagate_err;
-};
-
-static prof_gctx_t *
-prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque) {
- prof_gctx_t *ret;
- struct prof_gctx_dump_iter_arg_s *arg =
- (struct prof_gctx_dump_iter_arg_s *)opaque;
-
- malloc_mutex_lock(arg->tsdn, gctx->lock);
-
- if (prof_dump_gctx(arg->tsdn, arg->propagate_err, gctx, &gctx->bt,
- gctxs)) {
- ret = gctx;
- goto label_return;
- }
-
- ret = NULL;
-label_return:
- malloc_mutex_unlock(arg->tsdn, gctx->lock);
- return ret;
-}
-
-static void
-prof_dump_prep(tsd_t *tsd, prof_tdata_t *tdata,
- struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
- struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
- prof_gctx_tree_t *gctxs) {
- size_t tabind;
- union {
- prof_gctx_t *p;
- void *v;
- } gctx;
-
- prof_enter(tsd, tdata);
-
- /*
- * Put gctx's in limbo and clear their counters in preparation for
- * summing.
- */
- gctx_tree_new(gctxs);
- for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) {
- prof_dump_gctx_prep(tsd_tsdn(tsd), gctx.p, gctxs);
- }
-
- /*
- * Iterate over tdatas, and for the non-expired ones snapshot their tctx
- * stats and merge them into the associated gctx's.
- */
- prof_tdata_merge_iter_arg->tsdn = tsd_tsdn(tsd);
- memset(&prof_tdata_merge_iter_arg->cnt_all, 0, sizeof(prof_cnt_t));
- malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
- tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
- (void *)prof_tdata_merge_iter_arg);
- malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
-
- /* Merge tctx stats into gctx's. */
- prof_gctx_merge_iter_arg->tsdn = tsd_tsdn(tsd);
- prof_gctx_merge_iter_arg->leak_ngctx = 0;
- gctx_tree_iter(gctxs, NULL, prof_gctx_merge_iter,
- (void *)prof_gctx_merge_iter_arg);
-
- prof_leave(tsd, tdata);
-}
-
-static bool
-prof_dump_file(tsd_t *tsd, bool propagate_err, const char *filename,
- bool leakcheck, prof_tdata_t *tdata,
- struct prof_tdata_merge_iter_arg_s *prof_tdata_merge_iter_arg,
- struct prof_gctx_merge_iter_arg_s *prof_gctx_merge_iter_arg,
- struct prof_gctx_dump_iter_arg_s *prof_gctx_dump_iter_arg,
- prof_gctx_tree_t *gctxs) {
- /* Create dump file. */
- if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) {
- return true;
- }
-
- /* Dump profile header. */
- if (prof_dump_header(tsd_tsdn(tsd), propagate_err,
- &prof_tdata_merge_iter_arg->cnt_all)) {
- goto label_write_error;
- }
-
- /* Dump per gctx profile stats. */
- prof_gctx_dump_iter_arg->tsdn = tsd_tsdn(tsd);
- prof_gctx_dump_iter_arg->propagate_err = propagate_err;
- if (gctx_tree_iter(gctxs, NULL, prof_gctx_dump_iter,
- (void *)prof_gctx_dump_iter_arg) != NULL) {
- goto label_write_error;
- }
-
- /* Dump /proc//maps if possible. */
- if (prof_dump_maps(propagate_err)) {
- goto label_write_error;
- }
-
- if (prof_dump_close(propagate_err)) {
- return true;
- }
-
- return false;
-label_write_error:
- prof_dump_close(propagate_err);
- return true;
-}
-
-bool
-prof_dump(tsd_t *tsd, bool propagate_err, const char *filename,
- bool leakcheck) {
- cassert(config_prof);
- assert(tsd_reentrancy_level_get(tsd) == 0);
-
- prof_tdata_t * tdata = prof_tdata_get(tsd, true);
- if (tdata == NULL) {
- return true;
- }
-
- pre_reentrancy(tsd, NULL);
- malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
-
- prof_gctx_tree_t gctxs;
- struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
- struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
- struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg;
- prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
- &prof_gctx_merge_iter_arg, &gctxs);
- bool err = prof_dump_file(tsd, propagate_err, filename, leakcheck, tdata,
- &prof_tdata_merge_iter_arg, &prof_gctx_merge_iter_arg,
- &prof_gctx_dump_iter_arg, &gctxs);
- prof_gctx_finish(tsd, &gctxs);
-
- malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
- post_reentrancy(tsd);
-
- if (err) {
- return true;
- }
-
- if (leakcheck) {
- prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all,
- prof_gctx_merge_iter_arg.leak_ngctx, filename);
- }
- return false;
-}
-
-#ifdef JEMALLOC_JET
-void
-prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes, uint64_t *accumobjs,
- uint64_t *accumbytes) {
- tsd_t *tsd;
- prof_tdata_t *tdata;
- struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
- struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
- prof_gctx_tree_t gctxs;
-
- tsd = tsd_fetch();
- tdata = prof_tdata_get(tsd, false);
- if (tdata == NULL) {
- if (curobjs != NULL) {
- *curobjs = 0;
- }
- if (curbytes != NULL) {
- *curbytes = 0;
- }
- if (accumobjs != NULL) {
- *accumobjs = 0;
- }
- if (accumbytes != NULL) {
- *accumbytes = 0;
- }
- return;
- }
-
- prof_dump_prep(tsd, tdata, &prof_tdata_merge_iter_arg,
- &prof_gctx_merge_iter_arg, &gctxs);
- prof_gctx_finish(tsd, &gctxs);
-
- if (curobjs != NULL) {
- *curobjs = prof_tdata_merge_iter_arg.cnt_all.curobjs;
- }
- if (curbytes != NULL) {
- *curbytes = prof_tdata_merge_iter_arg.cnt_all.curbytes;
- }
- if (accumobjs != NULL) {
- *accumobjs = prof_tdata_merge_iter_arg.cnt_all.accumobjs;
- }
- if (accumbytes != NULL) {
- *accumbytes = prof_tdata_merge_iter_arg.cnt_all.accumbytes;
- }
-}
-#endif
-
-void
-prof_bt_hash(const void *key, size_t r_hash[2]) {
- prof_bt_t *bt = (prof_bt_t *)key;
-
- cassert(config_prof);
-
- hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
-}
-
-bool
-prof_bt_keycomp(const void *k1, const void *k2) {
- const prof_bt_t *bt1 = (prof_bt_t *)k1;
- const prof_bt_t *bt2 = (prof_bt_t *)k2;
-
- cassert(config_prof);
-
- if (bt1->len != bt2->len) {
- return false;
- }
- return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
-}
-
-prof_tdata_t *
-prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
- char *thread_name, bool active) {
- prof_tdata_t *tdata;
-
- cassert(config_prof);
-
- /* Initialize an empty cache for this thread. */
- tdata = (prof_tdata_t *)iallocztm(tsd_tsdn(tsd), sizeof(prof_tdata_t),
- sz_size2index(sizeof(prof_tdata_t)), false, NULL, true,
- arena_get(TSDN_NULL, 0, true), true);
- if (tdata == NULL) {
- return NULL;
- }
-
- tdata->lock = prof_tdata_mutex_choose(thr_uid);
- tdata->thr_uid = thr_uid;
- tdata->thr_discrim = thr_discrim;
- tdata->thread_name = thread_name;
- tdata->attached = true;
- tdata->expired = false;
- tdata->tctx_uid_next = 0;
-
- if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash,
- prof_bt_keycomp)) {
- idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
- return NULL;
- }
-
- tdata->prng_state = (uint64_t)(uintptr_t)tdata;
- prof_sample_threshold_update(tdata);
-
- tdata->enq = false;
- tdata->enq_idump = false;
- tdata->enq_gdump = false;
-
- tdata->dumping = false;
- tdata->active = active;
-
- malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
- tdata_tree_insert(&tdatas, tdata);
- malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
-
- return tdata;
-}
-
-static bool
-prof_tdata_should_destroy_unlocked(prof_tdata_t *tdata, bool even_if_attached) {
- if (tdata->attached && !even_if_attached) {
- return false;
- }
- if (ckh_count(&tdata->bt2tctx) != 0) {
- return false;
- }
- return true;
-}
-
-static bool
-prof_tdata_should_destroy(tsdn_t *tsdn, prof_tdata_t *tdata,
- bool even_if_attached) {
- malloc_mutex_assert_owner(tsdn, tdata->lock);
-
- return prof_tdata_should_destroy_unlocked(tdata, even_if_attached);
-}
-
-static void
-prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
- bool even_if_attached) {
- malloc_mutex_assert_owner(tsd_tsdn(tsd), &tdatas_mtx);
-
- tdata_tree_remove(&tdatas, tdata);
-
- assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
-
- if (tdata->thread_name != NULL) {
- idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, NULL, true,
- true);
- }
- ckh_delete(tsd, &tdata->bt2tctx);
- idalloctm(tsd_tsdn(tsd), tdata, NULL, NULL, true, true);
-}
-
-static void
-prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) {
- malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
- prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
- malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
-}
-
-void
-prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) {
- bool destroy_tdata;
-
- malloc_mutex_lock(tsd_tsdn(tsd), tdata->lock);
- if (tdata->attached) {
- destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata,
- true);
- /*
- * Only detach if !destroy_tdata, because detaching would allow
- * another thread to win the race to destroy tdata.
- */
- if (!destroy_tdata) {
- tdata->attached = false;
- }
- tsd_prof_tdata_set(tsd, NULL);
- } else {
- destroy_tdata = false;
- }
- malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
- if (destroy_tdata) {
- prof_tdata_destroy(tsd, tdata, true);
- }
-}
-
-static bool
-prof_tdata_expire(tsdn_t *tsdn, prof_tdata_t *tdata) {
- bool destroy_tdata;
-
- malloc_mutex_lock(tsdn, tdata->lock);
- if (!tdata->expired) {
- tdata->expired = true;
- destroy_tdata = tdata->attached ? false :
- prof_tdata_should_destroy(tsdn, tdata, false);
- } else {
- destroy_tdata = false;
- }
- malloc_mutex_unlock(tsdn, tdata->lock);
-
- return destroy_tdata;
-}
-
-static prof_tdata_t *
-prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
- void *arg) {
- tsdn_t *tsdn = (tsdn_t *)arg;
-
- return (prof_tdata_expire(tsdn, tdata) ? tdata : NULL);
-}
-
-void
-prof_reset(tsd_t *tsd, size_t lg_sample) {
- prof_tdata_t *next;
-
- assert(lg_sample < (sizeof(uint64_t) << 3));
-
- malloc_mutex_lock(tsd_tsdn(tsd), &prof_dump_mtx);
- malloc_mutex_lock(tsd_tsdn(tsd), &tdatas_mtx);
-
- lg_prof_sample = lg_sample;
-
- next = NULL;
- do {
- prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
- prof_tdata_reset_iter, (void *)tsd);
- if (to_destroy != NULL) {
- next = tdata_tree_next(&tdatas, to_destroy);
- prof_tdata_destroy_locked(tsd, to_destroy, false);
- } else {
- next = NULL;
- }
- } while (next != NULL);
-
- malloc_mutex_unlock(tsd_tsdn(tsd), &tdatas_mtx);
- malloc_mutex_unlock(tsd_tsdn(tsd), &prof_dump_mtx);
-}
-
-void
-prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) {
- prof_tdata_t *tdata = tctx->tdata;
- prof_gctx_t *gctx = tctx->gctx;
- bool destroy_tdata, destroy_tctx, destroy_gctx;
-
- malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock);
-
- assert(tctx->cnts.curobjs == 0);
- assert(tctx->cnts.curbytes == 0);
- assert(!opt_prof_accum);
- assert(tctx->cnts.accumobjs == 0);
- assert(tctx->cnts.accumbytes == 0);
-
- ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
- destroy_tdata = prof_tdata_should_destroy(tsd_tsdn(tsd), tdata, false);
- malloc_mutex_unlock(tsd_tsdn(tsd), tdata->lock);
-
- malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
- switch (tctx->state) {
- case prof_tctx_state_nominal:
- tctx_tree_remove(&gctx->tctxs, tctx);
- destroy_tctx = true;
- if (prof_gctx_should_destroy(gctx)) {
- /*
- * Increment gctx->nlimbo in order to keep another
- * thread from winning the race to destroy gctx while
- * this one has gctx->lock dropped. Without this, it
- * would be possible for another thread to:
- *
- * 1) Sample an allocation associated with gctx.
- * 2) Deallocate the sampled object.
- * 3) Successfully prof_gctx_try_destroy(gctx).
- *
- * The result would be that gctx no longer exists by the
- * time this thread accesses it in
- * prof_gctx_try_destroy().
- */
- gctx->nlimbo++;
- destroy_gctx = true;
- } else {
- destroy_gctx = false;
- }
- break;
- case prof_tctx_state_dumping:
- /*
- * A dumping thread needs tctx to remain valid until dumping
- * has finished. Change state such that the dumping thread will
- * complete destruction during a late dump iteration phase.
- */
- tctx->state = prof_tctx_state_purgatory;
- destroy_tctx = false;
- destroy_gctx = false;
- break;
- default:
- not_reached();
- destroy_tctx = false;
- destroy_gctx = false;
- }
- malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
- if (destroy_gctx) {
- prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
- tdata);
- }
-
- malloc_mutex_assert_not_owner(tsd_tsdn(tsd), tctx->tdata->lock);
-
- if (destroy_tdata) {
- prof_tdata_destroy(tsd, tdata, false);
- }
-
- if (destroy_tctx) {
- idalloctm(tsd_tsdn(tsd), tctx, NULL, NULL, true, true);
- }
-}
-
-/******************************************************************************/