project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement per thread heap profiling.

Browse Source 
Rename data structures (prof_thr_cnt_t-->prof_tctx_t,
prof_ctx_t-->prof_gctx_t), and convert to storing a prof_tctx_t for
sampled objects.

Convert PROF_ALLOC_PREP() to prof_alloc_prep(), since precise backtrace
depth within jemalloc functions is no longer an issue (pprof prunes
irrelevant frames).

Implement mallctl's:
- prof.reset implements full sample data reset, and optional change of
  sample interval.
- prof.lg_sample reads the current sample interval (opt.lg_prof_sample
  was the permanent source of truth prior to prof.reset).
- thread.prof.name provides naming capability for threads within heap
  profile dumps.
- thread.prof.active makes it possible to activate/deactivate heap
  profiling for individual threads.

Modify the heap dump files to contain per thread heap profile data.
This change is incompatible with the existing pprof, which will require
enhancements to read and process the enriched data.
This commit is contained in:
Jason Evans 2014-08-18 16:22:13 -07:00
parent 1628e8615e
commit 602c8e0971
11 changed files with 1235 additions and 724 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
doc/jemalloc.xml.in
View File

@ -1047,7 +1047,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<varlistentry id="opt.lg_prof_sample">
<term>
<mallctl>opt.lg_prof_sample</mallctl>
(<type>ssize_t</type>)
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-prof</option>]
</term>
@ -1243,6 +1243,35 @@ malloc_conf = "xmalloc:true";]]></programlisting>
the developer may find manual flushing useful.</para></listitem>
</varlistentry>
<varlistentry id="thread.prof.name">
<term>
<mallctl>thread.prof.name</mallctl>
(<type>const char *</type>)
<literal>rw</literal>
[<option>--enable-prof</option>]
</term>
<listitem><para>Get/set the descriptive name associated with the calling
thread in memory profile dumps. An internal copy of the name string is
created, so the input string need not be maintained after this interface
completes execution. The output string of this interface should be
copied for non-ephemeral uses, because multiple implementation details
can cause asynchronous string deallocation.</para></listitem>
</varlistentry>
<varlistentry id="thread.prof.active">
<term>
<mallctl>thread.prof.active</mallctl>
(<type>bool</type>)
<literal>rw</literal>
[<option>--enable-prof</option>]
</term>
<listitem><para>Control whether sampling is currently active for the
calling thread. This is a deactivation mechanism in addition to <link
linkend="prof.active"><mallctl>prof.active</mallctl></link>; both must
be active for the calling thread to sample. This flag is enabled by
default.</para></listitem>
</varlistentry>
<varlistentry id="arena.i.purge">
<term>
<mallctl>arena.&lt;i&gt;.purge</mallctl>
@ -1492,6 +1521,31 @@ malloc_conf = "xmalloc:true";]]></programlisting>
option.</para></listitem>
</varlistentry>
<varlistentry id="prof.reset">
<term>
<mallctl>prof.reset</mallctl>
(<type>size_t</type>)
<literal>-w</literal>
[<option>--enable-prof</option>]
</term>
<listitem><para>Reset all memory profile statistics, and optionally
update the sample rate (see <link
linkend="opt.lg_prof_sample"><mallctl>opt.lg_prof_sample</mallctl></link>).
</para></listitem>
</varlistentry>
<varlistentry id="prof.lg_sample">
<term>
<mallctl>prof.lg_sample</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
[<option>--enable-prof</option>]
</term>
<listitem><para>Get the sample rate (see <link
linkend="opt.lg_prof_sample"><mallctl>opt.lg_prof_sample</mallctl></link>).
</para></listitem>
</varlistentry>
<varlistentry id="prof.interval">
<term>
<mallctl>prof.interval</mallctl>

22
include/jemalloc/internal/arena.h
View File

@ -58,7 +58,7 @@ typedef struct arena_s arena_t;
struct arena_chunk_map_s {
#ifndef JEMALLOC_PROF
/*
* Overlay prof_ctx in order to allow it to be referenced by dead code.
* Overlay prof_tctx in order to allow it to be referenced by dead code.
* Such antics aren't warranted for per arena data structures, but
* chunk map overhead accounts for a percentage of memory, rather than
* being just a fixed cost.
@ -75,7 +75,7 @@ struct arena_chunk_map_s {
rb_node(arena_chunk_map_t) rb_link;
/* Profile counters, used for large object runs. */
prof_ctx_t *prof_ctx;
prof_tctx_t *prof_tctx;
#ifndef JEMALLOC_PROF
}; /* union { ... }; */
#endif
@ -472,8 +472,8 @@ size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
const void *ptr);
prof_ctx_t *arena_prof_ctx_get(const void *ptr);
void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
prof_tctx_t *arena_prof_tctx_get(const void *ptr);
void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
size_t arena_salloc(const void *ptr, bool demote);
void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache);
@ -987,10 +987,10 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
return (regind);
}
JEMALLOC_INLINE prof_ctx_t *
arena_prof_ctx_get(const void *ptr)
JEMALLOC_INLINE prof_tctx_t *
arena_prof_tctx_get(const void *ptr)
{
prof_ctx_t *ret;
prof_tctx_t *ret;
arena_chunk_t *chunk;
size_t pageind, mapbits;
@ -1003,15 +1003,15 @@ arena_prof_ctx_get(const void *ptr)
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
if ((mapbits & CHUNK_MAP_LARGE) == 0)
ret = (prof_ctx_t *)(uintptr_t)1U;
ret = (prof_tctx_t *)(uintptr_t)1U;
else
ret = arena_mapp_get(chunk, pageind)->prof_ctx;
ret = arena_mapp_get(chunk, pageind)->prof_tctx;
return (ret);
}
JEMALLOC_INLINE void
arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
{
arena_chunk_t *chunk;
size_t pageind;
@ -1025,7 +1025,7 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
if (arena_mapbits_large_get(chunk, pageind) != 0)
arena_mapp_get(chunk, pageind)->prof_ctx = ctx;
arena_mapp_get(chunk, pageind)->prof_tctx = tctx;
}
JEMALLOC_ALWAYS_INLINE void *

2
include/jemalloc/internal/extent.h
View File

@ -16,7 +16,7 @@ struct extent_node_s {
rb_node(extent_node_t) link_ad;
/* Profile counters, used for huge objects. */
prof_ctx_t *prof_ctx;
prof_tctx_t *prof_tctx;
/* Pointer to the extent that this tree node is responsible for. */
void *addr;

4
include/jemalloc/internal/huge.h
View File

@ -21,8 +21,8 @@ extern huge_dalloc_junk_t *huge_dalloc_junk;
#endif
void huge_dalloc(void *ptr);
size_t huge_salloc(const void *ptr);
prof_ctx_t *huge_prof_ctx_get(const void *ptr);
void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
prof_tctx_t *huge_prof_tctx_get(const void *ptr);
void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
bool huge_boot(void);
void huge_prefork(void);
void huge_postfork_parent(void);

21
include/jemalloc/internal/private_symbols.txt
View File

@ -48,9 +48,9 @@ arena_prefork
arena_prof_accum
arena_prof_accum_impl
arena_prof_accum_locked
arena_prof_ctx_get
arena_prof_ctx_set
arena_prof_promoted
arena_prof_tctx_get
arena_prof_tctx_set
arena_ptr_small_binind_get
arena_purge_all
arena_quarantine_junk_small
@ -208,8 +208,8 @@ huge_palloc
huge_postfork_child
huge_postfork_parent
huge_prefork
huge_prof_ctx_get
huge_prof_ctx_set
huge_prof_tctx_get
huge_prof_tctx_set
huge_ralloc
huge_ralloc_no_move
huge_salloc
@ -287,28 +287,31 @@ opt_zero
p2rz
pages_purge
pow2_ceil
prof_alloc_prep
prof_backtrace
prof_boot0
prof_boot1
prof_boot2
prof_bt_count
prof_ctx_get
prof_ctx_set
prof_dump_open
prof_free
prof_free_sampled_object
prof_gdump
prof_idump
prof_interval
prof_lookup
prof_malloc
prof_malloc_record_object
prof_malloc_sample_object
prof_mdump
prof_postfork_child
prof_postfork_parent
prof_prefork
prof_realloc
prof_reset
prof_sample_accum_update
prof_sample_threshold_update
prof_tctx_get
prof_tctx_set
prof_tdata_booted
prof_tdata_cleanup
prof_tdata_get
@ -322,6 +325,10 @@ prof_tdata_tsd_get
prof_tdata_tsd_get_wrapper
prof_tdata_tsd_init_head
prof_tdata_tsd_set
prof_thread_active_get
prof_thread_active_set
prof_thread_name_get
prof_thread_name_set
quarantine
quarantine_alloc_hook
quarantine_boot

444
include/jemalloc/internal/prof.h
View File

@ -1,11 +1,10 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
typedef uint64_t prof_thr_uid_t;
typedef struct prof_bt_s prof_bt_t;
typedef struct prof_cnt_s prof_cnt_t;
typedef struct prof_thr_cnt_s prof_thr_cnt_t;
typedef struct prof_ctx_s prof_ctx_t;
typedef struct prof_tctx_s prof_tctx_t;
typedef struct prof_gctx_s prof_gctx_t;
typedef struct prof_tdata_s prof_tdata_t;
/* Option defaults. */
@ -34,11 +33,17 @@ typedef struct prof_tdata_s prof_tdata_t;
#define PROF_PRINTF_BUFSIZE 128
/*
* Number of mutexes shared among all ctx's. No space is allocated for these
* Number of mutexes shared among all gctx's. No space is allocated for these
* unless profiling is enabled, so it's okay to over-provision.
*/
#define PROF_NCTX_LOCKS 1024
/*
* Number of mutexes shared among all tdata's. No space is allocated for these
* unless profiling is enabled, so it's okay to over-provision.
*/
#define PROF_NTDATA_LOCKS 256
/*
* prof_tdata pointers close to NULL are used to encode state information that
* is used for cleaning up during thread shutdown.
@ -66,87 +71,70 @@ typedef struct {
#endif
struct prof_cnt_s {
/*
* Profiling counters. An allocation/deallocation pair can operate on
* different prof_thr_cnt_t objects that are linked into the same
* prof_ctx_t cnts_ql, so it is possible for the cur* counters to go
* negative. In principle it is possible for the *bytes counters to
* overflow/underflow, but a general solution would require something
* like 128-bit counters; this implementation doesn't bother to solve
* that problem.
*/
int64_t curobjs;
int64_t curbytes;
/* Profiling counters. */
uint64_t curobjs;
uint64_t curbytes;
uint64_t accumobjs;
uint64_t accumbytes;
};
struct prof_thr_cnt_s {
prof_thr_uid_t thr_uid;
typedef enum {
prof_tctx_state_nominal,
prof_tctx_state_dumping,
prof_tctx_state_purgatory /* Dumper must finish destroying. */
} prof_tctx_state_t;
/* Linkage into prof_ctx_t's thr_cnts. */
rb_node(prof_thr_cnt_t) thr_cnt_link;
struct prof_tctx_s {
/* Thread data for thread that performed the allocation. */
prof_tdata_t *tdata;
/*
* Associated context. If a thread frees an object that it did not
* allocate, it is possible that the context is not present in the
* thread's hash table, in which case it must be able to look up the
* context, insert a new prof_thr_cnt_t into the thread's hash table,
* and link it into the prof_ctx_t's thr_cnts.
*/
prof_ctx_t *ctx;
/*
* Threads use memory barriers to update the counters. Since there is
* only ever one writer, the only challenge is for the reader to get a
* consistent read of the counters.
*
* The writer uses this series of operations:
*
* 1) Increment epoch to an odd number.
* 2) Update counters.
* 3) Increment epoch to an even number.
*
* The reader must assure 1) that the epoch is even while it reads the
* counters, and 2) that the epoch doesn't change between the time it
* starts and finishes reading the counters.
*/
unsigned epoch;
/* Profiling counters. */
/* Profiling counters, protected by tdata->lock. */
prof_cnt_t cnts;
};
typedef rb_tree(prof_thr_cnt_t) prof_thr_cnt_tree_t;
struct prof_ctx_s {
/* Protects nlimbo, cnt_merged, and thr_cnts. */
/* Associated global context. */
prof_gctx_t *gctx;
/* Linkage into gctx's tctxs. */
rb_node(prof_tctx_t) tctx_link;
/* Current dump-related state, protected by gctx->lock. */
prof_tctx_state_t state;
/*
* Copy of cnts snapshotted during early dump phase, protected by
* dump_mtx.
*/
prof_cnt_t dump_cnts;
};
typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
struct prof_gctx_s {
/* Protects nlimbo, cnt_summed, and tctxs. */
malloc_mutex_t *lock;
/*
* Number of threads that currently cause this ctx to be in a state of
* Number of threads that currently cause this gctx to be in a state of
* limbo due to one of:
* - Initializing per thread counters associated with this ctx.
* - Preparing to destroy this ctx.
* - Dumping a heap profile that includes this ctx.
* - Initializing this gctx.
* - Initializing per thread counters associated with this gctx.
* - Preparing to destroy this gctx.
* - Dumping a heap profile that includes this gctx.
* nlimbo must be 1 (single destroyer) in order to safely destroy the
* ctx.
* gctx.
*/
unsigned nlimbo;
/* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed;
/* When threads exit, they merge their stats into cnt_merged. */
prof_cnt_t cnt_merged;
/*
* Tree of profile counters, one for each thread that has allocated in
* this context.
*/
prof_thr_cnt_tree_t thr_cnts;
prof_tctx_tree_t tctxs;
/* Linkage for tree of contexts to be dumped. */
rb_node(prof_ctx_t) dump_link;
rb_node(prof_gctx_t) dump_link;
/* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed;
/* Associated backtrace. */
prof_bt_t bt;
@ -154,21 +142,34 @@ struct prof_ctx_s {
/* Backtrace vector, variable size, referred to by bt. */
void *vec[1];
};
typedef rb_tree(prof_ctx_t) prof_ctx_tree_t;
typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
typedef enum {
prof_tdata_state_attached, /* Active thread attached, data valid. */
prof_tdata_state_detached, /* Defunct thread, data remain valid. */
prof_tdata_state_expired /* Predates reset, omit data from dump. */
} prof_tdata_state_t;
struct prof_tdata_s {
malloc_mutex_t *lock;
/* Monotonically increasing unique thread identifier. */
uint64_t thr_uid;
/* Included in heap profile dumps if non-NULL. */
char *thread_name;
prof_tdata_state_t state;
rb_node(prof_tdata_t) tdata_link;
/*
* Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread tracks
* Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks
* backtraces for which it has non-zero allocation/deallocation counters
* associated with thread-specific prof_thr_cnt_t objects. Other
* threads may read the prof_thr_cnt_t contents, but no others will ever
* write them.
*
* Upon thread exit, the thread must merge all the prof_thr_cnt_t
* counter data into the associated prof_ctx_t objects, and unlink/free
* the prof_thr_cnt_t objects.
* associated with thread-specific prof_tctx_t objects. Other threads
* may write to prof_tctx_t contents when freeing associated objects.
*/
ckh_t bt2cnt;
ckh_t bt2tctx;
/* Sampling state. */
uint64_t prng_state;
@ -179,9 +180,27 @@ struct prof_tdata_s {
bool enq_idump;
bool enq_gdump;
/*
* Set to true during an early dump phase for tdata's which are
* currently being dumped. New threads' tdata's have this initialized
* to false so that they aren't accidentally included in later dump
* phases.
*/
bool dumping;
/*
* True if profiling is active for this tdata's thread
* (thread.prof.active mallctl).
*/
bool active;
/* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed;
/* Backtrace vector, used for calls to prof_backtrace(). */
void *vec[PROF_BT_MAX];
};
typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
@ -217,9 +236,18 @@ extern char opt_prof_prefix[
*/
extern uint64_t prof_interval;
/*
* Initialized as opt_lg_prof_sample, and potentially modified during profiling
* resets.
*/
extern size_t lg_prof_sample;
void prof_malloc_sample_object(const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_free_sampled_object(size_t usize, prof_tctx_t *tctx);
void bt_init(prof_bt_t *bt, void **vec);
void prof_backtrace(prof_bt_t *bt);
prof_thr_cnt_t *prof_lookup(prof_bt_t *bt);
prof_tctx_t *prof_lookup(prof_bt_t *bt);
#ifdef JEMALLOC_JET
size_t prof_bt_count(void);
typedef int (prof_dump_open_t)(bool, const char *);
@ -229,53 +257,44 @@ void prof_idump(void);
bool prof_mdump(const char *filename);
void prof_gdump(void);
prof_tdata_t *prof_tdata_init(void);
prof_tdata_t *prof_tdata_reinit(prof_tdata_t *tdata);
void prof_reset(size_t lg_sample);
void prof_tdata_cleanup(void *arg);
const char *prof_thread_name_get(void);
bool prof_thread_name_set(const char *thread_name);
bool prof_thread_active_get(void);
bool prof_thread_active_set(bool active);
void prof_boot0(void);
void prof_boot1(void);
bool prof_boot2(void);
void prof_prefork(void);
void prof_postfork_parent(void);
void prof_postfork_child(void);
void prof_sample_threshold_update(prof_tdata_t *prof_tdata);
void prof_sample_threshold_update(prof_tdata_t *tdata);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#define PROF_ALLOC_PREP(size, ret) do { \
prof_tdata_t *prof_tdata; \
prof_bt_t bt; \
\
assert(size == s2u(size)); \
\
if (!opt_prof_active || \
prof_sample_accum_update(size, false, &prof_tdata)) { \
ret = (prof_thr_cnt_t *)(uintptr_t)1U; \
} else { \
bt_init(&bt, prof_tdata->vec); \
prof_backtrace(&bt); \
ret = prof_lookup(&bt); \
} \
} while (0)
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
prof_tdata_t *prof_tdata_get(bool create);
bool prof_sample_accum_update(size_t size, bool commit,
prof_tdata_t **prof_tdata_out);
prof_ctx_t *prof_ctx_get(const void *ptr);
void prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
void prof_malloc_record_object(const void *ptr, size_t usize,
prof_thr_cnt_t *cnt);
void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt);
void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
size_t old_usize, prof_ctx_t *old_ctx);
void prof_free(const void *ptr, size_t size);
bool prof_sample_accum_update(size_t usize, bool commit,
prof_tdata_t **tdata_out);
prof_tctx_t *prof_alloc_prep(size_t usize);
prof_tctx_t *prof_tctx_get(const void *ptr);
void prof_tctx_set(const void *ptr, prof_tctx_t *tctx);
void prof_malloc_sample_object(const void *ptr, size_t usize,
prof_tctx_t *tctx);
void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx);
void prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx,
size_t old_usize, prof_tctx_t *old_tctx);
void prof_free(const void *ptr, size_t usize);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */
/* Thread-specific backtrace cache, used to reduce bt2gctx contention. */
malloc_tsd_externs(prof_tdata, prof_tdata_t *)
malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
prof_tdata_cleanup)
@ -283,21 +302,27 @@ malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
JEMALLOC_INLINE prof_tdata_t *
prof_tdata_get(bool create)
{
prof_tdata_t *prof_tdata;
prof_tdata_t *tdata;
cassert(config_prof);
prof_tdata = *prof_tdata_tsd_get();
if (create && prof_tdata == NULL)
prof_tdata = prof_tdata_init();
tdata = *prof_tdata_tsd_get();
if (create) {
if (tdata == NULL)
tdata = prof_tdata_init();
else if (tdata->state == prof_tdata_state_expired)
tdata = prof_tdata_reinit(tdata);
assert(tdata == NULL || tdata->state ==
prof_tdata_state_attached);
}
return (prof_tdata);
return (tdata);
}
JEMALLOC_INLINE prof_ctx_t *
prof_ctx_get(const void *ptr)
JEMALLOC_INLINE prof_tctx_t *
prof_tctx_get(const void *ptr)
{
prof_ctx_t *ret;
prof_tctx_t *ret;
arena_chunk_t *chunk;
cassert(config_prof);
@ -306,15 +331,15 @@ prof_ctx_get(const void *ptr)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) {
/* Region. */
ret = arena_prof_ctx_get(ptr);
ret = arena_prof_tctx_get(ptr);
} else
ret = huge_prof_ctx_get(ptr);
ret = huge_prof_tctx_get(ptr);
return (ret);
}
JEMALLOC_INLINE void
prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
{
arena_chunk_t *chunk;
@ -324,66 +349,62 @@ prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr) {
/* Region. */
arena_prof_ctx_set(ptr, ctx);
arena_prof_tctx_set(ptr, tctx);
} else
huge_prof_ctx_set(ptr, ctx);
huge_prof_tctx_set(ptr, tctx);
}
JEMALLOC_INLINE bool
prof_sample_accum_update(size_t size, bool commit,
prof_tdata_t **prof_tdata_out)
prof_sample_accum_update(size_t usize, bool commit, prof_tdata_t **tdata_out)
{
prof_tdata_t *prof_tdata;
prof_tdata_t *tdata;
cassert(config_prof);
prof_tdata = prof_tdata_get(true);
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
prof_tdata = NULL;
tdata = prof_tdata_get(true);
if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
tdata = NULL;
if (prof_tdata_out != NULL)
*prof_tdata_out = prof_tdata;
if (tdata_out != NULL)
*tdata_out = tdata;
if (prof_tdata == NULL)
if (tdata == NULL)
return (true);
if (prof_tdata->bytes_until_sample >= size) {
if (tdata->bytes_until_sample >= usize) {
if (commit)
prof_tdata->bytes_until_sample -= size;
tdata->bytes_until_sample -= usize;
return (true);
} else {
/* Compute new sample threshold. */
if (commit)
prof_sample_threshold_update(prof_tdata);
return (false);
prof_sample_threshold_update(tdata);
return (tdata->active == false);
}
}
JEMALLOC_INLINE void
prof_malloc_record_object(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) {
prof_ctx_set(ptr, cnt->ctx);
JEMALLOC_INLINE prof_tctx_t *
prof_alloc_prep(size_t usize)
{
prof_tctx_t *ret;
prof_tdata_t *tdata;
prof_bt_t bt;
cnt->epoch++;
/*********/
mb_write();
/*********/
cnt->cnts.curobjs++;
cnt->cnts.curbytes += usize;
if (opt_prof_accum) {
cnt->cnts.accumobjs++;
cnt->cnts.accumbytes += usize;
assert(usize == s2u(usize));
if (!opt_prof_active || prof_sample_accum_update(usize, false, &tdata))
ret = (prof_tctx_t *)(uintptr_t)1U;
else {
bt_init(&bt, tdata->vec);
prof_backtrace(&bt);
ret = prof_lookup(&bt);
}
/*********/
mb_write();
/*********/
cnt->epoch++;
/*********/
mb_write();
/*********/
return (ret);
}
JEMALLOC_INLINE void
prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt)
prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx)
{
cassert(config_prof);
@ -392,131 +413,60 @@ prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt)
if (prof_sample_accum_update(usize, true, NULL)) {
/*
* Don't sample. For malloc()-like allocation, it is
* always possible to tell in advance how large an
* object's usable size will be, so there should never
* be a difference between the usize passed to
* PROF_ALLOC_PREP() and prof_malloc().
* Don't sample. For malloc()-like allocation, it is always
* possible to tell in advance how large an object's usable size
* will be, so there should never be a difference between the
* usize passed to PROF_ALLOC_PREP() and prof_malloc().
*/
assert((uintptr_t)cnt == (uintptr_t)1U);
assert((uintptr_t)tctx == (uintptr_t)1U);
}
if ((uintptr_t)cnt > (uintptr_t)1U)
prof_malloc_record_object(ptr, usize, cnt);
if ((uintptr_t)tctx > (uintptr_t)1U)
prof_malloc_sample_object(ptr, usize, tctx);
else
prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U);
prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U);
}
JEMALLOC_INLINE void
prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt,
size_t old_usize, prof_ctx_t *old_ctx)
prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, size_t old_usize,
prof_tctx_t *old_tctx)
{
prof_thr_cnt_t *told_cnt;
cassert(config_prof);
assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U);
assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
if (ptr != NULL) {
assert(usize == isalloc(ptr, true));
if (prof_sample_accum_update(usize, true, NULL)) {
/*
* Don't sample. The usize passed to
* PROF_ALLOC_PREP() was larger than what
* actually got allocated, so a backtrace was
* captured for this allocation, even though
* its actual usize was insufficient to cross
* the sample threshold.
* Don't sample. The usize passed to PROF_ALLOC_PREP()
* was larger than what actually got allocated, so a
* backtrace was captured for this allocation, even
* though its actual usize was insufficient to cross the
* sample threshold.
*/
cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
tctx = (prof_tctx_t *)(uintptr_t)1U;
}
}
if ((uintptr_t)old_ctx > (uintptr_t)1U) {
told_cnt = prof_lookup(&old_ctx->bt);
if (told_cnt == NULL) {
/*
* It's too late to propagate OOM for this realloc(),
* so operate directly on old_cnt->ctx->cnt_merged.
*/
malloc_mutex_lock(old_ctx->lock);
old_ctx->cnt_merged.curobjs--;
old_ctx->cnt_merged.curbytes -= old_usize;
malloc_mutex_unlock(old_ctx->lock);
told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
}
} else
told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U;
if ((uintptr_t)told_cnt > (uintptr_t)1U)
told_cnt->epoch++;
if ((uintptr_t)cnt > (uintptr_t)1U) {
prof_ctx_set(ptr, cnt->ctx);
cnt->epoch++;
} else if (ptr != NULL)
prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U);
/*********/
mb_write();
/*********/
if ((uintptr_t)told_cnt > (uintptr_t)1U) {
told_cnt->cnts.curobjs--;
told_cnt->cnts.curbytes -= old_usize;
}
if ((uintptr_t)cnt > (uintptr_t)1U) {
cnt->cnts.curobjs++;
cnt->cnts.curbytes += usize;
if (opt_prof_accum) {
cnt->cnts.accumobjs++;
cnt->cnts.accumbytes += usize;
}
}
/*********/
mb_write();
/*********/
if ((uintptr_t)told_cnt > (uintptr_t)1U)
told_cnt->epoch++;
if ((uintptr_t)cnt > (uintptr_t)1U)
cnt->epoch++;
/*********/
mb_write(); /* Not strictly necessary. */
if ((uintptr_t)old_tctx > (uintptr_t)1U)
prof_free_sampled_object(old_usize, old_tctx);
if ((uintptr_t)tctx > (uintptr_t)1U)
prof_malloc_sample_object(ptr, usize, tctx);
else
prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U);
}
JEMALLOC_INLINE void
prof_free(const void *ptr, size_t size)
prof_free(const void *ptr, size_t usize)
{
prof_ctx_t *ctx = prof_ctx_get(ptr);
prof_tctx_t *tctx = prof_tctx_get(ptr);
cassert(config_prof);
assert(usize == isalloc(ptr, true));
if ((uintptr_t)ctx > (uintptr_t)1) {
prof_thr_cnt_t *tcnt;
assert(size == isalloc(ptr, true));
tcnt = prof_lookup(&ctx->bt);
if (tcnt != NULL) {
tcnt->epoch++;
/*********/
mb_write();
/*********/
tcnt->cnts.curobjs--;
tcnt->cnts.curbytes -= size;
/*********/
mb_write();
/*********/
tcnt->epoch++;
/*********/
mb_write();
/*********/
} else {
/*
* OOM during free() cannot be propagated, so operate
* directly on cnt->ctx->cnt_merged.
*/
malloc_mutex_lock(ctx->lock);
ctx->cnt_merged.curobjs--;
ctx->cnt_merged.curbytes -= size;
malloc_mutex_unlock(ctx->lock);
}
}
if ((uintptr_t)tctx > (uintptr_t)1U)
prof_free_sampled_object(usize, tctx);
}
#endif

97
src/ctl.c
View File

@ -68,6 +68,8 @@ CTL_PROTO(version)
CTL_PROTO(epoch)
CTL_PROTO(thread_tcache_enabled)
CTL_PROTO(thread_tcache_flush)
CTL_PROTO(thread_prof_name)
CTL_PROTO(thread_prof_active)
CTL_PROTO(thread_arena)
CTL_PROTO(thread_allocated)
CTL_PROTO(thread_allocatedp)
@ -132,7 +134,9 @@ CTL_PROTO(arenas_nlruns)
CTL_PROTO(arenas_extend)
CTL_PROTO(prof_active)
CTL_PROTO(prof_dump)
CTL_PROTO(prof_reset)
CTL_PROTO(prof_interval)
CTL_PROTO(lg_prof_sample)
CTL_PROTO(stats_chunks_current)
CTL_PROTO(stats_chunks_total)
CTL_PROTO(stats_chunks_high)
@ -196,18 +200,24 @@ CTL_PROTO(stats_mapped)
*/
#define INDEX(i) {false}, i##_index
static const ctl_named_node_t tcache_node[] = {
static const ctl_named_node_t thread_tcache_node[] = {
{NAME("enabled"), CTL(thread_tcache_enabled)},
{NAME("flush"), CTL(thread_tcache_flush)}
};
static const ctl_named_node_t thread_prof_node[] = {
{NAME("name"), CTL(thread_prof_name)},
{NAME("active"), CTL(thread_prof_active)}
};
static const ctl_named_node_t thread_node[] = {
{NAME("arena"), CTL(thread_arena)},
{NAME("allocated"), CTL(thread_allocated)},
{NAME("allocatedp"), CTL(thread_allocatedp)},
{NAME("deallocated"), CTL(thread_deallocated)},
{NAME("deallocatedp"), CTL(thread_deallocatedp)},
{NAME("tcache"), CHILD(named, tcache)}
{NAME("tcache"), CHILD(named, thread_tcache)},
{NAME("prof"), CHILD(named, thread_prof)}
};
static const ctl_named_node_t config_node[] = {
@ -311,7 +321,9 @@ static const ctl_named_node_t arenas_node[] = {
static const ctl_named_node_t prof_node[] = {
{NAME("active"), CTL(prof_active)},
{NAME("dump"), CTL(prof_dump)},
{NAME("interval"), CTL(prof_interval)}
{NAME("reset"), CTL(prof_reset)},
{NAME("interval"), CTL(prof_interval)},
{NAME("lg_sample"), CTL(lg_prof_sample)}
};
static const ctl_named_node_t stats_chunks_node[] = {
@ -1281,6 +1293,62 @@ label_return:
return (ret);
}
static int
thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
const char *oldname;
if (config_prof == false)
return (ENOENT);
oldname = prof_thread_name_get();
if (newp != NULL) {
if (newlen != sizeof(const char *)) {
ret = EINVAL;
goto label_return;
}
if (prof_thread_name_set(*(const char **)newp)) {
ret = EAGAIN;
goto label_return;
}
}
READ(oldname, const char *);
ret = 0;
label_return:
return (ret);
}
static int
thread_prof_active_ctl(const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
bool oldval;
if (config_prof == false)
return (ENOENT);
oldval = prof_thread_active_get();
if (newp != NULL) {
if (newlen != sizeof(bool)) {
ret = EINVAL;
goto label_return;
}
if (prof_thread_active_set(*(bool *)newp)) {
ret = EAGAIN;
goto label_return;
}
}
READ(oldval, bool);
ret = 0;
label_return:
return (ret);
}
/******************************************************************************/
/* ctl_mutex must be held during execution of this function. */
@ -1601,7 +1669,30 @@ label_return:
return (ret);
}
static int
prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
void *newp, size_t newlen)
{
int ret;
size_t lg_sample = lg_prof_sample;
if (config_prof == false)
return (ENOENT);
WRITEONLY();
WRITE(lg_sample, size_t);
if (lg_sample >= (sizeof(uint64_t) << 3))
lg_sample = (sizeof(uint64_t) << 3) - 1;
prof_reset(lg_sample);
ret = 0;
label_return:
return (ret);
}
CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t)
CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t)
/******************************************************************************/

12
src/huge.c
View File

@ -197,10 +197,10 @@ huge_salloc(const void *ptr)
return (ret);
}
prof_ctx_t *
huge_prof_ctx_get(const void *ptr)
prof_tctx_t *
huge_prof_tctx_get(const void *ptr)
{
prof_ctx_t *ret;
prof_tctx_t *ret;
extent_node_t *node, key;
malloc_mutex_lock(&huge_mtx);
@ -210,7 +210,7 @@ huge_prof_ctx_get(const void *ptr)
node = extent_tree_ad_search(&huge, &key);
assert(node != NULL);
ret = node->prof_ctx;
ret = node->prof_tctx;
malloc_mutex_unlock(&huge_mtx);
@ -218,7 +218,7 @@ huge_prof_ctx_get(const void *ptr)
}
void
huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
{
extent_node_t *node, key;
@ -229,7 +229,7 @@ huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
node = extent_tree_ad_search(&huge, &key);
assert(node != NULL);
node->prof_ctx = ctx;
node->prof_tctx = tctx;
malloc_mutex_unlock(&huge_mtx);
}

140
src/jemalloc.c
View File

@ -636,9 +636,9 @@ malloc_conf_init(void)
"prof_prefix", "jeprof")
CONF_HANDLE_BOOL(opt_prof_active, "prof_active",
true)
CONF_HANDLE_SSIZE_T(opt_lg_prof_sample,
CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
"lg_prof_sample", 0,
(sizeof(uint64_t) << 3) - 1)
(sizeof(uint64_t) << 3) - 1, true)
CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum",
true)
CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
@ -863,11 +863,11 @@ malloc_init_hard(void)
*/
static void *
imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
imalloc_prof_sample(size_t usize, prof_tctx_t *tctx)
{
void *p;
if (cnt == NULL)
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = imalloc(SMALL_MAXCLASS+1);
@ -884,16 +884,16 @@ JEMALLOC_ALWAYS_INLINE_C void *
imalloc_prof(size_t usize)
{
void *p;
prof_thr_cnt_t *cnt;
prof_tctx_t *tctx;
PROF_ALLOC_PREP(usize, cnt);
if ((uintptr_t)cnt != (uintptr_t)1U)
p = imalloc_prof_sample(usize, cnt);
tctx = prof_alloc_prep(usize);
if ((uintptr_t)tctx != (uintptr_t)1U)
p = imalloc_prof_sample(usize, tctx);
else
p = imalloc(usize);
if (p == NULL)
return (NULL);
prof_malloc(p, usize, cnt);
prof_malloc(p, usize, tctx);
return (p);
}
@ -943,11 +943,11 @@ je_malloc(size_t size)
}
static void *
imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx)
{
void *p;
if (cnt == NULL)
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0);
@ -963,17 +963,17 @@ imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
}
JEMALLOC_ALWAYS_INLINE_C void *
imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt)
imemalign_prof(size_t alignment, size_t usize, prof_tctx_t *tctx)
{
void *p;
if ((uintptr_t)cnt != (uintptr_t)1U)
p = imemalign_prof_sample(alignment, usize, cnt);
if ((uintptr_t)tctx != (uintptr_t)1U)
p = imemalign_prof_sample(alignment, usize, tctx);
else
p = ipalloc(usize, alignment, false);
if (p == NULL)
return (NULL);
prof_malloc(p, usize, cnt);
prof_malloc(p, usize, tctx);
return (p);
}
@ -1015,10 +1015,10 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
}
if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
prof_tctx_t *tctx;
PROF_ALLOC_PREP(usize, cnt);
result = imemalign_prof(alignment, usize, cnt);
tctx = prof_alloc_prep(usize);
result = imemalign_prof(alignment, usize, tctx);
} else
result = ipalloc(usize, alignment, false);
if (result == NULL)
@ -1070,11 +1070,11 @@ je_aligned_alloc(size_t alignment, size_t size)
}
static void *
icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
icalloc_prof_sample(size_t usize, prof_tctx_t *tctx)
{
void *p;
if (cnt == NULL)
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = icalloc(SMALL_MAXCLASS+1);
@ -1088,17 +1088,17 @@ icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt)
}
JEMALLOC_ALWAYS_INLINE_C void *
icalloc_prof(size_t usize, prof_thr_cnt_t *cnt)
icalloc_prof(size_t usize, prof_tctx_t *tctx)
{
void *p;
if ((uintptr_t)cnt != (uintptr_t)1U)
p = icalloc_prof_sample(usize, cnt);
if ((uintptr_t)tctx != (uintptr_t)1U)
p = icalloc_prof_sample(usize, tctx);
else
p = icalloc(usize);
if (p == NULL)
return (NULL);
prof_malloc(p, usize, cnt);
prof_malloc(p, usize, tctx);
return (p);
}
@ -1137,11 +1137,11 @@ je_calloc(size_t num, size_t size)
}
if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
prof_tctx_t *tctx;
usize = s2u(num_size);
PROF_ALLOC_PREP(usize, cnt);
ret = icalloc_prof(usize, cnt);
tctx = prof_alloc_prep(usize);
ret = icalloc_prof(usize, tctx);
} else {
if (config_stats || (config_valgrind && in_valgrind))
usize = s2u(num_size);
@ -1167,11 +1167,11 @@ label_return:
}
static void *
irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt)
irealloc_prof_sample(void *oldptr, size_t usize, prof_tctx_t *tctx)
{
void *p;
if (cnt == NULL)
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false);
@ -1185,19 +1185,19 @@ irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt)
}
JEMALLOC_ALWAYS_INLINE_C void *
irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt)
irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_tctx_t *tctx)
{
void *p;
prof_ctx_t *old_ctx;
prof_tctx_t *old_tctx;
old_ctx = prof_ctx_get(oldptr);
if ((uintptr_t)cnt != (uintptr_t)1U)
p = irealloc_prof_sample(oldptr, usize, cnt);
old_tctx = prof_tctx_get(oldptr);
if ((uintptr_t)tctx != (uintptr_t)1U)
p = irealloc_prof_sample(oldptr, usize, tctx);
else
p = iralloc(oldptr, usize, 0, 0, false);
if (p == NULL)
return (NULL);
prof_realloc(p, usize, cnt, old_usize, old_ctx);
prof_realloc(p, usize, tctx, old_usize, old_tctx);
return (p);
}
@ -1253,11 +1253,11 @@ je_realloc(void *ptr, size_t size)
old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
prof_tctx_t *tctx;
usize = s2u(size);
PROF_ALLOC_PREP(usize, cnt);
ret = irealloc_prof(ptr, old_usize, usize, cnt);
tctx = prof_alloc_prep(usize);
ret = irealloc_prof(ptr, old_usize, usize, tctx);
} else {
if (config_stats || (config_valgrind && in_valgrind))
usize = s2u(size);
@ -1379,11 +1379,11 @@ imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
static void *
imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena, prof_thr_cnt_t *cnt)
arena_t *arena, prof_tctx_t *tctx)
{
void *p;
if (cnt == NULL)
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
size_t usize_promoted = (alignment == 0) ?
@ -1402,18 +1402,18 @@ imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache,
JEMALLOC_ALWAYS_INLINE_C void *
imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache,
arena_t *arena, prof_thr_cnt_t *cnt)
arena_t *arena, prof_tctx_t *tctx)
{
void *p;
if ((uintptr_t)cnt != (uintptr_t)1U) {
if ((uintptr_t)tctx != (uintptr_t)1U) {
p = imallocx_prof_sample(usize, alignment, zero, try_tcache,
arena, cnt);
arena, tctx);
} else
p = imallocx(usize, alignment, zero, try_tcache, arena);
if (p == NULL)
return (NULL);
prof_malloc(p, usize, cnt);
prof_malloc(p, usize, tctx);
return (p);
}
@ -1447,11 +1447,11 @@ je_mallocx(size_t size, int flags)
assert(usize != 0);
if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
prof_tctx_t *tctx;
PROF_ALLOC_PREP(usize, cnt);
tctx = prof_alloc_prep(usize);
p = imallocx_prof(usize, alignment, zero, try_tcache, arena,
cnt);
tctx);
} else
p = imallocx(usize, alignment, zero, try_tcache, arena);
if (p == NULL)
@ -1476,11 +1476,11 @@ label_oom:
static void *
irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena,
prof_thr_cnt_t *cnt)
prof_tctx_t *tctx)
{
void *p;
if (cnt == NULL)
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
@ -1500,15 +1500,15 @@ irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize,
JEMALLOC_ALWAYS_INLINE_C void *
irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc,
arena_t *arena, prof_thr_cnt_t *cnt)
arena_t *arena, prof_tctx_t *tctx)
{
void *p;
prof_ctx_t *old_ctx;
prof_tctx_t *old_tctx;
old_ctx = prof_ctx_get(oldptr);
if ((uintptr_t)cnt != (uintptr_t)1U)
old_tctx = prof_tctx_get(oldptr);
if ((uintptr_t)tctx != (uintptr_t)1U)
p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero,
try_tcache_alloc, try_tcache_dalloc, arena, cnt);
try_tcache_alloc, try_tcache_dalloc, arena, tctx);
else {
p = iralloct(oldptr, size, 0, alignment, zero,
try_tcache_alloc, try_tcache_dalloc, arena);
@ -1527,7 +1527,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment,
*/
*usize = isalloc(p, config_prof);
}
prof_realloc(p, *usize, cnt, old_usize, old_ctx);
prof_realloc(p, *usize, tctx, old_usize, old_tctx);
return (p);
}
@ -1570,13 +1570,13 @@ je_rallocx(void *ptr, size_t size, int flags)
old_rzsize = u2rz(old_usize);
if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
prof_tctx_t *tctx;
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
assert(usize != 0);
PROF_ALLOC_PREP(usize, cnt);
tctx = prof_alloc_prep(usize);
p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero,
try_tcache_alloc, try_tcache_dalloc, arena, cnt);
try_tcache_alloc, try_tcache_dalloc, arena, tctx);
if (p == NULL)
goto label_oom;
} else {
@ -1623,11 +1623,11 @@ ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
static size_t
ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
size_t alignment, size_t max_usize, bool zero, arena_t *arena,
prof_thr_cnt_t *cnt)
prof_tctx_t *tctx)
{
size_t usize;
if (cnt == NULL)
if (tctx == NULL)
return (old_usize);
/* Use minimum usize to determine whether promotion may happen. */
if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <=
@ -1650,22 +1650,22 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
JEMALLOC_ALWAYS_INLINE_C size_t
ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra,
size_t alignment, size_t max_usize, bool zero, arena_t *arena,
prof_thr_cnt_t *cnt)
prof_tctx_t *tctx)
{
size_t usize;
prof_ctx_t *old_ctx;
prof_tctx_t *old_tctx;
old_ctx = prof_ctx_get(ptr);
if ((uintptr_t)cnt != (uintptr_t)1U) {
old_tctx = prof_tctx_get(ptr);
if ((uintptr_t)tctx != (uintptr_t)1U) {
usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
alignment, zero, max_usize, arena, cnt);
alignment, zero, max_usize, arena, tctx);
} else {
usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
zero, arena);
}
if (usize == old_usize)
return (usize);
prof_realloc(ptr, usize, cnt, old_usize, old_ctx);
prof_realloc(ptr, usize, tctx, old_usize, old_tctx);
return (usize);
}
@ -1697,19 +1697,19 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
old_rzsize = u2rz(old_usize);
if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
prof_tctx_t *tctx;
/*
* usize isn't knowable before ixalloc() returns when extra is
* non-zero. Therefore, compute its maximum possible value and
* use that in PROF_ALLOC_PREP() to decide whether to capture a
* use that in prof_alloc_prep() to decide whether to capture a
* backtrace. prof_realloc() will use the actual usize to
* decide whether to sample.
*/
size_t max_usize = (alignment == 0) ? s2u(size+extra) :
sa2u(size+extra, alignment);
PROF_ALLOC_PREP(max_usize, cnt);
tctx = prof_alloc_prep(max_usize);
usize = ixallocx_prof(ptr, old_usize, size, extra, alignment,
max_usize, zero, arena, cnt);
max_usize, zero, arena, tctx);
} else {
usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
zero, arena);

1159
src/prof.c
View File

File diff suppressed because it is too large Load Diff

2
src/stats.c
View File

@ -441,7 +441,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
}
if ((err = je_mallctl("opt.prof", &bv, &bsz, NULL, 0)) == 0 &&
bv) {
CTL_GET("opt.lg_prof_sample", &sv, size_t);
CTL_GET("prof.lg_sample", &sv, size_t);
malloc_cprintf(write_cb, cbopaque,
"Average profile sample interval: %"PRIu64
" (2^%zu)\n", (((uint64_t)1U) << sv), sv);