project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Stop depending on JEMALLOC_N() for function interception during testing.

Browse Source 
Instead, always define function pointers for interceptable functions,
but mark them const unless testing, so that the compiler can optimize
out the pointer dereferences.
This commit is contained in:
Jason Evans 2017-05-01 23:10:42 -07:00
parent b3b033eefd
commit a268af5085
13 changed files with 157 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/jemalloc/internal/arena_externs.h
View File

@ -59,12 +59,10 @@ void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes);
void arena_alloc_junk_small(void *ptr, const arena_bin_info_t *bin_info,
bool zero);
#ifdef JEMALLOC_JET
typedef void (arena_dalloc_junk_small_t)(void *, const arena_bin_info_t *);
extern arena_dalloc_junk_small_t *arena_dalloc_junk_small;
#else
void arena_dalloc_junk_small(void *ptr, const arena_bin_info_t *bin_info);
#endif
extern arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small;
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
szind_t ind, bool zero);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,

7
include/jemalloc/internal/jemalloc_internal_macros.h
View File

@ -30,4 +30,11 @@
# define restrict
#endif
/* Various function pointers are statick and immutable except during testing. */
#ifdef JEMALLOC_JET
# define JET_MUTABLE
#else
# define JET_MUTABLE const
#endif
#endif /* JEMALLOC_INTERNAL_MACROS_H */

20
include/jemalloc/internal/large_externs.h
View File

@ -2,21 +2,19 @@
#define JEMALLOC_INTERNAL_LARGE_EXTERNS_H
void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero);
void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero);
bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero);
void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
#ifdef JEMALLOC_JET
void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
size_t alignment, bool zero, tcache_t *tcache);
typedef void (large_dalloc_junk_t)(void *, size_t);
extern large_dalloc_junk_t *large_dalloc_junk;
extern large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk;
typedef void (large_dalloc_maybe_junk_t)(void *, size_t);
extern large_dalloc_maybe_junk_t *large_dalloc_maybe_junk;
#else
void large_dalloc_junk(void *ptr, size_t size);
void large_dalloc_maybe_junk(void *ptr, size_t size);
#endif
extern large_dalloc_maybe_junk_t *JET_MUTABLE large_dalloc_maybe_junk;
void large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent);
void large_dalloc_finish(tsdn_t *tsdn, extent_t *extent);
void large_dalloc(tsdn_t *tsdn, extent_t *extent);

14
include/jemalloc/internal/nstime.h
View File

@ -22,15 +22,11 @@ void nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
void nstime_imultiply(nstime_t *time, uint64_t multiplier);
void nstime_idivide(nstime_t *time, uint64_t divisor);
uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
#ifdef JEMALLOC_JET
typedef bool (nstime_monotonic_t)(void);
extern nstime_monotonic_t *nstime_monotonic;
typedef bool (nstime_update_t)(nstime_t *);
extern nstime_update_t *nstime_update;
#else
bool nstime_monotonic(void);
bool nstime_update(nstime_t *time);
#endif
typedef bool (nstime_monotonic_t)(void);
extern nstime_monotonic_t *JET_MUTABLE nstime_monotonic;
typedef bool (nstime_update_t)(nstime_t *);
extern nstime_update_t *JET_MUTABLE nstime_update;
#endif /* JEMALLOC_INTERNAL_NSTIME_H */

11
include/jemalloc/internal/prof_externs.h
View File

@ -50,12 +50,15 @@ prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
#ifdef JEMALLOC_JET
size_t prof_tdata_count(void);
size_t prof_bt_count(void);
#endif
typedef int (prof_dump_open_t)(bool, const char *);
extern prof_dump_open_t *prof_dump_open;
extern prof_dump_open_t *JET_MUTABLE prof_dump_open;
typedef bool (prof_dump_header_t)(tsdn_t *, bool, const prof_cnt_t *);
extern prof_dump_header_t *prof_dump_header;
void prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes,
uint64_t *accumobjs, uint64_t *accumbytes);
extern prof_dump_header_t *JET_MUTABLE prof_dump_header;
#ifdef JEMALLOC_JET
void prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes, uint64_t *accumobjs,
uint64_t *accumbytes);
#endif
bool prof_accum_init(tsdn_t *tsdn, prof_accum_t *prof_accum);
void prof_idump(tsdn_t *tsdn);

14
include/jemalloc/internal/rtree_externs.h
View File

@ -24,15 +24,19 @@ static const rtree_level_t rtree_levels[] = {
};
bool rtree_new(rtree_t *rtree, bool zeroed);
#ifdef JEMALLOC_JET
typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
extern rtree_node_alloc_t *rtree_node_alloc;
extern rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc;
typedef rtree_leaf_elm_t *(rtree_leaf_alloc_t)(tsdn_t *, rtree_t *, size_t);
extern rtree_leaf_alloc_t *rtree_leaf_alloc;
extern rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc;
typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_node_elm_t *);
extern rtree_node_dalloc_t *rtree_node_dalloc;
extern rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc;
typedef void (rtree_leaf_dalloc_t)(tsdn_t *, rtree_t *, rtree_leaf_elm_t *);
extern rtree_leaf_dalloc_t *rtree_leaf_dalloc;
extern rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc;
#ifdef JEMALLOC_JET
void rtree_delete(tsdn_t *tsdn, rtree_t *rtree);
#endif
rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,

30
include/jemalloc/internal/witness_externs.h
View File

@ -3,33 +3,19 @@
void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
witness_comp_t *comp, void *opaque);
#ifdef JEMALLOC_JET
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
extern witness_lock_error_t *witness_lock_error;
#else
void witness_lock_error(const witness_list_t *witnesses,
const witness_t *witness);
#endif
#ifdef JEMALLOC_JET
extern witness_lock_error_t *JET_MUTABLE witness_lock_error;
typedef void (witness_owner_error_t)(const witness_t *);
extern witness_owner_error_t *witness_owner_error;
#else
void witness_owner_error(const witness_t *witness);
#endif
#ifdef JEMALLOC_JET
extern witness_owner_error_t *JET_MUTABLE witness_owner_error;
typedef void (witness_not_owner_error_t)(const witness_t *);
extern witness_not_owner_error_t *witness_not_owner_error;
#else
void witness_not_owner_error(const witness_t *witness);
#endif
#ifdef JEMALLOC_JET
extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error;
typedef void (witness_depth_error_t)(const witness_list_t *,
witness_rank_t rank_inclusive, unsigned depth);
extern witness_depth_error_t *witness_depth_error;
#else
void witness_depth_error(const witness_list_t *witnesses,
witness_rank_t rank_inclusive, unsigned depth);
#endif
extern witness_depth_error_t *JET_MUTABLE witness_depth_error;
void witnesses_cleanup(tsd_t *tsd);
void witness_prefork(tsd_t *tsd);

16
src/arena.c
View File

@ -1407,20 +1407,12 @@ arena_alloc_junk_small(void *ptr, const arena_bin_info_t *bin_info, bool zero) {
}
}
#ifdef JEMALLOC_JET
#undef arena_dalloc_junk_small
#define arena_dalloc_junk_small JEMALLOC_N(n_arena_dalloc_junk_small)
#endif
void
arena_dalloc_junk_small(void *ptr, const arena_bin_info_t *bin_info) {
static void
arena_dalloc_junk_small_impl(void *ptr, const arena_bin_info_t *bin_info) {
memset(ptr, JEMALLOC_FREE_JUNK, bin_info->reg_size);
}
#ifdef JEMALLOC_JET
#undef arena_dalloc_junk_small
#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
arena_dalloc_junk_small_t *arena_dalloc_junk_small =
JEMALLOC_N(n_arena_dalloc_junk_small);
#endif
arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small =
arena_dalloc_junk_small_impl;
static void *
arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {

30
src/large.c
View File

@ -68,26 +68,14 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
return extent_addr_get(extent);
}
#ifdef JEMALLOC_JET
#undef large_dalloc_junk
#define large_dalloc_junk JEMALLOC_N(n_large_dalloc_junk)
#endif
void
large_dalloc_junk(void *ptr, size_t size) {
static void
large_dalloc_junk_impl(void *ptr, size_t size) {
memset(ptr, JEMALLOC_FREE_JUNK, size);
}
#ifdef JEMALLOC_JET
#undef large_dalloc_junk
#define large_dalloc_junk JEMALLOC_N(large_dalloc_junk)
large_dalloc_junk_t *large_dalloc_junk = JEMALLOC_N(n_large_dalloc_junk);
#endif
large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk = large_dalloc_junk_impl;
#ifdef JEMALLOC_JET
#undef large_dalloc_maybe_junk
#define large_dalloc_maybe_junk JEMALLOC_N(n_large_dalloc_maybe_junk)
#endif
void
large_dalloc_maybe_junk(void *ptr, size_t size) {
static void
large_dalloc_maybe_junk_impl(void *ptr, size_t size) {
if (config_fill && have_dss && unlikely(opt_junk_free)) {
/*
* Only bother junk filling if the extent isn't about to be
@ -98,12 +86,8 @@ large_dalloc_maybe_junk(void *ptr, size_t size) {
}
}
}
#ifdef JEMALLOC_JET
#undef large_dalloc_maybe_junk
#define large_dalloc_maybe_junk JEMALLOC_N(large_dalloc_maybe_junk)
large_dalloc_maybe_junk_t *large_dalloc_maybe_junk =
JEMALLOC_N(n_large_dalloc_maybe_junk);
#endif
large_dalloc_maybe_junk_t *JET_MUTABLE large_dalloc_maybe_junk =
large_dalloc_maybe_junk_impl;
static bool
large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) {

28
src/nstime.c
View File

@ -131,27 +131,15 @@ nstime_get(nstime_t *time) {
}
#endif
#ifdef JEMALLOC_JET
#undef nstime_monotonic
#define nstime_monotonic JEMALLOC_N(n_nstime_monotonic)
#endif
bool
nstime_monotonic(void) {
static bool
nstime_monotonic_impl(void) {
return NSTIME_MONOTONIC;
#undef NSTIME_MONOTONIC
}
#ifdef JEMALLOC_JET
#undef nstime_monotonic
#define nstime_monotonic JEMALLOC_N(nstime_monotonic)
nstime_monotonic_t *nstime_monotonic = JEMALLOC_N(n_nstime_monotonic);
#endif
nstime_monotonic_t *JET_MUTABLE nstime_monotonic = nstime_monotonic_impl;
#ifdef JEMALLOC_JET
#undef nstime_update
#define nstime_update JEMALLOC_N(n_nstime_update)
#endif
bool
nstime_update(nstime_t *time) {
static bool
nstime_update_impl(nstime_t *time) {
nstime_t old_time;
nstime_copy(&old_time, time);
@ -165,8 +153,4 @@ nstime_update(nstime_t *time) {
return false;
}
#ifdef JEMALLOC_JET
#undef nstime_update
#define nstime_update JEMALLOC_N(nstime_update)
nstime_update_t *nstime_update = JEMALLOC_N(n_nstime_update);
#endif
nstime_update_t *JET_MUTABLE nstime_update = nstime_update_impl;

27
src/prof.c
View File

@ -932,9 +932,7 @@ prof_tdata_count(void) {
return tdata_count;
}
#endif
#ifdef JEMALLOC_JET
size_t
prof_bt_count(void) {
size_t bt_count;
@ -955,12 +953,8 @@ prof_bt_count(void) {
}
#endif
#ifdef JEMALLOC_JET
#undef prof_dump_open
#define prof_dump_open JEMALLOC_N(prof_dump_open_impl)
#endif
static int
prof_dump_open(bool propagate_err, const char *filename) {
prof_dump_open_impl(bool propagate_err, const char *filename) {
int fd;
fd = creat(filename, 0644);
@ -974,11 +968,7 @@ prof_dump_open(bool propagate_err, const char *filename) {
return fd;
}
#ifdef JEMALLOC_JET
#undef prof_dump_open
#define prof_dump_open JEMALLOC_N(prof_dump_open)
prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl);
#endif
prof_dump_open_t *JET_MUTABLE prof_dump_open = prof_dump_open_impl;
static bool
prof_dump_flush(bool propagate_err) {
@ -1331,12 +1321,9 @@ prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
return NULL;
}
#ifdef JEMALLOC_JET
#undef prof_dump_header
#define prof_dump_header JEMALLOC_N(prof_dump_header_impl)
#endif
static bool
prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) {
prof_dump_header_impl(tsdn_t *tsdn, bool propagate_err,
const prof_cnt_t *cnt_all) {
bool ret;
if (prof_dump_printf(propagate_err,
@ -1353,11 +1340,7 @@ prof_dump_header(tsdn_t *tsdn, bool propagate_err, const prof_cnt_t *cnt_all) {
malloc_mutex_unlock(tsdn, &tdatas_mtx);
return ret;
}
#ifdef JEMALLOC_JET
#undef prof_dump_header
#define prof_dump_header JEMALLOC_N(prof_dump_header)
prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
#endif
prof_dump_header_t *JET_MUTABLE prof_dump_header = prof_dump_header_impl;
static bool
prof_dump_gctx(tsdn_t *tsdn, bool propagate_err, prof_gctx_t *gctx,

54
src/rtree.c
View File

@ -25,65 +25,35 @@ rtree_new(rtree_t *rtree, bool zeroed) {
return false;
}
#ifdef JEMALLOC_JET
#undef rtree_node_alloc
#define rtree_node_alloc JEMALLOC_N(rtree_node_alloc_impl)
#endif
static rtree_node_elm_t *
rtree_node_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
rtree_node_alloc_impl(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
return (rtree_node_elm_t *)base_alloc(tsdn, b0get(), nelms *
sizeof(rtree_node_elm_t), CACHELINE);
}
#ifdef JEMALLOC_JET
#undef rtree_node_alloc
#define rtree_node_alloc JEMALLOC_N(rtree_node_alloc)
rtree_node_alloc_t *rtree_node_alloc = JEMALLOC_N(rtree_node_alloc_impl);
#endif
rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc = rtree_node_alloc_impl;
#ifdef JEMALLOC_JET
#undef rtree_node_dalloc
#define rtree_node_dalloc JEMALLOC_N(rtree_node_dalloc_impl)
#endif
UNUSED static void
rtree_node_dalloc(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) {
static void
rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) {
/* Nodes are never deleted during normal operation. */
not_reached();
}
#ifdef JEMALLOC_JET
#undef rtree_node_dalloc
#define rtree_node_dalloc JEMALLOC_N(rtree_node_dalloc)
rtree_node_dalloc_t *rtree_node_dalloc = JEMALLOC_N(rtree_node_dalloc_impl);
#endif
UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc =
rtree_node_dalloc_impl;
#ifdef JEMALLOC_JET
#undef rtree_leaf_alloc
#define rtree_leaf_alloc JEMALLOC_N(rtree_leaf_alloc_impl)
#endif
static rtree_leaf_elm_t *
rtree_leaf_alloc(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
rtree_leaf_alloc_impl(tsdn_t *tsdn, rtree_t *rtree, size_t nelms) {
return (rtree_leaf_elm_t *)base_alloc(tsdn, b0get(), nelms *
sizeof(rtree_leaf_elm_t), CACHELINE);
}
#ifdef JEMALLOC_JET
#undef rtree_leaf_alloc
#define rtree_leaf_alloc JEMALLOC_N(rtree_leaf_alloc)
rtree_leaf_alloc_t *rtree_leaf_alloc = JEMALLOC_N(rtree_leaf_alloc_impl);
#endif
rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc = rtree_leaf_alloc_impl;
#ifdef JEMALLOC_JET
#undef rtree_leaf_dalloc
#define rtree_leaf_dalloc JEMALLOC_N(rtree_leaf_dalloc_impl)
#endif
UNUSED static void
rtree_leaf_dalloc(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) {
static void
rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) {
/* Leaves are never deleted during normal operation. */
not_reached();
}
#ifdef JEMALLOC_JET
#undef rtree_leaf_dalloc
#define rtree_leaf_dalloc JEMALLOC_N(rtree_leaf_dalloc)
rtree_leaf_dalloc_t *rtree_leaf_dalloc = JEMALLOC_N(rtree_leaf_dalloc_impl);
#endif
UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc =
rtree_leaf_dalloc_impl;
#ifdef JEMALLOC_JET
# if RTREE_HEIGHT > 1

61
src/witness.c
View File

@ -14,12 +14,9 @@ witness_init(witness_t *witness, const char *name, witness_rank_t rank,
witness->opaque = opaque;
}
#ifdef JEMALLOC_JET
#undef witness_lock_error
#define witness_lock_error JEMALLOC_N(n_witness_lock_error)
#endif
void
witness_lock_error(const witness_list_t *witnesses, const witness_t *witness) {
static void
witness_lock_error_impl(const witness_list_t *witnesses,
const witness_t *witness) {
witness_t *w;
malloc_printf("<jemalloc>: Lock rank order reversal:");
@ -29,51 +26,28 @@ witness_lock_error(const witness_list_t *witnesses, const witness_t *witness) {
malloc_printf(" %s(%u)\n", witness->name, witness->rank);
abort();
}
#ifdef JEMALLOC_JET
#undef witness_lock_error
#define witness_lock_error JEMALLOC_N(witness_lock_error)
witness_lock_error_t *witness_lock_error = JEMALLOC_N(n_witness_lock_error);
#endif
witness_lock_error_t *JET_MUTABLE witness_lock_error = witness_lock_error_impl;
#ifdef JEMALLOC_JET
#undef witness_owner_error
#define witness_owner_error JEMALLOC_N(n_witness_owner_error)
#endif
void
witness_owner_error(const witness_t *witness) {
static void
witness_owner_error_impl(const witness_t *witness) {
malloc_printf("<jemalloc>: Should own %s(%u)\n", witness->name,
witness->rank);
abort();
}
#ifdef JEMALLOC_JET
#undef witness_owner_error
#define witness_owner_error JEMALLOC_N(witness_owner_error)
witness_owner_error_t *witness_owner_error = JEMALLOC_N(n_witness_owner_error);
#endif
witness_owner_error_t *JET_MUTABLE witness_owner_error =
witness_owner_error_impl;
#ifdef JEMALLOC_JET
#undef witness_not_owner_error
#define witness_not_owner_error JEMALLOC_N(n_witness_not_owner_error)
#endif
void
witness_not_owner_error(const witness_t *witness) {
static void
witness_not_owner_error_impl(const witness_t *witness) {
malloc_printf("<jemalloc>: Should not own %s(%u)\n", witness->name,
witness->rank);
abort();
}
#ifdef JEMALLOC_JET
#undef witness_not_owner_error
#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error)
witness_not_owner_error_t *witness_not_owner_error =
JEMALLOC_N(n_witness_not_owner_error);
#endif
witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error =
witness_not_owner_error_impl;
#ifdef JEMALLOC_JET
#undef witness_depth_error
#define witness_depth_error JEMALLOC_N(n_witness_depth_error)
#endif
void
witness_depth_error(const witness_list_t *witnesses,
static void
witness_depth_error_impl(const witness_list_t *witnesses,
witness_rank_t rank_inclusive, unsigned depth) {
witness_t *w;
@ -85,11 +59,8 @@ witness_depth_error(const witness_list_t *witnesses,
malloc_printf("\n");
abort();
}
#ifdef JEMALLOC_JET
#undef witness_depth_error
#define witness_depth_error JEMALLOC_N(witness_depth_error)
witness_depth_error_t *witness_depth_error = JEMALLOC_N(n_witness_depth_error);
#endif
witness_depth_error_t *JET_MUTABLE witness_depth_error =
witness_depth_error_impl;
void
witnesses_cleanup(tsd_t *tsd) {