Add "hook" module.
The hook module allows a low-reader-overhead way of finding hooks to invoke and calling them. For now, none of the allocation pathways are tied into the hooks; this will come later.
This commit is contained in:
David Goldblatt
David Goldblatt
parent
06a8c40b36
commit
5ae6e7cbfa
@ -102,6 +102,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
|
||||
$(srcroot)src/extent_dss.c \
|
||||
$(srcroot)src/extent_mmap.c \
|
||||
$(srcroot)src/hash.c \
|
||||
$(srcroot)src/hook.c \
|
||||
$(srcroot)src/large.c \
|
||||
$(srcroot)src/log.c \
|
||||
$(srcroot)src/malloc_io.c \
|
||||
@ -172,6 +173,7 @@ TESTS_UNIT := \
|
||||
$(srcroot)test/unit/extent_quantize.c \
|
||||
$(srcroot)test/unit/fork.c \
|
||||
$(srcroot)test/unit/hash.c \
|
||||
$(srcroot)test/unit/hook.c \
|
||||
$(srcroot)test/unit/junk.c \
|
||||
$(srcroot)test/unit/junk_alloc.c \
|
||||
$(srcroot)test/unit/junk_free.c \
|
||||
|
||||
125
include/jemalloc/internal/hook.h
Normal file
125
include/jemalloc/internal/hook.h
Normal file
@ -0,0 +1,125 @@
|
||||
#ifndef JEMALLOC_INTERNAL_HOOK_H
|
||||
#define JEMALLOC_INTERNAL_HOOK_H
|
||||
|
||||
#include "jemalloc/internal/tsd.h"
|
||||
|
||||
/*
|
||||
* This API is *extremely* experimental, and may get ripped out, changed in API-
|
||||
* and ABI-incompatible ways, be insufficiently or incorrectly documented, etc.
|
||||
*
|
||||
* It allows hooking the stateful parts of the API to see changes as they
|
||||
* happen.
|
||||
*
|
||||
* Allocation hooks are called after the allocation is done, free hooks are
|
||||
* called before the free is done, and expand hooks are called after the
|
||||
* allocation is expanded.
|
||||
*
|
||||
* For realloc and rallocx, if the expansion happens in place, the expansion
|
||||
* hook is called. If it is moved, then the alloc hook is called on the new
|
||||
* location, and then the free hook is called on the old location.
|
||||
*
|
||||
* (We omit no-ops, like free(NULL), etc.).
|
||||
*
|
||||
* Reentrancy:
|
||||
* Is not protected against. If your hooks allocate, then the hooks will be
|
||||
* called again. Note that you can guard against this with a thread-local
|
||||
* "in_hook" bool.
|
||||
* Threading:
|
||||
* The installation of a hook synchronizes with all its uses. If you can
|
||||
* prove the installation of a hook happens-before a jemalloc entry point,
|
||||
* then the hook will get invoked (unless there's a racing removal).
|
||||
*
|
||||
* Hook insertion appears to be atomic at a per-thread level (i.e. if a thread
|
||||
* allocates and has the alloc hook invoked, then a subsequent free on the
|
||||
* same thread will also have the free hook invoked).
|
||||
*
|
||||
* The *removal* of a hook does *not* block until all threads are done with
|
||||
* the hook. Hook authors have to be resilient to this, and need some
|
||||
* out-of-band mechanism for cleaning up any dynamically allocated memory
|
||||
* associated with their hook.
|
||||
* Ordering:
|
||||
* Order of hook execution is unspecified, and may be different than insertion
|
||||
* order.
|
||||
*/
|
||||
|
||||
enum hook_alloc_e {
|
||||
hook_alloc_malloc,
|
||||
hook_alloc_posix_memalign,
|
||||
hook_alloc_aligned_alloc,
|
||||
hook_alloc_calloc,
|
||||
hook_alloc_memalign,
|
||||
hook_alloc_valloc,
|
||||
hook_alloc_mallocx,
|
||||
|
||||
/* The reallocating functions have both alloc and dalloc variants */
|
||||
hook_alloc_realloc,
|
||||
hook_alloc_rallocx,
|
||||
};
|
||||
/*
|
||||
* We put the enum typedef after the enum, since this file may get included by
|
||||
* jemalloc_cpp.cpp, and C++ disallows enum forward declarations.
|
||||
*/
|
||||
typedef enum hook_alloc_e hook_alloc_t;
|
||||
|
||||
enum hook_dalloc_e {
|
||||
hook_dalloc_free,
|
||||
hook_dalloc_dallocx,
|
||||
hook_dalloc_sdallocx,
|
||||
|
||||
/*
|
||||
* The dalloc halves of reallocation (not called if in-place expansion
|
||||
* happens).
|
||||
*/
|
||||
hook_dalloc_realloc,
|
||||
hook_dalloc_rallocx,
|
||||
};
|
||||
typedef enum hook_dalloc_e hook_dalloc_t;
|
||||
|
||||
|
||||
enum hook_expand_e {
|
||||
hook_expand_realloc,
|
||||
hook_expand_rallocx,
|
||||
hook_expand_xallocx,
|
||||
};
|
||||
typedef enum hook_expand_e hook_expand_t;
|
||||
|
||||
typedef void (*hook_alloc)(
|
||||
void *extra, hook_alloc_t type, void *result, uintptr_t result_raw,
|
||||
uintptr_t args_raw[3]);
|
||||
|
||||
typedef void (*hook_dalloc)(
|
||||
void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
|
||||
|
||||
typedef void (*hook_expand)(
|
||||
void *extra, hook_expand_t type, void *address, size_t old_usize,
|
||||
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
|
||||
|
||||
typedef struct hooks_s hooks_t;
|
||||
struct hooks_s {
|
||||
hook_alloc alloc_hook;
|
||||
hook_dalloc dalloc_hook;
|
||||
hook_expand expand_hook;
|
||||
};
|
||||
|
||||
/*
|
||||
* Returns an opaque handle to be used when removing the hook. NULL means that
|
||||
* we couldn't install the hook.
|
||||
*/
|
||||
bool hook_boot();
|
||||
|
||||
void *hook_install(tsdn_t *tsdn, hooks_t *hooks, void *extra);
|
||||
/* Uninstalls the hook with the handle previously returned from hook_install. */
|
||||
void hook_remove(tsdn_t *tsdn, void *opaque);
|
||||
|
||||
/* Hooks */
|
||||
|
||||
void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
|
||||
uintptr_t args_raw[3]);
|
||||
|
||||
void hook_invoke_dalloc(hook_dalloc_t type, void *address,
|
||||
uintptr_t args_raw[3]);
|
||||
|
||||
void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
|
||||
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_HOOK_H */
|
||||
@ -49,6 +49,7 @@
|
||||
#define WITNESS_RANK_RTREE 17U
|
||||
#define WITNESS_RANK_BASE 18U
|
||||
#define WITNESS_RANK_ARENA_LARGE 19U
|
||||
#define WITNESS_RANK_HOOK 20U
|
||||
|
||||
#define WITNESS_RANK_LEAF 0xffffffffU
|
||||
#define WITNESS_RANK_BIN WITNESS_RANK_LEAF
|
||||
|
||||
133
src/hook.c
Normal file
133
src/hook.c
Normal file
@ -0,0 +1,133 @@
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
|
||||
#include "jemalloc/internal/hook.h"
|
||||
|
||||
#include "jemalloc/internal/atomic.h"
|
||||
#include "jemalloc/internal/mutex.h"
|
||||
#include "jemalloc/internal/seq.h"
|
||||
|
||||
typedef struct hooks_internal_s hooks_internal_t;
|
||||
struct hooks_internal_s {
|
||||
hooks_t hooks;
|
||||
void *extra;
|
||||
bool in_use;
|
||||
};
|
||||
|
||||
seq_define(hooks_internal_t, hooks)
|
||||
|
||||
#define HOOKS_MAX 4
|
||||
static seq_hooks_t hooks[HOOKS_MAX];
|
||||
static malloc_mutex_t hooks_mu;
|
||||
|
||||
bool
|
||||
hook_boot() {
|
||||
return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK,
|
||||
malloc_mutex_rank_exclusive);
|
||||
}
|
||||
|
||||
static void *
|
||||
hook_install_locked(hooks_t *to_install, void *extra) {
|
||||
hooks_internal_t hooks_internal;
|
||||
for (int i = 0; i < HOOKS_MAX; i++) {
|
||||
bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]);
|
||||
/* We hold mu; no concurrent access. */
|
||||
assert(success);
|
||||
if (!hooks_internal.in_use) {
|
||||
hooks_internal.hooks = *to_install;
|
||||
hooks_internal.extra = extra;
|
||||
hooks_internal.in_use = true;
|
||||
seq_store_hooks(&hooks[i], &hooks_internal);
|
||||
return &hooks[i];
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *
|
||||
hook_install(tsdn_t *tsdn, hooks_t *to_install, void *extra) {
|
||||
malloc_mutex_lock(tsdn, &hooks_mu);
|
||||
void *ret = hook_install_locked(to_install, extra);
|
||||
if (ret != NULL) {
|
||||
tsd_global_slow_inc(tsdn);
|
||||
}
|
||||
malloc_mutex_unlock(tsdn, &hooks_mu);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void
|
||||
hook_remove_locked(seq_hooks_t *to_remove) {
|
||||
hooks_internal_t hooks_internal;
|
||||
bool success = seq_try_load_hooks(&hooks_internal, to_remove);
|
||||
/* We hold mu; no concurrent access. */
|
||||
assert(success);
|
||||
/* Should only remove hooks that were added. */
|
||||
assert(hooks_internal.in_use);
|
||||
hooks_internal.in_use = false;
|
||||
seq_store_hooks(to_remove, &hooks_internal);
|
||||
}
|
||||
|
||||
void
|
||||
hook_remove(tsdn_t *tsdn, void *opaque) {
|
||||
if (config_debug) {
|
||||
char *hooks_begin = (char *)&hooks[0];
|
||||
char *hooks_end = (char *)&hooks[HOOKS_MAX];
|
||||
char *hook = (char *)opaque;
|
||||
assert(hooks_begin <= hook && hook < hooks_end
|
||||
&& (hook - hooks_begin) % sizeof(seq_hooks_t) == 0);
|
||||
}
|
||||
malloc_mutex_lock(tsdn, &hooks_mu);
|
||||
hook_remove_locked((seq_hooks_t *)opaque);
|
||||
tsd_global_slow_dec(tsdn);
|
||||
malloc_mutex_unlock(tsdn, &hooks_mu);
|
||||
}
|
||||
|
||||
#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \
|
||||
for (int for_each_hook_counter = 0; \
|
||||
for_each_hook_counter < HOOKS_MAX; \
|
||||
for_each_hook_counter++) { \
|
||||
bool for_each_hook_success = seq_try_load_hooks( \
|
||||
(hooks_internal_ptr), &hooks[for_each_hook_counter]); \
|
||||
if (!for_each_hook_success) { \
|
||||
continue; \
|
||||
} \
|
||||
if (!(hooks_internal_ptr)->in_use) { \
|
||||
continue; \
|
||||
}
|
||||
#define FOR_EACH_HOOK_END \
|
||||
}
|
||||
|
||||
void
|
||||
hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
|
||||
uintptr_t args_raw[3]) {
|
||||
hooks_internal_t hook;
|
||||
FOR_EACH_HOOK_BEGIN(&hook)
|
||||
hook_alloc h = hook.hooks.alloc_hook;
|
||||
if (h != NULL) {
|
||||
h(hook.extra, type, result, result_raw, args_raw);
|
||||
}
|
||||
FOR_EACH_HOOK_END
|
||||
}
|
||||
|
||||
void
|
||||
hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) {
|
||||
hooks_internal_t hook;
|
||||
FOR_EACH_HOOK_BEGIN(&hook)
|
||||
hook_dalloc h = hook.hooks.dalloc_hook;
|
||||
if (h != NULL) {
|
||||
h(hook.extra, type, address, args_raw);
|
||||
}
|
||||
FOR_EACH_HOOK_END
|
||||
}
|
||||
|
||||
void
|
||||
hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
|
||||
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) {
|
||||
hooks_internal_t hook;
|
||||
FOR_EACH_HOOK_BEGIN(&hook)
|
||||
hook_expand h = hook.hooks.expand_hook;
|
||||
if (h != NULL) {
|
||||
h(hook.extra, type, address, old_usize, new_usize,
|
||||
result_raw, args_raw);
|
||||
}
|
||||
FOR_EACH_HOOK_END
|
||||
}
|
||||
@ -7,6 +7,7 @@
|
||||
#include "jemalloc/internal/ctl.h"
|
||||
#include "jemalloc/internal/extent_dss.h"
|
||||
#include "jemalloc/internal/extent_mmap.h"
|
||||
#include "jemalloc/internal/hook.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_types.h"
|
||||
#include "jemalloc/internal/log.h"
|
||||
#include "jemalloc/internal/malloc_io.h"
|
||||
@ -1311,6 +1312,7 @@ malloc_init_hard_a0_locked() {
|
||||
malloc_mutex_rank_exclusive)) {
|
||||
return true;
|
||||
}
|
||||
hook_boot();
|
||||
/*
|
||||
* Create enough scaffolding to allow recursive allocation in
|
||||
* malloc_ncpus().
|
||||
|
||||
180
test/unit/hook.c
Normal file
180
test/unit/hook.c
Normal file
@ -0,0 +1,180 @@
|
||||
#include "test/jemalloc_test.h"
|
||||
|
||||
#include "jemalloc/internal/hook.h"
|
||||
|
||||
static void *arg_extra;
|
||||
static int arg_type;
|
||||
static void *arg_result;
|
||||
static void *arg_address;
|
||||
static size_t arg_old_usize;
|
||||
static size_t arg_new_usize;
|
||||
static uintptr_t arg_result_raw;
|
||||
static uintptr_t arg_args_raw[4];
|
||||
|
||||
static int call_count = 0;
|
||||
|
||||
static void
|
||||
reset_args() {
|
||||
arg_extra = NULL;
|
||||
arg_type = 12345;
|
||||
arg_result = NULL;
|
||||
arg_address = NULL;
|
||||
arg_old_usize = 0;
|
||||
arg_new_usize = 0;
|
||||
arg_result_raw = 0;
|
||||
memset(arg_args_raw, 77, sizeof(arg_args_raw));
|
||||
}
|
||||
|
||||
static void
|
||||
set_args_raw(uintptr_t *args_raw, int nargs) {
|
||||
memcpy(arg_args_raw, args_raw, sizeof(uintptr_t) * nargs);
|
||||
}
|
||||
|
||||
static void
|
||||
assert_args_raw(uintptr_t *args_raw_expected, int nargs) {
|
||||
int cmp = memcmp(args_raw_expected, arg_args_raw,
|
||||
sizeof(uintptr_t) * nargs);
|
||||
assert_d_eq(cmp, 0, "Raw args mismatch");
|
||||
}
|
||||
|
||||
static void
|
||||
test_alloc_hook(void *extra, hook_alloc_t type, void *result,
|
||||
uintptr_t result_raw, uintptr_t args_raw[3]) {
|
||||
call_count++;
|
||||
arg_extra = extra;
|
||||
arg_type = (int)type;
|
||||
arg_result = result;
|
||||
arg_result_raw = result_raw;
|
||||
set_args_raw(args_raw, 3);
|
||||
}
|
||||
|
||||
static void
|
||||
test_dalloc_hook(void *extra, hook_dalloc_t type, void *address,
|
||||
uintptr_t args_raw[3]) {
|
||||
call_count++;
|
||||
arg_extra = extra;
|
||||
arg_type = (int)type;
|
||||
arg_address = address;
|
||||
set_args_raw(args_raw, 3);
|
||||
}
|
||||
|
||||
static void
|
||||
test_expand_hook(void *extra, hook_expand_t type, void *address,
|
||||
size_t old_usize, size_t new_usize, uintptr_t result_raw,
|
||||
uintptr_t args_raw[4]) {
|
||||
call_count++;
|
||||
arg_extra = extra;
|
||||
arg_type = (int)type;
|
||||
arg_address = address;
|
||||
arg_old_usize = old_usize;
|
||||
arg_new_usize = new_usize;
|
||||
arg_result_raw = result_raw;
|
||||
set_args_raw(args_raw, 4);
|
||||
}
|
||||
|
||||
TEST_BEGIN(test_hooks_basic) {
|
||||
/* Just verify that the record their arguments correctly. */
|
||||
hooks_t hooks = {
|
||||
&test_alloc_hook, &test_dalloc_hook, &test_expand_hook};
|
||||
void *handle = hook_install(TSDN_NULL, &hooks, (void *)111);
|
||||
uintptr_t args_raw[4] = {10, 20, 30, 40};
|
||||
|
||||
/* Alloc */
|
||||
reset_args();
|
||||
hook_invoke_alloc(hook_alloc_posix_memalign, (void *)222, 333,
|
||||
args_raw);
|
||||
assert_ptr_eq(arg_extra, (void *)111, "Passed wrong user pointer");
|
||||
assert_d_eq((int)hook_alloc_posix_memalign, arg_type,
|
||||
"Passed wrong alloc type");
|
||||
assert_ptr_eq((void *)222, arg_result, "Passed wrong result address");
|
||||
assert_u64_eq(333, arg_result_raw, "Passed wrong result");
|
||||
assert_args_raw(args_raw, 3);
|
||||
|
||||
/* Dalloc */
|
||||
reset_args();
|
||||
hook_invoke_dalloc(hook_dalloc_sdallocx, (void *)222, args_raw);
|
||||
assert_d_eq((int)hook_dalloc_sdallocx, arg_type,
|
||||
"Passed wrong dalloc type");
|
||||
assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer");
|
||||
assert_ptr_eq((void *)222, arg_address, "Passed wrong address");
|
||||
assert_args_raw(args_raw, 3);
|
||||
|
||||
/* Expand */
|
||||
reset_args();
|
||||
hook_invoke_expand(hook_expand_xallocx, (void *)222, 333, 444, 555,
|
||||
args_raw);
|
||||
assert_d_eq((int)hook_expand_xallocx, arg_type,
|
||||
"Passed wrong expand type");
|
||||
assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer");
|
||||
assert_ptr_eq((void *)222, arg_address, "Passed wrong address");
|
||||
assert_zu_eq(333, arg_old_usize, "Passed wrong old usize");
|
||||
assert_zu_eq(444, arg_new_usize, "Passed wrong new usize");
|
||||
assert_zu_eq(555, arg_result_raw, "Passed wrong result");
|
||||
assert_args_raw(args_raw, 4);
|
||||
|
||||
hook_remove(TSDN_NULL, handle);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_hooks_null) {
|
||||
/* Null hooks should be ignored, not crash. */
|
||||
hooks_t hooks1 = {NULL, NULL, NULL};
|
||||
hooks_t hooks2 = {&test_alloc_hook, NULL, NULL};
|
||||
hooks_t hooks3 = {NULL, &test_dalloc_hook, NULL};
|
||||
hooks_t hooks4 = {NULL, NULL, &test_expand_hook};
|
||||
|
||||
void *handle1 = hook_install(TSDN_NULL, &hooks1, NULL);
|
||||
void *handle2 = hook_install(TSDN_NULL, &hooks2, NULL);
|
||||
void *handle3 = hook_install(TSDN_NULL, &hooks3, NULL);
|
||||
void *handle4 = hook_install(TSDN_NULL, &hooks4, NULL);
|
||||
|
||||
assert_ptr_ne(handle1, NULL, "Hook installation failed");
|
||||
assert_ptr_ne(handle2, NULL, "Hook installation failed");
|
||||
assert_ptr_ne(handle3, NULL, "Hook installation failed");
|
||||
assert_ptr_ne(handle4, NULL, "Hook installation failed");
|
||||
|
||||
uintptr_t args_raw[4] = {10, 20, 30, 40};
|
||||
|
||||
call_count = 0;
|
||||
hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw);
|
||||
assert_d_eq(call_count, 1, "Called wrong number of times");
|
||||
|
||||
call_count = 0;
|
||||
hook_invoke_dalloc(hook_dalloc_free, NULL, args_raw);
|
||||
assert_d_eq(call_count, 1, "Called wrong number of times");
|
||||
|
||||
call_count = 0;
|
||||
hook_invoke_expand(hook_expand_realloc, NULL, 0, 0, 0, args_raw);
|
||||
assert_d_eq(call_count, 1, "Called wrong number of times");
|
||||
|
||||
hook_remove(TSDN_NULL, handle1);
|
||||
hook_remove(TSDN_NULL, handle2);
|
||||
hook_remove(TSDN_NULL, handle3);
|
||||
hook_remove(TSDN_NULL, handle4);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_hooks_remove) {
|
||||
hooks_t hooks = {&test_alloc_hook, NULL, NULL};
|
||||
void *handle = hook_install(TSDN_NULL, &hooks, NULL);
|
||||
assert_ptr_ne(handle, NULL, "Hook installation failed");
|
||||
call_count = 0;
|
||||
uintptr_t args_raw[4] = {10, 20, 30, 40};
|
||||
hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw);
|
||||
assert_d_eq(call_count, 1, "Hook not invoked");
|
||||
|
||||
call_count = 0;
|
||||
hook_remove(TSDN_NULL, handle);
|
||||
hook_invoke_alloc(hook_alloc_malloc, NULL, 0, NULL);
|
||||
assert_d_eq(call_count, 0, "Hook invoked after removal");
|
||||
|
||||
}
|
||||
TEST_END
|
||||
|
||||
int
|
||||
main(void) {
|
||||
return test(
|
||||
test_hooks_basic,
|
||||
test_hooks_null,
|
||||
test_hooks_remove);
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user