f8f0542194
This facilitates the application accessing its own extent allocator metadata during hook invocations. This resolves #259.
427 lines
11 KiB
C
427 lines
11 KiB
C
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_TYPES
|
|
|
|
typedef struct extent_s extent_t;
|
|
|
|
#define EXTENT_HOOKS_INITIALIZER NULL
|
|
|
|
#endif /* JEMALLOC_H_TYPES */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_STRUCTS
|
|
|
|
/* Extent (span of pages). Use accessor functions for e_* fields. */
|
|
struct extent_s {
|
|
/* Arena from which this extent came, if any. */
|
|
arena_t *e_arena;
|
|
|
|
/* Pointer to the extent that this structure is responsible for. */
|
|
void *e_addr;
|
|
|
|
/* Extent size. */
|
|
size_t e_size;
|
|
|
|
/*
|
|
* Usable size, typically smaller than extent size due to large_pad or
|
|
* promotion of sampled small regions.
|
|
*/
|
|
size_t e_usize;
|
|
|
|
/* True if extent is active (in use). */
|
|
bool e_active;
|
|
|
|
/*
|
|
* The zeroed flag is used by extent recycling code to track whether
|
|
* memory is zero-filled.
|
|
*/
|
|
bool e_zeroed;
|
|
|
|
/*
|
|
* True if physical memory is committed to the extent, whether
|
|
* explicitly or implicitly as on a system that overcommits and
|
|
* satisfies physical memory needs on demand via soft page faults.
|
|
*/
|
|
bool e_committed;
|
|
|
|
/*
|
|
* The slab flag indicates whether the extent is used for a slab of
|
|
* small regions. This helps differentiate small size classes, and it
|
|
* indicates whether interior pointers can be looked up via iealloc().
|
|
*/
|
|
bool e_slab;
|
|
|
|
union {
|
|
/* Small region slab metadata. */
|
|
arena_slab_data_t e_slab_data;
|
|
|
|
/* Profile counters, used for large objects. */
|
|
union {
|
|
void *e_prof_tctx_pun;
|
|
prof_tctx_t *e_prof_tctx;
|
|
};
|
|
};
|
|
|
|
/*
|
|
* Linkage for arena's extents_dirty and arena_bin_t's slabs_full rings.
|
|
*/
|
|
qr(extent_t) qr_link;
|
|
|
|
union {
|
|
/* Linkage for per size class address-ordered heaps. */
|
|
phn(extent_t) ph_link;
|
|
|
|
/* Linkage for arena's large and extent_cache lists. */
|
|
ql_elm(extent_t) ql_link;
|
|
};
|
|
};
|
|
typedef ph(extent_t) extent_heap_t;
|
|
|
|
#endif /* JEMALLOC_H_STRUCTS */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_EXTERNS
|
|
|
|
extern rtree_t extents_rtree;
|
|
extern const extent_hooks_t extent_hooks_default;
|
|
|
|
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
|
|
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
|
|
|
|
extent_hooks_t *extent_hooks_get(arena_t *arena);
|
|
extent_hooks_t *extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks);
|
|
|
|
#ifdef JEMALLOC_JET
|
|
typedef size_t (extent_size_quantize_t)(size_t);
|
|
extern extent_size_quantize_t *extent_size_quantize_floor;
|
|
extern extent_size_quantize_t *extent_size_quantize_ceil;
|
|
#else
|
|
size_t extent_size_quantize_floor(size_t size);
|
|
size_t extent_size_quantize_ceil(size_t size);
|
|
#endif
|
|
|
|
ph_proto(, extent_heap_, extent_heap_t, extent_t)
|
|
|
|
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
|
|
size_t alignment, bool *zero, bool slab);
|
|
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
|
|
size_t alignment, bool *zero, bool *commit, bool slab);
|
|
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent);
|
|
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent);
|
|
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
|
size_t length);
|
|
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
|
size_t length);
|
|
bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
|
size_t length);
|
|
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
|
|
size_t usize_a, size_t size_b, size_t usize_b);
|
|
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b);
|
|
void extent_prefork(tsdn_t *tsdn);
|
|
void extent_postfork_parent(tsdn_t *tsdn);
|
|
void extent_postfork_child(tsdn_t *tsdn);
|
|
|
|
bool extent_boot(void);
|
|
|
|
#endif /* JEMALLOC_H_EXTERNS */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_INLINES
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
extent_t *extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent);
|
|
arena_t *extent_arena_get(const extent_t *extent);
|
|
void *extent_base_get(const extent_t *extent);
|
|
void *extent_addr_get(const extent_t *extent);
|
|
size_t extent_size_get(const extent_t *extent);
|
|
size_t extent_usize_get(const extent_t *extent);
|
|
void *extent_before_get(const extent_t *extent);
|
|
void *extent_last_get(const extent_t *extent);
|
|
void *extent_past_get(const extent_t *extent);
|
|
bool extent_active_get(const extent_t *extent);
|
|
bool extent_retained_get(const extent_t *extent);
|
|
bool extent_zeroed_get(const extent_t *extent);
|
|
bool extent_committed_get(const extent_t *extent);
|
|
bool extent_slab_get(const extent_t *extent);
|
|
arena_slab_data_t *extent_slab_data_get(extent_t *extent);
|
|
const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent);
|
|
prof_tctx_t *extent_prof_tctx_get(const extent_t *extent);
|
|
void extent_arena_set(extent_t *extent, arena_t *arena);
|
|
void extent_addr_set(extent_t *extent, void *addr);
|
|
void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
|
|
void extent_size_set(extent_t *extent, size_t size);
|
|
void extent_usize_set(extent_t *extent, size_t usize);
|
|
void extent_active_set(extent_t *extent, bool active);
|
|
void extent_zeroed_set(extent_t *extent, bool zeroed);
|
|
void extent_committed_set(extent_t *extent, bool committed);
|
|
void extent_slab_set(extent_t *extent, bool slab);
|
|
void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
|
|
void extent_init(extent_t *extent, arena_t *arena, void *addr,
|
|
size_t size, size_t usize, bool active, bool zeroed, bool committed,
|
|
bool slab);
|
|
void extent_ring_insert(extent_t *sentinel, extent_t *extent);
|
|
void extent_ring_remove(extent_t *extent);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
|
|
JEMALLOC_INLINE extent_t *
|
|
extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent)
|
|
{
|
|
rtree_ctx_t rtree_ctx_fallback;
|
|
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
|
|
|
|
return (rtree_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
|
|
dependent));
|
|
}
|
|
|
|
JEMALLOC_INLINE arena_t *
|
|
extent_arena_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_arena);
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_base_get(const extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
|
|
!extent->e_slab);
|
|
return (PAGE_ADDR2BASE(extent->e_addr));
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_addr_get(const extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
|
|
!extent->e_slab);
|
|
return (extent->e_addr);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
extent_size_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_size);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
extent_usize_get(const extent_t *extent)
|
|
{
|
|
|
|
assert(!extent->e_slab);
|
|
return (extent->e_usize);
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_before_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((void *)(uintptr_t)extent->e_addr - PAGE);
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_last_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((void *)(uintptr_t)extent->e_addr + extent_size_get(extent) -
|
|
PAGE);
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_past_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((void *)(uintptr_t)extent->e_addr + extent_size_get(extent));
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_active_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_active);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_retained_get(const extent_t *extent)
|
|
{
|
|
|
|
return (qr_next(extent, qr_link) == extent);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_zeroed_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_zeroed);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_committed_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_committed);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_slab_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_slab);
|
|
}
|
|
|
|
JEMALLOC_INLINE arena_slab_data_t *
|
|
extent_slab_data_get(extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_slab);
|
|
return (&extent->e_slab_data);
|
|
}
|
|
|
|
JEMALLOC_INLINE const arena_slab_data_t *
|
|
extent_slab_data_get_const(const extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_slab);
|
|
return (&extent->e_slab_data);
|
|
}
|
|
|
|
JEMALLOC_INLINE prof_tctx_t *
|
|
extent_prof_tctx_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((prof_tctx_t *)atomic_read_p(
|
|
&((extent_t *)extent)->e_prof_tctx_pun));
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_arena_set(extent_t *extent, arena_t *arena)
|
|
{
|
|
|
|
extent->e_arena = arena;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_addr_set(extent_t *extent, void *addr)
|
|
{
|
|
|
|
extent->e_addr = addr;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment)
|
|
{
|
|
|
|
assert(extent_base_get(extent) == extent_addr_get(extent));
|
|
|
|
if (alignment < PAGE) {
|
|
unsigned lg_range = LG_PAGE -
|
|
lg_floor(CACHELINE_CEILING(alignment));
|
|
uint64_t r =
|
|
prng_lg_range(&extent_arena_get(extent)->offset_state,
|
|
lg_range, true);
|
|
uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
|
|
lg_range);
|
|
extent->e_addr = (void *)((uintptr_t)extent->e_addr +
|
|
random_offset);
|
|
assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==
|
|
extent->e_addr);
|
|
}
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_size_set(extent_t *extent, size_t size)
|
|
{
|
|
|
|
extent->e_size = size;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_usize_set(extent_t *extent, size_t usize)
|
|
{
|
|
|
|
extent->e_usize = usize;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_active_set(extent_t *extent, bool active)
|
|
{
|
|
|
|
extent->e_active = active;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_zeroed_set(extent_t *extent, bool zeroed)
|
|
{
|
|
|
|
extent->e_zeroed = zeroed;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_committed_set(extent_t *extent, bool committed)
|
|
{
|
|
|
|
extent->e_committed = committed;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_slab_set(extent_t *extent, bool slab)
|
|
{
|
|
|
|
extent->e_slab = slab;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx)
|
|
{
|
|
|
|
atomic_write_p(&extent->e_prof_tctx_pun, tctx);
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
|
|
size_t usize, bool active, bool zeroed, bool committed, bool slab)
|
|
{
|
|
|
|
assert(addr == PAGE_ADDR2BASE(addr) || !slab);
|
|
|
|
extent_arena_set(extent, arena);
|
|
extent_addr_set(extent, addr);
|
|
extent_size_set(extent, size);
|
|
extent_usize_set(extent, usize);
|
|
extent_active_set(extent, active);
|
|
extent_zeroed_set(extent, zeroed);
|
|
extent_committed_set(extent, committed);
|
|
extent_slab_set(extent, slab);
|
|
if (config_prof)
|
|
extent_prof_tctx_set(extent, NULL);
|
|
qr_new(extent, qr_link);
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_ring_insert(extent_t *sentinel, extent_t *extent)
|
|
{
|
|
|
|
qr_meld(sentinel, extent, qr_link);
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_ring_remove(extent_t *extent)
|
|
{
|
|
|
|
qr_remove(extent, qr_link);
|
|
}
|
|
#endif
|
|
|
|
#endif /* JEMALLOC_H_INLINES */
|
|
/******************************************************************************/
|
|
|
|
#include "jemalloc/internal/extent_dss.h"
|
|
#include "jemalloc/internal/extent_mmap.h"
|