577d4572b0
Rather than protecting dss operations with a mutex, use atomic operations. This has negligible impact on synchronization overhead during typical dss allocation, but is a substantial improvement for extent_in_dss() and the newly added extent_dss_mergeable(), which can be called multiple times during extent deallocations. This change also has the advantage of avoiding tsd in deallocation paths associated with purging, which resolves potential deadlocks during thread exit due to attempted tsd resurrection. This resolves #425.
429 lines
11 KiB
C
429 lines
11 KiB
C
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_TYPES
|
|
|
|
typedef struct extent_s extent_t;
|
|
|
|
#define EXTENT_HOOKS_INITIALIZER NULL
|
|
|
|
#endif /* JEMALLOC_H_TYPES */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_STRUCTS
|
|
|
|
/* Extent (span of pages). Use accessor functions for e_* fields. */
|
|
struct extent_s {
|
|
/* Arena from which this extent came, if any. */
|
|
arena_t *e_arena;
|
|
|
|
/* Pointer to the extent that this structure is responsible for. */
|
|
void *e_addr;
|
|
|
|
/* Extent size. */
|
|
size_t e_size;
|
|
|
|
/*
|
|
* Usable size, typically smaller than extent size due to large_pad or
|
|
* promotion of sampled small regions.
|
|
*/
|
|
size_t e_usize;
|
|
|
|
/* True if extent is active (in use). */
|
|
bool e_active;
|
|
|
|
/*
|
|
* The zeroed flag is used by extent recycling code to track whether
|
|
* memory is zero-filled.
|
|
*/
|
|
bool e_zeroed;
|
|
|
|
/*
|
|
* True if physical memory is committed to the extent, whether
|
|
* explicitly or implicitly as on a system that overcommits and
|
|
* satisfies physical memory needs on demand via soft page faults.
|
|
*/
|
|
bool e_committed;
|
|
|
|
/*
|
|
* The slab flag indicates whether the extent is used for a slab of
|
|
* small regions. This helps differentiate small size classes, and it
|
|
* indicates whether interior pointers can be looked up via iealloc().
|
|
*/
|
|
bool e_slab;
|
|
|
|
union {
|
|
/* Small region slab metadata. */
|
|
arena_slab_data_t e_slab_data;
|
|
|
|
/* Profile counters, used for large objects. */
|
|
union {
|
|
void *e_prof_tctx_pun;
|
|
prof_tctx_t *e_prof_tctx;
|
|
};
|
|
};
|
|
|
|
/*
|
|
* Linkage for arena's extents_dirty and arena_bin_t's slabs_full rings.
|
|
*/
|
|
qr(extent_t) qr_link;
|
|
|
|
union {
|
|
/* Linkage for per size class address-ordered heaps. */
|
|
phn(extent_t) ph_link;
|
|
|
|
/* Linkage for arena's large and extent_cache lists. */
|
|
ql_elm(extent_t) ql_link;
|
|
};
|
|
};
|
|
typedef ph(extent_t) extent_heap_t;
|
|
|
|
#endif /* JEMALLOC_H_STRUCTS */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_EXTERNS
|
|
|
|
extern rtree_t extents_rtree;
|
|
extern const extent_hooks_t extent_hooks_default;
|
|
|
|
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
|
|
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
|
|
|
|
extent_hooks_t *extent_hooks_get(arena_t *arena);
|
|
extent_hooks_t *extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks);
|
|
|
|
#ifdef JEMALLOC_JET
|
|
typedef size_t (extent_size_quantize_t)(size_t);
|
|
extern extent_size_quantize_t *extent_size_quantize_floor;
|
|
extern extent_size_quantize_t *extent_size_quantize_ceil;
|
|
#else
|
|
size_t extent_size_quantize_floor(size_t size);
|
|
size_t extent_size_quantize_ceil(size_t size);
|
|
#endif
|
|
|
|
ph_proto(, extent_heap_, extent_heap_t, extent_t)
|
|
|
|
extent_t *extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
|
|
size_t alignment, bool *zero, bool slab);
|
|
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
|
|
size_t alignment, bool *zero, bool slab);
|
|
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
|
|
size_t alignment, bool *zero, bool *commit, bool slab);
|
|
void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
|
|
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent);
|
|
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent);
|
|
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
|
size_t length);
|
|
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
|
size_t length);
|
|
bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
|
|
size_t length);
|
|
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
|
|
size_t usize_a, size_t size_b, size_t usize_b);
|
|
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
|
|
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b);
|
|
|
|
bool extent_boot(void);
|
|
|
|
#endif /* JEMALLOC_H_EXTERNS */
|
|
/******************************************************************************/
|
|
#ifdef JEMALLOC_H_INLINES
|
|
|
|
#ifndef JEMALLOC_ENABLE_INLINE
|
|
extent_t *extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent);
|
|
arena_t *extent_arena_get(const extent_t *extent);
|
|
void *extent_base_get(const extent_t *extent);
|
|
void *extent_addr_get(const extent_t *extent);
|
|
size_t extent_size_get(const extent_t *extent);
|
|
size_t extent_usize_get(const extent_t *extent);
|
|
void *extent_before_get(const extent_t *extent);
|
|
void *extent_last_get(const extent_t *extent);
|
|
void *extent_past_get(const extent_t *extent);
|
|
bool extent_active_get(const extent_t *extent);
|
|
bool extent_retained_get(const extent_t *extent);
|
|
bool extent_zeroed_get(const extent_t *extent);
|
|
bool extent_committed_get(const extent_t *extent);
|
|
bool extent_slab_get(const extent_t *extent);
|
|
arena_slab_data_t *extent_slab_data_get(extent_t *extent);
|
|
const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent);
|
|
prof_tctx_t *extent_prof_tctx_get(const extent_t *extent);
|
|
void extent_arena_set(extent_t *extent, arena_t *arena);
|
|
void extent_addr_set(extent_t *extent, void *addr);
|
|
void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
|
|
void extent_size_set(extent_t *extent, size_t size);
|
|
void extent_usize_set(extent_t *extent, size_t usize);
|
|
void extent_active_set(extent_t *extent, bool active);
|
|
void extent_zeroed_set(extent_t *extent, bool zeroed);
|
|
void extent_committed_set(extent_t *extent, bool committed);
|
|
void extent_slab_set(extent_t *extent, bool slab);
|
|
void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
|
|
void extent_init(extent_t *extent, arena_t *arena, void *addr,
|
|
size_t size, size_t usize, bool active, bool zeroed, bool committed,
|
|
bool slab);
|
|
void extent_ring_insert(extent_t *sentinel, extent_t *extent);
|
|
void extent_ring_remove(extent_t *extent);
|
|
#endif
|
|
|
|
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
|
|
JEMALLOC_INLINE extent_t *
|
|
extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent)
|
|
{
|
|
rtree_ctx_t rtree_ctx_fallback;
|
|
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
|
|
|
|
return (rtree_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
|
|
dependent));
|
|
}
|
|
|
|
JEMALLOC_INLINE arena_t *
|
|
extent_arena_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_arena);
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_base_get(const extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
|
|
!extent->e_slab);
|
|
return (PAGE_ADDR2BASE(extent->e_addr));
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_addr_get(const extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
|
|
!extent->e_slab);
|
|
return (extent->e_addr);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
extent_size_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_size);
|
|
}
|
|
|
|
JEMALLOC_INLINE size_t
|
|
extent_usize_get(const extent_t *extent)
|
|
{
|
|
|
|
assert(!extent->e_slab);
|
|
return (extent->e_usize);
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_before_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((void *)((uintptr_t)extent_base_get(extent) - PAGE));
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_last_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((void *)((uintptr_t)extent_base_get(extent) +
|
|
extent_size_get(extent) - PAGE));
|
|
}
|
|
|
|
JEMALLOC_INLINE void *
|
|
extent_past_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((void *)((uintptr_t)extent_base_get(extent) +
|
|
extent_size_get(extent)));
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_active_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_active);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_retained_get(const extent_t *extent)
|
|
{
|
|
|
|
return (qr_next(extent, qr_link) == extent);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_zeroed_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_zeroed);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_committed_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_committed);
|
|
}
|
|
|
|
JEMALLOC_INLINE bool
|
|
extent_slab_get(const extent_t *extent)
|
|
{
|
|
|
|
return (extent->e_slab);
|
|
}
|
|
|
|
JEMALLOC_INLINE arena_slab_data_t *
|
|
extent_slab_data_get(extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_slab);
|
|
return (&extent->e_slab_data);
|
|
}
|
|
|
|
JEMALLOC_INLINE const arena_slab_data_t *
|
|
extent_slab_data_get_const(const extent_t *extent)
|
|
{
|
|
|
|
assert(extent->e_slab);
|
|
return (&extent->e_slab_data);
|
|
}
|
|
|
|
JEMALLOC_INLINE prof_tctx_t *
|
|
extent_prof_tctx_get(const extent_t *extent)
|
|
{
|
|
|
|
return ((prof_tctx_t *)atomic_read_p(
|
|
&((extent_t *)extent)->e_prof_tctx_pun));
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_arena_set(extent_t *extent, arena_t *arena)
|
|
{
|
|
|
|
extent->e_arena = arena;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_addr_set(extent_t *extent, void *addr)
|
|
{
|
|
|
|
extent->e_addr = addr;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment)
|
|
{
|
|
|
|
assert(extent_base_get(extent) == extent_addr_get(extent));
|
|
|
|
if (alignment < PAGE) {
|
|
unsigned lg_range = LG_PAGE -
|
|
lg_floor(CACHELINE_CEILING(alignment));
|
|
uint64_t r =
|
|
prng_lg_range(&extent_arena_get(extent)->offset_state,
|
|
lg_range, true);
|
|
uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
|
|
lg_range);
|
|
extent->e_addr = (void *)((uintptr_t)extent->e_addr +
|
|
random_offset);
|
|
assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==
|
|
extent->e_addr);
|
|
}
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_size_set(extent_t *extent, size_t size)
|
|
{
|
|
|
|
extent->e_size = size;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_usize_set(extent_t *extent, size_t usize)
|
|
{
|
|
|
|
extent->e_usize = usize;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_active_set(extent_t *extent, bool active)
|
|
{
|
|
|
|
extent->e_active = active;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_zeroed_set(extent_t *extent, bool zeroed)
|
|
{
|
|
|
|
extent->e_zeroed = zeroed;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_committed_set(extent_t *extent, bool committed)
|
|
{
|
|
|
|
extent->e_committed = committed;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_slab_set(extent_t *extent, bool slab)
|
|
{
|
|
|
|
extent->e_slab = slab;
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx)
|
|
{
|
|
|
|
atomic_write_p(&extent->e_prof_tctx_pun, tctx);
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
|
|
size_t usize, bool active, bool zeroed, bool committed, bool slab)
|
|
{
|
|
|
|
assert(addr == PAGE_ADDR2BASE(addr) || !slab);
|
|
|
|
extent_arena_set(extent, arena);
|
|
extent_addr_set(extent, addr);
|
|
extent_size_set(extent, size);
|
|
extent_usize_set(extent, usize);
|
|
extent_active_set(extent, active);
|
|
extent_zeroed_set(extent, zeroed);
|
|
extent_committed_set(extent, committed);
|
|
extent_slab_set(extent, slab);
|
|
if (config_prof)
|
|
extent_prof_tctx_set(extent, NULL);
|
|
qr_new(extent, qr_link);
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_ring_insert(extent_t *sentinel, extent_t *extent)
|
|
{
|
|
|
|
qr_meld(sentinel, extent, qr_link);
|
|
}
|
|
|
|
JEMALLOC_INLINE void
|
|
extent_ring_remove(extent_t *extent)
|
|
{
|
|
|
|
qr_remove(extent, qr_link);
|
|
}
|
|
#endif
|
|
|
|
#endif /* JEMALLOC_H_INLINES */
|
|
/******************************************************************************/
|
|
|
|
#include "jemalloc/internal/extent_dss.h"
|
|
#include "jemalloc/internal/extent_mmap.h"
|