server-skynet-source-3rd-je.../include/jemalloc/internal/extent.h

429 lines
11 KiB
C
Raw Normal View History

/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
2016-03-24 12:09:28 +08:00
typedef struct extent_s extent_t;
#define EXTENT_HOOKS_INITIALIZER NULL
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
2016-03-24 12:09:28 +08:00
/* Extent (span of pages). Use accessor functions for e_* fields. */
struct extent_s {
/* Arena from which this extent came, if any. */
2016-03-24 12:09:28 +08:00
arena_t *e_arena;
2016-03-24 12:09:28 +08:00
/* Pointer to the extent that this structure is responsible for. */
void *e_addr;
/* Extent size. */
2016-03-24 12:09:28 +08:00
size_t e_size;
/*
* Usable size, typically smaller than extent size due to large_pad or
* promotion of sampled small regions.
*/
size_t e_usize;
/* True if extent is active (in use). */
bool e_active;
/*
* The zeroed flag is used by extent recycling code to track whether
* memory is zero-filled.
*/
2016-03-24 12:09:28 +08:00
bool e_zeroed;
/*
* True if physical memory is committed to the extent, whether
* explicitly or implicitly as on a system that overcommits and
* satisfies physical memory needs on demand via soft page faults.
*/
2016-03-24 12:09:28 +08:00
bool e_committed;
/*
* The slab flag indicates whether the extent is used for a slab of
* small regions. This helps differentiate small size classes, and it
* indicates whether interior pointers can be looked up via iealloc().
*/
bool e_slab;
2016-05-25 12:13:36 +08:00
union {
2016-05-30 09:34:50 +08:00
/* Small region slab metadata. */
arena_slab_data_t e_slab_data;
2016-06-01 05:50:21 +08:00
/* Profile counters, used for large objects. */
2016-05-30 09:34:50 +08:00
union {
void *e_prof_tctx_pun;
prof_tctx_t *e_prof_tctx;
};
2016-05-25 12:13:36 +08:00
};
2016-05-30 09:34:50 +08:00
/*
* Linkage for arena's extents_dirty and arena_bin_t's slabs_full rings.
*/
qr(extent_t) qr_link;
union {
/* Linkage for per size class address-ordered heaps. */
phn(extent_t) ph_link;
2016-06-01 05:50:21 +08:00
/* Linkage for arena's large and extent_cache lists. */
2016-03-24 12:09:28 +08:00
ql_elm(extent_t) ql_link;
};
};
typedef ph(extent_t) extent_heap_t;
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
extern rtree_t extents_rtree;
extern const extent_hooks_t extent_hooks_default;
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
extent_hooks_t *extent_hooks_get(arena_t *arena);
extent_hooks_t *extent_hooks_set(arena_t *arena, extent_hooks_t *extent_hooks);
#ifdef JEMALLOC_JET
typedef size_t (extent_size_quantize_t)(size_t);
extern extent_size_quantize_t *extent_size_quantize_floor;
extern extent_size_quantize_t *extent_size_quantize_ceil;
#else
size_t extent_size_quantize_floor(size_t size);
size_t extent_size_quantize_ceil(size_t size);
#endif
ph_proto(, extent_heap_, extent_heap_t, extent_t)
extent_t *extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab);
extent_t *extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool slab);
extent_t *extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, void *new_addr, size_t usize, size_t pad,
size_t alignment, bool *zero, bool *commit, bool slab);
void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
void extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent);
bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
bool extent_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset,
size_t length);
extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a,
size_t usize_a, size_t size_b, size_t usize_b);
bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b);
bool extent_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
extent_t *extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent);
2016-03-24 12:09:28 +08:00
arena_t *extent_arena_get(const extent_t *extent);
void *extent_base_get(const extent_t *extent);
2016-03-24 12:09:28 +08:00
void *extent_addr_get(const extent_t *extent);
size_t extent_size_get(const extent_t *extent);
size_t extent_usize_get(const extent_t *extent);
void *extent_before_get(const extent_t *extent);
void *extent_last_get(const extent_t *extent);
2016-04-07 22:34:26 +08:00
void *extent_past_get(const extent_t *extent);
bool extent_active_get(const extent_t *extent);
bool extent_retained_get(const extent_t *extent);
2016-03-24 12:09:28 +08:00
bool extent_zeroed_get(const extent_t *extent);
bool extent_committed_get(const extent_t *extent);
bool extent_slab_get(const extent_t *extent);
2016-05-30 09:34:50 +08:00
arena_slab_data_t *extent_slab_data_get(extent_t *extent);
const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent);
2016-03-24 12:09:28 +08:00
prof_tctx_t *extent_prof_tctx_get(const extent_t *extent);
void extent_arena_set(extent_t *extent, arena_t *arena);
void extent_addr_set(extent_t *extent, void *addr);
void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
2016-03-24 12:09:28 +08:00
void extent_size_set(extent_t *extent, size_t size);
void extent_usize_set(extent_t *extent, size_t usize);
void extent_active_set(extent_t *extent, bool active);
2016-03-24 12:09:28 +08:00
void extent_zeroed_set(extent_t *extent, bool zeroed);
void extent_committed_set(extent_t *extent, bool committed);
void extent_slab_set(extent_t *extent, bool slab);
2016-03-24 12:09:28 +08:00
void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
void extent_init(extent_t *extent, arena_t *arena, void *addr,
2016-05-30 09:34:50 +08:00
size_t size, size_t usize, bool active, bool zeroed, bool committed,
bool slab);
void extent_ring_insert(extent_t *sentinel, extent_t *extent);
void extent_ring_remove(extent_t *extent);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
JEMALLOC_INLINE extent_t *
extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent)
{
rtree_ctx_t rtree_ctx_fallback;
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
return (rtree_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
dependent));
}
JEMALLOC_INLINE arena_t *
2016-03-24 12:09:28 +08:00
extent_arena_get(const extent_t *extent)
{
2016-03-24 12:09:28 +08:00
return (extent->e_arena);
}
JEMALLOC_INLINE void *
extent_base_get(const extent_t *extent)
{
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent->e_slab);
return (PAGE_ADDR2BASE(extent->e_addr));
}
JEMALLOC_INLINE void *
2016-03-24 12:09:28 +08:00
extent_addr_get(const extent_t *extent)
{
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent->e_slab);
2016-03-24 12:09:28 +08:00
return (extent->e_addr);
}
JEMALLOC_INLINE size_t
2016-03-24 12:09:28 +08:00
extent_size_get(const extent_t *extent)
{
2016-03-24 12:09:28 +08:00
return (extent->e_size);
}
JEMALLOC_INLINE size_t
extent_usize_get(const extent_t *extent)
{
assert(!extent->e_slab);
return (extent->e_usize);
}
JEMALLOC_INLINE void *
extent_before_get(const extent_t *extent)
{
return ((void *)((uintptr_t)extent_base_get(extent) - PAGE));
}
JEMALLOC_INLINE void *
extent_last_get(const extent_t *extent)
{
return ((void *)((uintptr_t)extent_base_get(extent) +
extent_size_get(extent) - PAGE));
}
2016-04-07 22:34:26 +08:00
JEMALLOC_INLINE void *
extent_past_get(const extent_t *extent)
{
return ((void *)((uintptr_t)extent_base_get(extent) +
extent_size_get(extent)));
2016-04-07 22:34:26 +08:00
}
JEMALLOC_INLINE bool
extent_active_get(const extent_t *extent)
{
return (extent->e_active);
}
JEMALLOC_INLINE bool
extent_retained_get(const extent_t *extent)
{
2016-05-30 09:34:50 +08:00
return (qr_next(extent, qr_link) == extent);
}
JEMALLOC_INLINE bool
2016-03-24 12:09:28 +08:00
extent_zeroed_get(const extent_t *extent)
{
2016-03-24 12:09:28 +08:00
return (extent->e_zeroed);
}
JEMALLOC_INLINE bool
2016-03-24 12:09:28 +08:00
extent_committed_get(const extent_t *extent)
{
2016-03-24 12:09:28 +08:00
return (extent->e_committed);
}
JEMALLOC_INLINE bool
extent_slab_get(const extent_t *extent)
{
return (extent->e_slab);
}
2016-05-30 09:34:50 +08:00
JEMALLOC_INLINE arena_slab_data_t *
extent_slab_data_get(extent_t *extent)
{
assert(extent->e_slab);
return (&extent->e_slab_data);
}
JEMALLOC_INLINE const arena_slab_data_t *
extent_slab_data_get_const(const extent_t *extent)
{
assert(extent->e_slab);
return (&extent->e_slab_data);
}
JEMALLOC_INLINE prof_tctx_t *
2016-03-24 12:09:28 +08:00
extent_prof_tctx_get(const extent_t *extent)
{
2016-05-25 12:13:36 +08:00
return ((prof_tctx_t *)atomic_read_p(
&((extent_t *)extent)->e_prof_tctx_pun));
}
JEMALLOC_INLINE void
2016-03-24 12:09:28 +08:00
extent_arena_set(extent_t *extent, arena_t *arena)
{
2016-03-24 12:09:28 +08:00
extent->e_arena = arena;
}
JEMALLOC_INLINE void
2016-03-24 12:09:28 +08:00
extent_addr_set(extent_t *extent, void *addr)
{
2016-03-24 12:09:28 +08:00
extent->e_addr = addr;
}
JEMALLOC_INLINE void
extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment)
{
assert(extent_base_get(extent) == extent_addr_get(extent));
if (alignment < PAGE) {
unsigned lg_range = LG_PAGE -
lg_floor(CACHELINE_CEILING(alignment));
uint64_t r =
prng_lg_range(&extent_arena_get(extent)->offset_state,
lg_range, true);
uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
lg_range);
extent->e_addr = (void *)((uintptr_t)extent->e_addr +
random_offset);
assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==
extent->e_addr);
}
}
JEMALLOC_INLINE void
2016-03-24 12:09:28 +08:00
extent_size_set(extent_t *extent, size_t size)
{
2016-03-24 12:09:28 +08:00
extent->e_size = size;
}
JEMALLOC_INLINE void
extent_usize_set(extent_t *extent, size_t usize)
{
extent->e_usize = usize;
}
JEMALLOC_INLINE void
extent_active_set(extent_t *extent, bool active)
{
extent->e_active = active;
}
JEMALLOC_INLINE void
2016-03-24 12:09:28 +08:00
extent_zeroed_set(extent_t *extent, bool zeroed)
{
2016-03-24 12:09:28 +08:00
extent->e_zeroed = zeroed;
}
JEMALLOC_INLINE void
2016-03-24 12:09:28 +08:00
extent_committed_set(extent_t *extent, bool committed)
{
2016-03-24 12:09:28 +08:00
extent->e_committed = committed;
}
JEMALLOC_INLINE void
extent_slab_set(extent_t *extent, bool slab)
{
extent->e_slab = slab;
}
JEMALLOC_INLINE void
2016-03-24 12:09:28 +08:00
extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx)
{
2016-05-25 12:13:36 +08:00
atomic_write_p(&extent->e_prof_tctx_pun, tctx);
}
JEMALLOC_INLINE void
2016-03-24 12:09:28 +08:00
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
2016-05-30 09:34:50 +08:00
size_t usize, bool active, bool zeroed, bool committed, bool slab)
{
assert(addr == PAGE_ADDR2BASE(addr) || !slab);
2016-03-24 12:09:28 +08:00
extent_arena_set(extent, arena);
extent_addr_set(extent, addr);
extent_size_set(extent, size);
extent_usize_set(extent, usize);
extent_active_set(extent, active);
2016-03-24 12:09:28 +08:00
extent_zeroed_set(extent, zeroed);
extent_committed_set(extent, committed);
extent_slab_set(extent, slab);
if (config_prof)
2016-03-24 12:09:28 +08:00
extent_prof_tctx_set(extent, NULL);
2016-05-30 09:34:50 +08:00
qr_new(extent, qr_link);
}
JEMALLOC_INLINE void
2016-05-30 09:34:50 +08:00
extent_ring_insert(extent_t *sentinel, extent_t *extent)
{
2016-05-30 09:34:50 +08:00
qr_meld(sentinel, extent, qr_link);
}
JEMALLOC_INLINE void
2016-05-30 09:34:50 +08:00
extent_ring_remove(extent_t *extent)
{
2016-05-30 09:34:50 +08:00
qr_remove(extent, qr_link);
}
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/extent_mmap.h"