diff --git a/Makefile.in b/Makefile.in
index a735e0e6..29977bc0 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -107,6 +107,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/ehooks.c \
$(srcroot)src/eset.c \
$(srcroot)src/extent.c \
+ $(srcroot)src/extent2.c \
$(srcroot)src/extent_dss.c \
$(srcroot)src/extent_mmap.c \
$(srcroot)src/hash.c \
diff --git a/include/jemalloc/internal/bin.h b/include/jemalloc/internal/bin.h
index 0d6aff8b..92e8122d 100644
--- a/include/jemalloc/internal/bin.h
+++ b/include/jemalloc/internal/bin.h
@@ -4,8 +4,6 @@
#include "jemalloc/internal/bin_stats.h"
#include "jemalloc/internal/bin_types.h"
#include "jemalloc/internal/extent.h"
-#include "jemalloc/internal/extent_types.h"
-#include "jemalloc/internal/extent_structs.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/sc.h"
diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h
index fa7d1260..2fd6e906 100644
--- a/include/jemalloc/internal/extent.h
+++ b/include/jemalloc/internal/extent.h
@@ -620,4 +620,7 @@ extent_esnead_comp(const extent_t *a, const extent_t *b) {
return ret;
}
+ph_proto(, extent_avail_, extent_tree_t, extent_t)
+ph_proto(, extent_heap_, extent_heap_t, extent_t)
+
#endif /* JEMALLOC_INTERNAL_EXTENT_H */
diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent2.h
similarity index 62%
rename from include/jemalloc/internal/extent_externs.h
rename to include/jemalloc/internal/extent2.h
index 4e3803c6..22035bba 100644
--- a/include/jemalloc/internal/extent_externs.h
+++ b/include/jemalloc/internal/extent2.h
@@ -1,23 +1,56 @@
-#ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
-#define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
+#ifndef JEMALLOC_INTERNAL_EXTENT2_H
+#define JEMALLOC_INTERNAL_EXTENT2_H
#include "jemalloc/internal/ehooks.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/mutex_pool.h"
+#include "jemalloc/internal/eset.h"
#include "jemalloc/internal/ph.h"
#include "jemalloc/internal/rtree.h"
+/*
+ * This module contains the page-level allocator. It chooses the addresses that
+ * allocations requested by other modules will inhabit, and updates the global
+ * metadata to reflect allocation/deallocation/purging decisions.
+ *
+ * The naming ("extent2" for the module, and "extent_" or "extents_" for most of
+ * the functions) is historical. Eventually, the naming should be updated to
+ * reflect the functionality. Similarly, the utilization stats live here for no
+ * particular reason. This will also be changed, but much more immediately.
+ */
+
+/*
+ * The following two structs are for experimental purposes. See
+ * experimental_utilization_query_ctl and
+ * experimental_utilization_batch_query_ctl in src/ctl.c.
+ */
+typedef struct extent_util_stats_s extent_util_stats_t;
+struct extent_util_stats_s {
+ size_t nfree;
+ size_t nregs;
+ size_t size;
+};
+
+typedef struct extent_util_stats_verbose_s extent_util_stats_verbose_t;
+struct extent_util_stats_verbose_s {
+ void *slabcur_addr;
+ size_t nfree;
+ size_t nregs;
+ size_t size;
+ size_t bin_nfree;
+ size_t bin_nregs;
+};
+
+/*
+ * When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit)
+ * is the max ratio between the size of the active extent and the new extent.
+ */
+#define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6
extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree;
-extern mutex_pool_t extent_mutex_pool;
extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent);
-ph_proto(, extent_avail_, extent_tree_t, extent_t)
-ph_proto(, extent_heap_, extent_heap_t, extent_t)
-
extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit);
@@ -56,4 +89,4 @@ void extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size,
size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr);
-#endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */
+#endif /* JEMALLOC_INTERNAL_EXTENT2_H */
diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h
deleted file mode 100644
index 2647df8a..00000000
--- a/include/jemalloc/internal/extent_inlines.h
+++ /dev/null
@@ -1,38 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_EXTENT_INLINES_H
-#define JEMALLOC_INTERNAL_EXTENT_INLINES_H
-
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/mutex_pool.h"
-#include "jemalloc/internal/pages.h"
-#include "jemalloc/internal/prng.h"
-#include "jemalloc/internal/ql.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/sz.h"
-
-static inline void
-extent_lock(tsdn_t *tsdn, extent_t *extent) {
- assert(extent != NULL);
- mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
-}
-
-static inline void
-extent_unlock(tsdn_t *tsdn, extent_t *extent) {
- assert(extent != NULL);
- mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
-}
-
-static inline void
-extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
- assert(extent1 != NULL && extent2 != NULL);
- mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
- (uintptr_t)extent2);
-}
-
-static inline void
-extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
- assert(extent1 != NULL && extent2 != NULL);
- mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
- (uintptr_t)extent2);
-}
-
-#endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */
diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h
deleted file mode 100644
index 4e6e085c..00000000
--- a/include/jemalloc/internal/extent_structs.h
+++ /dev/null
@@ -1,33 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_EXTENT_STRUCTS_H
-#define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H
-
-#include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/bitmap.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/ql.h"
-#include "jemalloc/internal/ph.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/slab_data.h"
-
-/*
- * The following two structs are for experimental purposes. See
- * experimental_utilization_query_ctl and
- * experimental_utilization_batch_query_ctl in src/ctl.c.
- */
-
-struct extent_util_stats_s {
- size_t nfree;
- size_t nregs;
- size_t size;
-};
-
-struct extent_util_stats_verbose_s {
- void *slabcur_addr;
- size_t nfree;
- size_t nregs;
- size_t size;
- size_t bin_nfree;
- size_t bin_nregs;
-};
-
-#endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */
diff --git a/include/jemalloc/internal/extent_types.h b/include/jemalloc/internal/extent_types.h
deleted file mode 100644
index 25b360eb..00000000
--- a/include/jemalloc/internal/extent_types.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_EXTENT_TYPES_H
-#define JEMALLOC_INTERNAL_EXTENT_TYPES_H
-
-typedef struct extent_util_stats_s extent_util_stats_t;
-typedef struct extent_util_stats_verbose_s extent_util_stats_verbose_t;
-
-/*
- * When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit)
- * is the max ratio between the size of the active extent and the new extent.
- */
-#define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6
-
-#endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */
diff --git a/include/jemalloc/internal/jemalloc_internal_includes.h b/include/jemalloc/internal/jemalloc_internal_includes.h
index 6755b43e..75a94d3a 100644
--- a/include/jemalloc/internal/jemalloc_internal_includes.h
+++ b/include/jemalloc/internal/jemalloc_internal_includes.h
@@ -40,7 +40,6 @@
/* TYPES */
/******************************************************************************/
-#include "jemalloc/internal/extent_types.h"
#include "jemalloc/internal/base_types.h"
#include "jemalloc/internal/arena_types.h"
#include "jemalloc/internal/tcache_types.h"
@@ -61,7 +60,6 @@
/******************************************************************************/
#include "jemalloc/internal/jemalloc_internal_externs.h"
-#include "jemalloc/internal/extent_externs.h"
#include "jemalloc/internal/base_externs.h"
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/large_externs.h"
@@ -81,7 +79,6 @@
*/
#include "jemalloc/internal/prof_inlines_a.h"
#include "jemalloc/internal/arena_inlines_a.h"
-#include "jemalloc/internal/extent_inlines.h"
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
#include "jemalloc/internal/tcache_inlines.h"
#include "jemalloc/internal/arena_inlines_b.h"
diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/include/jemalloc/internal/jemalloc_internal_inlines_b.h
index f0b73d02..d4cb04c2 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_b.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_b.h
@@ -1,6 +1,7 @@
#ifndef JEMALLOC_INTERNAL_INLINES_B_H
#define JEMALLOC_INTERNAL_INLINES_B_H
+#include "jemalloc/internal/extent2.h"
#include "jemalloc/internal/rtree.h"
/* Choose an arena based on a per-thread value. */
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index e6803120..4118b911 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -47,6 +47,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index ce51930a..ed3b5248 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -47,6 +47,7 @@
+
diff --git a/src/extent.c b/src/extent.c
index ea7b8f2c..1a5a1fa6 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -1,1722 +1,6 @@
-#define JEMALLOC_EXTENT_C_
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
-#include "jemalloc/internal/assert.h"
-#include "jemalloc/internal/extent_dss.h"
-#include "jemalloc/internal/extent_mmap.h"
-#include "jemalloc/internal/ph.h"
-#include "jemalloc/internal/rtree.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/mutex_pool.h"
-
-/******************************************************************************/
-/* Data. */
-
-rtree_t extents_rtree;
-/* Keyed by the address of the extent_t being protected. */
-mutex_pool_t extent_mutex_pool;
-
-size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;
-
-static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset, size_t length, bool growing_retained);
-static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
- ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
- bool growing_retained);
-static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
- ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
- bool growing_retained);
-static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena,
- ehooks_t *ehooks, extent_t *extent, size_t size_a, szind_t szind_a,
- bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
- bool growing_retained);
-static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *a, extent_t *b, bool growing_retained);
-
-/* Used exclusively for gdump triggering. */
-static atomic_zu_t curpages;
-static atomic_zu_t highpages;
-
-/******************************************************************************/
-/*
- * Function prototypes for static functions that are referenced prior to
- * definition.
- */
-
-static void extent_deregister(tsdn_t *tsdn, extent_t *extent);
-static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment,
- bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained);
-static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
- ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent,
- bool *coalesced, bool growing_retained);
-static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- eset_t *eset, extent_t *extent, bool growing_retained);
-
-/******************************************************************************/
-
-#define ATTR_NONE /* does nothing */
-
-ph_gen(ATTR_NONE, extent_avail_, extent_tree_t, extent_t, ph_link,
+ph_gen(, extent_avail_, extent_tree_t, extent_t, ph_link,
extent_esnead_comp)
-
-#undef ATTR_NONE
-
-typedef enum {
- lock_result_success,
- lock_result_failure,
- lock_result_no_extent
-} lock_result_t;
-
-static lock_result_t
-extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm,
- extent_t **result, bool inactive_only) {
- extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree,
- elm, true);
-
- /* Slab implies active extents and should be skipped. */
- if (extent1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn,
- &extents_rtree, elm, true))) {
- return lock_result_no_extent;
- }
-
- /*
- * It's possible that the extent changed out from under us, and with it
- * the leaf->extent mapping. We have to recheck while holding the lock.
- */
- extent_lock(tsdn, extent1);
- extent_t *extent2 = rtree_leaf_elm_extent_read(tsdn,
- &extents_rtree, elm, true);
-
- if (extent1 == extent2) {
- *result = extent1;
- return lock_result_success;
- } else {
- extent_unlock(tsdn, extent1);
- return lock_result_failure;
- }
-}
-
-/*
- * Returns a pool-locked extent_t * if there's one associated with the given
- * address, and NULL otherwise.
- */
-static extent_t *
-extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr,
- bool inactive_only) {
- extent_t *ret = NULL;
- rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree,
- rtree_ctx, (uintptr_t)addr, false, false);
- if (elm == NULL) {
- return NULL;
- }
- lock_result_t lock_result;
- do {
- lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret,
- inactive_only);
- } while (lock_result == lock_result_failure);
- return ret;
-}
-
-static void
-extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
- size_t alignment) {
- assert(extent_base_get(extent) == extent_addr_get(extent));
-
- if (alignment < PAGE) {
- unsigned lg_range = LG_PAGE -
- lg_floor(CACHELINE_CEILING(alignment));
- size_t r;
- if (!tsdn_null(tsdn)) {
- tsd_t *tsd = tsdn_tsd(tsdn);
- r = (size_t)prng_lg_range_u64(
- tsd_prng_statep_get(tsd), lg_range);
- } else {
- uint64_t stack_value = (uint64_t)(uintptr_t)&r;
- r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
- }
- uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
- lg_range);
- extent->e_addr = (void *)((uintptr_t)extent->e_addr +
- random_offset);
- assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==
- extent->e_addr);
- }
-}
-
-extent_t *
-extent_alloc(tsdn_t *tsdn, arena_t *arena) {
- malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
- extent_t *extent = extent_avail_first(&arena->extent_avail);
- if (extent == NULL) {
- malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
- return base_alloc_extent(tsdn, arena->base);
- }
- extent_avail_remove(&arena->extent_avail, extent);
- atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED);
- malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
- return extent;
-}
-
-void
-extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
- malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
- extent_avail_insert(&arena->extent_avail, extent);
- atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED);
- malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
-}
-
ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_snad_comp)
-
-static bool
-extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent) {
- extent_state_set(extent, extent_state_active);
- bool coalesced;
- extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset,
- extent, &coalesced, false);
- extent_state_set(extent, eset_state_get(eset));
-
- if (!coalesced) {
- return true;
- }
- eset_insert_locked(tsdn, eset, extent);
- return false;
-}
-
-extent_t *
-extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
- szind_t szind, bool *zero, bool *commit) {
- assert(size + pad != 0);
- assert(alignment != 0);
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- extent_t *extent = extent_recycle(tsdn, arena, ehooks, eset, new_addr,
- size, pad, alignment, slab, szind, zero, commit, false);
- assert(extent == NULL || extent_dumpable_get(extent));
- return extent;
-}
-
-void
-extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- extent_t *extent) {
- assert(extent_base_get(extent) != NULL);
- assert(extent_size_get(extent) != 0);
- assert(extent_dumpable_get(extent));
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- extent_addr_set(extent, extent_base_get(extent));
- extent_zeroed_set(extent, false);
-
- extent_record(tsdn, arena, ehooks, eset, extent, false);
-}
-
-extent_t *
-extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- size_t npages_min) {
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
-
- malloc_mutex_lock(tsdn, &eset->mtx);
-
- /*
- * Get the LRU coalesced extent, if any. If coalescing was delayed,
- * the loop will iterate until the LRU extent is fully coalesced.
- */
- extent_t *extent;
- while (true) {
- /* Get the LRU extent, if any. */
- extent = extent_list_first(&eset->lru);
- if (extent == NULL) {
- goto label_return;
- }
- /* Check the eviction limit. */
- size_t extents_npages = atomic_load_zu(&eset->npages,
- ATOMIC_RELAXED);
- if (extents_npages <= npages_min) {
- extent = NULL;
- goto label_return;
- }
- eset_remove_locked(tsdn, eset, extent);
- if (!eset->delay_coalesce) {
- break;
- }
- /* Try to coalesce. */
- if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx,
- eset, extent)) {
- break;
- }
- /*
- * The LRU extent was just coalesced and the result placed in
- * the LRU at its neighbor's position. Start over.
- */
- }
-
- /*
- * Either mark the extent active or deregister it to protect against
- * concurrent operations.
- */
- switch (eset_state_get(eset)) {
- case extent_state_active:
- not_reached();
- case extent_state_dirty:
- case extent_state_muzzy:
- extent_state_set(extent, extent_state_active);
- break;
- case extent_state_retained:
- extent_deregister(tsdn, extent);
- break;
- default:
- not_reached();
- }
-
-label_return:
- malloc_mutex_unlock(tsdn, &eset->mtx);
- return extent;
-}
-
-/*
- * This can only happen when we fail to allocate a new extent struct (which
- * indicates OOM), e.g. when trying to split an existing extent.
- */
-static void
-extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- extent_t *extent, bool growing_retained) {
- size_t sz = extent_size_get(extent);
- if (config_stats) {
- arena_stats_accum_zu(&arena->stats.abandoned_vm, sz);
- }
- /*
- * Leak extent after making sure its pages have already been purged, so
- * that this is only a virtual memory leak.
- */
- if (eset_state_get(eset) == extent_state_dirty) {
- if (extent_purge_lazy_impl(tsdn, arena, ehooks, extent, 0, sz,
- growing_retained)) {
- extent_purge_forced_impl(tsdn, arena, ehooks, extent, 0,
- extent_size_get(extent), growing_retained);
- }
- }
- extent_dalloc(tsdn, arena, extent);
-}
-
-static void
-extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
- extent_t *extent) {
- assert(extent_arena_ind_get(extent) == arena_ind_get(arena));
- assert(extent_state_get(extent) == extent_state_active);
-
- extent_state_set(extent, eset_state_get(eset));
- eset_insert_locked(tsdn, eset, extent);
-}
-
-static void
-extent_deactivate(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
- extent_t *extent) {
- malloc_mutex_lock(tsdn, &eset->mtx);
- extent_deactivate_locked(tsdn, arena, eset, extent);
- malloc_mutex_unlock(tsdn, &eset->mtx);
-}
-
-static void
-extent_activate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
- extent_t *extent) {
- assert(extent_arena_ind_get(extent) == arena_ind_get(arena));
- assert(extent_state_get(extent) == eset_state_get(eset));
-
- eset_remove_locked(tsdn, eset, extent);
- extent_state_set(extent, extent_state_active);
-}
-
-static bool
-extent_rtree_leaf_elms_lookup(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
- const extent_t *extent, bool dependent, bool init_missing,
- rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) {
- *r_elm_a = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
- (uintptr_t)extent_base_get(extent), dependent, init_missing);
- if (!dependent && *r_elm_a == NULL) {
- return true;
- }
- assert(*r_elm_a != NULL);
-
- *r_elm_b = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
- (uintptr_t)extent_last_get(extent), dependent, init_missing);
- if (!dependent && *r_elm_b == NULL) {
- return true;
- }
- assert(*r_elm_b != NULL);
-
- return false;
-}
-
-static void
-extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a,
- rtree_leaf_elm_t *elm_b, extent_t *extent, szind_t szind, bool slab) {
- rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, extent, szind, slab);
- if (elm_b != NULL) {
- rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, extent, szind,
- slab);
- }
-}
-
-static void
-extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, extent_t *extent,
- szind_t szind) {
- assert(extent_slab_get(extent));
-
- /* Register interior. */
- for (size_t i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
- rtree_write(tsdn, &extents_rtree, rtree_ctx,
- (uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
- LG_PAGE), extent, szind, true);
- }
-}
-
-static void
-extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) {
- cassert(config_prof);
- /* prof_gdump() requirement. */
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- if (opt_prof && extent_state_get(extent) == extent_state_active) {
- size_t nadd = extent_size_get(extent) >> LG_PAGE;
- size_t cur = atomic_fetch_add_zu(&curpages, nadd,
- ATOMIC_RELAXED) + nadd;
- size_t high = atomic_load_zu(&highpages, ATOMIC_RELAXED);
- while (cur > high && !atomic_compare_exchange_weak_zu(
- &highpages, &high, cur, ATOMIC_RELAXED, ATOMIC_RELAXED)) {
- /*
- * Don't refresh cur, because it may have decreased
- * since this thread lost the highpages update race.
- * Note that high is updated in case of CAS failure.
- */
- }
- if (cur > high && prof_gdump_get_unlocked()) {
- prof_gdump(tsdn);
- }
- }
-}
-
-static void
-extent_gdump_sub(tsdn_t *tsdn, const extent_t *extent) {
- cassert(config_prof);
-
- if (opt_prof && extent_state_get(extent) == extent_state_active) {
- size_t nsub = extent_size_get(extent) >> LG_PAGE;
- assert(atomic_load_zu(&curpages, ATOMIC_RELAXED) >= nsub);
- atomic_fetch_sub_zu(&curpages, nsub, ATOMIC_RELAXED);
- }
-}
-
-static bool
-extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) {
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
- rtree_leaf_elm_t *elm_a, *elm_b;
-
- /*
- * We need to hold the lock to protect against a concurrent coalesce
- * operation that sees us in a partial state.
- */
- extent_lock(tsdn, extent);
-
- if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true,
- &elm_a, &elm_b)) {
- extent_unlock(tsdn, extent);
- return true;
- }
-
- szind_t szind = extent_szind_get_maybe_invalid(extent);
- bool slab = extent_slab_get(extent);
- extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab);
- if (slab) {
- extent_interior_register(tsdn, rtree_ctx, extent, szind);
- }
-
- extent_unlock(tsdn, extent);
-
- if (config_prof && gdump_add) {
- extent_gdump_add(tsdn, extent);
- }
-
- return false;
-}
-
-static bool
-extent_register(tsdn_t *tsdn, extent_t *extent) {
- return extent_register_impl(tsdn, extent, true);
-}
-
-static bool
-extent_register_no_gdump_add(tsdn_t *tsdn, extent_t *extent) {
- return extent_register_impl(tsdn, extent, false);
-}
-
-static void
-extent_reregister(tsdn_t *tsdn, extent_t *extent) {
- bool err = extent_register(tsdn, extent);
- assert(!err);
-}
-
-/*
- * Removes all pointers to the given extent from the global rtree indices for
- * its interior. This is relevant for slab extents, for which we need to do
- * metadata lookups at places other than the head of the extent. We deregister
- * on the interior, then, when an extent moves from being an active slab to an
- * inactive state.
- */
-static void
-extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
- extent_t *extent) {
- size_t i;
-
- assert(extent_slab_get(extent));
-
- for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
- rtree_clear(tsdn, &extents_rtree, rtree_ctx,
- (uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
- LG_PAGE));
- }
-}
-
-/*
- * Removes all pointers to the given extent from the global rtree.
- */
-static void
-extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool gdump) {
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
- rtree_leaf_elm_t *elm_a, *elm_b;
- extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, true, false,
- &elm_a, &elm_b);
-
- extent_lock(tsdn, extent);
-
- extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false);
- if (extent_slab_get(extent)) {
- extent_interior_deregister(tsdn, rtree_ctx, extent);
- extent_slab_set(extent, false);
- }
-
- extent_unlock(tsdn, extent);
-
- if (config_prof && gdump) {
- extent_gdump_sub(tsdn, extent);
- }
-}
-
-static void
-extent_deregister(tsdn_t *tsdn, extent_t *extent) {
- extent_deregister_impl(tsdn, extent, true);
-}
-
-static void
-extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) {
- extent_deregister_impl(tsdn, extent, false);
-}
-
-/*
- * Tries to find and remove an extent from eset that can be used for the
- * given allocation request.
- */
-static extent_t *
-extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
- size_t pad, size_t alignment, bool slab, bool growing_retained) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, growing_retained ? 1 : 0);
- assert(alignment > 0);
- if (config_debug && new_addr != NULL) {
- /*
- * Non-NULL new_addr has two use cases:
- *
- * 1) Recycle a known-extant extent, e.g. during purging.
- * 2) Perform in-place expanding reallocation.
- *
- * Regardless of use case, new_addr must either refer to a
- * non-existing extent, or to the base of an extant extent,
- * since only active slabs support interior lookups (which of
- * course cannot be recycled).
- */
- assert(PAGE_ADDR2BASE(new_addr) == new_addr);
- assert(pad == 0);
- assert(alignment <= PAGE);
- }
-
- size_t esize = size + pad;
- malloc_mutex_lock(tsdn, &eset->mtx);
- extent_t *extent;
- if (new_addr != NULL) {
- extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr,
- false);
- if (extent != NULL) {
- /*
- * We might null-out extent to report an error, but we
- * still need to unlock the associated mutex after.
- */
- extent_t *unlock_extent = extent;
- assert(extent_base_get(extent) == new_addr);
- if (extent_arena_ind_get(extent)
- != arena_ind_get(arena) ||
- extent_size_get(extent) < esize ||
- extent_state_get(extent) !=
- eset_state_get(eset)) {
- extent = NULL;
- }
- extent_unlock(tsdn, unlock_extent);
- }
- } else {
- extent = eset_fit_locked(tsdn, eset, esize, alignment);
- }
- if (extent == NULL) {
- malloc_mutex_unlock(tsdn, &eset->mtx);
- return NULL;
- }
-
- extent_activate_locked(tsdn, arena, eset, extent);
- malloc_mutex_unlock(tsdn, &eset->mtx);
-
- return extent;
-}
-
-/*
- * Given an allocation request and an extent guaranteed to be able to satisfy
- * it, this splits off lead and trail extents, leaving extent pointing to an
- * extent satisfying the allocation.
- * This function doesn't put lead or trail into any eset_t; it's the caller's
- * job to ensure that they can be reused.
- */
-typedef enum {
- /*
- * Split successfully. lead, extent, and trail, are modified to extents
- * describing the ranges before, in, and after the given allocation.
- */
- extent_split_interior_ok,
- /*
- * The extent can't satisfy the given allocation request. None of the
- * input extent_t *s are touched.
- */
- extent_split_interior_cant_alloc,
- /*
- * In a potentially invalid state. Must leak (if *to_leak is non-NULL),
- * and salvage what's still salvageable (if *to_salvage is non-NULL).
- * None of lead, extent, or trail are valid.
- */
- extent_split_interior_error
-} extent_split_interior_result_t;
-
-static extent_split_interior_result_t
-extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx,
- /* The result of splitting, in case of success. */
- extent_t **extent, extent_t **lead, extent_t **trail,
- /* The mess to clean up, in case of error. */
- extent_t **to_leak, extent_t **to_salvage,
- void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
- szind_t szind, bool growing_retained) {
- size_t esize = size + pad;
- size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(*extent),
- PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(*extent);
- assert(new_addr == NULL || leadsize == 0);
- if (extent_size_get(*extent) < leadsize + esize) {
- return extent_split_interior_cant_alloc;
- }
- size_t trailsize = extent_size_get(*extent) - leadsize - esize;
-
- *lead = NULL;
- *trail = NULL;
- *to_leak = NULL;
- *to_salvage = NULL;
-
- /* Split the lead. */
- if (leadsize != 0) {
- *lead = *extent;
- *extent = extent_split_impl(tsdn, arena, ehooks, *lead,
- leadsize, SC_NSIZES, false, esize + trailsize, szind, slab,
- growing_retained);
- if (*extent == NULL) {
- *to_leak = *lead;
- *lead = NULL;
- return extent_split_interior_error;
- }
- }
-
- /* Split the trail. */
- if (trailsize != 0) {
- *trail = extent_split_impl(tsdn, arena, ehooks, *extent, esize,
- szind, slab, trailsize, SC_NSIZES, false, growing_retained);
- if (*trail == NULL) {
- *to_leak = *extent;
- *to_salvage = *lead;
- *lead = NULL;
- *extent = NULL;
- return extent_split_interior_error;
- }
- }
-
- if (leadsize == 0 && trailsize == 0) {
- /*
- * Splitting causes szind to be set as a side effect, but no
- * splitting occurred.
- */
- extent_szind_set(*extent, szind);
- if (szind != SC_NSIZES) {
- rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,
- (uintptr_t)extent_addr_get(*extent), szind, slab);
- if (slab && extent_size_get(*extent) > PAGE) {
- rtree_szind_slab_update(tsdn, &extents_rtree,
- rtree_ctx,
- (uintptr_t)extent_past_get(*extent) -
- (uintptr_t)PAGE, szind, slab);
- }
- }
- }
-
- return extent_split_interior_ok;
-}
-
-/*
- * This fulfills the indicated allocation request out of the given extent (which
- * the caller should have ensured was big enough). If there's any unused space
- * before or after the resulting allocation, that space is given its own extent
- * and put back into eset.
- */
-static extent_t *
-extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
- size_t pad, size_t alignment, bool slab, szind_t szind, extent_t *extent,
- bool growing_retained) {
- extent_t *lead;
- extent_t *trail;
- extent_t *to_leak;
- extent_t *to_salvage;
-
- extent_split_interior_result_t result = extent_split_interior(
- tsdn, arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
- &to_salvage, new_addr, size, pad, alignment, slab, szind,
- growing_retained);
-
- if (!maps_coalesce && result != extent_split_interior_ok
- && !opt_retain) {
- /*
- * Split isn't supported (implies Windows w/o retain). Avoid
- * leaking the eset.
- */
- assert(to_leak != NULL && lead == NULL && trail == NULL);
- extent_deactivate(tsdn, arena, eset, to_leak);
- return NULL;
- }
-
- if (result == extent_split_interior_ok) {
- if (lead != NULL) {
- extent_deactivate(tsdn, arena, eset, lead);
- }
- if (trail != NULL) {
- extent_deactivate(tsdn, arena, eset, trail);
- }
- return extent;
- } else {
- /*
- * We should have picked an extent that was large enough to
- * fulfill our allocation request.
- */
- assert(result == extent_split_interior_error);
- if (to_salvage != NULL) {
- extent_deregister(tsdn, to_salvage);
- }
- if (to_leak != NULL) {
- void *leak = extent_base_get(to_leak);
- extent_deregister_no_gdump_sub(tsdn, to_leak);
- extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak,
- growing_retained);
- assert(extent_lock_from_addr(tsdn, rtree_ctx, leak,
- false) == NULL);
- }
- return NULL;
- }
- unreachable();
-}
-
-static bool
-extent_need_manual_zero(arena_t *arena) {
- /*
- * Need to manually zero the extent on repopulating if either; 1) non
- * default extent hooks installed (in which case the purge semantics may
- * change); or 2) transparent huge pages enabled.
- */
- return (!ehooks_are_default(arena_get_ehooks(arena)) ||
- (opt_thp == thp_mode_always));
-}
-
-/*
- * Tries to satisfy the given allocation request by reusing one of the extents
- * in the given eset_t.
- */
-static extent_t *
-extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
- szind_t szind, bool *zero, bool *commit, bool growing_retained) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, growing_retained ? 1 : 0);
- assert(new_addr == NULL || !slab);
- assert(pad == 0 || !slab);
- assert(!*zero || !slab);
-
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
-
- extent_t *extent = extent_recycle_extract(tsdn, arena, ehooks,
- rtree_ctx, eset, new_addr, size, pad, alignment, slab,
- growing_retained);
- if (extent == NULL) {
- return NULL;
- }
-
- extent = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset,
- new_addr, size, pad, alignment, slab, szind, extent,
- growing_retained);
- if (extent == NULL) {
- return NULL;
- }
-
- if (*commit && !extent_committed_get(extent)) {
- if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
- extent_size_get(extent), growing_retained)) {
- extent_record(tsdn, arena, ehooks, eset, extent,
- growing_retained);
- return NULL;
- }
- if (!extent_need_manual_zero(arena)) {
- extent_zeroed_set(extent, true);
- }
- }
-
- if (extent_committed_get(extent)) {
- *commit = true;
- }
- if (extent_zeroed_get(extent)) {
- *zero = true;
- }
-
- if (pad != 0) {
- extent_addr_randomize(tsdn, arena, extent, alignment);
- }
- assert(extent_state_get(extent) == extent_state_active);
- if (slab) {
- extent_slab_set(extent, slab);
- extent_interior_register(tsdn, rtree_ctx, extent, szind);
- }
-
- if (*zero) {
- void *addr = extent_base_get(extent);
- if (!extent_zeroed_get(extent)) {
- size_t size = extent_size_get(extent);
- if (extent_need_manual_zero(arena) ||
- pages_purge_forced(addr, size)) {
- memset(addr, 0, size);
- }
- } else if (config_debug) {
- size_t *p = (size_t *)(uintptr_t)addr;
- /* Check the first page only. */
- for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {
- assert(p[i] == 0);
- }
- }
- }
- return extent;
-}
-
-/*
- * If virtual memory is retained, create increasingly larger extents from which
- * to split requested extents in order to limit the total number of disjoint
- * virtual memory ranges retained by each arena.
- */
-static extent_t *
-extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
- bool *zero, bool *commit) {
- malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);
- assert(pad == 0 || !slab);
- assert(!*zero || !slab);
-
- size_t esize = size + pad;
- size_t alloc_size_min = esize + PAGE_CEILING(alignment) - PAGE;
- /* Beware size_t wrap-around. */
- if (alloc_size_min < esize) {
- goto label_err;
- }
- /*
- * Find the next extent size in the series that would be large enough to
- * satisfy this request.
- */
- pszind_t egn_skip = 0;
- size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
- while (alloc_size < alloc_size_min) {
- egn_skip++;
- if (arena->extent_grow_next + egn_skip >=
- sz_psz2ind(SC_LARGE_MAXCLASS)) {
- /* Outside legal range. */
- goto label_err;
- }
- alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
- }
-
- extent_t *extent = extent_alloc(tsdn, arena);
- if (extent == NULL) {
- goto label_err;
- }
- bool zeroed = false;
- bool committed = false;
-
- void *ptr = ehooks_alloc(tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed,
- &committed, arena_ind_get(arena));
-
- extent_init(extent, arena_ind_get(arena), ptr, alloc_size, false,
- SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed,
- committed, true, EXTENT_IS_HEAD);
- if (ptr == NULL) {
- extent_dalloc(tsdn, arena, extent);
- goto label_err;
- }
-
- if (extent_register_no_gdump_add(tsdn, extent)) {
- extent_dalloc(tsdn, arena, extent);
- goto label_err;
- }
-
- if (extent_zeroed_get(extent) && extent_committed_get(extent)) {
- *zero = true;
- }
- if (extent_committed_get(extent)) {
- *commit = true;
- }
-
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
-
- extent_t *lead;
- extent_t *trail;
- extent_t *to_leak;
- extent_t *to_salvage;
- extent_split_interior_result_t result = extent_split_interior(tsdn,
- arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
- &to_salvage, NULL, size, pad, alignment, slab, szind, true);
-
- if (result == extent_split_interior_ok) {
- if (lead != NULL) {
- extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, lead, true);
- }
- if (trail != NULL) {
- extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, trail, true);
- }
- } else {
- /*
- * We should have allocated a sufficiently large extent; the
- * cant_alloc case should not occur.
- */
- assert(result == extent_split_interior_error);
- if (to_salvage != NULL) {
- if (config_prof) {
- extent_gdump_add(tsdn, to_salvage);
- }
- extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, to_salvage, true);
- }
- if (to_leak != NULL) {
- extent_deregister_no_gdump_sub(tsdn, to_leak);
- extents_abandon_vm(tsdn, arena, ehooks,
- &arena->eset_retained, to_leak, true);
- }
- goto label_err;
- }
-
- if (*commit && !extent_committed_get(extent)) {
- if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
- extent_size_get(extent), true)) {
- extent_record(tsdn, arena, ehooks,
- &arena->eset_retained, extent, true);
- goto label_err;
- }
- if (!extent_need_manual_zero(arena)) {
- extent_zeroed_set(extent, true);
- }
- }
-
- /*
- * Increment extent_grow_next if doing so wouldn't exceed the allowed
- * range.
- */
- if (arena->extent_grow_next + egn_skip + 1 <=
- arena->retain_grow_limit) {
- arena->extent_grow_next += egn_skip + 1;
- } else {
- arena->extent_grow_next = arena->retain_grow_limit;
- }
- /* All opportunities for failure are past. */
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
-
- if (config_prof) {
- /* Adjust gdump stats now that extent is final size. */
- extent_gdump_add(tsdn, extent);
- }
- if (pad != 0) {
- extent_addr_randomize(tsdn, arena, extent, alignment);
- }
- if (slab) {
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
- &rtree_ctx_fallback);
-
- extent_slab_set(extent, true);
- extent_interior_register(tsdn, rtree_ctx, extent, szind);
- }
- if (*zero && !extent_zeroed_get(extent)) {
- void *addr = extent_base_get(extent);
- size_t size = extent_size_get(extent);
- if (extent_need_manual_zero(arena) ||
- pages_purge_forced(addr, size)) {
- memset(addr, 0, size);
- }
- }
-
- return extent;
-label_err:
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
- return NULL;
-}
-
-static extent_t *
-extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
- szind_t szind, bool *zero, bool *commit) {
- assert(size != 0);
- assert(alignment != 0);
-
- malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);
-
- extent_t *extent = extent_recycle(tsdn, arena, ehooks,
- &arena->eset_retained, new_addr, size, pad, alignment, slab,
- szind, zero, commit, true);
- if (extent != NULL) {
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
- if (config_prof) {
- extent_gdump_add(tsdn, extent);
- }
- } else if (opt_retain && new_addr == NULL) {
- extent = extent_grow_retained(tsdn, arena, ehooks, size, pad,
- alignment, slab, szind, zero, commit);
- /* extent_grow_retained() always releases extent_grow_mtx. */
- } else {
- malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
- }
- malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx);
-
- return extent;
-}
-
-static extent_t *
-extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
- szind_t szind, bool *zero, bool *commit) {
- size_t esize = size + pad;
- extent_t *extent = extent_alloc(tsdn, arena);
- if (extent == NULL) {
- return NULL;
- }
- size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
- void *addr = ehooks_alloc(tsdn, ehooks, new_addr, esize, palignment,
- zero, commit, arena_ind_get(arena));
- if (addr == NULL) {
- extent_dalloc(tsdn, arena, extent);
- return NULL;
- }
- extent_init(extent, arena_ind_get(arena), addr, esize, slab, szind,
- arena_extent_sn_next(arena), extent_state_active, *zero, *commit,
- true, EXTENT_NOT_HEAD);
- if (pad != 0) {
- extent_addr_randomize(tsdn, arena, extent, alignment);
- }
- if (extent_register(tsdn, extent)) {
- extent_dalloc(tsdn, arena, extent);
- return NULL;
- }
-
- return extent;
-}
-
-extent_t *
-extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
- szind_t szind, bool *zero, bool *commit) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- extent_t *extent = extent_alloc_retained(tsdn, arena, ehooks, new_addr,
- size, pad, alignment, slab, szind, zero, commit);
- if (extent == NULL) {
- if (opt_retain && new_addr != NULL) {
- /*
- * When retain is enabled and new_addr is set, we do not
- * attempt extent_alloc_wrapper_hard which does mmap
- * that is very unlikely to succeed (unless it happens
- * to be at the end).
- */
- return NULL;
- }
- extent = extent_alloc_wrapper_hard(tsdn, arena, ehooks,
- new_addr, size, pad, alignment, slab, szind, zero, commit);
- }
-
- assert(extent == NULL || extent_dumpable_get(extent));
- return extent;
-}
-
-static bool
-extent_can_coalesce(arena_t *arena, eset_t *eset, const extent_t *inner,
- const extent_t *outer) {
- assert(extent_arena_ind_get(inner) == arena_ind_get(arena));
- if (extent_arena_ind_get(outer) != arena_ind_get(arena)) {
- return false;
- }
-
- assert(extent_state_get(inner) == extent_state_active);
- if (extent_state_get(outer) != eset->state) {
- return false;
- }
-
- if (extent_committed_get(inner) != extent_committed_get(outer)) {
- return false;
- }
-
- return true;
-}
-
-static bool
-extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- extent_t *inner, extent_t *outer, bool forward, bool growing_retained) {
- assert(extent_can_coalesce(arena, eset, inner, outer));
-
- extent_activate_locked(tsdn, arena, eset, outer);
-
- malloc_mutex_unlock(tsdn, &eset->mtx);
- bool err = extent_merge_impl(tsdn, arena, ehooks,
- forward ? inner : outer, forward ? outer : inner, growing_retained);
- malloc_mutex_lock(tsdn, &eset->mtx);
-
- if (err) {
- extent_deactivate_locked(tsdn, arena, eset, outer);
- }
-
- return err;
-}
-
-static extent_t *
-extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
- bool growing_retained, bool inactive_only) {
- /*
- * We avoid checking / locking inactive neighbors for large size
- * classes, since they are eagerly coalesced on deallocation which can
- * cause lock contention.
- */
- /*
- * Continue attempting to coalesce until failure, to protect against
- * races with other threads that are thwarted by this one.
- */
- bool again;
- do {
- again = false;
-
- /* Try to coalesce forward. */
- extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx,
- extent_past_get(extent), inactive_only);
- if (next != NULL) {
- /*
- * eset->mtx only protects against races for
- * like-state eset, so call extent_can_coalesce()
- * before releasing next's pool lock.
- */
- bool can_coalesce = extent_can_coalesce(arena, eset,
- extent, next);
-
- extent_unlock(tsdn, next);
-
- if (can_coalesce && !extent_coalesce(tsdn, arena,
- ehooks, eset, extent, next, true,
- growing_retained)) {
- if (eset->delay_coalesce) {
- /* Do minimal coalescing. */
- *coalesced = true;
- return extent;
- }
- again = true;
- }
- }
-
- /* Try to coalesce backward. */
- extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx,
- extent_before_get(extent), inactive_only);
- if (prev != NULL) {
- bool can_coalesce = extent_can_coalesce(arena, eset,
- extent, prev);
- extent_unlock(tsdn, prev);
-
- if (can_coalesce && !extent_coalesce(tsdn, arena,
- ehooks, eset, extent, prev, false,
- growing_retained)) {
- extent = prev;
- if (eset->delay_coalesce) {
- /* Do minimal coalescing. */
- *coalesced = true;
- return extent;
- }
- again = true;
- }
- }
- } while (again);
-
- if (eset->delay_coalesce) {
- *coalesced = false;
- }
- return extent;
-}
-
-static extent_t *
-extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
- bool growing_retained) {
- return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
- extent, coalesced, growing_retained, false);
-}
-
-static extent_t *
-extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
- bool growing_retained) {
- return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
- extent, coalesced, growing_retained, true);
-}
-
-/*
- * Does the metadata management portions of putting an unused extent into the
- * given eset_t (coalesces, deregisters slab interiors, the heap operations).
- */
-static void
-extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
- extent_t *extent, bool growing_retained) {
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
-
- assert((eset_state_get(eset) != extent_state_dirty &&
- eset_state_get(eset) != extent_state_muzzy) ||
- !extent_zeroed_get(extent));
-
- malloc_mutex_lock(tsdn, &eset->mtx);
-
- extent_szind_set(extent, SC_NSIZES);
- if (extent_slab_get(extent)) {
- extent_interior_deregister(tsdn, rtree_ctx, extent);
- extent_slab_set(extent, false);
- }
-
- assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,
- (uintptr_t)extent_base_get(extent), true) == extent);
-
- if (!eset->delay_coalesce) {
- extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx,
- eset, extent, NULL, growing_retained);
- } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) {
- assert(eset == &arena->eset_dirty);
- /* Always coalesce large eset eagerly. */
- bool coalesced;
- do {
- assert(extent_state_get(extent) == extent_state_active);
- extent = extent_try_coalesce_large(tsdn, arena, ehooks,
- rtree_ctx, eset, extent, &coalesced,
- growing_retained);
- } while (coalesced);
- if (extent_size_get(extent) >= oversize_threshold) {
- /* Shortcut to purge the oversize extent eagerly. */
- malloc_mutex_unlock(tsdn, &eset->mtx);
- arena_decay_extent(tsdn, arena, ehooks, extent);
- return;
- }
- }
- extent_deactivate_locked(tsdn, arena, eset, extent);
-
- malloc_mutex_unlock(tsdn, &eset->mtx);
-}
-
-void
-extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
- ehooks_t *ehooks = arena_get_ehooks(arena);
-
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- if (extent_register(tsdn, extent)) {
- extent_dalloc(tsdn, arena, extent);
- return;
- }
- extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
-}
-
-static bool
-extent_may_dalloc(void) {
- /* With retain enabled, the default dalloc always fails. */
- return !opt_retain;
-}
-
-static bool
-extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent) {
- bool err;
-
- assert(extent_base_get(extent) != NULL);
- assert(extent_size_get(extent) != 0);
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- extent_addr_set(extent, extent_base_get(extent));
-
- /* Try to deallocate. */
- err = ehooks_dalloc(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), extent_committed_get(extent),
- arena_ind_get(arena));
-
- if (!err) {
- extent_dalloc(tsdn, arena, extent);
- }
-
- return err;
-}
-
-void
-extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent) {
- assert(extent_dumpable_get(extent));
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- /* Avoid calling the default extent_dalloc unless have to. */
- if (!ehooks_are_default(ehooks) || extent_may_dalloc()) {
- /*
- * Deregister first to avoid a race with other allocating
- * threads, and reregister if deallocation fails.
- */
- extent_deregister(tsdn, extent);
- if (!extent_dalloc_wrapper_try(tsdn, arena, ehooks, extent)) {
- return;
- }
- extent_reregister(tsdn, extent);
- }
-
- /* Try to decommit; purge if that fails. */
- bool zeroed;
- if (!extent_committed_get(extent)) {
- zeroed = true;
- } else if (!extent_decommit_wrapper(tsdn, arena, ehooks, extent, 0,
- extent_size_get(extent))) {
- zeroed = true;
- } else if (!ehooks_purge_forced(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), 0, extent_size_get(extent),
- arena_ind_get(arena))) {
- zeroed = true;
- } else if (extent_state_get(extent) == extent_state_muzzy ||
- !ehooks_purge_lazy(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), 0, extent_size_get(extent),
- arena_ind_get(arena))) {
- zeroed = false;
- } else {
- zeroed = false;
- }
- extent_zeroed_set(extent, zeroed);
-
- if (config_prof) {
- extent_gdump_sub(tsdn, extent);
- }
-
- extent_record(tsdn, arena, ehooks, &arena->eset_retained, extent,
- false);
-}
-
-void
-extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent) {
- assert(extent_base_get(extent) != NULL);
- assert(extent_size_get(extent) != 0);
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
-
- /* Deregister first to avoid a race with other allocating threads. */
- extent_deregister(tsdn, extent);
-
- extent_addr_set(extent, extent_base_get(extent));
-
- /* Try to destroy; silently fail otherwise. */
- ehooks_destroy(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), extent_committed_get(extent),
- arena_ind_get(arena));
-
- extent_dalloc(tsdn, arena, extent);
-}
-
-static bool
-extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset, size_t length, bool growing_retained) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, growing_retained ? 1 : 0);
- bool err = ehooks_commit(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), offset, length, arena_ind_get(arena));
- extent_committed_set(extent, extent_committed_get(extent) || !err);
- return err;
-}
-
-bool
-extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset,
- size_t length) {
- return extent_commit_impl(tsdn, arena, ehooks, extent, offset, length,
- false);
-}
-
-bool
-extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset, size_t length) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, 0);
- bool err = ehooks_decommit(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), offset, length, arena_ind_get(arena));
- extent_committed_set(extent, extent_committed_get(extent) && err);
- return err;
-}
-
-static bool
-extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset, size_t length, bool growing_retained) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, growing_retained ? 1 : 0);
- bool err = ehooks_purge_lazy(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), offset, length, arena_ind_get(arena));
- return err;
-}
-
-bool
-extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset, size_t length) {
- return extent_purge_lazy_impl(tsdn, arena, ehooks, extent, offset,
- length, false);
-}
-
-static bool
-extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset, size_t length, bool growing_retained) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, growing_retained ? 1 : 0);
- bool err = ehooks_purge_forced(tsdn, ehooks, extent_base_get(extent),
- extent_size_get(extent), offset, length, arena_ind_get(arena));
- return err;
-}
-
-bool
-extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t offset, size_t length) {
- return extent_purge_forced_impl(tsdn, arena, ehooks, extent,
- offset, length, false);
-}
-
-/*
- * Accepts the extent to split, and the characteristics of each side of the
- * split. The 'a' parameters go with the 'lead' of the resulting pair of
- * extents (the lower addressed portion of the split), and the 'b' parameters go
- * with the trail (the higher addressed portion). This makes 'extent' the lead,
- * and returns the trail (except in case of error).
- */
-static extent_t *
-extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
- size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained) {
- assert(extent_size_get(extent) == size_a + size_b);
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, growing_retained ? 1 : 0);
-
- if (ehooks_split_will_fail(ehooks)) {
- return NULL;
- }
-
- extent_t *trail = extent_alloc(tsdn, arena);
- if (trail == NULL) {
- goto label_error_a;
- }
-
- extent_init(trail, arena_ind_get(arena),
- (void *)((uintptr_t)extent_base_get(extent) + size_a), size_b,
- slab_b, szind_b, extent_sn_get(extent), extent_state_get(extent),
- extent_zeroed_get(extent), extent_committed_get(extent),
- extent_dumpable_get(extent), EXTENT_NOT_HEAD);
-
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
- rtree_leaf_elm_t *lead_elm_a, *lead_elm_b;
- {
- extent_t lead;
-
- extent_init(&lead, arena_ind_get(arena),
- extent_addr_get(extent), size_a,
- slab_a, szind_a, extent_sn_get(extent),
- extent_state_get(extent), extent_zeroed_get(extent),
- extent_committed_get(extent), extent_dumpable_get(extent),
- EXTENT_NOT_HEAD);
-
- extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false,
- true, &lead_elm_a, &lead_elm_b);
- }
- rtree_leaf_elm_t *trail_elm_a, *trail_elm_b;
- extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false, true,
- &trail_elm_a, &trail_elm_b);
-
- if (lead_elm_a == NULL || lead_elm_b == NULL || trail_elm_a == NULL
- || trail_elm_b == NULL) {
- goto label_error_b;
- }
-
- extent_lock2(tsdn, extent, trail);
-
- bool err = ehooks_split(tsdn, ehooks, extent_base_get(extent),
- size_a + size_b, size_a, size_b, extent_committed_get(extent),
- arena_ind_get(arena));
-
- if (err) {
- goto label_error_c;
- }
-
- extent_size_set(extent, size_a);
- extent_szind_set(extent, szind_a);
-
- extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent,
- szind_a, slab_a);
- extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail,
- szind_b, slab_b);
-
- extent_unlock2(tsdn, extent, trail);
-
- return trail;
-label_error_c:
- extent_unlock2(tsdn, extent, trail);
-label_error_b:
- extent_dalloc(tsdn, arena, trail);
-label_error_a:
- return NULL;
-}
-
-extent_t *
-extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
- size_t size_b, szind_t szind_b, bool slab_b) {
- return extent_split_impl(tsdn, arena, ehooks, extent, size_a, szind_a,
- slab_a, size_b, szind_b, slab_b, false);
-}
-
-/*
- * Returns true if the given extents can't be merged because of their head bit
- * settings. Assumes the second extent has the higher address.
- */
-bool
-extent_head_no_merge(extent_t *a, extent_t *b) {
- assert(extent_base_get(a) < extent_base_get(b));
- /*
- * When coalesce is not always allowed (Windows), only merge extents
- * from the same VirtualAlloc region under opt.retain (in which case
- * MEM_DECOMMIT is utilized for purging).
- */
- if (maps_coalesce) {
- return false;
- }
- if (!opt_retain) {
- return true;
- }
- /* If b is a head extent, disallow the cross-region merge. */
- if (extent_is_head_get(b)) {
- /*
- * Additionally, sn should not overflow with retain; sanity
- * check that different regions have unique sn.
- */
- assert(extent_sn_comp(a, b) != 0);
- return true;
- }
- assert(extent_sn_comp(a, b) == 0);
-
- return false;
-}
-
-static bool
-extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a,
- extent_t *b, bool growing_retained) {
- witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
- WITNESS_RANK_CORE, growing_retained ? 1 : 0);
- assert(extent_base_get(a) < extent_base_get(b));
-
- if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) {
- return true;
- }
-
- bool err = ehooks_merge(tsdn, ehooks, extent_base_get(a),
- extent_size_get(a), extent_base_get(b), extent_size_get(b),
- extent_committed_get(a), arena_ind_get(arena));
-
- if (err) {
- return true;
- }
-
- /*
- * The rtree writes must happen while all the relevant elements are
- * owned, so the following code uses decomposed helper functions rather
- * than extent_{,de}register() to do things in the right order.
- */
- rtree_ctx_t rtree_ctx_fallback;
- rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
- rtree_leaf_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
- extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true, false, &a_elm_a,
- &a_elm_b);
- extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true, false, &b_elm_a,
- &b_elm_b);
-
- extent_lock2(tsdn, a, b);
-
- if (a_elm_b != NULL) {
- rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL,
- SC_NSIZES, false);
- }
- if (b_elm_b != NULL) {
- rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL,
- SC_NSIZES, false);
- } else {
- b_elm_b = b_elm_a;
- }
-
- extent_size_set(a, extent_size_get(a) + extent_size_get(b));
- extent_szind_set(a, SC_NSIZES);
- extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ?
- extent_sn_get(a) : extent_sn_get(b));
- extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b));
-
- extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES,
- false);
-
- extent_unlock2(tsdn, a, b);
-
- /*
- * If we got here, we merged the extents; so they must be from the same
- * arena (i.e. this one).
- */
- assert(extent_arena_ind_get(b) == arena_ind_get(arena));
- extent_dalloc(tsdn, arena, b);
-
- return false;
-}
-
-bool
-extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
- extent_t *a, extent_t *b) {
- return extent_merge_impl(tsdn, arena, ehooks, a, b, false);
-}
-
-bool
-extent_boot(void) {
- if (rtree_new(&extents_rtree, true)) {
- return true;
- }
-
- if (mutex_pool_init(&extent_mutex_pool, "extent_mutex_pool",
- WITNESS_RANK_EXTENT_POOL)) {
- return true;
- }
-
- if (have_dss) {
- extent_dss_boot();
- }
-
- return false;
-}
-
-void
-extent_util_stats_get(tsdn_t *tsdn, const void *ptr,
- size_t *nfree, size_t *nregs, size_t *size) {
- assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL);
-
- const extent_t *extent = iealloc(tsdn, ptr);
- if (unlikely(extent == NULL)) {
- *nfree = *nregs = *size = 0;
- return;
- }
-
- *size = extent_size_get(extent);
- if (!extent_slab_get(extent)) {
- *nfree = 0;
- *nregs = 1;
- } else {
- *nfree = extent_nfree_get(extent);
- *nregs = bin_infos[extent_szind_get(extent)].nregs;
- assert(*nfree <= *nregs);
- assert(*nfree * extent_usize_get(extent) <= *size);
- }
-}
-
-void
-extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
- size_t *nfree, size_t *nregs, size_t *size,
- size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr) {
- assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL
- && bin_nfree != NULL && bin_nregs != NULL && slabcur_addr != NULL);
-
- const extent_t *extent = iealloc(tsdn, ptr);
- if (unlikely(extent == NULL)) {
- *nfree = *nregs = *size = *bin_nfree = *bin_nregs = 0;
- *slabcur_addr = NULL;
- return;
- }
-
- *size = extent_size_get(extent);
- if (!extent_slab_get(extent)) {
- *nfree = *bin_nfree = *bin_nregs = 0;
- *nregs = 1;
- *slabcur_addr = NULL;
- return;
- }
-
- *nfree = extent_nfree_get(extent);
- const szind_t szind = extent_szind_get(extent);
- *nregs = bin_infos[szind].nregs;
- assert(*nfree <= *nregs);
- assert(*nfree * extent_usize_get(extent) <= *size);
-
- const arena_t *arena = (arena_t *)atomic_load_p(
- &arenas[extent_arena_ind_get(extent)], ATOMIC_RELAXED);
- assert(arena != NULL);
- const unsigned binshard = extent_binshard_get(extent);
- bin_t *bin = &arena->bins[szind].bin_shards[binshard];
-
- malloc_mutex_lock(tsdn, &bin->lock);
- if (config_stats) {
- *bin_nregs = *nregs * bin->stats.curslabs;
- assert(*bin_nregs >= bin->stats.curregs);
- *bin_nfree = *bin_nregs - bin->stats.curregs;
- } else {
- *bin_nfree = *bin_nregs = 0;
- }
- extent_t *slab;
- if (bin->slabcur != NULL) {
- slab = bin->slabcur;
- } else {
- slab = extent_heap_first(&bin->slabs_nonfull);
- }
- *slabcur_addr = slab != NULL ? extent_addr_get(slab) : NULL;
- malloc_mutex_unlock(tsdn, &bin->lock);
-}
diff --git a/src/extent2.c b/src/extent2.c
new file mode 100644
index 00000000..4865beb1
--- /dev/null
+++ b/src/extent2.c
@@ -0,0 +1,1738 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/extent_dss.h"
+#include "jemalloc/internal/extent_mmap.h"
+#include "jemalloc/internal/ph.h"
+#include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/mutex_pool.h"
+
+/******************************************************************************/
+/* Data. */
+
+rtree_t extents_rtree;
+/* Keyed by the address of the extent_t being protected. */
+mutex_pool_t extent_mutex_pool;
+
+size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT;
+
+static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset, size_t length, bool growing_retained);
+static bool extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena,
+ ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
+ bool growing_retained);
+static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena,
+ ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length,
+ bool growing_retained);
+static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena,
+ ehooks_t *ehooks, extent_t *extent, size_t size_a, szind_t szind_a,
+ bool slab_a, size_t size_b, szind_t szind_b, bool slab_b,
+ bool growing_retained);
+static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *a, extent_t *b, bool growing_retained);
+
+/* Used exclusively for gdump triggering. */
+static atomic_zu_t curpages;
+static atomic_zu_t highpages;
+
+/******************************************************************************/
+/*
+ * Function prototypes for static functions that are referenced prior to
+ * definition.
+ */
+
+static void extent_deregister(tsdn_t *tsdn, extent_t *extent);
+static extent_t *extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ eset_t *eset, void *new_addr, size_t usize, size_t pad, size_t alignment,
+ bool slab, szind_t szind, bool *zero, bool *commit, bool growing_retained);
+static extent_t *extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
+ ehooks_t *ehooks, rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent,
+ bool *coalesced, bool growing_retained);
+static void extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ eset_t *eset, extent_t *extent, bool growing_retained);
+
+/******************************************************************************/
+
+typedef enum {
+ lock_result_success,
+ lock_result_failure,
+ lock_result_no_extent
+} lock_result_t;
+
+static inline void
+extent_lock(tsdn_t *tsdn, extent_t *extent) {
+ assert(extent != NULL);
+ mutex_pool_lock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
+}
+
+static inline void
+extent_unlock(tsdn_t *tsdn, extent_t *extent) {
+ assert(extent != NULL);
+ mutex_pool_unlock(tsdn, &extent_mutex_pool, (uintptr_t)extent);
+}
+
+static inline void
+extent_lock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
+ assert(extent1 != NULL && extent2 != NULL);
+ mutex_pool_lock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
+ (uintptr_t)extent2);
+}
+
+static inline void
+extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) {
+ assert(extent1 != NULL && extent2 != NULL);
+ mutex_pool_unlock2(tsdn, &extent_mutex_pool, (uintptr_t)extent1,
+ (uintptr_t)extent2);
+}
+
+static lock_result_t
+extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm,
+ extent_t **result, bool inactive_only) {
+ extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree,
+ elm, true);
+
+ /* Slab implies active extents and should be skipped. */
+ if (extent1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn,
+ &extents_rtree, elm, true))) {
+ return lock_result_no_extent;
+ }
+
+ /*
+ * It's possible that the extent changed out from under us, and with it
+ * the leaf->extent mapping. We have to recheck while holding the lock.
+ */
+ extent_lock(tsdn, extent1);
+ extent_t *extent2 = rtree_leaf_elm_extent_read(tsdn,
+ &extents_rtree, elm, true);
+
+ if (extent1 == extent2) {
+ *result = extent1;
+ return lock_result_success;
+ } else {
+ extent_unlock(tsdn, extent1);
+ return lock_result_failure;
+ }
+}
+
+/*
+ * Returns a pool-locked extent_t * if there's one associated with the given
+ * address, and NULL otherwise.
+ */
+static extent_t *
+extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr,
+ bool inactive_only) {
+ extent_t *ret = NULL;
+ rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree,
+ rtree_ctx, (uintptr_t)addr, false, false);
+ if (elm == NULL) {
+ return NULL;
+ }
+ lock_result_t lock_result;
+ do {
+ lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret,
+ inactive_only);
+ } while (lock_result == lock_result_failure);
+ return ret;
+}
+
+static void
+extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
+ size_t alignment) {
+ assert(extent_base_get(extent) == extent_addr_get(extent));
+
+ if (alignment < PAGE) {
+ unsigned lg_range = LG_PAGE -
+ lg_floor(CACHELINE_CEILING(alignment));
+ size_t r;
+ if (!tsdn_null(tsdn)) {
+ tsd_t *tsd = tsdn_tsd(tsdn);
+ r = (size_t)prng_lg_range_u64(
+ tsd_prng_statep_get(tsd), lg_range);
+ } else {
+ uint64_t stack_value = (uint64_t)(uintptr_t)&r;
+ r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
+ }
+ uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE -
+ lg_range);
+ extent->e_addr = (void *)((uintptr_t)extent->e_addr +
+ random_offset);
+ assert(ALIGNMENT_ADDR2BASE(extent->e_addr, alignment) ==
+ extent->e_addr);
+ }
+}
+
+extent_t *
+extent_alloc(tsdn_t *tsdn, arena_t *arena) {
+ malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
+ extent_t *extent = extent_avail_first(&arena->extent_avail);
+ if (extent == NULL) {
+ malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
+ return base_alloc_extent(tsdn, arena->base);
+ }
+ extent_avail_remove(&arena->extent_avail, extent);
+ atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED);
+ malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
+ return extent;
+}
+
+void
+extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
+ malloc_mutex_lock(tsdn, &arena->extent_avail_mtx);
+ extent_avail_insert(&arena->extent_avail, extent);
+ atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED);
+ malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx);
+}
+
+static bool
+extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent) {
+ extent_state_set(extent, extent_state_active);
+ bool coalesced;
+ extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx, eset,
+ extent, &coalesced, false);
+ extent_state_set(extent, eset_state_get(eset));
+
+ if (!coalesced) {
+ return true;
+ }
+ eset_insert_locked(tsdn, eset, extent);
+ return false;
+}
+
+extent_t *
+extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+ void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
+ szind_t szind, bool *zero, bool *commit) {
+ assert(size + pad != 0);
+ assert(alignment != 0);
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ extent_t *extent = extent_recycle(tsdn, arena, ehooks, eset, new_addr,
+ size, pad, alignment, slab, szind, zero, commit, false);
+ assert(extent == NULL || extent_dumpable_get(extent));
+ return extent;
+}
+
+void
+extents_dalloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+ extent_t *extent) {
+ assert(extent_base_get(extent) != NULL);
+ assert(extent_size_get(extent) != 0);
+ assert(extent_dumpable_get(extent));
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ extent_addr_set(extent, extent_base_get(extent));
+ extent_zeroed_set(extent, false);
+
+ extent_record(tsdn, arena, ehooks, eset, extent, false);
+}
+
+extent_t *
+extents_evict(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+ size_t npages_min) {
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+
+ malloc_mutex_lock(tsdn, &eset->mtx);
+
+ /*
+ * Get the LRU coalesced extent, if any. If coalescing was delayed,
+ * the loop will iterate until the LRU extent is fully coalesced.
+ */
+ extent_t *extent;
+ while (true) {
+ /* Get the LRU extent, if any. */
+ extent = extent_list_first(&eset->lru);
+ if (extent == NULL) {
+ goto label_return;
+ }
+ /* Check the eviction limit. */
+ size_t extents_npages = atomic_load_zu(&eset->npages,
+ ATOMIC_RELAXED);
+ if (extents_npages <= npages_min) {
+ extent = NULL;
+ goto label_return;
+ }
+ eset_remove_locked(tsdn, eset, extent);
+ if (!eset->delay_coalesce) {
+ break;
+ }
+ /* Try to coalesce. */
+ if (extent_try_delayed_coalesce(tsdn, arena, ehooks, rtree_ctx,
+ eset, extent)) {
+ break;
+ }
+ /*
+ * The LRU extent was just coalesced and the result placed in
+ * the LRU at its neighbor's position. Start over.
+ */
+ }
+
+ /*
+ * Either mark the extent active or deregister it to protect against
+ * concurrent operations.
+ */
+ switch (eset_state_get(eset)) {
+ case extent_state_active:
+ not_reached();
+ case extent_state_dirty:
+ case extent_state_muzzy:
+ extent_state_set(extent, extent_state_active);
+ break;
+ case extent_state_retained:
+ extent_deregister(tsdn, extent);
+ break;
+ default:
+ not_reached();
+ }
+
+label_return:
+ malloc_mutex_unlock(tsdn, &eset->mtx);
+ return extent;
+}
+
+/*
+ * This can only happen when we fail to allocate a new extent struct (which
+ * indicates OOM), e.g. when trying to split an existing extent.
+ */
+static void
+extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+ extent_t *extent, bool growing_retained) {
+ size_t sz = extent_size_get(extent);
+ if (config_stats) {
+ arena_stats_accum_zu(&arena->stats.abandoned_vm, sz);
+ }
+ /*
+ * Leak extent after making sure its pages have already been purged, so
+ * that this is only a virtual memory leak.
+ */
+ if (eset_state_get(eset) == extent_state_dirty) {
+ if (extent_purge_lazy_impl(tsdn, arena, ehooks, extent, 0, sz,
+ growing_retained)) {
+ extent_purge_forced_impl(tsdn, arena, ehooks, extent, 0,
+ extent_size_get(extent), growing_retained);
+ }
+ }
+ extent_dalloc(tsdn, arena, extent);
+}
+
+static void
+extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
+ extent_t *extent) {
+ assert(extent_arena_ind_get(extent) == arena_ind_get(arena));
+ assert(extent_state_get(extent) == extent_state_active);
+
+ extent_state_set(extent, eset_state_get(eset));
+ eset_insert_locked(tsdn, eset, extent);
+}
+
+static void
+extent_deactivate(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
+ extent_t *extent) {
+ malloc_mutex_lock(tsdn, &eset->mtx);
+ extent_deactivate_locked(tsdn, arena, eset, extent);
+ malloc_mutex_unlock(tsdn, &eset->mtx);
+}
+
+static void
+extent_activate_locked(tsdn_t *tsdn, arena_t *arena, eset_t *eset,
+ extent_t *extent) {
+ assert(extent_arena_ind_get(extent) == arena_ind_get(arena));
+ assert(extent_state_get(extent) == eset_state_get(eset));
+
+ eset_remove_locked(tsdn, eset, extent);
+ extent_state_set(extent, extent_state_active);
+}
+
+static bool
+extent_rtree_leaf_elms_lookup(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
+ const extent_t *extent, bool dependent, bool init_missing,
+ rtree_leaf_elm_t **r_elm_a, rtree_leaf_elm_t **r_elm_b) {
+ *r_elm_a = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
+ (uintptr_t)extent_base_get(extent), dependent, init_missing);
+ if (!dependent && *r_elm_a == NULL) {
+ return true;
+ }
+ assert(*r_elm_a != NULL);
+
+ *r_elm_b = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx,
+ (uintptr_t)extent_last_get(extent), dependent, init_missing);
+ if (!dependent && *r_elm_b == NULL) {
+ return true;
+ }
+ assert(*r_elm_b != NULL);
+
+ return false;
+}
+
+static void
+extent_rtree_write_acquired(tsdn_t *tsdn, rtree_leaf_elm_t *elm_a,
+ rtree_leaf_elm_t *elm_b, extent_t *extent, szind_t szind, bool slab) {
+ rtree_leaf_elm_write(tsdn, &extents_rtree, elm_a, extent, szind, slab);
+ if (elm_b != NULL) {
+ rtree_leaf_elm_write(tsdn, &extents_rtree, elm_b, extent, szind,
+ slab);
+ }
+}
+
+static void
+extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, extent_t *extent,
+ szind_t szind) {
+ assert(extent_slab_get(extent));
+
+ /* Register interior. */
+ for (size_t i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
+ rtree_write(tsdn, &extents_rtree, rtree_ctx,
+ (uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
+ LG_PAGE), extent, szind, true);
+ }
+}
+
+static void
+extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) {
+ cassert(config_prof);
+ /* prof_gdump() requirement. */
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ if (opt_prof && extent_state_get(extent) == extent_state_active) {
+ size_t nadd = extent_size_get(extent) >> LG_PAGE;
+ size_t cur = atomic_fetch_add_zu(&curpages, nadd,
+ ATOMIC_RELAXED) + nadd;
+ size_t high = atomic_load_zu(&highpages, ATOMIC_RELAXED);
+ while (cur > high && !atomic_compare_exchange_weak_zu(
+ &highpages, &high, cur, ATOMIC_RELAXED, ATOMIC_RELAXED)) {
+ /*
+ * Don't refresh cur, because it may have decreased
+ * since this thread lost the highpages update race.
+ * Note that high is updated in case of CAS failure.
+ */
+ }
+ if (cur > high && prof_gdump_get_unlocked()) {
+ prof_gdump(tsdn);
+ }
+ }
+}
+
+static void
+extent_gdump_sub(tsdn_t *tsdn, const extent_t *extent) {
+ cassert(config_prof);
+
+ if (opt_prof && extent_state_get(extent) == extent_state_active) {
+ size_t nsub = extent_size_get(extent) >> LG_PAGE;
+ assert(atomic_load_zu(&curpages, ATOMIC_RELAXED) >= nsub);
+ atomic_fetch_sub_zu(&curpages, nsub, ATOMIC_RELAXED);
+ }
+}
+
+static bool
+extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) {
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+ rtree_leaf_elm_t *elm_a, *elm_b;
+
+ /*
+ * We need to hold the lock to protect against a concurrent coalesce
+ * operation that sees us in a partial state.
+ */
+ extent_lock(tsdn, extent);
+
+ if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true,
+ &elm_a, &elm_b)) {
+ extent_unlock(tsdn, extent);
+ return true;
+ }
+
+ szind_t szind = extent_szind_get_maybe_invalid(extent);
+ bool slab = extent_slab_get(extent);
+ extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent, szind, slab);
+ if (slab) {
+ extent_interior_register(tsdn, rtree_ctx, extent, szind);
+ }
+
+ extent_unlock(tsdn, extent);
+
+ if (config_prof && gdump_add) {
+ extent_gdump_add(tsdn, extent);
+ }
+
+ return false;
+}
+
+static bool
+extent_register(tsdn_t *tsdn, extent_t *extent) {
+ return extent_register_impl(tsdn, extent, true);
+}
+
+static bool
+extent_register_no_gdump_add(tsdn_t *tsdn, extent_t *extent) {
+ return extent_register_impl(tsdn, extent, false);
+}
+
+static void
+extent_reregister(tsdn_t *tsdn, extent_t *extent) {
+ bool err = extent_register(tsdn, extent);
+ assert(!err);
+}
+
+/*
+ * Removes all pointers to the given extent from the global rtree indices for
+ * its interior. This is relevant for slab extents, for which we need to do
+ * metadata lookups at places other than the head of the extent. We deregister
+ * on the interior, then, when an extent moves from being an active slab to an
+ * inactive state.
+ */
+static void
+extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
+ extent_t *extent) {
+ size_t i;
+
+ assert(extent_slab_get(extent));
+
+ for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) {
+ rtree_clear(tsdn, &extents_rtree, rtree_ctx,
+ (uintptr_t)extent_base_get(extent) + (uintptr_t)(i <<
+ LG_PAGE));
+ }
+}
+
+/*
+ * Removes all pointers to the given extent from the global rtree.
+ */
+static void
+extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool gdump) {
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+ rtree_leaf_elm_t *elm_a, *elm_b;
+ extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, true, false,
+ &elm_a, &elm_b);
+
+ extent_lock(tsdn, extent);
+
+ extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false);
+ if (extent_slab_get(extent)) {
+ extent_interior_deregister(tsdn, rtree_ctx, extent);
+ extent_slab_set(extent, false);
+ }
+
+ extent_unlock(tsdn, extent);
+
+ if (config_prof && gdump) {
+ extent_gdump_sub(tsdn, extent);
+ }
+}
+
+static void
+extent_deregister(tsdn_t *tsdn, extent_t *extent) {
+ extent_deregister_impl(tsdn, extent, true);
+}
+
+static void
+extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) {
+ extent_deregister_impl(tsdn, extent, false);
+}
+
+/*
+ * Tries to find and remove an extent from eset that can be used for the
+ * given allocation request.
+ */
+static extent_t *
+extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
+ size_t pad, size_t alignment, bool slab, bool growing_retained) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, growing_retained ? 1 : 0);
+ assert(alignment > 0);
+ if (config_debug && new_addr != NULL) {
+ /*
+ * Non-NULL new_addr has two use cases:
+ *
+ * 1) Recycle a known-extant extent, e.g. during purging.
+ * 2) Perform in-place expanding reallocation.
+ *
+ * Regardless of use case, new_addr must either refer to a
+ * non-existing extent, or to the base of an extant extent,
+ * since only active slabs support interior lookups (which of
+ * course cannot be recycled).
+ */
+ assert(PAGE_ADDR2BASE(new_addr) == new_addr);
+ assert(pad == 0);
+ assert(alignment <= PAGE);
+ }
+
+ size_t esize = size + pad;
+ malloc_mutex_lock(tsdn, &eset->mtx);
+ extent_t *extent;
+ if (new_addr != NULL) {
+ extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr,
+ false);
+ if (extent != NULL) {
+ /*
+ * We might null-out extent to report an error, but we
+ * still need to unlock the associated mutex after.
+ */
+ extent_t *unlock_extent = extent;
+ assert(extent_base_get(extent) == new_addr);
+ if (extent_arena_ind_get(extent)
+ != arena_ind_get(arena) ||
+ extent_size_get(extent) < esize ||
+ extent_state_get(extent) !=
+ eset_state_get(eset)) {
+ extent = NULL;
+ }
+ extent_unlock(tsdn, unlock_extent);
+ }
+ } else {
+ extent = eset_fit_locked(tsdn, eset, esize, alignment);
+ }
+ if (extent == NULL) {
+ malloc_mutex_unlock(tsdn, &eset->mtx);
+ return NULL;
+ }
+
+ extent_activate_locked(tsdn, arena, eset, extent);
+ malloc_mutex_unlock(tsdn, &eset->mtx);
+
+ return extent;
+}
+
+/*
+ * Given an allocation request and an extent guaranteed to be able to satisfy
+ * it, this splits off lead and trail extents, leaving extent pointing to an
+ * extent satisfying the allocation.
+ * This function doesn't put lead or trail into any eset_t; it's the caller's
+ * job to ensure that they can be reused.
+ */
+typedef enum {
+ /*
+ * Split successfully. lead, extent, and trail, are modified to extents
+ * describing the ranges before, in, and after the given allocation.
+ */
+ extent_split_interior_ok,
+ /*
+ * The extent can't satisfy the given allocation request. None of the
+ * input extent_t *s are touched.
+ */
+ extent_split_interior_cant_alloc,
+ /*
+ * In a potentially invalid state. Must leak (if *to_leak is non-NULL),
+ * and salvage what's still salvageable (if *to_salvage is non-NULL).
+ * None of lead, extent, or trail are valid.
+ */
+ extent_split_interior_error
+} extent_split_interior_result_t;
+
+static extent_split_interior_result_t
+extent_split_interior(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ rtree_ctx_t *rtree_ctx,
+ /* The result of splitting, in case of success. */
+ extent_t **extent, extent_t **lead, extent_t **trail,
+ /* The mess to clean up, in case of error. */
+ extent_t **to_leak, extent_t **to_salvage,
+ void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
+ szind_t szind, bool growing_retained) {
+ size_t esize = size + pad;
+ size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(*extent),
+ PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(*extent);
+ assert(new_addr == NULL || leadsize == 0);
+ if (extent_size_get(*extent) < leadsize + esize) {
+ return extent_split_interior_cant_alloc;
+ }
+ size_t trailsize = extent_size_get(*extent) - leadsize - esize;
+
+ *lead = NULL;
+ *trail = NULL;
+ *to_leak = NULL;
+ *to_salvage = NULL;
+
+ /* Split the lead. */
+ if (leadsize != 0) {
+ *lead = *extent;
+ *extent = extent_split_impl(tsdn, arena, ehooks, *lead,
+ leadsize, SC_NSIZES, false, esize + trailsize, szind, slab,
+ growing_retained);
+ if (*extent == NULL) {
+ *to_leak = *lead;
+ *lead = NULL;
+ return extent_split_interior_error;
+ }
+ }
+
+ /* Split the trail. */
+ if (trailsize != 0) {
+ *trail = extent_split_impl(tsdn, arena, ehooks, *extent, esize,
+ szind, slab, trailsize, SC_NSIZES, false, growing_retained);
+ if (*trail == NULL) {
+ *to_leak = *extent;
+ *to_salvage = *lead;
+ *lead = NULL;
+ *extent = NULL;
+ return extent_split_interior_error;
+ }
+ }
+
+ if (leadsize == 0 && trailsize == 0) {
+ /*
+ * Splitting causes szind to be set as a side effect, but no
+ * splitting occurred.
+ */
+ extent_szind_set(*extent, szind);
+ if (szind != SC_NSIZES) {
+ rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx,
+ (uintptr_t)extent_addr_get(*extent), szind, slab);
+ if (slab && extent_size_get(*extent) > PAGE) {
+ rtree_szind_slab_update(tsdn, &extents_rtree,
+ rtree_ctx,
+ (uintptr_t)extent_past_get(*extent) -
+ (uintptr_t)PAGE, szind, slab);
+ }
+ }
+ }
+
+ return extent_split_interior_ok;
+}
+
+/*
+ * This fulfills the indicated allocation request out of the given extent (which
+ * the caller should have ensured was big enough). If there's any unused space
+ * before or after the resulting allocation, that space is given its own extent
+ * and put back into eset.
+ */
+static extent_t *
+extent_recycle_split(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ rtree_ctx_t *rtree_ctx, eset_t *eset, void *new_addr, size_t size,
+ size_t pad, size_t alignment, bool slab, szind_t szind, extent_t *extent,
+ bool growing_retained) {
+ extent_t *lead;
+ extent_t *trail;
+ extent_t *to_leak;
+ extent_t *to_salvage;
+
+ extent_split_interior_result_t result = extent_split_interior(
+ tsdn, arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
+ &to_salvage, new_addr, size, pad, alignment, slab, szind,
+ growing_retained);
+
+ if (!maps_coalesce && result != extent_split_interior_ok
+ && !opt_retain) {
+ /*
+ * Split isn't supported (implies Windows w/o retain). Avoid
+ * leaking the eset.
+ */
+ assert(to_leak != NULL && lead == NULL && trail == NULL);
+ extent_deactivate(tsdn, arena, eset, to_leak);
+ return NULL;
+ }
+
+ if (result == extent_split_interior_ok) {
+ if (lead != NULL) {
+ extent_deactivate(tsdn, arena, eset, lead);
+ }
+ if (trail != NULL) {
+ extent_deactivate(tsdn, arena, eset, trail);
+ }
+ return extent;
+ } else {
+ /*
+ * We should have picked an extent that was large enough to
+ * fulfill our allocation request.
+ */
+ assert(result == extent_split_interior_error);
+ if (to_salvage != NULL) {
+ extent_deregister(tsdn, to_salvage);
+ }
+ if (to_leak != NULL) {
+ void *leak = extent_base_get(to_leak);
+ extent_deregister_no_gdump_sub(tsdn, to_leak);
+ extents_abandon_vm(tsdn, arena, ehooks, eset, to_leak,
+ growing_retained);
+ assert(extent_lock_from_addr(tsdn, rtree_ctx, leak,
+ false) == NULL);
+ }
+ return NULL;
+ }
+ unreachable();
+}
+
+static bool
+extent_need_manual_zero(arena_t *arena) {
+ /*
+ * Need to manually zero the extent on repopulating if either; 1) non
+ * default extent hooks installed (in which case the purge semantics may
+ * change); or 2) transparent huge pages enabled.
+ */
+ return (!ehooks_are_default(arena_get_ehooks(arena)) ||
+ (opt_thp == thp_mode_always));
+}
+
+/*
+ * Tries to satisfy the given allocation request by reusing one of the extents
+ * in the given eset_t.
+ */
+static extent_t *
+extent_recycle(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+ void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
+ szind_t szind, bool *zero, bool *commit, bool growing_retained) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, growing_retained ? 1 : 0);
+ assert(new_addr == NULL || !slab);
+ assert(pad == 0 || !slab);
+ assert(!*zero || !slab);
+
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+
+ extent_t *extent = extent_recycle_extract(tsdn, arena, ehooks,
+ rtree_ctx, eset, new_addr, size, pad, alignment, slab,
+ growing_retained);
+ if (extent == NULL) {
+ return NULL;
+ }
+
+ extent = extent_recycle_split(tsdn, arena, ehooks, rtree_ctx, eset,
+ new_addr, size, pad, alignment, slab, szind, extent,
+ growing_retained);
+ if (extent == NULL) {
+ return NULL;
+ }
+
+ if (*commit && !extent_committed_get(extent)) {
+ if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
+ extent_size_get(extent), growing_retained)) {
+ extent_record(tsdn, arena, ehooks, eset, extent,
+ growing_retained);
+ return NULL;
+ }
+ if (!extent_need_manual_zero(arena)) {
+ extent_zeroed_set(extent, true);
+ }
+ }
+
+ if (extent_committed_get(extent)) {
+ *commit = true;
+ }
+ if (extent_zeroed_get(extent)) {
+ *zero = true;
+ }
+
+ if (pad != 0) {
+ extent_addr_randomize(tsdn, arena, extent, alignment);
+ }
+ assert(extent_state_get(extent) == extent_state_active);
+ if (slab) {
+ extent_slab_set(extent, slab);
+ extent_interior_register(tsdn, rtree_ctx, extent, szind);
+ }
+
+ if (*zero) {
+ void *addr = extent_base_get(extent);
+ if (!extent_zeroed_get(extent)) {
+ size_t size = extent_size_get(extent);
+ if (extent_need_manual_zero(arena) ||
+ pages_purge_forced(addr, size)) {
+ memset(addr, 0, size);
+ }
+ } else if (config_debug) {
+ size_t *p = (size_t *)(uintptr_t)addr;
+ /* Check the first page only. */
+ for (size_t i = 0; i < PAGE / sizeof(size_t); i++) {
+ assert(p[i] == 0);
+ }
+ }
+ }
+ return extent;
+}
+
+/*
+ * If virtual memory is retained, create increasingly larger extents from which
+ * to split requested extents in order to limit the total number of disjoint
+ * virtual memory ranges retained by each arena.
+ */
+static extent_t *
+extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ size_t size, size_t pad, size_t alignment, bool slab, szind_t szind,
+ bool *zero, bool *commit) {
+ malloc_mutex_assert_owner(tsdn, &arena->extent_grow_mtx);
+ assert(pad == 0 || !slab);
+ assert(!*zero || !slab);
+
+ size_t esize = size + pad;
+ size_t alloc_size_min = esize + PAGE_CEILING(alignment) - PAGE;
+ /* Beware size_t wrap-around. */
+ if (alloc_size_min < esize) {
+ goto label_err;
+ }
+ /*
+ * Find the next extent size in the series that would be large enough to
+ * satisfy this request.
+ */
+ pszind_t egn_skip = 0;
+ size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
+ while (alloc_size < alloc_size_min) {
+ egn_skip++;
+ if (arena->extent_grow_next + egn_skip >=
+ sz_psz2ind(SC_LARGE_MAXCLASS)) {
+ /* Outside legal range. */
+ goto label_err;
+ }
+ alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
+ }
+
+ extent_t *extent = extent_alloc(tsdn, arena);
+ if (extent == NULL) {
+ goto label_err;
+ }
+ bool zeroed = false;
+ bool committed = false;
+
+ void *ptr = ehooks_alloc(tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed,
+ &committed, arena_ind_get(arena));
+
+ extent_init(extent, arena_ind_get(arena), ptr, alloc_size, false,
+ SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed,
+ committed, true, EXTENT_IS_HEAD);
+ if (ptr == NULL) {
+ extent_dalloc(tsdn, arena, extent);
+ goto label_err;
+ }
+
+ if (extent_register_no_gdump_add(tsdn, extent)) {
+ extent_dalloc(tsdn, arena, extent);
+ goto label_err;
+ }
+
+ if (extent_zeroed_get(extent) && extent_committed_get(extent)) {
+ *zero = true;
+ }
+ if (extent_committed_get(extent)) {
+ *commit = true;
+ }
+
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+
+ extent_t *lead;
+ extent_t *trail;
+ extent_t *to_leak;
+ extent_t *to_salvage;
+ extent_split_interior_result_t result = extent_split_interior(tsdn,
+ arena, ehooks, rtree_ctx, &extent, &lead, &trail, &to_leak,
+ &to_salvage, NULL, size, pad, alignment, slab, szind, true);
+
+ if (result == extent_split_interior_ok) {
+ if (lead != NULL) {
+ extent_record(tsdn, arena, ehooks,
+ &arena->eset_retained, lead, true);
+ }
+ if (trail != NULL) {
+ extent_record(tsdn, arena, ehooks,
+ &arena->eset_retained, trail, true);
+ }
+ } else {
+ /*
+ * We should have allocated a sufficiently large extent; the
+ * cant_alloc case should not occur.
+ */
+ assert(result == extent_split_interior_error);
+ if (to_salvage != NULL) {
+ if (config_prof) {
+ extent_gdump_add(tsdn, to_salvage);
+ }
+ extent_record(tsdn, arena, ehooks,
+ &arena->eset_retained, to_salvage, true);
+ }
+ if (to_leak != NULL) {
+ extent_deregister_no_gdump_sub(tsdn, to_leak);
+ extents_abandon_vm(tsdn, arena, ehooks,
+ &arena->eset_retained, to_leak, true);
+ }
+ goto label_err;
+ }
+
+ if (*commit && !extent_committed_get(extent)) {
+ if (extent_commit_impl(tsdn, arena, ehooks, extent, 0,
+ extent_size_get(extent), true)) {
+ extent_record(tsdn, arena, ehooks,
+ &arena->eset_retained, extent, true);
+ goto label_err;
+ }
+ if (!extent_need_manual_zero(arena)) {
+ extent_zeroed_set(extent, true);
+ }
+ }
+
+ /*
+ * Increment extent_grow_next if doing so wouldn't exceed the allowed
+ * range.
+ */
+ if (arena->extent_grow_next + egn_skip + 1 <=
+ arena->retain_grow_limit) {
+ arena->extent_grow_next += egn_skip + 1;
+ } else {
+ arena->extent_grow_next = arena->retain_grow_limit;
+ }
+ /* All opportunities for failure are past. */
+ malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+
+ if (config_prof) {
+ /* Adjust gdump stats now that extent is final size. */
+ extent_gdump_add(tsdn, extent);
+ }
+ if (pad != 0) {
+ extent_addr_randomize(tsdn, arena, extent, alignment);
+ }
+ if (slab) {
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
+ &rtree_ctx_fallback);
+
+ extent_slab_set(extent, true);
+ extent_interior_register(tsdn, rtree_ctx, extent, szind);
+ }
+ if (*zero && !extent_zeroed_get(extent)) {
+ void *addr = extent_base_get(extent);
+ size_t size = extent_size_get(extent);
+ if (extent_need_manual_zero(arena) ||
+ pages_purge_forced(addr, size)) {
+ memset(addr, 0, size);
+ }
+ }
+
+ return extent;
+label_err:
+ malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+ return NULL;
+}
+
+static extent_t *
+extent_alloc_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
+ szind_t szind, bool *zero, bool *commit) {
+ assert(size != 0);
+ assert(alignment != 0);
+
+ malloc_mutex_lock(tsdn, &arena->extent_grow_mtx);
+
+ extent_t *extent = extent_recycle(tsdn, arena, ehooks,
+ &arena->eset_retained, new_addr, size, pad, alignment, slab,
+ szind, zero, commit, true);
+ if (extent != NULL) {
+ malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+ if (config_prof) {
+ extent_gdump_add(tsdn, extent);
+ }
+ } else if (opt_retain && new_addr == NULL) {
+ extent = extent_grow_retained(tsdn, arena, ehooks, size, pad,
+ alignment, slab, szind, zero, commit);
+ /* extent_grow_retained() always releases extent_grow_mtx. */
+ } else {
+ malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx);
+ }
+ malloc_mutex_assert_not_owner(tsdn, &arena->extent_grow_mtx);
+
+ return extent;
+}
+
+static extent_t *
+extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
+ szind_t szind, bool *zero, bool *commit) {
+ size_t esize = size + pad;
+ extent_t *extent = extent_alloc(tsdn, arena);
+ if (extent == NULL) {
+ return NULL;
+ }
+ size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
+ void *addr = ehooks_alloc(tsdn, ehooks, new_addr, esize, palignment,
+ zero, commit, arena_ind_get(arena));
+ if (addr == NULL) {
+ extent_dalloc(tsdn, arena, extent);
+ return NULL;
+ }
+ extent_init(extent, arena_ind_get(arena), addr, esize, slab, szind,
+ arena_extent_sn_next(arena), extent_state_active, *zero, *commit,
+ true, EXTENT_NOT_HEAD);
+ if (pad != 0) {
+ extent_addr_randomize(tsdn, arena, extent, alignment);
+ }
+ if (extent_register(tsdn, extent)) {
+ extent_dalloc(tsdn, arena, extent);
+ return NULL;
+ }
+
+ return extent;
+}
+
+extent_t *
+extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
+ szind_t szind, bool *zero, bool *commit) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ extent_t *extent = extent_alloc_retained(tsdn, arena, ehooks, new_addr,
+ size, pad, alignment, slab, szind, zero, commit);
+ if (extent == NULL) {
+ if (opt_retain && new_addr != NULL) {
+ /*
+ * When retain is enabled and new_addr is set, we do not
+ * attempt extent_alloc_wrapper_hard which does mmap
+ * that is very unlikely to succeed (unless it happens
+ * to be at the end).
+ */
+ return NULL;
+ }
+ extent = extent_alloc_wrapper_hard(tsdn, arena, ehooks,
+ new_addr, size, pad, alignment, slab, szind, zero, commit);
+ }
+
+ assert(extent == NULL || extent_dumpable_get(extent));
+ return extent;
+}
+
+static bool
+extent_can_coalesce(arena_t *arena, eset_t *eset, const extent_t *inner,
+ const extent_t *outer) {
+ assert(extent_arena_ind_get(inner) == arena_ind_get(arena));
+ if (extent_arena_ind_get(outer) != arena_ind_get(arena)) {
+ return false;
+ }
+
+ assert(extent_state_get(inner) == extent_state_active);
+ if (extent_state_get(outer) != eset->state) {
+ return false;
+ }
+
+ if (extent_committed_get(inner) != extent_committed_get(outer)) {
+ return false;
+ }
+
+ return true;
+}
+
+static bool
+extent_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+ extent_t *inner, extent_t *outer, bool forward, bool growing_retained) {
+ assert(extent_can_coalesce(arena, eset, inner, outer));
+
+ extent_activate_locked(tsdn, arena, eset, outer);
+
+ malloc_mutex_unlock(tsdn, &eset->mtx);
+ bool err = extent_merge_impl(tsdn, arena, ehooks,
+ forward ? inner : outer, forward ? outer : inner, growing_retained);
+ malloc_mutex_lock(tsdn, &eset->mtx);
+
+ if (err) {
+ extent_deactivate_locked(tsdn, arena, eset, outer);
+ }
+
+ return err;
+}
+
+static extent_t *
+extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
+ bool growing_retained, bool inactive_only) {
+ /*
+ * We avoid checking / locking inactive neighbors for large size
+ * classes, since they are eagerly coalesced on deallocation which can
+ * cause lock contention.
+ */
+ /*
+ * Continue attempting to coalesce until failure, to protect against
+ * races with other threads that are thwarted by this one.
+ */
+ bool again;
+ do {
+ again = false;
+
+ /* Try to coalesce forward. */
+ extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx,
+ extent_past_get(extent), inactive_only);
+ if (next != NULL) {
+ /*
+ * eset->mtx only protects against races for
+ * like-state eset, so call extent_can_coalesce()
+ * before releasing next's pool lock.
+ */
+ bool can_coalesce = extent_can_coalesce(arena, eset,
+ extent, next);
+
+ extent_unlock(tsdn, next);
+
+ if (can_coalesce && !extent_coalesce(tsdn, arena,
+ ehooks, eset, extent, next, true,
+ growing_retained)) {
+ if (eset->delay_coalesce) {
+ /* Do minimal coalescing. */
+ *coalesced = true;
+ return extent;
+ }
+ again = true;
+ }
+ }
+
+ /* Try to coalesce backward. */
+ extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx,
+ extent_before_get(extent), inactive_only);
+ if (prev != NULL) {
+ bool can_coalesce = extent_can_coalesce(arena, eset,
+ extent, prev);
+ extent_unlock(tsdn, prev);
+
+ if (can_coalesce && !extent_coalesce(tsdn, arena,
+ ehooks, eset, extent, prev, false,
+ growing_retained)) {
+ extent = prev;
+ if (eset->delay_coalesce) {
+ /* Do minimal coalescing. */
+ *coalesced = true;
+ return extent;
+ }
+ again = true;
+ }
+ }
+ } while (again);
+
+ if (eset->delay_coalesce) {
+ *coalesced = false;
+ }
+ return extent;
+}
+
+static extent_t *
+extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
+ bool growing_retained) {
+ return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
+ extent, coalesced, growing_retained, false);
+}
+
+static extent_t *
+extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ rtree_ctx_t *rtree_ctx, eset_t *eset, extent_t *extent, bool *coalesced,
+ bool growing_retained) {
+ return extent_try_coalesce_impl(tsdn, arena, ehooks, rtree_ctx, eset,
+ extent, coalesced, growing_retained, true);
+}
+
+/*
+ * Does the metadata management portions of putting an unused extent into the
+ * given eset_t (coalesces, deregisters slab interiors, the heap operations).
+ */
+static void
+extent_record(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
+ extent_t *extent, bool growing_retained) {
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+
+ assert((eset_state_get(eset) != extent_state_dirty &&
+ eset_state_get(eset) != extent_state_muzzy) ||
+ !extent_zeroed_get(extent));
+
+ malloc_mutex_lock(tsdn, &eset->mtx);
+
+ extent_szind_set(extent, SC_NSIZES);
+ if (extent_slab_get(extent)) {
+ extent_interior_deregister(tsdn, rtree_ctx, extent);
+ extent_slab_set(extent, false);
+ }
+
+ assert(rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,
+ (uintptr_t)extent_base_get(extent), true) == extent);
+
+ if (!eset->delay_coalesce) {
+ extent = extent_try_coalesce(tsdn, arena, ehooks, rtree_ctx,
+ eset, extent, NULL, growing_retained);
+ } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) {
+ assert(eset == &arena->eset_dirty);
+ /* Always coalesce large eset eagerly. */
+ bool coalesced;
+ do {
+ assert(extent_state_get(extent) == extent_state_active);
+ extent = extent_try_coalesce_large(tsdn, arena, ehooks,
+ rtree_ctx, eset, extent, &coalesced,
+ growing_retained);
+ } while (coalesced);
+ if (extent_size_get(extent) >= oversize_threshold) {
+ /* Shortcut to purge the oversize extent eagerly. */
+ malloc_mutex_unlock(tsdn, &eset->mtx);
+ arena_decay_extent(tsdn, arena, ehooks, extent);
+ return;
+ }
+ }
+ extent_deactivate_locked(tsdn, arena, eset, extent);
+
+ malloc_mutex_unlock(tsdn, &eset->mtx);
+}
+
+void
+extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) {
+ ehooks_t *ehooks = arena_get_ehooks(arena);
+
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ if (extent_register(tsdn, extent)) {
+ extent_dalloc(tsdn, arena, extent);
+ return;
+ }
+ extent_dalloc_wrapper(tsdn, arena, ehooks, extent);
+}
+
+static bool
+extent_may_dalloc(void) {
+ /* With retain enabled, the default dalloc always fails. */
+ return !opt_retain;
+}
+
+static bool
+extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent) {
+ bool err;
+
+ assert(extent_base_get(extent) != NULL);
+ assert(extent_size_get(extent) != 0);
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ extent_addr_set(extent, extent_base_get(extent));
+
+ /* Try to deallocate. */
+ err = ehooks_dalloc(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), extent_committed_get(extent),
+ arena_ind_get(arena));
+
+ if (!err) {
+ extent_dalloc(tsdn, arena, extent);
+ }
+
+ return err;
+}
+
+void
+extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent) {
+ assert(extent_dumpable_get(extent));
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ /* Avoid calling the default extent_dalloc unless have to. */
+ if (!ehooks_are_default(ehooks) || extent_may_dalloc()) {
+ /*
+ * Deregister first to avoid a race with other allocating
+ * threads, and reregister if deallocation fails.
+ */
+ extent_deregister(tsdn, extent);
+ if (!extent_dalloc_wrapper_try(tsdn, arena, ehooks, extent)) {
+ return;
+ }
+ extent_reregister(tsdn, extent);
+ }
+
+ /* Try to decommit; purge if that fails. */
+ bool zeroed;
+ if (!extent_committed_get(extent)) {
+ zeroed = true;
+ } else if (!extent_decommit_wrapper(tsdn, arena, ehooks, extent, 0,
+ extent_size_get(extent))) {
+ zeroed = true;
+ } else if (!ehooks_purge_forced(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), 0, extent_size_get(extent),
+ arena_ind_get(arena))) {
+ zeroed = true;
+ } else if (extent_state_get(extent) == extent_state_muzzy ||
+ !ehooks_purge_lazy(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), 0, extent_size_get(extent),
+ arena_ind_get(arena))) {
+ zeroed = false;
+ } else {
+ zeroed = false;
+ }
+ extent_zeroed_set(extent, zeroed);
+
+ if (config_prof) {
+ extent_gdump_sub(tsdn, extent);
+ }
+
+ extent_record(tsdn, arena, ehooks, &arena->eset_retained, extent,
+ false);
+}
+
+void
+extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent) {
+ assert(extent_base_get(extent) != NULL);
+ assert(extent_size_get(extent) != 0);
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+
+ /* Deregister first to avoid a race with other allocating threads. */
+ extent_deregister(tsdn, extent);
+
+ extent_addr_set(extent, extent_base_get(extent));
+
+ /* Try to destroy; silently fail otherwise. */
+ ehooks_destroy(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), extent_committed_get(extent),
+ arena_ind_get(arena));
+
+ extent_dalloc(tsdn, arena, extent);
+}
+
+static bool
+extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset, size_t length, bool growing_retained) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, growing_retained ? 1 : 0);
+ bool err = ehooks_commit(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), offset, length, arena_ind_get(arena));
+ extent_committed_set(extent, extent_committed_get(extent) || !err);
+ return err;
+}
+
+bool
+extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset,
+ size_t length) {
+ return extent_commit_impl(tsdn, arena, ehooks, extent, offset, length,
+ false);
+}
+
+bool
+extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset, size_t length) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, 0);
+ bool err = ehooks_decommit(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), offset, length, arena_ind_get(arena));
+ extent_committed_set(extent, extent_committed_get(extent) && err);
+ return err;
+}
+
+static bool
+extent_purge_lazy_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset, size_t length, bool growing_retained) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, growing_retained ? 1 : 0);
+ bool err = ehooks_purge_lazy(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), offset, length, arena_ind_get(arena));
+ return err;
+}
+
+bool
+extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset, size_t length) {
+ return extent_purge_lazy_impl(tsdn, arena, ehooks, extent, offset,
+ length, false);
+}
+
+static bool
+extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset, size_t length, bool growing_retained) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, growing_retained ? 1 : 0);
+ bool err = ehooks_purge_forced(tsdn, ehooks, extent_base_get(extent),
+ extent_size_get(extent), offset, length, arena_ind_get(arena));
+ return err;
+}
+
+bool
+extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t offset, size_t length) {
+ return extent_purge_forced_impl(tsdn, arena, ehooks, extent,
+ offset, length, false);
+}
+
+/*
+ * Accepts the extent to split, and the characteristics of each side of the
+ * split. The 'a' parameters go with the 'lead' of the resulting pair of
+ * extents (the lower addressed portion of the split), and the 'b' parameters go
+ * with the trail (the higher addressed portion). This makes 'extent' the lead,
+ * and returns the trail (except in case of error).
+ */
+static extent_t *
+extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
+ size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained) {
+ assert(extent_size_get(extent) == size_a + size_b);
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, growing_retained ? 1 : 0);
+
+ if (ehooks_split_will_fail(ehooks)) {
+ return NULL;
+ }
+
+ extent_t *trail = extent_alloc(tsdn, arena);
+ if (trail == NULL) {
+ goto label_error_a;
+ }
+
+ extent_init(trail, arena_ind_get(arena),
+ (void *)((uintptr_t)extent_base_get(extent) + size_a), size_b,
+ slab_b, szind_b, extent_sn_get(extent), extent_state_get(extent),
+ extent_zeroed_get(extent), extent_committed_get(extent),
+ extent_dumpable_get(extent), EXTENT_NOT_HEAD);
+
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+ rtree_leaf_elm_t *lead_elm_a, *lead_elm_b;
+ {
+ extent_t lead;
+
+ extent_init(&lead, arena_ind_get(arena),
+ extent_addr_get(extent), size_a,
+ slab_a, szind_a, extent_sn_get(extent),
+ extent_state_get(extent), extent_zeroed_get(extent),
+ extent_committed_get(extent), extent_dumpable_get(extent),
+ EXTENT_NOT_HEAD);
+
+ extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false,
+ true, &lead_elm_a, &lead_elm_b);
+ }
+ rtree_leaf_elm_t *trail_elm_a, *trail_elm_b;
+ extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, trail, false, true,
+ &trail_elm_a, &trail_elm_b);
+
+ if (lead_elm_a == NULL || lead_elm_b == NULL || trail_elm_a == NULL
+ || trail_elm_b == NULL) {
+ goto label_error_b;
+ }
+
+ extent_lock2(tsdn, extent, trail);
+
+ bool err = ehooks_split(tsdn, ehooks, extent_base_get(extent),
+ size_a + size_b, size_a, size_b, extent_committed_get(extent),
+ arena_ind_get(arena));
+
+ if (err) {
+ goto label_error_c;
+ }
+
+ extent_size_set(extent, size_a);
+ extent_szind_set(extent, szind_a);
+
+ extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent,
+ szind_a, slab_a);
+ extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail,
+ szind_b, slab_b);
+
+ extent_unlock2(tsdn, extent, trail);
+
+ return trail;
+label_error_c:
+ extent_unlock2(tsdn, extent, trail);
+label_error_b:
+ extent_dalloc(tsdn, arena, trail);
+label_error_a:
+ return NULL;
+}
+
+extent_t *
+extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a,
+ size_t size_b, szind_t szind_b, bool slab_b) {
+ return extent_split_impl(tsdn, arena, ehooks, extent, size_a, szind_a,
+ slab_a, size_b, szind_b, slab_b, false);
+}
+
+/*
+ * Returns true if the given extents can't be merged because of their head bit
+ * settings. Assumes the second extent has the higher address.
+ */
+bool
+extent_head_no_merge(extent_t *a, extent_t *b) {
+ assert(extent_base_get(a) < extent_base_get(b));
+ /*
+ * When coalesce is not always allowed (Windows), only merge extents
+ * from the same VirtualAlloc region under opt.retain (in which case
+ * MEM_DECOMMIT is utilized for purging).
+ */
+ if (maps_coalesce) {
+ return false;
+ }
+ if (!opt_retain) {
+ return true;
+ }
+ /* If b is a head extent, disallow the cross-region merge. */
+ if (extent_is_head_get(b)) {
+ /*
+ * Additionally, sn should not overflow with retain; sanity
+ * check that different regions have unique sn.
+ */
+ assert(extent_sn_comp(a, b) != 0);
+ return true;
+ }
+ assert(extent_sn_comp(a, b) == 0);
+
+ return false;
+}
+
+static bool
+extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a,
+ extent_t *b, bool growing_retained) {
+ witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
+ WITNESS_RANK_CORE, growing_retained ? 1 : 0);
+ assert(extent_base_get(a) < extent_base_get(b));
+
+ if (ehooks_merge_will_fail(ehooks) || extent_head_no_merge(a, b)) {
+ return true;
+ }
+
+ bool err = ehooks_merge(tsdn, ehooks, extent_base_get(a),
+ extent_size_get(a), extent_base_get(b), extent_size_get(b),
+ extent_committed_get(a), arena_ind_get(arena));
+
+ if (err) {
+ return true;
+ }
+
+ /*
+ * The rtree writes must happen while all the relevant elements are
+ * owned, so the following code uses decomposed helper functions rather
+ * than extent_{,de}register() to do things in the right order.
+ */
+ rtree_ctx_t rtree_ctx_fallback;
+ rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
+ rtree_leaf_elm_t *a_elm_a, *a_elm_b, *b_elm_a, *b_elm_b;
+ extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, a, true, false, &a_elm_a,
+ &a_elm_b);
+ extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, b, true, false, &b_elm_a,
+ &b_elm_b);
+
+ extent_lock2(tsdn, a, b);
+
+ if (a_elm_b != NULL) {
+ rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL,
+ SC_NSIZES, false);
+ }
+ if (b_elm_b != NULL) {
+ rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL,
+ SC_NSIZES, false);
+ } else {
+ b_elm_b = b_elm_a;
+ }
+
+ extent_size_set(a, extent_size_get(a) + extent_size_get(b));
+ extent_szind_set(a, SC_NSIZES);
+ extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ?
+ extent_sn_get(a) : extent_sn_get(b));
+ extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b));
+
+ extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES,
+ false);
+
+ extent_unlock2(tsdn, a, b);
+
+ /*
+ * If we got here, we merged the extents; so they must be from the same
+ * arena (i.e. this one).
+ */
+ assert(extent_arena_ind_get(b) == arena_ind_get(arena));
+ extent_dalloc(tsdn, arena, b);
+
+ return false;
+}
+
+bool
+extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
+ extent_t *a, extent_t *b) {
+ return extent_merge_impl(tsdn, arena, ehooks, a, b, false);
+}
+
+bool
+extent_boot(void) {
+ if (rtree_new(&extents_rtree, true)) {
+ return true;
+ }
+
+ if (mutex_pool_init(&extent_mutex_pool, "extent_mutex_pool",
+ WITNESS_RANK_EXTENT_POOL)) {
+ return true;
+ }
+
+ if (have_dss) {
+ extent_dss_boot();
+ }
+
+ return false;
+}
+
+void
+extent_util_stats_get(tsdn_t *tsdn, const void *ptr,
+ size_t *nfree, size_t *nregs, size_t *size) {
+ assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL);
+
+ const extent_t *extent = iealloc(tsdn, ptr);
+ if (unlikely(extent == NULL)) {
+ *nfree = *nregs = *size = 0;
+ return;
+ }
+
+ *size = extent_size_get(extent);
+ if (!extent_slab_get(extent)) {
+ *nfree = 0;
+ *nregs = 1;
+ } else {
+ *nfree = extent_nfree_get(extent);
+ *nregs = bin_infos[extent_szind_get(extent)].nregs;
+ assert(*nfree <= *nregs);
+ assert(*nfree * extent_usize_get(extent) <= *size);
+ }
+}
+
+void
+extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
+ size_t *nfree, size_t *nregs, size_t *size,
+ size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr) {
+ assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL
+ && bin_nfree != NULL && bin_nregs != NULL && slabcur_addr != NULL);
+
+ const extent_t *extent = iealloc(tsdn, ptr);
+ if (unlikely(extent == NULL)) {
+ *nfree = *nregs = *size = *bin_nfree = *bin_nregs = 0;
+ *slabcur_addr = NULL;
+ return;
+ }
+
+ *size = extent_size_get(extent);
+ if (!extent_slab_get(extent)) {
+ *nfree = *bin_nfree = *bin_nregs = 0;
+ *nregs = 1;
+ *slabcur_addr = NULL;
+ return;
+ }
+
+ *nfree = extent_nfree_get(extent);
+ const szind_t szind = extent_szind_get(extent);
+ *nregs = bin_infos[szind].nregs;
+ assert(*nfree <= *nregs);
+ assert(*nfree * extent_usize_get(extent) <= *size);
+
+ const arena_t *arena = (arena_t *)atomic_load_p(
+ &arenas[extent_arena_ind_get(extent)], ATOMIC_RELAXED);
+ assert(arena != NULL);
+ const unsigned binshard = extent_binshard_get(extent);
+ bin_t *bin = &arena->bins[szind].bin_shards[binshard];
+
+ malloc_mutex_lock(tsdn, &bin->lock);
+ if (config_stats) {
+ *bin_nregs = *nregs * bin->stats.curslabs;
+ assert(*bin_nregs >= bin->stats.curregs);
+ *bin_nfree = *bin_nregs - bin->stats.curregs;
+ } else {
+ *bin_nfree = *bin_nregs = 0;
+ }
+ extent_t *slab;
+ if (bin->slabcur != NULL) {
+ slab = bin->slabcur;
+ } else {
+ slab = extent_heap_first(&bin->slabs_nonfull);
+ }
+ *slabcur_addr = slab != NULL ? extent_addr_get(slab) : NULL;
+ malloc_mutex_unlock(tsdn, &bin->lock);
+}