project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pull out edata_t caching into its own module.

Browse Source
This commit is contained in:
David Goldblatt 2019-12-11 11:17:19 -08:00 committed by David Goldblatt
parent a7862df616
commit 7859184179
12 changed files with 106 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.in
View File

@ -105,6 +105,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/ctl.c \
$(srcroot)src/div.c \
$(srcroot)src/edata.c \
$(srcroot)src/edata_cache.c \
$(srcroot)src/ehooks.c \
$(srcroot)src/eset.c \
$(srcroot)src/extent2.c \

12
include/jemalloc/internal/arena_structs.h
View File

@ -5,6 +5,7 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bin.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/edata_cache.h"
#include "jemalloc/internal/eset.h"
#include "jemalloc/internal/extent_dss.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
@ -184,15 +185,8 @@ struct arena_s {
pszind_t retain_grow_limit;
malloc_mutex_t extent_grow_mtx;
/*
* Available edata structures that were allocated via
* base_alloc_edata().
*
* Synchronization: edata_avail_mtx.
*/
edata_tree_t edata_avail;
atomic_zu_t edata_avail_cnt;
malloc_mutex_t edata_avail_mtx;
/* The source of edata_t objects. */
edata_cache_t edata_cache;
/*
* bins is used to store heaps of free regions.

25
include/jemalloc/internal/edata_cache.h Normal file
View File

@ -0,0 +1,25 @@
#ifndef JEMALLOC_INTERNAL_EDATA_CACHE_H
#define JEMALLOC_INTERNAL_EDATA_CACHE_H
/*
* A cache of edata_t structures allocated via base_alloc_edata (as opposed to
* the underlying extents they describe). The contents of returned edata_t
* objects are garbage and cannot be relied upon.
*/
typedef struct edata_cache_s edata_cache_t;
struct edata_cache_s {
edata_tree_t avail;
atomic_zu_t count;
malloc_mutex_t mtx;
};
bool edata_cache_init(edata_cache_t *edata_cache);
edata_t *edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache,
base_t *base);
void edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata);
void edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache);
void edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache);
void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache);
#endif /* JEMALLOC_INTERNAL_EDATA_CACHE_H */

3
include/jemalloc/internal/extent2.h
View File

@ -26,9 +26,6 @@ extern size_t opt_lg_extent_max_active_fit;
extern rtree_t extents_rtree;
edata_t *extent_alloc(tsdn_t *tsdn, arena_t *arena);
void extent_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *edata);
edata_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
eset_t *eset, void *new_addr, size_t size, size_t pad, size_t alignment,
bool slab, szind_t szind, bool *zero, bool *commit);

2
include/jemalloc/internal/witness.h
View File

@ -43,7 +43,7 @@
#define WITNESS_RANK_TCACHE_QL 13U
#define WITNESS_RANK_EXTENT_GROW 14U
#define WITNESS_RANK_EXTENTS 15U
#define WITNESS_RANK_EDATA_AVAIL 16U
#define WITNESS_RANK_EDATA_CACHE 16U
#define WITNESS_RANK_EXTENT_POOL 17U
#define WITNESS_RANK_RTREE 18U

1
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
View File

@ -45,6 +45,7 @@
<ClCompile Include="..\..\..\..\src\ctl.c" />
<ClCompile Include="..\..\..\..\src\div.c" />
<ClCompile Include="..\..\..\..\src\edata.c" />
<ClCompile Include="..\..\..\..\src\edata_cache.c" />
<ClCompile Include="..\..\..\..\src\ehooks.c" />
<ClCompile Include="..\..\..\..\src\eset.c" />
<ClCompile Include="..\..\..\..\src\extent2.c" />

1
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
View File

@ -45,6 +45,7 @@
<ClCompile Include="..\..\..\..\src\ctl.c" />
<ClCompile Include="..\..\..\..\src\div.c" />
<ClCompile Include="..\..\..\..\src\edata.c" />
<ClCompile Include="..\..\..\..\src\edata_cache.c" />
<ClCompile Include="..\..\..\..\src\ehooks.c" />
<ClCompile Include="..\..\..\..\src\eset.c" />
<ClCompile Include="..\..\..\..\src\extent2.c" />

14
src/arena.c
View File

@ -103,7 +103,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
eset_npages_get(&arena->eset_retained) << LG_PAGE);
atomic_store_zu(&astats->edata_avail,
atomic_load_zu(&arena->edata_avail_cnt, ATOMIC_RELAXED),
atomic_load_zu(&arena->edata_cache.count, ATOMIC_RELAXED),
ATOMIC_RELAXED);
arena_stats_accum_u64(&astats->decay_dirty.npurge,
@ -224,7 +224,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
/* Gather per arena mutex profiling data. */
READ_ARENA_MUTEX_PROF_DATA(large_mtx, arena_prof_mutex_large);
READ_ARENA_MUTEX_PROF_DATA(edata_avail_mtx,
READ_ARENA_MUTEX_PROF_DATA(edata_cache.mtx,
arena_prof_mutex_extent_avail)
READ_ARENA_MUTEX_PROF_DATA(eset_dirty.mtx,
arena_prof_mutex_extents_dirty)
@ -2053,9 +2053,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
goto label_error;
}
edata_avail_new(&arena->edata_avail);
if (malloc_mutex_init(&arena->edata_avail_mtx, "edata_avail",
WITNESS_RANK_EDATA_AVAIL, malloc_mutex_rank_exclusive)) {
if (edata_cache_init(&arena->edata_cache)) {
goto label_error;
}
@ -2201,7 +2199,7 @@ arena_prefork3(tsdn_t *tsdn, arena_t *arena) {
void
arena_prefork4(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_prefork(tsdn, &arena->edata_avail_mtx);
edata_cache_prefork(tsdn, &arena->edata_cache);
}
void
@ -2235,7 +2233,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
}
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
base_postfork_parent(tsdn, arena->base);
malloc_mutex_postfork_parent(tsdn, &arena->edata_avail_mtx);
edata_cache_postfork_parent(tsdn, &arena->edata_cache);
eset_postfork_parent(tsdn, &arena->eset_dirty);
eset_postfork_parent(tsdn, &arena->eset_muzzy);
eset_postfork_parent(tsdn, &arena->eset_retained);
@ -2281,7 +2279,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
}
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
base_postfork_child(tsdn, arena->base);
malloc_mutex_postfork_child(tsdn, &arena->edata_avail_mtx);
edata_cache_postfork_child(tsdn, &arena->edata_cache);
eset_postfork_child(tsdn, &arena->eset_dirty);
eset_postfork_child(tsdn, &arena->eset_muzzy);
eset_postfork_child(tsdn, &arena->eset_retained);

2
src/ctl.c
View File

@ -3010,7 +3010,7 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib,
continue;
}
MUTEX_PROF_RESET(arena->large_mtx);
MUTEX_PROF_RESET(arena->edata_avail_mtx);
MUTEX_PROF_RESET(arena->edata_cache.mtx);
MUTEX_PROF_RESET(arena->eset_dirty.mtx);
MUTEX_PROF_RESET(arena->eset_muzzy.mtx);
MUTEX_PROF_RESET(arena->eset_retained.mtx);

47
src/edata_cache.c Normal file
View File

@ -0,0 +1,47 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
bool
edata_cache_init(edata_cache_t *edata_cache) {
if (malloc_mutex_init(&edata_cache->mtx, "edata_cache",
WITNESS_RANK_EDATA_CACHE, malloc_mutex_rank_exclusive)) {
return true;
}
edata_avail_new(&edata_cache->avail);
return false;
}
edata_t *
edata_cache_get(tsdn_t *tsdn, edata_cache_t *edata_cache, base_t *base) {
malloc_mutex_lock(tsdn, &edata_cache->mtx);
edata_t *edata = edata_avail_first(&edata_cache->avail);
if (edata == NULL) {
malloc_mutex_unlock(tsdn, &edata_cache->mtx);
return base_alloc_edata(tsdn, base);
}
edata_avail_remove(&edata_cache->avail, edata);
atomic_fetch_sub_zu(&edata_cache->count, 1, ATOMIC_RELAXED);
malloc_mutex_unlock(tsdn, &edata_cache->mtx);
return edata;
}
void
edata_cache_put(tsdn_t *tsdn, edata_cache_t *edata_cache, edata_t *edata) {
malloc_mutex_lock(tsdn, &edata_cache->mtx);
edata_avail_insert(&edata_cache->avail, edata);
atomic_fetch_add_zu(&edata_cache->count, 1, ATOMIC_RELAXED);
malloc_mutex_unlock(tsdn, &edata_cache->mtx);
}
void edata_cache_prefork(tsdn_t *tsdn, edata_cache_t *edata_cache) {
malloc_mutex_prefork(tsdn, &edata_cache->mtx);
}
void edata_cache_postfork_parent(tsdn_t *tsdn, edata_cache_t *edata_cache) {
malloc_mutex_postfork_parent(tsdn, &edata_cache->mtx);
}
void edata_cache_postfork_child(tsdn_t *tsdn, edata_cache_t *edata_cache) {
malloc_mutex_postfork_child(tsdn, &edata_cache->mtx);
}

51
src/extent2.c
View File

@ -163,28 +163,6 @@ extent_addr_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
}
}
edata_t *
extent_alloc(tsdn_t *tsdn, arena_t *arena) {
malloc_mutex_lock(tsdn, &arena->edata_avail_mtx);
edata_t *edata = edata_avail_first(&arena->edata_avail);
if (edata == NULL) {
malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
return base_alloc_edata(tsdn, arena->base);
}
edata_avail_remove(&arena->edata_avail, edata);
atomic_fetch_sub_zu(&arena->edata_avail_cnt, 1, ATOMIC_RELAXED);
malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
return edata;
}
void
extent_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
malloc_mutex_lock(tsdn, &arena->edata_avail_mtx);
edata_avail_insert(&arena->edata_avail, edata);
atomic_fetch_add_zu(&arena->edata_avail_cnt, 1, ATOMIC_RELAXED);
malloc_mutex_unlock(tsdn, &arena->edata_avail_mtx);
}
static bool
extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
rtree_ctx_t *rtree_ctx, eset_t *eset, edata_t *edata) {
@ -317,7 +295,7 @@ extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, eset_t *eset,
edata_size_get(edata), growing_retained);
}
}
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
}
static void
@ -858,7 +836,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip);
}
edata_t *edata = extent_alloc(tsdn, arena);
edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
arena->base);
if (edata == NULL) {
goto label_err;
}
@ -872,12 +851,12 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed,
committed, true, EXTENT_IS_HEAD);
if (ptr == NULL) {
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
goto label_err;
}
if (extent_register_no_gdump_add(tsdn, edata)) {
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
goto label_err;
}
@ -1021,7 +1000,8 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *new_addr, size_t size, size_t pad, size_t alignment, bool slab,
szind_t szind, bool *zero, bool *commit) {
size_t esize = size + pad;
edata_t *edata = extent_alloc(tsdn, arena);
edata_t *edata = edata_cache_get(tsdn, &arena->edata_cache,
arena->base);
if (edata == NULL) {
return NULL;
}
@ -1029,7 +1009,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
void *addr = ehooks_alloc(tsdn, ehooks, new_addr, esize, palignment,
zero, commit, arena_ind_get(arena));
if (addr == NULL) {
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
return NULL;
}
edata_init(edata, arena_ind_get(arena), addr, esize, slab, szind,
@ -1039,7 +1019,7 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
extent_addr_randomize(tsdn, arena, edata, alignment);
}
if (extent_register(tsdn, edata)) {
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
return NULL;
}
@ -1257,7 +1237,7 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
WITNESS_RANK_CORE, 0);
if (extent_register(tsdn, edata)) {
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
return;
}
extent_dalloc_wrapper(tsdn, arena, ehooks, edata);
@ -1287,7 +1267,7 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
arena_ind_get(arena));
if (!err) {
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
}
return err;
@ -1359,7 +1339,7 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
edata_size_get(edata), edata_committed_get(edata),
arena_ind_get(arena));
extent_dalloc(tsdn, arena, edata);
edata_cache_put(tsdn, &arena->edata_cache, edata);
}
static bool
@ -1445,7 +1425,8 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
return NULL;
}
edata_t *trail = extent_alloc(tsdn, arena);
edata_t *trail = edata_cache_get(tsdn, &arena->edata_cache,
arena->base);
if (trail == NULL) {
goto label_error_a;
}
@ -1505,7 +1486,7 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks,
label_error_c:
extent_unlock_edata2(tsdn, edata, trail);
label_error_b:
extent_dalloc(tsdn, arena, trail);
edata_cache_put(tsdn, &arena->edata_cache, trail);
label_error_a:
return NULL;
}
@ -1611,7 +1592,7 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, edata_t *a,
* arena (i.e. this one).
*/
assert(edata_arena_ind_get(b) == arena_ind_get(arena));
extent_dalloc(tsdn, arena, b);
edata_cache_put(tsdn, &arena->edata_cache, b);
return false;
}

7
src/extent_dss.c
View File

@ -123,7 +123,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
return NULL;
}
gap = extent_alloc(tsdn, arena);
gap = edata_cache_get(tsdn, &arena->edata_cache, arena->base);
if (gap == NULL) {
return NULL;
}
@ -188,7 +188,8 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
if (gap_size_page != 0) {
extent_dalloc_gap(tsdn, arena, gap);
} else {
extent_dalloc(tsdn, arena, gap);
edata_cache_put(tsdn,
&arena->edata_cache, gap);
}
if (!*commit) {
*commit = pages_decommit(ret, size);
@ -224,7 +225,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
}
label_oom:
extent_dss_extending_finish();
extent_dalloc(tsdn, arena, gap);
edata_cache_put(tsdn, &arena->edata_cache, gap);
return NULL;
}