project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Edata: Move sn into its own field.

Browse Source 
This lets the bins use a fragmentation avoidance policy that matches the HPA's
(without affecting the PAC).
This commit is contained in:
David Goldblatt 2021-02-06 09:29:01 -08:00 committed by David Goldblatt
parent fb327368db
commit d21d5b46b6
4 changed files with 19 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
include/jemalloc/internal/edata.h
View File

@ -87,9 +87,8 @@ struct edata_s {
* i: szind * i: szind
* f: nfree * f: nfree
* s: bin_shard * s: bin_shard
* n: sn
* *
* nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zpcbaaaa aaaaaaaa * 00000000 ... 000000ss ssssffff ffffffii iiiiiitt zpcbaaaa aaaaaaaa
* *
* arena_ind: Arena from which this extent came, or all 1 bits if * arena_ind: Arena from which this extent came, or all 1 bits if
* unassociated. * unassociated.
@ -120,16 +119,6 @@ struct edata_s {
* nfree: Number of free regions in slab. * nfree: Number of free regions in slab.
* *
* bin_shard: the shard of the bin from which this extent came. * bin_shard: the shard of the bin from which this extent came.
*
* sn: Serial number (potentially non-unique).
*
* Serial numbers may wrap around if !opt_retain, but as long as
* comparison functions fall back on address comparison for equal
* serial numbers, stable (if imperfect) ordering is maintained.
*
* Serial numbers may not be unique even in the absence of
* wrap-around, e.g. when splitting an extent and assigning the same
* serial number to both resulting adjacent extents.
*/ */
uint64_t e_bits; uint64_t e_bits;
#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) << (CURRENT_FIELD_SHIFT)) #define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) << (CURRENT_FIELD_SHIFT))
@ -174,9 +163,6 @@ struct edata_s {
#define EDATA_BITS_IS_HEAD_SHIFT (EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT) #define EDATA_BITS_IS_HEAD_SHIFT (EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT)
#define EDATA_BITS_IS_HEAD_MASK MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT) #define EDATA_BITS_IS_HEAD_MASK MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
#define EDATA_BITS_SN_SHIFT (EDATA_BITS_IS_HEAD_WIDTH + EDATA_BITS_IS_HEAD_SHIFT)
#define EDATA_BITS_SN_MASK (UINT64_MAX << EDATA_BITS_SN_SHIFT)
/* Pointer to the extent that this structure is responsible for. */ /* Pointer to the extent that this structure is responsible for. */
void *e_addr; void *e_addr;
@ -201,8 +187,11 @@ struct edata_s {
* into pageslabs). This tracks it. * into pageslabs). This tracks it.
*/ */
hpdata_t *e_ps; hpdata_t *e_ps;
/* Extra field reserved for HPA. */ /*
void *e_reserved; * Serial number. These are not necessarily unique; splitting an extent
* results in two extents with the same serial number.
*/
uint64_t e_sn;
union { union {
/* /*
@ -274,10 +263,9 @@ edata_binshard_get(const edata_t *edata) {
return binshard; return binshard;
} }
static inline size_t static inline uint64_t
edata_sn_get(const edata_t *edata) { edata_sn_get(const edata_t *edata) {
return (size_t)((edata->e_bits & EDATA_BITS_SN_MASK) >> return edata->e_sn;
EDATA_BITS_SN_SHIFT);
} }
static inline extent_state_t static inline extent_state_t
@ -488,9 +476,8 @@ edata_nfree_sub(edata_t *edata, uint64_t n) {
} }
static inline void static inline void
edata_sn_set(edata_t *edata, size_t sn) { edata_sn_set(edata_t *edata, uint64_t sn) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_SN_MASK) | edata->e_sn = sn;
((uint64_t)sn << EDATA_BITS_SN_SHIFT);
} }
static inline void static inline void
@ -566,7 +553,7 @@ edata_is_head_set(edata_t *edata, bool is_head) {
*/ */
static inline void static inline void
edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size, edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, bool slab, szind_t szind, uint64_t sn, extent_state_t state, bool zeroed,
bool committed, extent_pai_t pai, extent_head_state_t is_head) { bool committed, extent_pai_t pai, extent_head_state_t is_head) {
assert(addr == PAGE_ADDR2BASE(addr) || !slab); assert(addr == PAGE_ADDR2BASE(addr) || !slab);
@ -587,7 +574,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
} }
static inline void static inline void
edata_binit(edata_t *edata, void *addr, size_t bsize, size_t sn) { edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn) {
edata_arena_ind_set(edata, (1U << MALLOCX_ARENA_BITS) - 1); edata_arena_ind_set(edata, (1U << MALLOCX_ARENA_BITS) - 1);
edata_addr_set(edata, addr); edata_addr_set(edata, addr);
edata_bsize_set(edata, bsize); edata_bsize_set(edata, bsize);
@ -607,8 +594,8 @@ edata_binit(edata_t *edata, void *addr, size_t bsize, size_t sn) {
static inline int static inline int
edata_sn_comp(const edata_t *a, const edata_t *b) { edata_sn_comp(const edata_t *a, const edata_t *b) {
size_t a_sn = edata_sn_get(a); uint64_t a_sn = edata_sn_get(a);
size_t b_sn = edata_sn_get(b); uint64_t b_sn = edata_sn_get(b);
return (a_sn > b_sn) - (a_sn < b_sn); return (a_sn > b_sn) - (a_sn < b_sn);
} }

2
src/base.c
View File

@ -448,7 +448,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment,
ret = base_extent_bump_alloc(base, edata, usize, alignment); ret = base_extent_bump_alloc(base, edata, usize, alignment);
if (esn != NULL) { if (esn != NULL) {
*esn = edata_sn_get(edata); *esn = (size_t)edata_sn_get(edata);
} }
label_return: label_return:
malloc_mutex_unlock(tsdn, &base->mtx); malloc_mutex_unlock(tsdn, &base->mtx);

5
src/hpa.c
View File

@ -458,8 +458,9 @@ hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
void *addr = hpdata_reserve_alloc(ps, size); void *addr = hpdata_reserve_alloc(ps, size);
edata_init(edata, shard->ind, addr, size, /* slab */ false, edata_init(edata, shard->ind, addr, size, /* slab */ false,
SC_NSIZES, /* sn */ 0, extent_state_active, /* zeroed */ false, SC_NSIZES, /* sn */ hpdata_age_get(ps), extent_state_active,
/* committed */ true, EXTENT_PAI_HPA, EXTENT_NOT_HEAD); /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
EXTENT_NOT_HEAD);
edata_ps_set(edata, ps); edata_ps_set(edata, ps);
/* /*

2
test/unit/psset.c
View File

@ -91,7 +91,7 @@ edata_expect(edata_t *edata, size_t page_offset, size_t page_cnt) {
expect_false(edata_slab_get(edata), ""); expect_false(edata_slab_get(edata), "");
expect_u_eq(SC_NSIZES, edata_szind_get_maybe_invalid(edata), expect_u_eq(SC_NSIZES, edata_szind_get_maybe_invalid(edata),
""); "");
expect_zu_eq(0, edata_sn_get(edata), ""); expect_u64_eq(0, edata_sn_get(edata), "");
expect_d_eq(edata_state_get(edata), extent_state_active, ""); expect_d_eq(edata_state_get(edata), extent_state_active, "");
expect_false(edata_zeroed_get(edata), ""); expect_false(edata_zeroed_get(edata), "");
expect_true(edata_committed_get(edata), ""); expect_true(edata_committed_get(edata), "");