Move arena_slab_data_t's nfree into extent_t's e_bits.
Compact extent_t to 128 bytes on 64-bit systems by moving arena_slab_data_t's nfree into extent_t's e_bits. Cacheline-align extent_t structures so that they always cross the minimum number of cacheline boundaries. Re-order extent_t fields such that all fields except the slab bitmap (and overlaid heap profiling context pointer) are in the first cacheline. This resolves #461.
This commit is contained in:
parent
af3d737a9a
commit
07f4f93434
@ -2,9 +2,6 @@
|
|||||||
#define JEMALLOC_INTERNAL_ARENA_STRUCTS_A_H
|
#define JEMALLOC_INTERNAL_ARENA_STRUCTS_A_H
|
||||||
|
|
||||||
struct arena_slab_data_s {
|
struct arena_slab_data_s {
|
||||||
/* Number of free regions in slab. */
|
|
||||||
unsigned nfree;
|
|
||||||
|
|
||||||
/* Per region allocated/deallocated bitmap. */
|
/* Per region allocated/deallocated bitmap. */
|
||||||
bitmap_t bitmap[BITMAP_GROUPS_MAX];
|
bitmap_t bitmap[BITMAP_GROUPS_MAX];
|
||||||
};
|
};
|
||||||
|
@ -11,6 +11,7 @@ extent_state_t extent_state_get(const extent_t *extent);
|
|||||||
bool extent_zeroed_get(const extent_t *extent);
|
bool extent_zeroed_get(const extent_t *extent);
|
||||||
bool extent_committed_get(const extent_t *extent);
|
bool extent_committed_get(const extent_t *extent);
|
||||||
bool extent_slab_get(const extent_t *extent);
|
bool extent_slab_get(const extent_t *extent);
|
||||||
|
unsigned extent_nfree_get(const extent_t *extent);
|
||||||
void *extent_base_get(const extent_t *extent);
|
void *extent_base_get(const extent_t *extent);
|
||||||
void *extent_addr_get(const extent_t *extent);
|
void *extent_addr_get(const extent_t *extent);
|
||||||
size_t extent_size_get(const extent_t *extent);
|
size_t extent_size_get(const extent_t *extent);
|
||||||
@ -25,6 +26,9 @@ void extent_addr_set(extent_t *extent, void *addr);
|
|||||||
void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
|
void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
|
||||||
void extent_size_set(extent_t *extent, size_t size);
|
void extent_size_set(extent_t *extent, size_t size);
|
||||||
void extent_szind_set(extent_t *extent, szind_t szind);
|
void extent_szind_set(extent_t *extent, szind_t szind);
|
||||||
|
void extent_nfree_set(extent_t *extent, unsigned nfree);
|
||||||
|
void extent_nfree_inc(extent_t *extent);
|
||||||
|
void extent_nfree_dec(extent_t *extent);
|
||||||
void extent_sn_set(extent_t *extent, size_t sn);
|
void extent_sn_set(extent_t *extent, size_t sn);
|
||||||
void extent_state_set(extent_t *extent, extent_state_t state);
|
void extent_state_set(extent_t *extent, extent_state_t state);
|
||||||
void extent_zeroed_set(extent_t *extent, bool zeroed);
|
void extent_zeroed_set(extent_t *extent, bool zeroed);
|
||||||
@ -112,6 +116,13 @@ extent_slab_get(const extent_t *extent) {
|
|||||||
EXTENT_BITS_SLAB_SHIFT);
|
EXTENT_BITS_SLAB_SHIFT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
JEMALLOC_INLINE unsigned
|
||||||
|
extent_nfree_get(const extent_t *extent) {
|
||||||
|
assert(extent_slab_get(extent));
|
||||||
|
return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >>
|
||||||
|
EXTENT_BITS_NFREE_SHIFT);
|
||||||
|
}
|
||||||
|
|
||||||
JEMALLOC_INLINE void *
|
JEMALLOC_INLINE void *
|
||||||
extent_base_get(const extent_t *extent) {
|
extent_base_get(const extent_t *extent) {
|
||||||
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
|
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
|
||||||
@ -210,6 +221,25 @@ extent_szind_set(extent_t *extent, szind_t szind) {
|
|||||||
((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT);
|
((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
JEMALLOC_INLINE void
|
||||||
|
extent_nfree_set(extent_t *extent, unsigned nfree) {
|
||||||
|
assert(extent_slab_get(extent));
|
||||||
|
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) |
|
||||||
|
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
|
||||||
|
}
|
||||||
|
|
||||||
|
JEMALLOC_INLINE void
|
||||||
|
extent_nfree_inc(extent_t *extent) {
|
||||||
|
assert(extent_slab_get(extent));
|
||||||
|
extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
|
||||||
|
}
|
||||||
|
|
||||||
|
JEMALLOC_INLINE void
|
||||||
|
extent_nfree_dec(extent_t *extent) {
|
||||||
|
assert(extent_slab_get(extent));
|
||||||
|
extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
|
||||||
|
}
|
||||||
|
|
||||||
JEMALLOC_INLINE void
|
JEMALLOC_INLINE void
|
||||||
extent_sn_set(extent_t *extent, size_t sn) {
|
extent_sn_set(extent_t *extent, size_t sn) {
|
||||||
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
|
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
|
||||||
@ -260,10 +290,10 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
|
|||||||
extent_state_set(extent, state);
|
extent_state_set(extent, state);
|
||||||
extent_zeroed_set(extent, zeroed);
|
extent_zeroed_set(extent, zeroed);
|
||||||
extent_committed_set(extent, committed);
|
extent_committed_set(extent, committed);
|
||||||
|
ql_elm_new(extent, ql_link);
|
||||||
if (config_prof) {
|
if (config_prof) {
|
||||||
extent_prof_tctx_set(extent, NULL);
|
extent_prof_tctx_set(extent, NULL);
|
||||||
}
|
}
|
||||||
ql_elm_new(extent, ql_link);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_INLINE void
|
JEMALLOC_INLINE void
|
||||||
|
@ -20,8 +20,9 @@ struct extent_s {
|
|||||||
* t: state
|
* t: state
|
||||||
* i: szind
|
* i: szind
|
||||||
* n: sn
|
* n: sn
|
||||||
|
* f: nfree
|
||||||
*
|
*
|
||||||
* nnnnnnnn ... nnnnnnni iiiiiiit tzcbaaaa aaaaaaaa
|
* nnnnnnnn ... nnnnnfff fffffffi iiiiiiit tzcbaaaa aaaaaaaa
|
||||||
*
|
*
|
||||||
* arena_ind: Arena from which this extent came, or all 1 bits if
|
* arena_ind: Arena from which this extent came, or all 1 bits if
|
||||||
* unassociated.
|
* unassociated.
|
||||||
@ -47,6 +48,8 @@ struct extent_s {
|
|||||||
* even for non-slabs, either due to large_pad or promotion of
|
* even for non-slabs, either due to large_pad or promotion of
|
||||||
* sampled small regions.
|
* sampled small regions.
|
||||||
*
|
*
|
||||||
|
* nfree: Number of free regions in slab.
|
||||||
|
*
|
||||||
* sn: Serial number (potentially non-unique).
|
* sn: Serial number (potentially non-unique).
|
||||||
*
|
*
|
||||||
* Serial numbers may wrap around if JEMALLOC_MUNMAP is defined, but
|
* Serial numbers may wrap around if JEMALLOC_MUNMAP is defined, but
|
||||||
@ -61,26 +64,35 @@ struct extent_s {
|
|||||||
uint64_t e_bits;
|
uint64_t e_bits;
|
||||||
#define EXTENT_BITS_ARENA_SHIFT 0
|
#define EXTENT_BITS_ARENA_SHIFT 0
|
||||||
#define EXTENT_BITS_ARENA_MASK \
|
#define EXTENT_BITS_ARENA_MASK \
|
||||||
(((1U << MALLOCX_ARENA_BITS) - 1) << EXTENT_BITS_ARENA_SHIFT)
|
(((uint64_t)(1U << MALLOCX_ARENA_BITS) - 1) << EXTENT_BITS_ARENA_SHIFT)
|
||||||
|
|
||||||
#define EXTENT_BITS_SLAB_SHIFT MALLOCX_ARENA_BITS
|
#define EXTENT_BITS_SLAB_SHIFT MALLOCX_ARENA_BITS
|
||||||
#define EXTENT_BITS_SLAB_MASK (0x1U << EXTENT_BITS_SLAB_SHIFT)
|
#define EXTENT_BITS_SLAB_MASK \
|
||||||
|
((uint64_t)0x1U << EXTENT_BITS_SLAB_SHIFT)
|
||||||
|
|
||||||
#define EXTENT_BITS_COMMITTED_SHIFT (MALLOCX_ARENA_BITS + 1)
|
#define EXTENT_BITS_COMMITTED_SHIFT (MALLOCX_ARENA_BITS + 1)
|
||||||
#define EXTENT_BITS_COMMITTED_MASK (0x1U << EXTENT_BITS_COMMITTED_SHIFT)
|
#define EXTENT_BITS_COMMITTED_MASK \
|
||||||
|
((uint64_t)0x1U << EXTENT_BITS_COMMITTED_SHIFT)
|
||||||
|
|
||||||
#define EXTENT_BITS_ZEROED_SHIFT (MALLOCX_ARENA_BITS + 2)
|
#define EXTENT_BITS_ZEROED_SHIFT (MALLOCX_ARENA_BITS + 2)
|
||||||
#define EXTENT_BITS_ZEROED_MASK (0x1U << EXTENT_BITS_ZEROED_SHIFT)
|
#define EXTENT_BITS_ZEROED_MASK \
|
||||||
|
((uint64_t)0x1U << EXTENT_BITS_ZEROED_SHIFT)
|
||||||
|
|
||||||
#define EXTENT_BITS_STATE_SHIFT (MALLOCX_ARENA_BITS + 3)
|
#define EXTENT_BITS_STATE_SHIFT (MALLOCX_ARENA_BITS + 3)
|
||||||
#define EXTENT_BITS_STATE_MASK (0x3U << EXTENT_BITS_STATE_SHIFT)
|
#define EXTENT_BITS_STATE_MASK \
|
||||||
|
((uint64_t)0x3U << EXTENT_BITS_STATE_SHIFT)
|
||||||
|
|
||||||
#define EXTENT_BITS_SZIND_SHIFT (MALLOCX_ARENA_BITS + 5)
|
#define EXTENT_BITS_SZIND_SHIFT (MALLOCX_ARENA_BITS + 5)
|
||||||
#define EXTENT_BITS_SZIND_MASK \
|
#define EXTENT_BITS_SZIND_MASK \
|
||||||
(((1U << LG_CEIL_NSIZES) - 1) << EXTENT_BITS_SZIND_SHIFT)
|
(((uint64_t)(1U << LG_CEIL_NSIZES) - 1) << EXTENT_BITS_SZIND_SHIFT)
|
||||||
|
|
||||||
|
#define EXTENT_BITS_NFREE_SHIFT \
|
||||||
|
(MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES)
|
||||||
|
#define EXTENT_BITS_NFREE_MASK \
|
||||||
|
((uint64_t)((1U << (LG_SLAB_MAXREGS + 1)) - 1) << EXTENT_BITS_NFREE_SHIFT)
|
||||||
|
|
||||||
#define EXTENT_BITS_SN_SHIFT \
|
#define EXTENT_BITS_SN_SHIFT \
|
||||||
(MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES)
|
(MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES + (LG_SLAB_MAXREGS + 1))
|
||||||
#define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT)
|
#define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT)
|
||||||
|
|
||||||
/* Pointer to the extent that this structure is responsible for. */
|
/* Pointer to the extent that this structure is responsible for. */
|
||||||
@ -89,17 +101,6 @@ struct extent_s {
|
|||||||
/* Extent size. */
|
/* Extent size. */
|
||||||
size_t e_size;
|
size_t e_size;
|
||||||
|
|
||||||
union {
|
|
||||||
/* Small region slab metadata. */
|
|
||||||
arena_slab_data_t e_slab_data;
|
|
||||||
|
|
||||||
/* Profile counters, used for large objects. */
|
|
||||||
union {
|
|
||||||
void *e_prof_tctx_pun;
|
|
||||||
prof_tctx_t *e_prof_tctx;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* List linkage, used by a variety of lists:
|
* List linkage, used by a variety of lists:
|
||||||
* - arena_bin_t's slabs_full
|
* - arena_bin_t's slabs_full
|
||||||
@ -112,6 +113,17 @@ struct extent_s {
|
|||||||
|
|
||||||
/* Linkage for per size class sn/address-ordered heaps. */
|
/* Linkage for per size class sn/address-ordered heaps. */
|
||||||
phn(extent_t) ph_link;
|
phn(extent_t) ph_link;
|
||||||
|
|
||||||
|
union {
|
||||||
|
/* Small region slab metadata. */
|
||||||
|
arena_slab_data_t e_slab_data;
|
||||||
|
|
||||||
|
/* Profile counters, used for large objects. */
|
||||||
|
union {
|
||||||
|
void *e_prof_tctx_pun;
|
||||||
|
prof_tctx_t *e_prof_tctx;
|
||||||
|
};
|
||||||
|
};
|
||||||
};
|
};
|
||||||
typedef ql_head(extent_t) extent_list_t;
|
typedef ql_head(extent_t) extent_list_t;
|
||||||
typedef ph(extent_t) extent_heap_t;
|
typedef ph(extent_t) extent_heap_t;
|
||||||
|
@ -175,6 +175,10 @@ extent_list_last
|
|||||||
extent_list_remove
|
extent_list_remove
|
||||||
extent_list_replace
|
extent_list_replace
|
||||||
extent_merge_wrapper
|
extent_merge_wrapper
|
||||||
|
extent_nfree_dec
|
||||||
|
extent_nfree_get
|
||||||
|
extent_nfree_inc
|
||||||
|
extent_nfree_set
|
||||||
extent_past_get
|
extent_past_get
|
||||||
extent_prof_tctx_get
|
extent_prof_tctx_get
|
||||||
extent_prof_tctx_set
|
extent_prof_tctx_set
|
||||||
|
37
src/arena.c
37
src/arena.c
@ -361,13 +361,13 @@ arena_slab_reg_alloc(tsdn_t *tsdn, extent_t *slab,
|
|||||||
arena_slab_data_t *slab_data = extent_slab_data_get(slab);
|
arena_slab_data_t *slab_data = extent_slab_data_get(slab);
|
||||||
size_t regind;
|
size_t regind;
|
||||||
|
|
||||||
assert(slab_data->nfree > 0);
|
assert(extent_nfree_get(slab) > 0);
|
||||||
assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info));
|
assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info));
|
||||||
|
|
||||||
regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info);
|
regind = bitmap_sfu(slab_data->bitmap, &bin_info->bitmap_info);
|
||||||
ret = (void *)((uintptr_t)extent_addr_get(slab) +
|
ret = (void *)((uintptr_t)extent_addr_get(slab) +
|
||||||
(uintptr_t)(bin_info->reg_size * regind));
|
(uintptr_t)(bin_info->reg_size * regind));
|
||||||
slab_data->nfree--;
|
extent_nfree_dec(slab);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -416,12 +416,12 @@ arena_slab_reg_dalloc(tsdn_t *tsdn, extent_t *slab,
|
|||||||
const arena_bin_info_t *bin_info = &arena_bin_info[binind];
|
const arena_bin_info_t *bin_info = &arena_bin_info[binind];
|
||||||
size_t regind = arena_slab_regind(slab, binind, ptr);
|
size_t regind = arena_slab_regind(slab, binind, ptr);
|
||||||
|
|
||||||
assert(slab_data->nfree < bin_info->nregs);
|
assert(extent_nfree_get(slab) < bin_info->nregs);
|
||||||
/* Freeing an unallocated pointer can cause assertion failure. */
|
/* Freeing an unallocated pointer can cause assertion failure. */
|
||||||
assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind));
|
assert(bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, regind));
|
||||||
|
|
||||||
bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind);
|
bitmap_unset(slab_data->bitmap, &bin_info->bitmap_info, regind);
|
||||||
slab_data->nfree++;
|
extent_nfree_inc(slab);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
@ -999,7 +999,7 @@ arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) {
|
|||||||
|
|
||||||
static void
|
static void
|
||||||
arena_bin_slabs_nonfull_insert(arena_bin_t *bin, extent_t *slab) {
|
arena_bin_slabs_nonfull_insert(arena_bin_t *bin, extent_t *slab) {
|
||||||
assert(extent_slab_data_get(slab)->nfree > 0);
|
assert(extent_nfree_get(slab) > 0);
|
||||||
extent_heap_insert(&bin->slabs_nonfull, slab);
|
extent_heap_insert(&bin->slabs_nonfull, slab);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1022,7 +1022,7 @@ arena_bin_slabs_nonfull_tryget(arena_bin_t *bin) {
|
|||||||
|
|
||||||
static void
|
static void
|
||||||
arena_bin_slabs_full_insert(arena_bin_t *bin, extent_t *slab) {
|
arena_bin_slabs_full_insert(arena_bin_t *bin, extent_t *slab) {
|
||||||
assert(extent_slab_data_get(slab)->nfree == 0);
|
assert(extent_nfree_get(slab) == 0);
|
||||||
extent_list_append(&bin->slabs_full, slab);
|
extent_list_append(&bin->slabs_full, slab);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1209,7 +1209,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
|
|||||||
|
|
||||||
/* Initialize slab internals. */
|
/* Initialize slab internals. */
|
||||||
arena_slab_data_t *slab_data = extent_slab_data_get(slab);
|
arena_slab_data_t *slab_data = extent_slab_data_get(slab);
|
||||||
slab_data->nfree = bin_info->nregs;
|
extent_nfree_set(slab, bin_info->nregs);
|
||||||
bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false);
|
bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false);
|
||||||
|
|
||||||
arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE);
|
arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE);
|
||||||
@ -1277,7 +1277,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,
|
|||||||
* Another thread updated slabcur while this one ran without the
|
* Another thread updated slabcur while this one ran without the
|
||||||
* bin lock in arena_bin_nonfull_slab_get().
|
* bin lock in arena_bin_nonfull_slab_get().
|
||||||
*/
|
*/
|
||||||
if (extent_slab_data_get(bin->slabcur)->nfree > 0) {
|
if (extent_nfree_get(bin->slabcur) > 0) {
|
||||||
void *ret = arena_slab_reg_alloc(tsdn, bin->slabcur,
|
void *ret = arena_slab_reg_alloc(tsdn, bin->slabcur,
|
||||||
bin_info);
|
bin_info);
|
||||||
if (slab != NULL) {
|
if (slab != NULL) {
|
||||||
@ -1290,8 +1290,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,
|
|||||||
* arena_bin_lower_slab() must be called, as if
|
* arena_bin_lower_slab() must be called, as if
|
||||||
* a region were just deallocated from the slab.
|
* a region were just deallocated from the slab.
|
||||||
*/
|
*/
|
||||||
if (extent_slab_data_get(slab)->nfree ==
|
if (extent_nfree_get(slab) == bin_info->nregs) {
|
||||||
bin_info->nregs) {
|
|
||||||
arena_dalloc_bin_slab(tsdn, arena, slab,
|
arena_dalloc_bin_slab(tsdn, arena, slab,
|
||||||
bin);
|
bin);
|
||||||
} else {
|
} else {
|
||||||
@ -1311,7 +1310,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,
|
|||||||
}
|
}
|
||||||
bin->slabcur = slab;
|
bin->slabcur = slab;
|
||||||
|
|
||||||
assert(extent_slab_data_get(bin->slabcur)->nfree > 0);
|
assert(extent_nfree_get(bin->slabcur) > 0);
|
||||||
|
|
||||||
return arena_slab_reg_alloc(tsdn, slab, bin_info);
|
return arena_slab_reg_alloc(tsdn, slab, bin_info);
|
||||||
}
|
}
|
||||||
@ -1333,8 +1332,8 @@ arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_bin_t *tbin,
|
|||||||
tbin->lg_fill_div); i < nfill; i++) {
|
tbin->lg_fill_div); i < nfill; i++) {
|
||||||
extent_t *slab;
|
extent_t *slab;
|
||||||
void *ptr;
|
void *ptr;
|
||||||
if ((slab = bin->slabcur) != NULL &&
|
if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) >
|
||||||
extent_slab_data_get(slab)->nfree > 0) {
|
0) {
|
||||||
ptr = arena_slab_reg_alloc(tsdn, slab,
|
ptr = arena_slab_reg_alloc(tsdn, slab,
|
||||||
&arena_bin_info[binind]);
|
&arena_bin_info[binind]);
|
||||||
} else {
|
} else {
|
||||||
@ -1405,8 +1404,7 @@ arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {
|
|||||||
usize = index2size(binind);
|
usize = index2size(binind);
|
||||||
|
|
||||||
malloc_mutex_lock(tsdn, &bin->lock);
|
malloc_mutex_lock(tsdn, &bin->lock);
|
||||||
if ((slab = bin->slabcur) != NULL && extent_slab_data_get(slab)->nfree >
|
if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) {
|
||||||
0) {
|
|
||||||
ret = arena_slab_reg_alloc(tsdn, slab, &arena_bin_info[binind]);
|
ret = arena_slab_reg_alloc(tsdn, slab, &arena_bin_info[binind]);
|
||||||
} else {
|
} else {
|
||||||
ret = arena_bin_malloc_hard(tsdn, arena, bin, binind);
|
ret = arena_bin_malloc_hard(tsdn, arena, bin, binind);
|
||||||
@ -1582,7 +1580,7 @@ arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
|
|||||||
static void
|
static void
|
||||||
arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
|
arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
|
||||||
arena_bin_t *bin) {
|
arena_bin_t *bin) {
|
||||||
assert(extent_slab_data_get(slab)->nfree > 0);
|
assert(extent_nfree_get(slab) > 0);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Make sure that if bin->slabcur is non-NULL, it refers to the
|
* Make sure that if bin->slabcur is non-NULL, it refers to the
|
||||||
@ -1592,7 +1590,7 @@ arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
|
|||||||
*/
|
*/
|
||||||
if (bin->slabcur != NULL && extent_snad_comp(bin->slabcur, slab) > 0) {
|
if (bin->slabcur != NULL && extent_snad_comp(bin->slabcur, slab) > 0) {
|
||||||
/* Switch slabcur. */
|
/* Switch slabcur. */
|
||||||
if (extent_slab_data_get(bin->slabcur)->nfree > 0) {
|
if (extent_nfree_get(bin->slabcur) > 0) {
|
||||||
arena_bin_slabs_nonfull_insert(bin, bin->slabcur);
|
arena_bin_slabs_nonfull_insert(bin, bin->slabcur);
|
||||||
} else {
|
} else {
|
||||||
arena_bin_slabs_full_insert(bin, bin->slabcur);
|
arena_bin_slabs_full_insert(bin, bin->slabcur);
|
||||||
@ -1619,10 +1617,11 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab,
|
|||||||
}
|
}
|
||||||
|
|
||||||
arena_slab_reg_dalloc(tsdn, slab, slab_data, ptr);
|
arena_slab_reg_dalloc(tsdn, slab, slab_data, ptr);
|
||||||
if (slab_data->nfree == bin_info->nregs) {
|
unsigned nfree = extent_nfree_get(slab);
|
||||||
|
if (nfree == bin_info->nregs) {
|
||||||
arena_dissociate_bin_slab(slab, bin);
|
arena_dissociate_bin_slab(slab, bin);
|
||||||
arena_dalloc_bin_slab(tsdn, arena, slab, bin);
|
arena_dalloc_bin_slab(tsdn, arena, slab, bin);
|
||||||
} else if (slab_data->nfree == 1 && slab != bin->slabcur) {
|
} else if (nfree == 1 && slab != bin->slabcur) {
|
||||||
arena_bin_slabs_full_remove(bin, slab);
|
arena_bin_slabs_full_remove(bin, slab);
|
||||||
arena_bin_lower_slab(tsdn, arena, slab, bin);
|
arena_bin_lower_slab(tsdn, arena, slab, bin);
|
||||||
}
|
}
|
||||||
|
@ -94,7 +94,8 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) {
|
|||||||
extent = extent_list_last(&arena->extent_freelist);
|
extent = extent_list_last(&arena->extent_freelist);
|
||||||
if (extent == NULL) {
|
if (extent == NULL) {
|
||||||
malloc_mutex_unlock(tsdn, &arena->extent_freelist_mtx);
|
malloc_mutex_unlock(tsdn, &arena->extent_freelist_mtx);
|
||||||
return base_alloc(tsdn, arena->base, sizeof(extent_t), QUANTUM);
|
return base_alloc(tsdn, arena->base, sizeof(extent_t),
|
||||||
|
CACHELINE);
|
||||||
}
|
}
|
||||||
extent_list_remove(&arena->extent_freelist, extent);
|
extent_list_remove(&arena->extent_freelist, extent);
|
||||||
malloc_mutex_unlock(tsdn, &arena->extent_freelist_mtx);
|
malloc_mutex_unlock(tsdn, &arena->extent_freelist_mtx);
|
||||||
|
Loading…
Reference in New Issue
Block a user