Use the flat bitmap for eset and psset bitmaps.
This is simpler (note that the eset field comment was actually incorrect!), and slightly faster.
This commit is contained in:
parent
271a676dcd
commit
154aa5fcc1
@ -2,7 +2,7 @@
|
|||||||
#define JEMALLOC_INTERNAL_ESET_H
|
#define JEMALLOC_INTERNAL_ESET_H
|
||||||
|
|
||||||
#include "jemalloc/internal/atomic.h"
|
#include "jemalloc/internal/atomic.h"
|
||||||
#include "jemalloc/internal/bitmap.h"
|
#include "jemalloc/internal/flat_bitmap.h"
|
||||||
#include "jemalloc/internal/edata.h"
|
#include "jemalloc/internal/edata.h"
|
||||||
#include "jemalloc/internal/mutex.h"
|
#include "jemalloc/internal/mutex.h"
|
||||||
|
|
||||||
@ -22,7 +22,7 @@ struct eset_s {
|
|||||||
atomic_zu_t nbytes[SC_NPSIZES + 1];
|
atomic_zu_t nbytes[SC_NPSIZES + 1];
|
||||||
|
|
||||||
/* Bitmap for which set bits correspond to non-empty heaps. */
|
/* Bitmap for which set bits correspond to non-empty heaps. */
|
||||||
bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
|
fb_group_t bitmap[FB_NGROUPS(SC_NPSIZES + 1)];
|
||||||
|
|
||||||
/* LRU of all extents in heaps. */
|
/* LRU of all extents in heaps. */
|
||||||
edata_list_inactive_t lru;
|
edata_list_inactive_t lru;
|
||||||
|
@ -56,7 +56,8 @@ struct psset_s {
|
|||||||
* free run of pages in a pageslab.
|
* free run of pages in a pageslab.
|
||||||
*/
|
*/
|
||||||
hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
|
hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
|
||||||
bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
|
/* Bitmap for which set bits correspond to non-empty heaps. */
|
||||||
|
fb_group_t bitmap[FB_NGROUPS(PSSET_NPSIZES)];
|
||||||
/*
|
/*
|
||||||
* The sum of all bin stats in stats. This lets us quickly answer
|
* The sum of all bin stats in stats. This lets us quickly answer
|
||||||
* queries for the number of dirty, active, and retained pages in the
|
* queries for the number of dirty, active, and retained pages in the
|
||||||
|
30
src/eset.c
30
src/eset.c
@ -3,15 +3,14 @@
|
|||||||
|
|
||||||
#include "jemalloc/internal/eset.h"
|
#include "jemalloc/internal/eset.h"
|
||||||
|
|
||||||
const bitmap_info_t eset_bitmap_info =
|
#define ESET_NPSIZES (SC_NPSIZES + 1)
|
||||||
BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);
|
|
||||||
|
|
||||||
void
|
void
|
||||||
eset_init(eset_t *eset, extent_state_t state) {
|
eset_init(eset_t *eset, extent_state_t state) {
|
||||||
for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
|
for (unsigned i = 0; i < ESET_NPSIZES; i++) {
|
||||||
edata_heap_new(&eset->heaps[i]);
|
edata_heap_new(&eset->heaps[i]);
|
||||||
}
|
}
|
||||||
bitmap_init(eset->bitmap, &eset_bitmap_info, true);
|
fb_init(eset->bitmap, ESET_NPSIZES);
|
||||||
edata_list_inactive_init(&eset->lru);
|
edata_list_inactive_init(&eset->lru);
|
||||||
atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
|
atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
|
||||||
eset->state = state;
|
eset->state = state;
|
||||||
@ -56,8 +55,7 @@ eset_insert(eset_t *eset, edata_t *edata) {
|
|||||||
size_t psz = sz_psz_quantize_floor(size);
|
size_t psz = sz_psz_quantize_floor(size);
|
||||||
pszind_t pind = sz_psz2ind(psz);
|
pszind_t pind = sz_psz2ind(psz);
|
||||||
if (edata_heap_empty(&eset->heaps[pind])) {
|
if (edata_heap_empty(&eset->heaps[pind])) {
|
||||||
bitmap_unset(eset->bitmap, &eset_bitmap_info,
|
fb_set(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||||
(size_t)pind);
|
|
||||||
}
|
}
|
||||||
edata_heap_insert(&eset->heaps[pind], edata);
|
edata_heap_insert(&eset->heaps[pind], edata);
|
||||||
|
|
||||||
@ -92,8 +90,7 @@ eset_remove(eset_t *eset, edata_t *edata) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (edata_heap_empty(&eset->heaps[pind])) {
|
if (edata_heap_empty(&eset->heaps[pind])) {
|
||||||
bitmap_set(eset->bitmap, &eset_bitmap_info,
|
fb_unset(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||||
(size_t)pind);
|
|
||||||
}
|
}
|
||||||
edata_list_inactive_remove(&eset->lru, edata);
|
edata_list_inactive_remove(&eset->lru, edata);
|
||||||
size_t npages = size >> LG_PAGE;
|
size_t npages = size >> LG_PAGE;
|
||||||
@ -122,10 +119,10 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size,
|
|||||||
pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size));
|
pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size));
|
||||||
pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size));
|
pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size));
|
||||||
|
|
||||||
for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
|
for (pszind_t i =
|
||||||
&eset_bitmap_info, (size_t)pind); i < pind_max; i =
|
(pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||||
(pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
|
i < pind_max;
|
||||||
(size_t)i+1)) {
|
i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
|
||||||
assert(i < SC_NPSIZES);
|
assert(i < SC_NPSIZES);
|
||||||
assert(!edata_heap_empty(&eset->heaps[i]));
|
assert(!edata_heap_empty(&eset->heaps[i]));
|
||||||
edata_t *edata = edata_heap_first(&eset->heaps[i]);
|
edata_t *edata = edata_heap_first(&eset->heaps[i]);
|
||||||
@ -171,11 +168,10 @@ eset_first_fit(eset_t *eset, size_t size, bool exact_only,
|
|||||||
edata_heap_first(&eset->heaps[pind]);
|
edata_heap_first(&eset->heaps[pind]);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
|
for (pszind_t i =
|
||||||
&eset_bitmap_info, (size_t)pind);
|
(pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||||
i < SC_NPSIZES + 1;
|
i < ESET_NPSIZES;
|
||||||
i = (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
|
i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
|
||||||
(size_t)i+1)) {
|
|
||||||
assert(!edata_heap_empty(&eset->heaps[i]));
|
assert(!edata_heap_empty(&eset->heaps[i]));
|
||||||
edata_t *edata = edata_heap_first(&eset->heaps[i]);
|
edata_t *edata = edata_heap_first(&eset->heaps[i]);
|
||||||
assert(edata_size_get(edata) >= size);
|
assert(edata_size_get(edata) >= size);
|
||||||
|
11
src/psset.c
11
src/psset.c
@ -5,15 +5,12 @@
|
|||||||
|
|
||||||
#include "jemalloc/internal/flat_bitmap.h"
|
#include "jemalloc/internal/flat_bitmap.h"
|
||||||
|
|
||||||
static const bitmap_info_t psset_bitmap_info =
|
|
||||||
BITMAP_INFO_INITIALIZER(PSSET_NPSIZES);
|
|
||||||
|
|
||||||
void
|
void
|
||||||
psset_init(psset_t *psset) {
|
psset_init(psset_t *psset) {
|
||||||
for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
|
for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
|
||||||
hpdata_age_heap_new(&psset->pageslabs[i]);
|
hpdata_age_heap_new(&psset->pageslabs[i]);
|
||||||
}
|
}
|
||||||
bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
|
fb_init(psset->bitmap, PSSET_NPSIZES);
|
||||||
memset(&psset->merged_stats, 0, sizeof(psset->merged_stats));
|
memset(&psset->merged_stats, 0, sizeof(psset->merged_stats));
|
||||||
memset(&psset->stats, 0, sizeof(psset->stats));
|
memset(&psset->stats, 0, sizeof(psset->stats));
|
||||||
hpdata_empty_list_init(&psset->empty);
|
hpdata_empty_list_init(&psset->empty);
|
||||||
@ -101,14 +98,14 @@ static void
|
|||||||
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
||||||
hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
|
hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
|
||||||
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
|
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
|
||||||
bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
|
fb_unset(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
||||||
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
|
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
|
||||||
bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
|
fb_set(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
|
||||||
}
|
}
|
||||||
hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
|
hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
|
||||||
}
|
}
|
||||||
@ -266,7 +263,7 @@ psset_pick_alloc(psset_t *psset, size_t size) {
|
|||||||
assert(size <= HUGEPAGE);
|
assert(size <= HUGEPAGE);
|
||||||
|
|
||||||
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
|
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
|
||||||
pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
|
pszind_t pind = (pszind_t)fb_ffs(psset->bitmap, PSSET_NPSIZES,
|
||||||
(size_t)min_pind);
|
(size_t)min_pind);
|
||||||
if (pind == PSSET_NPSIZES) {
|
if (pind == PSSET_NPSIZES) {
|
||||||
return hpdata_empty_list_first(&psset->empty);
|
return hpdata_empty_list_first(&psset->empty);
|
||||||
|
Loading…
Reference in New Issue
Block a user