Use the flat bitmap for eset and psset bitmaps.
This is simpler (note that the eset field comment was actually incorrect!), and slightly faster.
This commit is contained in:
parent
271a676dcd
commit
154aa5fcc1
@ -2,7 +2,7 @@
|
||||
#define JEMALLOC_INTERNAL_ESET_H
|
||||
|
||||
#include "jemalloc/internal/atomic.h"
|
||||
#include "jemalloc/internal/bitmap.h"
|
||||
#include "jemalloc/internal/flat_bitmap.h"
|
||||
#include "jemalloc/internal/edata.h"
|
||||
#include "jemalloc/internal/mutex.h"
|
||||
|
||||
@ -22,7 +22,7 @@ struct eset_s {
|
||||
atomic_zu_t nbytes[SC_NPSIZES + 1];
|
||||
|
||||
/* Bitmap for which set bits correspond to non-empty heaps. */
|
||||
bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
|
||||
fb_group_t bitmap[FB_NGROUPS(SC_NPSIZES + 1)];
|
||||
|
||||
/* LRU of all extents in heaps. */
|
||||
edata_list_inactive_t lru;
|
||||
|
@ -56,7 +56,8 @@ struct psset_s {
|
||||
* free run of pages in a pageslab.
|
||||
*/
|
||||
hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
|
||||
bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
|
||||
/* Bitmap for which set bits correspond to non-empty heaps. */
|
||||
fb_group_t bitmap[FB_NGROUPS(PSSET_NPSIZES)];
|
||||
/*
|
||||
* The sum of all bin stats in stats. This lets us quickly answer
|
||||
* queries for the number of dirty, active, and retained pages in the
|
||||
|
30
src/eset.c
30
src/eset.c
@ -3,15 +3,14 @@
|
||||
|
||||
#include "jemalloc/internal/eset.h"
|
||||
|
||||
const bitmap_info_t eset_bitmap_info =
|
||||
BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);
|
||||
#define ESET_NPSIZES (SC_NPSIZES + 1)
|
||||
|
||||
void
|
||||
eset_init(eset_t *eset, extent_state_t state) {
|
||||
for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
|
||||
for (unsigned i = 0; i < ESET_NPSIZES; i++) {
|
||||
edata_heap_new(&eset->heaps[i]);
|
||||
}
|
||||
bitmap_init(eset->bitmap, &eset_bitmap_info, true);
|
||||
fb_init(eset->bitmap, ESET_NPSIZES);
|
||||
edata_list_inactive_init(&eset->lru);
|
||||
atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
|
||||
eset->state = state;
|
||||
@ -56,8 +55,7 @@ eset_insert(eset_t *eset, edata_t *edata) {
|
||||
size_t psz = sz_psz_quantize_floor(size);
|
||||
pszind_t pind = sz_psz2ind(psz);
|
||||
if (edata_heap_empty(&eset->heaps[pind])) {
|
||||
bitmap_unset(eset->bitmap, &eset_bitmap_info,
|
||||
(size_t)pind);
|
||||
fb_set(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||
}
|
||||
edata_heap_insert(&eset->heaps[pind], edata);
|
||||
|
||||
@ -92,8 +90,7 @@ eset_remove(eset_t *eset, edata_t *edata) {
|
||||
}
|
||||
|
||||
if (edata_heap_empty(&eset->heaps[pind])) {
|
||||
bitmap_set(eset->bitmap, &eset_bitmap_info,
|
||||
(size_t)pind);
|
||||
fb_unset(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||
}
|
||||
edata_list_inactive_remove(&eset->lru, edata);
|
||||
size_t npages = size >> LG_PAGE;
|
||||
@ -122,10 +119,10 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size,
|
||||
pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size));
|
||||
pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size));
|
||||
|
||||
for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
|
||||
&eset_bitmap_info, (size_t)pind); i < pind_max; i =
|
||||
(pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
|
||||
(size_t)i+1)) {
|
||||
for (pszind_t i =
|
||||
(pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||
i < pind_max;
|
||||
i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
|
||||
assert(i < SC_NPSIZES);
|
||||
assert(!edata_heap_empty(&eset->heaps[i]));
|
||||
edata_t *edata = edata_heap_first(&eset->heaps[i]);
|
||||
@ -171,11 +168,10 @@ eset_first_fit(eset_t *eset, size_t size, bool exact_only,
|
||||
edata_heap_first(&eset->heaps[pind]);
|
||||
}
|
||||
|
||||
for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
|
||||
&eset_bitmap_info, (size_t)pind);
|
||||
i < SC_NPSIZES + 1;
|
||||
i = (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
|
||||
(size_t)i+1)) {
|
||||
for (pszind_t i =
|
||||
(pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
|
||||
i < ESET_NPSIZES;
|
||||
i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
|
||||
assert(!edata_heap_empty(&eset->heaps[i]));
|
||||
edata_t *edata = edata_heap_first(&eset->heaps[i]);
|
||||
assert(edata_size_get(edata) >= size);
|
||||
|
11
src/psset.c
11
src/psset.c
@ -5,15 +5,12 @@
|
||||
|
||||
#include "jemalloc/internal/flat_bitmap.h"
|
||||
|
||||
static const bitmap_info_t psset_bitmap_info =
|
||||
BITMAP_INFO_INITIALIZER(PSSET_NPSIZES);
|
||||
|
||||
void
|
||||
psset_init(psset_t *psset) {
|
||||
for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
|
||||
hpdata_age_heap_new(&psset->pageslabs[i]);
|
||||
}
|
||||
bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
|
||||
fb_init(psset->bitmap, PSSET_NPSIZES);
|
||||
memset(&psset->merged_stats, 0, sizeof(psset->merged_stats));
|
||||
memset(&psset->stats, 0, sizeof(psset->stats));
|
||||
hpdata_empty_list_init(&psset->empty);
|
||||
@ -101,14 +98,14 @@ static void
|
||||
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
||||
hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
|
||||
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
|
||||
bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
|
||||
fb_unset(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
||||
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
|
||||
bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
|
||||
fb_set(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
|
||||
}
|
||||
hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
|
||||
}
|
||||
@ -266,7 +263,7 @@ psset_pick_alloc(psset_t *psset, size_t size) {
|
||||
assert(size <= HUGEPAGE);
|
||||
|
||||
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
|
||||
pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
|
||||
pszind_t pind = (pszind_t)fb_ffs(psset->bitmap, PSSET_NPSIZES,
|
||||
(size_t)min_pind);
|
||||
if (pind == PSSET_NPSIZES) {
|
||||
return hpdata_empty_list_first(&psset->empty);
|
||||
|
Loading…
Reference in New Issue
Block a user