From 154aa5fcc102172fcac0e111ff79df9d5ced7973 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Mon, 8 Feb 2021 11:04:46 -0800 Subject: [PATCH] Use the flat bitmap for eset and psset bitmaps. This is simpler (note that the eset field comment was actually incorrect!), and slightly faster. --- include/jemalloc/internal/eset.h | 4 ++-- include/jemalloc/internal/psset.h | 3 ++- src/eset.c | 30 +++++++++++++----------------- src/psset.c | 11 ++++------- 4 files changed, 21 insertions(+), 27 deletions(-) diff --git a/include/jemalloc/internal/eset.h b/include/jemalloc/internal/eset.h index d260bc13..7b53ecd8 100644 --- a/include/jemalloc/internal/eset.h +++ b/include/jemalloc/internal/eset.h @@ -2,7 +2,7 @@ #define JEMALLOC_INTERNAL_ESET_H #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/flat_bitmap.h" #include "jemalloc/internal/edata.h" #include "jemalloc/internal/mutex.h" @@ -22,7 +22,7 @@ struct eset_s { atomic_zu_t nbytes[SC_NPSIZES + 1]; /* Bitmap for which set bits correspond to non-empty heaps. */ - bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)]; + fb_group_t bitmap[FB_NGROUPS(SC_NPSIZES + 1)]; /* LRU of all extents in heaps. */ edata_list_inactive_t lru; diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index d2a8b24a..2b6ea7bc 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -56,7 +56,8 @@ struct psset_s { * free run of pages in a pageslab. */ hpdata_age_heap_t pageslabs[PSSET_NPSIZES]; - bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)]; + /* Bitmap for which set bits correspond to non-empty heaps. */ + fb_group_t bitmap[FB_NGROUPS(PSSET_NPSIZES)]; /* * The sum of all bin stats in stats. This lets us quickly answer * queries for the number of dirty, active, and retained pages in the diff --git a/src/eset.c b/src/eset.c index c9af80e1..a52a6f7c 100644 --- a/src/eset.c +++ b/src/eset.c @@ -3,15 +3,14 @@ #include "jemalloc/internal/eset.h" -const bitmap_info_t eset_bitmap_info = - BITMAP_INFO_INITIALIZER(SC_NPSIZES+1); +#define ESET_NPSIZES (SC_NPSIZES + 1) void eset_init(eset_t *eset, extent_state_t state) { - for (unsigned i = 0; i < SC_NPSIZES + 1; i++) { + for (unsigned i = 0; i < ESET_NPSIZES; i++) { edata_heap_new(&eset->heaps[i]); } - bitmap_init(eset->bitmap, &eset_bitmap_info, true); + fb_init(eset->bitmap, ESET_NPSIZES); edata_list_inactive_init(&eset->lru); atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED); eset->state = state; @@ -56,8 +55,7 @@ eset_insert(eset_t *eset, edata_t *edata) { size_t psz = sz_psz_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); if (edata_heap_empty(&eset->heaps[pind])) { - bitmap_unset(eset->bitmap, &eset_bitmap_info, - (size_t)pind); + fb_set(eset->bitmap, ESET_NPSIZES, (size_t)pind); } edata_heap_insert(&eset->heaps[pind], edata); @@ -92,8 +90,7 @@ eset_remove(eset_t *eset, edata_t *edata) { } if (edata_heap_empty(&eset->heaps[pind])) { - bitmap_set(eset->bitmap, &eset_bitmap_info, - (size_t)pind); + fb_unset(eset->bitmap, ESET_NPSIZES, (size_t)pind); } edata_list_inactive_remove(&eset->lru, edata); size_t npages = size >> LG_PAGE; @@ -122,10 +119,10 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size, pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size)); pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size)); - for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap, - &eset_bitmap_info, (size_t)pind); i < pind_max; i = - (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info, - (size_t)i+1)) { + for (pszind_t i = + (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind); + i < pind_max; + i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) { assert(i < SC_NPSIZES); assert(!edata_heap_empty(&eset->heaps[i])); edata_t *edata = edata_heap_first(&eset->heaps[i]); @@ -171,11 +168,10 @@ eset_first_fit(eset_t *eset, size_t size, bool exact_only, edata_heap_first(&eset->heaps[pind]); } - for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap, - &eset_bitmap_info, (size_t)pind); - i < SC_NPSIZES + 1; - i = (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info, - (size_t)i+1)) { + for (pszind_t i = + (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind); + i < ESET_NPSIZES; + i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) { assert(!edata_heap_empty(&eset->heaps[i])); edata_t *edata = edata_heap_first(&eset->heaps[i]); assert(edata_size_get(edata) >= size); diff --git a/src/psset.c b/src/psset.c index 08c9b6c5..a54e4b75 100644 --- a/src/psset.c +++ b/src/psset.c @@ -5,15 +5,12 @@ #include "jemalloc/internal/flat_bitmap.h" -static const bitmap_info_t psset_bitmap_info = - BITMAP_INFO_INITIALIZER(PSSET_NPSIZES); - void psset_init(psset_t *psset) { for (unsigned i = 0; i < PSSET_NPSIZES; i++) { hpdata_age_heap_new(&psset->pageslabs[i]); } - bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true); + fb_init(psset->bitmap, PSSET_NPSIZES); memset(&psset->merged_stats, 0, sizeof(psset->merged_stats)); memset(&psset->stats, 0, sizeof(psset->stats)); hpdata_empty_list_init(&psset->empty); @@ -101,14 +98,14 @@ static void psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) { hpdata_age_heap_remove(&psset->pageslabs[pind], ps); if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { - bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind); + fb_unset(psset->bitmap, PSSET_NPSIZES, (size_t)pind); } } static void psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) { if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { - bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind); + fb_set(psset->bitmap, PSSET_NPSIZES, (size_t)pind); } hpdata_age_heap_insert(&psset->pageslabs[pind], ps); } @@ -266,7 +263,7 @@ psset_pick_alloc(psset_t *psset, size_t size) { assert(size <= HUGEPAGE); pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size)); - pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info, + pszind_t pind = (pszind_t)fb_ffs(psset->bitmap, PSSET_NPSIZES, (size_t)min_pind); if (pind == PSSET_NPSIZES) { return hpdata_empty_list_first(&psset->empty);