project-base/server-skynet-source-3rd-jemalloc
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use the flat bitmap for eset and psset bitmaps.

Browse Source 
This is simpler (note that the eset field comment was actually incorrect!), and
slightly faster.
This commit is contained in:
David Goldblatt 2021-02-08 11:04:46 -08:00 committed by David Goldblatt
parent 271a676dcd
commit 154aa5fcc1
4 changed files with 21 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/jemalloc/internal/eset.h
View File

@ -2,7 +2,7 @@
#define JEMALLOC_INTERNAL_ESET_H
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/flat_bitmap.h"
#include "jemalloc/internal/edata.h"
#include "jemalloc/internal/mutex.h"
@ -22,7 +22,7 @@ struct eset_s {
atomic_zu_t nbytes[SC_NPSIZES + 1];
/* Bitmap for which set bits correspond to non-empty heaps. */
bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)];
fb_group_t bitmap[FB_NGROUPS(SC_NPSIZES + 1)];
/* LRU of all extents in heaps. */
edata_list_inactive_t lru;

3
include/jemalloc/internal/psset.h
View File

@ -56,7 +56,8 @@ struct psset_s {
* free run of pages in a pageslab.
*/
hpdata_age_heap_t pageslabs[PSSET_NPSIZES];
bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)];
/* Bitmap for which set bits correspond to non-empty heaps. */
fb_group_t bitmap[FB_NGROUPS(PSSET_NPSIZES)];
/*
* The sum of all bin stats in stats. This lets us quickly answer
* queries for the number of dirty, active, and retained pages in the

30
src/eset.c
View File

@ -3,15 +3,14 @@
#include "jemalloc/internal/eset.h"
const bitmap_info_t eset_bitmap_info =
BITMAP_INFO_INITIALIZER(SC_NPSIZES+1);
#define ESET_NPSIZES (SC_NPSIZES + 1)
void
eset_init(eset_t *eset, extent_state_t state) {
for (unsigned i = 0; i < SC_NPSIZES + 1; i++) {
for (unsigned i = 0; i < ESET_NPSIZES; i++) {
edata_heap_new(&eset->heaps[i]);
}
bitmap_init(eset->bitmap, &eset_bitmap_info, true);
fb_init(eset->bitmap, ESET_NPSIZES);
edata_list_inactive_init(&eset->lru);
atomic_store_zu(&eset->npages, 0, ATOMIC_RELAXED);
eset->state = state;
@ -56,8 +55,7 @@ eset_insert(eset_t *eset, edata_t *edata) {
size_t psz = sz_psz_quantize_floor(size);
pszind_t pind = sz_psz2ind(psz);
if (edata_heap_empty(&eset->heaps[pind])) {
bitmap_unset(eset->bitmap, &eset_bitmap_info,
(size_t)pind);
fb_set(eset->bitmap, ESET_NPSIZES, (size_t)pind);
}
edata_heap_insert(&eset->heaps[pind], edata);
@ -92,8 +90,7 @@ eset_remove(eset_t *eset, edata_t *edata) {
}
if (edata_heap_empty(&eset->heaps[pind])) {
bitmap_set(eset->bitmap, &eset_bitmap_info,
(size_t)pind);
fb_unset(eset->bitmap, ESET_NPSIZES, (size_t)pind);
}
edata_list_inactive_remove(&eset->lru, edata);
size_t npages = size >> LG_PAGE;
@ -122,10 +119,10 @@ eset_fit_alignment(eset_t *eset, size_t min_size, size_t max_size,
pszind_t pind = sz_psz2ind(sz_psz_quantize_ceil(min_size));
pszind_t pind_max = sz_psz2ind(sz_psz_quantize_ceil(max_size));
for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
&eset_bitmap_info, (size_t)pind); i < pind_max; i =
(pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
(size_t)i+1)) {
for (pszind_t i =
(pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
i < pind_max;
i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
assert(i < SC_NPSIZES);
assert(!edata_heap_empty(&eset->heaps[i]));
edata_t *edata = edata_heap_first(&eset->heaps[i]);
@ -171,11 +168,10 @@ eset_first_fit(eset_t *eset, size_t size, bool exact_only,
edata_heap_first(&eset->heaps[pind]);
}
for (pszind_t i = (pszind_t)bitmap_ffu(eset->bitmap,
&eset_bitmap_info, (size_t)pind);
i < SC_NPSIZES + 1;
i = (pszind_t)bitmap_ffu(eset->bitmap, &eset_bitmap_info,
(size_t)i+1)) {
for (pszind_t i =
(pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)pind);
i < ESET_NPSIZES;
i = (pszind_t)fb_ffs(eset->bitmap, ESET_NPSIZES, (size_t)i + 1)) {
assert(!edata_heap_empty(&eset->heaps[i]));
edata_t *edata = edata_heap_first(&eset->heaps[i]);
assert(edata_size_get(edata) >= size);

11
src/psset.c
View File

@ -5,15 +5,12 @@
#include "jemalloc/internal/flat_bitmap.h"
static const bitmap_info_t psset_bitmap_info =
BITMAP_INFO_INITIALIZER(PSSET_NPSIZES);
void
psset_init(psset_t *psset) {
for (unsigned i = 0; i < PSSET_NPSIZES; i++) {
hpdata_age_heap_new(&psset->pageslabs[i]);
}
bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
fb_init(psset->bitmap, PSSET_NPSIZES);
memset(&psset->merged_stats, 0, sizeof(psset->merged_stats));
memset(&psset->stats, 0, sizeof(psset->stats));
hpdata_empty_list_init(&psset->empty);
@ -101,14 +98,14 @@ static void
psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) {
hpdata_age_heap_remove(&psset->pageslabs[pind], ps);
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_set(psset->bitmap, &psset_bitmap_info, (size_t)pind);
fb_unset(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
}
}
static void
psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
if (hpdata_age_heap_empty(&psset->pageslabs[pind])) {
bitmap_unset(psset->bitmap, &psset_bitmap_info, (size_t)pind);
fb_set(psset->bitmap, PSSET_NPSIZES, (size_t)pind);
}
hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
}
@ -266,7 +263,7 @@ psset_pick_alloc(psset_t *psset, size_t size) {
assert(size <= HUGEPAGE);
pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size));
pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
pszind_t pind = (pszind_t)fb_ffs(psset->bitmap, PSSET_NPSIZES,
(size_t)min_pind);
if (pind == PSSET_NPSIZES) {
return hpdata_empty_list_first(&psset->empty);