Move hpdata bitmap logic out of the psset.
This commit is contained in:
parent
ca30b5db2b
commit
089f8fa442
@ -107,13 +107,15 @@ hpdata_longest_free_range_set(hpdata_t *hpdata, size_t longest_free_range) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static inline void
|
static inline void
|
||||||
hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
|
hpdata_assert_empty(hpdata_t *hpdata) {
|
||||||
hpdata_addr_set(hpdata, addr);
|
assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES));
|
||||||
hpdata_age_set(hpdata, age);
|
assert(hpdata_nfree_get(hpdata) == HUGEPAGE_PAGES);
|
||||||
hpdata_huge_set(hpdata, false);
|
}
|
||||||
hpdata_nfree_set(hpdata, HUGEPAGE_PAGES);
|
|
||||||
hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
|
static inline void
|
||||||
fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
|
hpdata_assert_consistent(hpdata_t *hpdata) {
|
||||||
|
assert(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES)
|
||||||
|
== hpdata_longest_free_range_get(hpdata));
|
||||||
}
|
}
|
||||||
|
|
||||||
TYPED_LIST(hpdata_list, hpdata_t, ql_link)
|
TYPED_LIST(hpdata_list, hpdata_t, ql_link)
|
||||||
@ -121,4 +123,12 @@ TYPED_LIST(hpdata_list, hpdata_t, ql_link)
|
|||||||
typedef ph(hpdata_t) hpdata_age_heap_t;
|
typedef ph(hpdata_t) hpdata_age_heap_t;
|
||||||
ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
|
ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t);
|
||||||
|
|
||||||
|
void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age);
|
||||||
|
/*
|
||||||
|
* Given an hpdata which can serve an allocation request, pick and reserve an
|
||||||
|
* offset within that allocation.
|
||||||
|
*/
|
||||||
|
size_t hpdata_reserve_alloc(hpdata_t *hpdata, size_t npages);
|
||||||
|
void hpdata_unreserve(hpdata_t *hpdata, size_t start, size_t npages);
|
||||||
|
|
||||||
#endif /* JEMALLOC_INTERNAL_HPDATA_H */
|
#endif /* JEMALLOC_INTERNAL_HPDATA_H */
|
||||||
|
96
src/hpdata.c
96
src/hpdata.c
@ -16,3 +16,99 @@ hpdata_age_comp(const hpdata_t *a, const hpdata_t *b) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ph_gen(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t, ph_link, hpdata_age_comp)
|
ph_gen(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t, ph_link, hpdata_age_comp)
|
||||||
|
|
||||||
|
|
||||||
|
void
|
||||||
|
hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age) {
|
||||||
|
hpdata_addr_set(hpdata, addr);
|
||||||
|
hpdata_age_set(hpdata, age);
|
||||||
|
hpdata_huge_set(hpdata, false);
|
||||||
|
hpdata_nfree_set(hpdata, HUGEPAGE_PAGES);
|
||||||
|
hpdata_longest_free_range_set(hpdata, HUGEPAGE_PAGES);
|
||||||
|
fb_init(hpdata->active_pages, HUGEPAGE_PAGES);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t
|
||||||
|
hpdata_reserve_alloc(hpdata_t *hpdata, size_t npages) {
|
||||||
|
assert(npages <= hpdata_longest_free_range_get(hpdata));
|
||||||
|
|
||||||
|
size_t result;
|
||||||
|
|
||||||
|
size_t start = 0;
|
||||||
|
/*
|
||||||
|
* These are dead stores, but the compiler will issue warnings on them
|
||||||
|
* since it can't tell statically that found is always true below.
|
||||||
|
*/
|
||||||
|
size_t begin = 0;
|
||||||
|
size_t len = 0;
|
||||||
|
|
||||||
|
size_t largest_unchosen_range = 0;
|
||||||
|
while (true) {
|
||||||
|
bool found = fb_urange_iter(hpdata->active_pages,
|
||||||
|
HUGEPAGE_PAGES, start, &begin, &len);
|
||||||
|
/*
|
||||||
|
* A precondition to this function is that hpdata must be able
|
||||||
|
* to serve the allocation.
|
||||||
|
*/
|
||||||
|
assert(found);
|
||||||
|
if (len >= npages) {
|
||||||
|
/*
|
||||||
|
* We use first-fit within the page slabs; this gives
|
||||||
|
* bounded worst-case fragmentation within a slab. It's
|
||||||
|
* not necessarily right; we could experiment with
|
||||||
|
* various other options.
|
||||||
|
*/
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (len > largest_unchosen_range) {
|
||||||
|
largest_unchosen_range = len;
|
||||||
|
}
|
||||||
|
start = begin + len;
|
||||||
|
}
|
||||||
|
/* We found a range; remember it. */
|
||||||
|
result = begin;
|
||||||
|
fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages);
|
||||||
|
hpdata_nfree_set(hpdata, hpdata_nfree_get(hpdata) - npages);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* We might have shrunk the longest free range. We have to keep
|
||||||
|
* scanning until the end of the hpdata to be sure.
|
||||||
|
*
|
||||||
|
* TODO: As an optimization, we should only do this when the range we
|
||||||
|
* just allocated from was equal to the longest free range size.
|
||||||
|
*/
|
||||||
|
start = begin + npages;
|
||||||
|
while (start < HUGEPAGE_PAGES) {
|
||||||
|
bool found = fb_urange_iter(hpdata->active_pages,
|
||||||
|
HUGEPAGE_PAGES, start, &begin, &len);
|
||||||
|
if (!found) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (len > largest_unchosen_range) {
|
||||||
|
largest_unchosen_range = len;
|
||||||
|
}
|
||||||
|
start = begin + len;
|
||||||
|
}
|
||||||
|
hpdata_longest_free_range_set(hpdata, largest_unchosen_range);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
hpdata_unreserve(hpdata_t *hpdata, size_t begin, size_t npages) {
|
||||||
|
size_t old_longest_range = hpdata_longest_free_range_get(hpdata);
|
||||||
|
|
||||||
|
fb_unset_range(hpdata->active_pages, HUGEPAGE_PAGES, begin, npages);
|
||||||
|
/* We might have just created a new, larger range. */
|
||||||
|
size_t new_begin = (fb_fls(hpdata->active_pages, HUGEPAGE_PAGES,
|
||||||
|
begin) + 1);
|
||||||
|
size_t new_end = fb_ffs(hpdata->active_pages, HUGEPAGE_PAGES,
|
||||||
|
begin + npages - 1);
|
||||||
|
size_t new_range_len = new_end - new_begin;
|
||||||
|
|
||||||
|
if (new_range_len > old_longest_range) {
|
||||||
|
hpdata_longest_free_range_set(hpdata, new_range_len);
|
||||||
|
}
|
||||||
|
|
||||||
|
hpdata_nfree_set(hpdata, hpdata_nfree_get(hpdata) + npages);
|
||||||
|
}
|
||||||
|
104
src/psset.c
104
src/psset.c
@ -105,15 +105,9 @@ psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
|
|||||||
psset_bin_stats_insert(&psset->stats.nonfull_slabs[pind], ps);
|
psset_bin_stats_insert(&psset->stats.nonfull_slabs[pind], ps);
|
||||||
}
|
}
|
||||||
|
|
||||||
JEMALLOC_ALWAYS_INLINE void
|
|
||||||
psset_assert_ps_consistent(hpdata_t *ps) {
|
|
||||||
assert(fb_urange_longest(ps->active_pages, HUGEPAGE_PAGES)
|
|
||||||
== hpdata_longest_free_range_get(ps));
|
|
||||||
}
|
|
||||||
|
|
||||||
void
|
void
|
||||||
psset_insert(psset_t *psset, hpdata_t *ps) {
|
psset_insert(psset_t *psset, hpdata_t *ps) {
|
||||||
psset_assert_ps_consistent(ps);
|
hpdata_assert_consistent(ps);
|
||||||
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
||||||
|
|
||||||
if (longest_free_range == 0) {
|
if (longest_free_range == 0) {
|
||||||
@ -137,7 +131,7 @@ psset_insert(psset_t *psset, hpdata_t *ps) {
|
|||||||
|
|
||||||
void
|
void
|
||||||
psset_remove(psset_t *psset, hpdata_t *ps) {
|
psset_remove(psset_t *psset, hpdata_t *ps) {
|
||||||
psset_assert_ps_consistent(ps);
|
hpdata_assert_consistent(ps);
|
||||||
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
||||||
|
|
||||||
if (longest_free_range == 0) {
|
if (longest_free_range == 0) {
|
||||||
@ -157,7 +151,7 @@ psset_remove(psset_t *psset, hpdata_t *ps) {
|
|||||||
void
|
void
|
||||||
psset_hugify(psset_t *psset, hpdata_t *ps) {
|
psset_hugify(psset_t *psset, hpdata_t *ps) {
|
||||||
assert(!hpdata_huge_get(ps));
|
assert(!hpdata_huge_get(ps));
|
||||||
psset_assert_ps_consistent(ps);
|
hpdata_assert_consistent(ps);
|
||||||
|
|
||||||
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
||||||
psset_bin_stats_t *bin_stats;
|
psset_bin_stats_t *bin_stats;
|
||||||
@ -196,7 +190,7 @@ psset_recycle_extract(psset_t *psset, size_t size) {
|
|||||||
bitmap_set(psset->bitmap, &psset_bitmap_info, pind);
|
bitmap_set(psset->bitmap, &psset_bitmap_info, pind);
|
||||||
}
|
}
|
||||||
|
|
||||||
psset_assert_ps_consistent(ps);
|
hpdata_assert_consistent(ps);
|
||||||
return ps;
|
return ps;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -207,76 +201,18 @@ psset_recycle_extract(psset_t *psset, size_t size) {
|
|||||||
static void
|
static void
|
||||||
psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
|
psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata,
|
||||||
size_t size) {
|
size_t size) {
|
||||||
size_t start = 0;
|
size_t npages = size / PAGE;
|
||||||
/*
|
size_t begin = hpdata_reserve_alloc(ps, npages);
|
||||||
* These are dead stores, but the compiler will issue warnings on them
|
|
||||||
* since it can't tell statically that found is always true below.
|
|
||||||
*/
|
|
||||||
size_t begin = 0;
|
|
||||||
size_t len = 0;
|
|
||||||
|
|
||||||
fb_group_t *ps_fb = ps->active_pages;
|
|
||||||
|
|
||||||
size_t npages = size >> LG_PAGE;
|
|
||||||
|
|
||||||
size_t largest_unchosen_range = 0;
|
|
||||||
while (true) {
|
|
||||||
bool found = fb_urange_iter(ps_fb, HUGEPAGE_PAGES, start,
|
|
||||||
&begin, &len);
|
|
||||||
/*
|
|
||||||
* A precondition to this function is that ps must be able to
|
|
||||||
* serve the allocation.
|
|
||||||
*/
|
|
||||||
assert(found);
|
|
||||||
if (len >= npages) {
|
|
||||||
/*
|
|
||||||
* We use first-fit within the page slabs; this gives
|
|
||||||
* bounded worst-case fragmentation within a slab. It's
|
|
||||||
* not necessarily right; we could experiment with
|
|
||||||
* various other options.
|
|
||||||
*/
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (len > largest_unchosen_range) {
|
|
||||||
largest_unchosen_range = len;
|
|
||||||
}
|
|
||||||
start = begin + len;
|
|
||||||
}
|
|
||||||
uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
|
uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE;
|
||||||
edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size,
|
edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size,
|
||||||
/* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active,
|
/* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active,
|
||||||
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
|
/* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA,
|
||||||
EXTENT_NOT_HEAD);
|
EXTENT_NOT_HEAD);
|
||||||
edata_ps_set(r_edata, ps);
|
edata_ps_set(r_edata, ps);
|
||||||
fb_set_range(ps_fb, HUGEPAGE_PAGES, begin, npages);
|
|
||||||
hpdata_nfree_set(ps, (uint32_t)(hpdata_nfree_get(ps) - npages));
|
|
||||||
/* The pageslab isn't in a bin, so no bin stats need to change. */
|
/* The pageslab isn't in a bin, so no bin stats need to change. */
|
||||||
|
|
||||||
/*
|
size_t longest_free_range = hpdata_longest_free_range_get(ps);
|
||||||
* OK, we've got to put the pageslab back. First we have to figure out
|
if (longest_free_range == 0) {
|
||||||
* where, though; we've only checked run sizes before the pageslab we
|
|
||||||
* picked. We also need to look for ones after the one we picked. Note
|
|
||||||
* that we want begin + npages as the start position, not begin + len;
|
|
||||||
* we might not have used the whole range.
|
|
||||||
*
|
|
||||||
* TODO: With a little bit more care, we can guarantee that the longest
|
|
||||||
* free range field in the edata is accurate upon entry, and avoid doing
|
|
||||||
* this check in the case where we're allocating from some smaller run.
|
|
||||||
*/
|
|
||||||
start = begin + npages;
|
|
||||||
while (start < HUGEPAGE_PAGES) {
|
|
||||||
bool found = fb_urange_iter(ps_fb, HUGEPAGE_PAGES, start, &begin,
|
|
||||||
&len);
|
|
||||||
if (!found) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (len > largest_unchosen_range) {
|
|
||||||
largest_unchosen_range = len;
|
|
||||||
}
|
|
||||||
start = begin + len;
|
|
||||||
}
|
|
||||||
hpdata_longest_free_range_set(ps, (uint32_t)largest_unchosen_range);
|
|
||||||
if (largest_unchosen_range == 0) {
|
|
||||||
psset_bin_stats_insert(&psset->stats.full_slabs, ps);
|
psset_bin_stats_insert(&psset->stats.full_slabs, ps);
|
||||||
} else {
|
} else {
|
||||||
psset_insert(psset, ps);
|
psset_insert(psset, ps);
|
||||||
@ -295,9 +231,7 @@ psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) {
|
|||||||
|
|
||||||
void
|
void
|
||||||
psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
|
psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) {
|
||||||
fb_group_t *ps_fb = ps->active_pages;
|
hpdata_assert_empty(ps);
|
||||||
assert(fb_empty(ps_fb, HUGEPAGE_PAGES));
|
|
||||||
assert(hpdata_nfree_get(ps) == HUGEPAGE_PAGES);
|
|
||||||
psset_ps_alloc_insert(psset, ps, r_edata, size);
|
psset_ps_alloc_insert(psset, ps, r_edata, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -307,7 +241,6 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
|
|||||||
assert(edata_ps_get(edata) != NULL);
|
assert(edata_ps_get(edata) != NULL);
|
||||||
hpdata_t *ps = edata_ps_get(edata);
|
hpdata_t *ps = edata_ps_get(edata);
|
||||||
|
|
||||||
fb_group_t *ps_fb = ps->active_pages;
|
|
||||||
size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps);
|
size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps);
|
||||||
pszind_t old_pind = SC_NPSIZES;
|
pszind_t old_pind = SC_NPSIZES;
|
||||||
if (ps_old_longest_free_range != 0) {
|
if (ps_old_longest_free_range != 0) {
|
||||||
@ -319,32 +252,23 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
|
|||||||
((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps))
|
((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps))
|
||||||
>> LG_PAGE;
|
>> LG_PAGE;
|
||||||
size_t len = edata_size_get(edata) >> LG_PAGE;
|
size_t len = edata_size_get(edata) >> LG_PAGE;
|
||||||
fb_unset_range(ps_fb, HUGEPAGE_PAGES, begin, len);
|
|
||||||
|
|
||||||
/* The pageslab is still in the bin; adjust its stats first. */
|
/* The pageslab is still in the bin; adjust its stats first. */
|
||||||
psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0
|
psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0
|
||||||
? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]);
|
? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]);
|
||||||
psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len);
|
psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len);
|
||||||
|
|
||||||
hpdata_nfree_set(ps, (uint32_t)(hpdata_nfree_get(ps) + len));
|
hpdata_unreserve(ps, begin, len);
|
||||||
|
size_t ps_new_longest_free_range = hpdata_longest_free_range_get(ps);
|
||||||
|
|
||||||
/* We might have just created a new, larger range. */
|
|
||||||
size_t new_begin = (size_t)(fb_fls(ps_fb, HUGEPAGE_PAGES, begin) + 1);
|
|
||||||
size_t new_end = fb_ffs(ps_fb, HUGEPAGE_PAGES, begin + len - 1);
|
|
||||||
size_t new_range_len = new_end - new_begin;
|
|
||||||
/*
|
/*
|
||||||
* If the new free range is no longer than the previous longest one,
|
* If the new free range is no longer than the previous longest one,
|
||||||
* then the pageslab is non-empty and doesn't need to change bins.
|
* then the pageslab is non-empty and doesn't need to change bins.
|
||||||
* We're done, and don't need to return a pageslab to evict.
|
* We're done, and don't need to return a pageslab to evict.
|
||||||
*/
|
*/
|
||||||
if (new_range_len <= ps_old_longest_free_range) {
|
if (ps_new_longest_free_range <= ps_old_longest_free_range) {
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
* Otherwise, it might need to get evicted from the set, or change its
|
|
||||||
* bin.
|
|
||||||
*/
|
|
||||||
hpdata_longest_free_range_set(ps, (uint32_t)new_range_len);
|
|
||||||
/*
|
/*
|
||||||
* If it was previously non-full, then it's in some (possibly now
|
* If it was previously non-full, then it's in some (possibly now
|
||||||
* incorrect) bin already; remove it.
|
* incorrect) bin already; remove it.
|
||||||
@ -371,12 +295,12 @@ psset_dalloc(psset_t *psset, edata_t *edata) {
|
|||||||
psset_bin_stats_remove(&psset->stats.full_slabs, ps);
|
psset_bin_stats_remove(&psset->stats.full_slabs, ps);
|
||||||
}
|
}
|
||||||
/* If the pageslab is empty, it gets evicted from the set. */
|
/* If the pageslab is empty, it gets evicted from the set. */
|
||||||
if (new_range_len == HUGEPAGE_PAGES) {
|
if (ps_new_longest_free_range == HUGEPAGE_PAGES) {
|
||||||
return ps;
|
return ps;
|
||||||
}
|
}
|
||||||
/* Otherwise, it gets reinserted. */
|
/* Otherwise, it gets reinserted. */
|
||||||
pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor(
|
pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor(
|
||||||
new_range_len << LG_PAGE));
|
ps_new_longest_free_range << LG_PAGE));
|
||||||
if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) {
|
if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) {
|
||||||
bitmap_unset(psset->bitmap, &psset_bitmap_info,
|
bitmap_unset(psset->bitmap, &psset_bitmap_info,
|
||||||
(size_t)new_pind);
|
(size_t)new_pind);
|
||||||
|
Loading…
Reference in New Issue
Block a user