HPA: Implement batch deallocation.
This saves O(n) mutex locks/unlocks during SEC flush.
This commit is contained in:
parent
f47b4c2cd8
commit
1944ebbe7f
@ -13,6 +13,7 @@ struct pai_s {
|
|||||||
bool (*shrink)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
bool (*shrink)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||||
size_t old_size, size_t new_size);
|
size_t old_size, size_t new_size);
|
||||||
void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata);
|
void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata);
|
||||||
|
/* This function empties out list as a side-effect of being called. */
|
||||||
void (*dalloc_batch)(tsdn_t *tsdn, pai_t *self,
|
void (*dalloc_batch)(tsdn_t *tsdn, pai_t *self,
|
||||||
edata_list_active_t *list);
|
edata_list_active_t *list);
|
||||||
};
|
};
|
||||||
|
63
src/hpa.c
63
src/hpa.c
@ -15,6 +15,8 @@ static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
|||||||
static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||||
size_t old_size, size_t new_size);
|
size_t old_size, size_t new_size);
|
||||||
static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata);
|
static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata);
|
||||||
|
static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
|
||||||
|
edata_list_active_t *list);
|
||||||
|
|
||||||
bool
|
bool
|
||||||
hpa_supported() {
|
hpa_supported() {
|
||||||
@ -91,7 +93,7 @@ hpa_shard_init(hpa_shard_t *shard, emap_t *emap, base_t *base,
|
|||||||
shard->pai.expand = &hpa_expand;
|
shard->pai.expand = &hpa_expand;
|
||||||
shard->pai.shrink = &hpa_shrink;
|
shard->pai.shrink = &hpa_shrink;
|
||||||
shard->pai.dalloc = &hpa_dalloc;
|
shard->pai.dalloc = &hpa_dalloc;
|
||||||
shard->pai.dalloc_batch = &pai_dalloc_batch_default;
|
shard->pai.dalloc_batch = &hpa_dalloc_batch;
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -663,11 +665,8 @@ hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
|
hpa_dalloc_prepare_unlocked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
|
||||||
hpa_shard_t *shard = hpa_from_pai(self);
|
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
|
||||||
|
|
||||||
edata_addr_set(edata, edata_base_get(edata));
|
|
||||||
edata_zeroed_set(edata, false);
|
|
||||||
|
|
||||||
assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
|
assert(edata_pai_get(edata) == EXTENT_PAI_HPA);
|
||||||
assert(edata_state_get(edata) == extent_state_active);
|
assert(edata_state_get(edata) == extent_state_active);
|
||||||
@ -677,32 +676,62 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
|
|||||||
assert(edata_committed_get(edata));
|
assert(edata_committed_get(edata));
|
||||||
assert(edata_base_get(edata) != NULL);
|
assert(edata_base_get(edata) != NULL);
|
||||||
|
|
||||||
hpdata_t *ps = edata_ps_get(edata);
|
edata_addr_set(edata, edata_base_get(edata));
|
||||||
/* Currently, all edatas come from pageslabs. */
|
edata_zeroed_set(edata, false);
|
||||||
assert(ps != NULL);
|
|
||||||
emap_deregister_boundary(tsdn, shard->emap, edata);
|
emap_deregister_boundary(tsdn, shard->emap, edata);
|
||||||
/*
|
}
|
||||||
* Note that the shard mutex protects ps's metadata too; it wouldn't be
|
|
||||||
* correct to try to read most information out of it without the lock.
|
static void
|
||||||
*/
|
hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
|
||||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Release the metadata early, to avoid having to remember to do it
|
* Release the metadata early, to avoid having to remember to do it
|
||||||
* while we're also doing tricky purging logic.
|
* while we're also doing tricky purging logic. First, we need to grab
|
||||||
|
* a few bits of metadata from it.
|
||||||
|
*
|
||||||
|
* Note that the shard mutex protects ps's metadata too; it wouldn't be
|
||||||
|
* correct to try to read most information out of it without the lock.
|
||||||
*/
|
*/
|
||||||
|
hpdata_t *ps = edata_ps_get(edata);
|
||||||
|
/* Currently, all edatas come from pageslabs. */
|
||||||
|
assert(ps != NULL);
|
||||||
void *unreserve_addr = edata_addr_get(edata);
|
void *unreserve_addr = edata_addr_get(edata);
|
||||||
size_t unreserve_size = edata_size_get(edata);
|
size_t unreserve_size = edata_size_get(edata);
|
||||||
edata_cache_small_put(tsdn, &shard->ecs, edata);
|
edata_cache_small_put(tsdn, &shard->ecs, edata);
|
||||||
|
|
||||||
psset_update_begin(&shard->psset, ps);
|
psset_update_begin(&shard->psset, ps);
|
||||||
hpdata_unreserve(ps, unreserve_addr, unreserve_size);
|
hpdata_unreserve(ps, unreserve_addr, unreserve_size);
|
||||||
|
|
||||||
hpa_update_purge_hugify_eligibility(shard, ps);
|
hpa_update_purge_hugify_eligibility(shard, ps);
|
||||||
psset_update_end(&shard->psset, ps);
|
psset_update_end(&shard->psset, ps);
|
||||||
|
|
||||||
hpa_do_deferred_work(tsdn, shard);
|
hpa_do_deferred_work(tsdn, shard);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
|
||||||
|
hpa_shard_t *shard = hpa_from_pai(self);
|
||||||
|
|
||||||
|
hpa_dalloc_prepare_unlocked(tsdn, shard, edata);
|
||||||
|
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||||
|
hpa_dalloc_locked(tsdn, shard, edata);
|
||||||
|
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list) {
|
||||||
|
hpa_shard_t *shard = hpa_from_pai(self);
|
||||||
|
|
||||||
|
edata_t *edata;
|
||||||
|
ql_foreach(edata, &list->head, ql_link_active) {
|
||||||
|
hpa_dalloc_prepare_unlocked(tsdn, shard, edata);
|
||||||
|
}
|
||||||
|
|
||||||
|
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||||
|
/* Now, remove from the list. */
|
||||||
|
while ((edata = edata_list_active_first(list)) != NULL) {
|
||||||
|
edata_list_active_remove(list, edata);
|
||||||
|
hpa_dalloc_locked(tsdn, shard, edata);
|
||||||
|
}
|
||||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -144,13 +144,6 @@ sec_do_flush_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
|
|||||||
edata_list_active_concat(&to_flush, &shard->freelist[i]);
|
edata_list_active_concat(&to_flush, &shard->freelist[i]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* A better way to do this would be to add a batch dalloc function to
|
|
||||||
* the pai_t. Practically, the current method turns into O(n) locks and
|
|
||||||
* unlocks at the fallback allocator. But some implementations (e.g.
|
|
||||||
* HPA) can straightforwardly do many deallocations in a single lock /
|
|
||||||
* unlock pair.
|
|
||||||
*/
|
|
||||||
pai_dalloc_batch(tsdn, sec->fallback, &to_flush);
|
pai_dalloc_batch(tsdn, sec->fallback, &to_flush);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user