lihao/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-24 05:33:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add batching to arena bins.

Browse Source 
This adds a fast-path for threads freeing a small number of allocations to
bins which are not their "home-base" and which encounter lock contention in
attempting to do so. In producer-consumer workflows, such small lock hold times
can cause lock convoying that greatly increases overall bin mutex contention.
This commit is contained in:
David Goldblatt
2024-02-09 16:08:45 -08:00
committed by David Goldblatt
parent 44d91cf243
commit fc615739cb
16 changed files with 722 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/bin.c
View File

@@ -6,6 +6,14 @@
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/witness.h"
#ifdef JEMALLOC_JET
unsigned bin_batching_test_ndalloc_slabs_max = (unsigned)-1;
void (*bin_batching_test_after_push_hook)(size_t push_idx);
void (*bin_batching_test_mid_pop_hook)(size_t nelems_to_pop);
void (*bin_batching_test_after_unlock_hook)(unsigned slab_dalloc_count,
bool list_empty);
#endif
bool
bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size,
size_t end_size, size_t nshards) {
@@ -39,7 +47,7 @@ bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) {
}
bool
bin_init(bin_t *bin) {
bin_init(bin_t *bin, unsigned binind) {
if (malloc_mutex_init(&bin->lock, "bin", WITNESS_RANK_BIN,
malloc_mutex_rank_exclusive)) {
return true;
@@ -50,6 +58,11 @@ bin_init(bin_t *bin) {
if (config_stats) {
memset(&bin->stats, 0, sizeof(bin_stats_t));
}
if (arena_bin_has_batch(binind)) {
bin_with_batch_t *batched_bin = (bin_with_batch_t *)bin;
batcher_init(&batched_bin->remote_frees,
opt_bin_info_remote_free_max);
}
return false;
}
@@ -57,8 +70,23 @@ void
bin_prefork(tsdn_t *tsdn, bin_t *bin, bool has_batch) {
malloc_mutex_prefork(tsdn, &bin->lock);
if (has_batch) {
/*
* The batch mutex has lower rank than the bin mutex (as it must
* -- it's acquired later). But during forking, we go
* bin-at-a-time, so that we acquire mutex on bin 0, then on
* the bin 0 batcher, then on bin 1. This is a safe ordering
* (it's ordered by the index of arenas and bins within those
* arenas), but will trigger witness errors that would
* otherwise force another level of arena forking that breaks
* bin encapsulation (because the witness API doesn't "know"
* about arena or bin ordering -- it just sees that the batcher
* has a lower rank than the bin). So instead we exclude the
* batcher mutex from witness checking during fork (which is
* the only time we touch multiple bins at once) by passing
* TSDN_NULL.
*/
bin_with_batch_t *batched = (bin_with_batch_t *)bin;
batcher_prefork(tsdn, &batched->remote_frees);
batcher_prefork(TSDN_NULL, &batched->remote_frees);
}
}
@@ -67,7 +95,7 @@ bin_postfork_parent(tsdn_t *tsdn, bin_t *bin, bool has_batch) {
malloc_mutex_postfork_parent(tsdn, &bin->lock);
if (has_batch) {
bin_with_batch_t *batched = (bin_with_batch_t *)bin;
batcher_postfork_parent(tsdn, &batched->remote_frees);
batcher_postfork_parent(TSDN_NULL, &batched->remote_frees);
}
}
@@ -76,6 +104,6 @@ bin_postfork_child(tsdn_t *tsdn, bin_t *bin, bool has_batch) {
malloc_mutex_postfork_child(tsdn, &bin->lock);
if (has_batch) {
bin_with_batch_t *batched = (bin_with_batch_t *)bin;
batcher_postfork_child(tsdn, &batched->remote_frees);
batcher_postfork_child(TSDN_NULL, &batched->remote_frees);
}
}