lihao/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-22 04:03:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make SEC stats lockless

Browse Source 
Per-bin SEC stats (bytes_cur, nmisses, nhits, ndalloc_flush,
ndalloc_noflush, noverfills) become atomic_zu_t fields directly inside
sec_bin_t.  Writers continue to hold bin->mtx; sec_stats_merge now reads
them lock-free with ATOMIC_RELAXED instead of acquiring bin->mtx.

This removes a lock-rank reversal that would otherwise occur whenever
stats aggregation runs while holding arena->stats.mtx.
This commit is contained in:
Bin Liu
2026-05-06 10:58:50 -07:00
committed by Guangli Dai
parent 9f37c70826
commit 2043c6ab58
2 changed files with 53 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
include/jemalloc/internal/sec.h
View File

@@ -40,15 +40,6 @@ struct sec_stats_s {
sec_bin_stats_t total; sec_bin_stats_t total;
}; };
static inline void
sec_bin_stats_init(sec_bin_stats_t *stats) {
stats->ndalloc_flush = 0;
stats->nmisses = 0;
stats->nhits = 0;
stats->ndalloc_noflush = 0;
stats->noverfills = 0;
}
static inline void static inline void
sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) { sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
dst->nmisses += src->nmisses; dst->nmisses += src->nmisses;
@@ -68,16 +59,20 @@ sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
typedef struct sec_bin_s sec_bin_t; typedef struct sec_bin_s sec_bin_t;
struct sec_bin_s { struct sec_bin_s {
/* /*
* Protects the data members of the bin. * Protects the freelist and synchronizes counter updates.
*/ */
malloc_mutex_t mtx; malloc_mutex_t mtx;
/* /*
* Number of bytes in this particular bin. * Number of bytes in this particular bin.
*/ */
size_t bytes_cur; atomic_zu_t bytes_cur;
edata_list_active_t freelist; edata_list_active_t freelist;
sec_bin_stats_t stats; atomic_zu_t nmisses;
atomic_zu_t nhits;
atomic_zu_t ndalloc_flush;
atomic_zu_t ndalloc_noflush;
atomic_zu_t noverfills;
}; };
typedef struct sec_s sec_t; typedef struct sec_s sec_t;
@@ -88,7 +83,7 @@ struct sec_s {
}; };
static inline bool static inline bool
sec_is_used(sec_t *sec) { sec_is_used(const sec_t *sec) {
return sec->opts.nshards != 0; return sec->opts.nshards != 0;
} }
@@ -142,7 +137,7 @@ void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
* lets them fit easily into the pa_shard stats framework (which also has this * lets them fit easily into the pa_shard stats framework (which also has this
* split), which simplifies the stats management. * split), which simplifies the stats management.
*/ */
void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats); void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats);
void sec_mutex_stats_read( void sec_mutex_stats_read(
tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data); tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data);

69
src/sec.c
View File

@@ -6,8 +6,12 @@
static bool static bool
sec_bin_init(sec_bin_t *bin) { sec_bin_init(sec_bin_t *bin) {
bin->bytes_cur = 0; atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
sec_bin_stats_init(&bin->stats); atomic_store_zu(&bin->ndalloc_flush, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->nmisses, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->nhits, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->ndalloc_noflush, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->noverfills, 0, ATOMIC_RELAXED);
edata_list_active_init(&bin->freelist); edata_list_active_init(&bin->freelist);
bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN, bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN,
malloc_mutex_rank_exclusive); malloc_mutex_rank_exclusive);
@@ -128,9 +132,11 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
assert(!edata_list_active_empty(&bin->freelist)); assert(!edata_list_active_empty(&bin->freelist));
edata_list_active_remove(&bin->freelist, edata); edata_list_active_remove(&bin->freelist, edata);
size_t sz = edata_size_get(edata); size_t sz = edata_size_get(edata);
assert(sz <= bin->bytes_cur && sz > 0); size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
bin->bytes_cur -= sz; assert(sz <= bytes_cur && sz > 0);
bin->stats.nhits++; bytes_cur -= sz;
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
atomic_load_add_store_zu(&bin->nhits, 1);
} }
return edata; return edata;
} }
@@ -170,7 +176,7 @@ sec_multishard_trylock_alloc(
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size); edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
if (edata == NULL) { if (edata == NULL) {
/* Only now we know it is a miss. */ /* Only now we know it is a miss. */
bin->stats.nmisses++; atomic_load_add_store_zu(&bin->nmisses, 1);
} }
malloc_mutex_unlock(tsdn, &bin->mtx); malloc_mutex_unlock(tsdn, &bin->mtx);
JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1); JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1);
@@ -195,7 +201,7 @@ sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
malloc_mutex_lock(tsdn, &bin->mtx); malloc_mutex_lock(tsdn, &bin->mtx);
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size); edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
if (edata == NULL) { if (edata == NULL) {
bin->stats.nmisses++; atomic_load_add_store_zu(&bin->nmisses, 1);
} }
malloc_mutex_unlock(tsdn, &bin->mtx); malloc_mutex_unlock(tsdn, &bin->mtx);
JE_USDT(sec_alloc, 5, sec, bin, edata, size, JE_USDT(sec_alloc, 5, sec, bin, edata, size,
@@ -210,7 +216,8 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
edata_list_active_t *dalloc_list) { edata_list_active_t *dalloc_list) {
malloc_mutex_assert_owner(tsdn, &bin->mtx); malloc_mutex_assert_owner(tsdn, &bin->mtx);
bin->bytes_cur += size; size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
bytes_cur += size;
edata_t *edata = edata_list_active_first(dalloc_list); edata_t *edata = edata_list_active_first(dalloc_list);
assert(edata != NULL); assert(edata != NULL);
edata_list_active_remove(dalloc_list, edata); edata_list_active_remove(dalloc_list, edata);
@@ -219,22 +226,24 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
/* Single extent can be returned to SEC */ /* Single extent can be returned to SEC */
assert(edata_list_active_empty(dalloc_list)); assert(edata_list_active_empty(dalloc_list));
if (bin->bytes_cur <= sec->opts.max_bytes) { if (bytes_cur <= sec->opts.max_bytes) {
bin->stats.ndalloc_noflush++; atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
atomic_load_add_store_zu(&bin->ndalloc_noflush, 1);
return; return;
} }
bin->stats.ndalloc_flush++; atomic_load_add_store_zu(&bin->ndalloc_flush, 1);
/* we want to flush 1/4 of max_bytes */ /* we want to flush 1/4 of max_bytes */
size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2); size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2);
while (bin->bytes_cur > bytes_target while (bytes_cur > bytes_target
&& !edata_list_active_empty(&bin->freelist)) { && !edata_list_active_empty(&bin->freelist)) {
edata_t *cur = edata_list_active_last(&bin->freelist); edata_t *cur = edata_list_active_last(&bin->freelist);
size_t sz = edata_size_get(cur); size_t sz = edata_size_get(cur);
assert(sz <= bin->bytes_cur && sz > 0); assert(sz <= bytes_cur && sz > 0);
bin->bytes_cur -= sz; bytes_cur -= sz;
edata_list_active_remove(&bin->freelist, cur); edata_list_active_remove(&bin->freelist, cur);
edata_list_active_append(dalloc_list, cur); edata_list_active_append(dalloc_list, cur);
} }
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
} }
static void static void
@@ -306,17 +315,19 @@ sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
malloc_mutex_assert_not_owner(tsdn, &bin->mtx); malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
malloc_mutex_lock(tsdn, &bin->mtx); malloc_mutex_lock(tsdn, &bin->mtx);
size_t new_cached_bytes = nallocs * size; size_t new_cached_bytes = nallocs * size;
if (bin->bytes_cur + new_cached_bytes <= sec->opts.max_bytes) { size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
if (bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
assert(!edata_list_active_empty(result)); assert(!edata_list_active_empty(result));
edata_list_active_concat(&bin->freelist, result); edata_list_active_concat(&bin->freelist, result);
bin->bytes_cur += new_cached_bytes; atomic_store_zu(&bin->bytes_cur, bytes_cur + new_cached_bytes,
ATOMIC_RELAXED);
} else { } else {
/* /*
* Unlikely case of many threads filling at the same time and * Unlikely case of many threads filling at the same time and
* going above max. * going above max.
*/ */
bin->stats.noverfills++; atomic_load_add_store_zu(&bin->noverfills, 1);
while (bin->bytes_cur + size <= sec->opts.max_bytes) { while (bytes_cur + size <= sec->opts.max_bytes) {
edata_t *edata = edata_list_active_first(result); edata_t *edata = edata_list_active_first(result);
if (edata == NULL) { if (edata == NULL) {
break; break;
@@ -324,8 +335,9 @@ sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
edata_list_active_remove(result, edata); edata_list_active_remove(result, edata);
assert(size == edata_size_get(edata)); assert(size == edata_size_get(edata));
edata_list_active_append(&bin->freelist, edata); edata_list_active_append(&bin->freelist, edata);
bin->bytes_cur += size; bytes_cur += size;
} }
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
} }
malloc_mutex_unlock(tsdn, &bin->mtx); malloc_mutex_unlock(tsdn, &bin->mtx);
} }
@@ -339,14 +351,14 @@ sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
for (pszind_t i = 0; i < ntotal_bins; i++) { for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i]; sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx); malloc_mutex_lock(tsdn, &bin->mtx);
bin->bytes_cur = 0; atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
edata_list_active_concat(to_flush, &bin->freelist); edata_list_active_concat(to_flush, &bin->freelist);
malloc_mutex_unlock(tsdn, &bin->mtx); malloc_mutex_unlock(tsdn, &bin->mtx);
} }
} }
void void
sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) { sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
if (!sec_is_used(sec)) { if (!sec_is_used(sec)) {
return; return;
} }
@@ -354,10 +366,17 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
size_t ntotal_bins = sec->opts.nshards * sec->npsizes; size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) { for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i]; sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx); sum += atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
sum += bin->bytes_cur; stats->total.nmisses +=
sec_bin_stats_accum(&stats->total, &bin->stats); atomic_load_zu(&bin->nmisses, ATOMIC_RELAXED);
malloc_mutex_unlock(tsdn, &bin->mtx); stats->total.nhits +=
atomic_load_zu(&bin->nhits, ATOMIC_RELAXED);
stats->total.ndalloc_flush +=
atomic_load_zu(&bin->ndalloc_flush, ATOMIC_RELAXED);
stats->total.ndalloc_noflush +=
atomic_load_zu(&bin->ndalloc_noflush, ATOMIC_RELAXED);
stats->total.noverfills +=
atomic_load_zu(&bin->noverfills, ATOMIC_RELAXED);
} }
stats->bytes += sum; stats->bytes += sum;
} }