lihao/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-23 21:23:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use SEC in PAC to reduce lock contention on the ecaches

Browse Source 
Add a small extent cache in front of the PAC ecaches. Allocs and dallocs
that fit are served from per-shard SEC bins without taking the ecache
mutex; overflow falls through to the backing ecaches, including
ecache_pinned for pinned extents.

The feature is gated behind experimental_pac_sec_nshards (default 0,
disabled). To support independent HPA and PAC SEC instances,
sec_alloc/sec_dalloc/sec_fill take an explicit shard argument, with HPA
and PAC using separate TSD shard slots.
This commit is contained in:
Bin Liu
2026-05-19 00:11:15 -07:00
committed by Guangli Dai
parent 2043c6ab58
commit 9c1a484e1d
19 changed files with 986 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
src/sec.c
View File

@@ -7,6 +7,7 @@
static bool
sec_bin_init(sec_bin_t *bin) {
atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->ndalloc_flush, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->nmisses, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->nhits, 0, ATOMIC_RELAXED);
@@ -24,8 +25,15 @@ sec_bin_init(sec_bin_t *bin) {
bool
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
/*
* Invariant: max_alloc == 0 whenever nshards == 0. This lets
* sec_size_supported() collapse to a single comparison.
*/
sec->opts = *opts;
sec->bins = NULL;
sec->npsizes = 0;
if (opts->nshards == 0) {
sec->opts.max_alloc = 0;
return false;
}
assert(opts->max_alloc >= PAGE);
@@ -44,11 +52,15 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
void *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
if (dynalloc == NULL) {
sec->opts.nshards = 0;
sec->opts.max_alloc = 0;
return true;
}
sec->bins = (sec_bin_t *)dynalloc;
for (pszind_t j = 0; j < ntotal_bins; j++) {
if (sec_bin_init(&sec->bins[j])) {
sec->opts.nshards = 0;
sec->opts.max_alloc = 0;
return true;
}
}
@@ -57,18 +69,16 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
return false;
}
static uint8_t
sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
uint8_t
sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp) {
/*
* Eventually, we should implement affinity, tracking source shard using
* the edata_t's newly freed up fields. For now, just randomly
* distribute across all shards.
*
* Callers must ensure sec->opts.nshards > 1.
*/
if (tsdn_null(tsdn)) {
return 0;
}
tsd_t *tsd = tsdn_tsd(tsdn);
uint8_t *idxp = tsd_sec_shardp_get(tsd);
assert(sec->opts.nshards > 1);
if (*idxp == (uint8_t)-1) {
/*
* First use; initialize using the trick from Daniel Lemire's
@@ -135,6 +145,14 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
assert(sz <= bytes_cur && sz > 0);
bytes_cur -= sz;
if (edata_pinned_get(edata)) {
size_t bytes_pinned_cur = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
assert(sz <= bytes_pinned_cur);
bytes_pinned_cur -= sz;
atomic_store_zu(&bin->bytes_pinned_cur,
bytes_pinned_cur, ATOMIC_RELAXED);
}
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
atomic_load_add_store_zu(&bin->nhits, 1);
}
@@ -143,10 +161,10 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
static edata_t *
sec_multishard_trylock_alloc(
tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind, uint8_t shard) {
assert(sec->opts.nshards > 0);
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
uint8_t cur_shard = shard;
sec_bin_t *bin;
for (size_t i = 0; i < sec->opts.nshards; ++i) {
bin = sec_bin_pick(sec, cur_shard, pszind);
@@ -170,7 +188,7 @@ sec_multishard_trylock_alloc(
* declaring a miss. That could recover more remote-shard hits under
* contention, but it also changes the allocation latency policy.
*/
assert(cur_shard == sec_shard_pick(tsdn, sec));
assert(cur_shard == shard);
bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
@@ -184,10 +202,8 @@ sec_multishard_trylock_alloc(
}
edata_t *
sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
if (!sec_size_supported(sec, size)) {
return NULL;
}
sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard) {
assert(sec_size_supported(sec, size));
assert((size & PAGE_MASK) == 0);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
@@ -208,7 +224,7 @@ sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
/* frequent_reuse */ 1);
return edata;
}
return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
return sec_multishard_trylock_alloc(tsdn, sec, size, pszind, shard);
}
static void
@@ -217,16 +233,23 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
malloc_mutex_assert_owner(tsdn, &bin->mtx);
size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
size_t bytes_pinned_cur = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
bytes_cur += size;
edata_t *edata = edata_list_active_first(dalloc_list);
assert(edata != NULL);
edata_list_active_remove(dalloc_list, edata);
JE_USDT(sec_dalloc, 3, sec, bin, edata);
edata_list_active_prepend(&bin->freelist, edata);
if (edata_pinned_get(edata)) {
bytes_pinned_cur += size;
}
/* Single extent can be returned to SEC */
assert(edata_list_active_empty(dalloc_list));
if (bytes_cur <= sec->opts.max_bytes) {
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
atomic_load_add_store_zu(&bin->ndalloc_noflush, 1);
return;
@@ -240,19 +263,25 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
size_t sz = edata_size_get(cur);
assert(sz <= bytes_cur && sz > 0);
bytes_cur -= sz;
if (edata_pinned_get(cur)) {
assert(sz <= bytes_pinned_cur);
bytes_pinned_cur -= sz;
}
edata_list_active_remove(&bin->freelist, cur);
edata_list_active_append(dalloc_list, cur);
}
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
}
static void
sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
pszind_t pszind, edata_list_active_t *dalloc_list) {
pszind_t pszind, edata_list_active_t *dalloc_list, uint8_t shard) {
assert(sec->opts.nshards > 0);
/* Try to dalloc in this threads bin first */
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
uint8_t cur_shard = shard;
for (size_t i = 0; i < sec->opts.nshards; ++i) {
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
@@ -267,7 +296,7 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
}
}
/* No bin had alloc or had the extent */
assert(cur_shard == sec_shard_pick(tsdn, sec));
assert(cur_shard == shard);
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
@@ -275,20 +304,16 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
}
void
sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
if (!sec_is_used(sec)) {
return;
}
sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
uint8_t shard) {
edata_t *edata = edata_list_active_first(dalloc_list);
size_t size = edata_size_get(edata);
if (size > sec->opts.max_alloc) {
return;
}
assert(sec_size_supported(sec, size));
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
/*
* If there's only one shard, skip the trylock optimization and
* If there's only one shard, skip the trylock optimization and
* go straight to the blocking lock.
*/
if (sec->opts.nshards == 1) {
@@ -298,27 +323,49 @@ sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
malloc_mutex_unlock(tsdn, &bin->mtx);
return;
}
sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
sec_multishard_trylock_dalloc(
tsdn, sec, size, pszind, dalloc_list, shard);
}
static void
sec_list_pinned_bytes_get(
edata_list_active_t *list, size_t size, size_t *pinned_bytes) {
*pinned_bytes = 0;
for (edata_t *edata = edata_list_active_first(list); edata != NULL;
edata = edata_list_active_next(list, edata)) {
assert(edata_size_get(edata) == size);
if (edata_pinned_get(edata)) {
*pinned_bytes += size;
}
}
}
void
sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
size_t nallocs) {
size_t nallocs, uint8_t shard) {
assert((size & PAGE_MASK) == 0);
assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
assert(sec_size_supported(sec, size));
assert(nallocs > 0);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
sec_bin_t *bin = sec_bin_pick(sec, shard, pszind);
malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
malloc_mutex_lock(tsdn, &bin->mtx);
size_t new_cached_bytes = nallocs * size;
size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
size_t bytes_pinned_cur = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
size_t new_cached_bytes = nallocs * size;
if (bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
assert(!edata_list_active_empty(result));
size_t new_cached_pinned_bytes;
sec_list_pinned_bytes_get(
result, size, &new_cached_pinned_bytes);
bytes_pinned_cur += new_cached_pinned_bytes;
edata_list_active_concat(&bin->freelist, result);
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur + new_cached_bytes,
ATOMIC_RELAXED);
} else {
@@ -336,7 +383,12 @@ sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
assert(size == edata_size_get(edata));
edata_list_active_append(&bin->freelist, edata);
bytes_cur += size;
if (edata_pinned_get(edata)) {
bytes_pinned_cur += size;
}
}
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
}
malloc_mutex_unlock(tsdn, &bin->mtx);
@@ -352,21 +404,47 @@ sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
edata_list_active_concat(to_flush, &bin->freelist);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
}
static void
sec_bin_bytes_get(const sec_bin_t *bin, size_t *bytes, size_t *bytes_pinned) {
*bytes = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
*bytes_pinned = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
*bytes_pinned = min_zu(*bytes_pinned, *bytes);
}
static void
sec_stats_merge_bin(
const sec_bin_t *bin, pszind_t pszind, sec_pszind_stats_t *stats) {
size_t bytes;
size_t bytes_pinned;
sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
stats->bytes += bytes;
stats->bytes_pinned += bytes_pinned;
size_t size = sz_pind2sz(pszind);
stats->nextents += bytes / size;
stats->nextents_pinned += bytes_pinned / size;
}
void
sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
if (!sec_is_used(sec)) {
return;
}
size_t sum = 0;
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
sum += atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
size_t bytes;
size_t bytes_pinned;
sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
stats->bytes += bytes;
stats->bytes_pinned += bytes_pinned;
stats->total.nmisses +=
atomic_load_zu(&bin->nmisses, ATOMIC_RELAXED);
stats->total.nhits +=
@@ -378,7 +456,20 @@ sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
stats->total.noverfills +=
atomic_load_zu(&bin->noverfills, ATOMIC_RELAXED);
}
stats->bytes += sum;
}
void
sec_stats_merge_pszind(tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
sec_pszind_stats_t *stats) {
if (!sec_is_used(sec) || pszind >= sec->npsizes) {
return;
}
for (size_t shard = 0; shard < sec->opts.nshards; shard++) {
size_t ind = shard * sec->npsizes + pszind;
assert(ind < sec->npsizes * sec->opts.nshards);
sec_stats_merge_bin(&sec->bins[ind], pszind, stats);
}
}
void