lihao/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-23 13:13:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use SEC in PAC to reduce lock contention on the ecaches

Browse Source 
Add a small extent cache in front of the PAC ecaches. Allocs and dallocs
that fit are served from per-shard SEC bins without taking the ecache
mutex; overflow falls through to the backing ecaches, including
ecache_pinned for pinned extents.

The feature is gated behind experimental_pac_sec_nshards (default 0,
disabled). To support independent HPA and PAC SEC instances,
sec_alloc/sec_dalloc/sec_fill take an explicit shard argument, with HPA
and PAC using separate TSD shard slots.
This commit is contained in:
Bin Liu
2026-05-19 00:11:15 -07:00
committed by Guangli Dai
parent 2043c6ab58
commit 9c1a484e1d
19 changed files with 986 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/conf.c
View File

@@ -950,6 +950,17 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
"hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_pac_sec_opts.nshards,
"experimental_pac_sec_nshards", 0, 0,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_alloc,
"experimental_pac_sec_max_alloc", PAGE,
USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN,
CONF_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_bytes,
"experimental_pac_sec_max_bytes",
SEC_OPTS_MAX_BYTES_DEFAULT, 0,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
if (CONF_MATCH("slab_sizes")) {
if (CONF_MATCH_VALUE("default")) {

51
src/ctl.c
View File

@@ -147,6 +147,9 @@ CTL_PROTO(opt_hpa_dirty_mult)
CTL_PROTO(opt_hpa_sec_nshards)
CTL_PROTO(opt_hpa_sec_max_alloc)
CTL_PROTO(opt_hpa_sec_max_bytes)
CTL_PROTO(opt_experimental_pac_sec_nshards)
CTL_PROTO(opt_experimental_pac_sec_max_alloc)
CTL_PROTO(opt_experimental_pac_sec_max_bytes)
CTL_PROTO(opt_huge_arena_pac_thp)
CTL_PROTO(opt_metadata_thp)
CTL_PROTO(opt_retain)
@@ -382,6 +385,11 @@ CTL_PROTO(stats_arenas_i_hpa_sec_misses)
CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_flush)
CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_noflush)
CTL_PROTO(stats_arenas_i_hpa_sec_overfills)
CTL_PROTO(stats_arenas_i_pac_sec_bytes)
CTL_PROTO(stats_arenas_i_pac_sec_hits)
CTL_PROTO(stats_arenas_i_pac_sec_misses)
CTL_PROTO(stats_arenas_i_pac_sec_dalloc_flush)
CTL_PROTO(stats_arenas_i_pac_sec_dalloc_noflush)
INDEX_PROTO(stats_arenas_i)
CTL_PROTO(stats_allocated)
CTL_PROTO(stats_active)
@@ -520,6 +528,12 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
{NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
{NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
{NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
{NAME("experimental_pac_sec_nshards"),
CTL(opt_experimental_pac_sec_nshards)},
{NAME("experimental_pac_sec_max_alloc"),
CTL(opt_experimental_pac_sec_max_alloc)},
{NAME("experimental_pac_sec_max_bytes"),
CTL(opt_experimental_pac_sec_max_bytes)},
{NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
{NAME("metadata_thp"), CTL(opt_metadata_thp)},
{NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)},
@@ -883,6 +897,12 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
CTL(stats_arenas_i_hpa_sec_dalloc_noflush)},
{NAME("hpa_sec_dalloc_flush"), CTL(stats_arenas_i_hpa_sec_dalloc_flush)},
{NAME("hpa_sec_overfills"), CTL(stats_arenas_i_hpa_sec_overfills)},
{NAME("pac_sec_bytes"), CTL(stats_arenas_i_pac_sec_bytes)},
{NAME("pac_sec_hits"), CTL(stats_arenas_i_pac_sec_hits)},
{NAME("pac_sec_misses"), CTL(stats_arenas_i_pac_sec_misses)},
{NAME("pac_sec_dalloc_noflush"),
CTL(stats_arenas_i_pac_sec_dalloc_noflush)},
{NAME("pac_sec_dalloc_flush"), CTL(stats_arenas_i_pac_sec_dalloc_flush)},
{NAME("small"), CHILD(named, stats_arenas_i_small)},
{NAME("large"), CHILD(named, stats_arenas_i_large)},
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
@@ -1269,6 +1289,10 @@ ctl_arena_stats_sdmerge(
&sdstats->astats.pa_shard_stats.pac_stats.abandoned_vm,
&astats->astats.pa_shard_stats.pac_stats.abandoned_vm);
sec_stats_accum(
&sdstats->astats.pa_shard_stats.pac_stats.pac_sec_stats,
&astats->astats.pa_shard_stats.pac_stats.pac_sec_stats);
sdstats->astats.tcache_bytes += astats->astats.tcache_bytes;
sdstats->astats.tcache_stashed_bytes +=
astats->astats.tcache_stashed_bytes;
@@ -2235,6 +2259,12 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
CTL_RO_NL_GEN(opt_experimental_pac_sec_nshards,
opt_pac_sec_opts.nshards, size_t)
CTL_RO_NL_GEN(opt_experimental_pac_sec_max_alloc,
opt_pac_sec_opts.max_alloc, size_t)
CTL_RO_NL_GEN(opt_experimental_pac_sec_max_bytes,
opt_pac_sec_opts.max_bytes, size_t)
CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
CTL_RO_NL_GEN(
opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
@@ -3829,6 +3859,27 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_noflush,
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_overfills,
arenas_i(mib[2])->astats->hpastats.secstats.total.noverfills, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_bytes,
arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
.pac_sec_stats.bytes,
size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_hits,
arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
.pac_sec_stats.total.nhits,
size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_misses,
arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
.pac_sec_stats.total.nmisses,
size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_dalloc_flush,
arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
.pac_sec_stats.total.ndalloc_flush,
size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_dalloc_noflush,
arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
.pac_sec_stats.total.ndalloc_noflush,
size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
arenas_i(mib[2])->astats->allocated_small, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc,

38
src/hpa.c
View File

@@ -14,6 +14,18 @@ const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
bool opt_experimental_hpa_start_huge_if_thp_always = true;
bool opt_experimental_hpa_enforce_hugify = false;
static inline uint8_t
hpa_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
if (sec->opts.nshards <= 1) {
return 0;
}
if (tsdn_null(tsdn)) {
return 0;
}
tsd_t *tsd = tsdn_tsd(tsdn);
return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd));
}
JET_EXTERN bool
hpa_hugepage_size_exceeds_limit(void) {
return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE;
@@ -945,9 +957,13 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
&& (size > shard->opts.slab_max_alloc)) {
return NULL;
}
edata_t *edata = sec_alloc(tsdn, &shard->sec, size);
if (edata != NULL) {
return edata;
edata_t *edata = NULL;
if (sec_size_supported(&shard->sec, size)) {
edata = sec_alloc(tsdn, &shard->sec, size,
hpa_sec_shard_pick(tsdn, &shard->sec));
if (edata != NULL) {
return edata;
}
}
edata_list_active_t results;
edata_list_active_init(&results);
@@ -966,7 +982,8 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
}
if (nsuccess > 0) {
assert(sec_size_supported(&shard->sec, size));
sec_fill(tsdn, &shard->sec, size, &results, nsuccess);
sec_fill(tsdn, &shard->sec, size, &results, nsuccess,
hpa_sec_shard_pick(tsdn, &shard->sec));
/* Unlikely rollback in case of overfill */
if (!edata_list_active_empty(&results)) {
hpa_dalloc_batch(
@@ -1059,11 +1076,14 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
edata_list_active_init(&dalloc_list);
edata_list_active_append(&dalloc_list, edata);
sec_dalloc(tsdn, &shard->sec, &dalloc_list);
if (edata_list_active_empty(&dalloc_list)) {
/* sec consumed the pointer */
*deferred_work_generated = false;
return;
if (sec_size_supported(&shard->sec, edata_size_get(edata))) {
sec_dalloc(tsdn, &shard->sec, &dalloc_list,
hpa_sec_shard_pick(tsdn, &shard->sec));
if (edata_list_active_empty(&dalloc_list)) {
/* sec consumed the pointer */
*deferred_work_generated = false;
return;
}
}
/* We may have more than one pointer to flush now */
hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated);

3
src/jemalloc.c
View File

@@ -198,6 +198,9 @@ size_t opt_calloc_madvise_threshold = CALLOC_MADVISE_THRESHOLD_DEFAULT;
bool opt_hpa = false;
hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT;
sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT;
sec_opts_t opt_pac_sec_opts = {0,
(32 * 1024) > (PAGE * 2) ? (32 * 1024) : (PAGE * 2),
SEC_OPTS_MAX_BYTES_DEFAULT};
/* False should be the common case. Set to true to trigger initialization. */
bool malloc_slow = true;

1
src/pa.c
View File

@@ -94,6 +94,7 @@ pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
void
pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
pac_sec_flush(tsdn, &shard->pac);
if (shard->ever_used_hpa) {
hpa_shard_flush(tsdn, &shard->hpa);
}

52
src/pa_extra.c
View File

@@ -17,6 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
void
pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
sec_prefork2(tsdn, &shard->pac.sec);
if (shard->ever_used_hpa) {
hpa_shard_prefork2(tsdn, &shard->hpa);
}
@@ -54,6 +55,7 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
ecache_postfork_parent(tsdn, &shard->pac.ecache_pinned);
malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
sec_postfork_parent(tsdn, &shard->pac.sec);
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
@@ -69,6 +71,7 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
ecache_postfork_child(tsdn, &shard->pac.ecache_pinned);
malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
sec_postfork_child(tsdn, &shard->pac.sec);
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
@@ -82,7 +85,17 @@ pa_shard_nactive(const pa_shard_t *shard) {
}
static size_t
pa_shard_ndirty(const pa_shard_t *shard) {
pac_sec_pinned_bytes_get(const sec_stats_t *stats) {
return min_zu(stats->bytes_pinned, stats->bytes);
}
static size_t
pac_sec_dirty_npages_get(const sec_stats_t *stats) {
return (stats->bytes - pac_sec_pinned_bytes_get(stats)) >> LG_PAGE;
}
static size_t
pa_shard_ndirty_no_pac_sec(const pa_shard_t *shard) {
size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty);
if (shard->ever_used_hpa) {
ndirty += psset_ndirty(&shard->hpa.psset);
@@ -90,6 +103,14 @@ pa_shard_ndirty(const pa_shard_t *shard) {
return ndirty;
}
static size_t
pa_shard_ndirty(const pa_shard_t *shard) {
sec_stats_t pac_sec_stats = {0};
sec_stats_merge(TSDN_NULL, &shard->pac.sec, &pac_sec_stats);
return pa_shard_ndirty_no_pac_sec(shard)
+ pac_sec_dirty_npages_get(&pac_sec_stats);
}
static size_t
pa_shard_nmuzzy(const pa_shard_t *shard) {
return ecache_npages_get(&shard->pac.ecache_muzzy);
@@ -109,17 +130,29 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
hpa_shard_stats_t *hpa_stats_out, size_t *resident) {
cassert(config_stats);
sec_stats_t pac_sec_stats = {0};
sec_stats_merge(tsdn, &shard->pac.sec, &pac_sec_stats);
sec_stats_accum(&pa_shard_stats_out->pac_stats.pac_sec_stats,
&pac_sec_stats);
size_t pac_sec_pinned_bytes = pac_sec_pinned_bytes_get(&pac_sec_stats);
size_t pac_sec_pinned_npages = pac_sec_pinned_bytes >> LG_PAGE;
size_t pac_sec_dirty_npages =
pac_sec_dirty_npages_get(&pac_sec_stats);
pa_shard_stats_out->pac_stats.retained +=
ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
pa_shard_stats_out->pac_stats.pinned +=
ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE;
(ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE)
+ pac_sec_pinned_bytes;
pa_shard_stats_out->edata_avail += atomic_load_zu(
&shard->edata_cache.count, ATOMIC_RELAXED);
size_t resident_pgs = 0;
resident_pgs += pa_shard_nactive(shard);
resident_pgs += pa_shard_ndirty(shard);
resident_pgs += pa_shard_ndirty_no_pac_sec(shard);
resident_pgs += pac_sec_dirty_npages;
resident_pgs += ecache_npages_get(&shard->pac.ecache_pinned);
resident_pgs += pac_sec_pinned_npages;
*resident += (resident_pgs << LG_PAGE);
/* Dirty decay stats */
@@ -167,6 +200,16 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
pinned_bytes = ecache_nbytes_get(
&shard->pac.ecache_pinned, i);
sec_pszind_stats_t pac_sec_pszind_stats = {0};
sec_stats_merge_pszind(
tsdn, &shard->pac.sec, i, &pac_sec_pszind_stats);
dirty += pac_sec_pszind_stats.nextents
- pac_sec_pszind_stats.nextents_pinned;
dirty_bytes += pac_sec_pszind_stats.bytes
- pac_sec_pszind_stats.bytes_pinned;
pinned += pac_sec_pszind_stats.nextents_pinned;
pinned_bytes += pac_sec_pszind_stats.bytes_pinned;
estats_out[i].ndirty = dirty;
estats_out[i].nmuzzy = muzzy;
estats_out[i].nretained = retained;
@@ -208,6 +251,9 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);
sec_mutex_stats_read(tsdn, &shard->pac.sec,
&mutex_prof_data[arena_prof_mutex_pac_sec]);
if (shard->ever_used_hpa) {
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->hpa.mtx, arena_prof_mutex_hpa_shard);

96
src/pac.c
View File

@@ -7,6 +7,18 @@
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/witness.h"
static inline uint8_t
pac_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
if (sec->opts.nshards <= 1) {
return 0;
}
if (tsdn_null(tsdn)) {
return 0;
}
tsd_t *tsd = tsdn_tsd(tsdn);
return sec_shard_pick(tsd, sec, tsd_pac_sec_shardp_get(tsd));
}
static inline void
pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay,
pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) {
@@ -95,6 +107,16 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
pac->stats_mtx = stats_mtx;
atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
if (sec_init(tsdn, &pac->sec, base, &opt_pac_sec_opts)) {
/* sec_init already zeroed nshards and max_alloc. */
}
if (!sec_is_used(&pac->sec) || dirty_decay_ms == 0) {
atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED);
} else {
atomic_store_zu(&pac->sec_max_alloc,
pac->sec.opts.max_alloc, ATOMIC_RELAXED);
}
return false;
}
@@ -136,6 +158,24 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
edata_t *edata = NULL;
if (!guarded && !zero && alignment <= PAGE
&& size <= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) {
/*
* sec_max_alloc mirrors sec.opts.max_alloc when SEC is
* enabled, 0 when dirty decay is disabled.
*/
edata = sec_alloc(tsdn, &pac->sec, size,
pac_sec_shard_pick(tsdn, &pac->sec));
if (edata != NULL) {
return edata;
}
/*
* Unlike HPA, PAC has no batch allocation primitive; we
* intentionally do not refill SEC on a miss. Extents enter the
* cache only through the dalloc path.
*/
}
/*
* Guarded allocations need surrounding guard pages, which the pinned
* pool does not maintain; skip ecache_pinned in that case.
@@ -398,6 +438,37 @@ pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
san_unguard_pages_two_sided(
tsdn, ehooks, edata, pac->emap);
}
} else if (edata_size_get(edata)
<= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) {
/*
* A dalloc can race with disabling SEC and cache an extent after
* the flush. Avoid a hot-path gate lock; such extents remain
* stats-tracked and are flushed by reset/destroy or a later
* disable.
*/
edata_list_active_t dalloc_list;
edata_list_active_init(&dalloc_list);
edata_list_active_append(&dalloc_list, edata);
sec_dalloc(tsdn, &pac->sec, &dalloc_list,
pac_sec_shard_pick(tsdn, &pac->sec));
if (edata_list_active_empty(&dalloc_list)) {
*deferred_work_generated = false;
return;
}
/* Flush overflow extents to their backing ecaches. */
bool any_deferred_work = false;
edata_t *flush_edata;
while ((flush_edata =
edata_list_active_first(&dalloc_list)) != NULL) {
edata_list_active_remove(&dalloc_list,
flush_edata);
if (!edata_pinned_get(flush_edata)) {
any_deferred_work = true;
}
pac_ecache_dalloc(tsdn, pac, ehooks, flush_edata);
}
*deferred_work_generated = any_deferred_work;
return;
}
bool pinned = edata_pinned_get(edata);
@@ -720,6 +791,13 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
return true;
}
bool update_pac_sec = (state == extent_state_dirty)
&& sec_is_used(&pac->sec);
if (update_pac_sec && decay_ms == 0) {
atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED);
pac_sec_flush(tsdn, pac);
}
malloc_mutex_lock(tsdn, &decay->mtx);
/*
* Restart decay backlog from scratch, which may cause many dirty pages
@@ -735,6 +813,11 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness);
malloc_mutex_unlock(tsdn, &decay->mtx);
if (update_pac_sec && decay_ms != 0) {
atomic_store_zu(&pac->sec_max_alloc,
pac->sec.opts.max_alloc, ATOMIC_RELAXED);
}
return false;
}
@@ -809,3 +892,16 @@ pac_destroy(tsdn_t *tsdn, pac_t *pac) {
extent_destroy_wrapper(tsdn, pac, ehooks, edata);
}
}
void
pac_sec_flush(tsdn_t *tsdn, pac_t *pac) {
ehooks_t *ehooks = pac_ehooks_get(pac);
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
sec_flush(tsdn, &pac->sec, &to_flush);
edata_t *edata;
while ((edata = edata_list_active_first(&to_flush)) != NULL) {
edata_list_active_remove(&to_flush, edata);
pac_ecache_dalloc(tsdn, pac, ehooks, edata);
}
}

159
src/sec.c
View File

@@ -7,6 +7,7 @@
static bool
sec_bin_init(sec_bin_t *bin) {
atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->ndalloc_flush, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->nmisses, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->nhits, 0, ATOMIC_RELAXED);
@@ -24,8 +25,15 @@ sec_bin_init(sec_bin_t *bin) {
bool
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
/*
* Invariant: max_alloc == 0 whenever nshards == 0. This lets
* sec_size_supported() collapse to a single comparison.
*/
sec->opts = *opts;
sec->bins = NULL;
sec->npsizes = 0;
if (opts->nshards == 0) {
sec->opts.max_alloc = 0;
return false;
}
assert(opts->max_alloc >= PAGE);
@@ -44,11 +52,15 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
void *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
if (dynalloc == NULL) {
sec->opts.nshards = 0;
sec->opts.max_alloc = 0;
return true;
}
sec->bins = (sec_bin_t *)dynalloc;
for (pszind_t j = 0; j < ntotal_bins; j++) {
if (sec_bin_init(&sec->bins[j])) {
sec->opts.nshards = 0;
sec->opts.max_alloc = 0;
return true;
}
}
@@ -57,18 +69,16 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
return false;
}
static uint8_t
sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
uint8_t
sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp) {
/*
* Eventually, we should implement affinity, tracking source shard using
* the edata_t's newly freed up fields. For now, just randomly
* distribute across all shards.
*
* Callers must ensure sec->opts.nshards > 1.
*/
if (tsdn_null(tsdn)) {
return 0;
}
tsd_t *tsd = tsdn_tsd(tsdn);
uint8_t *idxp = tsd_sec_shardp_get(tsd);
assert(sec->opts.nshards > 1);
if (*idxp == (uint8_t)-1) {
/*
* First use; initialize using the trick from Daniel Lemire's
@@ -135,6 +145,14 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
assert(sz <= bytes_cur && sz > 0);
bytes_cur -= sz;
if (edata_pinned_get(edata)) {
size_t bytes_pinned_cur = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
assert(sz <= bytes_pinned_cur);
bytes_pinned_cur -= sz;
atomic_store_zu(&bin->bytes_pinned_cur,
bytes_pinned_cur, ATOMIC_RELAXED);
}
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
atomic_load_add_store_zu(&bin->nhits, 1);
}
@@ -143,10 +161,10 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
static edata_t *
sec_multishard_trylock_alloc(
tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind, uint8_t shard) {
assert(sec->opts.nshards > 0);
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
uint8_t cur_shard = shard;
sec_bin_t *bin;
for (size_t i = 0; i < sec->opts.nshards; ++i) {
bin = sec_bin_pick(sec, cur_shard, pszind);
@@ -170,7 +188,7 @@ sec_multishard_trylock_alloc(
* declaring a miss. That could recover more remote-shard hits under
* contention, but it also changes the allocation latency policy.
*/
assert(cur_shard == sec_shard_pick(tsdn, sec));
assert(cur_shard == shard);
bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
@@ -184,10 +202,8 @@ sec_multishard_trylock_alloc(
}
edata_t *
sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
if (!sec_size_supported(sec, size)) {
return NULL;
}
sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard) {
assert(sec_size_supported(sec, size));
assert((size & PAGE_MASK) == 0);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
@@ -208,7 +224,7 @@ sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
/* frequent_reuse */ 1);
return edata;
}
return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
return sec_multishard_trylock_alloc(tsdn, sec, size, pszind, shard);
}
static void
@@ -217,16 +233,23 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
malloc_mutex_assert_owner(tsdn, &bin->mtx);
size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
size_t bytes_pinned_cur = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
bytes_cur += size;
edata_t *edata = edata_list_active_first(dalloc_list);
assert(edata != NULL);
edata_list_active_remove(dalloc_list, edata);
JE_USDT(sec_dalloc, 3, sec, bin, edata);
edata_list_active_prepend(&bin->freelist, edata);
if (edata_pinned_get(edata)) {
bytes_pinned_cur += size;
}
/* Single extent can be returned to SEC */
assert(edata_list_active_empty(dalloc_list));
if (bytes_cur <= sec->opts.max_bytes) {
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
atomic_load_add_store_zu(&bin->ndalloc_noflush, 1);
return;
@@ -240,19 +263,25 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
size_t sz = edata_size_get(cur);
assert(sz <= bytes_cur && sz > 0);
bytes_cur -= sz;
if (edata_pinned_get(cur)) {
assert(sz <= bytes_pinned_cur);
bytes_pinned_cur -= sz;
}
edata_list_active_remove(&bin->freelist, cur);
edata_list_active_append(dalloc_list, cur);
}
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
}
static void
sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
pszind_t pszind, edata_list_active_t *dalloc_list) {
pszind_t pszind, edata_list_active_t *dalloc_list, uint8_t shard) {
assert(sec->opts.nshards > 0);
/* Try to dalloc in this threads bin first */
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
uint8_t cur_shard = shard;
for (size_t i = 0; i < sec->opts.nshards; ++i) {
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
@@ -267,7 +296,7 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
}
}
/* No bin had alloc or had the extent */
assert(cur_shard == sec_shard_pick(tsdn, sec));
assert(cur_shard == shard);
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
@@ -275,20 +304,16 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
}
void
sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
if (!sec_is_used(sec)) {
return;
}
sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
uint8_t shard) {
edata_t *edata = edata_list_active_first(dalloc_list);
size_t size = edata_size_get(edata);
if (size > sec->opts.max_alloc) {
return;
}
assert(sec_size_supported(sec, size));
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
/*
* If there's only one shard, skip the trylock optimization and
* If there's only one shard, skip the trylock optimization and
* go straight to the blocking lock.
*/
if (sec->opts.nshards == 1) {
@@ -298,27 +323,49 @@ sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
malloc_mutex_unlock(tsdn, &bin->mtx);
return;
}
sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
sec_multishard_trylock_dalloc(
tsdn, sec, size, pszind, dalloc_list, shard);
}
static void
sec_list_pinned_bytes_get(
edata_list_active_t *list, size_t size, size_t *pinned_bytes) {
*pinned_bytes = 0;
for (edata_t *edata = edata_list_active_first(list); edata != NULL;
edata = edata_list_active_next(list, edata)) {
assert(edata_size_get(edata) == size);
if (edata_pinned_get(edata)) {
*pinned_bytes += size;
}
}
}
void
sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
size_t nallocs) {
size_t nallocs, uint8_t shard) {
assert((size & PAGE_MASK) == 0);
assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
assert(sec_size_supported(sec, size));
assert(nallocs > 0);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
sec_bin_t *bin = sec_bin_pick(sec, shard, pszind);
malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
malloc_mutex_lock(tsdn, &bin->mtx);
size_t new_cached_bytes = nallocs * size;
size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
size_t bytes_pinned_cur = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
size_t new_cached_bytes = nallocs * size;
if (bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
assert(!edata_list_active_empty(result));
size_t new_cached_pinned_bytes;
sec_list_pinned_bytes_get(
result, size, &new_cached_pinned_bytes);
bytes_pinned_cur += new_cached_pinned_bytes;
edata_list_active_concat(&bin->freelist, result);
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur + new_cached_bytes,
ATOMIC_RELAXED);
} else {
@@ -336,7 +383,12 @@ sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
assert(size == edata_size_get(edata));
edata_list_active_append(&bin->freelist, edata);
bytes_cur += size;
if (edata_pinned_get(edata)) {
bytes_pinned_cur += size;
}
}
atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
}
malloc_mutex_unlock(tsdn, &bin->mtx);
@@ -352,21 +404,47 @@ sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
edata_list_active_concat(to_flush, &bin->freelist);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
}
static void
sec_bin_bytes_get(const sec_bin_t *bin, size_t *bytes, size_t *bytes_pinned) {
*bytes = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
*bytes_pinned = atomic_load_zu(
&bin->bytes_pinned_cur, ATOMIC_RELAXED);
*bytes_pinned = min_zu(*bytes_pinned, *bytes);
}
static void
sec_stats_merge_bin(
const sec_bin_t *bin, pszind_t pszind, sec_pszind_stats_t *stats) {
size_t bytes;
size_t bytes_pinned;
sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
stats->bytes += bytes;
stats->bytes_pinned += bytes_pinned;
size_t size = sz_pind2sz(pszind);
stats->nextents += bytes / size;
stats->nextents_pinned += bytes_pinned / size;
}
void
sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
if (!sec_is_used(sec)) {
return;
}
size_t sum = 0;
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
sum += atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
size_t bytes;
size_t bytes_pinned;
sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
stats->bytes += bytes;
stats->bytes_pinned += bytes_pinned;
stats->total.nmisses +=
atomic_load_zu(&bin->nmisses, ATOMIC_RELAXED);
stats->total.nhits +=
@@ -378,7 +456,20 @@ sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
stats->total.noverfills +=
atomic_load_zu(&bin->noverfills, ATOMIC_RELAXED);
}
stats->bytes += sum;
}
void
sec_stats_merge_pszind(tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
sec_pszind_stats_t *stats) {
if (!sec_is_used(sec) || pszind >= sec->npsizes) {
return;
}
for (size_t shard = 0; shard < sec->opts.nshards; shard++) {
size_t ind = shard * sec->npsizes + pszind;
assert(ind < sec->npsizes * sec->opts.nshards);
sec_stats_merge_bin(&sec->bins[ind], pszind, stats);
}
}
void

38
src/stats.c
View File

@@ -838,6 +838,37 @@ stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) {
&sec_overfills);
}
static void
stats_arena_pac_sec_print(emitter_t *emitter, unsigned i) {
size_t sec_bytes;
size_t sec_hits;
size_t sec_misses;
size_t sec_dalloc_flush;
size_t sec_dalloc_noflush;
CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec_bytes, size_t);
emitter_kv(emitter, "pac_sec_bytes",
"Bytes in PAC small extent cache",
emitter_type_size, &sec_bytes);
CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec_hits, size_t);
emitter_kv(emitter, "pac_sec_hits",
"Total hits in PAC small extent cache",
emitter_type_size, &sec_hits);
CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec_misses, size_t);
emitter_kv(emitter, "pac_sec_misses",
"Total misses in PAC small extent cache",
emitter_type_size, &sec_misses);
CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i,
&sec_dalloc_noflush, size_t);
emitter_kv(emitter, "pac_sec_dalloc_noflush",
"Dalloc calls without flush in PAC small extent cache",
emitter_type_size, &sec_dalloc_noflush);
CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i, &sec_dalloc_flush,
size_t);
emitter_kv(emitter, "pac_sec_dalloc_flush",
"Dalloc calls with flush in PAC small extent cache",
emitter_type_size, &sec_dalloc_flush);
}
static void
stats_arena_hpa_shard_counters_print(
emitter_t *emitter, unsigned i, uint64_t uptime) {
@@ -1570,6 +1601,10 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
GET_AND_EMIT_MEM_STAT(extent_avail)
#undef GET_AND_EMIT_MEM_STAT
if (opt_pac_sec_opts.nshards > 0) {
stats_arena_pac_sec_print(emitter, i);
}
if (mutex) {
stats_arena_mutexes_print(emitter, i, uptime);
}
@@ -1764,6 +1799,9 @@ stats_general_print(emitter_t *emitter) {
OPT_WRITE_SIZE_T("hpa_sec_nshards")
OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
OPT_WRITE_SIZE_T("experimental_pac_sec_nshards")
OPT_WRITE_SIZE_T("experimental_pac_sec_max_alloc")
OPT_WRITE_SIZE_T("experimental_pac_sec_max_bytes")
OPT_WRITE_BOOL("huge_arena_pac_thp")
OPT_WRITE_CHAR_P("metadata_thp")
OPT_WRITE_INT64("mutex_max_spin")