mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-07-23 21:23:06 +00:00
[SEC] Make SEC owned by hpa_shard, simplify the code, add stats, lock per bin
This commit is contained in:
Slobodan Predolac
committed by
Guangli Dai
Guangli Dai
parent
c7690e92da
commit
6016d86c18
564
src/sec.c
564
src/sec.c
@@ -4,95 +4,56 @@
|
||||
#include "jemalloc/internal/sec.h"
|
||||
#include "jemalloc/internal/jemalloc_probe.h"
|
||||
|
||||
static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
|
||||
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
|
||||
bool *deferred_work_generated);
|
||||
static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
|
||||
static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
|
||||
size_t old_size, size_t new_size, bool *deferred_work_generated);
|
||||
static void sec_dalloc(
|
||||
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
|
||||
|
||||
static void
|
||||
static bool
|
||||
sec_bin_init(sec_bin_t *bin) {
|
||||
bin->being_batch_filled = false;
|
||||
bin->bytes_cur = 0;
|
||||
sec_bin_stats_init(&bin->stats);
|
||||
edata_list_active_init(&bin->freelist);
|
||||
bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN,
|
||||
malloc_mutex_rank_exclusive);
|
||||
if (err) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
|
||||
const sec_opts_t *opts) {
|
||||
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
|
||||
sec->opts = *opts;
|
||||
if (opts->nshards == 0) {
|
||||
return false;
|
||||
}
|
||||
assert(opts->max_alloc >= PAGE);
|
||||
|
||||
/*
|
||||
* Same as tcache, sec do not cache allocs/dallocs larger than
|
||||
* USIZE_GROW_SLOW_THRESHOLD because the usize above this increases
|
||||
* by PAGE and the number of usizes is too large.
|
||||
*/
|
||||
assert(!sz_large_size_classes_disabled()
|
||||
|| opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
|
||||
assert(opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
|
||||
|
||||
size_t max_alloc = PAGE_FLOOR(opts->max_alloc);
|
||||
pszind_t npsizes = sz_psz2ind(max_alloc) + 1;
|
||||
|
||||
size_t sz_shards = opts->nshards * sizeof(sec_shard_t);
|
||||
size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t);
|
||||
size_t sz_alloc = sz_shards + sz_bins;
|
||||
void *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE);
|
||||
size_t ntotal_bins = opts->nshards * (size_t)npsizes;
|
||||
size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
|
||||
void *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
|
||||
if (dynalloc == NULL) {
|
||||
return true;
|
||||
}
|
||||
sec_shard_t *shard_cur = (sec_shard_t *)dynalloc;
|
||||
sec->shards = shard_cur;
|
||||
sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards];
|
||||
/* Just for asserts, below. */
|
||||
sec_bin_t *bin_start = bin_cur;
|
||||
|
||||
for (size_t i = 0; i < opts->nshards; i++) {
|
||||
sec_shard_t *shard = shard_cur;
|
||||
shard_cur++;
|
||||
bool err = malloc_mutex_init(&shard->mtx, "sec_shard",
|
||||
WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive);
|
||||
if (err) {
|
||||
sec->bins = (sec_bin_t *)dynalloc;
|
||||
for (pszind_t j = 0; j < ntotal_bins; j++) {
|
||||
if (sec_bin_init(&sec->bins[j])) {
|
||||
return true;
|
||||
}
|
||||
shard->enabled = true;
|
||||
shard->bins = bin_cur;
|
||||
for (pszind_t j = 0; j < npsizes; j++) {
|
||||
sec_bin_init(&shard->bins[j]);
|
||||
bin_cur++;
|
||||
}
|
||||
shard->bytes_cur = 0;
|
||||
shard->to_flush_next = 0;
|
||||
}
|
||||
/*
|
||||
* Should have exactly matched the bin_start to the first unused byte
|
||||
* after the shards.
|
||||
*/
|
||||
assert((void *)shard_cur == (void *)bin_start);
|
||||
/* And the last bin to use up the last bytes of the allocation. */
|
||||
assert((char *)bin_cur == ((char *)dynalloc + sz_alloc));
|
||||
sec->fallback = fallback;
|
||||
|
||||
sec->opts = *opts;
|
||||
sec->npsizes = npsizes;
|
||||
|
||||
/*
|
||||
* Initialize these last so that an improper use of an SEC whose
|
||||
* initialization failed will segfault in an easy-to-spot way.
|
||||
*/
|
||||
sec->pai.alloc = &sec_alloc;
|
||||
sec->pai.alloc_batch = &pai_alloc_batch_default;
|
||||
sec->pai.expand = &sec_expand;
|
||||
sec->pai.shrink = &sec_shrink;
|
||||
sec->pai.dalloc = &sec_dalloc;
|
||||
sec->pai.dalloc_batch = &pai_dalloc_batch_default;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static sec_shard_t *
|
||||
static uint8_t
|
||||
sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
|
||||
/*
|
||||
* Eventually, we should implement affinity, tracking source shard using
|
||||
@@ -100,7 +61,7 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
|
||||
* distribute across all shards.
|
||||
*/
|
||||
if (tsdn_null(tsdn)) {
|
||||
return &sec->shards[0];
|
||||
return 0;
|
||||
}
|
||||
tsd_t *tsd = tsdn_tsd(tsdn);
|
||||
uint8_t *idxp = tsd_sec_shardp_get(tsd);
|
||||
@@ -118,284 +79,252 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
|
||||
assert(idx < (uint32_t)sec->opts.nshards);
|
||||
*idxp = (uint8_t)idx;
|
||||
}
|
||||
return &sec->shards[*idxp];
|
||||
return *idxp;
|
||||
}
|
||||
|
||||
/*
|
||||
* Perhaps surprisingly, this can be called on the alloc pathways; if we hit an
|
||||
* empty cache, we'll try to fill it, which can push the shard over it's limit.
|
||||
*/
|
||||
static void
|
||||
sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
|
||||
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||
edata_list_active_t to_flush;
|
||||
edata_list_active_init(&to_flush);
|
||||
while (shard->bytes_cur > sec->opts.bytes_after_flush) {
|
||||
/* Pick a victim. */
|
||||
sec_bin_t *bin = &shard->bins[shard->to_flush_next];
|
||||
|
||||
/* Update our victim-picking state. */
|
||||
shard->to_flush_next++;
|
||||
if (shard->to_flush_next == sec->npsizes) {
|
||||
shard->to_flush_next = 0;
|
||||
}
|
||||
|
||||
assert(shard->bytes_cur >= bin->bytes_cur);
|
||||
if (bin->bytes_cur != 0) {
|
||||
shard->bytes_cur -= bin->bytes_cur;
|
||||
bin->bytes_cur = 0;
|
||||
edata_list_active_concat(&to_flush, &bin->freelist);
|
||||
}
|
||||
/*
|
||||
* Either bin->bytes_cur was 0, in which case we didn't touch
|
||||
* the bin list but it should be empty anyways (or else we
|
||||
* missed a bytes_cur update on a list modification), or it
|
||||
* *was* 0 and we emptied it ourselves. Either way, it should
|
||||
* be empty now.
|
||||
*/
|
||||
assert(edata_list_active_empty(&bin->freelist));
|
||||
}
|
||||
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
bool deferred_work_generated = false;
|
||||
pai_dalloc_batch(
|
||||
tsdn, sec->fallback, &to_flush, &deferred_work_generated);
|
||||
static sec_bin_t *
|
||||
sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) {
|
||||
assert(shard < sec->opts.nshards);
|
||||
size_t ind = (size_t)shard * sec->npsizes + pszind;
|
||||
assert(ind < sec->npsizes * sec->opts.nshards);
|
||||
return &sec->bins[ind];
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
sec_shard_alloc_locked(
|
||||
tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, sec_bin_t *bin) {
|
||||
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||
if (!shard->enabled) {
|
||||
return NULL;
|
||||
}
|
||||
sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
|
||||
malloc_mutex_assert_owner(tsdn, &bin->mtx);
|
||||
|
||||
edata_t *edata = edata_list_active_first(&bin->freelist);
|
||||
if (edata != NULL) {
|
||||
assert(!edata_list_active_empty(&bin->freelist));
|
||||
edata_list_active_remove(&bin->freelist, edata);
|
||||
assert(edata_size_get(edata) <= bin->bytes_cur);
|
||||
bin->bytes_cur -= edata_size_get(edata);
|
||||
assert(edata_size_get(edata) <= shard->bytes_cur);
|
||||
shard->bytes_cur -= edata_size_get(edata);
|
||||
size_t sz = edata_size_get(edata);
|
||||
assert(sz <= bin->bytes_cur && sz > 0);
|
||||
bin->bytes_cur -= sz;
|
||||
bin->stats.nhits++;
|
||||
}
|
||||
return edata;
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
|
||||
sec_bin_t *bin, size_t size, bool frequent_reuse) {
|
||||
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
|
||||
sec_multishard_trylock_alloc(
|
||||
tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
|
||||
assert(sec->opts.nshards > 0);
|
||||
|
||||
edata_list_active_t result;
|
||||
edata_list_active_init(&result);
|
||||
bool deferred_work_generated = false;
|
||||
size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size,
|
||||
1 + sec->opts.batch_fill_extra, &result, frequent_reuse,
|
||||
&deferred_work_generated);
|
||||
|
||||
edata_t *ret = edata_list_active_first(&result);
|
||||
if (ret != NULL) {
|
||||
edata_list_active_remove(&result, ret);
|
||||
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
|
||||
sec_bin_t *bin;
|
||||
for (size_t i = 0; i < sec->opts.nshards; ++i) {
|
||||
bin = sec_bin_pick(sec, cur_shard, pszind);
|
||||
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
|
||||
edata_t *edata = sec_bin_alloc_locked(
|
||||
tsdn, sec, bin, size);
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
if (edata != NULL) {
|
||||
JE_USDT(sec_alloc, 5, sec, bin, edata, size,
|
||||
/* frequent_reuse */ 1);
|
||||
return edata;
|
||||
}
|
||||
}
|
||||
cur_shard++;
|
||||
if (cur_shard == sec->opts.nshards) {
|
||||
cur_shard = 0;
|
||||
}
|
||||
}
|
||||
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
bin->being_batch_filled = false;
|
||||
/*
|
||||
* Handle the easy case first: nothing to cache. Note that this can
|
||||
* only happen in case of OOM, since sec_alloc checks the expected
|
||||
* number of allocs, and doesn't bother going down the batch_fill
|
||||
* pathway if there won't be anything left to cache. So to be in this
|
||||
* code path, we must have asked for > 1 alloc, but only gotten 1 back.
|
||||
*/
|
||||
if (nalloc <= 1) {
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
return ret;
|
||||
/* No bin had alloc or had the extent */
|
||||
assert(cur_shard == sec_shard_pick(tsdn, sec));
|
||||
bin = sec_bin_pick(sec, cur_shard, pszind);
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
|
||||
if (edata == NULL) {
|
||||
/* Only now we know it is a miss */
|
||||
bin->stats.nmisses++;
|
||||
}
|
||||
|
||||
size_t new_cached_bytes = (nalloc - 1) * size;
|
||||
|
||||
edata_list_active_concat(&bin->freelist, &result);
|
||||
bin->bytes_cur += new_cached_bytes;
|
||||
shard->bytes_cur += new_cached_bytes;
|
||||
|
||||
if (shard->bytes_cur > sec->opts.max_bytes) {
|
||||
sec_flush_some_and_unlock(tsdn, sec, shard);
|
||||
} else {
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
}
|
||||
|
||||
return ret;
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1);
|
||||
return edata;
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
|
||||
bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
|
||||
edata_t *
|
||||
sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
|
||||
if (!sec_size_supported(sec, size)) {
|
||||
return NULL;
|
||||
}
|
||||
assert((size & PAGE_MASK) == 0);
|
||||
assert(!guarded);
|
||||
|
||||
sec_t *sec = (sec_t *)self;
|
||||
|
||||
if (zero || alignment > PAGE || sec->opts.nshards == 0
|
||||
|| size > sec->opts.max_alloc) {
|
||||
return pai_alloc(tsdn, sec->fallback, size, alignment, zero,
|
||||
/* guarded */ false, frequent_reuse,
|
||||
deferred_work_generated);
|
||||
}
|
||||
pszind_t pszind = sz_psz2ind(size);
|
||||
assert(pszind < sec->npsizes);
|
||||
|
||||
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
|
||||
sec_bin_t *bin = &shard->bins[pszind];
|
||||
bool do_batch_fill = false;
|
||||
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin);
|
||||
if (edata == NULL) {
|
||||
if (!bin->being_batch_filled
|
||||
&& sec->opts.batch_fill_extra > 0) {
|
||||
bin->being_batch_filled = true;
|
||||
do_batch_fill = true;
|
||||
/*
|
||||
* If there's only one shard, skip the trylock optimization and
|
||||
* go straight to the blocking lock.
|
||||
*/
|
||||
if (sec->opts.nshards == 1) {
|
||||
sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
|
||||
if (edata == NULL) {
|
||||
bin->stats.nmisses++;
|
||||
}
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
JE_USDT(sec_alloc, 5, sec, bin, edata, size,
|
||||
/* frequent_reuse */ 1);
|
||||
return edata;
|
||||
}
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
if (edata == NULL) {
|
||||
if (do_batch_fill) {
|
||||
edata = sec_batch_fill_and_alloc(
|
||||
tsdn, sec, shard, bin, size, frequent_reuse);
|
||||
} else {
|
||||
edata = pai_alloc(tsdn, sec->fallback, size, alignment,
|
||||
zero, /* guarded */ false, frequent_reuse,
|
||||
deferred_work_generated);
|
||||
}
|
||||
}
|
||||
JE_USDT(sec_alloc, 5, sec, shard, edata, size, frequent_reuse);
|
||||
return edata;
|
||||
}
|
||||
|
||||
static bool
|
||||
sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
|
||||
size_t new_size, bool zero, bool *deferred_work_generated) {
|
||||
sec_t *sec = (sec_t *)self;
|
||||
JE_USDT(sec_expand, 4, sec, edata, old_size, new_size);
|
||||
return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero,
|
||||
deferred_work_generated);
|
||||
}
|
||||
|
||||
static bool
|
||||
sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
|
||||
size_t new_size, bool *deferred_work_generated) {
|
||||
sec_t *sec = (sec_t *)self;
|
||||
JE_USDT(sec_shrink, 4, sec, edata, old_size, new_size);
|
||||
return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size,
|
||||
deferred_work_generated);
|
||||
return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
|
||||
}
|
||||
|
||||
static void
|
||||
sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
|
||||
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||
shard->bytes_cur = 0;
|
||||
edata_list_active_t to_flush;
|
||||
edata_list_active_init(&to_flush);
|
||||
for (pszind_t i = 0; i < sec->npsizes; i++) {
|
||||
sec_bin_t *bin = &shard->bins[i];
|
||||
bin->bytes_cur = 0;
|
||||
edata_list_active_concat(&to_flush, &bin->freelist);
|
||||
}
|
||||
sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
|
||||
edata_list_active_t *dalloc_list) {
|
||||
malloc_mutex_assert_owner(tsdn, &bin->mtx);
|
||||
|
||||
/*
|
||||
* Ordinarily we would try to avoid doing the batch deallocation while
|
||||
* holding the shard mutex, but the flush_all pathways only happen when
|
||||
* we're disabling the HPA or resetting the arena, both of which are
|
||||
* rare pathways.
|
||||
*/
|
||||
bool deferred_work_generated = false;
|
||||
pai_dalloc_batch(
|
||||
tsdn, sec->fallback, &to_flush, &deferred_work_generated);
|
||||
}
|
||||
|
||||
static void
|
||||
sec_shard_dalloc_and_unlock(
|
||||
tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_t *edata) {
|
||||
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||
assert(shard->bytes_cur <= sec->opts.max_bytes);
|
||||
size_t size = edata_size_get(edata);
|
||||
pszind_t pszind = sz_psz2ind(size);
|
||||
assert(pszind < sec->npsizes);
|
||||
/*
|
||||
* Prepending here results in LIFO allocation per bin, which seems
|
||||
* reasonable.
|
||||
*/
|
||||
sec_bin_t *bin = &shard->bins[pszind];
|
||||
edata_list_active_prepend(&bin->freelist, edata);
|
||||
bin->bytes_cur += size;
|
||||
shard->bytes_cur += size;
|
||||
if (shard->bytes_cur > sec->opts.max_bytes) {
|
||||
/*
|
||||
* We've exceeded the shard limit. We make two nods in the
|
||||
* direction of fragmentation avoidance: we flush everything in
|
||||
* the shard, rather than one particular bin, and we hold the
|
||||
* lock while flushing (in case one of the extents we flush is
|
||||
* highly preferred from a fragmentation-avoidance perspective
|
||||
* in the backing allocator). This has the extra advantage of
|
||||
* not requiring advanced cache balancing strategies.
|
||||
*/
|
||||
sec_flush_some_and_unlock(tsdn, sec, shard);
|
||||
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
|
||||
} else {
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
}
|
||||
}
|
||||
edata_t *edata = edata_list_active_first(dalloc_list);
|
||||
assert(edata != NULL);
|
||||
edata_list_active_remove(dalloc_list, edata);
|
||||
JE_USDT(sec_dalloc, 3, sec, bin, edata);
|
||||
edata_list_active_prepend(&bin->freelist, edata);
|
||||
/* Single extent can be returned to SEC */
|
||||
assert(edata_list_active_empty(dalloc_list));
|
||||
|
||||
static void
|
||||
sec_dalloc(
|
||||
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
|
||||
sec_t *sec = (sec_t *)self;
|
||||
if (sec->opts.nshards == 0
|
||||
|| edata_size_get(edata) > sec->opts.max_alloc) {
|
||||
pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
|
||||
if (bin->bytes_cur <= sec->opts.max_bytes) {
|
||||
bin->stats.ndalloc_noflush++;
|
||||
return;
|
||||
}
|
||||
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
|
||||
JE_USDT(sec_dalloc, 3, sec, shard, edata);
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
if (shard->enabled) {
|
||||
sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata);
|
||||
bin->stats.ndalloc_flush++;
|
||||
/* we want to flush 1/4 of max_bytes */
|
||||
size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2);
|
||||
while (bin->bytes_cur > bytes_target
|
||||
&& !edata_list_active_empty(&bin->freelist)) {
|
||||
edata_t *cur = edata_list_active_last(&bin->freelist);
|
||||
size_t sz = edata_size_get(cur);
|
||||
assert(sz <= bin->bytes_cur && sz > 0);
|
||||
bin->bytes_cur -= sz;
|
||||
edata_list_active_remove(&bin->freelist, cur);
|
||||
edata_list_active_append(dalloc_list, cur);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
|
||||
pszind_t pszind, edata_list_active_t *dalloc_list) {
|
||||
assert(sec->opts.nshards > 0);
|
||||
|
||||
/* Try to dalloc in this threads bin first */
|
||||
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
|
||||
for (size_t i = 0; i < sec->opts.nshards; ++i) {
|
||||
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
|
||||
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
|
||||
sec_bin_dalloc_locked(
|
||||
tsdn, sec, bin, size, dalloc_list);
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
return;
|
||||
}
|
||||
cur_shard++;
|
||||
if (cur_shard == sec->opts.nshards) {
|
||||
cur_shard = 0;
|
||||
}
|
||||
}
|
||||
/* No bin had alloc or had the extent */
|
||||
assert(cur_shard == sec_shard_pick(tsdn, sec));
|
||||
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
}
|
||||
|
||||
void
|
||||
sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
|
||||
if (!sec_is_used(sec)) {
|
||||
return;
|
||||
}
|
||||
edata_t *edata = edata_list_active_first(dalloc_list);
|
||||
size_t size = edata_size_get(edata);
|
||||
if (size > sec->opts.max_alloc) {
|
||||
return;
|
||||
}
|
||||
pszind_t pszind = sz_psz2ind(size);
|
||||
assert(pszind < sec->npsizes);
|
||||
|
||||
/*
|
||||
* If there's only one shard, skip the trylock optimization and
|
||||
* go straight to the blocking lock.
|
||||
*/
|
||||
if (sec->opts.nshards == 1) {
|
||||
sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
return;
|
||||
}
|
||||
sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
|
||||
}
|
||||
|
||||
void
|
||||
sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
|
||||
size_t nallocs) {
|
||||
assert((size & PAGE_MASK) == 0);
|
||||
assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
|
||||
assert(nallocs > 0);
|
||||
|
||||
pszind_t pszind = sz_psz2ind(size);
|
||||
assert(pszind < sec->npsizes);
|
||||
|
||||
sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
|
||||
malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
size_t new_cached_bytes = nallocs * size;
|
||||
if (bin->bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
|
||||
assert(!edata_list_active_empty(result));
|
||||
edata_list_active_concat(&bin->freelist, result);
|
||||
bin->bytes_cur += new_cached_bytes;
|
||||
} else {
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
|
||||
/*
|
||||
* Unlikely case of many threads filling at the same time and
|
||||
* going above max.
|
||||
*/
|
||||
bin->stats.noverfills++;
|
||||
while (bin->bytes_cur + size <= sec->opts.max_bytes) {
|
||||
edata_t *edata = edata_list_active_first(result);
|
||||
if (edata == NULL) {
|
||||
break;
|
||||
}
|
||||
edata_list_active_remove(result, edata);
|
||||
assert(size == edata_size_get(edata));
|
||||
edata_list_active_append(&bin->freelist, edata);
|
||||
bin->bytes_cur += size;
|
||||
}
|
||||
}
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
}
|
||||
|
||||
void
|
||||
sec_flush(tsdn_t *tsdn, sec_t *sec) {
|
||||
for (size_t i = 0; i < sec->opts.nshards; i++) {
|
||||
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
|
||||
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
|
||||
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
|
||||
sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
|
||||
if (!sec_is_used(sec)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
sec_disable(tsdn_t *tsdn, sec_t *sec) {
|
||||
for (size_t i = 0; i < sec->opts.nshards; i++) {
|
||||
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
|
||||
sec->shards[i].enabled = false;
|
||||
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
|
||||
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
|
||||
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
|
||||
for (pszind_t i = 0; i < ntotal_bins; i++) {
|
||||
sec_bin_t *bin = &sec->bins[i];
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
bin->bytes_cur = 0;
|
||||
edata_list_active_concat(to_flush, &bin->freelist);
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
|
||||
if (!sec_is_used(sec)) {
|
||||
return;
|
||||
}
|
||||
size_t sum = 0;
|
||||
for (size_t i = 0; i < sec->opts.nshards; i++) {
|
||||
/*
|
||||
* We could save these lock acquisitions by making bytes_cur
|
||||
* atomic, but stats collection is rare anyways and we expect
|
||||
* the number and type of stats to get more interesting.
|
||||
*/
|
||||
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
|
||||
sum += sec->shards[i].bytes_cur;
|
||||
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
|
||||
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
|
||||
for (pszind_t i = 0; i < ntotal_bins; i++) {
|
||||
sec_bin_t *bin = &sec->bins[i];
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
sum += bin->bytes_cur;
|
||||
sec_bin_stats_accum(&stats->total, &bin->stats);
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
}
|
||||
stats->bytes += sum;
|
||||
}
|
||||
@@ -403,31 +332,50 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
|
||||
void
|
||||
sec_mutex_stats_read(
|
||||
tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data) {
|
||||
for (size_t i = 0; i < sec->opts.nshards; i++) {
|
||||
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
|
||||
malloc_mutex_prof_accum(
|
||||
tsdn, mutex_prof_data, &sec->shards[i].mtx);
|
||||
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
|
||||
if (!sec_is_used(sec)) {
|
||||
return;
|
||||
}
|
||||
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
|
||||
for (pszind_t i = 0; i < ntotal_bins; i++) {
|
||||
sec_bin_t *bin = &sec->bins[i];
|
||||
malloc_mutex_lock(tsdn, &bin->mtx);
|
||||
malloc_mutex_prof_accum(tsdn, mutex_prof_data, &bin->mtx);
|
||||
malloc_mutex_unlock(tsdn, &bin->mtx);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
sec_prefork2(tsdn_t *tsdn, sec_t *sec) {
|
||||
for (size_t i = 0; i < sec->opts.nshards; i++) {
|
||||
malloc_mutex_prefork(tsdn, &sec->shards[i].mtx);
|
||||
if (!sec_is_used(sec)) {
|
||||
return;
|
||||
}
|
||||
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
|
||||
for (pszind_t i = 0; i < ntotal_bins; i++) {
|
||||
sec_bin_t *bin = &sec->bins[i];
|
||||
malloc_mutex_prefork(tsdn, &bin->mtx);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) {
|
||||
for (size_t i = 0; i < sec->opts.nshards; i++) {
|
||||
malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx);
|
||||
if (!sec_is_used(sec)) {
|
||||
return;
|
||||
}
|
||||
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
|
||||
for (pszind_t i = 0; i < ntotal_bins; i++) {
|
||||
sec_bin_t *bin = &sec->bins[i];
|
||||
malloc_mutex_postfork_parent(tsdn, &bin->mtx);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
sec_postfork_child(tsdn_t *tsdn, sec_t *sec) {
|
||||
for (size_t i = 0; i < sec->opts.nshards; i++) {
|
||||
malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx);
|
||||
if (!sec_is_used(sec)) {
|
||||
return;
|
||||
}
|
||||
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
|
||||
for (pszind_t i = 0; i < ntotal_bins; i++) {
|
||||
sec_bin_t *bin = &sec->bins[i];
|
||||
malloc_mutex_postfork_child(tsdn, &bin->mtx);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user