lihao/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-23 21:23:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Wake up background threads on demand

Browse Source 
This change allows every allocator conforming to PAI communicate that it
deferred some work for the future. Without it if a background thread goes into
indefinite sleep, there is no way to notify it about upcoming deferred work.
This commit is contained in:
Alex Lapenkou
2021-08-18 19:24:37 -07:00
committed by Alexander Lapenkov
parent 97da57c13a
commit 8229cc77c5
21 changed files with 445 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
src/arena.c
View File

@@ -199,15 +199,17 @@ arena_background_thread_inactivity_check(tsdn_t *tsdn, arena_t *arena,
}
}
void arena_handle_new_dirty_pages(tsdn_t *tsdn, arena_t *arena) {
/*
* React to deferred work generated by a PAI function.
*/
void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
if (arena_decay_ms_get(arena, extent_state_dirty) == 0) {
if (decay_immediately(&arena->pa_shard.pac.decay_dirty)) {
arena_decay_dirty(tsdn, arena, false, true);
} else {
arena_background_thread_inactivity_check(tsdn, arena, false);
}
arena_background_thread_inactivity_check(tsdn, arena, false);
}
static void *
@@ -316,11 +318,14 @@ arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize,
edata_t *
arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero) {
bool deferred_work_generated;
szind_t szind = sz_size2index(usize);
size_t esize = usize + sz_large_pad;
edata_t *edata = pa_alloc(tsdn, &arena->pa_shard, esize, alignment,
/* slab */ false, szind, zero);
/* slab */ false, szind, zero, &deferred_work_generated);
assert(deferred_work_generated == false);
if (edata != NULL) {
if (config_stats) {
@@ -471,6 +476,45 @@ arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
arena_decay_muzzy(tsdn, arena, is_background_thread, all);
}
static bool
arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
background_thread_info_t *info, nstime_t *remaining_sleep,
size_t npages_new) {
malloc_mutex_assert_owner(tsdn, &info->mtx);
if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
return false;
}
if (!decay_gradually(decay)) {
malloc_mutex_unlock(tsdn, &decay->mtx);
return false;
}
nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
malloc_mutex_unlock(tsdn, &decay->mtx);
return false;
}
nstime_subtract(remaining_sleep, &decay->epoch);
if (npages_new > 0) {
uint64_t npurge_new = decay_npages_purge_in(decay,
remaining_sleep, npages_new);
info->npages_to_purge_new += npurge_new;
}
malloc_mutex_unlock(tsdn, &decay->mtx);
return info->npages_to_purge_new >
ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD;
}
/*
* Check if deferred work needs to be done sooner than planned.
* For decay we might want to wake up earlier because of an influx of dirty
* pages. Rather than waiting for previously estimated time, we proactively
* purge those pages.
* If background thread sleeps indefinitely, always wake up because some
* deferred work has been generated.
*/
static void
arena_maybe_do_deferred_work(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
size_t npages_new) {
@@ -485,47 +529,18 @@ arena_maybe_do_deferred_work(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
*/
return;
}
if (!background_thread_running(info)) {
if (!background_thread_is_started(info)) {
goto label_done;
}
if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
goto label_done;
}
if (!decay_gradually(decay)) {
goto label_done_unlock2;
}
nstime_t diff;
nstime_init(&diff, background_thread_wakeup_time_get(info));
if (nstime_compare(&diff, &decay->epoch) <= 0) {
goto label_done_unlock2;
}
nstime_subtract(&diff, &decay->epoch);
if (npages_new > 0) {
uint64_t npurge_new = decay_npages_purge_in(decay, &diff,
npages_new);
info->npages_to_purge_new += npurge_new;
}
bool should_signal;
if (info->npages_to_purge_new > ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD) {
should_signal = true;
} else if (unlikely(background_thread_indefinite_sleep(info)) &&
(ecache_npages_get(&arena->pa_shard.pac.ecache_dirty) > 0 ||
ecache_npages_get(&arena->pa_shard.pac.ecache_muzzy) > 0 ||
info->npages_to_purge_new > 0)) {
should_signal = true;
} else {
should_signal = false;
}
if (should_signal) {
nstime_t remaining_sleep;
if (background_thread_indefinite_sleep(info)) {
background_thread_wakeup_early(info, NULL);
} else if (arena_should_decay_early(tsdn, arena, decay, info,
&remaining_sleep, npages_new)) {
info->npages_to_purge_new = 0;
background_thread_wakeup_early(info, &diff);
background_thread_wakeup_early(info, &remaining_sleep);
}
label_done_unlock2:
malloc_mutex_unlock(tsdn, &decay->mtx);
label_done:
malloc_mutex_unlock(tsdn, &info->mtx);
}
@@ -539,10 +554,10 @@ arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) {
void
arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) {
bool generated_dirty;
pa_dalloc(tsdn, &arena->pa_shard, slab, &generated_dirty);
if (generated_dirty) {
arena_handle_new_dirty_pages(tsdn, arena);
bool deferred_work_generated;
pa_dalloc(tsdn, &arena->pa_shard, slab, &deferred_work_generated);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
}
@@ -803,11 +818,17 @@ arena_destroy(tsd_t *tsd, arena_t *arena) {
static edata_t *
arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard,
const bin_info_t *bin_info) {
bool deferred_work_generated;
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
edata_t *slab = pa_alloc(tsdn, &arena->pa_shard, bin_info->slab_size,
PAGE, /* slab */ true, /* szind */ binind, /* zero */ false);
PAGE, /* slab */ true, /* szind */ binind, /* zero */ false,
&deferred_work_generated);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
if (slab == NULL) {
return NULL;

20
src/background_thread.c
View File

@@ -119,7 +119,8 @@ background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info,
int ret;
if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
assert(background_thread_indefinite_sleep(info));
background_thread_wakeup_time_set(tsdn, info,
BACKGROUND_THREAD_INDEFINITE_SLEEP);
ret = pthread_cond_wait(&info->cond, &info->mtx.lock);
assert(ret == 0);
} else {
@@ -144,8 +145,6 @@ background_thread_sleep(tsdn_t *tsdn, background_thread_info_t *info,
assert(!background_thread_indefinite_sleep(info));
ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, &ts);
assert(ret == ETIMEDOUT || ret == 0);
background_thread_wakeup_time_set(tsdn, info,
BACKGROUND_THREAD_INDEFINITE_SLEEP);
}
if (config_stats) {
gettimeofday(&tv, NULL);
@@ -177,13 +176,21 @@ background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info,
unsigned ind) {
uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX;
unsigned narenas = narenas_total_get();
bool slept_indefinitely = background_thread_indefinite_sleep(info);
for (unsigned i = ind; i < narenas; i += max_background_threads) {
arena_t *arena = arena_get(tsdn, i, false);
if (!arena) {
continue;
}
arena_do_deferred_work(tsdn, arena);
/*
* If thread was woken up from the indefinite sleep, don't
* do the work instantly, but rather check when the deferred
* work that caused this thread to wake up is scheduled for.
*/
if (!slept_indefinitely) {
arena_do_deferred_work(tsdn, arena);
}
if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) {
/* Min interval will be used. */
continue;
@@ -574,7 +581,7 @@ background_threads_disable(tsd_t *tsd) {
}
bool
background_thread_running(background_thread_info_t *info) {
background_thread_is_started(background_thread_info_t *info) {
return info->state == background_thread_started;
}
@@ -586,7 +593,8 @@ background_thread_wakeup_early(background_thread_info_t *info,
* we know that background thread wakes up soon, so the time to cache
* the just freed memory is bounded and low.
*/
if (nstime_ns(remaining_sleep) < BACKGROUND_THREAD_MIN_INTERVAL_NS) {
if (remaining_sleep && nstime_ns(remaining_sleep) <
BACKGROUND_THREAD_MIN_INTERVAL_NS) {
return;
}
pthread_cond_signal(&info->cond);

2
src/decay.c
View File

@@ -3,7 +3,7 @@
#include "jemalloc/internal/decay.h"
const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = {
static const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = {
#define STEP(step, h, x, y) \
h,
SMOOTHSTEP

122
src/hpa.c
View File

@@ -9,16 +9,17 @@
#define HPA_EDEN_SIZE (128 * HUGEPAGE)
static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero);
size_t alignment, bool zero, bool *deferred_work_generated);
static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results);
size_t nallocs, edata_list_active_t *results, bool *deferred_work_generated);
static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero);
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size);
static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata);
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated);
static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list);
edata_list_active_t *list, bool *deferred_work_generated);
static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
bool
@@ -366,6 +367,13 @@ hpa_update_purge_hugify_eligibility(tsdn_t *tsdn, hpa_shard_t *shard,
}
}
static bool
hpa_shard_has_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
return to_hugify != NULL || hpa_should_purge(tsdn, shard);
}
/* Returns whether or not we purged anything. */
static bool
hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
@@ -429,6 +437,7 @@ hpa_try_purge(tsdn_t *tsdn, hpa_shard_t *shard) {
shard->npending_purge -= num_to_purge;
shard->stats.npurge_passes++;
shard->stats.npurges += purges_this_pass;
shard->central->hooks.curtime(&shard->last_purge);
if (dehugify) {
shard->stats.ndehugifies++;
}
@@ -615,7 +624,8 @@ hpa_try_alloc_one_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
static size_t
hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
bool *oom, size_t nallocs, edata_list_active_t *results) {
bool *oom, size_t nallocs, edata_list_active_t *results,
bool *deferred_work_generated) {
malloc_mutex_lock(tsdn, &shard->mtx);
size_t nsuccess = 0;
for (; nsuccess < nallocs; nsuccess++) {
@@ -628,18 +638,20 @@ hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
}
hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
*deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard);
malloc_mutex_unlock(tsdn, &shard->mtx);
return nsuccess;
}
static size_t
hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
size_t nallocs, edata_list_active_t *results) {
size_t nallocs, edata_list_active_t *results,
bool *deferred_work_generated) {
assert(size <= shard->opts.slab_max_alloc);
bool oom = false;
size_t nsuccess = hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
nallocs, results);
nallocs, results, deferred_work_generated);
if (nsuccess == nallocs || oom) {
return nsuccess;
@@ -655,7 +667,7 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
* in between when we dropped the main mutex and grabbed the grow mutex.
*/
nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
nallocs - nsuccess, results);
nallocs - nsuccess, results, deferred_work_generated);
if (nsuccess == nallocs || oom) {
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
return nsuccess;
@@ -683,7 +695,7 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
malloc_mutex_unlock(tsdn, &shard->mtx);
nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
nallocs - nsuccess, results);
nallocs - nsuccess, results, deferred_work_generated);
/*
* Drop grow_mtx before doing deferred work; other threads blocked on it
* should be allowed to proceed while we're working.
@@ -704,7 +716,7 @@ hpa_from_pai(pai_t *self) {
static size_t
hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results) {
edata_list_active_t *results, bool *deferred_work_generated) {
assert(nallocs > 0);
assert((size & PAGE_MASK) == 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
@@ -716,7 +728,7 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
}
size_t nsuccess = hpa_alloc_batch_psset(tsdn, shard, size, nallocs,
results);
results, deferred_work_generated);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
@@ -737,7 +749,8 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
}
static edata_t *
hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) {
hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool *deferred_work_generated) {
assert((size & PAGE_MASK) == 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
@@ -753,23 +766,25 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) {
edata_list_active_t results;
edata_list_active_init(&results);
size_t nallocs = hpa_alloc_batch(tsdn, self, size, /* nallocs */ 1,
&results);
&results, deferred_work_generated);
assert(nallocs == 0 || nallocs == 1);
edata_t *edata = edata_list_active_first(&results);
return edata;
}
static bool
hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero) {
hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated) {
/* Expand not yet supported. */
*deferred_work_generated = false;
return true;
}
static bool
hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size) {
size_t old_size, size_t new_size, bool *deferred_work_generated) {
/* Shrink not yet supported. */
*deferred_work_generated = false;
return true;
}
@@ -825,7 +840,8 @@ hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
}
static void
hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list) {
hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
bool *deferred_work_generated) {
hpa_shard_t *shard = hpa_from_pai(self);
edata_t *edata;
@@ -840,21 +856,83 @@ hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list) {
hpa_dalloc_locked(tsdn, shard, edata);
}
hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
*deferred_work_generated =
hpa_shard_has_deferred_work(tsdn, shard);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
static void
hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated) {
/* Just a dalloc_batch of size 1; this lets us share logic. */
edata_list_active_t dalloc_list;
edata_list_active_init(&dalloc_list);
edata_list_active_append(&dalloc_list, edata);
hpa_dalloc_batch(tsdn, self, &dalloc_list);
hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
}
/*
* Calculate time until either purging or hugification ought to happen.
* Called by background threads.
*/
static uint64_t
hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
return opt_background_thread_hpa_interval_max_ms;
hpa_shard_t *shard = hpa_from_pai(self);
uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
malloc_mutex_lock(tsdn, &shard->mtx);
hpdata_t *to_hugify = psset_pick_hugify(&shard->psset);
if (to_hugify != NULL) {
nstime_t time_hugify_allowed =
hpdata_time_hugify_allowed(to_hugify);
nstime_t nstime;
shard->central->hooks.curtime(&nstime);
nstime_subtract(&nstime, &time_hugify_allowed);
uint64_t since_hugify_allowed_ms = nstime_msec(&nstime);
/*
* If not enough time has passed since hugification was allowed,
* sleep for the rest.
*/
if (since_hugify_allowed_ms < shard->opts.hugify_delay_ms) {
time_ns = shard->opts.hugify_delay_ms - since_hugify_allowed_ms;
time_ns *= 1000 * 1000;
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
return BACKGROUND_THREAD_DEFERRED_MIN;
}
}
if (hpa_should_purge(tsdn, shard)) {
/*
* If we haven't purged before, no need to check interval
* between purges. Simply purge as soon as possible.
*/
if (shard->stats.npurge_passes == 0) {
malloc_mutex_unlock(tsdn, &shard->mtx);
return BACKGROUND_THREAD_DEFERRED_MIN;
}
nstime_t nstime;
shard->central->hooks.curtime(&nstime);
nstime_subtract(&nstime, &shard->last_purge);
uint64_t since_last_purge_ms = nstime_msec(&nstime);
if (since_last_purge_ms < shard->opts.min_purge_interval_ms) {
uint64_t until_purge_ns;
until_purge_ns = shard->opts.min_purge_interval_ms -
since_last_purge_ms;
until_purge_ns *= 1000 * 1000;
if (until_purge_ns < time_ns) {
time_ns = until_purge_ns;
}
} else {
time_ns = BACKGROUND_THREAD_DEFERRED_MIN;
}
}
malloc_mutex_unlock(tsdn, &shard->mtx);
return time_ns;
}
void

26
src/large.c
View File

@@ -64,14 +64,15 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
return true;
}
bool generated_dirty;
bool deferred_work_generated;
bool err = pa_shrink(tsdn, &arena->pa_shard, edata, old_size,
usize + sz_large_pad, sz_size2index(usize), &generated_dirty);
usize + sz_large_pad, sz_size2index(usize),
&deferred_work_generated);
if (err) {
return true;
}
if (generated_dirty) {
arena_handle_new_dirty_pages(tsdn, arena);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize);
@@ -88,8 +89,15 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, edata_t *edata, size_t usize,
size_t new_size = usize + sz_large_pad;
szind_t szind = sz_size2index(usize);
bool deferred_work_generated;
bool err = pa_expand(tsdn, &arena->pa_shard, edata, old_size, new_size,
szind, zero);
szind, zero, &deferred_work_generated);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
if (err) {
return true;
}
@@ -241,10 +249,10 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
static void
large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
bool generated_dirty;
pa_dalloc(tsdn, &arena->pa_shard, edata, &generated_dirty);
if (generated_dirty) {
arena_handle_new_dirty_pages(tsdn, arena);
bool deferred_work_generated;
pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated);
if (deferred_work_generated) {
arena_handle_deferred_work(tsdn, arena);
}
}

25
src/pa.c
View File

@@ -118,21 +118,23 @@ pa_get_pai(pa_shard_t *shard, edata_t *edata) {
edata_t *
pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
bool slab, szind_t szind, bool zero) {
bool slab, szind_t szind, bool zero, bool *deferred_work_generated) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE, 0);
edata_t *edata = NULL;
*deferred_work_generated = false;
if (pa_shard_uses_hpa(shard)) {
edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment,
zero);
zero, deferred_work_generated);
}
/*
* Fall back to the PAC if the HPA is off or couldn't serve the given
* allocation request.
*/
if (edata == NULL) {
edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero);
edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero,
deferred_work_generated);
}
if (edata != NULL) {
@@ -152,7 +154,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
bool
pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
size_t new_size, szind_t szind, bool zero) {
size_t new_size, szind_t szind, bool zero, bool *deferred_work_generated) {
assert(new_size > old_size);
assert(edata_size_get(edata) == old_size);
assert((new_size & PAGE_MASK) == 0);
@@ -161,7 +163,8 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
pai_t *pai = pa_get_pai(shard, edata);
bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero);
bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero,
deferred_work_generated);
if (error) {
return true;
}
@@ -174,20 +177,19 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
bool
pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
size_t new_size, szind_t szind, bool *generated_dirty) {
size_t new_size, szind_t szind, bool *deferred_work_generated) {
assert(new_size < old_size);
assert(edata_size_get(edata) == old_size);
assert((new_size & PAGE_MASK) == 0);
size_t shrink_amount = old_size - new_size;
*generated_dirty = false;
pai_t *pai = pa_get_pai(shard, edata);
bool error = pai_shrink(tsdn, pai, edata, old_size, new_size);
bool error = pai_shrink(tsdn, pai, edata, old_size, new_size,
deferred_work_generated);
if (error) {
return true;
}
pa_nactive_sub(shard, shrink_amount >> LG_PAGE);
*generated_dirty = (edata_pai_get(edata) == EXTENT_PAI_PAC);
edata_szind_set(edata, szind);
emap_remap(tsdn, shard->emap, edata, szind, /* slab */ false);
@@ -196,7 +198,7 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
void
pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
bool *generated_dirty) {
bool *deferred_work_generated) {
emap_remap(tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false);
if (edata_slab_get(edata)) {
emap_deregister_interior(tsdn, shard->emap, edata);
@@ -206,8 +208,7 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
edata_szind_set(edata, SC_NSIZES);
pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE);
pai_t *pai = pa_get_pai(shard, edata);
pai_dalloc(tsdn, pai, edata);
*generated_dirty = (edata_pai_get(edata) == EXTENT_PAI_PAC);
pai_dalloc(tsdn, pai, edata, deferred_work_generated);
}
bool

28
src/pac.c
View File

@@ -4,12 +4,13 @@
#include "jemalloc/internal/pac.h"
static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero);
size_t alignment, bool zero, bool *deferred_work_generated);
static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero);
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size);
static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata);
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated);
static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
static ehooks_t *
@@ -109,9 +110,11 @@ pac_may_have_muzzy(pac_t *pac) {
static edata_t *
pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
bool zero) {
bool zero, bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
*deferred_work_generated = false;
ehooks_t *ehooks = pac_ehooks_get(pac);
edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
NULL, size, alignment, zero);
@@ -133,10 +136,12 @@ pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
static bool
pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero) {
size_t new_size, bool zero, bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
*deferred_work_generated = false;
size_t mapped_add = 0;
size_t expand_amount = new_size - old_size;
@@ -171,12 +176,13 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
static bool
pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size) {
size_t new_size, bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
size_t shrink_amount = old_size - new_size;
*deferred_work_generated = false;
if (ehooks_split_will_fail(ehooks)) {
return true;
@@ -188,14 +194,18 @@ pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
return true;
}
ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail);
*deferred_work_generated = true;
return false;
}
static void
pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
pac_dalloc_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated) {
pac_t *pac = (pac_t *)self;
ehooks_t *ehooks = pac_ehooks_get(pac);
ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
/* Purging of deallocated pages is deferred */
*deferred_work_generated = true;
}
static uint64_t

14
src/pai.c
View File

@@ -2,11 +2,13 @@
#include "jemalloc/internal/jemalloc_internal_includes.h"
size_t
pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results) {
pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool *deferred_work_generated) {
for (size_t i = 0; i < nallocs; i++) {
bool deferred_by_alloc = false;
edata_t *edata = pai_alloc(tsdn, self, size, PAGE,
/* zero */ false);
/* zero */ false, &deferred_by_alloc);
*deferred_work_generated |= deferred_by_alloc;
if (edata == NULL) {
return i;
}
@@ -17,10 +19,12 @@ pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size,
void
pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list) {
edata_list_active_t *list, bool *deferred_work_generated) {
edata_t *edata;
while ((edata = edata_list_active_first(list)) != NULL) {
bool deferred_by_dalloc = false;
edata_list_active_remove(list, edata);
pai_dalloc(tsdn, self, edata);
pai_dalloc(tsdn, self, edata, &deferred_by_dalloc);
*deferred_work_generated |= deferred_by_dalloc;
}
}

49
src/sec.c
View File

@@ -4,12 +4,13 @@
#include "jemalloc/internal/sec.h"
static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero);
size_t alignment, bool zero, bool *deferred_work_generated);
static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero);
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size);
static void sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata);
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated);
static void
sec_bin_init(sec_bin_t *bin) {
@@ -147,7 +148,9 @@ sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
}
malloc_mutex_unlock(tsdn, &shard->mtx);
pai_dalloc_batch(tsdn, sec->fallback, &to_flush);
bool deferred_work_generated;
pai_dalloc_batch(tsdn, sec->fallback, &to_flush,
&deferred_work_generated);
}
static edata_t *
@@ -175,8 +178,9 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
edata_list_active_t result;
edata_list_active_init(&result);
bool deferred_work_generated;
size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size,
1 + sec->opts.batch_fill_extra, &result);
1 + sec->opts.batch_fill_extra, &result, &deferred_work_generated);
edata_t *ret = edata_list_active_first(&result);
if (ret != NULL) {
@@ -213,14 +217,17 @@ sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
}
static edata_t *
sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) {
sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool *deferred_work_generated) {
assert((size & PAGE_MASK) == 0);
sec_t *sec = (sec_t *)self;
*deferred_work_generated = false;
if (zero || alignment > PAGE || sec->opts.nshards == 0
|| size > sec->opts.max_alloc) {
return pai_alloc(tsdn, sec->fallback, size, alignment, zero);
return pai_alloc(tsdn, sec->fallback, size, alignment, zero,
deferred_work_generated);
}
pszind_t pszind = sz_psz2ind(size);
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
@@ -243,7 +250,7 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) {
size);
} else {
edata = pai_alloc(tsdn, sec->fallback, size, alignment,
zero);
zero, deferred_work_generated);
}
}
return edata;
@@ -251,16 +258,18 @@ sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) {
static bool
sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero) {
size_t new_size, bool zero, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero);
return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero,
deferred_work_generated);
}
static bool
sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size) {
size_t new_size, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size);
return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size,
deferred_work_generated);
}
static void
@@ -281,7 +290,9 @@ sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
* we're disabling the HPA or resetting the arena, both of which are
* rare pathways.
*/
pai_dalloc_batch(tsdn, sec->fallback, &to_flush);
bool deferred_work_generated;
pai_dalloc_batch(tsdn, sec->fallback, &to_flush,
&deferred_work_generated);
}
static void
@@ -317,20 +328,24 @@ sec_shard_dalloc_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
}
static void
sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) {
sec_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
if (sec->opts.nshards == 0
|| edata_size_get(edata) > sec->opts.max_alloc) {
pai_dalloc(tsdn, sec->fallback, edata);
pai_dalloc(tsdn, sec->fallback, edata,
deferred_work_generated);
return;
}
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
malloc_mutex_lock(tsdn, &shard->mtx);
if (shard->enabled) {
*deferred_work_generated = false;
sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata);
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
pai_dalloc(tsdn, sec->fallback, edata);
pai_dalloc(tsdn, sec->fallback, edata,
deferred_work_generated);
}
}