lihao/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-24 13:53:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reformat the codebase with the clang-format 18.

Browse Source
This commit is contained in:
guangli-dai
2025-06-13 12:31:12 -07:00
committed by Guangli Dai
parent a952a3b8b0
commit 6200e8987f
346 changed files with 18286 additions and 17770 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/jemalloc/internal/activity_callback.h
View File

@@ -13,9 +13,10 @@
* *
* The calls to this thunk get driven by the peak_event module. * The calls to this thunk get driven by the peak_event module.
*/ */
#define ACTIVITY_CALLBACK_THUNK_INITIALIZER {NULL, NULL} #define ACTIVITY_CALLBACK_THUNK_INITIALIZER \
typedef void (*activity_callback_t)(void *uctx, uint64_t allocated, { NULL, NULL }
uint64_t deallocated); typedef void (*activity_callback_t)(
void *uctx, uint64_t allocated, uint64_t deallocated);
typedef struct activity_callback_thunk_s activity_callback_thunk_t; typedef struct activity_callback_thunk_s activity_callback_thunk_t;
struct activity_callback_thunk_s { struct activity_callback_thunk_s {
activity_callback_t callback; activity_callback_t callback;

70
include/jemalloc/internal/arena_externs.h
View File

@@ -39,28 +39,28 @@ extern pac_thp_t huge_arena_pac_thp;
*/ */
extern uint32_t arena_bin_offsets[SC_NBINS]; extern uint32_t arena_bin_offsets[SC_NBINS];
void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_data_t *bstats, arena_stats_large_t *lstats, bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
pac_estats_t *estats, hpa_shard_stats_t *hpastats, sec_stats_t *secstats); hpa_shard_stats_t *hpastats, sec_stats_t *secstats);
void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena); void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
edata_t *arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, edata_t *arena_extent_alloc_large(
size_t usize, size_t alignment, bool zero); tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
void arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, void arena_extent_dalloc_large_prep(
edata_t *edata); tsdn_t *tsdn, arena_t *arena, edata_t *edata);
void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, void arena_extent_ralloc_large_shrink(
edata_t *edata, size_t oldusize); tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, void arena_extent_ralloc_large_expand(
edata_t *edata, size_t oldusize); tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
bool arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, extent_state_t state, bool arena_decay_ms_set(
ssize_t decay_ms); tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state); ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
void arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, void arena_decay(
bool all); tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena); uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena); void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
void arena_reset(tsd_t *tsd, arena_t *arena); void arena_reset(tsd_t *tsd, arena_t *arena);
@@ -69,20 +69,20 @@ void arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena,
cache_bin_t *cache_bin, szind_t binind, const cache_bin_sz_t nfill_min, cache_bin_t *cache_bin, szind_t binind, const cache_bin_sz_t nfill_min,
const cache_bin_sz_t nfill_max); const cache_bin_sz_t nfill_max);
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
szind_t ind, bool zero, bool slab); bool zero, bool slab);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
size_t alignment, bool zero, bool slab, tcache_t *tcache); bool zero, bool slab, tcache_t *tcache);
void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize, void arena_prof_promote(
size_t bumped_usize); tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, void arena_dalloc_promoted(
bool slow_path); tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab); void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
void arena_dalloc_bin_locked_handle_newly_empty(tsdn_t *tsdn, arena_t *arena, void arena_dalloc_bin_locked_handle_newly_empty(
edata_t *slab, bin_t *bin); tsdn_t *tsdn, arena_t *arena, edata_t *slab, bin_t *bin);
void arena_dalloc_bin_locked_handle_newly_nonempty(tsdn_t *tsdn, arena_t *arena, void arena_dalloc_bin_locked_handle_newly_nonempty(
edata_t *slab, bin_t *bin); tsdn_t *tsdn, arena_t *arena, edata_t *slab, bin_t *bin);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr); void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero, size_t *newsize); size_t extra, bool zero, size_t *newsize);
@@ -91,8 +91,8 @@ void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
hook_ralloc_args_t *hook_args); hook_ralloc_args_t *hook_args);
dss_prec_t arena_dss_prec_get(arena_t *arena); dss_prec_t arena_dss_prec_get(arena_t *arena);
ehooks_t *arena_get_ehooks(arena_t *arena); ehooks_t *arena_get_ehooks(arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(tsd_t *tsd, arena_t *arena, extent_hooks_t *arena_set_extent_hooks(
extent_hooks_t *extent_hooks); tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
void arena_name_get(arena_t *arena, char *name); void arena_name_get(arena_t *arena, char *name);
void arena_name_set(arena_t *arena, const char *name); void arena_name_set(arena_t *arena, const char *name);
@@ -100,16 +100,16 @@ ssize_t arena_dirty_decay_ms_default_get(void);
bool arena_dirty_decay_ms_default_set(ssize_t decay_ms); bool arena_dirty_decay_ms_default_set(ssize_t decay_ms);
ssize_t arena_muzzy_decay_ms_default_get(void); ssize_t arena_muzzy_decay_ms_default_get(void);
bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms); bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, bool arena_retain_grow_limit_get_set(
size_t *old_limit, size_t *new_limit); tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
unsigned arena_nthreads_get(arena_t *arena, bool internal); unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
bool arena_init_huge(tsdn_t *tsdn, arena_t *a0); bool arena_init_huge(tsdn_t *tsdn, arena_t *a0);
arena_t *arena_choose_huge(tsd_t *tsd); arena_t *arena_choose_huge(tsd_t *tsd);
bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, bin_t *arena_bin_choose(
unsigned *binshard); tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned *binshard);
size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind, size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
void **ptrs, size_t nfill, bool zero); void **ptrs, size_t nfill, bool zero);
bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa); bool arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);

226
include/jemalloc/internal/arena_inlines_b.h
View File

@@ -21,8 +21,8 @@
static inline arena_t * static inline arena_t *
arena_get_from_edata(edata_t *edata) { arena_get_from_edata(edata_t *edata) {
return (arena_t *)atomic_load_p(&arenas[edata_arena_ind_get(edata)], return (arena_t *)atomic_load_p(
ATOMIC_RELAXED); &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
} }
JEMALLOC_ALWAYS_INLINE arena_t * JEMALLOC_ALWAYS_INLINE arena_t *
@@ -61,15 +61,17 @@ large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
* The cost is low enough (as edata will be accessed anyway) to be * The cost is low enough (as edata will be accessed anyway) to be
* enabled all the time. * enabled all the time.
*/ */
if (unlikely(edata == NULL || if (unlikely(edata == NULL
edata_state_get(edata) != extent_state_active)) { || edata_state_get(edata) != extent_state_active)) {
safety_check_fail("Invalid deallocation detected: " safety_check_fail(
"Invalid deallocation detected: "
"pages being freed (%p) not currently active, " "pages being freed (%p) not currently active, "
"possibly caused by double free bugs.", ptr); "possibly caused by double free bugs.",
ptr);
return true; return true;
} }
if (unlikely(input_size != edata_usize_get(edata) || if (unlikely(input_size != edata_usize_get(edata)
input_size > SC_LARGE_MAXCLASS)) { || input_size > SC_LARGE_MAXCLASS)) {
safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr, safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
/* true_size */ edata_usize_get(edata), input_size); /* true_size */ edata_usize_get(edata), input_size);
return true; return true;
@@ -90,21 +92,22 @@ arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
/* Static check. */ /* Static check. */
if (alloc_ctx == NULL) { if (alloc_ctx == NULL) {
edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, edata = emap_edata_lookup(
ptr); tsd_tsdn(tsd), &arena_emap_global, ptr);
is_slab = edata_slab_get(edata); is_slab = edata_slab_get(edata);
} else if (unlikely(!(is_slab = alloc_ctx->slab))) { } else if (unlikely(!(is_slab = alloc_ctx->slab))) {
edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, edata = emap_edata_lookup(
ptr); tsd_tsdn(tsd), &arena_emap_global, ptr);
} }
if (unlikely(!is_slab)) { if (unlikely(!is_slab)) {
/* edata must have been initialized at this point. */ /* edata must have been initialized at this point. */
assert(edata != NULL); assert(edata != NULL);
size_t usize = (alloc_ctx == NULL)? edata_usize_get(edata): size_t usize = (alloc_ctx == NULL)
emap_alloc_ctx_usize_get(alloc_ctx); ? edata_usize_get(edata)
if (reset_recent && : emap_alloc_ctx_usize_get(alloc_ctx);
large_dalloc_safety_checks(edata, ptr, usize)) { if (reset_recent
&& large_dalloc_safety_checks(edata, ptr, usize)) {
prof_info->alloc_tctx = PROF_TCTX_SENTINEL; prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
return; return;
} }
@@ -119,22 +122,22 @@ arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_prof_tctx_reset(tsd_t *tsd, const void *ptr, arena_prof_tctx_reset(
emap_alloc_ctx_t *alloc_ctx) { tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
/* Static check. */ /* Static check. */
if (alloc_ctx == NULL) { if (alloc_ctx == NULL) {
edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), edata_t *edata = emap_edata_lookup(
&arena_emap_global, ptr); tsd_tsdn(tsd), &arena_emap_global, ptr);
if (unlikely(!edata_slab_get(edata))) { if (unlikely(!edata_slab_get(edata))) {
large_prof_tctx_reset(edata); large_prof_tctx_reset(edata);
} }
} else { } else {
if (unlikely(!alloc_ctx->slab)) { if (unlikely(!alloc_ctx->slab)) {
edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), edata_t *edata = emap_edata_lookup(
&arena_emap_global, ptr); tsd_tsdn(tsd), &arena_emap_global, ptr);
large_prof_tctx_reset(edata); large_prof_tctx_reset(edata);
} }
} }
@@ -145,16 +148,16 @@ arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
cassert(config_prof); cassert(config_prof);
assert(ptr != NULL); assert(ptr != NULL);
edata_t *edata = emap_edata_lookup(tsd_tsdn(tsd), &arena_emap_global, edata_t *edata = emap_edata_lookup(
ptr); tsd_tsdn(tsd), &arena_emap_global, ptr);
assert(!edata_slab_get(edata)); assert(!edata_slab_get(edata));
large_prof_tctx_reset(edata); large_prof_tctx_reset(edata);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_prof_info_set(tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, arena_prof_info_set(
size_t size) { tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
cassert(config_prof); cassert(config_prof);
assert(!edata_slab_get(edata)); assert(!edata_slab_get(edata));
@@ -197,14 +200,13 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
if (likely(tcache != NULL)) { if (likely(tcache != NULL)) {
if (likely(slab)) { if (likely(slab)) {
assert(sz_can_use_slab(size)); assert(sz_can_use_slab(size));
return tcache_alloc_small(tsdn_tsd(tsdn), arena, return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
tcache, size, ind, zero, slow_path); size, ind, zero, slow_path);
} else if (likely( } else if (likely(ind < tcache_nbins_get(tcache->tcache_slow)
ind < tcache_nbins_get(tcache->tcache_slow) && && !tcache_bin_disabled(ind, &tcache->bins[ind],
!tcache_bin_disabled(ind, &tcache->bins[ind],
tcache->tcache_slow))) { tcache->tcache_slow))) {
return tcache_alloc_large(tsdn_tsd(tsdn), arena, return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
tcache, size, ind, zero, slow_path); size, ind, zero, slow_path);
} }
/* (size > tcache_max) case falls through. */ /* (size > tcache_max) case falls through. */
} }
@@ -241,8 +243,8 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
*/ */
emap_full_alloc_ctx_t full_alloc_ctx; emap_full_alloc_ctx_t full_alloc_ctx;
bool missing = emap_full_alloc_ctx_try_lookup(tsdn, &arena_emap_global, bool missing = emap_full_alloc_ctx_try_lookup(
ptr, &full_alloc_ctx); tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
if (missing) { if (missing) {
return 0; return 0;
} }
@@ -261,8 +263,8 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
} }
static inline void static inline void
arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind, arena_dalloc_large_no_tcache(
size_t usize) { tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
/* /*
* szind is still needed in this function mainly becuase * szind is still needed in this function mainly becuase
* szind < SC_NBINS determines not only if this is a small alloc, * szind < SC_NBINS determines not only if this is a small alloc,
@@ -272,8 +274,8 @@ arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind,
if (config_prof && unlikely(szind < SC_NBINS)) { if (config_prof && unlikely(szind < SC_NBINS)) {
arena_dalloc_promoted(tsdn, ptr, NULL, true); arena_dalloc_promoted(tsdn, ptr, NULL, true);
} else { } else {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, edata_t *edata = emap_edata_lookup(
ptr); tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) { if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */ /* See the comment in isfree. */
return; return;
@@ -290,13 +292,13 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx); emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
if (config_debug) { if (config_debug) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, edata_t *edata = emap_edata_lookup(
ptr); tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES); assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata)); assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == assert(emap_alloc_ctx_usize_get(&alloc_ctx)
edata_usize_get(edata)); == edata_usize_get(edata));
} }
if (likely(alloc_ctx.slab)) { if (likely(alloc_ctx.slab)) {
@@ -311,19 +313,19 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind, arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
size_t usize, bool slow_path) { size_t usize, bool slow_path) {
assert (!tsdn_null(tsdn) && tcache != NULL); assert(!tsdn_null(tsdn) && tcache != NULL);
bool is_sample_promoted = config_prof && szind < SC_NBINS; bool is_sample_promoted = config_prof && szind < SC_NBINS;
if (unlikely(is_sample_promoted)) { if (unlikely(is_sample_promoted)) {
arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
} else { } else {
if (szind < tcache_nbins_get(tcache->tcache_slow) && if (szind < tcache_nbins_get(tcache->tcache_slow)
!tcache_bin_disabled(szind, &tcache->bins[szind], && !tcache_bin_disabled(
tcache->tcache_slow)) { szind, &tcache->bins[szind], tcache->tcache_slow)) {
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind, tcache_dalloc_large(
slow_path); tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
} else { } else {
edata_t *edata = emap_edata_lookup(tsdn, edata_t *edata = emap_edata_lookup(
&arena_emap_global, ptr); tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) { if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */ /* See the comment in isfree. */
return; return;
@@ -335,16 +337,17 @@ arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
/* Find the region index of a pointer. */ /* Find the region index of a pointer. */
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
arena_slab_regind_impl(div_info_t* div_info, szind_t binind, arena_slab_regind_impl(
edata_t *slab, const void *ptr) { div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) {
size_t diff, regind; size_t diff, regind;
/* Freeing a pointer outside the slab can cause assertion failure. */ /* Freeing a pointer outside the slab can cause assertion failure. */
assert((uintptr_t)ptr >= (uintptr_t)edata_addr_get(slab)); assert((uintptr_t)ptr >= (uintptr_t)edata_addr_get(slab));
assert((uintptr_t)ptr < (uintptr_t)edata_past_get(slab)); assert((uintptr_t)ptr < (uintptr_t)edata_past_get(slab));
/* Freeing an interior pointer can cause assertion failure. */ /* Freeing an interior pointer can cause assertion failure. */
assert(((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab)) % assert(((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab))
(uintptr_t)bin_infos[binind].reg_size == 0); % (uintptr_t)bin_infos[binind].reg_size
== 0);
diff = (size_t)((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab)); diff = (size_t)((uintptr_t)ptr - (uintptr_t)edata_addr_get(slab));
@@ -371,11 +374,12 @@ arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
slab_data_t *slab_data = edata_slab_data_get(edata); slab_data_t *slab_data = edata_slab_data_get(edata);
const bin_info_t *bin_info = &bin_infos[binind]; const bin_info_t *bin_info = &bin_infos[binind];
assert(edata_nfree_get(edata) < bin_info->nregs); assert(edata_nfree_get(edata) < bin_info->nregs);
if (unlikely(!bitmap_get(slab_data->bitmap, &bin_info->bitmap_info, if (unlikely(!bitmap_get(
regind))) { slab_data->bitmap, &bin_info->bitmap_info, regind))) {
safety_check_fail( safety_check_fail(
"Invalid deallocation detected: the pointer being freed (%p) not " "Invalid deallocation detected: the pointer being freed (%p) not "
"currently active, possibly caused by double free bugs.\n", ptr); "currently active, possibly caused by double free bugs.\n",
ptr);
return true; return true;
} }
return false; return false;
@@ -397,18 +401,18 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
alloc_ctx = *caller_alloc_ctx; alloc_ctx = *caller_alloc_ctx;
} else { } else {
util_assume(tsdn != NULL); util_assume(tsdn != NULL);
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, emap_alloc_ctx_lookup(
&alloc_ctx); tsdn, &arena_emap_global, ptr, &alloc_ctx);
} }
if (config_debug) { if (config_debug) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, edata_t *edata = emap_edata_lookup(
ptr); tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES); assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata)); assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == assert(emap_alloc_ctx_usize_get(&alloc_ctx)
edata_usize_get(edata)); == edata_usize_get(edata));
} }
if (likely(alloc_ctx.slab)) { if (likely(alloc_ctx.slab)) {
@@ -416,8 +420,8 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) { if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
return; return;
} }
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, tcache_dalloc_small(
alloc_ctx.szind, slow_path); tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else { } else {
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind, arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx), slow_path); emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
@@ -436,21 +440,21 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
* object, so base szind and slab on the given size. * object, so base szind and slab on the given size.
*/ */
szind_t szind = sz_size2index(size); szind_t szind = sz_size2index(size);
emap_alloc_ctx_init(&alloc_ctx, szind, (szind < SC_NBINS), emap_alloc_ctx_init(
size); &alloc_ctx, szind, (szind < SC_NBINS), size);
} }
if ((config_prof && opt_prof) || config_debug) { if ((config_prof && opt_prof) || config_debug) {
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, emap_alloc_ctx_lookup(
&alloc_ctx); tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size)); assert(alloc_ctx.szind == sz_size2index(size));
assert((config_prof && opt_prof) assert((config_prof && opt_prof)
|| alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS)); || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
if (config_debug) { if (config_debug) {
edata_t *edata = emap_edata_lookup(tsdn, edata_t *edata = emap_edata_lookup(
&arena_emap_global, ptr); tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata)); assert(alloc_ctx.slab == edata_slab_get(edata));
} }
@@ -481,8 +485,8 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
if (caller_alloc_ctx == NULL) { if (caller_alloc_ctx == NULL) {
/* Uncommon case and should be a static check. */ /* Uncommon case and should be a static check. */
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, emap_alloc_ctx_lookup(
&alloc_ctx); tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size)); assert(alloc_ctx.szind == sz_size2index(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size); assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
} else { } else {
@@ -498,14 +502,14 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
} }
if (config_debug) { if (config_debug) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, edata_t *edata = emap_edata_lookup(
ptr); tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata)); assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata)); assert(alloc_ctx.slab == edata_slab_get(edata));
emap_alloc_ctx_init(&alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, emap_alloc_ctx_init(
sz_s2u(size)); &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == assert(emap_alloc_ctx_usize_get(&alloc_ctx)
edata_usize_get(edata)); == edata_usize_get(edata));
} }
if (likely(alloc_ctx.slab)) { if (likely(alloc_ctx.slab)) {
@@ -513,8 +517,8 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) { if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
return; return;
} }
tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, tcache_dalloc_small(
alloc_ctx.szind, slow_path); tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else { } else {
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind, arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
sz_s2u(size), slow_path); sz_s2u(size), slow_path);
@@ -522,13 +526,13 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
} }
static inline void static inline void
arena_cache_oblivious_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata, arena_cache_oblivious_randomize(
size_t alignment) { tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
assert(edata_base_get(edata) == edata_addr_get(edata)); assert(edata_base_get(edata) == edata_addr_get(edata));
if (alignment < PAGE) { if (alignment < PAGE) {
unsigned lg_range = LG_PAGE - unsigned lg_range = LG_PAGE
lg_floor(CACHELINE_CEILING(alignment)); - lg_floor(CACHELINE_CEILING(alignment));
size_t r; size_t r;
if (!tsdn_null(tsdn)) { if (!tsdn_null(tsdn)) {
tsd_t *tsd = tsdn_tsd(tsdn); tsd_t *tsd = tsdn_tsd(tsdn);
@@ -538,12 +542,12 @@ arena_cache_oblivious_randomize(tsdn_t *tsdn, arena_t *arena, edata_t *edata,
uint64_t stack_value = (uint64_t)(uintptr_t)&r; uint64_t stack_value = (uint64_t)(uintptr_t)&r;
r = (size_t)prng_lg_range_u64(&stack_value, lg_range); r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
} }
uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE - uintptr_t random_offset = ((uintptr_t)r)
lg_range); << (LG_PAGE - lg_range);
edata->e_addr = (void *)((byte_t *)edata->e_addr + edata->e_addr = (void *)((byte_t *)edata->e_addr
random_offset); + random_offset);
assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment) == assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
edata->e_addr); == edata->e_addr);
} }
} }
@@ -563,13 +567,14 @@ struct arena_dalloc_bin_locked_info_s {
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
arena_slab_regind(arena_dalloc_bin_locked_info_t *info, szind_t binind, arena_slab_regind(arena_dalloc_bin_locked_info_t *info, szind_t binind,
edata_t *slab, const void *ptr) { edata_t *slab, const void *ptr) {
size_t regind = arena_slab_regind_impl(&info->div_info, binind, slab, ptr); size_t regind = arena_slab_regind_impl(
&info->div_info, binind, slab, ptr);
return regind; return regind;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
arena_dalloc_bin_locked_begin(arena_dalloc_bin_locked_info_t *info, arena_dalloc_bin_locked_begin(
szind_t binind) { arena_dalloc_bin_locked_info_t *info, szind_t binind) {
info->div_info = arena_binind_div_info[binind]; info->div_info = arena_binind_div_info[binind];
info->nregs = bin_infos[binind].nregs; info->nregs = bin_infos[binind].nregs;
info->ndalloc = 0; info->ndalloc = 0;
@@ -605,8 +610,8 @@ arena_dalloc_bin_locked_step(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
unsigned nfree = edata_nfree_get(slab); unsigned nfree = edata_nfree_get(slab);
if (nfree == bin_info->nregs) { if (nfree == bin_info->nregs) {
arena_dalloc_bin_locked_handle_newly_empty(tsdn, arena, slab, arena_dalloc_bin_locked_handle_newly_empty(
bin); tsdn, arena, slab, bin);
if (*dalloc_slabs_count < ndalloc_slabs) { if (*dalloc_slabs_count < ndalloc_slabs) {
dalloc_slabs[*dalloc_slabs_count] = slab; dalloc_slabs[*dalloc_slabs_count] = slab;
@@ -615,8 +620,8 @@ arena_dalloc_bin_locked_step(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
edata_list_active_append(dalloc_slabs_extra, slab); edata_list_active_append(dalloc_slabs_extra, slab);
} }
} else if (nfree == 1 && slab != bin->slabcur) { } else if (nfree == 1 && slab != bin->slabcur) {
arena_dalloc_bin_locked_handle_newly_nonempty(tsdn, arena, slab, arena_dalloc_bin_locked_handle_newly_nonempty(
bin); tsdn, arena, slab, bin);
} }
} }
@@ -637,21 +642,20 @@ arena_bin_flush_batch_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
edata_list_active_t *dalloc_slabs_extra) { edata_list_active_t *dalloc_slabs_extra) {
assert(binind < bin_info_nbatched_sizes); assert(binind < bin_info_nbatched_sizes);
bin_with_batch_t *batched_bin = (bin_with_batch_t *)bin; bin_with_batch_t *batched_bin = (bin_with_batch_t *)bin;
size_t nelems_to_pop = batcher_pop_begin(tsdn, size_t nelems_to_pop = batcher_pop_begin(
&batched_bin->remote_frees); tsdn, &batched_bin->remote_frees);
bin_batching_test_mid_pop(nelems_to_pop); bin_batching_test_mid_pop(nelems_to_pop);
if (nelems_to_pop == BATCHER_NO_IDX) { if (nelems_to_pop == BATCHER_NO_IDX) {
malloc_mutex_assert_not_owner(tsdn, malloc_mutex_assert_not_owner(
&batched_bin->remote_frees.mtx); tsdn, &batched_bin->remote_frees.mtx);
return; return;
} else { } else {
malloc_mutex_assert_owner(tsdn, malloc_mutex_assert_owner(tsdn, &batched_bin->remote_frees.mtx);
&batched_bin->remote_frees.mtx);
} }
size_t npushes = batcher_pop_get_pushes(tsdn, size_t npushes = batcher_pop_get_pushes(
&batched_bin->remote_frees); tsdn, &batched_bin->remote_frees);
bin_remote_free_data_t remote_free_data[BIN_REMOTE_FREE_ELEMS_MAX]; bin_remote_free_data_t remote_free_data[BIN_REMOTE_FREE_ELEMS_MAX];
for (size_t i = 0; i < nelems_to_pop; i++) { for (size_t i = 0; i < nelems_to_pop; i++) {
remote_free_data[i] = batched_bin->remote_free_data[i]; remote_free_data[i] = batched_bin->remote_free_data[i];
@@ -712,8 +716,8 @@ arena_bin_flush_batch_after_lock(tsdn_t *tsdn, arena_t *arena, bin_t *bin,
preallocated_slabs); preallocated_slabs);
arena_bin_flush_batch_impl(tsdn, arena, bin, &state->info, binind, arena_bin_flush_batch_impl(tsdn, arena, bin, &state->info, binind,
state->dalloc_slabs, ndalloc_slabs, state->dalloc_slabs, ndalloc_slabs, &state->dalloc_slab_count,
&state->dalloc_slab_count, &state->dalloc_slabs_extra); &state->dalloc_slabs_extra);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
@@ -769,8 +773,8 @@ arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
ret = shard0 + binshard; ret = shard0 + binshard;
} }
assert(binind >= SC_NBINS - 1 assert(binind >= SC_NBINS - 1
|| (uintptr_t)ret < (uintptr_t)arena || (uintptr_t)ret
+ arena_bin_offsets[binind + 1]); < (uintptr_t)arena + arena_bin_offsets[binind + 1]);
return ret; return ret;
} }

4
include/jemalloc/internal/arena_stats.h
View File

@@ -115,8 +115,8 @@ arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats,
arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS]; arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS];
locked_inc_u64(tsdn, LOCKEDINT_MTX(arena_stats->mtx), locked_inc_u64(tsdn, LOCKEDINT_MTX(arena_stats->mtx),
&lstats->nrequests, nrequests); &lstats->nrequests, nrequests);
locked_inc_u64(tsdn, LOCKEDINT_MTX(arena_stats->mtx), locked_inc_u64(
&lstats->nflushes, 1); tsdn, LOCKEDINT_MTX(arena_stats->mtx), &lstats->nflushes, 1);
LOCKEDINT_MTX_UNLOCK(tsdn, arena_stats->mtx); LOCKEDINT_MTX_UNLOCK(tsdn, arena_stats->mtx);
} }

3
include/jemalloc/internal/arena_structs.h
View File

@@ -101,7 +101,8 @@ struct arena_s {
* Enforcing cacheline-alignment to minimize the number of cachelines * Enforcing cacheline-alignment to minimize the number of cachelines
* touched on the hot paths. * touched on the hot paths.
*/ */
JEMALLOC_WARN_ON_USAGE("Do not use this field directly. " JEMALLOC_WARN_ON_USAGE(
"Do not use this field directly. "
"Use `arena_get_bin` instead.") "Use `arena_get_bin` instead.")
JEMALLOC_ALIGNED(CACHELINE) JEMALLOC_ALIGNED(CACHELINE)
bin_with_batch_t all_bins[0]; bin_with_batch_t all_bins[0];

28
include/jemalloc/internal/assert.h
View File

@@ -7,18 +7,20 @@
* assertion failure. * assertion failure.
*/ */
#ifndef assert #ifndef assert
#define assert(e) do { \ # define assert(e) \
do { \
if (unlikely(config_debug && !(e))) { \ if (unlikely(config_debug && !(e))) { \
malloc_printf( \ malloc_printf( \
"<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \ "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n", \
__FILE__, __LINE__, #e); \ __FILE__, __LINE__, #e); \
abort(); \ abort(); \
} \ } \
} while (0) } while (0)
#endif #endif
#ifndef not_reached #ifndef not_reached
#define not_reached() do { \ # define not_reached() \
do { \
if (config_debug) { \ if (config_debug) { \
malloc_printf( \ malloc_printf( \
"<jemalloc>: %s:%d: Unreachable code reached\n", \ "<jemalloc>: %s:%d: Unreachable code reached\n", \
@@ -26,32 +28,36 @@
abort(); \ abort(); \
} \ } \
unreachable(); \ unreachable(); \
} while (0) } while (0)
#endif #endif
#ifndef not_implemented #ifndef not_implemented
#define not_implemented() do { \ # define not_implemented() \
do { \
if (config_debug) { \ if (config_debug) { \
malloc_printf("<jemalloc>: %s:%d: Not implemented\n", \ malloc_printf( \
"<jemalloc>: %s:%d: Not implemented\n", \
__FILE__, __LINE__); \ __FILE__, __LINE__); \
abort(); \ abort(); \
} \ } \
} while (0) } while (0)
#endif #endif
#ifndef assert_not_implemented #ifndef assert_not_implemented
#define assert_not_implemented(e) do { \ # define assert_not_implemented(e) \
do { \
if (unlikely(config_debug && !(e))) { \ if (unlikely(config_debug && !(e))) { \
not_implemented(); \ not_implemented(); \
} \ } \
} while (0) } while (0)
#endif #endif
/* Use to assert a particular configuration, e.g., cassert(config_debug). */ /* Use to assert a particular configuration, e.g., cassert(config_debug). */
#ifndef cassert #ifndef cassert
#define cassert(c) do { \ # define cassert(c) \
do { \
if (unlikely(!(c))) { \ if (unlikely(!(c))) { \
not_reached(); \ not_reached(); \
} \ } \
} while (0) } while (0)
#endif #endif

13
include/jemalloc/internal/atomic.h
View File

@@ -56,19 +56,16 @@
/* /*
* Another convenience -- simple atomic helper functions. * Another convenience -- simple atomic helper functions.
*/ */
#define JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(type, short_type, \ #define JEMALLOC_GENERATE_EXPANDED_INT_ATOMICS(type, short_type, lg_size) \
lg_size) \
JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \ JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \
ATOMIC_INLINE void \ ATOMIC_INLINE void atomic_load_add_store_##short_type( \
atomic_load_add_store_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type inc) { \
type inc) { \
type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \ type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \
type newval = oldval + inc; \ type newval = oldval + inc; \
atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \ atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \
} \ } \
ATOMIC_INLINE void \ ATOMIC_INLINE void atomic_load_sub_store_##short_type( \
atomic_load_sub_store_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type inc) { \
type inc) { \
type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \ type oldval = atomic_load_##short_type(a, ATOMIC_RELAXED); \
type newval = oldval - inc; \ type newval = oldval - inc; \
atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \ atomic_store_##short_type(a, newval, ATOMIC_RELAXED); \

40
include/jemalloc/internal/atomic_c11.h
View File

@@ -66,35 +66,29 @@ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
* Integral types have some special operations available that non-integral ones * Integral types have some special operations available that non-integral ones
* lack. * lack.
*/ */
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \ #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size) \
/* unused */ lg_size) \ JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_add_##short_type( \
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_add_explicit(a, val, mo); \ return atomic_fetch_add_explicit(a, val, mo); \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_sub_##short_type( \
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_sub_explicit(a, val, mo); \ return atomic_fetch_sub_explicit(a, val, mo); \
} \ } \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_and_##short_type( \
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_and_explicit(a, val, mo); \ return atomic_fetch_and_explicit(a, val, mo); \
} \ } \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_or_##short_type( \
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_or_explicit(a, val, mo); \ return atomic_fetch_or_explicit(a, val, mo); \
} \ } \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_xor_##short_type( \
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
type val, atomic_memory_order_t mo) { \
return atomic_fetch_xor_explicit(a, val, mo); \ return atomic_fetch_xor_explicit(a, val, mo); \
} }
#endif /* JEMALLOC_INTERNAL_ATOMIC_C11_H */ #endif /* JEMALLOC_INTERNAL_ATOMIC_C11_H */

113
include/jemalloc/internal/atomic_gcc_atomic.h
View File

@@ -6,7 +6,8 @@
#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE #define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
#define ATOMIC_INIT(...) {__VA_ARGS__} #define ATOMIC_INIT(...) \
{ __VA_ARGS__ }
typedef enum { typedef enum {
atomic_memory_order_relaxed, atomic_memory_order_relaxed,
@@ -39,95 +40,81 @@ atomic_fence(atomic_memory_order_t mo) {
__atomic_thread_fence(atomic_enum_to_builtin(mo)); __atomic_thread_fence(atomic_enum_to_builtin(mo));
} }
#define JEMALLOC_GENERATE_ATOMICS(type, short_type, \ #define JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
/* unused */ lg_size) \ typedef struct { \
typedef struct { \
type repr; \ type repr; \
} atomic_##short_type##_t; \ } atomic_##short_type##_t; \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_load_##short_type( \
atomic_load_##short_type(const atomic_##short_type##_t *a, \ const atomic_##short_type##_t *a, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
type result; \ type result; \
__atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo)); \ __atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo)); \
return result; \ return result; \
} \ } \
\ \
ATOMIC_INLINE void \ ATOMIC_INLINE void atomic_store_##short_type( \
atomic_store_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
__atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo)); \ __atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo)); \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_exchange_##short_type( \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
type result; \ type result; \
__atomic_exchange(&a->repr, &val, &result, \ __atomic_exchange( \
atomic_enum_to_builtin(mo)); \ &a->repr, &val, &result, atomic_enum_to_builtin(mo)); \
return result; \ return result; \
} \ } \
\ \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool atomic_compare_exchange_weak_##short_type( \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, UNUSED type *expected, type desired, \
UNUSED type *expected, type desired, \
atomic_memory_order_t success_mo, \ atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \ return __atomic_compare_exchange(&a->repr, expected, &desired, \
true, atomic_enum_to_builtin(success_mo), \ true, atomic_enum_to_builtin(success_mo), \
atomic_enum_to_builtin(failure_mo)); \ atomic_enum_to_builtin(failure_mo)); \
} \ } \
\ \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool atomic_compare_exchange_strong_##short_type( \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, UNUSED type *expected, type desired, \
UNUSED type *expected, type desired, \
atomic_memory_order_t success_mo, \ atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t failure_mo) { \
return __atomic_compare_exchange(&a->repr, expected, &desired, \ return __atomic_compare_exchange(&a->repr, expected, &desired, \
false, \ false, atomic_enum_to_builtin(success_mo), \
atomic_enum_to_builtin(success_mo), \
atomic_enum_to_builtin(failure_mo)); \ atomic_enum_to_builtin(failure_mo)); \
} }
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size) \
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \ JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
/* unused */ lg_size) \
JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_add_##short_type( \
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \ return __atomic_fetch_add( \
return __atomic_fetch_add(&a->repr, val, \ &a->repr, val, atomic_enum_to_builtin(mo)); \
atomic_enum_to_builtin(mo)); \ } \
} \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_sub_##short_type( \
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \ return __atomic_fetch_sub( \
return __atomic_fetch_sub(&a->repr, val, \ &a->repr, val, atomic_enum_to_builtin(mo)); \
atomic_enum_to_builtin(mo)); \ } \
} \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_and_##short_type( \
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \ return __atomic_fetch_and( \
return __atomic_fetch_and(&a->repr, val, \ &a->repr, val, atomic_enum_to_builtin(mo)); \
atomic_enum_to_builtin(mo)); \ } \
} \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_or_##short_type( \
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \ return __atomic_fetch_or( \
return __atomic_fetch_or(&a->repr, val, \ &a->repr, val, atomic_enum_to_builtin(mo)); \
atomic_enum_to_builtin(mo)); \ } \
} \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_xor_##short_type( \
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \ return __atomic_fetch_xor( \
return __atomic_fetch_xor(&a->repr, val, \ &a->repr, val, atomic_enum_to_builtin(mo)); \
atomic_enum_to_builtin(mo)); \ }
}
#undef ATOMIC_INLINE #undef ATOMIC_INLINE

72
include/jemalloc/internal/atomic_gcc_sync.h
View File

@@ -5,7 +5,8 @@
#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE #define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
#define ATOMIC_INIT(...) {__VA_ARGS__} #define ATOMIC_INIT(...) \
{ __VA_ARGS__ }
typedef enum { typedef enum {
atomic_memory_order_relaxed, atomic_memory_order_relaxed,
@@ -29,13 +30,13 @@ atomic_fence(atomic_memory_order_t mo) {
return; return;
} }
asm volatile("" ::: "memory"); asm volatile("" ::: "memory");
# if defined(__i386__) || defined(__x86_64__) #if defined(__i386__) || defined(__x86_64__)
/* This is implicit on x86. */ /* This is implicit on x86. */
# elif defined(__ppc64__) #elif defined(__ppc64__)
asm volatile("lwsync"); asm volatile("lwsync");
# elif defined(__ppc__) #elif defined(__ppc__)
asm volatile("sync"); asm volatile("sync");
# elif defined(__sparc__) && defined(__arch64__) #elif defined(__sparc__) && defined(__arch64__)
if (mo == atomic_memory_order_acquire) { if (mo == atomic_memory_order_acquire) {
asm volatile("membar #LoadLoad | #LoadStore"); asm volatile("membar #LoadLoad | #LoadStore");
} else if (mo == atomic_memory_order_release) { } else if (mo == atomic_memory_order_release) {
@@ -43,9 +44,9 @@ atomic_fence(atomic_memory_order_t mo) {
} else { } else {
asm volatile("membar #LoadLoad | #LoadStore | #StoreStore"); asm volatile("membar #LoadLoad | #LoadStore | #StoreStore");
} }
# else #else
__sync_synchronize(); __sync_synchronize();
# endif #endif
asm volatile("" ::: "memory"); asm volatile("" ::: "memory");
} }
@@ -68,23 +69,22 @@ atomic_fence(atomic_memory_order_t mo) {
ATOMIC_INLINE void ATOMIC_INLINE void
atomic_pre_sc_load_fence() { atomic_pre_sc_load_fence() {
# if defined(__i386__) || defined(__x86_64__) || \ #if defined(__i386__) || defined(__x86_64__) \
(defined(__sparc__) && defined(__arch64__)) || (defined(__sparc__) && defined(__arch64__))
atomic_fence(atomic_memory_order_relaxed); atomic_fence(atomic_memory_order_relaxed);
# else #else
atomic_fence(atomic_memory_order_seq_cst); atomic_fence(atomic_memory_order_seq_cst);
# endif #endif
} }
ATOMIC_INLINE void ATOMIC_INLINE void
atomic_post_sc_store_fence() { atomic_post_sc_store_fence() {
# if defined(__i386__) || defined(__x86_64__) || \ #if defined(__i386__) || defined(__x86_64__) \
(defined(__sparc__) && defined(__arch64__)) || (defined(__sparc__) && defined(__arch64__))
atomic_fence(atomic_memory_order_seq_cst); atomic_fence(atomic_memory_order_seq_cst);
# else #else
atomic_fence(atomic_memory_order_relaxed); atomic_fence(atomic_memory_order_relaxed);
# endif #endif
} }
/* clang-format off */ /* clang-format off */
@@ -164,39 +164,33 @@ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \
} }
/* clang-format on */ /* clang-format on */
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, \ #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, /* unused */ lg_size) \
/* unused */ lg_size) \ JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_add_##short_type( \
atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
return __sync_fetch_and_add(&a->repr, val); \ return __sync_fetch_and_add(&a->repr, val); \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_sub_##short_type( \
atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
return __sync_fetch_and_sub(&a->repr, val); \ return __sync_fetch_and_sub(&a->repr, val); \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_and_##short_type( \
atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
return __sync_fetch_and_and(&a->repr, val); \ return __sync_fetch_and_and(&a->repr, val); \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_or_##short_type( \
atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
return __sync_fetch_and_or(&a->repr, val); \ return __sync_fetch_and_or(&a->repr, val); \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_fetch_xor_##short_type( \
atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
return __sync_fetch_and_xor(&a->repr, val); \ return __sync_fetch_and_xor(&a->repr, val); \
} }
#undef ATOMIC_INLINE #undef ATOMIC_INLINE

83
include/jemalloc/internal/atomic_msvc.h
View File

@@ -5,7 +5,8 @@
#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE #define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE
#define ATOMIC_INIT(...) {__VA_ARGS__} #define ATOMIC_INIT(...) \
{ __VA_ARGS__ }
typedef enum { typedef enum {
atomic_memory_order_relaxed, atomic_memory_order_relaxed,
@@ -23,29 +24,29 @@ typedef __int64 atomic_repr_3_t;
ATOMIC_INLINE void ATOMIC_INLINE void
atomic_fence(atomic_memory_order_t mo) { atomic_fence(atomic_memory_order_t mo) {
_ReadWriteBarrier(); _ReadWriteBarrier();
# if defined(_M_ARM) || defined(_M_ARM64) #if defined(_M_ARM) || defined(_M_ARM64)
/* ARM needs a barrier for everything but relaxed. */ /* ARM needs a barrier for everything but relaxed. */
if (mo != atomic_memory_order_relaxed) { if (mo != atomic_memory_order_relaxed) {
MemoryBarrier(); MemoryBarrier();
} }
# elif defined(_M_IX86) || defined (_M_X64) #elif defined(_M_IX86) || defined(_M_X64)
/* x86 needs a barrier only for seq_cst. */ /* x86 needs a barrier only for seq_cst. */
if (mo == atomic_memory_order_seq_cst) { if (mo == atomic_memory_order_seq_cst) {
MemoryBarrier(); MemoryBarrier();
} }
# else #else
# error "Don't know how to create atomics for this platform for MSVC." # error "Don't know how to create atomics for this platform for MSVC."
# endif #endif
_ReadWriteBarrier(); _ReadWriteBarrier();
} }
#define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_ ## lg_size ## _t #define ATOMIC_INTERLOCKED_REPR(lg_size) atomic_repr_##lg_size##_t
#define ATOMIC_CONCAT(a, b) ATOMIC_RAW_CONCAT(a, b) #define ATOMIC_CONCAT(a, b) ATOMIC_RAW_CONCAT(a, b)
#define ATOMIC_RAW_CONCAT(a, b) a ## b #define ATOMIC_RAW_CONCAT(a, b) a##b
#define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) ATOMIC_CONCAT( \ #define ATOMIC_INTERLOCKED_NAME(base_name, lg_size) \
base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size)) ATOMIC_CONCAT(base_name, ATOMIC_INTERLOCKED_SUFFIX(lg_size))
#define ATOMIC_INTERLOCKED_SUFFIX(lg_size) \ #define ATOMIC_INTERLOCKED_SUFFIX(lg_size) \
ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size) ATOMIC_CONCAT(ATOMIC_INTERLOCKED_SUFFIX_, lg_size)
@@ -56,67 +57,63 @@ atomic_fence(atomic_memory_order_t mo) {
#define ATOMIC_INTERLOCKED_SUFFIX_3 64 #define ATOMIC_INTERLOCKED_SUFFIX_3 64
#define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \ #define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \
typedef struct { \ typedef struct { \
ATOMIC_INTERLOCKED_REPR(lg_size) repr; \ ATOMIC_INTERLOCKED_REPR(lg_size) repr; \
} atomic_##short_type##_t; \ } atomic_##short_type##_t; \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_load_##short_type( \
atomic_load_##short_type(const atomic_##short_type##_t *a, \ const atomic_##short_type##_t *a, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr; \ ATOMIC_INTERLOCKED_REPR(lg_size) ret = a->repr; \
if (mo != atomic_memory_order_relaxed) { \ if (mo != atomic_memory_order_relaxed) { \
atomic_fence(atomic_memory_order_acquire); \ atomic_fence(atomic_memory_order_acquire); \
} \ } \
return (type) ret; \ return (type)ret; \
} \ } \
\ \
ATOMIC_INLINE void \ ATOMIC_INLINE void atomic_store_##short_type( \
atomic_store_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
type val, atomic_memory_order_t mo) { \
if (mo != atomic_memory_order_relaxed) { \ if (mo != atomic_memory_order_relaxed) { \
atomic_fence(atomic_memory_order_release); \ atomic_fence(atomic_memory_order_release); \
} \ } \
a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size)) val; \ a->repr = (ATOMIC_INTERLOCKED_REPR(lg_size))val; \
if (mo == atomic_memory_order_seq_cst) { \ if (mo == atomic_memory_order_seq_cst) { \
atomic_fence(atomic_memory_order_seq_cst); \ atomic_fence(atomic_memory_order_seq_cst); \
} \ } \
} \ } \
\ \
ATOMIC_INLINE type \ ATOMIC_INLINE type atomic_exchange_##short_type( \
atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ atomic_##short_type##_t *a, type val, atomic_memory_order_t mo) { \
atomic_memory_order_t mo) { \
return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange, \ return (type)ATOMIC_INTERLOCKED_NAME(_InterlockedExchange, \
lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \ lg_size)(&a->repr, (ATOMIC_INTERLOCKED_REPR(lg_size))val); \
} \ } \
\ \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool atomic_compare_exchange_weak_##short_type( \
atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type *expected, type desired, \
type *expected, type desired, atomic_memory_order_t success_mo, \ atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t failure_mo) { \
ATOMIC_INTERLOCKED_REPR(lg_size) e = \ ATOMIC_INTERLOCKED_REPR(lg_size) \
(ATOMIC_INTERLOCKED_REPR(lg_size))*expected; \ e = (ATOMIC_INTERLOCKED_REPR(lg_size)) * expected; \
ATOMIC_INTERLOCKED_REPR(lg_size) d = \ ATOMIC_INTERLOCKED_REPR(lg_size) \
(ATOMIC_INTERLOCKED_REPR(lg_size))desired; \ d = (ATOMIC_INTERLOCKED_REPR(lg_size))desired; \
ATOMIC_INTERLOCKED_REPR(lg_size) old = \ ATOMIC_INTERLOCKED_REPR(lg_size) \
ATOMIC_INTERLOCKED_NAME(_InterlockedCompareExchange, \ old = ATOMIC_INTERLOCKED_NAME( \
lg_size)(&a->repr, d, e); \ _InterlockedCompareExchange, lg_size)(&a->repr, d, e); \
if (old == e) { \ if (old == e) { \
return true; \ return true; \
} else { \ } else { \
*expected = (type)old; \ *expected = (type)old; \
return false; \ return false; \
} \ } \
} \ } \
\ \
ATOMIC_INLINE bool \ ATOMIC_INLINE bool atomic_compare_exchange_strong_##short_type( \
atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ atomic_##short_type##_t *a, type *expected, type desired, \
type *expected, type desired, atomic_memory_order_t success_mo, \ atomic_memory_order_t success_mo, \
atomic_memory_order_t failure_mo) { \ atomic_memory_order_t failure_mo) { \
/* We implement the weak version with strong semantics. */ \ /* We implement the weak version with strong semantics. */ \
return atomic_compare_exchange_weak_##short_type(a, expected, \ return atomic_compare_exchange_weak_##short_type( \
desired, success_mo, failure_mo); \ a, expected, desired, success_mo, failure_mo); \
} }
/* clang-format off */ /* clang-format off */
#define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \ #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \

10
include/jemalloc/internal/background_thread_externs.h
View File

@@ -17,15 +17,15 @@ extern background_thread_info_t *background_thread_info;
bool background_thread_create(tsd_t *tsd, unsigned arena_ind); bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
bool background_threads_enable(tsd_t *tsd); bool background_threads_enable(tsd_t *tsd);
bool background_threads_disable(tsd_t *tsd); bool background_threads_disable(tsd_t *tsd);
bool background_thread_is_started(background_thread_info_t* info); bool background_thread_is_started(background_thread_info_t *info);
void background_thread_wakeup_early(background_thread_info_t *info, void background_thread_wakeup_early(
nstime_t *remaining_sleep); background_thread_info_t *info, nstime_t *remaining_sleep);
void background_thread_prefork0(tsdn_t *tsdn); void background_thread_prefork0(tsdn_t *tsdn);
void background_thread_prefork1(tsdn_t *tsdn); void background_thread_prefork1(tsdn_t *tsdn);
void background_thread_postfork_parent(tsdn_t *tsdn); void background_thread_postfork_parent(tsdn_t *tsdn);
void background_thread_postfork_child(tsdn_t *tsdn); void background_thread_postfork_child(tsdn_t *tsdn);
bool background_thread_stats_read(tsdn_t *tsdn, bool background_thread_stats_read(
background_thread_stats_t *stats); tsdn_t *tsdn, background_thread_stats_t *stats);
void background_thread_ctl_init(tsdn_t *tsdn); void background_thread_ctl_init(tsdn_t *tsdn);
#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER

8
include/jemalloc/internal/background_thread_inlines.h
View File

@@ -36,14 +36,14 @@ background_thread_info_get(size_t ind) {
JEMALLOC_ALWAYS_INLINE uint64_t JEMALLOC_ALWAYS_INLINE uint64_t
background_thread_wakeup_time_get(background_thread_info_t *info) { background_thread_wakeup_time_get(background_thread_info_t *info) {
uint64_t next_wakeup = nstime_ns(&info->next_wakeup); uint64_t next_wakeup = nstime_ns(&info->next_wakeup);
assert(atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE) == assert(atomic_load_b(&info->indefinite_sleep, ATOMIC_ACQUIRE)
(next_wakeup == BACKGROUND_THREAD_INDEFINITE_SLEEP)); == (next_wakeup == BACKGROUND_THREAD_INDEFINITE_SLEEP));
return next_wakeup; return next_wakeup;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
background_thread_wakeup_time_set(tsdn_t *tsdn, background_thread_info_t *info, background_thread_wakeup_time_set(
uint64_t wakeup_time) { tsdn_t *tsdn, background_thread_info_t *info, uint64_t wakeup_time) {
malloc_mutex_assert_owner(tsdn, &info->mtx); malloc_mutex_assert_owner(tsdn, &info->mtx);
atomic_store_b(&info->indefinite_sleep, atomic_store_b(&info->indefinite_sleep,
wakeup_time == BACKGROUND_THREAD_INDEFINITE_SLEEP, ATOMIC_RELEASE); wakeup_time == BACKGROUND_THREAD_INDEFINITE_SLEEP, ATOMIC_RELEASE);

8
include/jemalloc/internal/base.h
View File

@@ -102,13 +102,13 @@ metadata_thp_enabled(void) {
} }
base_t *b0get(void); base_t *b0get(void);
base_t *base_new(tsdn_t *tsdn, unsigned ind, base_t *base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
const extent_hooks_t *extent_hooks, bool metadata_use_hooks); bool metadata_use_hooks);
void base_delete(tsdn_t *tsdn, base_t *base); void base_delete(tsdn_t *tsdn, base_t *base);
ehooks_t *base_ehooks_get(base_t *base); ehooks_t *base_ehooks_get(base_t *base);
ehooks_t *base_ehooks_get_for_metadata(base_t *base); ehooks_t *base_ehooks_get_for_metadata(base_t *base);
extent_hooks_t *base_extent_hooks_set(base_t *base, extent_hooks_t *base_extent_hooks_set(
extent_hooks_t *extent_hooks); base_t *base, extent_hooks_t *extent_hooks);
void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);
edata_t *base_alloc_edata(tsdn_t *tsdn, base_t *base); edata_t *base_alloc_edata(tsdn_t *tsdn, base_t *base);
void *base_alloc_rtree(tsdn_t *tsdn, base_t *base, size_t size); void *base_alloc_rtree(tsdn_t *tsdn, base_t *base, size_t size);

6
include/jemalloc/internal/batcher.h
View File

@@ -5,7 +5,7 @@
#include "jemalloc/internal/atomic.h" #include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#define BATCHER_NO_IDX ((size_t)-1) #define BATCHER_NO_IDX ((size_t) - 1)
typedef struct batcher_s batcher_t; typedef struct batcher_s batcher_t;
struct batcher_s { struct batcher_s {
@@ -27,8 +27,8 @@ void batcher_init(batcher_t *batcher, size_t nelems_max);
* BATCHER_NO_IDX if no index is free. If the former, the caller must call * BATCHER_NO_IDX if no index is free. If the former, the caller must call
* batcher_push_end once done. * batcher_push_end once done.
*/ */
size_t batcher_push_begin(tsdn_t *tsdn, batcher_t *batcher, size_t batcher_push_begin(
size_t elems_to_push); tsdn_t *tsdn, batcher_t *batcher, size_t elems_to_push);
void batcher_push_end(tsdn_t *tsdn, batcher_t *batcher); void batcher_push_end(tsdn_t *tsdn, batcher_t *batcher);
/* /*

8
include/jemalloc/internal/bin.h
View File

@@ -14,8 +14,8 @@
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
extern void (*bin_batching_test_after_push_hook)(size_t idx); extern void (*bin_batching_test_after_push_hook)(size_t idx);
extern void (*bin_batching_test_mid_pop_hook)(size_t elems_to_pop); extern void (*bin_batching_test_mid_pop_hook)(size_t elems_to_pop);
extern void (*bin_batching_test_after_unlock_hook)(unsigned slab_dalloc_count, extern void (*bin_batching_test_after_unlock_hook)(
bool list_empty); unsigned slab_dalloc_count, bool list_empty);
#endif #endif
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
@@ -50,8 +50,8 @@ bin_batching_test_after_unlock(unsigned slab_dalloc_count, bool list_empty) {
(void)list_empty; (void)list_empty;
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
if (bin_batching_test_after_unlock_hook != NULL) { if (bin_batching_test_after_unlock_hook != NULL) {
bin_batching_test_after_unlock_hook(slab_dalloc_count, bin_batching_test_after_unlock_hook(
list_empty); slab_dalloc_count, list_empty);
} }
#endif #endif
} }

5
include/jemalloc/internal/bin_types.h
View File

@@ -8,7 +8,10 @@
#define N_BIN_SHARDS_DEFAULT 1 #define N_BIN_SHARDS_DEFAULT 1
/* Used in TSD static initializer only. Real init in arena_bind(). */ /* Used in TSD static initializer only. Real init in arena_bind(). */
#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}} #define TSD_BINSHARDS_ZERO_INITIALIZER \
{ \
{ UINT8_MAX } \
}
typedef struct tsd_binshards_s tsd_binshards_t; typedef struct tsd_binshards_s tsd_binshards_t;
struct tsd_binshards_s { struct tsd_binshards_s {

40
include/jemalloc/internal/bit_util.h
View File

@@ -110,15 +110,17 @@ fls_u(unsigned x) {
} }
#elif defined(_MSC_VER) #elif defined(_MSC_VER)
#if LG_SIZEOF_PTR == 3 # if LG_SIZEOF_PTR == 3
#define DO_BSR64(bit, x) _BitScanReverse64(&bit, x) # define DO_BSR64(bit, x) _BitScanReverse64(&bit, x)
#else # else
/* /*
* This never actually runs; we're just dodging a compiler error for the * This never actually runs; we're just dodging a compiler error for the
* never-taken branch where sizeof(void *) == 8. * never-taken branch where sizeof(void *) == 8.
*/ */
#define DO_BSR64(bit, x) bit = 0; unreachable() # define DO_BSR64(bit, x) \
#endif bit = 0; \
unreachable()
# endif
/* clang-format off */ /* clang-format off */
#define DO_FLS(x) do { \ #define DO_FLS(x) do { \
@@ -164,8 +166,8 @@ fls_u(unsigned x) {
DO_FLS(x); DO_FLS(x);
} }
#undef DO_FLS # undef DO_FLS
#undef DO_BSR64 # undef DO_BSR64
#else #else
static inline unsigned static inline unsigned
@@ -284,7 +286,7 @@ popcount_llu(unsigned long long bitmap) {
*/ */
static inline size_t static inline size_t
cfs_lu(unsigned long* bitmap) { cfs_lu(unsigned long *bitmap) {
util_assume(*bitmap != 0); util_assume(*bitmap != 0);
size_t bit = ffs_lu(*bitmap); size_t bit = ffs_lu(*bitmap);
*bitmap ^= ZU(1) << bit; *bitmap ^= ZU(1) << bit;
@@ -300,7 +302,7 @@ ffs_zu(size_t x) {
#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
return ffs_llu(x); return ffs_llu(x);
#else #else
#error No implementation for size_t ffs() # error No implementation for size_t ffs()
#endif #endif
} }
@@ -313,11 +315,10 @@ fls_zu(size_t x) {
#elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG #elif LG_SIZEOF_PTR == LG_SIZEOF_LONG_LONG
return fls_llu(x); return fls_llu(x);
#else #else
#error No implementation for size_t fls() # error No implementation for size_t fls()
#endif #endif
} }
static inline unsigned static inline unsigned
ffs_u64(uint64_t x) { ffs_u64(uint64_t x) {
#if LG_SIZEOF_LONG == 3 #if LG_SIZEOF_LONG == 3
@@ -325,7 +326,7 @@ ffs_u64(uint64_t x) {
#elif LG_SIZEOF_LONG_LONG == 3 #elif LG_SIZEOF_LONG_LONG == 3
return ffs_llu(x); return ffs_llu(x);
#else #else
#error No implementation for 64-bit ffs() # error No implementation for 64-bit ffs()
#endif #endif
} }
@@ -336,7 +337,7 @@ fls_u64(uint64_t x) {
#elif LG_SIZEOF_LONG_LONG == 3 #elif LG_SIZEOF_LONG_LONG == 3
return fls_llu(x); return fls_llu(x);
#else #else
#error No implementation for 64-bit fls() # error No implementation for 64-bit fls()
#endif #endif
} }
@@ -345,7 +346,7 @@ ffs_u32(uint32_t x) {
#if LG_SIZEOF_INT == 2 #if LG_SIZEOF_INT == 2
return ffs_u(x); return ffs_u(x);
#else #else
#error No implementation for 32-bit ffs() # error No implementation for 32-bit ffs()
#endif #endif
} }
@@ -354,7 +355,7 @@ fls_u32(uint32_t x) {
#if LG_SIZEOF_INT == 2 #if LG_SIZEOF_INT == 2
return fls_u(x); return fls_u(x);
#else #else
#error No implementation for 32-bit fls() # error No implementation for 32-bit fls()
#endif #endif
} }
@@ -413,9 +414,12 @@ lg_ceil(size_t x) {
#define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1)) #define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1))
#define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2)) #define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2))
#define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4)) #define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4))
#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8)) #define LG_FLOOR_16(x) \
#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16)) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8))
#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32)) #define LG_FLOOR_32(x) \
(x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16))
#define LG_FLOOR_64(x) \
(x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32))
#if LG_SIZEOF_PTR == 2 #if LG_SIZEOF_PTR == 2
# define LG_FLOOR(x) LG_FLOOR_32((x)) # define LG_FLOOR(x) LG_FLOOR_32((x))
#else #else

117
include/jemalloc/internal/bitmap.h
View File

@@ -21,7 +21,7 @@ typedef unsigned long bitmap_t;
/* Number of bits per group. */ /* Number of bits per group. */
#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3) #define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3)
#define BITMAP_GROUP_NBITS (1U << LG_BITMAP_GROUP_NBITS) #define BITMAP_GROUP_NBITS (1U << LG_BITMAP_GROUP_NBITS)
#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1) #define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS - 1)
/* /*
* Do some analysis on how big the bitmap is before we use a tree. For a brute * Do some analysis on how big the bitmap is before we use a tree. For a brute
@@ -39,15 +39,13 @@ typedef unsigned long bitmap_t;
/* /*
* Number of groups required at a particular level for a given number of bits. * Number of groups required at a particular level for a given number of bits.
*/ */
#define BITMAP_GROUPS_L0(nbits) \ #define BITMAP_GROUPS_L0(nbits) BITMAP_BITS2GROUPS(nbits)
BITMAP_BITS2GROUPS(nbits) #define BITMAP_GROUPS_L1(nbits) BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits))
#define BITMAP_GROUPS_L1(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits))
#define BITMAP_GROUPS_L2(nbits) \ #define BITMAP_GROUPS_L2(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))) BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))
#define BITMAP_GROUPS_L3(nbits) \ #define BITMAP_GROUPS_L3(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \ BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \
BITMAP_BITS2GROUPS((nbits))))) BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))))
#define BITMAP_GROUPS_L4(nbits) \ #define BITMAP_GROUPS_L4(nbits) \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \ BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \
BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))))) BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits))))))
@@ -56,8 +54,7 @@ typedef unsigned long bitmap_t;
* Assuming the number of levels, number of groups required for a given number * Assuming the number of levels, number of groups required for a given number
* of bits. * of bits.
*/ */
#define BITMAP_GROUPS_1_LEVEL(nbits) \ #define BITMAP_GROUPS_1_LEVEL(nbits) BITMAP_GROUPS_L0(nbits)
BITMAP_GROUPS_L0(nbits)
#define BITMAP_GROUPS_2_LEVEL(nbits) \ #define BITMAP_GROUPS_2_LEVEL(nbits) \
(BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits)) (BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits))
#define BITMAP_GROUPS_3_LEVEL(nbits) \ #define BITMAP_GROUPS_3_LEVEL(nbits) \
@@ -72,24 +69,24 @@ typedef unsigned long bitmap_t;
*/ */
#ifdef BITMAP_USE_TREE #ifdef BITMAP_USE_TREE
#if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS # if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_1_LEVEL(nbits) # define BITMAP_GROUPS(nbits) BITMAP_GROUPS_1_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS) # define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2 # elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_2_LEVEL(nbits) # define BITMAP_GROUPS(nbits) BITMAP_GROUPS_2_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS) # define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3 # elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_3_LEVEL(nbits) # define BITMAP_GROUPS(nbits) BITMAP_GROUPS_3_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS) # define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4 # elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_4_LEVEL(nbits) # define BITMAP_GROUPS(nbits) BITMAP_GROUPS_4_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS) # define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS)
#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 5 # elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 5
# define BITMAP_GROUPS(nbits) BITMAP_GROUPS_5_LEVEL(nbits) # define BITMAP_GROUPS(nbits) BITMAP_GROUPS_5_LEVEL(nbits)
# define BITMAP_GROUPS_MAX BITMAP_GROUPS_5_LEVEL(BITMAP_MAXBITS) # define BITMAP_GROUPS_MAX BITMAP_GROUPS_5_LEVEL(BITMAP_MAXBITS)
#else # else
# error "Unsupported bitmap size" # error "Unsupported bitmap size"
#endif # endif
/* /*
* Maximum number of levels possible. This could be statically computed based * Maximum number of levels possible. This could be statically computed based
@@ -105,42 +102,53 @@ typedef unsigned long bitmap_t;
* unused trailing entries in bitmap_info_t structures; the bitmaps themselves * unused trailing entries in bitmap_info_t structures; the bitmaps themselves
* are not impacted. * are not impacted.
*/ */
#define BITMAP_MAX_LEVELS 5 # define BITMAP_MAX_LEVELS 5
#define BITMAP_INFO_INITIALIZER(nbits) { \ # define BITMAP_INFO_INITIALIZER(nbits) \
/* nbits. */ \
nbits, \
/* nlevels. */ \
(BITMAP_GROUPS_L0(nbits) > BITMAP_GROUPS_L1(nbits)) + \
(BITMAP_GROUPS_L1(nbits) > BITMAP_GROUPS_L2(nbits)) + \
(BITMAP_GROUPS_L2(nbits) > BITMAP_GROUPS_L3(nbits)) + \
(BITMAP_GROUPS_L3(nbits) > BITMAP_GROUPS_L4(nbits)) + 1, \
/* levels. */ \
{ \ { \
{0}, \ /* nbits. */ \
{BITMAP_GROUPS_L0(nbits)}, \ nbits, /* nlevels. */ \
{BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)}, \ (BITMAP_GROUPS_L0(nbits) \
{BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits) + \ > BITMAP_GROUPS_L1(nbits)) \
BITMAP_GROUPS_L0(nbits)}, \ + (BITMAP_GROUPS_L1(nbits) \
{BITMAP_GROUPS_L3(nbits) + BITMAP_GROUPS_L2(nbits) + \ > BITMAP_GROUPS_L2(nbits)) \
BITMAP_GROUPS_L1(nbits) + BITMAP_GROUPS_L0(nbits)}, \ + (BITMAP_GROUPS_L2(nbits) \
{BITMAP_GROUPS_L4(nbits) + BITMAP_GROUPS_L3(nbits) + \ > BITMAP_GROUPS_L3(nbits)) \
BITMAP_GROUPS_L2(nbits) + BITMAP_GROUPS_L1(nbits) \ + (BITMAP_GROUPS_L3(nbits) \
+ BITMAP_GROUPS_L0(nbits)} \ > BITMAP_GROUPS_L4(nbits)) \
+ 1, /* levels. */ \
{ \
{0}, {BITMAP_GROUPS_L0(nbits)}, \
{BITMAP_GROUPS_L1(nbits) \
+ BITMAP_GROUPS_L0(nbits)}, \
{BITMAP_GROUPS_L2(nbits) \
+ BITMAP_GROUPS_L1(nbits) \
+ BITMAP_GROUPS_L0(nbits)}, \
{BITMAP_GROUPS_L3(nbits) \
+ BITMAP_GROUPS_L2(nbits) \
+ BITMAP_GROUPS_L1(nbits) \
+ BITMAP_GROUPS_L0(nbits)}, \
{ \
BITMAP_GROUPS_L4(nbits) \
+ BITMAP_GROUPS_L3(nbits) \
+ BITMAP_GROUPS_L2(nbits) \
+ BITMAP_GROUPS_L1(nbits) \
+ BITMAP_GROUPS_L0(nbits) \
} \ } \
} } \
}
#else /* BITMAP_USE_TREE */ #else /* BITMAP_USE_TREE */
#define BITMAP_GROUPS(nbits) BITMAP_BITS2GROUPS(nbits) # define BITMAP_GROUPS(nbits) BITMAP_BITS2GROUPS(nbits)
#define BITMAP_GROUPS_MAX BITMAP_BITS2GROUPS(BITMAP_MAXBITS) # define BITMAP_GROUPS_MAX BITMAP_BITS2GROUPS(BITMAP_MAXBITS)
#define BITMAP_INFO_INITIALIZER(nbits) { \ # define BITMAP_INFO_INITIALIZER(nbits) \
{ \
/* nbits. */ \ /* nbits. */ \
nbits, \ nbits, /* ngroups. */ \
/* ngroups. */ \
BITMAP_BITS2GROUPS(nbits) \ BITMAP_BITS2GROUPS(nbits) \
} }
#endif /* BITMAP_USE_TREE */ #endif /* BITMAP_USE_TREE */
@@ -161,7 +169,7 @@ typedef struct bitmap_info_s {
* Only the first (nlevels+1) elements are used, and levels are ordered * Only the first (nlevels+1) elements are used, and levels are ordered
* bottom to top (e.g. the bottom level is stored in levels[0]). * bottom to top (e.g. the bottom level is stored in levels[0]).
*/ */
bitmap_level_t levels[BITMAP_MAX_LEVELS+1]; bitmap_level_t levels[BITMAP_MAX_LEVELS + 1];
#else /* BITMAP_USE_TREE */ #else /* BITMAP_USE_TREE */
/* Number of groups necessary for nbits. */ /* Number of groups necessary for nbits. */
size_t ngroups; size_t ngroups;
@@ -245,12 +253,13 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
#ifdef BITMAP_USE_TREE #ifdef BITMAP_USE_TREE
size_t bit = 0; size_t bit = 0;
for (unsigned level = binfo->nlevels; level--;) { for (unsigned level = binfo->nlevels; level--;) {
size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS * (level + size_t lg_bits_per_group = (LG_BITMAP_GROUP_NBITS
1)); * (level + 1));
bitmap_t group = bitmap[binfo->levels[level].group_offset + (bit bitmap_t group = bitmap[binfo->levels[level].group_offset
>> lg_bits_per_group)]; + (bit >> lg_bits_per_group)];
unsigned group_nmask = (unsigned)(((min_bit > bit) ? (min_bit - unsigned group_nmask =
bit) : 0) >> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS)); (unsigned)(((min_bit > bit) ? (min_bit - bit) : 0)
>> (lg_bits_per_group - LG_BITMAP_GROUP_NBITS));
assert(group_nmask <= BITMAP_GROUP_NBITS); assert(group_nmask <= BITMAP_GROUP_NBITS);
bitmap_t group_mask = ~((1LU << group_nmask) - 1); bitmap_t group_mask = ~((1LU << group_nmask) - 1);
bitmap_t group_masked = group & group_mask; bitmap_t group_masked = group & group_mask;
@@ -273,16 +282,16 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
} }
return bitmap_ffu(bitmap, binfo, sib_base); return bitmap_ffu(bitmap, binfo, sib_base);
} }
bit += ((size_t)ffs_lu(group_masked)) << bit += ((size_t)ffs_lu(group_masked))
(lg_bits_per_group - LG_BITMAP_GROUP_NBITS); << (lg_bits_per_group - LG_BITMAP_GROUP_NBITS);
} }
assert(bit >= min_bit); assert(bit >= min_bit);
assert(bit < binfo->nbits); assert(bit < binfo->nbits);
return bit; return bit;
#else #else
size_t i = min_bit >> LG_BITMAP_GROUP_NBITS; size_t i = min_bit >> LG_BITMAP_GROUP_NBITS;
bitmap_t g = bitmap[i] & ~((1LU << (min_bit & BITMAP_GROUP_NBITS_MASK)) bitmap_t g = bitmap[i]
- 1); & ~((1LU << (min_bit & BITMAP_GROUP_NBITS_MASK)) - 1);
size_t bit; size_t bit;
while (1) { while (1) {
if (g != 0) { if (g != 0) {

6
include/jemalloc/internal/buf_writer.h
View File

@@ -29,8 +29,8 @@ void buf_writer_flush(buf_writer_t *buf_writer);
write_cb_t buf_writer_cb; write_cb_t buf_writer_cb;
void buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer); void buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer);
typedef ssize_t (read_cb_t)(void *read_cbopaque, void *buf, size_t limit); typedef ssize_t(read_cb_t)(void *read_cbopaque, void *buf, size_t limit);
void buf_writer_pipe(buf_writer_t *buf_writer, read_cb_t *read_cb, void buf_writer_pipe(
void *read_cbopaque); buf_writer_t *buf_writer, read_cb_t *read_cb, void *read_cbopaque);
#endif /* JEMALLOC_INTERNAL_BUF_WRITER_H */ #endif /* JEMALLOC_INTERNAL_BUF_WRITER_H */

91
include/jemalloc/internal/cache_bin.h
View File

@@ -45,8 +45,8 @@ extern const uintptr_t disabled_bin;
* 1 << (sizeof(cache_bin_sz_t) * 8) * 1 << (sizeof(cache_bin_sz_t) * 8)
* bytes spread across pointer sized objects to get the maximum. * bytes spread across pointer sized objects to get the maximum.
*/ */
#define CACHE_BIN_NCACHED_MAX (((size_t)1 << sizeof(cache_bin_sz_t) * 8) \ #define CACHE_BIN_NCACHED_MAX \
/ sizeof(void *) - 1) (((size_t)1 << sizeof(cache_bin_sz_t) * 8) / sizeof(void *) - 1)
/* /*
* This lives inside the cache_bin (for locality reasons), and is initialized * This lives inside the cache_bin (for locality reasons), and is initialized
@@ -152,8 +152,8 @@ struct cache_bin_array_descriptor_s {
}; };
static inline void static inline void
cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor, cache_bin_array_descriptor_init(
cache_bin_t *bins) { cache_bin_array_descriptor_t *descriptor, cache_bin_t *bins) {
ql_elm_new(descriptor, link); ql_elm_new(descriptor, link);
descriptor->bins = bins; descriptor->bins = bins;
} }
@@ -222,7 +222,8 @@ cache_bin_ncached_max_get(cache_bin_t *bin) {
* with later. * with later.
*/ */
static inline void static inline void
cache_bin_assert_earlier(cache_bin_t *bin, cache_bin_sz_t earlier, cache_bin_sz_t later) { cache_bin_assert_earlier(
cache_bin_t *bin, cache_bin_sz_t earlier, cache_bin_sz_t later) {
if (earlier > later) { if (earlier > later) {
assert(bin->low_bits_full > bin->low_bits_empty); assert(bin->low_bits_full > bin->low_bits_empty);
} }
@@ -305,8 +306,8 @@ cache_bin_empty_position_get(cache_bin_t *bin) {
*/ */
static inline cache_bin_sz_t static inline cache_bin_sz_t
cache_bin_low_bits_low_bound_get(cache_bin_t *bin) { cache_bin_low_bits_low_bound_get(cache_bin_t *bin) {
return (cache_bin_sz_t)bin->low_bits_empty - return (cache_bin_sz_t)bin->low_bits_empty
cache_bin_ncached_max_get(bin) * sizeof(void *); - cache_bin_ncached_max_get(bin) * sizeof(void *);
} }
/* /*
@@ -340,8 +341,8 @@ cache_bin_assert_empty(cache_bin_t *bin) {
*/ */
static inline cache_bin_sz_t static inline cache_bin_sz_t
cache_bin_low_water_get_internal(cache_bin_t *bin) { cache_bin_low_water_get_internal(cache_bin_t *bin) {
return cache_bin_diff(bin, bin->low_bits_low_water, return cache_bin_diff(bin, bin->low_bits_low_water, bin->low_bits_empty)
bin->low_bits_empty) / sizeof(void *); / sizeof(void *);
} }
/* Returns the numeric value of low water in [0, ncached]. */ /* Returns the numeric value of low water in [0, ncached]. */
@@ -351,7 +352,8 @@ cache_bin_low_water_get(cache_bin_t *bin) {
assert(low_water <= cache_bin_ncached_max_get(bin)); assert(low_water <= cache_bin_ncached_max_get(bin));
assert(low_water <= cache_bin_ncached_get_local(bin)); assert(low_water <= cache_bin_ncached_get_local(bin));
cache_bin_assert_earlier(bin, (cache_bin_sz_t)(uintptr_t)bin->stack_head, cache_bin_assert_earlier(bin,
(cache_bin_sz_t)(uintptr_t)bin->stack_head,
bin->low_bits_low_water); bin->low_bits_low_water);
return low_water; return low_water;
@@ -455,7 +457,8 @@ cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) {
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
cache_bin_full(cache_bin_t *bin) { cache_bin_full(cache_bin_t *bin) {
return ((cache_bin_sz_t)(uintptr_t)bin->stack_head == bin->low_bits_full); return (
(cache_bin_sz_t)(uintptr_t)bin->stack_head == bin->low_bits_full);
} }
/* /*
@@ -516,9 +519,11 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
} }
/* Stash at the full position, in the [full, head) range. */ /* Stash at the full position, in the [full, head) range. */
cache_bin_sz_t low_bits_head = (cache_bin_sz_t)(uintptr_t)bin->stack_head; cache_bin_sz_t low_bits_head = (cache_bin_sz_t)(uintptr_t)
bin->stack_head;
/* Wraparound handled as well. */ /* Wraparound handled as well. */
cache_bin_sz_t diff = cache_bin_diff(bin, bin->low_bits_full, low_bits_head); cache_bin_sz_t diff = cache_bin_diff(
bin, bin->low_bits_full, low_bits_head);
*(void **)((byte_t *)bin->stack_head - diff) = ptr; *(void **)((byte_t *)bin->stack_head - diff) = ptr;
assert(!cache_bin_full(bin)); assert(!cache_bin_full(bin));
@@ -532,16 +537,19 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_internal(cache_bin_t *bin) { cache_bin_nstashed_get_internal(cache_bin_t *bin) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin); cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin); cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
bin);
cache_bin_sz_t n = cache_bin_diff(bin, low_bits_low_bound, cache_bin_sz_t n = cache_bin_diff(
bin->low_bits_full) / sizeof(void *); bin, low_bits_low_bound, bin->low_bits_full)
/ sizeof(void *);
assert(n <= ncached_max); assert(n <= ncached_max);
if (config_debug && n != 0) { if (config_debug && n != 0) {
/* Below are for assertions only. */ /* Below are for assertions only. */
void **low_bound = cache_bin_low_bound_get(bin); void **low_bound = cache_bin_low_bound_get(bin);
assert((cache_bin_sz_t)(uintptr_t)low_bound == low_bits_low_bound); assert(
(cache_bin_sz_t)(uintptr_t)low_bound == low_bits_low_bound);
void *stashed = *(low_bound + n - 1); void *stashed = *(low_bound + n - 1);
bool aligned = cache_bin_nonfast_aligned(stashed); bool aligned = cache_bin_nonfast_aligned(stashed);
#ifdef JEMALLOC_JET #ifdef JEMALLOC_JET
@@ -582,16 +590,17 @@ cache_bin_nstashed_get_local(cache_bin_t *bin) {
* they help access values that will not be concurrently modified. * they help access values that will not be concurrently modified.
*/ */
static inline void static inline void
cache_bin_nitems_get_remote(cache_bin_t *bin, cache_bin_sz_t *ncached, cache_bin_nitems_get_remote(
cache_bin_sz_t *nstashed) { cache_bin_t *bin, cache_bin_sz_t *ncached, cache_bin_sz_t *nstashed) {
/* Racy version of cache_bin_ncached_get_internal. */ /* Racy version of cache_bin_ncached_get_internal. */
cache_bin_sz_t diff = bin->low_bits_empty - cache_bin_sz_t diff = bin->low_bits_empty
(cache_bin_sz_t)(uintptr_t)bin->stack_head; - (cache_bin_sz_t)(uintptr_t)bin->stack_head;
cache_bin_sz_t n = diff / sizeof(void *); cache_bin_sz_t n = diff / sizeof(void *);
*ncached = n; *ncached = n;
/* Racy version of cache_bin_nstashed_get_internal. */ /* Racy version of cache_bin_nstashed_get_internal. */
cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(bin); cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
bin);
n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *); n = (bin->low_bits_full - low_bits_low_bound) / sizeof(void *);
*nstashed = n; *nstashed = n;
/* /*
@@ -616,7 +625,8 @@ struct cache_bin_fill_ctl_s {
* This is to avoid stack overflow when we do batch edata look up, which * This is to avoid stack overflow when we do batch edata look up, which
* reserves a nflush * sizeof(emap_batch_lookup_result_t) stack variable. * reserves a nflush * sizeof(emap_batch_lookup_result_t) stack variable.
*/ */
#define CACHE_BIN_NFLUSH_BATCH_MAX ((VARIABLE_ARRAY_SIZE_MAX >> LG_SIZEOF_PTR) - 1) #define CACHE_BIN_NFLUSH_BATCH_MAX \
((VARIABLE_ARRAY_SIZE_MAX >> LG_SIZEOF_PTR) - 1)
/* /*
* Filling and flushing are done in batch, on arrays of void *s. For filling, * Filling and flushing are done in batch, on arrays of void *s. For filling,
@@ -659,8 +669,8 @@ struct cache_bin_ptr_array_s {
* finish_fill call before doing any alloc/dalloc operations on the bin. * finish_fill call before doing any alloc/dalloc operations on the bin.
*/ */
static inline void static inline void
cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_init_ptr_array_for_fill(
cache_bin_sz_t nfill) { cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nfill) {
cache_bin_assert_empty(bin); cache_bin_assert_empty(bin);
arr->ptr = cache_bin_empty_position_get(bin) - nfill; arr->ptr = cache_bin_empty_position_get(bin) - nfill;
} }
@@ -671,8 +681,8 @@ cache_bin_init_ptr_array_for_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
* case of OOM. * case of OOM.
*/ */
static inline void static inline void
cache_bin_finish_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_finish_fill(
cache_bin_sz_t nfilled) { cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nfilled) {
cache_bin_assert_empty(bin); cache_bin_assert_empty(bin);
void **empty_position = cache_bin_empty_position_get(bin); void **empty_position = cache_bin_empty_position_get(bin);
if (nfilled < arr->n) { if (nfilled < arr->n) {
@@ -687,19 +697,18 @@ cache_bin_finish_fill(cache_bin_t *bin, cache_bin_ptr_array_t *arr,
* everything we give them. * everything we give them.
*/ */
static inline void static inline void
cache_bin_init_ptr_array_for_flush(cache_bin_t *bin, cache_bin_init_ptr_array_for_flush(
cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) { cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nflush) {
arr->ptr = cache_bin_empty_position_get(bin) - nflush; arr->ptr = cache_bin_empty_position_get(bin) - nflush;
assert(cache_bin_ncached_get_local(bin) == 0 assert(cache_bin_ncached_get_local(bin) == 0 || *arr->ptr != NULL);
|| *arr->ptr != NULL);
} }
static inline void static inline void
cache_bin_finish_flush(cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_finish_flush(
cache_bin_sz_t nflushed) { cache_bin_t *bin, cache_bin_ptr_array_t *arr, cache_bin_sz_t nflushed) {
unsigned rem = cache_bin_ncached_get_local(bin) - nflushed; unsigned rem = cache_bin_ncached_get_local(bin) - nflushed;
memmove(bin->stack_head + nflushed, bin->stack_head, memmove(
rem * sizeof(void *)); bin->stack_head + nflushed, bin->stack_head, rem * sizeof(void *));
bin->stack_head += nflushed; bin->stack_head += nflushed;
cache_bin_low_water_adjust(bin); cache_bin_low_water_adjust(bin);
} }
@@ -728,14 +737,14 @@ cache_bin_finish_flush_stashed(cache_bin_t *bin) {
* Initialize a cache_bin_info to represent up to the given number of items in * Initialize a cache_bin_info to represent up to the given number of items in
* the cache_bins it is associated with. * the cache_bins it is associated with.
*/ */
void cache_bin_info_init(cache_bin_info_t *bin_info, void cache_bin_info_init(
cache_bin_sz_t ncached_max); cache_bin_info_t *bin_info, cache_bin_sz_t ncached_max);
/* /*
* Given an array of initialized cache_bin_info_ts, determine how big an * Given an array of initialized cache_bin_info_ts, determine how big an
* allocation is required to initialize a full set of cache_bin_ts. * allocation is required to initialize a full set of cache_bin_ts.
*/ */
void cache_bin_info_compute_alloc(const cache_bin_info_t *infos, void cache_bin_info_compute_alloc(const cache_bin_info_t *infos, szind_t ninfos,
szind_t ninfos, size_t *size, size_t *alignment); size_t *size, size_t *alignment);
/* /*
* Actually initialize some cache bins. Callers should allocate the backing * Actually initialize some cache bins. Callers should allocate the backing
@@ -747,8 +756,8 @@ void cache_bin_info_compute_alloc(const cache_bin_info_t *infos,
void cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos, void cache_bin_preincrement(const cache_bin_info_t *infos, szind_t ninfos,
void *alloc, size_t *cur_offset); void *alloc, size_t *cur_offset);
void cache_bin_postincrement(void *alloc, size_t *cur_offset); void cache_bin_postincrement(void *alloc, size_t *cur_offset);
void cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
void *alloc, size_t *cur_offset); size_t *cur_offset);
void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max); void cache_bin_init_disabled(cache_bin_t *bin, cache_bin_sz_t ncached_max);
bool cache_bin_stack_use_thp(void); bool cache_bin_stack_use_thp(void);

8
include/jemalloc/internal/ckh.h
View File

@@ -22,8 +22,8 @@
#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) #define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1)
/* Typedefs to allow easy function pointer passing. */ /* Typedefs to allow easy function pointer passing. */
typedef void ckh_hash_t (const void *, size_t[2]); typedef void ckh_hash_t(const void *, size_t[2]);
typedef bool ckh_keycomp_t (const void *, const void *); typedef bool ckh_keycomp_t(const void *, const void *);
/* Hash table cell. */ /* Hash table cell. */
typedef struct { typedef struct {
@@ -89,8 +89,8 @@ bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
* the key and value, and doesn't do any lifetime management. * the key and value, and doesn't do any lifetime management.
*/ */
bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data); bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, bool ckh_remove(
void **data); tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data);
bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data); bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
/* Some useful hash and comparison functions for strings and pointers. */ /* Some useful hash and comparison functions for strings and pointers. */

55
include/jemalloc/internal/ctl.h
View File

@@ -26,14 +26,14 @@ typedef struct ctl_named_node_s {
/* If (nchildren == 0), this is a terminal node. */ /* If (nchildren == 0), this is a terminal node. */
size_t nchildren; size_t nchildren;
const ctl_node_t *children; const ctl_node_t *children;
int (*ctl)(tsd_t *, const size_t *, size_t, void *, size_t *, void *, int (*ctl)(
size_t); tsd_t *, const size_t *, size_t, void *, size_t *, void *, size_t);
} ctl_named_node_t; } ctl_named_node_t;
typedef struct ctl_indexed_node_s { typedef struct ctl_indexed_node_s {
struct ctl_node_s node; struct ctl_node_s node;
const ctl_named_node_t *(*index)(tsdn_t *, const size_t *, size_t, const ctl_named_node_t *(*index)(
size_t); tsdn_t *, const size_t *, size_t, size_t);
} ctl_indexed_node_t; } ctl_indexed_node_t;
typedef struct ctl_arena_stats_s { typedef struct ctl_arena_stats_s {
@@ -107,8 +107,8 @@ int ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
int ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp); int ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp);
int ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, int ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen); size_t *oldlenp, void *newp, size_t newlen);
int ctl_mibnametomib(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, int ctl_mibnametomib(
size_t *miblenp); tsd_t *tsd, size_t *mib, size_t miblen, const char *name, size_t *miblenp);
int ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name, int ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen); size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
bool ctl_boot(void); bool ctl_boot(void);
@@ -117,50 +117,57 @@ void ctl_postfork_parent(tsdn_t *tsdn);
void ctl_postfork_child(tsdn_t *tsdn); void ctl_postfork_child(tsdn_t *tsdn);
void ctl_mtx_assert_held(tsdn_t *tsdn); void ctl_mtx_assert_held(tsdn_t *tsdn);
#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ #define xmallctl(name, oldp, oldlenp, newp, newlen) \
if (je_mallctl(name, oldp, oldlenp, newp, newlen) \ do { \
!= 0) { \ if (je_mallctl(name, oldp, oldlenp, newp, newlen) != 0) { \
malloc_printf( \ malloc_printf( \
"<jemalloc>: Failure in xmallctl(\"%s\", ...)\n", \ "<jemalloc>: Failure in xmallctl(\"%s\", ...)\n", \
name); \ name); \
abort(); \ abort(); \
} \ } \
} while (0) } while (0)
#define xmallctlnametomib(name, mibp, miblenp) do { \ #define xmallctlnametomib(name, mibp, miblenp) \
do { \
if (je_mallctlnametomib(name, mibp, miblenp) != 0) { \ if (je_mallctlnametomib(name, mibp, miblenp) != 0) { \
malloc_printf("<jemalloc>: Failure in " \ malloc_printf( \
"xmallctlnametomib(\"%s\", ...)\n", name); \ "<jemalloc>: Failure in " \
"xmallctlnametomib(\"%s\", ...)\n", \
name); \
abort(); \ abort(); \
} \ } \
} while (0) } while (0)
#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \ #define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) \
if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, \ do { \
newlen) != 0) { \ if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) \
!= 0) { \
malloc_write( \ malloc_write( \
"<jemalloc>: Failure in xmallctlbymib()\n"); \ "<jemalloc>: Failure in xmallctlbymib()\n"); \
abort(); \ abort(); \
} \ } \
} while (0) } while (0)
#define xmallctlmibnametomib(mib, miblen, name, miblenp) do { \ #define xmallctlmibnametomib(mib, miblen, name, miblenp) \
do { \
if (ctl_mibnametomib(tsd_fetch(), mib, miblen, name, miblenp) \ if (ctl_mibnametomib(tsd_fetch(), mib, miblen, name, miblenp) \
!= 0) { \ != 0) { \
malloc_write( \ malloc_write( \
"<jemalloc>: Failure in ctl_mibnametomib()\n"); \ "<jemalloc>: Failure in ctl_mibnametomib()\n"); \
abort(); \ abort(); \
} \ } \
} while (0) } while (0)
#define xmallctlbymibname(mib, miblen, name, miblenp, oldp, oldlenp, \ #define xmallctlbymibname( \
newp, newlen) do { \ mib, miblen, name, miblenp, oldp, oldlenp, newp, newlen) \
do { \
if (ctl_bymibname(tsd_fetch(), mib, miblen, name, miblenp, \ if (ctl_bymibname(tsd_fetch(), mib, miblen, name, miblenp, \
oldp, oldlenp, newp, newlen) != 0) { \ oldp, oldlenp, newp, newlen) \
!= 0) { \
malloc_write( \ malloc_write( \
"<jemalloc>: Failure in ctl_bymibname()\n"); \ "<jemalloc>: Failure in ctl_bymibname()\n"); \
abort(); \ abort(); \
} \ } \
} while (0) } while (0)
#endif /* JEMALLOC_INTERNAL_CTL_H */ #endif /* JEMALLOC_INTERNAL_CTL_H */

14
include/jemalloc/internal/decay.h
View File

@@ -5,7 +5,7 @@
#include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/smoothstep.h"
#define DECAY_UNBOUNDED_TIME_TO_PURGE ((uint64_t)-1) #define DECAY_UNBOUNDED_TIME_TO_PURGE ((uint64_t) - 1)
/* /*
* The decay_t computes the number of pages we should purge at any given time. * The decay_t computes the number of pages we should purge at any given time.
@@ -168,12 +168,12 @@ void decay_reinit(decay_t *decay, nstime_t *cur_time, ssize_t decay_ms);
/* /*
* Compute how many of 'npages_new' pages we would need to purge in 'time'. * Compute how many of 'npages_new' pages we would need to purge in 'time'.
*/ */
uint64_t decay_npages_purge_in(decay_t *decay, nstime_t *time, uint64_t decay_npages_purge_in(
size_t npages_new); decay_t *decay, nstime_t *time, size_t npages_new);
/* Returns true if the epoch advanced and there are pages to purge. */ /* Returns true if the epoch advanced and there are pages to purge. */
bool decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time, bool decay_maybe_advance_epoch(
size_t current_npages); decay_t *decay, nstime_t *new_time, size_t current_npages);
/* /*
* Calculates wait time until a number of pages in the interval * Calculates wait time until a number of pages in the interval
@@ -182,7 +182,7 @@ bool decay_maybe_advance_epoch(decay_t *decay, nstime_t *new_time,
* Returns number of nanoseconds or DECAY_UNBOUNDED_TIME_TO_PURGE in case of * Returns number of nanoseconds or DECAY_UNBOUNDED_TIME_TO_PURGE in case of
* indefinite wait. * indefinite wait.
*/ */
uint64_t decay_ns_until_purge(decay_t *decay, size_t npages_current, uint64_t decay_ns_until_purge(
uint64_t npages_threshold); decay_t *decay, size_t npages_current, uint64_t npages_threshold);
#endif /* JEMALLOC_INTERNAL_DECAY_H */ #endif /* JEMALLOC_INTERNAL_DECAY_H */

12
include/jemalloc/internal/ecache.h
View File

@@ -24,22 +24,22 @@ struct ecache_s {
static inline size_t static inline size_t
ecache_npages_get(ecache_t *ecache) { ecache_npages_get(ecache_t *ecache) {
return eset_npages_get(&ecache->eset) + return eset_npages_get(&ecache->eset)
eset_npages_get(&ecache->guarded_eset); + eset_npages_get(&ecache->guarded_eset);
} }
/* Get the number of extents in the given page size index. */ /* Get the number of extents in the given page size index. */
static inline size_t static inline size_t
ecache_nextents_get(ecache_t *ecache, pszind_t ind) { ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
return eset_nextents_get(&ecache->eset, ind) + return eset_nextents_get(&ecache->eset, ind)
eset_nextents_get(&ecache->guarded_eset, ind); + eset_nextents_get(&ecache->guarded_eset, ind);
} }
/* Get the sum total bytes of the extents in the given page size index. */ /* Get the sum total bytes of the extents in the given page size index. */
static inline size_t static inline size_t
ecache_nbytes_get(ecache_t *ecache, pszind_t ind) { ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
return eset_nbytes_get(&ecache->eset, ind) + return eset_nbytes_get(&ecache->eset, ind)
eset_nbytes_get(&ecache->guarded_eset, ind); + eset_nbytes_get(&ecache->guarded_eset, ind);
} }
static inline unsigned static inline unsigned

205
include/jemalloc/internal/edata.h
View File

@@ -50,10 +50,7 @@ typedef enum extent_head_state_e extent_head_state_t;
* Which implementation of the page allocator interface, (PAI, defined in * Which implementation of the page allocator interface, (PAI, defined in
* pai.h) owns the given extent? * pai.h) owns the given extent?
*/ */
enum extent_pai_e { enum extent_pai_e { EXTENT_PAI_PAC = 0, EXTENT_PAI_HPA = 1 };
EXTENT_PAI_PAC = 0,
EXTENT_PAI_HPA = 1
};
typedef enum extent_pai_e extent_pai_t; typedef enum extent_pai_e extent_pai_t;
struct e_prof_info_s { struct e_prof_info_s {
@@ -150,51 +147,68 @@ struct edata_s {
* bin_shard: the shard of the bin from which this extent came. * bin_shard: the shard of the bin from which this extent came.
*/ */
uint64_t e_bits; uint64_t e_bits;
#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) << (CURRENT_FIELD_SHIFT)) #define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) \
((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) \
<< (CURRENT_FIELD_SHIFT))
#define EDATA_BITS_ARENA_WIDTH MALLOCX_ARENA_BITS #define EDATA_BITS_ARENA_WIDTH MALLOCX_ARENA_BITS
#define EDATA_BITS_ARENA_SHIFT 0 #define EDATA_BITS_ARENA_SHIFT 0
#define EDATA_BITS_ARENA_MASK MASK(EDATA_BITS_ARENA_WIDTH, EDATA_BITS_ARENA_SHIFT) #define EDATA_BITS_ARENA_MASK \
MASK(EDATA_BITS_ARENA_WIDTH, EDATA_BITS_ARENA_SHIFT)
#define EDATA_BITS_SLAB_WIDTH 1 #define EDATA_BITS_SLAB_WIDTH 1
#define EDATA_BITS_SLAB_SHIFT (EDATA_BITS_ARENA_WIDTH + EDATA_BITS_ARENA_SHIFT) #define EDATA_BITS_SLAB_SHIFT (EDATA_BITS_ARENA_WIDTH + EDATA_BITS_ARENA_SHIFT)
#define EDATA_BITS_SLAB_MASK MASK(EDATA_BITS_SLAB_WIDTH, EDATA_BITS_SLAB_SHIFT) #define EDATA_BITS_SLAB_MASK MASK(EDATA_BITS_SLAB_WIDTH, EDATA_BITS_SLAB_SHIFT)
#define EDATA_BITS_COMMITTED_WIDTH 1 #define EDATA_BITS_COMMITTED_WIDTH 1
#define EDATA_BITS_COMMITTED_SHIFT (EDATA_BITS_SLAB_WIDTH + EDATA_BITS_SLAB_SHIFT) #define EDATA_BITS_COMMITTED_SHIFT \
#define EDATA_BITS_COMMITTED_MASK MASK(EDATA_BITS_COMMITTED_WIDTH, EDATA_BITS_COMMITTED_SHIFT) (EDATA_BITS_SLAB_WIDTH + EDATA_BITS_SLAB_SHIFT)
#define EDATA_BITS_COMMITTED_MASK \
MASK(EDATA_BITS_COMMITTED_WIDTH, EDATA_BITS_COMMITTED_SHIFT)
#define EDATA_BITS_PAI_WIDTH 1 #define EDATA_BITS_PAI_WIDTH 1
#define EDATA_BITS_PAI_SHIFT (EDATA_BITS_COMMITTED_WIDTH + EDATA_BITS_COMMITTED_SHIFT) #define EDATA_BITS_PAI_SHIFT \
(EDATA_BITS_COMMITTED_WIDTH + EDATA_BITS_COMMITTED_SHIFT)
#define EDATA_BITS_PAI_MASK MASK(EDATA_BITS_PAI_WIDTH, EDATA_BITS_PAI_SHIFT) #define EDATA_BITS_PAI_MASK MASK(EDATA_BITS_PAI_WIDTH, EDATA_BITS_PAI_SHIFT)
#define EDATA_BITS_ZEROED_WIDTH 1 #define EDATA_BITS_ZEROED_WIDTH 1
#define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT) #define EDATA_BITS_ZEROED_SHIFT (EDATA_BITS_PAI_WIDTH + EDATA_BITS_PAI_SHIFT)
#define EDATA_BITS_ZEROED_MASK MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT) #define EDATA_BITS_ZEROED_MASK \
MASK(EDATA_BITS_ZEROED_WIDTH, EDATA_BITS_ZEROED_SHIFT)
#define EDATA_BITS_GUARDED_WIDTH 1 #define EDATA_BITS_GUARDED_WIDTH 1
#define EDATA_BITS_GUARDED_SHIFT (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT) #define EDATA_BITS_GUARDED_SHIFT \
#define EDATA_BITS_GUARDED_MASK MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT) (EDATA_BITS_ZEROED_WIDTH + EDATA_BITS_ZEROED_SHIFT)
#define EDATA_BITS_GUARDED_MASK \
MASK(EDATA_BITS_GUARDED_WIDTH, EDATA_BITS_GUARDED_SHIFT)
#define EDATA_BITS_STATE_WIDTH 3 #define EDATA_BITS_STATE_WIDTH 3
#define EDATA_BITS_STATE_SHIFT (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT) #define EDATA_BITS_STATE_SHIFT \
#define EDATA_BITS_STATE_MASK MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT) (EDATA_BITS_GUARDED_WIDTH + EDATA_BITS_GUARDED_SHIFT)
#define EDATA_BITS_STATE_MASK \
MASK(EDATA_BITS_STATE_WIDTH, EDATA_BITS_STATE_SHIFT)
#define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) #define EDATA_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES)
#define EDATA_BITS_SZIND_SHIFT (EDATA_BITS_STATE_WIDTH + EDATA_BITS_STATE_SHIFT) #define EDATA_BITS_SZIND_SHIFT (EDATA_BITS_STATE_WIDTH + EDATA_BITS_STATE_SHIFT)
#define EDATA_BITS_SZIND_MASK MASK(EDATA_BITS_SZIND_WIDTH, EDATA_BITS_SZIND_SHIFT) #define EDATA_BITS_SZIND_MASK \
MASK(EDATA_BITS_SZIND_WIDTH, EDATA_BITS_SZIND_SHIFT)
#define EDATA_BITS_NFREE_WIDTH (SC_LG_SLAB_MAXREGS + 1) #define EDATA_BITS_NFREE_WIDTH (SC_LG_SLAB_MAXREGS + 1)
#define EDATA_BITS_NFREE_SHIFT (EDATA_BITS_SZIND_WIDTH + EDATA_BITS_SZIND_SHIFT) #define EDATA_BITS_NFREE_SHIFT (EDATA_BITS_SZIND_WIDTH + EDATA_BITS_SZIND_SHIFT)
#define EDATA_BITS_NFREE_MASK MASK(EDATA_BITS_NFREE_WIDTH, EDATA_BITS_NFREE_SHIFT) #define EDATA_BITS_NFREE_MASK \
MASK(EDATA_BITS_NFREE_WIDTH, EDATA_BITS_NFREE_SHIFT)
#define EDATA_BITS_BINSHARD_WIDTH 6 #define EDATA_BITS_BINSHARD_WIDTH 6
#define EDATA_BITS_BINSHARD_SHIFT (EDATA_BITS_NFREE_WIDTH + EDATA_BITS_NFREE_SHIFT) #define EDATA_BITS_BINSHARD_SHIFT \
#define EDATA_BITS_BINSHARD_MASK MASK(EDATA_BITS_BINSHARD_WIDTH, EDATA_BITS_BINSHARD_SHIFT) (EDATA_BITS_NFREE_WIDTH + EDATA_BITS_NFREE_SHIFT)
#define EDATA_BITS_BINSHARD_MASK \
MASK(EDATA_BITS_BINSHARD_WIDTH, EDATA_BITS_BINSHARD_SHIFT)
#define EDATA_BITS_IS_HEAD_WIDTH 1 #define EDATA_BITS_IS_HEAD_WIDTH 1
#define EDATA_BITS_IS_HEAD_SHIFT (EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT) #define EDATA_BITS_IS_HEAD_SHIFT \
#define EDATA_BITS_IS_HEAD_MASK MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT) (EDATA_BITS_BINSHARD_WIDTH + EDATA_BITS_BINSHARD_SHIFT)
#define EDATA_BITS_IS_HEAD_MASK \
MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
/* Pointer to the extent that this structure is responsible for. */ /* Pointer to the extent that this structure is responsible for. */
void *e_addr; void *e_addr;
@@ -208,8 +222,8 @@ struct edata_s {
* ssssssss [...] ssssssss ssssnnnn nnnnnnnn * ssssssss [...] ssssssss ssssnnnn nnnnnnnn
*/ */
size_t e_size_esn; size_t e_size_esn;
#define EDATA_SIZE_MASK ((size_t)~(PAGE-1)) #define EDATA_SIZE_MASK ((size_t) ~(PAGE - 1))
#define EDATA_ESN_MASK ((size_t)PAGE-1) #define EDATA_ESN_MASK ((size_t)PAGE - 1)
/* Base extent size, which may not be a multiple of PAGE. */ /* Base extent size, which may not be a multiple of PAGE. */
size_t e_bsize; size_t e_bsize;
}; };
@@ -265,8 +279,8 @@ TYPED_LIST(edata_list_inactive, edata_t, ql_link_inactive)
static inline unsigned static inline unsigned
edata_arena_ind_get(const edata_t *edata) { edata_arena_ind_get(const edata_t *edata) {
unsigned arena_ind = (unsigned)((edata->e_bits & unsigned arena_ind = (unsigned)((edata->e_bits & EDATA_BITS_ARENA_MASK)
EDATA_BITS_ARENA_MASK) >> EDATA_BITS_ARENA_SHIFT); >> EDATA_BITS_ARENA_SHIFT);
assert(arena_ind < MALLOCX_ARENA_LIMIT); assert(arena_ind < MALLOCX_ARENA_LIMIT);
return arena_ind; return arena_ind;
@@ -274,8 +288,8 @@ edata_arena_ind_get(const edata_t *edata) {
static inline szind_t static inline szind_t
edata_szind_get_maybe_invalid(const edata_t *edata) { edata_szind_get_maybe_invalid(const edata_t *edata) {
szind_t szind = (szind_t)((edata->e_bits & EDATA_BITS_SZIND_MASK) >> szind_t szind = (szind_t)((edata->e_bits & EDATA_BITS_SZIND_MASK)
EDATA_BITS_SZIND_SHIFT); >> EDATA_BITS_SZIND_SHIFT);
assert(szind <= SC_NSIZES); assert(szind <= SC_NSIZES);
return szind; return szind;
} }
@@ -318,8 +332,8 @@ edata_usize_get(const edata_t *edata) {
if (!sz_large_size_classes_disabled() || szind < SC_NBINS) { if (!sz_large_size_classes_disabled() || szind < SC_NBINS) {
size_t usize_from_ind = sz_index2size(szind); size_t usize_from_ind = sz_index2size(szind);
if (!sz_large_size_classes_disabled() && if (!sz_large_size_classes_disabled()
usize_from_ind >= SC_LARGE_MINCLASS) { && usize_from_ind >= SC_LARGE_MINCLASS) {
size_t size = (edata->e_size_esn & EDATA_SIZE_MASK); size_t size = (edata->e_size_esn & EDATA_SIZE_MASK);
assert(size > sz_large_pad); assert(size > sz_large_pad);
size_t usize_from_size = size - sz_large_pad; size_t usize_from_size = size - sz_large_pad;
@@ -341,8 +355,9 @@ edata_usize_get(const edata_t *edata) {
static inline unsigned static inline unsigned
edata_binshard_get(const edata_t *edata) { edata_binshard_get(const edata_t *edata) {
unsigned binshard = (unsigned)((edata->e_bits & unsigned binshard = (unsigned)((edata->e_bits
EDATA_BITS_BINSHARD_MASK) >> EDATA_BITS_BINSHARD_SHIFT); & EDATA_BITS_BINSHARD_MASK)
>> EDATA_BITS_BINSHARD_SHIFT);
assert(binshard < bin_infos[edata_szind_get(edata)].n_shards); assert(binshard < bin_infos[edata_szind_get(edata)].n_shards);
return binshard; return binshard;
} }
@@ -354,58 +369,58 @@ edata_sn_get(const edata_t *edata) {
static inline extent_state_t static inline extent_state_t
edata_state_get(const edata_t *edata) { edata_state_get(const edata_t *edata) {
return (extent_state_t)((edata->e_bits & EDATA_BITS_STATE_MASK) >> return (extent_state_t)((edata->e_bits & EDATA_BITS_STATE_MASK)
EDATA_BITS_STATE_SHIFT); >> EDATA_BITS_STATE_SHIFT);
} }
static inline bool static inline bool
edata_guarded_get(const edata_t *edata) { edata_guarded_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK) >> return (bool)((edata->e_bits & EDATA_BITS_GUARDED_MASK)
EDATA_BITS_GUARDED_SHIFT); >> EDATA_BITS_GUARDED_SHIFT);
} }
static inline bool static inline bool
edata_zeroed_get(const edata_t *edata) { edata_zeroed_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK) >> return (bool)((edata->e_bits & EDATA_BITS_ZEROED_MASK)
EDATA_BITS_ZEROED_SHIFT); >> EDATA_BITS_ZEROED_SHIFT);
} }
static inline bool static inline bool
edata_committed_get(const edata_t *edata) { edata_committed_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_COMMITTED_MASK) >> return (bool)((edata->e_bits & EDATA_BITS_COMMITTED_MASK)
EDATA_BITS_COMMITTED_SHIFT); >> EDATA_BITS_COMMITTED_SHIFT);
} }
static inline extent_pai_t static inline extent_pai_t
edata_pai_get(const edata_t *edata) { edata_pai_get(const edata_t *edata) {
return (extent_pai_t)((edata->e_bits & EDATA_BITS_PAI_MASK) >> return (extent_pai_t)((edata->e_bits & EDATA_BITS_PAI_MASK)
EDATA_BITS_PAI_SHIFT); >> EDATA_BITS_PAI_SHIFT);
} }
static inline bool static inline bool
edata_slab_get(const edata_t *edata) { edata_slab_get(const edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK) >> return (bool)((edata->e_bits & EDATA_BITS_SLAB_MASK)
EDATA_BITS_SLAB_SHIFT); >> EDATA_BITS_SLAB_SHIFT);
} }
static inline unsigned static inline unsigned
edata_nfree_get(const edata_t *edata) { edata_nfree_get(const edata_t *edata) {
assert(edata_slab_get(edata)); assert(edata_slab_get(edata));
return (unsigned)((edata->e_bits & EDATA_BITS_NFREE_MASK) >> return (unsigned)((edata->e_bits & EDATA_BITS_NFREE_MASK)
EDATA_BITS_NFREE_SHIFT); >> EDATA_BITS_NFREE_SHIFT);
} }
static inline void * static inline void *
edata_base_get(const edata_t *edata) { edata_base_get(const edata_t *edata) {
assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) || assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr)
!edata_slab_get(edata)); || !edata_slab_get(edata));
return PAGE_ADDR2BASE(edata->e_addr); return PAGE_ADDR2BASE(edata->e_addr);
} }
static inline void * static inline void *
edata_addr_get(const edata_t *edata) { edata_addr_get(const edata_t *edata) {
assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr) || assert(edata->e_addr == PAGE_ADDR2BASE(edata->e_addr)
!edata_slab_get(edata)); || !edata_slab_get(edata));
return edata->e_addr; return edata->e_addr;
} }
@@ -437,14 +452,14 @@ edata_before_get(const edata_t *edata) {
static inline void * static inline void *
edata_last_get(const edata_t *edata) { edata_last_get(const edata_t *edata) {
return (void *)((byte_t *)edata_base_get(edata) + return (void *)((byte_t *)edata_base_get(edata) + edata_size_get(edata)
edata_size_get(edata) - PAGE); - PAGE);
} }
static inline void * static inline void *
edata_past_get(const edata_t *edata) { edata_past_get(const edata_t *edata) {
return (void *)((byte_t *)edata_base_get(edata) + return (
edata_size_get(edata)); void *)((byte_t *)edata_base_get(edata) + edata_size_get(edata));
} }
static inline slab_data_t * static inline slab_data_t *
@@ -461,8 +476,8 @@ edata_slab_data_get_const(const edata_t *edata) {
static inline prof_tctx_t * static inline prof_tctx_t *
edata_prof_tctx_get(const edata_t *edata) { edata_prof_tctx_get(const edata_t *edata) {
return (prof_tctx_t *)atomic_load_p(&edata->e_prof_info.e_prof_tctx, return (prof_tctx_t *)atomic_load_p(
ATOMIC_ACQUIRE); &edata->e_prof_info.e_prof_tctx, ATOMIC_ACQUIRE);
} }
static inline const nstime_t * static inline const nstime_t *
@@ -483,16 +498,16 @@ edata_prof_recent_alloc_get_dont_call_directly(const edata_t *edata) {
static inline void static inline void
edata_arena_ind_set(edata_t *edata, unsigned arena_ind) { edata_arena_ind_set(edata_t *edata, unsigned arena_ind) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_ARENA_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_ARENA_MASK)
((uint64_t)arena_ind << EDATA_BITS_ARENA_SHIFT); | ((uint64_t)arena_ind << EDATA_BITS_ARENA_SHIFT);
} }
static inline void static inline void
edata_binshard_set(edata_t *edata, unsigned binshard) { edata_binshard_set(edata_t *edata, unsigned binshard) {
/* The assertion assumes szind is set already. */ /* The assertion assumes szind is set already. */
assert(binshard < bin_infos[edata_szind_get(edata)].n_shards); assert(binshard < bin_infos[edata_szind_get(edata)].n_shards);
edata->e_bits = (edata->e_bits & ~EDATA_BITS_BINSHARD_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_BINSHARD_MASK)
((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT); | ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT);
} }
static inline void static inline void
@@ -508,8 +523,8 @@ edata_size_set(edata_t *edata, size_t size) {
static inline void static inline void
edata_esn_set(edata_t *edata, size_t esn) { edata_esn_set(edata_t *edata, size_t esn) {
edata->e_size_esn = (edata->e_size_esn & ~EDATA_ESN_MASK) | (esn & edata->e_size_esn = (edata->e_size_esn & ~EDATA_ESN_MASK)
EDATA_ESN_MASK); | (esn & EDATA_ESN_MASK);
} }
static inline void static inline void
@@ -526,25 +541,26 @@ edata_ps_set(edata_t *edata, hpdata_t *ps) {
static inline void static inline void
edata_szind_set(edata_t *edata, szind_t szind) { edata_szind_set(edata_t *edata, szind_t szind) {
assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
edata->e_bits = (edata->e_bits & ~EDATA_BITS_SZIND_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_SZIND_MASK)
((uint64_t)szind << EDATA_BITS_SZIND_SHIFT); | ((uint64_t)szind << EDATA_BITS_SZIND_SHIFT);
} }
static inline void static inline void
edata_nfree_set(edata_t *edata, unsigned nfree) { edata_nfree_set(edata_t *edata, unsigned nfree) {
assert(edata_slab_get(edata)); assert(edata_slab_get(edata));
edata->e_bits = (edata->e_bits & ~EDATA_BITS_NFREE_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_NFREE_MASK)
((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT); | ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT);
} }
static inline void static inline void
edata_nfree_binshard_set(edata_t *edata, unsigned nfree, unsigned binshard) { edata_nfree_binshard_set(edata_t *edata, unsigned nfree, unsigned binshard) {
/* The assertion assumes szind is set already. */ /* The assertion assumes szind is set already. */
assert(binshard < bin_infos[edata_szind_get(edata)].n_shards); assert(binshard < bin_infos[edata_szind_get(edata)].n_shards);
edata->e_bits = (edata->e_bits & edata->e_bits = (edata->e_bits
(~EDATA_BITS_NFREE_MASK & ~EDATA_BITS_BINSHARD_MASK)) | & (~EDATA_BITS_NFREE_MASK
((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT) | & ~EDATA_BITS_BINSHARD_MASK))
((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT); | ((uint64_t)binshard << EDATA_BITS_BINSHARD_SHIFT)
| ((uint64_t)nfree << EDATA_BITS_NFREE_SHIFT);
} }
static inline void static inline void
@@ -572,38 +588,38 @@ edata_sn_set(edata_t *edata, uint64_t sn) {
static inline void static inline void
edata_state_set(edata_t *edata, extent_state_t state) { edata_state_set(edata_t *edata, extent_state_t state) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_STATE_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_STATE_MASK)
((uint64_t)state << EDATA_BITS_STATE_SHIFT); | ((uint64_t)state << EDATA_BITS_STATE_SHIFT);
} }
static inline void static inline void
edata_guarded_set(edata_t *edata, bool guarded) { edata_guarded_set(edata_t *edata, bool guarded) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_GUARDED_MASK)
((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT); | ((uint64_t)guarded << EDATA_BITS_GUARDED_SHIFT);
} }
static inline void static inline void
edata_zeroed_set(edata_t *edata, bool zeroed) { edata_zeroed_set(edata_t *edata, bool zeroed) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_ZEROED_MASK)
((uint64_t)zeroed << EDATA_BITS_ZEROED_SHIFT); | ((uint64_t)zeroed << EDATA_BITS_ZEROED_SHIFT);
} }
static inline void static inline void
edata_committed_set(edata_t *edata, bool committed) { edata_committed_set(edata_t *edata, bool committed) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_COMMITTED_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_COMMITTED_MASK)
((uint64_t)committed << EDATA_BITS_COMMITTED_SHIFT); | ((uint64_t)committed << EDATA_BITS_COMMITTED_SHIFT);
} }
static inline void static inline void
edata_pai_set(edata_t *edata, extent_pai_t pai) { edata_pai_set(edata_t *edata, extent_pai_t pai) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_PAI_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_PAI_MASK)
((uint64_t)pai << EDATA_BITS_PAI_SHIFT); | ((uint64_t)pai << EDATA_BITS_PAI_SHIFT);
} }
static inline void static inline void
edata_slab_set(edata_t *edata, bool slab) { edata_slab_set(edata_t *edata, bool slab) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_SLAB_MASK)
((uint64_t)slab << EDATA_BITS_SLAB_SHIFT); | ((uint64_t)slab << EDATA_BITS_SLAB_SHIFT);
} }
static inline void static inline void
@@ -622,22 +638,22 @@ edata_prof_alloc_size_set(edata_t *edata, size_t size) {
} }
static inline void static inline void
edata_prof_recent_alloc_set_dont_call_directly(edata_t *edata, edata_prof_recent_alloc_set_dont_call_directly(
prof_recent_t *recent_alloc) { edata_t *edata, prof_recent_t *recent_alloc) {
atomic_store_p(&edata->e_prof_info.e_prof_recent_alloc, recent_alloc, atomic_store_p(&edata->e_prof_info.e_prof_recent_alloc, recent_alloc,
ATOMIC_RELAXED); ATOMIC_RELAXED);
} }
static inline bool static inline bool
edata_is_head_get(edata_t *edata) { edata_is_head_get(edata_t *edata) {
return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK) >> return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK)
EDATA_BITS_IS_HEAD_SHIFT); >> EDATA_BITS_IS_HEAD_SHIFT);
} }
static inline void static inline void
edata_is_head_set(edata_t *edata, bool is_head) { edata_is_head_set(edata_t *edata, bool is_head) {
edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK) | edata->e_bits = (edata->e_bits & ~EDATA_BITS_IS_HEAD_MASK)
((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT); | ((uint64_t)is_head << EDATA_BITS_IS_HEAD_SHIFT);
} }
static inline bool static inline bool
@@ -676,8 +692,8 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
} }
static inline void static inline void
edata_binit(edata_t *edata, void *addr, size_t bsize, uint64_t sn, edata_binit(
bool reused) { edata_t *edata, void *addr, size_t bsize, uint64_t sn, bool reused) {
edata_arena_ind_set(edata, (1U << MALLOCX_ARENA_BITS) - 1); edata_arena_ind_set(edata, (1U << MALLOCX_ARENA_BITS) - 1);
edata_addr_set(edata, addr); edata_addr_set(edata, addr);
edata_bsize_set(edata, bsize); edata_bsize_set(edata, bsize);
@@ -731,8 +747,10 @@ static inline int
edata_cmp_summary_comp(edata_cmp_summary_t a, edata_cmp_summary_t b) { edata_cmp_summary_comp(edata_cmp_summary_t a, edata_cmp_summary_t b) {
unsigned __int128 a_encoded = edata_cmp_summary_encode(a); unsigned __int128 a_encoded = edata_cmp_summary_encode(a);
unsigned __int128 b_encoded = edata_cmp_summary_encode(b); unsigned __int128 b_encoded = edata_cmp_summary_encode(b);
if (a_encoded < b_encoded) return -1; if (a_encoded < b_encoded)
if (a_encoded == b_encoded) return 0; return -1;
if (a_encoded == b_encoded)
return 0;
return 1; return 1;
} }
#else #else
@@ -750,8 +768,8 @@ edata_cmp_summary_comp(edata_cmp_summary_t a, edata_cmp_summary_t b) {
* prediction accuracy is not great. As a result, this implementation * prediction accuracy is not great. As a result, this implementation
* is measurably faster (by around 30%). * is measurably faster (by around 30%).
*/ */
return (2 * ((a.sn > b.sn) - (a.sn < b.sn))) + return (2 * ((a.sn > b.sn) - (a.sn < b.sn)))
((a.addr > b.addr) - (a.addr < b.addr)); + ((a.addr > b.addr) - (a.addr < b.addr));
} }
#endif #endif
@@ -772,7 +790,6 @@ edata_esnead_comp(const edata_t *a, const edata_t *b) {
return (2 * edata_esn_comp(a, b)) + edata_ead_comp(a, b); return (2 * edata_esn_comp(a, b)) + edata_ead_comp(a, b);
} }
ph_proto(, edata_avail, edata_t) ph_proto(, edata_avail, edata_t) ph_proto(, edata_heap, edata_t)
ph_proto(, edata_heap, edata_t)
#endif /* JEMALLOC_INTERNAL_EDATA_H */ #endif /* JEMALLOC_INTERNAL_EDATA_H */

4
include/jemalloc/internal/edata_cache.h
View File

@@ -43,8 +43,8 @@ struct edata_cache_fast_s {
void edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback); void edata_cache_fast_init(edata_cache_fast_t *ecs, edata_cache_t *fallback);
edata_t *edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs); edata_t *edata_cache_fast_get(tsdn_t *tsdn, edata_cache_fast_t *ecs);
void edata_cache_fast_put(tsdn_t *tsdn, edata_cache_fast_t *ecs, void edata_cache_fast_put(
edata_t *edata); tsdn_t *tsdn, edata_cache_fast_t *ecs, edata_t *edata);
void edata_cache_fast_disable(tsdn_t *tsdn, edata_cache_fast_t *ecs); void edata_cache_fast_disable(tsdn_t *tsdn, edata_cache_fast_t *ecs);
#endif /* JEMALLOC_INTERNAL_EDATA_CACHE_H */ #endif /* JEMALLOC_INTERNAL_EDATA_CACHE_H */

21
include/jemalloc/internal/ehooks.h
View File

@@ -116,8 +116,8 @@ ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
static inline bool static inline bool
ehooks_are_default(ehooks_t *ehooks) { ehooks_are_default(ehooks_t *ehooks) {
return ehooks_get_extent_hooks_ptr(ehooks) == return ehooks_get_extent_hooks_ptr(ehooks)
&ehooks_default_extent_hooks; == &ehooks_default_extent_hooks;
} }
/* /*
@@ -189,7 +189,6 @@ ehooks_debug_zero_check(void *addr, size_t size) {
} }
} }
static inline void * static inline void *
ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size, ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit) { size_t alignment, bool *zero, bool *commit) {
@@ -197,8 +196,8 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
void *ret; void *ret;
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks == &ehooks_default_extent_hooks) { if (extent_hooks == &ehooks_default_extent_hooks) {
ret = ehooks_default_alloc_impl(tsdn, new_addr, size, ret = ehooks_default_alloc_impl(tsdn, new_addr, size, alignment,
alignment, zero, commit, ehooks_ind_get(ehooks)); zero, commit, ehooks_ind_get(ehooks));
} else { } else {
ehooks_pre_reentrancy(tsdn); ehooks_pre_reentrancy(tsdn);
ret = extent_hooks->alloc(extent_hooks, new_addr, size, ret = extent_hooks->alloc(extent_hooks, new_addr, size,
@@ -214,8 +213,8 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
} }
static inline bool static inline bool
ehooks_dalloc(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, ehooks_dalloc(
bool committed) { tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, bool committed) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks == &ehooks_default_extent_hooks) { if (extent_hooks == &ehooks_default_extent_hooks) {
return ehooks_default_dalloc_impl(addr, size); return ehooks_default_dalloc_impl(addr, size);
@@ -231,8 +230,8 @@ ehooks_dalloc(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
} }
static inline void static inline void
ehooks_destroy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, ehooks_destroy(
bool committed) { tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, bool committed) {
extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
if (extent_hooks == &ehooks_default_extent_hooks) { if (extent_hooks == &ehooks_default_extent_hooks) {
ehooks_default_destroy_impl(addr, size); ehooks_default_destroy_impl(addr, size);
@@ -257,8 +256,8 @@ ehooks_commit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size,
err = true; err = true;
} else { } else {
ehooks_pre_reentrancy(tsdn); ehooks_pre_reentrancy(tsdn);
err = extent_hooks->commit(extent_hooks, addr, size, err = extent_hooks->commit(extent_hooks, addr, size, offset,
offset, length, ehooks_ind_get(ehooks)); length, ehooks_ind_get(ehooks));
ehooks_post_reentrancy(tsdn); ehooks_post_reentrancy(tsdn);
} }
if (!err) { if (!err) {

83
include/jemalloc/internal/emap.h
View File

@@ -36,11 +36,11 @@ struct emap_full_alloc_ctx_s {
bool emap_init(emap_t *emap, base_t *base, bool zeroed); bool emap_init(emap_t *emap, base_t *base, bool zeroed);
void emap_remap(tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, void emap_remap(
bool slab); tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab);
void emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata, void emap_update_edata_state(
extent_state_t state); tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t state);
/* /*
* The two acquire functions below allow accessing neighbor edatas, if it's safe * The two acquire functions below allow accessing neighbor edatas, if it's safe
@@ -62,16 +62,16 @@ edata_t *emap_try_acquire_edata_neighbor(tsdn_t *tsdn, emap_t *emap,
bool forward); bool forward);
edata_t *emap_try_acquire_edata_neighbor_expand(tsdn_t *tsdn, emap_t *emap, edata_t *emap_try_acquire_edata_neighbor_expand(tsdn_t *tsdn, emap_t *emap,
edata_t *edata, extent_pai_t pai, extent_state_t expected_state); edata_t *edata, extent_pai_t pai, extent_state_t expected_state);
void emap_release_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata, void emap_release_edata(
extent_state_t new_state); tsdn_t *tsdn, emap_t *emap, edata_t *edata, extent_state_t new_state);
/* /*
* Associate the given edata with its beginning and end address, setting the * Associate the given edata with its beginning and end address, setting the
* szind and slab info appropriately. * szind and slab info appropriately.
* Returns true on error (i.e. resource exhaustion). * Returns true on error (i.e. resource exhaustion).
*/ */
bool emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata, bool emap_register_boundary(
szind_t szind, bool slab); tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind, bool slab);
/* /*
* Does the same thing, but with the interior of the range, for slab * Does the same thing, but with the interior of the range, for slab
@@ -92,8 +92,8 @@ bool emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
* touched, so no allocation is necessary to fill the interior once the boundary * touched, so no allocation is necessary to fill the interior once the boundary
* has been touched. * has been touched.
*/ */
void emap_register_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata, void emap_register_interior(
szind_t szind); tsdn_t *tsdn, emap_t *emap, edata_t *edata, szind_t szind);
void emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata); void emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata);
void emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata); void emap_deregister_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata);
@@ -161,8 +161,8 @@ emap_edata_in_transition(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
emap_assert_mapped(tsdn, emap, edata); emap_assert_mapped(tsdn, emap, edata);
EMAP_DECLARE_RTREE_CTX; EMAP_DECLARE_RTREE_CTX;
rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx, rtree_contents_t contents = rtree_read(
(uintptr_t)edata_base_get(edata)); tsdn, &emap->rtree, rtree_ctx, (uintptr_t)edata_base_get(edata));
return edata_state_in_transition(contents.metadata.state); return edata_state_in_transition(contents.metadata.state);
} }
@@ -194,9 +194,9 @@ emap_edata_is_acquired(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
} }
rtree_contents_t contents = rtree_leaf_elm_read(tsdn, &emap->rtree, elm, rtree_contents_t contents = rtree_leaf_elm_read(tsdn, &emap->rtree, elm,
/* dependent */ false); /* dependent */ false);
if (contents.edata == NULL || if (contents.edata == NULL
contents.metadata.state == extent_state_active || || contents.metadata.state == extent_state_active
edata_state_in_transition(contents.metadata.state)) { || edata_state_in_transition(contents.metadata.state)) {
return true; return true;
} }
@@ -211,8 +211,8 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) {
assert(edata_state_get(inner) == extent_state_active); assert(edata_state_get(inner) == extent_state_active);
assert(edata_state_get(outer) == extent_state_merging); assert(edata_state_get(outer) == extent_state_merging);
assert(!edata_guarded_get(inner) && !edata_guarded_get(outer)); assert(!edata_guarded_get(inner) && !edata_guarded_get(outer));
assert(edata_base_get(inner) == edata_past_get(outer) || assert(edata_base_get(inner) == edata_past_get(outer)
edata_base_get(outer) == edata_past_get(inner)); || edata_base_get(outer) == edata_past_get(inner));
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
@@ -232,13 +232,13 @@ emap_edata_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr) {
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
emap_alloc_ctx_init(emap_alloc_ctx_t *alloc_ctx, szind_t szind, bool slab, emap_alloc_ctx_init(
size_t usize) { emap_alloc_ctx_t *alloc_ctx, szind_t szind, bool slab, size_t usize) {
alloc_ctx->szind = szind; alloc_ctx->szind = szind;
alloc_ctx->slab = slab; alloc_ctx->slab = slab;
alloc_ctx->usize = usize; alloc_ctx->usize = usize;
assert(sz_large_size_classes_disabled() || assert(
usize == sz_index2size(szind)); sz_large_size_classes_disabled() || usize == sz_index2size(szind));
} }
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
@@ -248,27 +248,29 @@ emap_alloc_ctx_usize_get(emap_alloc_ctx_t *alloc_ctx) {
assert(alloc_ctx->usize == sz_index2size(alloc_ctx->szind)); assert(alloc_ctx->usize == sz_index2size(alloc_ctx->szind));
return sz_index2size(alloc_ctx->szind); return sz_index2size(alloc_ctx->szind);
} }
assert(sz_large_size_classes_disabled() || assert(sz_large_size_classes_disabled()
alloc_ctx->usize == sz_index2size(alloc_ctx->szind)); || alloc_ctx->usize == sz_index2size(alloc_ctx->szind));
assert(alloc_ctx->usize <= SC_LARGE_MAXCLASS); assert(alloc_ctx->usize <= SC_LARGE_MAXCLASS);
return alloc_ctx->usize; return alloc_ctx->usize;
} }
/* Fills in alloc_ctx with the info in the map. */ /* Fills in alloc_ctx with the info in the map. */
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
emap_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr, emap_alloc_ctx_lookup(
emap_alloc_ctx_t *alloc_ctx) { tsdn_t *tsdn, emap_t *emap, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
EMAP_DECLARE_RTREE_CTX; EMAP_DECLARE_RTREE_CTX;
rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_contents_t contents = rtree_read(
rtree_ctx, (uintptr_t)ptr); tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr);
/* /*
* If the alloc is invalid, do not calculate usize since edata * If the alloc is invalid, do not calculate usize since edata
* could be corrupted. * could be corrupted.
*/ */
emap_alloc_ctx_init(alloc_ctx, contents.metadata.szind, emap_alloc_ctx_init(alloc_ctx, contents.metadata.szind,
contents.metadata.slab, (contents.metadata.szind == SC_NSIZES contents.metadata.slab,
|| contents.edata == NULL)? 0: edata_usize_get(contents.edata)); (contents.metadata.szind == SC_NSIZES || contents.edata == NULL)
? 0
: edata_usize_get(contents.edata));
} }
/* The pointer must be mapped. */ /* The pointer must be mapped. */
@@ -277,8 +279,8 @@ emap_full_alloc_ctx_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
emap_full_alloc_ctx_t *full_alloc_ctx) { emap_full_alloc_ctx_t *full_alloc_ctx) {
EMAP_DECLARE_RTREE_CTX; EMAP_DECLARE_RTREE_CTX;
rtree_contents_t contents = rtree_read(tsdn, &emap->rtree, rtree_ctx, rtree_contents_t contents = rtree_read(
(uintptr_t)ptr); tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr);
full_alloc_ctx->edata = contents.edata; full_alloc_ctx->edata = contents.edata;
full_alloc_ctx->szind = contents.metadata.szind; full_alloc_ctx->szind = contents.metadata.szind;
full_alloc_ctx->slab = contents.metadata.slab; full_alloc_ctx->slab = contents.metadata.slab;
@@ -295,8 +297,8 @@ emap_full_alloc_ctx_try_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
EMAP_DECLARE_RTREE_CTX; EMAP_DECLARE_RTREE_CTX;
rtree_contents_t contents; rtree_contents_t contents;
bool err = rtree_read_independent(tsdn, &emap->rtree, rtree_ctx, bool err = rtree_read_independent(
(uintptr_t)ptr, &contents); tsdn, &emap->rtree, rtree_ctx, (uintptr_t)ptr, &contents);
if (err) { if (err) {
return true; return true;
} }
@@ -311,14 +313,14 @@ emap_full_alloc_ctx_try_lookup(tsdn_t *tsdn, emap_t *emap, const void *ptr,
* fast path, e.g. when the metadata key is not cached. * fast path, e.g. when the metadata key is not cached.
*/ */
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
emap_alloc_ctx_try_lookup_fast(tsd_t *tsd, emap_t *emap, const void *ptr, emap_alloc_ctx_try_lookup_fast(
emap_alloc_ctx_t *alloc_ctx) { tsd_t *tsd, emap_t *emap, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
/* Use the unsafe getter since this may gets called during exit. */ /* Use the unsafe getter since this may gets called during exit. */
rtree_ctx_t *rtree_ctx = tsd_rtree_ctxp_get_unsafe(tsd); rtree_ctx_t *rtree_ctx = tsd_rtree_ctxp_get_unsafe(tsd);
rtree_metadata_t metadata; rtree_metadata_t metadata;
bool err = rtree_metadata_try_read_fast(tsd_tsdn(tsd), &emap->rtree, bool err = rtree_metadata_try_read_fast(
rtree_ctx, (uintptr_t)ptr, &metadata); tsd_tsdn(tsd), &emap->rtree, rtree_ctx, (uintptr_t)ptr, &metadata);
if (err) { if (err) {
return true; return true;
} }
@@ -345,7 +347,8 @@ typedef const void *(*emap_ptr_getter)(void *ctx, size_t ind);
* This allows size-checking assertions, which we can only do while we're in the * This allows size-checking assertions, which we can only do while we're in the
* process of edata lookups. * process of edata lookups.
*/ */
typedef void (*emap_metadata_visitor)(void *ctx, emap_full_alloc_ctx_t *alloc_ctx); typedef void (*emap_metadata_visitor)(
void *ctx, emap_full_alloc_ctx_t *alloc_ctx);
typedef union emap_batch_lookup_result_u emap_batch_lookup_result_t; typedef union emap_batch_lookup_result_u emap_batch_lookup_result_t;
union emap_batch_lookup_result_u { union emap_batch_lookup_result_u {
@@ -375,8 +378,8 @@ emap_edata_lookup_batch(tsd_t *tsd, emap_t *emap, size_t nptrs,
for (size_t i = 0; i < nptrs; i++) { for (size_t i = 0; i < nptrs; i++) {
rtree_leaf_elm_t *elm = result[i].rtree_leaf; rtree_leaf_elm_t *elm = result[i].rtree_leaf;
rtree_contents_t contents = rtree_leaf_elm_read(tsd_tsdn(tsd), rtree_contents_t contents = rtree_leaf_elm_read(
&emap->rtree, elm, /* dependent */ true); tsd_tsdn(tsd), &emap->rtree, elm, /* dependent */ true);
result[i].edata = contents.edata; result[i].edata = contents.edata;
emap_full_alloc_ctx_t alloc_ctx; emap_full_alloc_ctx_t alloc_ctx;
/* /*

66
include/jemalloc/internal/emitter.h
View File

@@ -83,8 +83,8 @@ struct emitter_s {
static inline bool static inline bool
emitter_outputs_json(emitter_t *emitter) { emitter_outputs_json(emitter_t *emitter) {
return emitter->output == emitter_output_json || return emitter->output == emitter_output_json
emitter->output == emitter_output_json_compact; || emitter->output == emitter_output_json_compact;
} }
/* Internal convenience function. Write to the emitter the given string. */ /* Internal convenience function. Write to the emitter the given string. */
@@ -98,20 +98,20 @@ emitter_printf(emitter_t *emitter, const char *format, ...) {
va_end(ap); va_end(ap);
} }
static inline const char * JEMALLOC_FORMAT_ARG(3) static inline const char *
emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, JEMALLOC_FORMAT_ARG(3) emitter_gen_fmt(char *out_fmt, size_t out_size,
emitter_justify_t justify, int width) { const char *fmt_specifier, emitter_justify_t justify, int width) {
size_t written; size_t written;
fmt_specifier++; fmt_specifier++;
if (justify == emitter_justify_none) { if (justify == emitter_justify_none) {
written = malloc_snprintf(out_fmt, out_size, written = malloc_snprintf(
"%%%s", fmt_specifier); out_fmt, out_size, "%%%s", fmt_specifier);
} else if (justify == emitter_justify_left) { } else if (justify == emitter_justify_left) {
written = malloc_snprintf(out_fmt, out_size, written = malloc_snprintf(
"%%-%d%s", width, fmt_specifier); out_fmt, out_size, "%%-%d%s", width, fmt_specifier);
} else { } else {
written = malloc_snprintf(out_fmt, out_size, written = malloc_snprintf(
"%%%d%s", width, fmt_specifier); out_fmt, out_size, "%%%d%s", width, fmt_specifier);
} }
/* Only happens in case of bad format string, which *we* choose. */ /* Only happens in case of bad format string, which *we* choose. */
assert(written < out_size); assert(written < out_size);
@@ -124,8 +124,8 @@ emitter_emit_str(emitter_t *emitter, emitter_justify_t justify, int width,
#define BUF_SIZE 256 #define BUF_SIZE 256
char buf[BUF_SIZE]; char buf[BUF_SIZE];
size_t str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", str); size_t str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", str);
emitter_printf(emitter, emitter_printf(
emitter_gen_fmt(fmt, fmt_size, "%s", justify, width), buf); emitter, emitter_gen_fmt(fmt, fmt_size, "%s", justify, width), buf);
if (str_written < BUF_SIZE) { if (str_written < BUF_SIZE) {
return; return;
} }
@@ -213,7 +213,6 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width,
#undef FMT_SIZE #undef FMT_SIZE
} }
/* Internal functions. In json mode, tracks nesting state. */ /* Internal functions. In json mode, tracks nesting state. */
static inline void static inline void
emitter_nest_inc(emitter_t *emitter) { emitter_nest_inc(emitter_t *emitter) {
@@ -291,12 +290,12 @@ emitter_json_key(emitter_t *emitter, const char *json_key) {
} }
static inline void static inline void
emitter_json_value(emitter_t *emitter, emitter_type_t value_type, emitter_json_value(
const void *value) { emitter_t *emitter, emitter_type_t value_type, const void *value) {
if (emitter_outputs_json(emitter)) { if (emitter_outputs_json(emitter)) {
emitter_json_key_prefix(emitter); emitter_json_key_prefix(emitter);
emitter_print_value(emitter, emitter_justify_none, -1, emitter_print_value(
value_type, value); emitter, emitter_justify_none, -1, value_type, value);
emitter->item_at_depth = true; emitter->item_at_depth = true;
} }
} }
@@ -367,7 +366,6 @@ emitter_json_object_end(emitter_t *emitter) {
} }
} }
/******************************************************************************/ /******************************************************************************/
/* Table public API. */ /* Table public API. */
@@ -389,14 +387,13 @@ emitter_table_dict_end(emitter_t *emitter) {
static inline void static inline void
emitter_table_kv_note(emitter_t *emitter, const char *table_key, emitter_table_kv_note(emitter_t *emitter, const char *table_key,
emitter_type_t value_type, const void *value, emitter_type_t value_type, const void *value, const char *table_note_key,
const char *table_note_key, emitter_type_t table_note_value_type, emitter_type_t table_note_value_type, const void *table_note_value) {
const void *table_note_value) {
if (emitter->output == emitter_output_table) { if (emitter->output == emitter_output_table) {
emitter_indent(emitter); emitter_indent(emitter);
emitter_printf(emitter, "%s: ", table_key); emitter_printf(emitter, "%s: ", table_key);
emitter_print_value(emitter, emitter_justify_none, -1, emitter_print_value(
value_type, value); emitter, emitter_justify_none, -1, value_type, value);
if (table_note_key != NULL) { if (table_note_key != NULL) {
emitter_printf(emitter, " (%s: ", table_note_key); emitter_printf(emitter, " (%s: ", table_note_key);
emitter_print_value(emitter, emitter_justify_none, -1, emitter_print_value(emitter, emitter_justify_none, -1,
@@ -415,7 +412,6 @@ emitter_table_kv(emitter_t *emitter, const char *table_key,
emitter_type_bool, NULL); emitter_type_bool, NULL);
} }
/* Write to the emitter the given string, but only in table mode. */ /* Write to the emitter the given string, but only in table mode. */
JEMALLOC_FORMAT_PRINTF(2, 3) JEMALLOC_FORMAT_PRINTF(2, 3)
static inline void static inline void
@@ -423,7 +419,8 @@ emitter_table_printf(emitter_t *emitter, const char *format, ...) {
if (emitter->output == emitter_output_table) { if (emitter->output == emitter_output_table) {
va_list ap; va_list ap;
va_start(ap, format); va_start(ap, format);
malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); malloc_vcprintf(
emitter->write_cb, emitter->cbopaque, format, ap);
va_end(ap); va_end(ap);
} }
} }
@@ -434,7 +431,7 @@ emitter_table_row(emitter_t *emitter, emitter_row_t *row) {
return; return;
} }
emitter_col_t *col; emitter_col_t *col;
ql_foreach(col, &row->cols, link) { ql_foreach (col, &row->cols, link) {
emitter_print_value(emitter, col->justify, col->width, emitter_print_value(emitter, col->justify, col->width,
col->type, (const void *)&col->bool_val); col->type, (const void *)&col->bool_val);
} }
@@ -452,7 +449,6 @@ emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
ql_tail_insert(&row->cols, col, link); ql_tail_insert(&row->cols, col, link);
} }
/******************************************************************************/ /******************************************************************************/
/* /*
* Generalized public API. Emits using either JSON or table, according to * Generalized public API. Emits using either JSON or table, according to
@@ -464,9 +460,8 @@ emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
*/ */
static inline void static inline void
emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key,
emitter_type_t value_type, const void *value, emitter_type_t value_type, const void *value, const char *table_note_key,
const char *table_note_key, emitter_type_t table_note_value_type, emitter_type_t table_note_value_type, const void *table_note_value) {
const void *table_note_value) {
if (emitter_outputs_json(emitter)) { if (emitter_outputs_json(emitter)) {
emitter_json_key(emitter, json_key); emitter_json_key(emitter, json_key);
emitter_json_value(emitter, value_type, value); emitter_json_value(emitter, value_type, value);
@@ -485,8 +480,8 @@ emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key,
} }
static inline void static inline void
emitter_dict_begin(emitter_t *emitter, const char *json_key, emitter_dict_begin(
const char *table_header) { emitter_t *emitter, const char *json_key, const char *table_header) {
if (emitter_outputs_json(emitter)) { if (emitter_outputs_json(emitter)) {
emitter_json_key(emitter, json_key); emitter_json_key(emitter, json_key);
emitter_json_object_begin(emitter); emitter_json_object_begin(emitter);
@@ -526,8 +521,9 @@ emitter_end(emitter_t *emitter) {
if (emitter_outputs_json(emitter)) { if (emitter_outputs_json(emitter)) {
assert(emitter->nesting_depth == 1); assert(emitter->nesting_depth == 1);
emitter_nest_dec(emitter); emitter_nest_dec(emitter);
emitter_printf(emitter, "%s", emitter->output == emitter_printf(emitter, "%s",
emitter_output_json_compact ? "}" : "\n}\n"); emitter->output == emitter_output_json_compact ? "}"
: "\n}\n");
} }
} }

4
include/jemalloc/internal/exp_grow.h
View File

@@ -27,8 +27,7 @@ exp_grow_size_prepare(exp_grow_t *exp_grow, size_t alloc_size_min,
*r_alloc_size = sz_pind2sz(exp_grow->next + *r_skip); *r_alloc_size = sz_pind2sz(exp_grow->next + *r_skip);
while (*r_alloc_size < alloc_size_min) { while (*r_alloc_size < alloc_size_min) {
(*r_skip)++; (*r_skip)++;
if (exp_grow->next + *r_skip >= if (exp_grow->next + *r_skip >= sz_psz2ind(SC_LARGE_MAXCLASS)) {
sz_psz2ind(SC_LARGE_MAXCLASS)) {
/* Outside legal range. */ /* Outside legal range. */
return true; return true;
} }
@@ -44,7 +43,6 @@ exp_grow_size_commit(exp_grow_t *exp_grow, pszind_t skip) {
} else { } else {
exp_grow->next = exp_grow->limit; exp_grow->next = exp_grow->limit;
} }
} }
void exp_grow_init(exp_grow_t *exp_grow); void exp_grow_init(exp_grow_t *exp_grow);

55
include/jemalloc/internal/extent.h
View File

@@ -26,9 +26,10 @@ extern size_t opt_process_madvise_max_batch;
#ifdef JEMALLOC_HAVE_PROCESS_MADVISE #ifdef JEMALLOC_HAVE_PROCESS_MADVISE
/* The iovec is on stack. Limit the max batch to avoid stack overflow. */ /* The iovec is on stack. Limit the max batch to avoid stack overflow. */
#define PROCESS_MADVISE_MAX_BATCH_LIMIT (VARIABLE_ARRAY_SIZE_MAX / sizeof(struct iovec)) # define PROCESS_MADVISE_MAX_BATCH_LIMIT \
(VARIABLE_ARRAY_SIZE_MAX / sizeof(struct iovec))
#else #else
#define PROCESS_MADVISE_MAX_BATCH_LIMIT 0 # define PROCESS_MADVISE_MAX_BATCH_LIMIT 0
#endif #endif
edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
@@ -37,44 +38,43 @@ edata_t *ecache_alloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *ecache_alloc_grow(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment, ecache_t *ecache, edata_t *expand_edata, size_t size, size_t alignment,
bool zero, bool guarded); bool zero, bool guarded);
void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
ecache_t *ecache, edata_t *edata); edata_t *edata);
edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, size_t npages_min); ecache_t *ecache, size_t npages_min);
void extent_gdump_add(tsdn_t *tsdn, const edata_t *edata); void extent_gdump_add(tsdn_t *tsdn, const edata_t *edata);
void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache, void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
edata_t *edata); edata_t *edata);
void extent_dalloc_gap(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void extent_dalloc_gap(
edata_t *edata); tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
edata_t *extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
void *new_addr, size_t size, size_t alignment, bool zero, bool *commit, void *new_addr, size_t size, size_t alignment, bool zero, bool *commit,
bool growing_retained); bool growing_retained);
void extent_dalloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void extent_dalloc_wrapper(
edata_t *edata); tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
void extent_dalloc_wrapper_purged(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void extent_dalloc_wrapper_purged(
edata_t *edata); tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
void extent_destroy_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void extent_destroy_wrapper(
edata_t *edata); tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata);
bool extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, bool extent_commit_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
size_t offset, size_t length); size_t offset, size_t length);
bool extent_purge_lazy_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, bool extent_purge_lazy_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
size_t offset, size_t length); size_t offset, size_t length);
bool extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, bool extent_purge_forced_wrapper(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
size_t offset, size_t length); size_t offset, size_t length);
edata_t *extent_split_wrapper(tsdn_t *tsdn, pac_t *pac, edata_t *extent_split_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ehooks_t *ehooks, edata_t *edata, size_t size_a, size_t size_b, edata_t *edata, size_t size_a, size_t size_b, bool holding_core_locks);
bool holding_core_locks); bool extent_merge_wrapper(
bool extent_merge_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a, edata_t *b);
edata_t *a, edata_t *b);
bool extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, bool extent_commit_zero(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
bool commit, bool zero, bool growing_retained); bool commit, bool zero, bool growing_retained);
size_t extent_sn_next(pac_t *pac); size_t extent_sn_next(pac_t *pac);
bool extent_boot(void); bool extent_boot(void);
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
extent_neighbor_head_state_mergeable(bool edata_is_head, extent_neighbor_head_state_mergeable(
bool neighbor_is_head, bool forward) { bool edata_is_head, bool neighbor_is_head, bool forward) {
/* /*
* Head states checking: disallow merging if the higher addr extent is a * Head states checking: disallow merging if the higher addr extent is a
* head extent. This helps preserve first-fit, and more importantly * head extent. This helps preserve first-fit, and more importantly
@@ -102,8 +102,8 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
} }
/* It's not safe to access *neighbor yet; must verify states first. */ /* It's not safe to access *neighbor yet; must verify states first. */
bool neighbor_is_head = contents.metadata.is_head; bool neighbor_is_head = contents.metadata.is_head;
if (!extent_neighbor_head_state_mergeable(edata_is_head_get(edata), if (!extent_neighbor_head_state_mergeable(
neighbor_is_head, forward)) { edata_is_head_get(edata), neighbor_is_head, forward)) {
return false; return false;
} }
extent_state_t neighbor_state = contents.metadata.state; extent_state_t neighbor_state = contents.metadata.state;
@@ -112,8 +112,9 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
return false; return false;
} }
/* From this point, it's safe to access *neighbor. */ /* From this point, it's safe to access *neighbor. */
if (!expanding && (edata_committed_get(edata) != if (!expanding
edata_committed_get(neighbor))) { && (edata_committed_get(edata)
!= edata_committed_get(neighbor))) {
/* /*
* Some platforms (e.g. Windows) require an explicit * Some platforms (e.g. Windows) require an explicit
* commit step (and writing to uncommitted memory is not * commit step (and writing to uncommitted memory is not
@@ -133,11 +134,11 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
return false; return false;
} }
if (opt_retain) { if (opt_retain) {
assert(edata_arena_ind_get(edata) == assert(edata_arena_ind_get(edata)
edata_arena_ind_get(neighbor)); == edata_arena_ind_get(neighbor));
} else { } else {
if (edata_arena_ind_get(edata) != if (edata_arena_ind_get(edata)
edata_arena_ind_get(neighbor)) { != edata_arena_ind_get(neighbor)) {
return false; return false;
} }
} }

4
include/jemalloc/internal/extent_mmap.h
View File

@@ -5,8 +5,8 @@
extern bool opt_retain; extern bool opt_retain;
void *extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, void *extent_alloc_mmap(
bool *zero, bool *commit); void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit);
bool extent_dalloc_mmap(void *addr, size_t size); bool extent_dalloc_mmap(void *addr, size_t size);
#endif /* JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H */ #endif /* JEMALLOC_INTERNAL_EXTENT_MMAP_EXTERNS_H */

25
include/jemalloc/internal/fb.h
View File

@@ -15,8 +15,8 @@
typedef unsigned long fb_group_t; typedef unsigned long fb_group_t;
#define FB_GROUP_BITS (ZU(1) << (LG_SIZEOF_LONG + 3)) #define FB_GROUP_BITS (ZU(1) << (LG_SIZEOF_LONG + 3))
#define FB_NGROUPS(nbits) ((nbits) / FB_GROUP_BITS \ #define FB_NGROUPS(nbits) \
+ ((nbits) % FB_GROUP_BITS == 0 ? 0 : 1)) ((nbits) / FB_GROUP_BITS + ((nbits) % FB_GROUP_BITS == 0 ? 0 : 1))
static inline void static inline void
fb_init(fb_group_t *fb, size_t nbits) { fb_init(fb_group_t *fb, size_t nbits) {
@@ -75,7 +75,6 @@ fb_unset(fb_group_t *fb, size_t nbits, size_t bit) {
fb[group_ind] &= ~((fb_group_t)1 << bit_ind); fb[group_ind] &= ~((fb_group_t)1 << bit_ind);
} }
/* /*
* Some implementation details. This visitation function lets us apply a group * Some implementation details. This visitation function lets us apply a group
* visitor to each group in the bitmap (potentially modifying it). The mask * visitor to each group in the bitmap (potentially modifying it). The mask
@@ -94,7 +93,8 @@ fb_visit_impl(fb_group_t *fb, size_t nbits, fb_group_visitor_t visit, void *ctx,
* to from bit 0. * to from bit 0.
*/ */
size_t first_group_cnt = (start_bit_ind + cnt > FB_GROUP_BITS size_t first_group_cnt = (start_bit_ind + cnt > FB_GROUP_BITS
? FB_GROUP_BITS - start_bit_ind : cnt); ? FB_GROUP_BITS - start_bit_ind
: cnt);
/* /*
* We can basically split affected words into: * We can basically split affected words into:
* - The first group, where we touch only the high bits * - The first group, where we touch only the high bits
@@ -104,8 +104,8 @@ fb_visit_impl(fb_group_t *fb, size_t nbits, fb_group_visitor_t visit, void *ctx,
* this can lead to bad codegen for those middle words. * this can lead to bad codegen for those middle words.
*/ */
/* First group */ /* First group */
fb_group_t mask = ((~(fb_group_t)0) fb_group_t mask =
>> (FB_GROUP_BITS - first_group_cnt)) ((~(fb_group_t)0) >> (FB_GROUP_BITS - first_group_cnt))
<< start_bit_ind; << start_bit_ind;
visit(ctx, &fb[group_ind], mask); visit(ctx, &fb[group_ind], mask);
@@ -176,8 +176,8 @@ fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
* Returns the number of bits in the bitmap if no such bit exists. * Returns the number of bits in the bitmap if no such bit exists.
*/ */
JEMALLOC_ALWAYS_INLINE ssize_t JEMALLOC_ALWAYS_INLINE ssize_t
fb_find_impl(fb_group_t *fb, size_t nbits, size_t start, bool val, fb_find_impl(
bool forward) { fb_group_t *fb, size_t nbits, size_t start, bool val, bool forward) {
assert(start < nbits); assert(start < nbits);
size_t ngroups = FB_NGROUPS(nbits); size_t ngroups = FB_NGROUPS(nbits);
ssize_t group_ind = start / FB_GROUP_BITS; ssize_t group_ind = start / FB_GROUP_BITS;
@@ -265,8 +265,8 @@ fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
return false; return false;
} }
/* Half open range; the set bits are [begin, end). */ /* Half open range; the set bits are [begin, end). */
ssize_t next_range_end = fb_find_impl(fb, nbits, next_range_begin, !val, ssize_t next_range_end = fb_find_impl(
forward); fb, nbits, next_range_begin, !val, forward);
if (forward) { if (forward) {
*r_begin = next_range_begin; *r_begin = next_range_begin;
*r_len = next_range_end - next_range_begin; *r_len = next_range_end - next_range_begin;
@@ -324,8 +324,9 @@ fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
size_t begin = 0; size_t begin = 0;
size_t longest_len = 0; size_t longest_len = 0;
size_t len = 0; size_t len = 0;
while (begin < nbits && fb_iter_range_impl(fb, nbits, begin, &begin, while (begin < nbits
&len, val, /* forward */ true)) { && fb_iter_range_impl(
fb, nbits, begin, &begin, &len, val, /* forward */ true)) {
if (len > longest_len) { if (len > longest_len) {
longest_len = len; longest_len = len;
} }

292
include/jemalloc/internal/hash.h
View File

@@ -25,7 +25,7 @@ hash_rotl_64(uint64_t x, int8_t r) {
static inline uint32_t static inline uint32_t
hash_get_block_32(const uint32_t *p, int i) { hash_get_block_32(const uint32_t *p, int i) {
/* Handle unaligned read. */ /* Handle unaligned read. */
if (unlikely((uintptr_t)p & (sizeof(uint32_t)-1)) != 0) { if (unlikely((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0) {
uint32_t ret; uint32_t ret;
memcpy(&ret, (uint8_t *)(p + i), sizeof(uint32_t)); memcpy(&ret, (uint8_t *)(p + i), sizeof(uint32_t));
@@ -38,7 +38,7 @@ hash_get_block_32(const uint32_t *p, int i) {
static inline uint64_t static inline uint64_t
hash_get_block_64(const uint64_t *p, int i) { hash_get_block_64(const uint64_t *p, int i) {
/* Handle unaligned read. */ /* Handle unaligned read. */
if (unlikely((uintptr_t)p & (sizeof(uint64_t)-1)) != 0) { if (unlikely((uintptr_t)p & (sizeof(uint64_t) - 1)) != 0) {
uint64_t ret; uint64_t ret;
memcpy(&ret, (uint8_t *)(p + i), sizeof(uint64_t)); memcpy(&ret, (uint8_t *)(p + i), sizeof(uint64_t));
@@ -72,7 +72,7 @@ hash_fmix_64(uint64_t k) {
static inline uint32_t static inline uint32_t
hash_x86_32(const void *key, int len, uint32_t seed) { hash_x86_32(const void *key, int len, uint32_t seed) {
const uint8_t *data = (const uint8_t *) key; const uint8_t *data = (const uint8_t *)key;
const int nblocks = len / 4; const int nblocks = len / 4;
uint32_t h1 = seed; uint32_t h1 = seed;
@@ -82,7 +82,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
/* body */ /* body */
{ {
const uint32_t *blocks = (const uint32_t *) (data + nblocks*4); const uint32_t *blocks = (const uint32_t *)(data + nblocks * 4);
int i; int i;
for (i = -nblocks; i; i++) { for (i = -nblocks; i; i++) {
@@ -94,21 +94,29 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
h1 ^= k1; h1 ^= k1;
h1 = hash_rotl_32(h1, 13); h1 = hash_rotl_32(h1, 13);
h1 = h1*5 + 0xe6546b64; h1 = h1 * 5 + 0xe6546b64;
} }
} }
/* tail */ /* tail */
{ {
const uint8_t *tail = (const uint8_t *) (data + nblocks*4); const uint8_t *tail = (const uint8_t *)(data + nblocks * 4);
uint32_t k1 = 0; uint32_t k1 = 0;
switch (len & 3) { switch (len & 3) {
case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH; case 3:
case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH; k1 ^= tail[2] << 16;
case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); JEMALLOC_FALLTHROUGH;
k1 *= c2; h1 ^= k1; case 2:
k1 ^= tail[1] << 8;
JEMALLOC_FALLTHROUGH;
case 1:
k1 ^= tail[0];
k1 *= c1;
k1 = hash_rotl_32(k1, 15);
k1 *= c2;
h1 ^= k1;
} }
} }
@@ -121,9 +129,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
} }
static inline void static inline void
hash_x86_128(const void *key, const int len, uint32_t seed, hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) {
uint64_t r_out[2]) { const uint8_t *data = (const uint8_t *)key;
const uint8_t * data = (const uint8_t *) key;
const int nblocks = len / 16; const int nblocks = len / 16;
uint32_t h1 = seed; uint32_t h1 = seed;
@@ -138,94 +145,160 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
/* body */ /* body */
{ {
const uint32_t *blocks = (const uint32_t *) (data + nblocks*16); const uint32_t *blocks = (const uint32_t *)(data
+ nblocks * 16);
int i; int i;
for (i = -nblocks; i; i++) { for (i = -nblocks; i; i++) {
uint32_t k1 = hash_get_block_32(blocks, i*4 + 0); uint32_t k1 = hash_get_block_32(blocks, i * 4 + 0);
uint32_t k2 = hash_get_block_32(blocks, i*4 + 1); uint32_t k2 = hash_get_block_32(blocks, i * 4 + 1);
uint32_t k3 = hash_get_block_32(blocks, i*4 + 2); uint32_t k3 = hash_get_block_32(blocks, i * 4 + 2);
uint32_t k4 = hash_get_block_32(blocks, i*4 + 3); uint32_t k4 = hash_get_block_32(blocks, i * 4 + 3);
k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; k1 *= c1;
k1 = hash_rotl_32(k1, 15);
k1 *= c2;
h1 ^= k1;
h1 = hash_rotl_32(h1, 19); h1 += h2; h1 = hash_rotl_32(h1, 19);
h1 = h1*5 + 0x561ccd1b; h1 += h2;
h1 = h1 * 5 + 0x561ccd1b;
k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; k2 *= c2;
k2 = hash_rotl_32(k2, 16);
k2 *= c3;
h2 ^= k2;
h2 = hash_rotl_32(h2, 17); h2 += h3; h2 = hash_rotl_32(h2, 17);
h2 = h2*5 + 0x0bcaa747; h2 += h3;
h2 = h2 * 5 + 0x0bcaa747;
k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; k3 *= c3;
k3 = hash_rotl_32(k3, 17);
k3 *= c4;
h3 ^= k3;
h3 = hash_rotl_32(h3, 15); h3 += h4; h3 = hash_rotl_32(h3, 15);
h3 = h3*5 + 0x96cd1c35; h3 += h4;
h3 = h3 * 5 + 0x96cd1c35;
k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; k4 *= c4;
k4 = hash_rotl_32(k4, 18);
k4 *= c1;
h4 ^= k4;
h4 = hash_rotl_32(h4, 13); h4 += h1; h4 = hash_rotl_32(h4, 13);
h4 = h4*5 + 0x32ac3b17; h4 += h1;
h4 = h4 * 5 + 0x32ac3b17;
} }
} }
/* tail */ /* tail */
{ {
const uint8_t *tail = (const uint8_t *) (data + nblocks*16); const uint8_t *tail = (const uint8_t *)(data + nblocks * 16);
uint32_t k1 = 0; uint32_t k1 = 0;
uint32_t k2 = 0; uint32_t k2 = 0;
uint32_t k3 = 0; uint32_t k3 = 0;
uint32_t k4 = 0; uint32_t k4 = 0;
switch (len & 15) { switch (len & 15) {
case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH; case 15:
case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH; k4 ^= tail[14] << 16;
case 13: k4 ^= tail[12] << 0;
k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case 12: k3 ^= (uint32_t) tail[11] << 24; JEMALLOC_FALLTHROUGH; case 14:
case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH; k4 ^= tail[13] << 8;
case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH;
case 9: k3 ^= tail[ 8] << 0;
k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case 8: k2 ^= (uint32_t) tail[ 7] << 24; JEMALLOC_FALLTHROUGH; case 13:
case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH; k4 ^= tail[12] << 0;
case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH; k4 *= c4;
case 5: k2 ^= tail[ 4] << 0; k4 = hash_rotl_32(k4, 18);
k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; k4 *= c1;
h4 ^= k4;
JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case 4: k1 ^= (uint32_t) tail[ 3] << 24; JEMALLOC_FALLTHROUGH; case 12:
case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH; k3 ^= (uint32_t)tail[11] << 24;
case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case 1: k1 ^= tail[ 0] << 0; case 11:
k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; k3 ^= tail[10] << 16;
JEMALLOC_FALLTHROUGH;
case 10:
k3 ^= tail[9] << 8;
JEMALLOC_FALLTHROUGH;
case 9:
k3 ^= tail[8] << 0;
k3 *= c3;
k3 = hash_rotl_32(k3, 17);
k3 *= c4;
h3 ^= k3;
JEMALLOC_FALLTHROUGH;
case 8:
k2 ^= (uint32_t)tail[7] << 24;
JEMALLOC_FALLTHROUGH;
case 7:
k2 ^= tail[6] << 16;
JEMALLOC_FALLTHROUGH;
case 6:
k2 ^= tail[5] << 8;
JEMALLOC_FALLTHROUGH;
case 5:
k2 ^= tail[4] << 0;
k2 *= c2;
k2 = hash_rotl_32(k2, 16);
k2 *= c3;
h2 ^= k2;
JEMALLOC_FALLTHROUGH;
case 4:
k1 ^= (uint32_t)tail[3] << 24;
JEMALLOC_FALLTHROUGH;
case 3:
k1 ^= tail[2] << 16;
JEMALLOC_FALLTHROUGH;
case 2:
k1 ^= tail[1] << 8;
JEMALLOC_FALLTHROUGH;
case 1:
k1 ^= tail[0] << 0;
k1 *= c1;
k1 = hash_rotl_32(k1, 15);
k1 *= c2;
h1 ^= k1;
break; break;
} }
} }
/* finalization */ /* finalization */
h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; h1 ^= len;
h2 ^= len;
h3 ^= len;
h4 ^= len;
h1 += h2; h1 += h3; h1 += h4; h1 += h2;
h2 += h1; h3 += h1; h4 += h1; h1 += h3;
h1 += h4;
h2 += h1;
h3 += h1;
h4 += h1;
h1 = hash_fmix_32(h1); h1 = hash_fmix_32(h1);
h2 = hash_fmix_32(h2); h2 = hash_fmix_32(h2);
h3 = hash_fmix_32(h3); h3 = hash_fmix_32(h3);
h4 = hash_fmix_32(h4); h4 = hash_fmix_32(h4);
h1 += h2; h1 += h3; h1 += h4; h1 += h2;
h2 += h1; h3 += h1; h4 += h1; h1 += h3;
h1 += h4;
h2 += h1;
h3 += h1;
h4 += h1;
r_out[0] = (((uint64_t) h2) << 32) | h1; r_out[0] = (((uint64_t)h2) << 32) | h1;
r_out[1] = (((uint64_t) h4) << 32) | h3; r_out[1] = (((uint64_t)h4) << 32) | h3;
} }
static inline void static inline void
hash_x64_128(const void *key, const int len, const uint32_t seed, hash_x64_128(
uint64_t r_out[2]) { const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) {
const uint8_t *data = (const uint8_t *) key; const uint8_t *data = (const uint8_t *)key;
const int nblocks = len / 16; const int nblocks = len / 16;
uint64_t h1 = seed; uint64_t h1 = seed;
@@ -236,56 +309,99 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
/* body */ /* body */
{ {
const uint64_t *blocks = (const uint64_t *) (data); const uint64_t *blocks = (const uint64_t *)(data);
int i; int i;
for (i = 0; i < nblocks; i++) { for (i = 0; i < nblocks; i++) {
uint64_t k1 = hash_get_block_64(blocks, i*2 + 0); uint64_t k1 = hash_get_block_64(blocks, i * 2 + 0);
uint64_t k2 = hash_get_block_64(blocks, i*2 + 1); uint64_t k2 = hash_get_block_64(blocks, i * 2 + 1);
k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; k1 *= c1;
k1 = hash_rotl_64(k1, 31);
k1 *= c2;
h1 ^= k1;
h1 = hash_rotl_64(h1, 27); h1 += h2; h1 = hash_rotl_64(h1, 27);
h1 = h1*5 + 0x52dce729; h1 += h2;
h1 = h1 * 5 + 0x52dce729;
k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; k2 *= c2;
k2 = hash_rotl_64(k2, 33);
k2 *= c1;
h2 ^= k2;
h2 = hash_rotl_64(h2, 31); h2 += h1; h2 = hash_rotl_64(h2, 31);
h2 = h2*5 + 0x38495ab5; h2 += h1;
h2 = h2 * 5 + 0x38495ab5;
} }
} }
/* tail */ /* tail */
{ {
const uint8_t *tail = (const uint8_t*)(data + nblocks*16); const uint8_t *tail = (const uint8_t *)(data + nblocks * 16);
uint64_t k1 = 0; uint64_t k1 = 0;
uint64_t k2 = 0; uint64_t k2 = 0;
switch (len & 15) { switch (len & 15) {
case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH; case 15:
case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH; k2 ^= ((uint64_t)(tail[14])) << 48;
case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH;
case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH;
case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH;
case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH;
case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0;
k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH; case 14:
case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH; k2 ^= ((uint64_t)(tail[13])) << 40;
case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH; case 13:
case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH; k2 ^= ((uint64_t)(tail[12])) << 32;
case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH; JEMALLOC_FALLTHROUGH;
case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH; case 12:
case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; k2 ^= ((uint64_t)(tail[11])) << 24;
k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; JEMALLOC_FALLTHROUGH;
case 11:
k2 ^= ((uint64_t)(tail[10])) << 16;
JEMALLOC_FALLTHROUGH;
case 10:
k2 ^= ((uint64_t)(tail[9])) << 8;
JEMALLOC_FALLTHROUGH;
case 9:
k2 ^= ((uint64_t)(tail[8])) << 0;
k2 *= c2;
k2 = hash_rotl_64(k2, 33);
k2 *= c1;
h2 ^= k2;
JEMALLOC_FALLTHROUGH;
case 8:
k1 ^= ((uint64_t)(tail[7])) << 56;
JEMALLOC_FALLTHROUGH;
case 7:
k1 ^= ((uint64_t)(tail[6])) << 48;
JEMALLOC_FALLTHROUGH;
case 6:
k1 ^= ((uint64_t)(tail[5])) << 40;
JEMALLOC_FALLTHROUGH;
case 5:
k1 ^= ((uint64_t)(tail[4])) << 32;
JEMALLOC_FALLTHROUGH;
case 4:
k1 ^= ((uint64_t)(tail[3])) << 24;
JEMALLOC_FALLTHROUGH;
case 3:
k1 ^= ((uint64_t)(tail[2])) << 16;
JEMALLOC_FALLTHROUGH;
case 2:
k1 ^= ((uint64_t)(tail[1])) << 8;
JEMALLOC_FALLTHROUGH;
case 1:
k1 ^= ((uint64_t)(tail[0])) << 0;
k1 *= c1;
k1 = hash_rotl_64(k1, 31);
k1 *= c2;
h1 ^= k1;
break; break;
} }
} }
/* finalization */ /* finalization */
h1 ^= len; h2 ^= len; h1 ^= len;
h2 ^= len;
h1 += h2; h1 += h2;
h2 += h1; h2 += h1;

16
include/jemalloc/internal/hook.h
View File

@@ -83,7 +83,6 @@ enum hook_dalloc_e {
}; };
typedef enum hook_dalloc_e hook_dalloc_t; typedef enum hook_dalloc_e hook_dalloc_t;
enum hook_expand_e { enum hook_expand_e {
hook_expand_realloc, hook_expand_realloc,
hook_expand_rallocx, hook_expand_rallocx,
@@ -91,16 +90,15 @@ enum hook_expand_e {
}; };
typedef enum hook_expand_e hook_expand_t; typedef enum hook_expand_e hook_expand_t;
typedef void (*hook_alloc)( typedef void (*hook_alloc)(void *extra, hook_alloc_t type, void *result,
void *extra, hook_alloc_t type, void *result, uintptr_t result_raw, uintptr_t result_raw, uintptr_t args_raw[3]);
uintptr_t args_raw[3]);
typedef void (*hook_dalloc)( typedef void (*hook_dalloc)(
void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]); void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
typedef void (*hook_expand)( typedef void (*hook_expand)(void *extra, hook_expand_t type, void *address,
void *extra, hook_expand_t type, void *address, size_t old_usize, size_t old_usize, size_t new_usize, uintptr_t result_raw,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); uintptr_t args_raw[4]);
typedef struct hooks_s hooks_t; typedef struct hooks_s hooks_t;
struct hooks_s { struct hooks_s {
@@ -156,8 +154,8 @@ void hook_remove(tsdn_t *tsdn, void *opaque);
void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
uintptr_t args_raw[3]); uintptr_t args_raw[3]);
void hook_invoke_dalloc(hook_dalloc_t type, void *address, void hook_invoke_dalloc(
uintptr_t args_raw[3]); hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);

11
include/jemalloc/internal/hpa.h
View File

@@ -156,14 +156,15 @@ bool hpa_hugepage_size_exceeds_limit(void);
* just that it can function properly given the system it's running on. * just that it can function properly given the system it's running on.
*/ */
bool hpa_supported(void); bool hpa_supported(void);
bool hpa_central_init(hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks); bool hpa_central_init(
hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
base_t *base, edata_cache_t *edata_cache, unsigned ind, base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts); const hpa_shard_opts_t *opts);
void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src); void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard, void hpa_shard_stats_merge(
hpa_shard_stats_t *dst); tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst);
/* /*
* Notify the shard that we won't use it for allocations much longer. Due to * Notify the shard that we won't use it for allocations much longer. Due to
@@ -173,8 +174,8 @@ void hpa_shard_stats_merge(tsdn_t *tsdn, hpa_shard_t *shard,
void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_set_deferral_allowed(tsdn_t *tsdn, hpa_shard_t *shard, void hpa_shard_set_deferral_allowed(
bool deferral_allowed); tsdn_t *tsdn, hpa_shard_t *shard, bool deferral_allowed);
void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
/* /*

2
include/jemalloc/internal/hpa_hooks.h
View File

@@ -13,7 +13,7 @@ struct hpa_hooks_s {
void (*dehugify)(void *ptr, size_t size); void (*dehugify)(void *ptr, size_t size);
void (*curtime)(nstime_t *r_time, bool first_reading); void (*curtime)(nstime_t *r_time, bool first_reading);
uint64_t (*ms_since)(nstime_t *r_time); uint64_t (*ms_since)(nstime_t *r_time);
bool (*vectorized_purge)(void* vec, size_t vlen, size_t nbytes); bool (*vectorized_purge)(void *vec, size_t vlen, size_t nbytes);
}; };
extern const hpa_hooks_t hpa_hooks_default; extern const hpa_hooks_t hpa_hooks_default;

3
include/jemalloc/internal/hpa_utils.h
View File

@@ -25,7 +25,8 @@ hpa_try_vectorized_purge(
* failed, we could avoid that penalty. * failed, we could avoid that penalty.
*/ */
for (size_t i = 0; i < vlen; ++i) { for (size_t i = 0; i < vlen; ++i) {
shard->central->hooks.purge(vec[i].iov_base, vec[i].iov_len); shard->central->hooks.purge(
vec[i].iov_base, vec[i].iov_len);
} }
} }
} }

7
include/jemalloc/internal/hpdata.h
View File

@@ -250,7 +250,6 @@ hpdata_changing_state_get(const hpdata_t *hpdata) {
return hpdata->h_mid_purge || hpdata->h_mid_hugify; return hpdata->h_mid_purge || hpdata->h_mid_hugify;
} }
static inline bool static inline bool
hpdata_updating_get(const hpdata_t *hpdata) { hpdata_updating_get(const hpdata_t *hpdata) {
return hpdata->h_updating; return hpdata->h_updating;
@@ -317,7 +316,7 @@ hpdata_assert_empty(hpdata_t *hpdata) {
*/ */
static inline bool static inline bool
hpdata_consistent(hpdata_t *hpdata) { hpdata_consistent(hpdata_t *hpdata) {
if(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES) if (fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES)
!= hpdata_longest_free_range_get(hpdata)) { != hpdata_longest_free_range_get(hpdata)) {
return false; return false;
} }
@@ -398,8 +397,8 @@ struct hpdata_purge_state_s {
* Returns the number of dirty pages that will be purged and sets nranges * Returns the number of dirty pages that will be purged and sets nranges
* to number of ranges with dirty pages that will be purged. * to number of ranges with dirty pages that will be purged.
*/ */
size_t hpdata_purge_begin(hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t hpdata_purge_begin(
size_t *nranges); hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t *nranges);
/* /*
* If there are more extents to purge, sets *r_purge_addr and *r_purge_size to * If there are more extents to purge, sets *r_purge_addr and *r_purge_size to

8
include/jemalloc/internal/inspect.h
View File

@@ -34,10 +34,10 @@ struct inspect_extent_util_stats_verbose_s {
size_t bin_nregs; size_t bin_nregs;
}; };
void inspect_extent_util_stats_get(tsdn_t *tsdn, const void *ptr, void inspect_extent_util_stats_get(
size_t *nfree, size_t *nregs, size_t *size); tsdn_t *tsdn, const void *ptr, size_t *nfree, size_t *nregs, size_t *size);
void inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr, void inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
size_t *nfree, size_t *nregs, size_t *size, size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree,
size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr); size_t *bin_nregs, void **slabcur_addr);
#endif /* JEMALLOC_INTERNAL_INSPECT_H */ #endif /* JEMALLOC_INTERNAL_INSPECT_H */

13
include/jemalloc/internal/jemalloc_internal_decls.h
View File

@@ -25,14 +25,15 @@
# define SYS_write __NR_write # define SYS_write __NR_write
# endif # endif
# if defined(SYS_open) && defined(__aarch64__) # if defined(SYS_open) && defined(__aarch64__)
/* Android headers may define SYS_open to __NR_open even though /* Android headers may define SYS_open to __NR_open even though
* __NR_open may not exist on AArch64 (superseded by __NR_openat). */ * __NR_open may not exist on AArch64 (superseded by __NR_openat). */
# undef SYS_open # undef SYS_open
# endif # endif
# include <sys/uio.h> # include <sys/uio.h>
# endif # endif
# include <pthread.h> # include <pthread.h>
# if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) # if defined(__FreeBSD__) || defined(__DragonFly__) \
|| defined(__OpenBSD__)
# include <pthread_np.h> # include <pthread_np.h>
# include <sched.h> # include <sched.h>
# if defined(__FreeBSD__) # if defined(__FreeBSD__)
@@ -69,7 +70,7 @@
#include <stdint.h> #include <stdint.h>
#include <stddef.h> #include <stddef.h>
#ifndef offsetof #ifndef offsetof
# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) # define offsetof(type, member) ((size_t) & (((type *)NULL)->member))
#endif #endif
#include <string.h> #include <string.h>
#include <strings.h> #include <strings.h>
@@ -84,13 +85,13 @@ typedef intptr_t ssize_t;
# define restrict __restrict # define restrict __restrict
# endif # endif
/* Disable warnings about deprecated system functions. */ /* Disable warnings about deprecated system functions. */
# pragma warning(disable: 4996) # pragma warning(disable : 4996)
#if _MSC_VER < 1800 # if _MSC_VER < 1800
static int static int
isblank(int c) { isblank(int c) {
return (c == '\t' || c == ' '); return (c == '\t' || c == ' ');
} }
#endif # endif
#else #else
# include <unistd.h> # include <unistd.h>
#endif #endif

8
include/jemalloc/internal/jemalloc_internal_inlines_a.h
View File

@@ -20,12 +20,12 @@ malloc_getcpu(void) {
return (malloc_cpuid_t)sched_getcpu(); return (malloc_cpuid_t)sched_getcpu();
#elif defined(JEMALLOC_HAVE_RDTSCP) #elif defined(JEMALLOC_HAVE_RDTSCP)
unsigned int ecx; unsigned int ecx;
asm volatile("rdtscp" : "=c" (ecx) :: "eax", "edx"); asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx");
return (malloc_cpuid_t)(ecx & 0xfff); return (malloc_cpuid_t)(ecx & 0xfff);
#elif defined(__aarch64__) && defined(__APPLE__) #elif defined(__aarch64__) && defined(__APPLE__)
/* Other oses most likely use tpidr_el0 instead */ /* Other oses most likely use tpidr_el0 instead */
uintptr_t c; uintptr_t c;
asm volatile("mrs %x0, tpidrro_el0" : "=r"(c) :: "memory"); asm volatile("mrs %x0, tpidrro_el0" : "=r"(c)::"memory");
return (malloc_cpuid_t)(c & (1 << 3) - 1); return (malloc_cpuid_t)(c & (1 << 3) - 1);
#else #else
not_reached(); not_reached();
@@ -42,8 +42,8 @@ percpu_arena_choose(void) {
assert(cpuid >= 0); assert(cpuid >= 0);
unsigned arena_ind; unsigned arena_ind;
if ((opt_percpu_arena == percpu_arena) || ((unsigned)cpuid < ncpus / if ((opt_percpu_arena == percpu_arena)
2)) { || ((unsigned)cpuid < ncpus / 2)) {
arena_ind = cpuid; arena_ind = cpuid;
} else { } else {
assert(opt_percpu_arena == per_phycpu_arena); assert(opt_percpu_arena == per_phycpu_arena);

21
include/jemalloc/internal/jemalloc_internal_inlines_b.h
View File

@@ -24,13 +24,12 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
if (tcache != NULL) { if (tcache != NULL) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd); tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
assert(tcache_slow->arena != NULL); assert(tcache_slow->arena != NULL);
tcache_arena_reassociate(tsd_tsdn(tsd), tcache_slow, tcache_arena_reassociate(
tcache, newarena); tsd_tsdn(tsd), tcache_slow, tcache, newarena);
} }
} }
} }
/* Choose an arena based on a per-thread value. */ /* Choose an arena based on a per-thread value. */
static inline arena_t * static inline arena_t *
arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
@@ -54,15 +53,15 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
tcache_t *tcache = tsd_tcachep_get(tsd); tcache_t *tcache = tsd_tcachep_get(tsd);
if (tcache_slow->arena != NULL) { if (tcache_slow->arena != NULL) {
/* See comments in tsd_tcache_data_init().*/ /* See comments in tsd_tcache_data_init().*/
assert(tcache_slow->arena == assert(tcache_slow->arena
arena_get(tsd_tsdn(tsd), 0, false)); == arena_get(tsd_tsdn(tsd), 0, false));
if (tcache_slow->arena != ret) { if (tcache_slow->arena != ret) {
tcache_arena_reassociate(tsd_tsdn(tsd), tcache_arena_reassociate(tsd_tsdn(tsd),
tcache_slow, tcache, ret); tcache_slow, tcache, ret);
} }
} else { } else {
tcache_arena_associate(tsd_tsdn(tsd), tcache_arena_associate(
tcache_slow, tcache, ret); tsd_tsdn(tsd), tcache_slow, tcache, ret);
} }
} }
} }
@@ -72,10 +71,10 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
* auto percpu arena range, (i.e. thread is assigned to a manually * auto percpu arena range, (i.e. thread is assigned to a manually
* managed arena), then percpu arena is skipped. * managed arena), then percpu arena is skipped.
*/ */
if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena) && if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
!internal && (arena_ind_get(ret) < && !internal
percpu_arena_ind_limit(opt_percpu_arena)) && (ret->last_thd != && (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
tsd_tsdn(tsd))) { && (ret->last_thd != tsd_tsdn(tsd))) {
unsigned ind = percpu_arena_choose(); unsigned ind = percpu_arena_choose();
if (arena_ind_get(ret) != ind) { if (arena_ind_get(ret) != ind) {
percpu_arena_update(tsd, ind); percpu_arena_update(tsd, ind);

127
include/jemalloc/internal/jemalloc_internal_inlines_c.h
View File

@@ -63,11 +63,12 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
assert(!is_internal || tcache == NULL); assert(!is_internal || tcache == NULL);
assert(!is_internal || arena == NULL || arena_is_auto(arena)); assert(!is_internal || arena == NULL || arena_is_auto(arena));
if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(
WITNESS_RANK_CORE, 0); tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
} }
ret = arena_malloc(tsdn, arena, size, ind, zero, slab, tcache, slow_path); ret = arena_malloc(
tsdn, arena, size, ind, zero, slab, tcache, slow_path);
if (config_stats && is_internal && likely(ret != NULL)) { if (config_stats && is_internal && likely(ret != NULL)) {
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
} }
@@ -78,8 +79,8 @@ JEMALLOC_ALWAYS_INLINE void *
iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
bool is_internal, arena_t *arena, bool slow_path) { bool is_internal, arena_t *arena, bool slow_path) {
bool slab = sz_can_use_slab(size); bool slab = sz_can_use_slab(size);
return iallocztm_explicit_slab(tsdn, size, ind, zero, slab, tcache, return iallocztm_explicit_slab(
is_internal, arena, slow_path); tsdn, size, ind, zero, slab, tcache, is_internal, arena, slow_path);
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
@@ -89,8 +90,8 @@ ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) {
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
bool slab, tcache_t *tcache, bool is_internal, arena_t *arena) { bool zero, bool slab, tcache_t *tcache, bool is_internal, arena_t *arena) {
void *ret; void *ret;
assert(!slab || sz_can_use_slab(usize)); /* slab && large is illegal */ assert(!slab || sz_can_use_slab(usize)); /* slab && large is illegal */
@@ -98,8 +99,8 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero
assert(usize == sz_sa2u(usize, alignment)); assert(usize == sz_sa2u(usize, alignment));
assert(!is_internal || tcache == NULL); assert(!is_internal || tcache == NULL);
assert(!is_internal || arena == NULL || arena_is_auto(arena)); assert(!is_internal || arena == NULL || arena_is_auto(arena));
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(
WITNESS_RANK_CORE, 0); tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache); ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
@@ -123,10 +124,10 @@ ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
ipalloct_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment, ipalloct_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
bool zero, bool slab, tcache_t *tcache, arena_t *arena) { bool slab, tcache_t *tcache, arena_t *arena) {
return ipallocztm_explicit_slab(tsdn, usize, alignment, zero, slab, return ipallocztm_explicit_slab(
tcache, false, arena); tsdn, usize, alignment, zero, slab, tcache, false, arena);
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
@@ -146,13 +147,13 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
assert(ptr != NULL); assert(ptr != NULL);
assert(!is_internal || tcache == NULL); assert(!is_internal || tcache == NULL);
assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr))); assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr)));
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(
WITNESS_RANK_CORE, 0); tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
if (config_stats && is_internal) { if (config_stats && is_internal) {
arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr));
} }
if (!is_internal && !tsdn_null(tsdn) && if (!is_internal && !tsdn_null(tsdn)
tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) { && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
assert(tcache == NULL); assert(tcache == NULL);
} }
arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path); arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
@@ -166,8 +167,8 @@ idalloc(tsd_t *tsd, void *ptr) {
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *alloc_ctx, bool slow_path) { emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(
WITNESS_RANK_CORE, 0); tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path); arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
} }
@@ -175,8 +176,8 @@ JEMALLOC_ALWAYS_INLINE void *
iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena, size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
hook_ralloc_args_t *hook_args) { hook_ralloc_args_t *hook_args) {
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(
WITNESS_RANK_CORE, 0); tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
void *p; void *p;
size_t usize, copysize; size_t usize, copysize;
@@ -184,8 +185,8 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
return NULL; return NULL;
} }
p = ipalloct_explicit_slab(tsdn, usize, alignment, zero, slab, p = ipalloct_explicit_slab(
tcache, arena); tsdn, usize, alignment, zero, slab, tcache, arena);
if (p == NULL) { if (p == NULL) {
return NULL; return NULL;
} }
@@ -195,11 +196,12 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
*/ */
copysize = (size < oldsize) ? size : oldsize; copysize = (size < oldsize) ? size : oldsize;
memcpy(p, ptr, copysize); memcpy(p, ptr, copysize);
hook_invoke_alloc(hook_args->is_realloc hook_invoke_alloc(
? hook_alloc_realloc : hook_alloc_rallocx, p, (uintptr_t)p, hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, p,
hook_args->args); (uintptr_t)p, hook_args->args);
hook_invoke_dalloc(hook_args->is_realloc hook_invoke_dalloc(
? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
ptr, hook_args->args);
isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
return p; return p;
} }
@@ -214,15 +216,14 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena, size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
hook_ralloc_args_t *hook_args) hook_ralloc_args_t *hook_args) {
{
assert(ptr != NULL); assert(ptr != NULL);
assert(size != 0); assert(size != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(
WITNESS_RANK_CORE, 0); tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) if (alignment != 0
!= 0) { && ((uintptr_t)ptr & ((uintptr_t)alignment - 1)) != 0) {
/* /*
* Existing object alignment is inadequate; allocate new space * Existing object alignment is inadequate; allocate new space
* and copy. * and copy.
@@ -238,8 +239,7 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
size_t usize, bool zero, tcache_t *tcache, arena_t *arena, size_t usize, bool zero, tcache_t *tcache, arena_t *arena,
hook_ralloc_args_t *hook_args) hook_ralloc_args_t *hook_args) {
{
bool slab = sz_can_use_slab(usize); bool slab = sz_can_use_slab(usize);
return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero, return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero,
slab, tcache, arena, hook_args); slab, tcache, arena, hook_args);
@@ -257,23 +257,23 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
size_t alignment, bool zero, size_t *newsize) { size_t alignment, bool zero, size_t *newsize) {
assert(ptr != NULL); assert(ptr != NULL);
assert(size != 0); assert(size != 0);
witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), witness_assert_depth_to_rank(
WITNESS_RANK_CORE, 0); tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) if (alignment != 0
!= 0) { && ((uintptr_t)ptr & ((uintptr_t)alignment - 1)) != 0) {
/* Existing object alignment is inadequate. */ /* Existing object alignment is inadequate. */
*newsize = oldsize; *newsize = oldsize;
return true; return true;
} }
return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero, return arena_ralloc_no_move(
newsize); tsdn, ptr, oldsize, size, extra, zero, newsize);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
fastpath_success_finish(tsd_t *tsd, uint64_t allocated_after, fastpath_success_finish(
cache_bin_t *bin, void *ret) { tsd_t *tsd, uint64_t allocated_after, cache_bin_t *bin, void *ret) {
thread_allocated_set(tsd, allocated_after); thread_allocated_set(tsd, allocated_after);
if (config_stats) { if (config_stats) {
bin->tstats.nrequests++; bin->tstats.nrequests++;
@@ -331,8 +331,8 @@ imalloc_fastpath(size_t size, void *(fallback_alloc)(size_t)) {
sz_size2index_usize_fastpath(size, &ind, &usize); sz_size2index_usize_fastpath(size, &ind, &usize);
/* Fast path relies on size being a bin. */ /* Fast path relies on size being a bin. */
assert(ind < SC_NBINS); assert(ind < SC_NBINS);
assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS) && assert((SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS)
(size <= SC_SMALL_MAXCLASS)); && (size <= SC_SMALL_MAXCLASS));
uint64_t allocated, threshold; uint64_t allocated, threshold;
te_malloc_fastpath_ctx(tsd, &allocated, &threshold); te_malloc_fastpath_ctx(tsd, &allocated, &threshold);
@@ -394,8 +394,8 @@ tcache_get_from_ind(tsd_t *tsd, unsigned tcache_ind, bool slow, bool is_alloc) {
/* Getting tcache ptr unconditionally. */ /* Getting tcache ptr unconditionally. */
tcache = tsd_tcachep_get(tsd); tcache = tsd_tcachep_get(tsd);
assert(tcache == tcache_get(tsd)); assert(tcache == tcache_get(tsd));
} else if (is_alloc || } else if (is_alloc
likely(tsd_reentrancy_level_get(tsd) == 0)) { || likely(tsd_reentrancy_level_get(tsd) == 0)) {
tcache = tcache_get(tsd); tcache = tcache_get(tsd);
} else { } else {
tcache = NULL; tcache = NULL;
@@ -405,8 +405,8 @@ tcache_get_from_ind(tsd_t *tsd, unsigned tcache_ind, bool slow, bool is_alloc) {
* Should not specify tcache on deallocation path when being * Should not specify tcache on deallocation path when being
* reentrant. * reentrant.
*/ */
assert(is_alloc || tsd_reentrancy_level_get(tsd) == 0 || assert(is_alloc || tsd_reentrancy_level_get(tsd) == 0
tsd_state_nocleanup(tsd)); || tsd_state_nocleanup(tsd));
if (tcache_ind == TCACHE_IND_NONE) { if (tcache_ind == TCACHE_IND_NONE) {
tcache = NULL; tcache = NULL;
} else { } else {
@@ -420,14 +420,14 @@ JEMALLOC_ALWAYS_INLINE bool
maybe_check_alloc_ctx(tsd_t *tsd, void *ptr, emap_alloc_ctx_t *alloc_ctx) { maybe_check_alloc_ctx(tsd_t *tsd, void *ptr, emap_alloc_ctx_t *alloc_ctx) {
if (config_opt_size_checks) { if (config_opt_size_checks) {
emap_alloc_ctx_t dbg_ctx; emap_alloc_ctx_t dbg_ctx;
emap_alloc_ctx_lookup(tsd_tsdn(tsd), &arena_emap_global, ptr, emap_alloc_ctx_lookup(
&dbg_ctx); tsd_tsdn(tsd), &arena_emap_global, ptr, &dbg_ctx);
if (alloc_ctx->szind != dbg_ctx.szind) { if (alloc_ctx->szind != dbg_ctx.szind) {
safety_check_fail_sized_dealloc( safety_check_fail_sized_dealloc(
/* current_dealloc */ true, ptr, /* current_dealloc */ true, ptr,
/* true_size */ emap_alloc_ctx_usize_get(&dbg_ctx), /* true_size */ emap_alloc_ctx_usize_get(&dbg_ctx),
/* input_size */ emap_alloc_ctx_usize_get( /* input_size */
alloc_ctx)); emap_alloc_ctx_usize_get(alloc_ctx));
return true; return true;
} }
if (alloc_ctx->slab != dbg_ctx.slab) { if (alloc_ctx->slab != dbg_ctx.slab) {
@@ -482,7 +482,8 @@ free_fastpath_nonfast_aligned(void *ptr, bool check_prof) {
/* Returns whether or not the free attempt was successful. */ /* Returns whether or not the free attempt was successful. */
JEMALLOC_ALWAYS_INLINE JEMALLOC_ALWAYS_INLINE
bool free_fastpath(void *ptr, size_t size, bool size_hint) { bool
free_fastpath(void *ptr, size_t size, bool size_hint) {
tsd_t *tsd = tsd_get(false); tsd_t *tsd = tsd_get(false);
/* The branch gets optimized away unless tsd_get_allocates(). */ /* The branch gets optimized away unless tsd_get_allocates(). */
if (unlikely(tsd == NULL)) { if (unlikely(tsd == NULL)) {
@@ -493,18 +494,18 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
* testing (deallocated_after >= threshold) later in this function. * testing (deallocated_after >= threshold) later in this function.
* The threshold will be set to 0 when !tsd_fast. * The threshold will be set to 0 when !tsd_fast.
*/ */
assert(tsd_fast(tsd) || assert(tsd_fast(tsd)
*tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0); || *tsd_thread_deallocated_next_event_fastp_get_unsafe(tsd) == 0);
emap_alloc_ctx_t alloc_ctx JEMALLOC_CC_SILENCE_INIT({0, 0, false}); emap_alloc_ctx_t alloc_ctx JEMALLOC_CC_SILENCE_INIT({0, 0, false});
size_t usize; size_t usize;
if (!size_hint) { if (!size_hint) {
bool err = emap_alloc_ctx_try_lookup_fast(tsd, bool err = emap_alloc_ctx_try_lookup_fast(
&arena_emap_global, ptr, &alloc_ctx); tsd, &arena_emap_global, ptr, &alloc_ctx);
/* Note: profiled objects will have alloc_ctx.slab set */ /* Note: profiled objects will have alloc_ctx.slab set */
if (unlikely(err || !alloc_ctx.slab || if (unlikely(err || !alloc_ctx.slab
free_fastpath_nonfast_aligned(ptr, || free_fastpath_nonfast_aligned(ptr,
/* check_prof */ false))) { /* check_prof */ false))) {
return false; return false;
} }
@@ -516,8 +517,8 @@ bool free_fastpath(void *ptr, size_t size, bool size_hint) {
* special aligned objects. The alignment check will also check * special aligned objects. The alignment check will also check
* for null ptr. * for null ptr.
*/ */
if (unlikely(size > SC_LOOKUP_MAXCLASS || if (unlikely(size > SC_LOOKUP_MAXCLASS
free_fastpath_nonfast_aligned(ptr, || free_fastpath_nonfast_aligned(ptr,
/* check_prof */ true))) { /* check_prof */ true))) {
return false; return false;
} }

17
include/jemalloc/internal/jemalloc_internal_macros.h
View File

@@ -7,7 +7,8 @@
# ifdef _MSC_VER # ifdef _MSC_VER
# define JEMALLOC_ALWAYS_INLINE static __forceinline # define JEMALLOC_ALWAYS_INLINE static __forceinline
# else # else
# define JEMALLOC_ALWAYS_INLINE JEMALLOC_ATTR(always_inline) static inline # define JEMALLOC_ALWAYS_INLINE \
JEMALLOC_ATTR(always_inline) static inline
# endif # endif
#endif #endif
#ifdef _MSC_VER #ifdef _MSC_VER
@@ -50,7 +51,7 @@
#if defined(_MSC_VER) && !defined(__clang__) #if defined(_MSC_VER) && !defined(__clang__)
# define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push)) # define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push))
# define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop)) # define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop))
# define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable:W)) # define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable : W))
# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS # define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
# define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS # define JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS
# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS # define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS
@@ -58,8 +59,9 @@
# define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED # define JEMALLOC_DIAGNOSTIC_IGNORE_DEPRECATED
# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS # define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
/* #pragma GCC diagnostic first appeared in gcc 4.6. */ /* #pragma GCC diagnostic first appeared in gcc 4.6. */
#elif (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && \ #elif (defined(__GNUC__) \
(__GNUC_MINOR__ > 5)))) || defined(__clang__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) \
|| defined(__clang__)
/* /*
* The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang * The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang
* diagnostic suppression macros and should not be used anywhere else. * diagnostic suppression macros and should not be used anywhere else.
@@ -75,10 +77,11 @@
* all clang versions up to version 7 (currently trunk, unreleased). This macro * all clang versions up to version 7 (currently trunk, unreleased). This macro
* suppresses the warning for the affected compiler versions only. * suppresses the warning for the affected compiler versions only.
*/ */
# if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) || \ # if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) \
defined(__clang__) || defined(__clang__)
# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \ # define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \
JEMALLOC_DIAGNOSTIC_IGNORE("-Wmissing-field-initializers") JEMALLOC_DIAGNOSTIC_IGNORE( \
"-Wmissing-field-initializers")
# else # else
# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS # define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
# endif # endif

9
include/jemalloc/internal/jemalloc_internal_overrides.h
View File

@@ -9,13 +9,14 @@
*/ */
#ifdef JEMALLOC_OVERRIDE_LG_PAGE #ifdef JEMALLOC_OVERRIDE_LG_PAGE
#undef LG_PAGE # undef LG_PAGE
#define LG_PAGE JEMALLOC_OVERRIDE_LG_PAGE # define LG_PAGE JEMALLOC_OVERRIDE_LG_PAGE
#endif #endif
#ifdef JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF #ifdef JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF
#undef JEMALLOC_CONFIG_MALLOC_CONF # undef JEMALLOC_CONFIG_MALLOC_CONF
#define JEMALLOC_CONFIG_MALLOC_CONF JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF # define JEMALLOC_CONFIG_MALLOC_CONF \
JEMALLOC_OVERRIDE_JEMALLOC_CONFIG_MALLOC_CONF
#endif #endif
#endif /* JEMALLOC_INTERNAL_OVERRIDES_H */ #endif /* JEMALLOC_INTERNAL_OVERRIDES_H */

28
include/jemalloc/internal/jemalloc_internal_types.h
View File

@@ -18,7 +18,7 @@ enum zero_realloc_action_e {
typedef enum zero_realloc_action_e zero_realloc_action_t; typedef enum zero_realloc_action_e zero_realloc_action_t;
/* Signature of write callback. */ /* Signature of write callback. */
typedef void (write_cb_t)(void *, const char *); typedef void(write_cb_t)(void *, const char *);
enum malloc_init_e { enum malloc_init_e {
malloc_init_uninitialized = 3, malloc_init_uninitialized = 3,
@@ -56,12 +56,12 @@ typedef enum malloc_init_e malloc_init_t;
#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ #define MALLOCX_ALIGN_GET_SPECIFIED(flags) \
(ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK))
#define MALLOCX_ALIGN_GET(flags) \ #define MALLOCX_ALIGN_GET(flags) \
(MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1)) (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX - 1))
#define MALLOCX_ZERO_GET(flags) \ #define MALLOCX_ZERO_GET(flags) ((bool)(flags & MALLOCX_ZERO))
((bool)(flags & MALLOCX_ZERO))
#define MALLOCX_TCACHE_GET(flags) \ #define MALLOCX_TCACHE_GET(flags) \
(((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2) (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) \
- 2)
#define MALLOCX_ARENA_GET(flags) \ #define MALLOCX_ARENA_GET(flags) \
(((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1) (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1)
@@ -72,15 +72,13 @@ typedef enum malloc_init_e malloc_init_t;
#define LONG_MASK (LONG - 1) #define LONG_MASK (LONG - 1)
/* Return the smallest long multiple that is >= a. */ /* Return the smallest long multiple that is >= a. */
#define LONG_CEILING(a) \ #define LONG_CEILING(a) (((a) + LONG_MASK) & ~LONG_MASK)
(((a) + LONG_MASK) & ~LONG_MASK)
#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) #define SIZEOF_PTR (1U << LG_SIZEOF_PTR)
#define PTR_MASK (SIZEOF_PTR - 1) #define PTR_MASK (SIZEOF_PTR - 1)
/* Return the smallest (void *) multiple that is >= a. */ /* Return the smallest (void *) multiple that is >= a. */
#define PTR_CEILING(a) \ #define PTR_CEILING(a) (((a) + PTR_MASK) & ~PTR_MASK)
(((a) + PTR_MASK) & ~PTR_MASK)
/* /*
* Maximum size of L1 cache line. This is used to avoid cache line aliasing. * Maximum size of L1 cache line. This is used to avoid cache line aliasing.
@@ -94,13 +92,12 @@ typedef enum malloc_init_e malloc_init_t;
#define CACHELINE_MASK (CACHELINE - 1) #define CACHELINE_MASK (CACHELINE - 1)
/* Return the smallest cacheline multiple that is >= s. */ /* Return the smallest cacheline multiple that is >= s. */
#define CACHELINE_CEILING(s) \ #define CACHELINE_CEILING(s) (((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
(((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
/* Return the nearest aligned address at or below a. */ /* Return the nearest aligned address at or below a. */
#define ALIGNMENT_ADDR2BASE(a, alignment) \ #define ALIGNMENT_ADDR2BASE(a, alignment) \
((void *)(((byte_t *)(a)) - (((uintptr_t)(a)) - \ ((void *)(((byte_t *)(a)) \
((uintptr_t)(a) & ((~(alignment)) + 1))))) - (((uintptr_t)(a)) - ((uintptr_t)(a) & ((~(alignment)) + 1)))))
/* Return the offset between a and the nearest aligned address at or below a. */ /* Return the offset between a and the nearest aligned address at or below a. */
#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ #define ALIGNMENT_ADDR2OFFSET(a, alignment) \
@@ -120,8 +117,9 @@ typedef enum malloc_init_e malloc_init_t;
* definition of `byte_t` for more details. * definition of `byte_t` for more details.
*/ */
#define ALIGNMENT_ADDR2CEILING(a, alignment) \ #define ALIGNMENT_ADDR2CEILING(a, alignment) \
((void *)(((byte_t *)(a)) + (((((uintptr_t)(a)) + \ ((void *)(((byte_t *)(a)) \
(alignment - 1)) & ((~(alignment)) + 1)) - ((uintptr_t)(a))))) + (((((uintptr_t)(a)) + (alignment - 1)) & ((~(alignment)) + 1)) \
- ((uintptr_t)(a)))))
/* Declare a variable-length array. */ /* Declare a variable-length array. */
#if __STDC_VERSION__ < 199901L || defined(__STDC_NO_VLA__) #if __STDC_VERSION__ < 199901L || defined(__STDC_NO_VLA__)

8
include/jemalloc/internal/large_externs.h
View File

@@ -6,8 +6,8 @@
#include "jemalloc/internal/hook.h" #include "jemalloc/internal/hook.h"
void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero); void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, void *large_palloc(
bool zero); tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
bool large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min, bool large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
size_t usize_max, bool zero); size_t usize_max, bool zero);
void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
@@ -18,8 +18,8 @@ void large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata);
void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata); void large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);
void large_dalloc(tsdn_t *tsdn, edata_t *edata); void large_dalloc(tsdn_t *tsdn, edata_t *edata);
size_t large_salloc(tsdn_t *tsdn, const edata_t *edata); size_t large_salloc(tsdn_t *tsdn, const edata_t *edata);
void large_prof_info_get(tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, void large_prof_info_get(
bool reset_recent); tsd_t *tsd, edata_t *edata, prof_info_t *prof_info, bool reset_recent);
void large_prof_tctx_reset(edata_t *edata); void large_prof_tctx_reset(edata_t *edata);
void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size); void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size);

19
include/jemalloc/internal/lockedint.h
View File

@@ -36,7 +36,8 @@ struct locked_zu_s {
# define LOCKEDINT_MTX(mtx) (&(mtx)) # define LOCKEDINT_MTX(mtx) (&(mtx))
# define LOCKEDINT_MTX_LOCK(tsdn, mu) malloc_mutex_lock(tsdn, &(mu)) # define LOCKEDINT_MTX_LOCK(tsdn, mu) malloc_mutex_lock(tsdn, &(mu))
# define LOCKEDINT_MTX_UNLOCK(tsdn, mu) malloc_mutex_unlock(tsdn, &(mu)) # define LOCKEDINT_MTX_UNLOCK(tsdn, mu) malloc_mutex_unlock(tsdn, &(mu))
# define LOCKEDINT_MTX_PREFORK(tsdn, mu) malloc_mutex_prefork(tsdn, &(mu)) # define LOCKEDINT_MTX_PREFORK(tsdn, mu) \
malloc_mutex_prefork(tsdn, &(mu))
# define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu) \ # define LOCKEDINT_MTX_POSTFORK_PARENT(tsdn, mu) \
malloc_mutex_postfork_parent(tsdn, &(mu)) malloc_mutex_postfork_parent(tsdn, &(mu))
# define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu) \ # define LOCKEDINT_MTX_POSTFORK_CHILD(tsdn, mu) \
@@ -70,8 +71,7 @@ locked_read_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p) {
} }
static inline void static inline void
locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, uint64_t x) {
uint64_t x) {
LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
#ifdef JEMALLOC_ATOMIC_U64 #ifdef JEMALLOC_ATOMIC_U64
atomic_fetch_add_u64(&p->val, x, ATOMIC_RELAXED); atomic_fetch_add_u64(&p->val, x, ATOMIC_RELAXED);
@@ -81,8 +81,7 @@ locked_inc_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p,
} }
static inline void static inline void
locked_dec_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, locked_dec_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p, uint64_t x) {
uint64_t x) {
LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
#ifdef JEMALLOC_ATOMIC_U64 #ifdef JEMALLOC_ATOMIC_U64
uint64_t r = atomic_fetch_sub_u64(&p->val, x, ATOMIC_RELAXED); uint64_t r = atomic_fetch_sub_u64(&p->val, x, ATOMIC_RELAXED);
@@ -109,8 +108,8 @@ locked_inc_mod_u64(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_u64_t *p,
if (overflow) { if (overflow) {
after %= modulus; after %= modulus;
} }
} while (!atomic_compare_exchange_weak_u64(&p->val, &before, after, } while (!atomic_compare_exchange_weak_u64(
ATOMIC_RELAXED, ATOMIC_RELAXED)); &p->val, &before, after, ATOMIC_RELAXED, ATOMIC_RELAXED));
#else #else
before = p->val; before = p->val;
after = before + x; after = before + x;
@@ -167,8 +166,7 @@ locked_read_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p) {
} }
static inline void static inline void
locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, size_t x) {
size_t x) {
LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
#ifdef JEMALLOC_ATOMIC_U64 #ifdef JEMALLOC_ATOMIC_U64
atomic_fetch_add_zu(&p->val, x, ATOMIC_RELAXED); atomic_fetch_add_zu(&p->val, x, ATOMIC_RELAXED);
@@ -179,8 +177,7 @@ locked_inc_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p,
} }
static inline void static inline void
locked_dec_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, locked_dec_zu(tsdn_t *tsdn, malloc_mutex_t *mtx, locked_zu_t *p, size_t x) {
size_t x) {
LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx); LOCKEDINT_MTX_ASSERT_INTERNAL(tsdn, mtx);
#ifdef JEMALLOC_ATOMIC_U64 #ifdef JEMALLOC_ATOMIC_U64
size_t r = atomic_fetch_sub_zu(&p->val, x, ATOMIC_RELAXED); size_t r = atomic_fetch_sub_zu(&p->val, x, ATOMIC_RELAXED);

24
include/jemalloc/internal/log.h
View File

@@ -53,7 +53,8 @@ struct log_var_s {
#define LOG_INITIALIZED_NOT_ENABLED 1U #define LOG_INITIALIZED_NOT_ENABLED 1U
#define LOG_ENABLED 2U #define LOG_ENABLED 2U
#define LOG_VAR_INIT(name_str) {ATOMIC_INIT(LOG_NOT_INITIALIZED), name_str} #define LOG_VAR_INIT(name_str) \
{ ATOMIC_INIT(LOG_NOT_INITIALIZED), name_str }
/* /*
* Returns the value we should assume for state (which is not necessarily * Returns the value we should assume for state (which is not necessarily
@@ -64,20 +65,20 @@ unsigned log_var_update_state(log_var_t *log_var);
/* We factor out the metadata management to allow us to test more easily. */ /* We factor out the metadata management to allow us to test more easily. */
#define log_do_begin(log_var) \ #define log_do_begin(log_var) \
if (config_log) { \ if (config_log) { \
unsigned log_state = atomic_load_u(&(log_var).state, \ unsigned log_state = atomic_load_u( \
ATOMIC_RELAXED); \ &(log_var).state, ATOMIC_RELAXED); \
if (unlikely(log_state == LOG_NOT_INITIALIZED)) { \ if (unlikely(log_state == LOG_NOT_INITIALIZED)) { \
log_state = log_var_update_state(&(log_var)); \ log_state = log_var_update_state(&(log_var)); \
assert(log_state != LOG_NOT_INITIALIZED); \ assert(log_state != LOG_NOT_INITIALIZED); \
} \ } \
if (log_state == LOG_ENABLED) { \ if (log_state == LOG_ENABLED) { \
{ {
/* User code executes here. */ /* User code executes here. */
#define log_do_end(log_var) \ #define log_do_end(log_var) \
} \ } \
} \ } \
} }
/* /*
* MSVC has some preprocessor bugs in its expansion of __VA_ARGS__ during * MSVC has some preprocessor bugs in its expansion of __VA_ARGS__ during
@@ -95,9 +96,10 @@ log_impl_varargs(const char *name, ...) {
const char *format = va_arg(ap, const char *); const char *format = va_arg(ap, const char *);
size_t dst_offset = 0; size_t dst_offset = 0;
dst_offset += malloc_snprintf(buf, JEMALLOC_LOG_BUFSIZE, "%s: ", name); dst_offset += malloc_snprintf(buf, JEMALLOC_LOG_BUFSIZE, "%s: ", name);
dst_offset += malloc_vsnprintf(buf + dst_offset, dst_offset += malloc_vsnprintf(
JEMALLOC_LOG_BUFSIZE - dst_offset, format, ap); buf + dst_offset, JEMALLOC_LOG_BUFSIZE - dst_offset, format, ap);
malloc_snprintf(buf + dst_offset, JEMALLOC_LOG_BUFSIZE - dst_offset, "\n"); malloc_snprintf(
buf + dst_offset, JEMALLOC_LOG_BUFSIZE - dst_offset, "\n");
va_end(ap); va_end(ap);
malloc_write(buf); malloc_write(buf);
@@ -105,11 +107,11 @@ log_impl_varargs(const char *name, ...) {
/* Call as log("log.var.str", "format_string %d", arg_for_format_string); */ /* Call as log("log.var.str", "format_string %d", arg_for_format_string); */
#define LOG(log_var_str, ...) \ #define LOG(log_var_str, ...) \
do { \ do { \
static log_var_t log_var = LOG_VAR_INIT(log_var_str); \ static log_var_t log_var = LOG_VAR_INIT(log_var_str); \
log_do_begin(log_var) \ log_do_begin(log_var) \
log_impl_varargs((log_var).name, __VA_ARGS__); \ log_impl_varargs((log_var).name, __VA_ARGS__); \
log_do_end(log_var) \ log_do_end(log_var) \
} while (0) } while (0)
#endif /* JEMALLOC_INTERNAL_LOG_H */ #endif /* JEMALLOC_INTERNAL_LOG_H */

32
include/jemalloc/internal/malloc_io.h
View File

@@ -45,24 +45,23 @@
write_cb_t wrtmessage; write_cb_t wrtmessage;
int buferror(int err, char *buf, size_t buflen); int buferror(int err, char *buf, size_t buflen);
uintmax_t malloc_strtoumax(const char *restrict nptr, char **restrict endptr, uintmax_t malloc_strtoumax(
int base); const char *restrict nptr, char **restrict endptr, int base);
void malloc_write(const char *s); void malloc_write(const char *s);
/* /*
* malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
* point math. * point math.
*/ */
size_t malloc_vsnprintf(char *str, size_t size, const char *format, size_t malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap);
va_list ap);
size_t malloc_snprintf(char *str, size_t size, const char *format, ...) size_t malloc_snprintf(char *str, size_t size, const char *format, ...)
JEMALLOC_FORMAT_PRINTF(3, 4); JEMALLOC_FORMAT_PRINTF(3, 4);
/* /*
* The caller can set write_cb to null to choose to print with the * The caller can set write_cb to null to choose to print with the
* je_malloc_message hook. * je_malloc_message hook.
*/ */
void malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format, void malloc_vcprintf(
va_list ap); write_cb_t *write_cb, void *cbopaque, const char *format, va_list ap);
void malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, void malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format,
...) JEMALLOC_FORMAT_PRINTF(3, 4); ...) JEMALLOC_FORMAT_PRINTF(3, 4);
void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
@@ -81,9 +80,9 @@ malloc_write_fd_syscall(int fd, const void *buf, size_t count) {
long result = syscall(SYS_write, fd, buf, count); long result = syscall(SYS_write, fd, buf, count);
#else #else
ssize_t result = (ssize_t)write(fd, buf, ssize_t result = (ssize_t)write(fd, buf,
#ifdef _WIN32 # ifdef _WIN32
(unsigned int) (unsigned int)
#endif # endif
count); count);
#endif #endif
return (ssize_t)result; return (ssize_t)result;
@@ -110,9 +109,9 @@ malloc_read_fd_syscall(int fd, void *buf, size_t count) {
long result = syscall(SYS_read, fd, buf, count); long result = syscall(SYS_read, fd, buf, count);
#else #else
ssize_t result = read(fd, buf, ssize_t result = read(fd, buf,
#ifdef _WIN32 # ifdef _WIN32
(unsigned int) (unsigned int)
#endif # endif
count); count);
#endif #endif
return (ssize_t)result; return (ssize_t)result;
@@ -122,8 +121,8 @@ static inline ssize_t
malloc_read_fd(int fd, void *buf, size_t count) { malloc_read_fd(int fd, void *buf, size_t count) {
size_t bytes_read = 0; size_t bytes_read = 0;
do { do {
ssize_t result = malloc_read_fd_syscall(fd, ssize_t result = malloc_read_fd_syscall(
&((byte_t *)buf)[bytes_read], count - bytes_read); fd, &((byte_t *)buf)[bytes_read], count - bytes_read);
if (result < 0) { if (result < 0) {
return result; return result;
} else if (result == 0) { } else if (result == 0) {
@@ -134,7 +133,8 @@ malloc_read_fd(int fd, void *buf, size_t count) {
return bytes_read; return bytes_read;
} }
static inline int malloc_open(const char *path, int flags) { static inline int
malloc_open(const char *path, int flags) {
int fd; int fd;
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
fd = (int)syscall(SYS_open, path, flags); fd = (int)syscall(SYS_open, path, flags);
@@ -146,7 +146,8 @@ static inline int malloc_open(const char *path, int flags) {
return fd; return fd;
} }
static inline int malloc_close(int fd) { static inline int
malloc_close(int fd) {
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
return (int)syscall(SYS_close, fd); return (int)syscall(SYS_close, fd);
#else #else
@@ -154,7 +155,8 @@ static inline int malloc_close(int fd) {
#endif #endif
} }
static inline off_t malloc_lseek(int fd, off_t offset, int whence) { static inline off_t
malloc_lseek(int fd, off_t offset, int whence) {
#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_lseek) #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_lseek)
return (off_t)syscall(SYS_lseek, fd, offset, whence); return (off_t)syscall(SYS_lseek, fd, offset, whence);
#else #else

94
include/jemalloc/internal/mutex.h
View File

@@ -76,12 +76,15 @@ struct malloc_mutex_s {
#ifdef _WIN32 #ifdef _WIN32
# if _WIN32_WINNT >= 0x0600 # if _WIN32_WINNT >= 0x0600
# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) # define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
# define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) \
# define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock)) ReleaseSRWLockExclusive(&(m)->lock)
# define MALLOC_MUTEX_TRYLOCK(m) \
(!TryAcquireSRWLockExclusive(&(m)->lock))
# else # else
# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) # define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
# define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock)) # define MALLOC_MUTEX_TRYLOCK(m) \
(!TryEnterCriticalSection(&(m)->lock))
# endif # endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) #elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
@@ -94,42 +97,75 @@ struct malloc_mutex_s {
#endif #endif
#define LOCK_PROF_DATA_INITIALIZER \ #define LOCK_PROF_DATA_INITIALIZER \
{NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \ { \
ATOMIC_INIT(0), 0, NULL, 0} NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0, \
ATOMIC_INIT(0), 0, NULL, 0 \
}
#ifdef _WIN32 #ifdef _WIN32
# define MALLOC_MUTEX_INITIALIZER # define MALLOC_MUTEX_INITIALIZER
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) #elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
# if defined(JEMALLOC_DEBUG) # if defined(JEMALLOC_DEBUG)
# define MALLOC_MUTEX_INITIALIZER \ # define MALLOC_MUTEX_INITIALIZER \
{{{LOCK_PROF_DATA_INITIALIZER, ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \ { \
WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} {{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT), \
0 \
}
# else # else
# define MALLOC_MUTEX_INITIALIZER \ # define MALLOC_MUTEX_INITIALIZER \
{{{LOCK_PROF_DATA_INITIALIZER, ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \ { \
WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} {{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT) \
}
# endif # endif
#elif (defined(JEMALLOC_MUTEX_INIT_CB)) #elif (defined(JEMALLOC_MUTEX_INIT_CB))
# if (defined(JEMALLOC_DEBUG)) # if (defined(JEMALLOC_DEBUG))
# define MALLOC_MUTEX_INITIALIZER \ # define MALLOC_MUTEX_INITIALIZER \
{{{LOCK_PROF_DATA_INITIALIZER, ATOMIC_INIT(false), PTHREAD_MUTEX_INITIALIZER, NULL}}, \ { \
WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} {{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER, NULL}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT), \
0 \
}
# else # else
# define MALLOC_MUTEX_INITIALIZER \ # define MALLOC_MUTEX_INITIALIZER \
{{{LOCK_PROF_DATA_INITIALIZER, ATOMIC_INIT(false), PTHREAD_MUTEX_INITIALIZER, NULL}}, \ { \
WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} {{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER, NULL}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT) \
}
# endif # endif
#else #else
# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
# if defined(JEMALLOC_DEBUG) # if defined(JEMALLOC_DEBUG)
# define MALLOC_MUTEX_INITIALIZER \ # define MALLOC_MUTEX_INITIALIZER \
{{{LOCK_PROF_DATA_INITIALIZER, ATOMIC_INIT(false), PTHREAD_MUTEX_INITIALIZER}}, \ { \
WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} {{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT), \
0 \
}
# else # else
# define MALLOC_MUTEX_INITIALIZER \ # define MALLOC_MUTEX_INITIALIZER \
{{{LOCK_PROF_DATA_INITIALIZER, ATOMIC_INIT(false), PTHREAD_MUTEX_INITIALIZER}}, \ { \
WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} {{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT) \
}
# endif # endif
#endif #endif
@@ -214,12 +250,12 @@ malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
if (sum->max_n_thds < data->max_n_thds) { if (sum->max_n_thds < data->max_n_thds) {
sum->max_n_thds = data->max_n_thds; sum->max_n_thds = data->max_n_thds;
} }
uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds, uint32_t cur_n_waiting_thds = atomic_load_u32(
ATOMIC_RELAXED); &sum->n_waiting_thds, ATOMIC_RELAXED);
uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32( uint32_t new_n_waiting_thds = cur_n_waiting_thds
&data->n_waiting_thds, ATOMIC_RELAXED); + atomic_load_u32(&data->n_waiting_thds, ATOMIC_RELAXED);
atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds, atomic_store_u32(
ATOMIC_RELAXED); &sum->n_waiting_thds, new_n_waiting_thds, ATOMIC_RELAXED);
sum->n_owner_switches += data->n_owner_switches; sum->n_owner_switches += data->n_owner_switches;
sum->n_lock_ops += data->n_lock_ops; sum->n_lock_ops += data->n_lock_ops;
} }
@@ -274,16 +310,16 @@ malloc_mutex_prof_copy(mutex_prof_data_t *dst, mutex_prof_data_t *source) {
/* Copy the prof data from mutex for processing. */ /* Copy the prof data from mutex for processing. */
static inline void static inline void
malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_prof_read(
malloc_mutex_t *mutex) { tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) {
/* Can only read holding the mutex. */ /* Can only read holding the mutex. */
malloc_mutex_assert_owner(tsdn, mutex); malloc_mutex_assert_owner(tsdn, mutex);
malloc_mutex_prof_copy(data, &mutex->prof_data); malloc_mutex_prof_copy(data, &mutex->prof_data);
} }
static inline void static inline void
malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_prof_accum(
malloc_mutex_t *mutex) { tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) {
mutex_prof_data_t *source = &mutex->prof_data; mutex_prof_data_t *source = &mutex->prof_data;
/* Can only read holding the mutex. */ /* Can only read holding the mutex. */
malloc_mutex_assert_owner(tsdn, mutex); malloc_mutex_assert_owner(tsdn, mutex);
@@ -305,8 +341,8 @@ malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
/* Compare the prof data and update to the maximum. */ /* Compare the prof data and update to the maximum. */
static inline void static inline void
malloc_mutex_prof_max_update(tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_prof_max_update(
malloc_mutex_t *mutex) { tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) {
mutex_prof_data_t *source = &mutex->prof_data; mutex_prof_data_t *source = &mutex->prof_data;
/* Can only read holding the mutex. */ /* Can only read holding the mutex. */
malloc_mutex_assert_owner(tsdn, mutex); malloc_mutex_assert_owner(tsdn, mutex);

6
include/jemalloc/internal/mutex_prof.h
View File

@@ -56,7 +56,8 @@ typedef enum {
OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \ OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \
OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \ OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \
OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \ OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \
OP(num_owner_switch, uint64_t, "n_owner_switch", false, num_owner_switch) \ OP(num_owner_switch, uint64_t, "n_owner_switch", false, \
num_owner_switch) \
OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \ OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \
OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \ OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \
OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \ OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \
@@ -73,8 +74,7 @@ typedef enum {
#define COUNTER_ENUM(counter_list, t) \ #define COUNTER_ENUM(counter_list, t) \
typedef enum { \ typedef enum { \
counter_list \ counter_list mutex_prof_num_##t##_counters \
mutex_prof_num_##t##_counters \
} mutex_prof_##t##_counter_ind_t; } mutex_prof_##t##_counter_ind_t;
COUNTER_ENUM(MUTEX_PROF_UINT64_COUNTERS, uint64_t) COUNTER_ENUM(MUTEX_PROF_UINT64_COUNTERS, uint64_t)

17
include/jemalloc/internal/nstime.h
View File

@@ -9,9 +9,11 @@
#define NSTIME_MAGIC ((uint32_t)0xb8a9ce37) #define NSTIME_MAGIC ((uint32_t)0xb8a9ce37)
#ifdef JEMALLOC_DEBUG #ifdef JEMALLOC_DEBUG
# define NSTIME_ZERO_INITIALIZER {0, NSTIME_MAGIC} # define NSTIME_ZERO_INITIALIZER \
{ 0, NSTIME_MAGIC }
#else #else
# define NSTIME_ZERO_INITIALIZER {0} # define NSTIME_ZERO_INITIALIZER \
{ 0 }
#endif #endif
typedef struct { typedef struct {
@@ -41,22 +43,19 @@ uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
uint64_t nstime_ns_since(const nstime_t *past); uint64_t nstime_ns_since(const nstime_t *past);
uint64_t nstime_ms_since(const nstime_t *past); uint64_t nstime_ms_since(const nstime_t *past);
typedef bool (nstime_monotonic_t)(void); typedef bool(nstime_monotonic_t)(void);
extern nstime_monotonic_t *JET_MUTABLE nstime_monotonic; extern nstime_monotonic_t *JET_MUTABLE nstime_monotonic;
typedef void (nstime_update_t)(nstime_t *); typedef void(nstime_update_t)(nstime_t *);
extern nstime_update_t *JET_MUTABLE nstime_update; extern nstime_update_t *JET_MUTABLE nstime_update;
typedef void (nstime_prof_update_t)(nstime_t *); typedef void(nstime_prof_update_t)(nstime_t *);
extern nstime_prof_update_t *JET_MUTABLE nstime_prof_update; extern nstime_prof_update_t *JET_MUTABLE nstime_prof_update;
void nstime_init_update(nstime_t *time); void nstime_init_update(nstime_t *time);
void nstime_prof_init_update(nstime_t *time); void nstime_prof_init_update(nstime_t *time);
enum prof_time_res_e { enum prof_time_res_e { prof_time_res_default = 0, prof_time_res_high = 1 };
prof_time_res_default = 0,
prof_time_res_high = 1
};
typedef enum prof_time_res_e prof_time_res_t; typedef enum prof_time_res_e prof_time_res_t;
extern prof_time_res_t opt_prof_time_res; extern prof_time_res_t opt_prof_time_res;

12
include/jemalloc/internal/pa.h
View File

@@ -121,8 +121,8 @@ struct pa_shard_s {
static inline bool static inline bool
pa_shard_dont_decay_muzzy(pa_shard_t *shard) { pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
return ecache_npages_get(&shard->pac.ecache_muzzy) == 0 && return ecache_npages_get(&shard->pac.ecache_muzzy) == 0
pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0; && pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0;
} }
static inline ehooks_t * static inline ehooks_t *
@@ -199,8 +199,8 @@ ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state);
* though, the arena, background thread, and PAC modules are tightly interwoven * though, the arena, background thread, and PAC modules are tightly interwoven
* in a way that's tricky to extricate, so we only do the HPA-specific parts. * in a way that's tricky to extricate, so we only do the HPA-specific parts.
*/ */
void pa_shard_set_deferral_allowed(tsdn_t *tsdn, pa_shard_t *shard, void pa_shard_set_deferral_allowed(
bool deferral_allowed); tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed);
void pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); void pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
void pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); void pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard); uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
@@ -228,8 +228,8 @@ size_t pa_shard_nactive(pa_shard_t *shard);
size_t pa_shard_ndirty(pa_shard_t *shard); size_t pa_shard_ndirty(pa_shard_t *shard);
size_t pa_shard_nmuzzy(pa_shard_t *shard); size_t pa_shard_nmuzzy(pa_shard_t *shard);
void pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, void pa_shard_basic_stats_merge(
size_t *ndirty, size_t *nmuzzy); pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out, pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,

4
include/jemalloc/internal/pac.h
View File

@@ -195,8 +195,8 @@ bool pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
* *
* Returns true on error (if the new limit is not valid). * Returns true on error (if the new limit is not valid).
*/ */
bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit, bool pac_retain_grow_limit_get_set(
size_t *new_limit); tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit);
bool pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, bool pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
ssize_t decay_ms, pac_purge_eagerness_t eagerness); ssize_t decay_ms, pac_purge_eagerness_t eagerness);

24
include/jemalloc/internal/pages.h
View File

@@ -14,14 +14,11 @@ extern size_t os_page;
#define PAGE ((size_t)(1U << LG_PAGE)) #define PAGE ((size_t)(1U << LG_PAGE))
#define PAGE_MASK ((size_t)(PAGE - 1)) #define PAGE_MASK ((size_t)(PAGE - 1))
/* Return the page base address for the page containing address a. */ /* Return the page base address for the page containing address a. */
#define PAGE_ADDR2BASE(a) \ #define PAGE_ADDR2BASE(a) ALIGNMENT_ADDR2BASE(a, PAGE)
ALIGNMENT_ADDR2BASE(a, PAGE)
/* Return the smallest pagesize multiple that is >= s. */ /* Return the smallest pagesize multiple that is >= s. */
#define PAGE_CEILING(s) \ #define PAGE_CEILING(s) (((s) + PAGE_MASK) & ~PAGE_MASK)
(((s) + PAGE_MASK) & ~PAGE_MASK)
/* Return the largest pagesize multiple that is <=s. */ /* Return the largest pagesize multiple that is <=s. */
#define PAGE_FLOOR(s) \ #define PAGE_FLOOR(s) ((s) & ~PAGE_MASK)
((s) & ~PAGE_MASK)
/* Huge page size. LG_HUGEPAGE is determined by the configure script. */ /* Huge page size. LG_HUGEPAGE is determined by the configure script. */
#define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE)) #define HUGEPAGE ((size_t)(1U << LG_HUGEPAGE))
@@ -49,11 +46,9 @@ extern size_t os_page;
#endif #endif
/* Return the huge page base address for the huge page containing address a. */ /* Return the huge page base address for the huge page containing address a. */
#define HUGEPAGE_ADDR2BASE(a) \ #define HUGEPAGE_ADDR2BASE(a) ALIGNMENT_ADDR2BASE(a, HUGEPAGE)
ALIGNMENT_ADDR2BASE(a, HUGEPAGE)
/* Return the smallest pagesize multiple that is >= s. */ /* Return the smallest pagesize multiple that is >= s. */
#define HUGEPAGE_CEILING(s) \ #define HUGEPAGE_CEILING(s) (((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
(((s) + HUGEPAGE_MASK) & ~HUGEPAGE_MASK)
/* PAGES_CAN_PURGE_LAZY is defined if lazy purging is supported. */ /* PAGES_CAN_PURGE_LAZY is defined if lazy purging is supported. */
#if defined(_WIN32) || defined(JEMALLOC_PURGE_MADVISE_FREE) #if defined(_WIN32) || defined(JEMALLOC_PURGE_MADVISE_FREE)
@@ -68,9 +63,10 @@ extern size_t os_page;
* next step after purging on Windows anyway, there's no point in adding such * next step after purging on Windows anyway, there's no point in adding such
* complexity. * complexity.
*/ */
#if !defined(_WIN32) && ((defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \ #if !defined(_WIN32) \
defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)) || \ && ((defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \
defined(JEMALLOC_MAPS_COALESCE)) && defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)) \
|| defined(JEMALLOC_MAPS_COALESCE))
# define PAGES_CAN_PURGE_FORCED # define PAGES_CAN_PURGE_FORCED
#endif #endif
@@ -128,7 +124,7 @@ bool pages_collapse(void *addr, size_t size);
bool pages_dontdump(void *addr, size_t size); bool pages_dontdump(void *addr, size_t size);
bool pages_dodump(void *addr, size_t size); bool pages_dodump(void *addr, size_t size);
bool pages_boot(void); bool pages_boot(void);
void pages_set_thp_state (void *ptr, size_t size); void pages_set_thp_state(void *ptr, size_t size);
void pages_mark_guards(void *head, void *tail); void pages_mark_guards(void *head, void *tail);
void pages_unmark_guards(void *head, void *tail); void pages_unmark_guards(void *head, void *tail);

13
include/jemalloc/internal/pai.h
View File

@@ -41,9 +41,8 @@ struct pai_s {
*/ */
static inline edata_t * static inline edata_t *
pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool zero, bool guarded, bool frequent_reuse, bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
bool *deferred_work_generated) {
return self->alloc(tsdn, self, size, alignment, zero, guarded, return self->alloc(tsdn, self, size, alignment, zero, guarded,
frequent_reuse, deferred_work_generated); frequent_reuse, deferred_work_generated);
} }
@@ -66,13 +65,13 @@ pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
static inline bool static inline bool
pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool *deferred_work_generated) { size_t new_size, bool *deferred_work_generated) {
return self->shrink(tsdn, self, edata, old_size, new_size, return self->shrink(
deferred_work_generated); tsdn, self, edata, old_size, new_size, deferred_work_generated);
} }
static inline void static inline void
pai_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata, pai_dalloc(
bool *deferred_work_generated) { tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
self->dalloc(tsdn, self, edata, deferred_work_generated); self->dalloc(tsdn, self, edata, deferred_work_generated);
} }

3
include/jemalloc/internal/peak.h
View File

@@ -14,7 +14,8 @@ struct peak_s {
uint64_t adjustment; uint64_t adjustment;
}; };
#define PEAK_INITIALIZER {0, 0} #define PEAK_INITIALIZER \
{ 0, 0 }
static inline uint64_t static inline uint64_t
peak_max(peak_t *peak) { peak_max(peak_t *peak) {

134
include/jemalloc/internal/ph.h
View File

@@ -129,8 +129,7 @@ phn_prev_set(void *phn, void *prev, size_t offset) {
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
phn_merge_ordered(void *phn0, void *phn1, size_t offset, phn_merge_ordered(void *phn0, void *phn1, size_t offset, ph_cmp_t cmp) {
ph_cmp_t cmp) {
void *phn0child; void *phn0child;
assert(phn0 != NULL); assert(phn0 != NULL);
@@ -361,8 +360,7 @@ ph_insert(ph_t *ph, void *phn, size_t offset, ph_cmp_t cmp) {
phn_next_set(phn, phn_next_get(ph->root, offset), offset); phn_next_set(phn, phn_next_get(ph->root, offset), offset);
if (phn_next_get(ph->root, offset) != NULL) { if (phn_next_get(ph->root, offset) != NULL) {
phn_prev_set(phn_next_get(ph->root, offset), phn, phn_prev_set(phn_next_get(ph->root, offset), phn, offset);
offset);
} }
phn_prev_set(phn, ph->root, offset); phn_prev_set(phn, ph->root, offset);
phn_next_set(ph->root, phn, offset); phn_next_set(ph->root, phn, offset);
@@ -387,7 +385,6 @@ ph_remove_first(ph_t *ph, size_t offset, ph_cmp_t cmp) {
ph->root = ph_merge_children(ph->root, offset, cmp); ph->root = ph_merge_children(ph->root, offset, cmp);
return ret; return ret;
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
@@ -398,11 +395,11 @@ ph_remove(ph_t *ph, void *phn, size_t offset, ph_cmp_t cmp) {
return; return;
} }
void* prev = phn_prev_get(phn, offset); void *prev = phn_prev_get(phn, offset);
void* next = phn_next_get(phn, offset); void *next = phn_next_get(phn, offset);
/* If we have children, then we integrate them back in the heap. */ /* If we have children, then we integrate them back in the heap. */
void* replace = ph_merge_children(phn, offset, cmp); void *replace = ph_merge_children(phn, offset, cmp);
if (replace != NULL) { if (replace != NULL) {
phn_next_set(replace, next, offset); phn_next_set(replace, next, offset);
if (next != NULL) { if (next != NULL) {
@@ -438,16 +435,16 @@ ph_enumerate_vars_init(ph_enumerate_vars_t *vars, uint16_t max_visit_num,
* max_queue_size must be able to support max_visit_num, which means * max_queue_size must be able to support max_visit_num, which means
* the queue will not overflow before reaching max_visit_num. * the queue will not overflow before reaching max_visit_num.
*/ */
assert(vars->max_queue_size >= (vars->max_visit_num + 1)/2); assert(vars->max_queue_size >= (vars->max_visit_num + 1) / 2);
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
ph_enumerate_queue_push(void *phn, void **bfs_queue, ph_enumerate_queue_push(
ph_enumerate_vars_t *vars) { void *phn, void **bfs_queue, ph_enumerate_vars_t *vars) {
assert(vars->queue_size < vars->max_queue_size); assert(vars->queue_size < vars->max_queue_size);
bfs_queue[vars->rear] = phn; bfs_queue[vars->rear] = phn;
vars->rear = (vars->rear + 1) % vars->max_queue_size; vars->rear = (vars->rear + 1) % vars->max_queue_size;
(vars->queue_size) ++; (vars->queue_size)++;
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
@@ -456,11 +453,10 @@ ph_enumerate_queue_pop(void **bfs_queue, ph_enumerate_vars_t *vars) {
assert(vars->queue_size <= vars->max_queue_size); assert(vars->queue_size <= vars->max_queue_size);
void *ret = bfs_queue[vars->front]; void *ret = bfs_queue[vars->front];
vars->front = (vars->front + 1) % vars->max_queue_size; vars->front = (vars->front + 1) % vars->max_queue_size;
(vars->queue_size) --; (vars->queue_size)--;
return ret; return ret;
} }
/* /*
* The two functions below offer a solution to enumerate the pairing heap. * The two functions below offer a solution to enumerate the pairing heap.
* Whe enumerating, always call ph_enumerate_prepare first to prepare the queue * Whe enumerating, always call ph_enumerate_prepare first to prepare the queue
@@ -478,13 +474,13 @@ ph_enumerate_prepare(ph_t *ph, void **bfs_queue, ph_enumerate_vars_t *vars,
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
ph_enumerate_next(ph_t *ph, size_t offset, void **bfs_queue, ph_enumerate_next(
ph_enumerate_vars_t *vars) { ph_t *ph, size_t offset, void **bfs_queue, ph_enumerate_vars_t *vars) {
if (vars->queue_size == 0) { if (vars->queue_size == 0) {
return NULL; return NULL;
} }
(vars->visited_num) ++; (vars->visited_num)++;
if (vars->visited_num > vars->max_visit_num) { if (vars->visited_num > vars->max_visit_num) {
return NULL; return NULL;
} }
@@ -503,19 +499,18 @@ ph_enumerate_next(ph_t *ph, size_t offset, void **bfs_queue,
} }
#define ph_structs(a_prefix, a_type, a_max_queue_size) \ #define ph_structs(a_prefix, a_type, a_max_queue_size) \
typedef struct { \ typedef struct { \
phn_link_t link; \ phn_link_t link; \
} a_prefix##_link_t; \ } a_prefix##_link_t; \
\ \
typedef struct { \ typedef struct { \
ph_t ph; \ ph_t ph; \
} a_prefix##_t; \ } a_prefix##_t; \
\ \
typedef struct { \ typedef struct { \
void *bfs_queue[a_max_queue_size]; \ void *bfs_queue[a_max_queue_size]; \
ph_enumerate_vars_t vars; \ ph_enumerate_vars_t vars; \
} a_prefix##_enumerate_helper_t; } a_prefix##_enumerate_helper_t;
/* /*
* The ph_proto() macro generates function prototypes that correspond to the * The ph_proto() macro generates function prototypes that correspond to the
@@ -523,88 +518,77 @@ typedef struct { \
*/ */
#define ph_proto(a_attr, a_prefix, a_type) \ #define ph_proto(a_attr, a_prefix, a_type) \
\ \
a_attr void a_prefix##_new(a_prefix##_t *ph); \ a_attr void a_prefix##_new(a_prefix##_t *ph); \
a_attr bool a_prefix##_empty(a_prefix##_t *ph); \ a_attr bool a_prefix##_empty(a_prefix##_t *ph); \
a_attr a_type *a_prefix##_first(a_prefix##_t *ph); \ a_attr a_type *a_prefix##_first(a_prefix##_t *ph); \
a_attr a_type *a_prefix##_any(a_prefix##_t *ph); \ a_attr a_type *a_prefix##_any(a_prefix##_t *ph); \
a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn); \ a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn); \
a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph); \ a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph); \
a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn); \ a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn); \
a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph); \ a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph); \
a_attr void a_prefix##_enumerate_prepare(a_prefix##_t *ph, \ a_attr void a_prefix##_enumerate_prepare(a_prefix##_t *ph, \
a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num, \ a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num, \
uint16_t max_queue_size); \ uint16_t max_queue_size); \
a_attr a_type *a_prefix##_enumerate_next(a_prefix##_t *ph, \ a_attr a_type *a_prefix##_enumerate_next( \
a_prefix##_enumerate_helper_t *helper); a_prefix##_t *ph, a_prefix##_enumerate_helper_t *helper);
/* The ph_gen() macro generates a type-specific pairing heap implementation. */ /* The ph_gen() macro generates a type-specific pairing heap implementation. */
#define ph_gen(a_attr, a_prefix, a_type, a_field, a_cmp) \ #define ph_gen(a_attr, a_prefix, a_type, a_field, a_cmp) \
JEMALLOC_ALWAYS_INLINE int \ JEMALLOC_ALWAYS_INLINE int a_prefix##_ph_cmp(void *a, void *b) { \
a_prefix##_ph_cmp(void *a, void *b) { \
return a_cmp((a_type *)a, (a_type *)b); \ return a_cmp((a_type *)a, (a_type *)b); \
} \ } \
\ \
a_attr void \ a_attr void a_prefix##_new(a_prefix##_t *ph) { \
a_prefix##_new(a_prefix##_t *ph) { \
ph_new(&ph->ph); \ ph_new(&ph->ph); \
} \ } \
\ \
a_attr bool \ a_attr bool a_prefix##_empty(a_prefix##_t *ph) { \
a_prefix##_empty(a_prefix##_t *ph) { \
return ph_empty(&ph->ph); \ return ph_empty(&ph->ph); \
} \ } \
\ \
a_attr a_type * \ a_attr a_type *a_prefix##_first(a_prefix##_t *ph) { \
a_prefix##_first(a_prefix##_t *ph) { \ return ph_first( \
return ph_first(&ph->ph, offsetof(a_type, a_field), \ &ph->ph, offsetof(a_type, a_field), &a_prefix##_ph_cmp); \
&a_prefix##_ph_cmp); \ } \
} \
\ \
a_attr a_type * \ a_attr a_type *a_prefix##_any(a_prefix##_t *ph) { \
a_prefix##_any(a_prefix##_t *ph) { \
return ph_any(&ph->ph, offsetof(a_type, a_field)); \ return ph_any(&ph->ph, offsetof(a_type, a_field)); \
} \ } \
\ \
a_attr void \ a_attr void a_prefix##_insert(a_prefix##_t *ph, a_type *phn) { \
a_prefix##_insert(a_prefix##_t *ph, a_type *phn) { \
ph_insert(&ph->ph, phn, offsetof(a_type, a_field), \ ph_insert(&ph->ph, phn, offsetof(a_type, a_field), \
a_prefix##_ph_cmp); \ a_prefix##_ph_cmp); \
} \ } \
\ \
a_attr a_type * \ a_attr a_type *a_prefix##_remove_first(a_prefix##_t *ph) { \
a_prefix##_remove_first(a_prefix##_t *ph) { \ return ph_remove_first( \
return ph_remove_first(&ph->ph, offsetof(a_type, a_field), \ &ph->ph, offsetof(a_type, a_field), a_prefix##_ph_cmp); \
a_prefix##_ph_cmp); \ } \
} \
\ \
a_attr void \ a_attr void a_prefix##_remove(a_prefix##_t *ph, a_type *phn) { \
a_prefix##_remove(a_prefix##_t *ph, a_type *phn) { \
ph_remove(&ph->ph, phn, offsetof(a_type, a_field), \ ph_remove(&ph->ph, phn, offsetof(a_type, a_field), \
a_prefix##_ph_cmp); \ a_prefix##_ph_cmp); \
} \ } \
\ \
a_attr a_type * \ a_attr a_type *a_prefix##_remove_any(a_prefix##_t *ph) { \
a_prefix##_remove_any(a_prefix##_t *ph) { \
a_type *ret = a_prefix##_any(ph); \ a_type *ret = a_prefix##_any(ph); \
if (ret != NULL) { \ if (ret != NULL) { \
a_prefix##_remove(ph, ret); \ a_prefix##_remove(ph, ret); \
} \ } \
return ret; \ return ret; \
} \ } \
\ \
a_attr void \ a_attr void a_prefix##_enumerate_prepare(a_prefix##_t *ph, \
a_prefix##_enumerate_prepare(a_prefix##_t *ph, \
a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num, \ a_prefix##_enumerate_helper_t *helper, uint16_t max_visit_num, \
uint16_t max_queue_size) { \ uint16_t max_queue_size) { \
ph_enumerate_prepare(&ph->ph, helper->bfs_queue, &helper->vars, \ ph_enumerate_prepare(&ph->ph, helper->bfs_queue, \
max_visit_num, max_queue_size); \ &helper->vars, max_visit_num, max_queue_size); \
} \ } \
\ \
a_attr a_type * \ a_attr a_type *a_prefix##_enumerate_next( \
a_prefix##_enumerate_next(a_prefix##_t *ph, \ a_prefix##_t *ph, a_prefix##_enumerate_helper_t *helper) { \
a_prefix##_enumerate_helper_t *helper) { \
return ph_enumerate_next(&ph->ph, offsetof(a_type, a_field), \ return ph_enumerate_next(&ph->ph, offsetof(a_type, a_field), \
helper->bfs_queue, &helper->vars); \ helper->bfs_queue, &helper->vars); \
} }
#endif /* JEMALLOC_INTERNAL_PH_H */ #endif /* JEMALLOC_INTERNAL_PH_H */

2
include/jemalloc/internal/prng.h
View File

@@ -49,7 +49,7 @@ prng_state_next_zu(size_t state) {
#elif LG_SIZEOF_PTR == 3 #elif LG_SIZEOF_PTR == 3
return (state * PRNG_A_64) + PRNG_C_64; return (state * PRNG_A_64) + PRNG_C_64;
#else #else
#error Unsupported pointer size # error Unsupported pointer size
#endif #endif
} }

2
include/jemalloc/internal/prof_data.h
View File

@@ -23,7 +23,7 @@ int prof_thread_name_set_impl(tsd_t *tsd, const char *thread_name);
void prof_unbias_map_init(void); void prof_unbias_map_init(void);
void prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque, void prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
prof_tdata_t *tdata, bool leakcheck); prof_tdata_t *tdata, bool leakcheck);
prof_tdata_t * prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, prof_tdata_t *prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid,
uint64_t thr_discrim, char *thread_name, bool active); uint64_t thr_discrim, char *thread_name, bool active);
void prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata); void prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
void prof_reset(tsd_t *tsd, size_t lg_sample); void prof_reset(tsd_t *tsd, size_t lg_sample);

21
include/jemalloc/internal/prof_externs.h
View File

@@ -12,7 +12,8 @@ extern bool opt_prof_active;
extern bool opt_prof_thread_active_init; extern bool opt_prof_thread_active_init;
extern unsigned opt_prof_bt_max; extern unsigned opt_prof_bt_max;
extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */
extern size_t opt_experimental_lg_prof_threshold; /* Mean bytes between thresholds. */ extern size_t
opt_experimental_lg_prof_threshold; /* Mean bytes between thresholds. */
extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
extern bool opt_prof_gdump; /* High-water memory dumping. */ extern bool opt_prof_gdump; /* High-water memory dumping. */
extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_final; /* Final profile dumping. */
@@ -21,7 +22,7 @@ extern bool opt_prof_leak_error; /* Exit with error code if memory leaked */
extern bool opt_prof_accum; /* Report cumulative bytes. */ extern bool opt_prof_accum; /* Report cumulative bytes. */
extern bool opt_prof_log; /* Turn logging on at boot. */ extern bool opt_prof_log; /* Turn logging on at boot. */
extern char opt_prof_prefix[ extern char opt_prof_prefix[
/* Minimize memory bloat for non-prof builds. */ /* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF #ifdef JEMALLOC_PROF
PATH_MAX + PATH_MAX +
#endif #endif
@@ -77,10 +78,10 @@ prof_tdata_t *prof_tdata_init(tsd_t *tsd);
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx); void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
void prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t size, void prof_malloc_sample_object(
size_t usize, prof_tctx_t *tctx); tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
void prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, void prof_free_sampled_object(
prof_info_t *prof_info); tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
prof_tctx_t *prof_tctx_create(tsd_t *tsd); prof_tctx_t *prof_tctx_create(tsd_t *tsd);
void prof_idump(tsdn_t *tsdn); void prof_idump(tsdn_t *tsdn);
bool prof_mdump(tsd_t *tsd, const char *filename); bool prof_mdump(tsd_t *tsd, const char *filename);
@@ -130,8 +131,8 @@ uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
*/ */
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
te_prof_sample_event_lookahead_surplus(tsd_t *tsd, size_t usize, te_prof_sample_event_lookahead_surplus(
size_t *surplus) { tsd_t *tsd, size_t usize, size_t *surplus) {
if (surplus != NULL) { if (surplus != NULL) {
/* /*
* This is a dead store: the surplus will be overwritten before * This is a dead store: the surplus will be overwritten before
@@ -146,8 +147,8 @@ te_prof_sample_event_lookahead_surplus(tsd_t *tsd, size_t usize,
return false; return false;
} }
/* The subtraction is intentionally susceptible to underflow. */ /* The subtraction is intentionally susceptible to underflow. */
uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize - uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize
tsd_thread_allocated_last_event_get(tsd); - tsd_thread_allocated_last_event_get(tsd);
uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd); uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd);
if (accumbytes < sample_wait) { if (accumbytes < sample_wait) {
return false; return false;

6
include/jemalloc/internal/prof_hook.h
View File

@@ -21,7 +21,8 @@ typedef void (*prof_backtrace_hook_t)(void **, unsigned *, unsigned);
typedef void (*prof_dump_hook_t)(const char *filename); typedef void (*prof_dump_hook_t)(const char *filename);
/* ptr, size, backtrace vector, backtrace vector length, usize */ /* ptr, size, backtrace vector, backtrace vector length, usize */
typedef void (*prof_sample_hook_t)(const void *ptr, size_t size, void **backtrace, unsigned backtrace_length, size_t usize); typedef void (*prof_sample_hook_t)(const void *ptr, size_t size,
void **backtrace, unsigned backtrace_length, size_t usize);
/* ptr, size */ /* ptr, size */
typedef void (*prof_sample_free_hook_t)(const void *, size_t); typedef void (*prof_sample_free_hook_t)(const void *, size_t);
@@ -29,6 +30,7 @@ typedef void (*prof_sample_free_hook_t)(const void *, size_t);
/* /*
* A callback hook that notifies when an allocation threshold has been crossed. * A callback hook that notifies when an allocation threshold has been crossed.
*/ */
typedef void (*prof_threshold_hook_t)(uint64_t alloc, uint64_t dealloc, uint64_t peak); typedef void (*prof_threshold_hook_t)(
uint64_t alloc, uint64_t dealloc, uint64_t peak);
#endif /* JEMALLOC_INTERNAL_PROF_HOOK_H */ #endif /* JEMALLOC_INTERNAL_PROF_HOOK_H */

19
include/jemalloc/internal/prof_inlines.h
View File

@@ -164,8 +164,8 @@ JEMALLOC_ALWAYS_INLINE prof_tctx_t *
prof_alloc_prep(tsd_t *tsd, bool prof_active, bool sample_event) { prof_alloc_prep(tsd_t *tsd, bool prof_active, bool sample_event) {
prof_tctx_t *ret; prof_tctx_t *ret;
if (!prof_active || if (!prof_active
likely(prof_sample_should_skip(tsd, sample_event))) { || likely(prof_sample_should_skip(tsd, sample_event))) {
ret = PROF_TCTX_SENTINEL; ret = PROF_TCTX_SENTINEL;
} else { } else {
ret = prof_tctx_create(tsd); ret = prof_tctx_create(tsd);
@@ -242,8 +242,8 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t size, size_t usize,
* counters. * counters.
*/ */
if (unlikely(old_sampled)) { if (unlikely(old_sampled)) {
prof_free_sampled_object(tsd, old_ptr, old_usize, prof_free_sampled_object(
old_prof_info); tsd, old_ptr, old_usize, old_prof_info);
} }
} }
@@ -254,9 +254,10 @@ prof_sample_align(size_t usize, size_t orig_align) {
* w/o metadata lookup. * w/o metadata lookup.
*/ */
assert(opt_prof); assert(opt_prof);
return (orig_align < PROF_SAMPLE_ALIGNMENT && return (orig_align < PROF_SAMPLE_ALIGNMENT
(sz_can_use_slab(usize) || opt_cache_oblivious)) ? && (sz_can_use_slab(usize) || opt_cache_oblivious))
PROF_SAMPLE_ALIGNMENT : orig_align; ? PROF_SAMPLE_ALIGNMENT
: orig_align;
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
@@ -271,8 +272,8 @@ prof_sampled(tsd_t *tsd, const void *ptr) {
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
prof_free(tsd_t *tsd, const void *ptr, size_t usize, prof_free(
emap_alloc_ctx_t *alloc_ctx) { tsd_t *tsd, const void *ptr, size_t usize, emap_alloc_ctx_t *alloc_ctx) {
prof_info_t prof_info; prof_info_t prof_info;
prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info); prof_info_get_and_reset_recent(tsd, ptr, alloc_ctx, &prof_info);

8
include/jemalloc/internal/prof_sys.h
View File

@@ -23,13 +23,13 @@ void prof_gdump_impl(tsd_t *tsd);
int prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high); int prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high);
/* Used in unit tests. */ /* Used in unit tests. */
typedef int (prof_sys_thread_name_read_t)(char *buf, size_t limit); typedef int(prof_sys_thread_name_read_t)(char *buf, size_t limit);
extern prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read; extern prof_sys_thread_name_read_t *JET_MUTABLE prof_sys_thread_name_read;
typedef int (prof_dump_open_file_t)(const char *, int); typedef int(prof_dump_open_file_t)(const char *, int);
extern prof_dump_open_file_t *JET_MUTABLE prof_dump_open_file; extern prof_dump_open_file_t *JET_MUTABLE prof_dump_open_file;
typedef ssize_t (prof_dump_write_file_t)(int, const void *, size_t); typedef ssize_t(prof_dump_write_file_t)(int, const void *, size_t);
extern prof_dump_write_file_t *JET_MUTABLE prof_dump_write_file; extern prof_dump_write_file_t *JET_MUTABLE prof_dump_write_file;
typedef int (prof_dump_open_maps_t)(void); typedef int(prof_dump_open_maps_t)(void);
extern prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps; extern prof_dump_open_maps_t *JET_MUTABLE prof_dump_open_maps;
#endif /* JEMALLOC_INTERNAL_PROF_SYS_H */ #endif /* JEMALLOC_INTERNAL_PROF_SYS_H */

4
include/jemalloc/internal/prof_types.h
View File

@@ -69,9 +69,9 @@ typedef struct prof_recent_s prof_recent_t;
/* Minimize memory bloat for non-prof builds. */ /* Minimize memory bloat for non-prof builds. */
#ifdef JEMALLOC_PROF #ifdef JEMALLOC_PROF
#define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1) # define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
#else #else
#define PROF_DUMP_FILENAME_LEN 1 # define PROF_DUMP_FILENAME_LEN 1
#endif #endif
/* Default number of recent allocations to record. */ /* Default number of recent allocations to record. */

75
include/jemalloc/internal/ql.h
View File

@@ -29,12 +29,13 @@
/* List definitions. */ /* List definitions. */
#define ql_head(a_type) \ #define ql_head(a_type) \
struct { \ struct { \
a_type *qlh_first; \ a_type *qlh_first; \
} }
/* Static initializer for an empty list. */ /* Static initializer for an empty list. */
#define ql_head_initializer(a_head) {NULL} #define ql_head_initializer(a_head) \
{ NULL }
/* The field definition. */ /* The field definition. */
#define ql_elm(a_type) qr(a_type) #define ql_elm(a_type) qr(a_type)
@@ -43,18 +44,20 @@ struct { \
#define ql_first(a_head) ((a_head)->qlh_first) #define ql_first(a_head) ((a_head)->qlh_first)
/* Dynamically initializes a list. */ /* Dynamically initializes a list. */
#define ql_new(a_head) do { \ #define ql_new(a_head) \
do { \
ql_first(a_head) = NULL; \ ql_first(a_head) = NULL; \
} while (0) } while (0)
/* /*
* Sets dest to be the contents of src (overwriting any elements there), leaving * Sets dest to be the contents of src (overwriting any elements there), leaving
* src empty. * src empty.
*/ */
#define ql_move(a_head_dest, a_head_src) do { \ #define ql_move(a_head_dest, a_head_src) \
do { \
ql_first(a_head_dest) = ql_first(a_head_src); \ ql_first(a_head_dest) = ql_first(a_head_src); \
ql_new(a_head_src); \ ql_new(a_head_src); \
} while (0) } while (0)
/* True if the list is empty, otherwise false. */ /* True if the list is empty, otherwise false. */
#define ql_empty(a_head) (ql_first(a_head) == NULL) #define ql_empty(a_head) (ql_first(a_head) == NULL)
@@ -76,56 +79,60 @@ struct { \
* the end or retreat before the beginning of the list returns NULL. * the end or retreat before the beginning of the list returns NULL.
*/ */
#define ql_next(a_head, a_elm, a_field) \ #define ql_next(a_head, a_elm, a_field) \
((ql_last(a_head, a_field) != (a_elm)) \ ((ql_last(a_head, a_field) != (a_elm)) ? qr_next((a_elm), a_field) \
? qr_next((a_elm), a_field) : NULL)
#define ql_prev(a_head, a_elm, a_field) \
((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) \
: NULL) : NULL)
#define ql_prev(a_head, a_elm, a_field) \
((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) : NULL)
/* Inserts a_elm before a_qlelm in the list. */ /* Inserts a_elm before a_qlelm in the list. */
#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ #define ql_before_insert(a_head, a_qlelm, a_elm, a_field) \
do { \
qr_before_insert((a_qlelm), (a_elm), a_field); \ qr_before_insert((a_qlelm), (a_elm), a_field); \
if (ql_first(a_head) == (a_qlelm)) { \ if (ql_first(a_head) == (a_qlelm)) { \
ql_first(a_head) = (a_elm); \ ql_first(a_head) = (a_elm); \
} \ } \
} while (0) } while (0)
/* Inserts a_elm after a_qlelm in the list. */ /* Inserts a_elm after a_qlelm in the list. */
#define ql_after_insert(a_qlelm, a_elm, a_field) \ #define ql_after_insert(a_qlelm, a_elm, a_field) \
qr_after_insert((a_qlelm), (a_elm), a_field) qr_after_insert((a_qlelm), (a_elm), a_field)
/* Inserts a_elm as the first item in the list. */ /* Inserts a_elm as the first item in the list. */
#define ql_head_insert(a_head, a_elm, a_field) do { \ #define ql_head_insert(a_head, a_elm, a_field) \
do { \
if (!ql_empty(a_head)) { \ if (!ql_empty(a_head)) { \
qr_before_insert(ql_first(a_head), (a_elm), a_field); \ qr_before_insert(ql_first(a_head), (a_elm), a_field); \
} \ } \
ql_first(a_head) = (a_elm); \ ql_first(a_head) = (a_elm); \
} while (0) } while (0)
/* Inserts a_elm as the last item in the list. */ /* Inserts a_elm as the last item in the list. */
#define ql_tail_insert(a_head, a_elm, a_field) do { \ #define ql_tail_insert(a_head, a_elm, a_field) \
do { \
if (!ql_empty(a_head)) { \ if (!ql_empty(a_head)) { \
qr_before_insert(ql_first(a_head), (a_elm), a_field); \ qr_before_insert(ql_first(a_head), (a_elm), a_field); \
} \ } \
ql_first(a_head) = qr_next((a_elm), a_field); \ ql_first(a_head) = qr_next((a_elm), a_field); \
} while (0) } while (0)
/* /*
* Given lists a = [a_1, ..., a_n] and [b_1, ..., b_n], results in: * Given lists a = [a_1, ..., a_n] and [b_1, ..., b_n], results in:
* a = [a1, ..., a_n, b_1, ..., b_n] and b = []. * a = [a1, ..., a_n, b_1, ..., b_n] and b = [].
*/ */
#define ql_concat(a_head_a, a_head_b, a_field) do { \ #define ql_concat(a_head_a, a_head_b, a_field) \
do { \
if (ql_empty(a_head_a)) { \ if (ql_empty(a_head_a)) { \
ql_move(a_head_a, a_head_b); \ ql_move(a_head_a, a_head_b); \
} else if (!ql_empty(a_head_b)) { \ } else if (!ql_empty(a_head_b)) { \
qr_meld(ql_first(a_head_a), ql_first(a_head_b), \ qr_meld( \
a_field); \ ql_first(a_head_a), ql_first(a_head_b), a_field); \
ql_new(a_head_b); \ ql_new(a_head_b); \
} \ } \
} while (0) } while (0)
/* Removes a_elm from the list. */ /* Removes a_elm from the list. */
#define ql_remove(a_head, a_elm, a_field) do { \ #define ql_remove(a_head, a_elm, a_field) \
do { \
if (ql_first(a_head) == (a_elm)) { \ if (ql_first(a_head) == (a_elm)) { \
ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ ql_first(a_head) = qr_next(ql_first(a_head), a_field); \
} \ } \
@@ -134,19 +141,21 @@ struct { \
} else { \ } else { \
ql_new(a_head); \ ql_new(a_head); \
} \ } \
} while (0) } while (0)
/* Removes the first item in the list. */ /* Removes the first item in the list. */
#define ql_head_remove(a_head, a_type, a_field) do { \ #define ql_head_remove(a_head, a_type, a_field) \
do { \
a_type *t = ql_first(a_head); \ a_type *t = ql_first(a_head); \
ql_remove((a_head), t, a_field); \ ql_remove((a_head), t, a_field); \
} while (0) } while (0)
/* Removes the last item in the list. */ /* Removes the last item in the list. */
#define ql_tail_remove(a_head, a_type, a_field) do { \ #define ql_tail_remove(a_head, a_type, a_field) \
do { \
a_type *t = ql_last(a_head, a_field); \ a_type *t = ql_last(a_head, a_field); \
ql_remove((a_head), t, a_field); \ ql_remove((a_head), t, a_field); \
} while (0) } while (0)
/* /*
* Given a = [a_1, a_2, ..., a_n-1, a_n, a_n+1, ...], * Given a = [a_1, a_2, ..., a_n-1, a_n, a_n+1, ...],
@@ -155,14 +164,15 @@ struct { \
* and replaces b's contents with: * and replaces b's contents with:
* b = [a_n, a_n+1, ...] * b = [a_n, a_n+1, ...]
*/ */
#define ql_split(a_head_a, a_elm, a_head_b, a_field) do { \ #define ql_split(a_head_a, a_elm, a_head_b, a_field) \
do { \
if (ql_first(a_head_a) == (a_elm)) { \ if (ql_first(a_head_a) == (a_elm)) { \
ql_move(a_head_b, a_head_a); \ ql_move(a_head_b, a_head_a); \
} else { \ } else { \
qr_split(ql_first(a_head_a), (a_elm), a_field); \ qr_split(ql_first(a_head_a), (a_elm), a_field); \
ql_first(a_head_b) = (a_elm); \ ql_first(a_head_b) = (a_elm); \
} \ } \
} while (0) } while (0)
/* /*
* An optimized version of: * An optimized version of:
@@ -170,9 +180,10 @@ struct { \
* ql_remove((a_head), t, a_field); * ql_remove((a_head), t, a_field);
* ql_tail_insert((a_head), t, a_field); * ql_tail_insert((a_head), t, a_field);
*/ */
#define ql_rotate(a_head, a_field) do { \ #define ql_rotate(a_head, a_field) \
do { \
ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ ql_first(a_head) = qr_next(ql_first(a_head), a_field); \
} while (0) } while (0)
/* /*
* Helper macro to iterate over each element in a list in order, starting from * Helper macro to iterate over each element in a list in order, starting from
@@ -190,7 +201,7 @@ struct { \
*/ */
#define ql_foreach(a_var, a_head, a_field) \ #define ql_foreach(a_var, a_head, a_field) \
qr_foreach((a_var), ql_first(a_head), a_field) qr_foreach ((a_var), ql_first(a_head), a_field)
#define ql_reverse_foreach(a_var, a_head, a_field) \ #define ql_reverse_foreach(a_var, a_head, a_field) \
qr_reverse_foreach((a_var), ql_first(a_head), a_field) qr_reverse_foreach((a_var), ql_first(a_head), a_field)

26
include/jemalloc/internal/qr.h
View File

@@ -18,20 +18,21 @@
/* Ring definitions. */ /* Ring definitions. */
#define qr(a_type) \ #define qr(a_type) \
struct { \ struct { \
a_type *qre_next; \ a_type *qre_next; \
a_type *qre_prev; \ a_type *qre_prev; \
} }
/* /*
* Initialize a qr link. Every link must be initialized before being used, even * Initialize a qr link. Every link must be initialized before being used, even
* if that initialization is going to be immediately overwritten (say, by being * if that initialization is going to be immediately overwritten (say, by being
* passed into an insertion macro). * passed into an insertion macro).
*/ */
#define qr_new(a_qr, a_field) do { \ #define qr_new(a_qr, a_field) \
do { \
(a_qr)->a_field.qre_next = (a_qr); \ (a_qr)->a_field.qre_next = (a_qr); \
(a_qr)->a_field.qre_prev = (a_qr); \ (a_qr)->a_field.qre_prev = (a_qr); \
} while (0) } while (0)
/* /*
* Go forwards or backwards in the ring. Note that (the ring being circular), this * Go forwards or backwards in the ring. Note that (the ring being circular), this
@@ -58,7 +59,8 @@ struct { \
* *
* a_qr_a can directly be a qr_next() macro, but a_qr_b cannot. * a_qr_a can directly be a qr_next() macro, but a_qr_b cannot.
*/ */
#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ #define qr_meld(a_qr_a, a_qr_b, a_field) \
do { \
(a_qr_b)->a_field.qre_prev->a_field.qre_next = \ (a_qr_b)->a_field.qre_prev->a_field.qre_next = \
(a_qr_a)->a_field.qre_prev; \ (a_qr_a)->a_field.qre_prev; \
(a_qr_a)->a_field.qre_prev = (a_qr_b)->a_field.qre_prev; \ (a_qr_a)->a_field.qre_prev = (a_qr_b)->a_field.qre_prev; \
@@ -66,7 +68,7 @@ struct { \
(a_qr_b)->a_field.qre_prev->a_field.qre_next; \ (a_qr_b)->a_field.qre_prev->a_field.qre_next; \
(a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \ (a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \
(a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \ (a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \
} while (0) } while (0)
/* /*
* Logically, this is just a meld. The intent, though, is that a_qrelm is a * Logically, this is just a meld. The intent, though, is that a_qrelm is a
@@ -98,8 +100,7 @@ struct { \
* qr_meld() and qr_split() are functionally equivalent, so there's no need to * qr_meld() and qr_split() are functionally equivalent, so there's no need to
* have two copies of the code. * have two copies of the code.
*/ */
#define qr_split(a_qr_a, a_qr_b, a_field) \ #define qr_split(a_qr_a, a_qr_b, a_field) qr_meld((a_qr_a), (a_qr_b), a_field)
qr_meld((a_qr_a), (a_qr_b), a_field)
/* /*
* Splits off a_qr from the rest of its ring, so that it becomes a * Splits off a_qr from the rest of its ring, so that it becomes a
@@ -122,10 +123,10 @@ struct { \
* } * }
*/ */
#define qr_foreach(var, a_qr, a_field) \ #define qr_foreach(var, a_qr, a_field) \
for ((var) = (a_qr); \ for ((var) = (a_qr); (var) != NULL; \
(var) != NULL; \
(var) = (((var)->a_field.qre_next != (a_qr)) \ (var) = (((var)->a_field.qre_next != (a_qr)) \
? (var)->a_field.qre_next : NULL)) ? (var)->a_field.qre_next \
: NULL))
/* /*
* The same (and with the same usage) as qr_foreach, but in the opposite order, * The same (and with the same usage) as qr_foreach, but in the opposite order,
@@ -134,7 +135,6 @@ struct { \
#define qr_reverse_foreach(var, a_qr, a_field) \ #define qr_reverse_foreach(var, a_qr, a_field) \
for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \ for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \
(var) != NULL; \ (var) != NULL; \
(var) = (((var) != (a_qr)) \ (var) = (((var) != (a_qr)) ? (var)->a_field.qre_prev : NULL))
? (var)->a_field.qre_prev : NULL))
#endif /* JEMALLOC_INTERNAL_QR_H */ #endif /* JEMALLOC_INTERNAL_QR_H */

18
include/jemalloc/internal/quantum.h
View File

@@ -15,7 +15,8 @@
# ifdef __alpha__ # ifdef __alpha__
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) # if (defined(__sparc64__) || defined(__sparcv9) \
|| defined(__sparc_v9__))
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) # if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
@@ -49,7 +50,8 @@
# ifdef __or1k__ # ifdef __or1k__
# define LG_QUANTUM 3 # define LG_QUANTUM 3
# endif # endif
# if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__ppc64__) # if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) \
|| defined(__ppc64__)
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# if defined(__riscv) || defined(__riscv__) # if defined(__riscv) || defined(__riscv__)
@@ -58,8 +60,8 @@
# ifdef __s390__ # ifdef __s390__
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ # if (defined(__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) \
defined(__SH4_SINGLE_ONLY__)) || defined(__SH4_SINGLE_ONLY__))
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# ifdef __tile__ # ifdef __tile__
@@ -72,8 +74,9 @@
# define LG_QUANTUM 3 # define LG_QUANTUM 3
# endif # endif
# ifndef LG_QUANTUM # ifndef LG_QUANTUM
# error "Unknown minimum alignment for architecture; specify via " # error \
"--with-lg-quantum" "Unknown minimum alignment for architecture; specify via "
"--with-lg-quantum"
# endif # endif
#endif #endif
@@ -81,7 +84,6 @@
#define QUANTUM_MASK (QUANTUM - 1) #define QUANTUM_MASK (QUANTUM - 1)
/* Return the smallest quantum multiple that is >= a. */ /* Return the smallest quantum multiple that is >= a. */
#define QUANTUM_CEILING(a) \ #define QUANTUM_CEILING(a) (((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
(((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
#endif /* JEMALLOC_INTERNAL_QUANTUM_H */ #endif /* JEMALLOC_INTERNAL_QUANTUM_H */

2
include/jemalloc/internal/rb.h
View File

@@ -26,7 +26,7 @@
*/ */
#ifndef __PGI #ifndef __PGI
#define RB_COMPACT # define RB_COMPACT
#endif #endif
/* /*

155
include/jemalloc/internal/rtree.h
View File

@@ -18,7 +18,7 @@
*/ */
/* Number of high insignificant bits. */ /* Number of high insignificant bits. */
#define RTREE_NHIB ((1U << (LG_SIZEOF_PTR+3)) - LG_VADDR) #define RTREE_NHIB ((1U << (LG_SIZEOF_PTR + 3)) - LG_VADDR)
/* Number of low insigificant bits. */ /* Number of low insigificant bits. */
#define RTREE_NLIB LG_PAGE #define RTREE_NLIB LG_PAGE
/* Number of significant bits. */ /* Number of significant bits. */
@@ -59,7 +59,8 @@ struct rtree_contents_s {
#define RTREE_LEAF_STATE_WIDTH EDATA_BITS_STATE_WIDTH #define RTREE_LEAF_STATE_WIDTH EDATA_BITS_STATE_WIDTH
#define RTREE_LEAF_STATE_SHIFT 2 #define RTREE_LEAF_STATE_SHIFT 2
#define RTREE_LEAF_STATE_MASK MASK(RTREE_LEAF_STATE_WIDTH, RTREE_LEAF_STATE_SHIFT) #define RTREE_LEAF_STATE_MASK \
MASK(RTREE_LEAF_STATE_WIDTH, RTREE_LEAF_STATE_SHIFT)
struct rtree_leaf_elm_s { struct rtree_leaf_elm_s {
#ifdef RTREE_LEAF_COMPACT #ifdef RTREE_LEAF_COMPACT
@@ -104,9 +105,9 @@ struct rtree_s {
malloc_mutex_t init_lock; malloc_mutex_t init_lock;
/* Number of elements based on rtree_levels[0].bits. */ /* Number of elements based on rtree_levels[0].bits. */
#if RTREE_HEIGHT > 1 #if RTREE_HEIGHT > 1
rtree_node_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; rtree_node_elm_t root[1U << (RTREE_NSB / RTREE_HEIGHT)];
#else #else
rtree_leaf_elm_t root[1U << (RTREE_NSB/RTREE_HEIGHT)]; rtree_leaf_elm_t root[1U << (RTREE_NSB / RTREE_HEIGHT)];
#endif #endif
}; };
@@ -120,13 +121,13 @@ static const rtree_level_t rtree_levels[] = {
#if RTREE_HEIGHT == 1 #if RTREE_HEIGHT == 1
{RTREE_NSB, RTREE_NHIB + RTREE_NSB} {RTREE_NSB, RTREE_NHIB + RTREE_NSB}
#elif RTREE_HEIGHT == 2 #elif RTREE_HEIGHT == 2
{RTREE_NSB/2, RTREE_NHIB + RTREE_NSB/2}, {RTREE_NSB / 2, RTREE_NHIB + RTREE_NSB / 2},
{RTREE_NSB/2 + RTREE_NSB%2, RTREE_NHIB + RTREE_NSB} {RTREE_NSB / 2 + RTREE_NSB % 2, RTREE_NHIB + RTREE_NSB}
#elif RTREE_HEIGHT == 3 #elif RTREE_HEIGHT == 3
{RTREE_NSB/3, RTREE_NHIB + RTREE_NSB/3}, {RTREE_NSB / 3, RTREE_NHIB + RTREE_NSB / 3},
{RTREE_NSB/3 + RTREE_NSB%3/2, {RTREE_NSB / 3 + RTREE_NSB % 3 / 2,
RTREE_NHIB + RTREE_NSB/3*2 + RTREE_NSB%3/2}, RTREE_NHIB + RTREE_NSB / 3 * 2 + RTREE_NSB % 3 / 2},
{RTREE_NSB/3 + RTREE_NSB%3 - RTREE_NSB%3/2, RTREE_NHIB + RTREE_NSB} {RTREE_NSB / 3 + RTREE_NSB % 3 - RTREE_NSB % 3 / 2, RTREE_NHIB + RTREE_NSB}
#else #else
# error Unsupported rtree height # error Unsupported rtree height
#endif #endif
@@ -139,9 +140,9 @@ rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
JEMALLOC_ALWAYS_INLINE unsigned JEMALLOC_ALWAYS_INLINE unsigned
rtree_leaf_maskbits(void) { rtree_leaf_maskbits(void) {
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3); unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR + 3);
unsigned cumbits = (rtree_levels[RTREE_HEIGHT-1].cumbits - unsigned cumbits = (rtree_levels[RTREE_HEIGHT - 1].cumbits
rtree_levels[RTREE_HEIGHT-1].bits); - rtree_levels[RTREE_HEIGHT - 1].bits);
return ptrbits - cumbits; return ptrbits - cumbits;
} }
@@ -153,13 +154,13 @@ rtree_leafkey(uintptr_t key) {
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
rtree_cache_direct_map(uintptr_t key) { rtree_cache_direct_map(uintptr_t key) {
return (size_t)((key >> rtree_leaf_maskbits()) & return (
(RTREE_CTX_NCACHE - 1)); size_t)((key >> rtree_leaf_maskbits()) & (RTREE_CTX_NCACHE - 1));
} }
JEMALLOC_ALWAYS_INLINE uintptr_t JEMALLOC_ALWAYS_INLINE uintptr_t
rtree_subkey(uintptr_t key, unsigned level) { rtree_subkey(uintptr_t key, unsigned level) {
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3); unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR + 3);
unsigned cumbits = rtree_levels[level].cumbits; unsigned cumbits = rtree_levels[level].cumbits;
unsigned shiftbits = ptrbits - cumbits; unsigned shiftbits = ptrbits - cumbits;
unsigned maskbits = rtree_levels[level].bits; unsigned maskbits = rtree_levels[level].bits;
@@ -178,12 +179,12 @@ rtree_subkey(uintptr_t key, unsigned level) {
* dependent on a previous rtree write, which means a stale read * dependent on a previous rtree write, which means a stale read
* could result if synchronization were omitted here. * could result if synchronization were omitted here.
*/ */
# ifdef RTREE_LEAF_COMPACT #ifdef RTREE_LEAF_COMPACT
JEMALLOC_ALWAYS_INLINE uintptr_t JEMALLOC_ALWAYS_INLINE uintptr_t
rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_bits_read(
rtree_leaf_elm_t *elm, bool dependent) { tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) {
return (uintptr_t)atomic_load_p(&elm->le_bits, dependent return (uintptr_t)atomic_load_p(
? ATOMIC_RELAXED : ATOMIC_ACQUIRE); &elm->le_bits, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE);
} }
JEMALLOC_ALWAYS_INLINE uintptr_t JEMALLOC_ALWAYS_INLINE uintptr_t
@@ -195,10 +196,10 @@ rtree_leaf_elm_bits_encode(rtree_contents_t contents) {
uintptr_t szind_bits = (uintptr_t)contents.metadata.szind << LG_VADDR; uintptr_t szind_bits = (uintptr_t)contents.metadata.szind << LG_VADDR;
uintptr_t slab_bits = (uintptr_t)contents.metadata.slab; uintptr_t slab_bits = (uintptr_t)contents.metadata.slab;
uintptr_t is_head_bits = (uintptr_t)contents.metadata.is_head << 1; uintptr_t is_head_bits = (uintptr_t)contents.metadata.is_head << 1;
uintptr_t state_bits = (uintptr_t)contents.metadata.state << uintptr_t state_bits = (uintptr_t)contents.metadata.state
RTREE_LEAF_STATE_SHIFT; << RTREE_LEAF_STATE_SHIFT;
uintptr_t metadata_bits = szind_bits | state_bits | is_head_bits | uintptr_t metadata_bits = szind_bits | state_bits | is_head_bits
slab_bits; | slab_bits;
assert((edata_bits & metadata_bits) == 0); assert((edata_bits & metadata_bits) == 0);
return edata_bits | metadata_bits; return edata_bits | metadata_bits;
@@ -212,8 +213,8 @@ rtree_leaf_elm_bits_decode(uintptr_t bits) {
contents.metadata.slab = (bool)(bits & 1); contents.metadata.slab = (bool)(bits & 1);
contents.metadata.is_head = (bool)(bits & (1 << 1)); contents.metadata.is_head = (bool)(bits & (1 << 1));
uintptr_t state_bits = (bits & RTREE_LEAF_STATE_MASK) >> uintptr_t state_bits = (bits & RTREE_LEAF_STATE_MASK)
RTREE_LEAF_STATE_SHIFT; >> RTREE_LEAF_STATE_SHIFT;
assert(state_bits <= extent_state_max); assert(state_bits <= extent_state_max);
contents.metadata.state = (extent_state_t)state_bits; contents.metadata.state = (extent_state_t)state_bits;
@@ -232,45 +233,46 @@ rtree_leaf_elm_bits_decode(uintptr_t bits) {
/* Restore sign-extended high bits, mask metadata bits. */ /* Restore sign-extended high bits, mask metadata bits. */
/* NOLINTNEXTLINE(performance-no-int-to-ptr) */ /* NOLINTNEXTLINE(performance-no-int-to-ptr) */
contents.edata = (edata_t *)((uintptr_t)((intptr_t)(bits << RTREE_NHIB) contents.edata = (edata_t *)((uintptr_t)((intptr_t)(bits << RTREE_NHIB)
>> RTREE_NHIB) & low_bit_mask); >> RTREE_NHIB)
& low_bit_mask);
# endif # endif
assert((uintptr_t)contents.edata % (uintptr_t)EDATA_ALIGNMENT == 0); assert((uintptr_t)contents.edata % (uintptr_t)EDATA_ALIGNMENT == 0);
return contents; return contents;
} }
# endif /* RTREE_LEAF_COMPACT */ #endif /* RTREE_LEAF_COMPACT */
JEMALLOC_ALWAYS_INLINE rtree_contents_t JEMALLOC_ALWAYS_INLINE rtree_contents_t
rtree_leaf_elm_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, rtree_leaf_elm_read(
bool dependent) { tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) {
#ifdef RTREE_LEAF_COMPACT #ifdef RTREE_LEAF_COMPACT
uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent);
rtree_contents_t contents = rtree_leaf_elm_bits_decode(bits); rtree_contents_t contents = rtree_leaf_elm_bits_decode(bits);
return contents; return contents;
#else #else
rtree_contents_t contents; rtree_contents_t contents;
unsigned metadata_bits = atomic_load_u(&elm->le_metadata, dependent unsigned metadata_bits = atomic_load_u(
? ATOMIC_RELAXED : ATOMIC_ACQUIRE); &elm->le_metadata, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE);
contents.metadata.slab = (bool)(metadata_bits & 1); contents.metadata.slab = (bool)(metadata_bits & 1);
contents.metadata.is_head = (bool)(metadata_bits & (1 << 1)); contents.metadata.is_head = (bool)(metadata_bits & (1 << 1));
uintptr_t state_bits = (metadata_bits & RTREE_LEAF_STATE_MASK) >> uintptr_t state_bits = (metadata_bits & RTREE_LEAF_STATE_MASK)
RTREE_LEAF_STATE_SHIFT; >> RTREE_LEAF_STATE_SHIFT;
assert(state_bits <= extent_state_max); assert(state_bits <= extent_state_max);
contents.metadata.state = (extent_state_t)state_bits; contents.metadata.state = (extent_state_t)state_bits;
contents.metadata.szind = metadata_bits >> (RTREE_LEAF_STATE_SHIFT + contents.metadata.szind = metadata_bits
RTREE_LEAF_STATE_WIDTH); >> (RTREE_LEAF_STATE_SHIFT + RTREE_LEAF_STATE_WIDTH);
contents.edata = (edata_t *)atomic_load_p(&elm->le_edata, dependent contents.edata = (edata_t *)atomic_load_p(
? ATOMIC_RELAXED : ATOMIC_ACQUIRE); &elm->le_edata, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE);
return contents; return contents;
#endif #endif
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
rtree_contents_encode(rtree_contents_t contents, void **bits, rtree_contents_encode(
unsigned *additional) { rtree_contents_t contents, void **bits, unsigned *additional) {
#ifdef RTREE_LEAF_COMPACT #ifdef RTREE_LEAF_COMPACT
/* NOLINTNEXTLINE(performance-no-int-to-ptr) */ /* NOLINTNEXTLINE(performance-no-int-to-ptr) */
*bits = (void *)rtree_leaf_elm_bits_encode(contents); *bits = (void *)rtree_leaf_elm_bits_encode(contents);
@@ -282,15 +284,15 @@ rtree_contents_encode(rtree_contents_t contents, void **bits,
*additional = (unsigned)contents.metadata.slab *additional = (unsigned)contents.metadata.slab
| ((unsigned)contents.metadata.is_head << 1) | ((unsigned)contents.metadata.is_head << 1)
| ((unsigned)contents.metadata.state << RTREE_LEAF_STATE_SHIFT) | ((unsigned)contents.metadata.state << RTREE_LEAF_STATE_SHIFT)
| ((unsigned)contents.metadata.szind << (RTREE_LEAF_STATE_SHIFT + | ((unsigned)contents.metadata.szind
RTREE_LEAF_STATE_WIDTH)); << (RTREE_LEAF_STATE_SHIFT + RTREE_LEAF_STATE_WIDTH));
*bits = contents.edata; *bits = contents.edata;
#endif #endif
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
rtree_leaf_elm_write_commit(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_write_commit(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
rtree_leaf_elm_t *elm, void *bits, unsigned additional) { void *bits, unsigned additional) {
#ifdef RTREE_LEAF_COMPACT #ifdef RTREE_LEAF_COMPACT
atomic_store_p(&elm->le_bits, bits, ATOMIC_RELEASE); atomic_store_p(&elm->le_bits, bits, ATOMIC_RELEASE);
#else #else
@@ -304,8 +306,8 @@ rtree_leaf_elm_write_commit(tsdn_t *tsdn, rtree_t *rtree,
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
rtree_leaf_elm_t *elm, rtree_contents_t contents) { rtree_contents_t contents) {
assert((uintptr_t)contents.edata % EDATA_ALIGNMENT == 0); assert((uintptr_t)contents.edata % EDATA_ALIGNMENT == 0);
void *bits; void *bits;
unsigned additional; unsigned additional;
@@ -358,7 +360,7 @@ rtree_leaf_elm_lookup_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf;
assert(leaf != NULL); assert(leaf != NULL);
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT - 1);
*elm = &leaf[subkey]; *elm = &leaf[subkey];
return false; return false;
@@ -378,14 +380,15 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) { if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) {
rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf;
assert(leaf != NULL); assert(leaf != NULL);
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT - 1);
return &leaf[subkey]; return &leaf[subkey];
} }
/* /*
* Search the L2 LRU cache. On hit, swap the matching element into the * Search the L2 LRU cache. On hit, swap the matching element into the
* slot in L1 cache, and move the position in L2 up by 1. * slot in L1 cache, and move the position in L2 up by 1.
*/ */
#define RTREE_CACHE_CHECK_L2(i) do { \ #define RTREE_CACHE_CHECK_L2(i) \
do { \
if (likely(rtree_ctx->l2_cache[i].leafkey == leafkey)) { \ if (likely(rtree_ctx->l2_cache[i].leafkey == leafkey)) { \
rtree_leaf_elm_t *leaf = rtree_ctx->l2_cache[i].leaf; \ rtree_leaf_elm_t *leaf = rtree_ctx->l2_cache[i].leaf; \
assert(leaf != NULL); \ assert(leaf != NULL); \
@@ -407,10 +410,11 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
} \ } \
rtree_ctx->cache[slot].leafkey = leafkey; \ rtree_ctx->cache[slot].leafkey = leafkey; \
rtree_ctx->cache[slot].leaf = leaf; \ rtree_ctx->cache[slot].leaf = leaf; \
uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); \ uintptr_t subkey = rtree_subkey( \
key, RTREE_HEIGHT - 1); \
return &leaf[subkey]; \ return &leaf[subkey]; \
} \ } \
} while (0) } while (0)
/* Check the first cache entry. */ /* Check the first cache entry. */
RTREE_CACHE_CHECK_L2(0); RTREE_CACHE_CHECK_L2(0);
/* Search the remaining cache elements. */ /* Search the remaining cache elements. */
@@ -419,8 +423,8 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
} }
#undef RTREE_CACHE_CHECK_L2 #undef RTREE_CACHE_CHECK_L2
return rtree_leaf_elm_lookup_hard(tsdn, rtree, rtree_ctx, key, return rtree_leaf_elm_lookup_hard(
dependent, init_missing); tsdn, rtree, rtree_ctx, key, dependent, init_missing);
} }
/* /*
@@ -440,8 +444,8 @@ rtree_read_independent(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
} }
static inline rtree_contents_t static inline rtree_contents_t
rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_read(
uintptr_t key) { tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, /* dependent */ true, /* init_missing */ false); key, /* dependent */ true, /* init_missing */ false);
assert(elm != NULL); assert(elm != NULL);
@@ -449,21 +453,22 @@ rtree_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
} }
static inline rtree_metadata_t static inline rtree_metadata_t
rtree_metadata_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_metadata_read(
uintptr_t key) { tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, /* dependent */ true, /* init_missing */ false); key, /* dependent */ true, /* init_missing */ false);
assert(elm != NULL); assert(elm != NULL);
return rtree_leaf_elm_read(tsdn, rtree, elm, return rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ true).metadata; /* dependent */ true)
.metadata;
} }
/* /*
* Returns true when the request cannot be fulfilled by fastpath. * Returns true when the request cannot be fulfilled by fastpath.
*/ */
static inline bool static inline bool
rtree_metadata_try_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_metadata_try_read_fast(tsdn_t *tsdn, rtree_t *rtree,
uintptr_t key, rtree_metadata_t *r_rtree_metadata) { rtree_ctx_t *rtree_ctx, uintptr_t key, rtree_metadata_t *r_rtree_metadata) {
rtree_leaf_elm_t *elm; rtree_leaf_elm_t *elm;
/* /*
* Should check the bool return value (lookup success or not) instead of * Should check the bool return value (lookup success or not) instead of
@@ -476,7 +481,8 @@ rtree_metadata_try_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ct
} }
assert(elm != NULL); assert(elm != NULL);
*r_rtree_metadata = rtree_leaf_elm_read(tsdn, rtree, elm, *r_rtree_metadata = rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ true).metadata; /* dependent */ true)
.metadata;
return false; return false;
} }
@@ -496,16 +502,21 @@ rtree_write_range_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
rtree_leaf_elm_t *elm = NULL; /* Dead store. */ rtree_leaf_elm_t *elm = NULL; /* Dead store. */
for (uintptr_t addr = base; addr <= end; addr += PAGE) { for (uintptr_t addr = base; addr <= end; addr += PAGE) {
if (addr == base || if (addr == base
(addr & ((ZU(1) << rtree_leaf_maskbits()) - 1)) == 0) { || (addr & ((ZU(1) << rtree_leaf_maskbits()) - 1)) == 0) {
elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, addr, elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
addr,
/* dependent */ true, /* init_missing */ false); /* dependent */ true, /* init_missing */ false);
assert(elm != NULL); assert(elm != NULL);
} }
assert(elm == rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, addr, assert(elm
== rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, addr,
/* dependent */ true, /* init_missing */ false)); /* dependent */ true, /* init_missing */ false));
assert(!clearing || rtree_leaf_elm_read(tsdn, rtree, elm, assert(!clearing
/* dependent */ true).edata != NULL); || rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ true)
.edata
!= NULL);
rtree_leaf_elm_write_commit(tsdn, rtree, elm, bits, additional); rtree_leaf_elm_write_commit(tsdn, rtree, elm, bits, additional);
elm++; elm++;
} }
@@ -533,13 +544,15 @@ rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
} }
static inline void static inline void
rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_clear(
uintptr_t key) { tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
key, /* dependent */ true, /* init_missing */ false); key, /* dependent */ true, /* init_missing */ false);
assert(elm != NULL); assert(elm != NULL);
assert(rtree_leaf_elm_read(tsdn, rtree, elm, assert(rtree_leaf_elm_read(tsdn, rtree, elm,
/* dependent */ true).edata != NULL); /* dependent */ true)
.edata
!= NULL);
rtree_contents_t contents; rtree_contents_t contents;
contents.edata = NULL; contents.edata = NULL;
contents.metadata.szind = SC_NSIZES; contents.metadata.szind = SC_NSIZES;

11
include/jemalloc/internal/rtree_tsd.h
View File

@@ -25,7 +25,8 @@
/* Needed for initialization only. */ /* Needed for initialization only. */
#define RTREE_LEAFKEY_INVALID ((uintptr_t)1) #define RTREE_LEAFKEY_INVALID ((uintptr_t)1)
#define RTREE_CTX_CACHE_ELM_INVALID {RTREE_LEAFKEY_INVALID, NULL} #define RTREE_CTX_CACHE_ELM_INVALID \
{ RTREE_LEAFKEY_INVALID, NULL }
#define RTREE_CTX_INIT_ELM_1 RTREE_CTX_CACHE_ELM_INVALID #define RTREE_CTX_INIT_ELM_1 RTREE_CTX_CACHE_ELM_INVALID
#define RTREE_CTX_INIT_ELM_2 RTREE_CTX_INIT_ELM_1, RTREE_CTX_INIT_ELM_1 #define RTREE_CTX_INIT_ELM_2 RTREE_CTX_INIT_ELM_1, RTREE_CTX_INIT_ELM_1
@@ -40,8 +41,12 @@
* Static initializer (to invalidate the cache entries) is required because the * Static initializer (to invalidate the cache entries) is required because the
* free fastpath may access the rtree cache before a full tsd initialization. * free fastpath may access the rtree cache before a full tsd initialization.
*/ */
#define RTREE_CTX_INITIALIZER {{RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE)}, \ #define RTREE_CTX_INITIALIZER \
{RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE_L2)}} { \
{RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE)}, { \
RTREE_CTX_INIT_ELM_DATA(RTREE_CTX_NCACHE_L2) \
} \
}
typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; typedef struct rtree_leaf_elm_s rtree_leaf_elm_t;

27
include/jemalloc/internal/safety_check.h
View File

@@ -7,8 +7,8 @@
#define SAFETY_CHECK_DOUBLE_FREE_MAX_SCAN_DEFAULT 32 #define SAFETY_CHECK_DOUBLE_FREE_MAX_SCAN_DEFAULT 32
void safety_check_fail_sized_dealloc(bool current_dealloc, const void *ptr, void safety_check_fail_sized_dealloc(
size_t true_size, size_t input_size); bool current_dealloc, const void *ptr, size_t true_size, size_t input_size);
void safety_check_fail(const char *format, ...); void safety_check_fail(const char *format, ...);
typedef void (*safety_check_abort_hook_t)(const char *message); typedef void (*safety_check_abort_hook_t)(const char *message);
@@ -16,7 +16,7 @@ typedef void (*safety_check_abort_hook_t)(const char *message);
/* Can set to NULL for a default. */ /* Can set to NULL for a default. */
void safety_check_set_abort(safety_check_abort_hook_t abort_fn); void safety_check_set_abort(safety_check_abort_hook_t abort_fn);
#define REDZONE_SIZE ((size_t) 32) #define REDZONE_SIZE ((size_t)32)
#define REDZONE_FILL_VALUE 0xBC #define REDZONE_FILL_VALUE 0xBC
/* /*
@@ -27,9 +27,10 @@ void safety_check_set_abort(safety_check_abort_hook_t abort_fn);
*/ */
JEMALLOC_ALWAYS_INLINE const unsigned char * JEMALLOC_ALWAYS_INLINE const unsigned char *
compute_redzone_end(const void *_ptr, size_t usize, size_t bumped_usize) { compute_redzone_end(const void *_ptr, size_t usize, size_t bumped_usize) {
const unsigned char *ptr = (const unsigned char *) _ptr; const unsigned char *ptr = (const unsigned char *)_ptr;
const unsigned char *redzone_end = usize + REDZONE_SIZE < bumped_usize ? const unsigned char *redzone_end = usize + REDZONE_SIZE < bumped_usize
&ptr[usize + REDZONE_SIZE] : &ptr[bumped_usize]; ? &ptr[usize + REDZONE_SIZE]
: &ptr[bumped_usize];
const unsigned char *page_end = (const unsigned char *) const unsigned char *page_end = (const unsigned char *)
ALIGNMENT_ADDR2CEILING(&ptr[usize], os_page); ALIGNMENT_ADDR2CEILING(&ptr[usize], os_page);
return redzone_end < page_end ? redzone_end : page_end; return redzone_end < page_end ? redzone_end : page_end;
@@ -38,8 +39,8 @@ compute_redzone_end(const void *_ptr, size_t usize, size_t bumped_usize) {
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
safety_check_set_redzone(void *ptr, size_t usize, size_t bumped_usize) { safety_check_set_redzone(void *ptr, size_t usize, size_t bumped_usize) {
assert(usize <= bumped_usize); assert(usize <= bumped_usize);
const unsigned char *redzone_end = const unsigned char *redzone_end = compute_redzone_end(
compute_redzone_end(ptr, usize, bumped_usize); ptr, usize, bumped_usize);
for (unsigned char *curr = &((unsigned char *)ptr)[usize]; for (unsigned char *curr = &((unsigned char *)ptr)[usize];
curr < redzone_end; curr++) { curr < redzone_end; curr++) {
*curr = REDZONE_FILL_VALUE; *curr = REDZONE_FILL_VALUE;
@@ -47,11 +48,11 @@ safety_check_set_redzone(void *ptr, size_t usize, size_t bumped_usize) {
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
safety_check_verify_redzone(const void *ptr, size_t usize, size_t bumped_usize) safety_check_verify_redzone(
{ const void *ptr, size_t usize, size_t bumped_usize) {
const unsigned char *redzone_end = const unsigned char *redzone_end = compute_redzone_end(
compute_redzone_end(ptr, usize, bumped_usize); ptr, usize, bumped_usize);
for (const unsigned char *curr= &((const unsigned char *)ptr)[usize]; for (const unsigned char *curr = &((const unsigned char *)ptr)[usize];
curr < redzone_end; curr++) { curr < redzone_end; curr++) {
if (unlikely(*curr != REDZONE_FILL_VALUE)) { if (unlikely(*curr != REDZONE_FILL_VALUE)) {
safety_check_fail("Use after free error\n"); safety_check_fail("Use after free error\n");

44
include/jemalloc/internal/san.h
View File

@@ -32,22 +32,22 @@ void san_unguard_pages(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
* Unguard the extent, but don't modify emap boundaries. Must be called on an * Unguard the extent, but don't modify emap boundaries. Must be called on an
* extent that has been erased from emap and shouldn't be placed back. * extent that has been erased from emap and shouldn't be placed back.
*/ */
void san_unguard_pages_pre_destroy(tsdn_t *tsdn, ehooks_t *ehooks, void san_unguard_pages_pre_destroy(
edata_t *edata, emap_t *emap); tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap);
void san_check_stashed_ptrs(void **ptrs, size_t nstashed, size_t usize); void san_check_stashed_ptrs(void **ptrs, size_t nstashed, size_t usize);
void tsd_san_init(tsd_t *tsd); void tsd_san_init(tsd_t *tsd);
void san_init(ssize_t lg_san_uaf_align); void san_init(ssize_t lg_san_uaf_align);
static inline void static inline void
san_guard_pages_two_sided(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, san_guard_pages_two_sided(
emap_t *emap, bool remap) { tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap, bool remap) {
san_guard_pages(tsdn, ehooks, edata, emap, true, true, remap); san_guard_pages(tsdn, ehooks, edata, emap, true, true, remap);
} }
static inline void static inline void
san_unguard_pages_two_sided(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, san_unguard_pages_two_sided(
emap_t *emap) { tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata, emap_t *emap) {
san_unguard_pages(tsdn, ehooks, edata, emap, true, true); san_unguard_pages(tsdn, ehooks, edata, emap, true, true);
} }
@@ -83,10 +83,10 @@ san_guard_enabled(void) {
} }
static inline bool static inline bool
san_large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size, san_large_extent_decide_guard(
size_t alignment) { tsdn_t *tsdn, ehooks_t *ehooks, size_t size, size_t alignment) {
if (opt_san_guard_large == 0 || ehooks_guard_will_fail(ehooks) || if (opt_san_guard_large == 0 || ehooks_guard_will_fail(ehooks)
tsdn_null(tsdn)) { || tsdn_null(tsdn)) {
return false; return false;
} }
@@ -101,10 +101,10 @@ san_large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size,
*tsd_san_extents_until_guard_largep_get(tsd) = n - 1; *tsd_san_extents_until_guard_largep_get(tsd) = n - 1;
} }
if (n == 1 && (alignment <= PAGE) && if (n == 1 && (alignment <= PAGE)
(san_two_side_guarded_sz(size) <= SC_LARGE_MAXCLASS)) { && (san_two_side_guarded_sz(size) <= SC_LARGE_MAXCLASS)) {
*tsd_san_extents_until_guard_largep_get(tsd) = *tsd_san_extents_until_guard_largep_get(
opt_san_guard_large; tsd) = opt_san_guard_large;
return true; return true;
} else { } else {
assert(tsd_san_extents_until_guard_large_get(tsd) >= 1); assert(tsd_san_extents_until_guard_large_get(tsd) >= 1);
@@ -114,8 +114,8 @@ san_large_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks, size_t size,
static inline bool static inline bool
san_slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) { san_slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) {
if (opt_san_guard_small == 0 || ehooks_guard_will_fail(ehooks) || if (opt_san_guard_small == 0 || ehooks_guard_will_fail(ehooks)
tsdn_null(tsdn)) { || tsdn_null(tsdn)) {
return false; return false;
} }
@@ -123,8 +123,8 @@ san_slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) {
uint64_t n = tsd_san_extents_until_guard_small_get(tsd); uint64_t n = tsd_san_extents_until_guard_small_get(tsd);
assert(n >= 1); assert(n >= 1);
if (n == 1) { if (n == 1) {
*tsd_san_extents_until_guard_smallp_get(tsd) = *tsd_san_extents_until_guard_smallp_get(
opt_san_guard_small; tsd) = opt_san_guard_small;
return true; return true;
} else { } else {
*tsd_san_extents_until_guard_smallp_get(tsd) = n - 1; *tsd_san_extents_until_guard_smallp_get(tsd) = n - 1;
@@ -134,8 +134,8 @@ san_slab_extent_decide_guard(tsdn_t *tsdn, ehooks_t *ehooks) {
} }
static inline void static inline void
san_junk_ptr_locations(void *ptr, size_t usize, void **first, void **mid, san_junk_ptr_locations(
void **last) { void *ptr, size_t usize, void **first, void **mid, void **last) {
size_t ptr_sz = sizeof(void *); size_t ptr_sz = sizeof(void *);
*first = ptr; *first = ptr;
@@ -184,8 +184,8 @@ static inline bool
san_uaf_detection_enabled(void) { san_uaf_detection_enabled(void) {
bool ret = config_uaf_detection && (opt_lg_san_uaf_align != -1); bool ret = config_uaf_detection && (opt_lg_san_uaf_align != -1);
if (config_uaf_detection && ret) { if (config_uaf_detection && ret) {
assert(san_cache_bin_nonfast_mask == ((uintptr_t)1 << assert(san_cache_bin_nonfast_mask
opt_lg_san_uaf_align) - 1); == ((uintptr_t)1 << opt_lg_san_uaf_align) - 1);
} }
return ret; return ret;

7
include/jemalloc/internal/san_bump.h
View File

@@ -36,7 +36,7 @@ san_bump_enabled(void) {
} }
static inline bool static inline bool
san_bump_alloc_init(san_bump_alloc_t* sba) { san_bump_alloc_init(san_bump_alloc_t *sba) {
bool err = malloc_mutex_init(&sba->mtx, "sanitizer_bump_allocator", bool err = malloc_mutex_init(&sba->mtx, "sanitizer_bump_allocator",
WITNESS_RANK_SAN_BUMP_ALLOC, malloc_mutex_rank_exclusive); WITNESS_RANK_SAN_BUMP_ALLOC, malloc_mutex_rank_exclusive);
if (err) { if (err) {
@@ -47,8 +47,7 @@ san_bump_alloc_init(san_bump_alloc_t* sba) {
return false; return false;
} }
edata_t * edata_t *san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t *sba, pac_t *pac,
san_bump_alloc(tsdn_t *tsdn, san_bump_alloc_t* sba, pac_t *pac, ehooks_t *ehooks, ehooks_t *ehooks, size_t size, bool zero);
size_t size, bool zero);
#endif /* JEMALLOC_INTERNAL_SAN_BUMP_H */ #endif /* JEMALLOC_INTERNAL_SAN_BUMP_H */

32
include/jemalloc/internal/sc.h
View File

@@ -174,7 +174,7 @@
#if SC_LG_TINY_MIN == 0 #if SC_LG_TINY_MIN == 0
/* The div module doesn't support division by 1, which this would require. */ /* The div module doesn't support division by 1, which this would require. */
#error "Unsupported LG_TINY_MIN" # error "Unsupported LG_TINY_MIN"
#endif #endif
/* /*
@@ -194,8 +194,8 @@
* We could probably save some space in arenas by capping this at LG_VADDR size. * We could probably save some space in arenas by capping this at LG_VADDR size.
*/ */
#define SC_LG_BASE_MAX (SC_PTR_BITS - 2) #define SC_LG_BASE_MAX (SC_PTR_BITS - 2)
#define SC_NREGULAR (SC_NGROUP * \ #define SC_NREGULAR \
(SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1) (SC_NGROUP * (SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1)
#define SC_NSIZES (SC_NTINY + SC_NPSEUDO + SC_NREGULAR) #define SC_NSIZES (SC_NTINY + SC_NPSEUDO + SC_NREGULAR)
/* /*
@@ -222,21 +222,21 @@
* *
* This gives us the quantity we seek. * This gives us the quantity we seek.
*/ */
#define SC_NPSIZES ( \ #define SC_NPSIZES \
SC_NGROUP \ (SC_NGROUP + (SC_LG_BASE_MAX - (LG_PAGE + SC_LG_NGROUP)) * SC_NGROUP \
+ (SC_LG_BASE_MAX - (LG_PAGE + SC_LG_NGROUP)) * SC_NGROUP \
+ SC_NGROUP - 1) + SC_NGROUP - 1)
/* /*
* We declare a size class is binnable if size < page size * group. Or, in other * We declare a size class is binnable if size < page size * group. Or, in other
* words, lg(size) < lg(page size) + lg(group size). * words, lg(size) < lg(page size) + lg(group size).
*/ */
#define SC_NBINS ( \ #define SC_NBINS \
/* Sub-regular size classes. */ \ (/* Sub-regular size classes. */ \
SC_NTINY + SC_NPSEUDO \ SC_NTINY \
/* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \ + SC_NPSEUDO /* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \
+ SC_NGROUP * (LG_PAGE + SC_LG_NGROUP - SC_LG_FIRST_REGULAR_BASE) \ + SC_NGROUP \
/* Last SC of the last group hits the bound exactly; exclude it. */ \ * (LG_PAGE + SC_LG_NGROUP \
- SC_LG_FIRST_REGULAR_BASE) /* Last SC of the last group hits the bound exactly; exclude it. */ \
- 1) - 1)
/* /*
@@ -256,8 +256,8 @@
#define SC_SMALL_MAX_DELTA (1 << (LG_PAGE - 1)) #define SC_SMALL_MAX_DELTA (1 << (LG_PAGE - 1))
/* The largest size class allocated out of a slab. */ /* The largest size class allocated out of a slab. */
#define SC_SMALL_MAXCLASS (SC_SMALL_MAX_BASE \ #define SC_SMALL_MAXCLASS \
+ (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) (SC_SMALL_MAX_BASE + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA)
/* The fastpath assumes all lookup-able sizes are small. */ /* The fastpath assumes all lookup-able sizes are small. */
#if (SC_SMALL_MAXCLASS < SC_LOOKUP_MAXCLASS) #if (SC_SMALL_MAXCLASS < SC_LOOKUP_MAXCLASS)
@@ -369,8 +369,8 @@ void sc_data_init(sc_data_t *data);
* Updates slab sizes in [begin, end] to be pgs pages in length, if possible. * Updates slab sizes in [begin, end] to be pgs pages in length, if possible.
* Otherwise, does its best to accommodate the request. * Otherwise, does its best to accommodate the request.
*/ */
void sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, void sc_data_update_slab_size(
int pgs); sc_data_t *data, size_t begin, size_t end, int pgs);
void sc_boot(sc_data_t *data); void sc_boot(sc_data_t *data);
#endif /* JEMALLOC_INTERNAL_SC_H */ #endif /* JEMALLOC_INTERNAL_SC_H */

4
include/jemalloc/internal/sec.h
View File

@@ -110,8 +110,8 @@ void sec_disable(tsdn_t *tsdn, sec_t *sec);
* split), which simplifies the stats management. * split), which simplifies the stats management.
*/ */
void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats); void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats);
void sec_mutex_stats_read(tsdn_t *tsdn, sec_t *sec, void sec_mutex_stats_read(
mutex_prof_data_t *mutex_prof_data); tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data);
/* /*
* We use the arena lock ordering; these are acquired in phase 2 of forking, but * We use the arena lock ordering; these are acquired in phase 2 of forking, but

18
include/jemalloc/internal/sec_opts.h
View File

@@ -44,18 +44,14 @@ struct sec_opts_s {
size_t batch_fill_extra; size_t batch_fill_extra;
}; };
#define SEC_OPTS_DEFAULT { \ #define SEC_OPTS_DEFAULT \
{ \
/* nshards */ \ /* nshards */ \
4, \ 4, /* max_alloc */ \
/* max_alloc */ \ (32 * 1024) < PAGE ? PAGE : (32 * 1024), /* max_bytes */ \
(32 * 1024) < PAGE ? PAGE : (32 * 1024), \ 256 * 1024, /* bytes_after_flush */ \
/* max_bytes */ \ 128 * 1024, /* batch_fill_extra */ \
256 * 1024, \
/* bytes_after_flush */ \
128 * 1024, \
/* batch_fill_extra */ \
0 \ 0 \
} }
#endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */ #endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

400
include/jemalloc/internal/smoothstep.h
View File

@@ -28,205 +28,205 @@
#define SMOOTHSTEP_BFP 24 #define SMOOTHSTEP_BFP 24
#define SMOOTHSTEP \ #define SMOOTHSTEP \
/* STEP(step, h, x, y) */ \ /* STEP(step, h, x, y) */ \
STEP( 1, UINT64_C(0x0000000000000014), 0.005, 0.000001240643750) \ STEP(1, UINT64_C(0x0000000000000014), 0.005, 0.000001240643750) \
STEP( 2, UINT64_C(0x00000000000000a5), 0.010, 0.000009850600000) \ STEP(2, UINT64_C(0x00000000000000a5), 0.010, 0.000009850600000) \
STEP( 3, UINT64_C(0x0000000000000229), 0.015, 0.000032995181250) \ STEP(3, UINT64_C(0x0000000000000229), 0.015, 0.000032995181250) \
STEP( 4, UINT64_C(0x0000000000000516), 0.020, 0.000077619200000) \ STEP(4, UINT64_C(0x0000000000000516), 0.020, 0.000077619200000) \
STEP( 5, UINT64_C(0x00000000000009dc), 0.025, 0.000150449218750) \ STEP(5, UINT64_C(0x00000000000009dc), 0.025, 0.000150449218750) \
STEP( 6, UINT64_C(0x00000000000010e8), 0.030, 0.000257995800000) \ STEP(6, UINT64_C(0x00000000000010e8), 0.030, 0.000257995800000) \
STEP( 7, UINT64_C(0x0000000000001aa4), 0.035, 0.000406555756250) \ STEP(7, UINT64_C(0x0000000000001aa4), 0.035, 0.000406555756250) \
STEP( 8, UINT64_C(0x0000000000002777), 0.040, 0.000602214400000) \ STEP(8, UINT64_C(0x0000000000002777), 0.040, 0.000602214400000) \
STEP( 9, UINT64_C(0x00000000000037c2), 0.045, 0.000850847793750) \ STEP(9, UINT64_C(0x00000000000037c2), 0.045, 0.000850847793750) \
STEP( 10, UINT64_C(0x0000000000004be6), 0.050, 0.001158125000000) \ STEP(10, UINT64_C(0x0000000000004be6), 0.050, 0.001158125000000) \
STEP( 11, UINT64_C(0x000000000000643c), 0.055, 0.001529510331250) \ STEP(11, UINT64_C(0x000000000000643c), 0.055, 0.001529510331250) \
STEP( 12, UINT64_C(0x000000000000811f), 0.060, 0.001970265600000) \ STEP(12, UINT64_C(0x000000000000811f), 0.060, 0.001970265600000) \
STEP( 13, UINT64_C(0x000000000000a2e2), 0.065, 0.002485452368750) \ STEP(13, UINT64_C(0x000000000000a2e2), 0.065, 0.002485452368750) \
STEP( 14, UINT64_C(0x000000000000c9d8), 0.070, 0.003079934200000) \ STEP(14, UINT64_C(0x000000000000c9d8), 0.070, 0.003079934200000) \
STEP( 15, UINT64_C(0x000000000000f64f), 0.075, 0.003758378906250) \ STEP(15, UINT64_C(0x000000000000f64f), 0.075, 0.003758378906250) \
STEP( 16, UINT64_C(0x0000000000012891), 0.080, 0.004525260800000) \ STEP(16, UINT64_C(0x0000000000012891), 0.080, 0.004525260800000) \
STEP( 17, UINT64_C(0x00000000000160e7), 0.085, 0.005384862943750) \ STEP(17, UINT64_C(0x00000000000160e7), 0.085, 0.005384862943750) \
STEP( 18, UINT64_C(0x0000000000019f95), 0.090, 0.006341279400000) \ STEP(18, UINT64_C(0x0000000000019f95), 0.090, 0.006341279400000) \
STEP( 19, UINT64_C(0x000000000001e4dc), 0.095, 0.007398417481250) \ STEP(19, UINT64_C(0x000000000001e4dc), 0.095, 0.007398417481250) \
STEP( 20, UINT64_C(0x00000000000230fc), 0.100, 0.008560000000000) \ STEP(20, UINT64_C(0x00000000000230fc), 0.100, 0.008560000000000) \
STEP( 21, UINT64_C(0x0000000000028430), 0.105, 0.009829567518750) \ STEP(21, UINT64_C(0x0000000000028430), 0.105, 0.009829567518750) \
STEP( 22, UINT64_C(0x000000000002deb0), 0.110, 0.011210480600000) \ STEP(22, UINT64_C(0x000000000002deb0), 0.110, 0.011210480600000) \
STEP( 23, UINT64_C(0x00000000000340b1), 0.115, 0.012705922056250) \ STEP(23, UINT64_C(0x00000000000340b1), 0.115, 0.012705922056250) \
STEP( 24, UINT64_C(0x000000000003aa67), 0.120, 0.014318899200000) \ STEP(24, UINT64_C(0x000000000003aa67), 0.120, 0.014318899200000) \
STEP( 25, UINT64_C(0x0000000000041c00), 0.125, 0.016052246093750) \ STEP(25, UINT64_C(0x0000000000041c00), 0.125, 0.016052246093750) \
STEP( 26, UINT64_C(0x00000000000495a8), 0.130, 0.017908625800000) \ STEP(26, UINT64_C(0x00000000000495a8), 0.130, 0.017908625800000) \
STEP( 27, UINT64_C(0x000000000005178b), 0.135, 0.019890532631250) \ STEP(27, UINT64_C(0x000000000005178b), 0.135, 0.019890532631250) \
STEP( 28, UINT64_C(0x000000000005a1cf), 0.140, 0.022000294400000) \ STEP(28, UINT64_C(0x000000000005a1cf), 0.140, 0.022000294400000) \
STEP( 29, UINT64_C(0x0000000000063498), 0.145, 0.024240074668750) \ STEP(29, UINT64_C(0x0000000000063498), 0.145, 0.024240074668750) \
STEP( 30, UINT64_C(0x000000000006d009), 0.150, 0.026611875000000) \ STEP(30, UINT64_C(0x000000000006d009), 0.150, 0.026611875000000) \
STEP( 31, UINT64_C(0x000000000007743f), 0.155, 0.029117537206250) \ STEP(31, UINT64_C(0x000000000007743f), 0.155, 0.029117537206250) \
STEP( 32, UINT64_C(0x0000000000082157), 0.160, 0.031758745600000) \ STEP(32, UINT64_C(0x0000000000082157), 0.160, 0.031758745600000) \
STEP( 33, UINT64_C(0x000000000008d76b), 0.165, 0.034537029243750) \ STEP(33, UINT64_C(0x000000000008d76b), 0.165, 0.034537029243750) \
STEP( 34, UINT64_C(0x0000000000099691), 0.170, 0.037453764200000) \ STEP(34, UINT64_C(0x0000000000099691), 0.170, 0.037453764200000) \
STEP( 35, UINT64_C(0x00000000000a5edf), 0.175, 0.040510175781250) \ STEP(35, UINT64_C(0x00000000000a5edf), 0.175, 0.040510175781250) \
STEP( 36, UINT64_C(0x00000000000b3067), 0.180, 0.043707340800000) \ STEP(36, UINT64_C(0x00000000000b3067), 0.180, 0.043707340800000) \
STEP( 37, UINT64_C(0x00000000000c0b38), 0.185, 0.047046189818750) \ STEP(37, UINT64_C(0x00000000000c0b38), 0.185, 0.047046189818750) \
STEP( 38, UINT64_C(0x00000000000cef5e), 0.190, 0.050527509400000) \ STEP(38, UINT64_C(0x00000000000cef5e), 0.190, 0.050527509400000) \
STEP( 39, UINT64_C(0x00000000000ddce6), 0.195, 0.054151944356250) \ STEP(39, UINT64_C(0x00000000000ddce6), 0.195, 0.054151944356250) \
STEP( 40, UINT64_C(0x00000000000ed3d8), 0.200, 0.057920000000000) \ STEP(40, UINT64_C(0x00000000000ed3d8), 0.200, 0.057920000000000) \
STEP( 41, UINT64_C(0x00000000000fd439), 0.205, 0.061832044393750) \ STEP(41, UINT64_C(0x00000000000fd439), 0.205, 0.061832044393750) \
STEP( 42, UINT64_C(0x000000000010de0e), 0.210, 0.065888310600000) \ STEP(42, UINT64_C(0x000000000010de0e), 0.210, 0.065888310600000) \
STEP( 43, UINT64_C(0x000000000011f158), 0.215, 0.070088898931250) \ STEP(43, UINT64_C(0x000000000011f158), 0.215, 0.070088898931250) \
STEP( 44, UINT64_C(0x0000000000130e17), 0.220, 0.074433779200000) \ STEP(44, UINT64_C(0x0000000000130e17), 0.220, 0.074433779200000) \
STEP( 45, UINT64_C(0x0000000000143448), 0.225, 0.078922792968750) \ STEP(45, UINT64_C(0x0000000000143448), 0.225, 0.078922792968750) \
STEP( 46, UINT64_C(0x00000000001563e7), 0.230, 0.083555655800000) \ STEP(46, UINT64_C(0x00000000001563e7), 0.230, 0.083555655800000) \
STEP( 47, UINT64_C(0x0000000000169cec), 0.235, 0.088331959506250) \ STEP(47, UINT64_C(0x0000000000169cec), 0.235, 0.088331959506250) \
STEP( 48, UINT64_C(0x000000000017df4f), 0.240, 0.093251174400000) \ STEP(48, UINT64_C(0x000000000017df4f), 0.240, 0.093251174400000) \
STEP( 49, UINT64_C(0x0000000000192b04), 0.245, 0.098312651543750) \ STEP(49, UINT64_C(0x0000000000192b04), 0.245, 0.098312651543750) \
STEP( 50, UINT64_C(0x00000000001a8000), 0.250, 0.103515625000000) \ STEP(50, UINT64_C(0x00000000001a8000), 0.250, 0.103515625000000) \
STEP( 51, UINT64_C(0x00000000001bde32), 0.255, 0.108859214081250) \ STEP(51, UINT64_C(0x00000000001bde32), 0.255, 0.108859214081250) \
STEP( 52, UINT64_C(0x00000000001d458b), 0.260, 0.114342425600000) \ STEP(52, UINT64_C(0x00000000001d458b), 0.260, 0.114342425600000) \
STEP( 53, UINT64_C(0x00000000001eb5f8), 0.265, 0.119964156118750) \ STEP(53, UINT64_C(0x00000000001eb5f8), 0.265, 0.119964156118750) \
STEP( 54, UINT64_C(0x0000000000202f65), 0.270, 0.125723194200000) \ STEP(54, UINT64_C(0x0000000000202f65), 0.270, 0.125723194200000) \
STEP( 55, UINT64_C(0x000000000021b1bb), 0.275, 0.131618222656250) \ STEP(55, UINT64_C(0x000000000021b1bb), 0.275, 0.131618222656250) \
STEP( 56, UINT64_C(0x0000000000233ce3), 0.280, 0.137647820800000) \ STEP(56, UINT64_C(0x0000000000233ce3), 0.280, 0.137647820800000) \
STEP( 57, UINT64_C(0x000000000024d0c3), 0.285, 0.143810466693750) \ STEP(57, UINT64_C(0x000000000024d0c3), 0.285, 0.143810466693750) \
STEP( 58, UINT64_C(0x0000000000266d40), 0.290, 0.150104539400000) \ STEP(58, UINT64_C(0x0000000000266d40), 0.290, 0.150104539400000) \
STEP( 59, UINT64_C(0x000000000028123d), 0.295, 0.156528321231250) \ STEP(59, UINT64_C(0x000000000028123d), 0.295, 0.156528321231250) \
STEP( 60, UINT64_C(0x000000000029bf9c), 0.300, 0.163080000000000) \ STEP(60, UINT64_C(0x000000000029bf9c), 0.300, 0.163080000000000) \
STEP( 61, UINT64_C(0x00000000002b753d), 0.305, 0.169757671268750) \ STEP(61, UINT64_C(0x00000000002b753d), 0.305, 0.169757671268750) \
STEP( 62, UINT64_C(0x00000000002d32fe), 0.310, 0.176559340600000) \ STEP(62, UINT64_C(0x00000000002d32fe), 0.310, 0.176559340600000) \
STEP( 63, UINT64_C(0x00000000002ef8bc), 0.315, 0.183482925806250) \ STEP(63, UINT64_C(0x00000000002ef8bc), 0.315, 0.183482925806250) \
STEP( 64, UINT64_C(0x000000000030c654), 0.320, 0.190526259200000) \ STEP(64, UINT64_C(0x000000000030c654), 0.320, 0.190526259200000) \
STEP( 65, UINT64_C(0x0000000000329b9f), 0.325, 0.197687089843750) \ STEP(65, UINT64_C(0x0000000000329b9f), 0.325, 0.197687089843750) \
STEP( 66, UINT64_C(0x0000000000347875), 0.330, 0.204963085800000) \ STEP(66, UINT64_C(0x0000000000347875), 0.330, 0.204963085800000) \
STEP( 67, UINT64_C(0x0000000000365cb0), 0.335, 0.212351836381250) \ STEP(67, UINT64_C(0x0000000000365cb0), 0.335, 0.212351836381250) \
STEP( 68, UINT64_C(0x0000000000384825), 0.340, 0.219850854400000) \ STEP(68, UINT64_C(0x0000000000384825), 0.340, 0.219850854400000) \
STEP( 69, UINT64_C(0x00000000003a3aa8), 0.345, 0.227457578418750) \ STEP(69, UINT64_C(0x00000000003a3aa8), 0.345, 0.227457578418750) \
STEP( 70, UINT64_C(0x00000000003c340f), 0.350, 0.235169375000000) \ STEP(70, UINT64_C(0x00000000003c340f), 0.350, 0.235169375000000) \
STEP( 71, UINT64_C(0x00000000003e342b), 0.355, 0.242983540956250) \ STEP(71, UINT64_C(0x00000000003e342b), 0.355, 0.242983540956250) \
STEP( 72, UINT64_C(0x0000000000403ace), 0.360, 0.250897305600000) \ STEP(72, UINT64_C(0x0000000000403ace), 0.360, 0.250897305600000) \
STEP( 73, UINT64_C(0x00000000004247c8), 0.365, 0.258907832993750) \ STEP(73, UINT64_C(0x00000000004247c8), 0.365, 0.258907832993750) \
STEP( 74, UINT64_C(0x0000000000445ae9), 0.370, 0.267012224200000) \ STEP(74, UINT64_C(0x0000000000445ae9), 0.370, 0.267012224200000) \
STEP( 75, UINT64_C(0x0000000000467400), 0.375, 0.275207519531250) \ STEP(75, UINT64_C(0x0000000000467400), 0.375, 0.275207519531250) \
STEP( 76, UINT64_C(0x00000000004892d8), 0.380, 0.283490700800000) \ STEP(76, UINT64_C(0x00000000004892d8), 0.380, 0.283490700800000) \
STEP( 77, UINT64_C(0x00000000004ab740), 0.385, 0.291858693568750) \ STEP(77, UINT64_C(0x00000000004ab740), 0.385, 0.291858693568750) \
STEP( 78, UINT64_C(0x00000000004ce102), 0.390, 0.300308369400000) \ STEP(78, UINT64_C(0x00000000004ce102), 0.390, 0.300308369400000) \
STEP( 79, UINT64_C(0x00000000004f0fe9), 0.395, 0.308836548106250) \ STEP(79, UINT64_C(0x00000000004f0fe9), 0.395, 0.308836548106250) \
STEP( 80, UINT64_C(0x00000000005143bf), 0.400, 0.317440000000000) \ STEP(80, UINT64_C(0x00000000005143bf), 0.400, 0.317440000000000) \
STEP( 81, UINT64_C(0x0000000000537c4d), 0.405, 0.326115448143750) \ STEP(81, UINT64_C(0x0000000000537c4d), 0.405, 0.326115448143750) \
STEP( 82, UINT64_C(0x000000000055b95b), 0.410, 0.334859570600000) \ STEP(82, UINT64_C(0x000000000055b95b), 0.410, 0.334859570600000) \
STEP( 83, UINT64_C(0x000000000057fab1), 0.415, 0.343669002681250) \ STEP(83, UINT64_C(0x000000000057fab1), 0.415, 0.343669002681250) \
STEP( 84, UINT64_C(0x00000000005a4015), 0.420, 0.352540339200000) \ STEP(84, UINT64_C(0x00000000005a4015), 0.420, 0.352540339200000) \
STEP( 85, UINT64_C(0x00000000005c894e), 0.425, 0.361470136718750) \ STEP(85, UINT64_C(0x00000000005c894e), 0.425, 0.361470136718750) \
STEP( 86, UINT64_C(0x00000000005ed622), 0.430, 0.370454915800000) \ STEP(86, UINT64_C(0x00000000005ed622), 0.430, 0.370454915800000) \
STEP( 87, UINT64_C(0x0000000000612655), 0.435, 0.379491163256250) \ STEP(87, UINT64_C(0x0000000000612655), 0.435, 0.379491163256250) \
STEP( 88, UINT64_C(0x00000000006379ac), 0.440, 0.388575334400000) \ STEP(88, UINT64_C(0x00000000006379ac), 0.440, 0.388575334400000) \
STEP( 89, UINT64_C(0x000000000065cfeb), 0.445, 0.397703855293750) \ STEP(89, UINT64_C(0x000000000065cfeb), 0.445, 0.397703855293750) \
STEP( 90, UINT64_C(0x00000000006828d6), 0.450, 0.406873125000000) \ STEP(90, UINT64_C(0x00000000006828d6), 0.450, 0.406873125000000) \
STEP( 91, UINT64_C(0x00000000006a842f), 0.455, 0.416079517831250) \ STEP(91, UINT64_C(0x00000000006a842f), 0.455, 0.416079517831250) \
STEP( 92, UINT64_C(0x00000000006ce1bb), 0.460, 0.425319385600000) \ STEP(92, UINT64_C(0x00000000006ce1bb), 0.460, 0.425319385600000) \
STEP( 93, UINT64_C(0x00000000006f413a), 0.465, 0.434589059868750) \ STEP(93, UINT64_C(0x00000000006f413a), 0.465, 0.434589059868750) \
STEP( 94, UINT64_C(0x000000000071a270), 0.470, 0.443884854200000) \ STEP(94, UINT64_C(0x000000000071a270), 0.470, 0.443884854200000) \
STEP( 95, UINT64_C(0x000000000074051d), 0.475, 0.453203066406250) \ STEP(95, UINT64_C(0x000000000074051d), 0.475, 0.453203066406250) \
STEP( 96, UINT64_C(0x0000000000766905), 0.480, 0.462539980800000) \ STEP(96, UINT64_C(0x0000000000766905), 0.480, 0.462539980800000) \
STEP( 97, UINT64_C(0x000000000078cde7), 0.485, 0.471891870443750) \ STEP(97, UINT64_C(0x000000000078cde7), 0.485, 0.471891870443750) \
STEP( 98, UINT64_C(0x00000000007b3387), 0.490, 0.481254999400000) \ STEP(98, UINT64_C(0x00000000007b3387), 0.490, 0.481254999400000) \
STEP( 99, UINT64_C(0x00000000007d99a4), 0.495, 0.490625624981250) \ STEP(99, UINT64_C(0x00000000007d99a4), 0.495, 0.490625624981250) \
STEP( 100, UINT64_C(0x0000000000800000), 0.500, 0.500000000000000) \ STEP(100, UINT64_C(0x0000000000800000), 0.500, 0.500000000000000) \
STEP( 101, UINT64_C(0x000000000082665b), 0.505, 0.509374375018750) \ STEP(101, UINT64_C(0x000000000082665b), 0.505, 0.509374375018750) \
STEP( 102, UINT64_C(0x000000000084cc78), 0.510, 0.518745000600000) \ STEP(102, UINT64_C(0x000000000084cc78), 0.510, 0.518745000600000) \
STEP( 103, UINT64_C(0x0000000000873218), 0.515, 0.528108129556250) \ STEP(103, UINT64_C(0x0000000000873218), 0.515, 0.528108129556250) \
STEP( 104, UINT64_C(0x00000000008996fa), 0.520, 0.537460019200000) \ STEP(104, UINT64_C(0x00000000008996fa), 0.520, 0.537460019200000) \
STEP( 105, UINT64_C(0x00000000008bfae2), 0.525, 0.546796933593750) \ STEP(105, UINT64_C(0x00000000008bfae2), 0.525, 0.546796933593750) \
STEP( 106, UINT64_C(0x00000000008e5d8f), 0.530, 0.556115145800000) \ STEP(106, UINT64_C(0x00000000008e5d8f), 0.530, 0.556115145800000) \
STEP( 107, UINT64_C(0x000000000090bec5), 0.535, 0.565410940131250) \ STEP(107, UINT64_C(0x000000000090bec5), 0.535, 0.565410940131250) \
STEP( 108, UINT64_C(0x0000000000931e44), 0.540, 0.574680614400000) \ STEP(108, UINT64_C(0x0000000000931e44), 0.540, 0.574680614400000) \
STEP( 109, UINT64_C(0x0000000000957bd0), 0.545, 0.583920482168750) \ STEP(109, UINT64_C(0x0000000000957bd0), 0.545, 0.583920482168750) \
STEP( 110, UINT64_C(0x000000000097d729), 0.550, 0.593126875000000) \ STEP(110, UINT64_C(0x000000000097d729), 0.550, 0.593126875000000) \
STEP( 111, UINT64_C(0x00000000009a3014), 0.555, 0.602296144706250) \ STEP(111, UINT64_C(0x00000000009a3014), 0.555, 0.602296144706250) \
STEP( 112, UINT64_C(0x00000000009c8653), 0.560, 0.611424665600000) \ STEP(112, UINT64_C(0x00000000009c8653), 0.560, 0.611424665600000) \
STEP( 113, UINT64_C(0x00000000009ed9aa), 0.565, 0.620508836743750) \ STEP(113, UINT64_C(0x00000000009ed9aa), 0.565, 0.620508836743750) \
STEP( 114, UINT64_C(0x0000000000a129dd), 0.570, 0.629545084200000) \ STEP(114, UINT64_C(0x0000000000a129dd), 0.570, 0.629545084200000) \
STEP( 115, UINT64_C(0x0000000000a376b1), 0.575, 0.638529863281250) \ STEP(115, UINT64_C(0x0000000000a376b1), 0.575, 0.638529863281250) \
STEP( 116, UINT64_C(0x0000000000a5bfea), 0.580, 0.647459660800000) \ STEP(116, UINT64_C(0x0000000000a5bfea), 0.580, 0.647459660800000) \
STEP( 117, UINT64_C(0x0000000000a8054e), 0.585, 0.656330997318750) \ STEP(117, UINT64_C(0x0000000000a8054e), 0.585, 0.656330997318750) \
STEP( 118, UINT64_C(0x0000000000aa46a4), 0.590, 0.665140429400000) \ STEP(118, UINT64_C(0x0000000000aa46a4), 0.590, 0.665140429400000) \
STEP( 119, UINT64_C(0x0000000000ac83b2), 0.595, 0.673884551856250) \ STEP(119, UINT64_C(0x0000000000ac83b2), 0.595, 0.673884551856250) \
STEP( 120, UINT64_C(0x0000000000aebc40), 0.600, 0.682560000000000) \ STEP(120, UINT64_C(0x0000000000aebc40), 0.600, 0.682560000000000) \
STEP( 121, UINT64_C(0x0000000000b0f016), 0.605, 0.691163451893750) \ STEP(121, UINT64_C(0x0000000000b0f016), 0.605, 0.691163451893750) \
STEP( 122, UINT64_C(0x0000000000b31efd), 0.610, 0.699691630600000) \ STEP(122, UINT64_C(0x0000000000b31efd), 0.610, 0.699691630600000) \
STEP( 123, UINT64_C(0x0000000000b548bf), 0.615, 0.708141306431250) \ STEP(123, UINT64_C(0x0000000000b548bf), 0.615, 0.708141306431250) \
STEP( 124, UINT64_C(0x0000000000b76d27), 0.620, 0.716509299200000) \ STEP(124, UINT64_C(0x0000000000b76d27), 0.620, 0.716509299200000) \
STEP( 125, UINT64_C(0x0000000000b98c00), 0.625, 0.724792480468750) \ STEP(125, UINT64_C(0x0000000000b98c00), 0.625, 0.724792480468750) \
STEP( 126, UINT64_C(0x0000000000bba516), 0.630, 0.732987775800000) \ STEP(126, UINT64_C(0x0000000000bba516), 0.630, 0.732987775800000) \
STEP( 127, UINT64_C(0x0000000000bdb837), 0.635, 0.741092167006250) \ STEP(127, UINT64_C(0x0000000000bdb837), 0.635, 0.741092167006250) \
STEP( 128, UINT64_C(0x0000000000bfc531), 0.640, 0.749102694400000) \ STEP(128, UINT64_C(0x0000000000bfc531), 0.640, 0.749102694400000) \
STEP( 129, UINT64_C(0x0000000000c1cbd4), 0.645, 0.757016459043750) \ STEP(129, UINT64_C(0x0000000000c1cbd4), 0.645, 0.757016459043750) \
STEP( 130, UINT64_C(0x0000000000c3cbf0), 0.650, 0.764830625000000) \ STEP(130, UINT64_C(0x0000000000c3cbf0), 0.650, 0.764830625000000) \
STEP( 131, UINT64_C(0x0000000000c5c557), 0.655, 0.772542421581250) \ STEP(131, UINT64_C(0x0000000000c5c557), 0.655, 0.772542421581250) \
STEP( 132, UINT64_C(0x0000000000c7b7da), 0.660, 0.780149145600000) \ STEP(132, UINT64_C(0x0000000000c7b7da), 0.660, 0.780149145600000) \
STEP( 133, UINT64_C(0x0000000000c9a34f), 0.665, 0.787648163618750) \ STEP(133, UINT64_C(0x0000000000c9a34f), 0.665, 0.787648163618750) \
STEP( 134, UINT64_C(0x0000000000cb878a), 0.670, 0.795036914200000) \ STEP(134, UINT64_C(0x0000000000cb878a), 0.670, 0.795036914200000) \
STEP( 135, UINT64_C(0x0000000000cd6460), 0.675, 0.802312910156250) \ STEP(135, UINT64_C(0x0000000000cd6460), 0.675, 0.802312910156250) \
STEP( 136, UINT64_C(0x0000000000cf39ab), 0.680, 0.809473740800000) \ STEP(136, UINT64_C(0x0000000000cf39ab), 0.680, 0.809473740800000) \
STEP( 137, UINT64_C(0x0000000000d10743), 0.685, 0.816517074193750) \ STEP(137, UINT64_C(0x0000000000d10743), 0.685, 0.816517074193750) \
STEP( 138, UINT64_C(0x0000000000d2cd01), 0.690, 0.823440659400000) \ STEP(138, UINT64_C(0x0000000000d2cd01), 0.690, 0.823440659400000) \
STEP( 139, UINT64_C(0x0000000000d48ac2), 0.695, 0.830242328731250) \ STEP(139, UINT64_C(0x0000000000d48ac2), 0.695, 0.830242328731250) \
STEP( 140, UINT64_C(0x0000000000d64063), 0.700, 0.836920000000000) \ STEP(140, UINT64_C(0x0000000000d64063), 0.700, 0.836920000000000) \
STEP( 141, UINT64_C(0x0000000000d7edc2), 0.705, 0.843471678768750) \ STEP(141, UINT64_C(0x0000000000d7edc2), 0.705, 0.843471678768750) \
STEP( 142, UINT64_C(0x0000000000d992bf), 0.710, 0.849895460600000) \ STEP(142, UINT64_C(0x0000000000d992bf), 0.710, 0.849895460600000) \
STEP( 143, UINT64_C(0x0000000000db2f3c), 0.715, 0.856189533306250) \ STEP(143, UINT64_C(0x0000000000db2f3c), 0.715, 0.856189533306250) \
STEP( 144, UINT64_C(0x0000000000dcc31c), 0.720, 0.862352179200000) \ STEP(144, UINT64_C(0x0000000000dcc31c), 0.720, 0.862352179200000) \
STEP( 145, UINT64_C(0x0000000000de4e44), 0.725, 0.868381777343750) \ STEP(145, UINT64_C(0x0000000000de4e44), 0.725, 0.868381777343750) \
STEP( 146, UINT64_C(0x0000000000dfd09a), 0.730, 0.874276805800000) \ STEP(146, UINT64_C(0x0000000000dfd09a), 0.730, 0.874276805800000) \
STEP( 147, UINT64_C(0x0000000000e14a07), 0.735, 0.880035843881250) \ STEP(147, UINT64_C(0x0000000000e14a07), 0.735, 0.880035843881250) \
STEP( 148, UINT64_C(0x0000000000e2ba74), 0.740, 0.885657574400000) \ STEP(148, UINT64_C(0x0000000000e2ba74), 0.740, 0.885657574400000) \
STEP( 149, UINT64_C(0x0000000000e421cd), 0.745, 0.891140785918750) \ STEP(149, UINT64_C(0x0000000000e421cd), 0.745, 0.891140785918750) \
STEP( 150, UINT64_C(0x0000000000e58000), 0.750, 0.896484375000000) \ STEP(150, UINT64_C(0x0000000000e58000), 0.750, 0.896484375000000) \
STEP( 151, UINT64_C(0x0000000000e6d4fb), 0.755, 0.901687348456250) \ STEP(151, UINT64_C(0x0000000000e6d4fb), 0.755, 0.901687348456250) \
STEP( 152, UINT64_C(0x0000000000e820b0), 0.760, 0.906748825600000) \ STEP(152, UINT64_C(0x0000000000e820b0), 0.760, 0.906748825600000) \
STEP( 153, UINT64_C(0x0000000000e96313), 0.765, 0.911668040493750) \ STEP(153, UINT64_C(0x0000000000e96313), 0.765, 0.911668040493750) \
STEP( 154, UINT64_C(0x0000000000ea9c18), 0.770, 0.916444344200000) \ STEP(154, UINT64_C(0x0000000000ea9c18), 0.770, 0.916444344200000) \
STEP( 155, UINT64_C(0x0000000000ebcbb7), 0.775, 0.921077207031250) \ STEP(155, UINT64_C(0x0000000000ebcbb7), 0.775, 0.921077207031250) \
STEP( 156, UINT64_C(0x0000000000ecf1e8), 0.780, 0.925566220800000) \ STEP(156, UINT64_C(0x0000000000ecf1e8), 0.780, 0.925566220800000) \
STEP( 157, UINT64_C(0x0000000000ee0ea7), 0.785, 0.929911101068750) \ STEP(157, UINT64_C(0x0000000000ee0ea7), 0.785, 0.929911101068750) \
STEP( 158, UINT64_C(0x0000000000ef21f1), 0.790, 0.934111689400000) \ STEP(158, UINT64_C(0x0000000000ef21f1), 0.790, 0.934111689400000) \
STEP( 159, UINT64_C(0x0000000000f02bc6), 0.795, 0.938167955606250) \ STEP(159, UINT64_C(0x0000000000f02bc6), 0.795, 0.938167955606250) \
STEP( 160, UINT64_C(0x0000000000f12c27), 0.800, 0.942080000000000) \ STEP(160, UINT64_C(0x0000000000f12c27), 0.800, 0.942080000000000) \
STEP( 161, UINT64_C(0x0000000000f22319), 0.805, 0.945848055643750) \ STEP(161, UINT64_C(0x0000000000f22319), 0.805, 0.945848055643750) \
STEP( 162, UINT64_C(0x0000000000f310a1), 0.810, 0.949472490600000) \ STEP(162, UINT64_C(0x0000000000f310a1), 0.810, 0.949472490600000) \
STEP( 163, UINT64_C(0x0000000000f3f4c7), 0.815, 0.952953810181250) \ STEP(163, UINT64_C(0x0000000000f3f4c7), 0.815, 0.952953810181250) \
STEP( 164, UINT64_C(0x0000000000f4cf98), 0.820, 0.956292659200000) \ STEP(164, UINT64_C(0x0000000000f4cf98), 0.820, 0.956292659200000) \
STEP( 165, UINT64_C(0x0000000000f5a120), 0.825, 0.959489824218750) \ STEP(165, UINT64_C(0x0000000000f5a120), 0.825, 0.959489824218750) \
STEP( 166, UINT64_C(0x0000000000f6696e), 0.830, 0.962546235800000) \ STEP(166, UINT64_C(0x0000000000f6696e), 0.830, 0.962546235800000) \
STEP( 167, UINT64_C(0x0000000000f72894), 0.835, 0.965462970756250) \ STEP(167, UINT64_C(0x0000000000f72894), 0.835, 0.965462970756250) \
STEP( 168, UINT64_C(0x0000000000f7dea8), 0.840, 0.968241254400000) \ STEP(168, UINT64_C(0x0000000000f7dea8), 0.840, 0.968241254400000) \
STEP( 169, UINT64_C(0x0000000000f88bc0), 0.845, 0.970882462793750) \ STEP(169, UINT64_C(0x0000000000f88bc0), 0.845, 0.970882462793750) \
STEP( 170, UINT64_C(0x0000000000f92ff6), 0.850, 0.973388125000000) \ STEP(170, UINT64_C(0x0000000000f92ff6), 0.850, 0.973388125000000) \
STEP( 171, UINT64_C(0x0000000000f9cb67), 0.855, 0.975759925331250) \ STEP(171, UINT64_C(0x0000000000f9cb67), 0.855, 0.975759925331250) \
STEP( 172, UINT64_C(0x0000000000fa5e30), 0.860, 0.977999705600000) \ STEP(172, UINT64_C(0x0000000000fa5e30), 0.860, 0.977999705600000) \
STEP( 173, UINT64_C(0x0000000000fae874), 0.865, 0.980109467368750) \ STEP(173, UINT64_C(0x0000000000fae874), 0.865, 0.980109467368750) \
STEP( 174, UINT64_C(0x0000000000fb6a57), 0.870, 0.982091374200000) \ STEP(174, UINT64_C(0x0000000000fb6a57), 0.870, 0.982091374200000) \
STEP( 175, UINT64_C(0x0000000000fbe400), 0.875, 0.983947753906250) \ STEP(175, UINT64_C(0x0000000000fbe400), 0.875, 0.983947753906250) \
STEP( 176, UINT64_C(0x0000000000fc5598), 0.880, 0.985681100800000) \ STEP(176, UINT64_C(0x0000000000fc5598), 0.880, 0.985681100800000) \
STEP( 177, UINT64_C(0x0000000000fcbf4e), 0.885, 0.987294077943750) \ STEP(177, UINT64_C(0x0000000000fcbf4e), 0.885, 0.987294077943750) \
STEP( 178, UINT64_C(0x0000000000fd214f), 0.890, 0.988789519400000) \ STEP(178, UINT64_C(0x0000000000fd214f), 0.890, 0.988789519400000) \
STEP( 179, UINT64_C(0x0000000000fd7bcf), 0.895, 0.990170432481250) \ STEP(179, UINT64_C(0x0000000000fd7bcf), 0.895, 0.990170432481250) \
STEP( 180, UINT64_C(0x0000000000fdcf03), 0.900, 0.991440000000000) \ STEP(180, UINT64_C(0x0000000000fdcf03), 0.900, 0.991440000000000) \
STEP( 181, UINT64_C(0x0000000000fe1b23), 0.905, 0.992601582518750) \ STEP(181, UINT64_C(0x0000000000fe1b23), 0.905, 0.992601582518750) \
STEP( 182, UINT64_C(0x0000000000fe606a), 0.910, 0.993658720600000) \ STEP(182, UINT64_C(0x0000000000fe606a), 0.910, 0.993658720600000) \
STEP( 183, UINT64_C(0x0000000000fe9f18), 0.915, 0.994615137056250) \ STEP(183, UINT64_C(0x0000000000fe9f18), 0.915, 0.994615137056250) \
STEP( 184, UINT64_C(0x0000000000fed76e), 0.920, 0.995474739200000) \ STEP(184, UINT64_C(0x0000000000fed76e), 0.920, 0.995474739200000) \
STEP( 185, UINT64_C(0x0000000000ff09b0), 0.925, 0.996241621093750) \ STEP(185, UINT64_C(0x0000000000ff09b0), 0.925, 0.996241621093750) \
STEP( 186, UINT64_C(0x0000000000ff3627), 0.930, 0.996920065800000) \ STEP(186, UINT64_C(0x0000000000ff3627), 0.930, 0.996920065800000) \
STEP( 187, UINT64_C(0x0000000000ff5d1d), 0.935, 0.997514547631250) \ STEP(187, UINT64_C(0x0000000000ff5d1d), 0.935, 0.997514547631250) \
STEP( 188, UINT64_C(0x0000000000ff7ee0), 0.940, 0.998029734400000) \ STEP(188, UINT64_C(0x0000000000ff7ee0), 0.940, 0.998029734400000) \
STEP( 189, UINT64_C(0x0000000000ff9bc3), 0.945, 0.998470489668750) \ STEP(189, UINT64_C(0x0000000000ff9bc3), 0.945, 0.998470489668750) \
STEP( 190, UINT64_C(0x0000000000ffb419), 0.950, 0.998841875000000) \ STEP(190, UINT64_C(0x0000000000ffb419), 0.950, 0.998841875000000) \
STEP( 191, UINT64_C(0x0000000000ffc83d), 0.955, 0.999149152206250) \ STEP(191, UINT64_C(0x0000000000ffc83d), 0.955, 0.999149152206250) \
STEP( 192, UINT64_C(0x0000000000ffd888), 0.960, 0.999397785600000) \ STEP(192, UINT64_C(0x0000000000ffd888), 0.960, 0.999397785600000) \
STEP( 193, UINT64_C(0x0000000000ffe55b), 0.965, 0.999593444243750) \ STEP(193, UINT64_C(0x0000000000ffe55b), 0.965, 0.999593444243750) \
STEP( 194, UINT64_C(0x0000000000ffef17), 0.970, 0.999742004200000) \ STEP(194, UINT64_C(0x0000000000ffef17), 0.970, 0.999742004200000) \
STEP( 195, UINT64_C(0x0000000000fff623), 0.975, 0.999849550781250) \ STEP(195, UINT64_C(0x0000000000fff623), 0.975, 0.999849550781250) \
STEP( 196, UINT64_C(0x0000000000fffae9), 0.980, 0.999922380800000) \ STEP(196, UINT64_C(0x0000000000fffae9), 0.980, 0.999922380800000) \
STEP( 197, UINT64_C(0x0000000000fffdd6), 0.985, 0.999967004818750) \ STEP(197, UINT64_C(0x0000000000fffdd6), 0.985, 0.999967004818750) \
STEP( 198, UINT64_C(0x0000000000ffff5a), 0.990, 0.999990149400000) \ STEP(198, UINT64_C(0x0000000000ffff5a), 0.990, 0.999990149400000) \
STEP( 199, UINT64_C(0x0000000000ffffeb), 0.995, 0.999998759356250) \ STEP(199, UINT64_C(0x0000000000ffffeb), 0.995, 0.999998759356250) \
STEP( 200, UINT64_C(0x0000000001000000), 1.000, 1.000000000000000) \ STEP(200, UINT64_C(0x0000000001000000), 1.000, 1.000000000000000)
#endif /* JEMALLOC_INTERNAL_SMOOTHSTEP_H */ #endif /* JEMALLOC_INTERNAL_SMOOTHSTEP_H */

9
include/jemalloc/internal/spin.h
View File

@@ -3,7 +3,8 @@
#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_preamble.h"
#define SPIN_INITIALIZER {0U} #define SPIN_INITIALIZER \
{ 0U }
typedef struct { typedef struct {
unsigned iteration; unsigned iteration;
@@ -11,12 +12,12 @@ typedef struct {
static inline void static inline void
spin_cpu_spinwait(void) { spin_cpu_spinwait(void) {
# if HAVE_CPU_SPINWAIT #if HAVE_CPU_SPINWAIT
CPU_SPINWAIT; CPU_SPINWAIT;
# else #else
volatile int x = 0; volatile int x = 0;
x = x; x = x;
# endif #endif
} }
static inline void static inline void

4
include/jemalloc/internal/stats.h
View File

@@ -28,11 +28,11 @@ enum {
/* Options for stats_print. */ /* Options for stats_print. */
extern bool opt_stats_print; extern bool opt_stats_print;
extern char opt_stats_print_opts[stats_print_tot_num_options+1]; extern char opt_stats_print_opts[stats_print_tot_num_options + 1];
/* Utilities for stats_interval. */ /* Utilities for stats_interval. */
extern int64_t opt_stats_interval; extern int64_t opt_stats_interval;
extern char opt_stats_interval_opts[stats_print_tot_num_options+1]; extern char opt_stats_interval_opts[stats_print_tot_num_options + 1];
#define STATS_INTERVAL_DEFAULT -1 #define STATS_INTERVAL_DEFAULT -1
/* /*

70
include/jemalloc/internal/sz.h
View File

@@ -76,8 +76,9 @@ sz_psz2ind(size_t psz) {
* SC_NGROUP. off_to_first_ps_rg begins from 1, instead of 0. e.g. * SC_NGROUP. off_to_first_ps_rg begins from 1, instead of 0. e.g.
* off_to_first_ps_rg is 1 when psz is (PAGE * SC_NGROUP + 1). * off_to_first_ps_rg is 1 when psz is (PAGE * SC_NGROUP + 1).
*/ */
pszind_t off_to_first_ps_rg = (x < SC_LG_NGROUP + LG_PAGE) ? pszind_t off_to_first_ps_rg = (x < SC_LG_NGROUP + LG_PAGE)
0 : x - (SC_LG_NGROUP + LG_PAGE); ? 0
: x - (SC_LG_NGROUP + LG_PAGE);
/* /*
* Same as sc_s::lg_delta. * Same as sc_s::lg_delta.
@@ -85,8 +86,9 @@ sz_psz2ind(size_t psz) {
* for each increase in offset, it's multiplied by two. * for each increase in offset, it's multiplied by two.
* Therefore, lg_delta = LG_PAGE + (off_to_first_ps_rg - 1). * Therefore, lg_delta = LG_PAGE + (off_to_first_ps_rg - 1).
*/ */
pszind_t lg_delta = (off_to_first_ps_rg == 0) ? pszind_t lg_delta = (off_to_first_ps_rg == 0)
LG_PAGE : LG_PAGE + (off_to_first_ps_rg - 1); ? LG_PAGE
: LG_PAGE + (off_to_first_ps_rg - 1);
/* /*
* Let's write psz in binary, e.g. 0011 for 0x3, 0111 for 0x7. * Let's write psz in binary, e.g. 0011 for 0x3, 0111 for 0x7.
@@ -118,13 +120,13 @@ sz_pind2sz_compute(pszind_t pind) {
size_t grp = pind >> SC_LG_NGROUP; size_t grp = pind >> SC_LG_NGROUP;
size_t mod = pind & ((ZU(1) << SC_LG_NGROUP) - 1); size_t mod = pind & ((ZU(1) << SC_LG_NGROUP) - 1);
size_t grp_size_mask = ~((!!grp)-1); size_t grp_size_mask = ~((!!grp) - 1);
size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP-1))) << grp) size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP - 1))) << grp)
& grp_size_mask; & grp_size_mask;
size_t shift = (grp == 0) ? 1 : grp; size_t shift = (grp == 0) ? 1 : grp;
size_t lg_delta = shift + (LG_PAGE-1); size_t lg_delta = shift + (LG_PAGE - 1);
size_t mod_size = (mod+1) << lg_delta; size_t mod_size = (mod + 1) << lg_delta;
size_t sz = grp_size + mod_size; size_t sz = grp_size + mod_size;
return sz; return sz;
@@ -148,9 +150,10 @@ sz_psz2u(size_t psz) {
if (unlikely(psz > SC_LARGE_MAXCLASS)) { if (unlikely(psz > SC_LARGE_MAXCLASS)) {
return SC_LARGE_MAXCLASS + PAGE; return SC_LARGE_MAXCLASS + PAGE;
} }
size_t x = lg_floor((psz<<1)-1); size_t x = lg_floor((psz << 1) - 1);
size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1)
LG_PAGE : x - SC_LG_NGROUP - 1; ? LG_PAGE
: x - SC_LG_NGROUP - 1;
size_t delta = ZU(1) << lg_delta; size_t delta = ZU(1) << lg_delta;
size_t delta_mask = delta - 1; size_t delta_mask = delta - 1;
size_t usize = (psz + delta_mask) & ~delta_mask; size_t usize = (psz + delta_mask) & ~delta_mask;
@@ -174,17 +177,19 @@ sz_size2index_compute_inline(size_t size) {
} }
#endif #endif
{ {
szind_t x = lg_floor((size<<1)-1); szind_t x = lg_floor((size << 1) - 1);
szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM) ? 0 : szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM)
x - (SC_LG_NGROUP + LG_QUANTUM); ? 0
: x - (SC_LG_NGROUP + LG_QUANTUM);
szind_t grp = shift << SC_LG_NGROUP; szind_t grp = shift << SC_LG_NGROUP;
szind_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) szind_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1)
? LG_QUANTUM : x - SC_LG_NGROUP - 1; ? LG_QUANTUM
: x - SC_LG_NGROUP - 1;
size_t delta_inverse_mask = ZU(-1) << lg_delta; size_t delta_inverse_mask = ZU(-1) << lg_delta;
szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & szind_t mod = ((((size - 1) & delta_inverse_mask) >> lg_delta))
((ZU(1) << SC_LG_NGROUP) - 1); & ((ZU(1) << SC_LG_NGROUP) - 1);
szind_t index = SC_NTINY + grp + mod; szind_t index = SC_NTINY + grp + mod;
return index; return index;
@@ -228,16 +233,16 @@ sz_index2size_compute_inline(szind_t index) {
{ {
size_t reduced_index = index - SC_NTINY; size_t reduced_index = index - SC_NTINY;
size_t grp = reduced_index >> SC_LG_NGROUP; size_t grp = reduced_index >> SC_LG_NGROUP;
size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - 1);
1);
size_t grp_size_mask = ~((!!grp)-1); size_t grp_size_mask = ~((!!grp) - 1);
size_t grp_size = ((ZU(1) << (LG_QUANTUM + size_t grp_size = ((ZU(1) << (LG_QUANTUM + (SC_LG_NGROUP - 1)))
(SC_LG_NGROUP-1))) << grp) & grp_size_mask; << grp)
& grp_size_mask;
size_t shift = (grp == 0) ? 1 : grp; size_t shift = (grp == 0) ? 1 : grp;
size_t lg_delta = shift + (LG_QUANTUM-1); size_t lg_delta = shift + (LG_QUANTUM - 1);
size_t mod_size = (mod+1) << lg_delta; size_t mod_size = (mod + 1) << lg_delta;
size_t usize = grp_size + mod_size; size_t usize = grp_size + mod_size;
return usize; return usize;
@@ -269,8 +274,8 @@ sz_index2size_unsafe(szind_t index) {
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
sz_index2size(szind_t index) { sz_index2size(szind_t index) {
assert(!sz_large_size_classes_disabled() || assert(!sz_large_size_classes_disabled()
index <= sz_size2index(USIZE_GROW_SLOW_THRESHOLD)); || index <= sz_size2index(USIZE_GROW_SLOW_THRESHOLD));
size_t size = sz_index2size_unsafe(index); size_t size = sz_index2size_unsafe(index);
/* /*
* With large size classes disabled, the usize above * With large size classes disabled, the usize above
@@ -285,8 +290,8 @@ sz_index2size(szind_t index) {
* the size is no larger than USIZE_GROW_SLOW_THRESHOLD here * the size is no larger than USIZE_GROW_SLOW_THRESHOLD here
* instead of SC_LARGE_MINCLASS. * instead of SC_LARGE_MINCLASS.
*/ */
assert(!sz_large_size_classes_disabled() || assert(!sz_large_size_classes_disabled()
size <= USIZE_GROW_SLOW_THRESHOLD); || size <= USIZE_GROW_SLOW_THRESHOLD);
return size; return size;
} }
@@ -309,9 +314,10 @@ sz_size2index_usize_fastpath(size_t size, szind_t *ind, size_t *usize) {
JEMALLOC_ALWAYS_INLINE size_t JEMALLOC_ALWAYS_INLINE size_t
sz_s2u_compute_using_delta(size_t size) { sz_s2u_compute_using_delta(size_t size) {
size_t x = lg_floor((size<<1)-1); size_t x = lg_floor((size << 1) - 1);
size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1)
? LG_QUANTUM : x - SC_LG_NGROUP - 1; ? LG_QUANTUM
: x - SC_LG_NGROUP - 1;
size_t delta = ZU(1) << lg_delta; size_t delta = ZU(1) << lg_delta;
size_t delta_mask = delta - 1; size_t delta_mask = delta - 1;
size_t usize = (size + delta_mask) & ~delta_mask; size_t usize = (size + delta_mask) & ~delta_mask;
@@ -331,8 +337,8 @@ sz_s2u_compute(size_t size) {
if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) {
size_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; size_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1;
size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); size_t lg_ceil = lg_floor(pow2_ceil_zu(size));
return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin)
(ZU(1) << lg_ceil)); : (ZU(1) << lg_ceil));
} }
#endif #endif
if (size <= SC_SMALL_MAXCLASS || !sz_large_size_classes_disabled()) { if (size <= SC_SMALL_MAXCLASS || !sz_large_size_classes_disabled()) {

16
include/jemalloc/internal/tcache_externs.h
View File

@@ -56,13 +56,13 @@ void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, unsigned rem); cache_bin_t *cache_bin, szind_t binind, unsigned rem);
void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
cache_bin_t *cache_bin, szind_t binind, bool is_small); cache_bin_t *cache_bin, szind_t binind, bool is_small);
bool tcache_bin_info_default_init(const char *bin_settings_segment_cur, bool tcache_bin_info_default_init(
size_t len_left); const char *bin_settings_segment_cur, size_t len_left);
bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len); bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
bool tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size, bool tcache_bin_ncached_max_read(
cache_bin_sz_t *ncached_max); tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
void tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, void tcache_arena_reassociate(
tcache_t *tcache, arena_t *arena); tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
tcache_t *tcache_create_explicit(tsd_t *tsd); tcache_t *tcache_create_explicit(tsd_t *tsd);
void thread_tcache_max_set(tsd_t *tsd, size_t tcache_max); void thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
void tcache_cleanup(tsd_t *tsd); void tcache_cleanup(tsd_t *tsd);
@@ -71,8 +71,8 @@ bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_flush(tsd_t *tsd, unsigned ind);
void tcaches_destroy(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind);
bool tcache_boot(tsdn_t *tsdn, base_t *base); bool tcache_boot(tsdn_t *tsdn, base_t *base);
void tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, void tcache_arena_associate(
tcache_t *tcache, arena_t *arena); tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
void tcache_prefork(tsdn_t *tsdn); void tcache_prefork(tsdn_t *tsdn);
void tcache_postfork_parent(tsdn_t *tsdn); void tcache_postfork_parent(tsdn_t *tsdn);
void tcache_postfork_child(tsdn_t *tsdn); void tcache_postfork_child(tsdn_t *tsdn);

48
include/jemalloc/internal/tcache_inlines.h
View File

@@ -42,8 +42,8 @@ tcache_max_set(tcache_slow_t *tcache_slow, size_t tcache_max) {
} }
static inline void static inline void
tcache_bin_settings_backup(tcache_t *tcache, tcache_bin_settings_backup(
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) { tcache_t *tcache, cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) { for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
cache_bin_info_init(&tcache_bin_info[i], cache_bin_info_init(&tcache_bin_info[i],
cache_bin_ncached_max_get_unsafe(&tcache->bins[i])); cache_bin_ncached_max_get_unsafe(&tcache->bins[i]));
@@ -51,8 +51,7 @@ tcache_bin_settings_backup(tcache_t *tcache,
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_slow_t *tcache_slow) {
tcache_slow_t *tcache_slow) {
assert(bin != NULL); assert(bin != NULL);
assert(ind < TCACHE_NBINS_MAX); assert(ind < TCACHE_NBINS_MAX);
bool disabled = cache_bin_disabled(bin); bool disabled = cache_bin_disabled(bin);
@@ -88,8 +87,8 @@ tcache_bin_disabled(szind_t ind, cache_bin_t *bin,
} }
JEMALLOC_ALWAYS_INLINE void * JEMALLOC_ALWAYS_INLINE void *
tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
size_t size, szind_t binind, bool zero, bool slow_path) { szind_t binind, bool zero, bool slow_path) {
void *ret; void *ret;
bool tcache_success; bool tcache_success;
@@ -103,8 +102,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
if (unlikely(arena == NULL)) { if (unlikely(arena == NULL)) {
return NULL; return NULL;
} }
if (unlikely(tcache_bin_disabled(binind, bin, if (unlikely(tcache_bin_disabled(
tcache->tcache_slow))) { binind, bin, tcache->tcache_slow))) {
/* stats and zero are handled directly by the arena. */ /* stats and zero are handled directly by the arena. */
return arena_malloc_hard(tsd_tsdn(tsd), arena, size, return arena_malloc_hard(tsd_tsdn(tsd), arena, size,
binind, zero, /* slab */ true); binind, zero, /* slab */ true);
@@ -112,8 +111,8 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
tcache_bin_flush_stashed(tsd, tcache, bin, binind, tcache_bin_flush_stashed(tsd, tcache, bin, binind,
/* is_small */ true); /* is_small */ true);
ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, bin,
bin, binind, &tcache_hard_success); binind, &tcache_hard_success);
if (tcache_hard_success == false) { if (tcache_hard_success == false) {
return NULL; return NULL;
} }
@@ -138,8 +137,8 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
bool tcache_success; bool tcache_success;
cache_bin_t *bin = &tcache->bins[binind]; cache_bin_t *bin = &tcache->bins[binind];
assert(binind >= SC_NBINS && assert(binind >= SC_NBINS
!tcache_bin_disabled(binind, bin, tcache->tcache_slow)); && !tcache_bin_disabled(binind, bin, tcache->tcache_slow));
ret = cache_bin_alloc(bin, &tcache_success); ret = cache_bin_alloc(bin, &tcache_success);
assert(tcache_success == (ret != NULL)); assert(tcache_success == (ret != NULL));
if (unlikely(!tcache_success)) { if (unlikely(!tcache_success)) {
@@ -174,8 +173,8 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, tcache_dalloc_small(
bool slow_path) { tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) {
assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SC_SMALL_MAXCLASS); assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SC_SMALL_MAXCLASS);
cache_bin_t *bin = &tcache->bins[binind]; cache_bin_t *bin = &tcache->bins[binind];
@@ -195,8 +194,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
} }
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
if (unlikely(tcache_bin_disabled(binind, bin, if (unlikely(tcache_bin_disabled(
tcache->tcache_slow))) { binind, bin, tcache->tcache_slow))) {
arena_dalloc_small(tsd_tsdn(tsd), ptr); arena_dalloc_small(tsd_tsdn(tsd), ptr);
return; return;
} }
@@ -209,19 +208,18 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind,
} }
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, tcache_dalloc_large(
bool slow_path) { tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) {
assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SC_SMALL_MAXCLASS); assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SC_SMALL_MAXCLASS);
assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= assert(tcache_salloc(tsd_tsdn(tsd), ptr)
tcache_max_get(tcache->tcache_slow)); <= tcache_max_get(tcache->tcache_slow));
assert(!tcache_bin_disabled(binind, &tcache->bins[binind], assert(!tcache_bin_disabled(
tcache->tcache_slow)); binind, &tcache->bins[binind], tcache->tcache_slow));
cache_bin_t *bin = &tcache->bins[binind]; cache_bin_t *bin = &tcache->bins[binind];
if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) { if (unlikely(!cache_bin_dalloc_easy(bin, ptr))) {
unsigned remain = cache_bin_ncached_max_get(bin) >> unsigned remain = cache_bin_ncached_max_get(bin)
opt_lg_tcache_flush_large_div; >> opt_lg_tcache_flush_large_div;
tcache_bin_flush_large(tsd, tcache, bin, binind, remain); tcache_bin_flush_large(tsd, tcache, bin, binind, remain);
bool ret = cache_bin_dalloc_easy(bin, ptr); bool ret = cache_bin_dalloc_easy(bin, ptr);
assert(ret); assert(ret);

14
include/jemalloc/internal/tcache_types.h
View File

@@ -9,8 +9,12 @@ typedef struct tcache_s tcache_t;
typedef struct tcaches_s tcaches_t; typedef struct tcaches_s tcaches_t;
/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */ /* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
#define TCACHE_ZERO_INITIALIZER {0} #define TCACHE_ZERO_INITIALIZER \
#define TCACHE_SLOW_ZERO_INITIALIZER {{0}} { 0 }
#define TCACHE_SLOW_ZERO_INITIALIZER \
{ \
{ 0 } \
}
/* Used in TSD static initializer only. Will be initialized to opt_tcache. */ /* Used in TSD static initializer only. Will be initialized to opt_tcache. */
#define TCACHE_ENABLED_ZERO_INITIALIZER false #define TCACHE_ENABLED_ZERO_INITIALIZER false
@@ -21,8 +25,10 @@ typedef struct tcaches_s tcaches_t;
#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD #define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT) #define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
#define TCACHE_NBINS_MAX (SC_NBINS + SC_NGROUP * \ #define TCACHE_NBINS_MAX \
(TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS) + 1) (SC_NBINS \
+ SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS) \
+ 1)
#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21) /* 2M */ #define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21) /* 2M */
#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */ #define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1) #define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)

10
include/jemalloc/internal/test_hooks.h
View File

@@ -7,7 +7,8 @@ extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(void);
extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void); extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void);
#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST) #if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
# define JEMALLOC_TEST_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) # define JEMALLOC_TEST_HOOK(fn, hook) \
((void)(hook != NULL && (hook(), 0)), fn)
# define open JEMALLOC_TEST_HOOK(open, test_hooks_libc_hook) # define open JEMALLOC_TEST_HOOK(open, test_hooks_libc_hook)
# define read JEMALLOC_TEST_HOOK(read, test_hooks_libc_hook) # define read JEMALLOC_TEST_HOOK(read, test_hooks_libc_hook)
@@ -15,12 +16,13 @@ extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void);
# define readlink JEMALLOC_TEST_HOOK(readlink, test_hooks_libc_hook) # define readlink JEMALLOC_TEST_HOOK(readlink, test_hooks_libc_hook)
# define close JEMALLOC_TEST_HOOK(close, test_hooks_libc_hook) # define close JEMALLOC_TEST_HOOK(close, test_hooks_libc_hook)
# define creat JEMALLOC_TEST_HOOK(creat, test_hooks_libc_hook) # define creat JEMALLOC_TEST_HOOK(creat, test_hooks_libc_hook)
# define secure_getenv JEMALLOC_TEST_HOOK(secure_getenv, test_hooks_libc_hook) # define secure_getenv \
JEMALLOC_TEST_HOOK(secure_getenv, test_hooks_libc_hook)
/* Note that this is undef'd and re-define'd in src/prof.c. */ /* Note that this is undef'd and re-define'd in src/prof.c. */
# define _Unwind_Backtrace JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) # define _Unwind_Backtrace \
JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
#else #else
# define JEMALLOC_TEST_HOOK(fn, hook) fn # define JEMALLOC_TEST_HOOK(fn, hook) fn
#endif #endif
#endif /* JEMALLOC_INTERNAL_TEST_HOOKS_H */ #endif /* JEMALLOC_INTERNAL_TEST_HOOKS_H */

13
include/jemalloc/internal/thread_event.h
View File

@@ -48,8 +48,7 @@ void te_assert_invariants_debug(tsd_t *tsd);
void te_event_trigger(tsd_t *tsd, te_ctx_t *ctx); void te_event_trigger(tsd_t *tsd, te_ctx_t *ctx);
void te_recompute_fast_threshold(tsd_t *tsd); void te_recompute_fast_threshold(tsd_t *tsd);
void tsd_te_init(tsd_t *tsd); void tsd_te_init(tsd_t *tsd);
void te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx, void te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx, uint64_t wait);
uint64_t wait);
/* List of all thread event counters. */ /* List of all thread event counters. */
#define ITERATE_OVER_ALL_COUNTERS \ #define ITERATE_OVER_ALL_COUNTERS \
@@ -60,10 +59,9 @@ void te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx,
/* Getters directly wrap TSD getters. */ /* Getters directly wrap TSD getters. */
#define C(counter) \ #define C(counter) \
JEMALLOC_ALWAYS_INLINE uint64_t \ JEMALLOC_ALWAYS_INLINE uint64_t counter##_get(tsd_t *tsd) { \
counter##_get(tsd_t *tsd) { \
return tsd_##counter##_get(tsd); \ return tsd_##counter##_get(tsd); \
} }
ITERATE_OVER_ALL_COUNTERS ITERATE_OVER_ALL_COUNTERS
#undef C #undef C
@@ -76,10 +74,9 @@ ITERATE_OVER_ALL_COUNTERS
* allocation call. * allocation call.
*/ */
#define C(counter) \ #define C(counter) \
JEMALLOC_ALWAYS_INLINE void \ JEMALLOC_ALWAYS_INLINE void counter##_set(tsd_t *tsd, uint64_t v) { \
counter##_set(tsd_t *tsd, uint64_t v) { \
*tsd_##counter##p_get(tsd) = v; \ *tsd_##counter##p_get(tsd) = v; \
} }
ITERATE_OVER_ALL_COUNTERS ITERATE_OVER_ALL_COUNTERS
#undef C #undef C

16
include/jemalloc/internal/ticker.h
View File

@@ -129,7 +129,8 @@ struct ticker_geom_s {
* the behavior over long periods of time rather than the exact timing of the * the behavior over long periods of time rather than the exact timing of the
* initial ticks. * initial ticks.
*/ */
#define TICKER_GEOM_INIT(nticks) {nticks, nticks} #define TICKER_GEOM_INIT(nticks) \
{ nticks, nticks }
static inline void static inline void
ticker_geom_init(ticker_geom_t *ticker, int32_t nticks) { ticker_geom_init(ticker_geom_t *ticker, int32_t nticks) {
@@ -155,17 +156,16 @@ ticker_geom_read(const ticker_geom_t *ticker) {
JEMALLOC_NOINLINE JEMALLOC_NOINLINE
#endif #endif
static bool static bool
ticker_geom_fixup(ticker_geom_t *ticker, uint64_t *prng_state, ticker_geom_fixup(
bool delay_trigger) { ticker_geom_t *ticker, uint64_t *prng_state, bool delay_trigger) {
if (delay_trigger) { if (delay_trigger) {
ticker->tick = 0; ticker->tick = 0;
return false; return false;
} }
uint64_t idx = prng_lg_range_u64(prng_state, TICKER_GEOM_NBITS); uint64_t idx = prng_lg_range_u64(prng_state, TICKER_GEOM_NBITS);
ticker->tick = (uint32_t)( ticker->tick = (uint32_t)((uint64_t)ticker->nticks
(uint64_t)ticker->nticks * (uint64_t)ticker_geom_table[idx] * (uint64_t)ticker_geom_table[idx] / (uint64_t)TICKER_GEOM_MUL);
/ (uint64_t)TICKER_GEOM_MUL);
return true; return true;
} }
@@ -181,8 +181,8 @@ ticker_geom_ticks(ticker_geom_t *ticker, uint64_t *prng_state, int32_t nticks,
} }
static inline bool static inline bool
ticker_geom_tick(ticker_geom_t *ticker, uint64_t *prng_state, ticker_geom_tick(
bool delay_trigger) { ticker_geom_t *ticker, uint64_t *prng_state, bool delay_trigger) {
return ticker_geom_ticks(ticker, prng_state, 1, delay_trigger); return ticker_geom_ticks(ticker, prng_state, 1, delay_trigger);
} }

42
include/jemalloc/internal/tsd.h
View File

@@ -7,14 +7,14 @@
* tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates, tsd_get, and tsd_set. * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates, tsd_get, and tsd_set.
*/ */
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
#include "jemalloc/internal/jemalloc_preamble.h" # include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/tsd_malloc_thread_cleanup.h" # include "jemalloc/internal/tsd_malloc_thread_cleanup.h"
#elif (defined(JEMALLOC_TLS)) #elif (defined(JEMALLOC_TLS))
#include "jemalloc/internal/tsd_tls.h" # include "jemalloc/internal/tsd_tls.h"
#elif (defined(_WIN32)) #elif (defined(_WIN32))
#include "jemalloc/internal/tsd_win.h" # include "jemalloc/internal/tsd_win.h"
#else #else
#include "jemalloc/internal/tsd_generic.h" # include "jemalloc/internal/tsd_generic.h"
#endif #endif
/* /*
@@ -23,10 +23,9 @@
* initialization and cleanup. * initialization and cleanup.
*/ */
#define O(n, t, nt) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE t * \ JEMALLOC_ALWAYS_INLINE t *tsd_##n##p_get_unsafe(tsd_t *tsd) { \
tsd_##n##p_get_unsafe(tsd_t *tsd) { \
return &tsd->TSD_MANGLE(n); \ return &tsd->TSD_MANGLE(n); \
} }
TSD_DATA_SLOW TSD_DATA_SLOW
TSD_DATA_FAST TSD_DATA_FAST
TSD_DATA_SLOWER TSD_DATA_SLOWER
@@ -60,14 +59,13 @@ TSD_DATA_SLOWER
* isn't NULL), or NULL (if tsdn is NULL), cast to the nullable pointer type. * isn't NULL), or NULL (if tsdn is NULL), cast to the nullable pointer type.
*/ */
#define O(n, t, nt) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE nt * \ JEMALLOC_ALWAYS_INLINE nt *tsdn_##n##p_get(tsdn_t *tsdn) { \
tsdn_##n##p_get(tsdn_t *tsdn) { \
if (tsdn_null(tsdn)) { \ if (tsdn_null(tsdn)) { \
return NULL; \ return NULL; \
} \ } \
tsd_t *tsd = tsdn_tsd(tsdn); \ tsd_t *tsd = tsdn_tsd(tsdn); \
return (nt *)tsd_##n##p_get(tsd); \ return (nt *)tsd_##n##p_get(tsd); \
} }
TSD_DATA_SLOW TSD_DATA_SLOW
TSD_DATA_FAST TSD_DATA_FAST
TSD_DATA_SLOWER TSD_DATA_SLOWER
@@ -75,10 +73,9 @@ TSD_DATA_SLOWER
/* tsd_foo_get(tsd) returns the value of the thread-local instance of foo. */ /* tsd_foo_get(tsd) returns the value of the thread-local instance of foo. */
#define O(n, t, nt) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE t \ JEMALLOC_ALWAYS_INLINE t tsd_##n##_get(tsd_t *tsd) { \
tsd_##n##_get(tsd_t *tsd) { \
return *tsd_##n##p_get(tsd); \ return *tsd_##n##p_get(tsd); \
} }
TSD_DATA_SLOW TSD_DATA_SLOW
TSD_DATA_FAST TSD_DATA_FAST
TSD_DATA_SLOWER TSD_DATA_SLOWER
@@ -86,12 +83,11 @@ TSD_DATA_SLOWER
/* tsd_foo_set(tsd, val) updates the thread-local instance of foo to be val. */ /* tsd_foo_set(tsd, val) updates the thread-local instance of foo to be val. */
#define O(n, t, nt) \ #define O(n, t, nt) \
JEMALLOC_ALWAYS_INLINE void \ JEMALLOC_ALWAYS_INLINE void tsd_##n##_set(tsd_t *tsd, t val) { \
tsd_##n##_set(tsd_t *tsd, t val) { \ assert(tsd_state_get(tsd) != tsd_state_reincarnated \
assert(tsd_state_get(tsd) != tsd_state_reincarnated && \ && tsd_state_get(tsd) != tsd_state_minimal_initialized); \
tsd_state_get(tsd) != tsd_state_minimal_initialized); \
*tsd_##n##p_get(tsd) = val; \ *tsd_##n##p_get(tsd) = val; \
} }
TSD_DATA_SLOW TSD_DATA_SLOW
TSD_DATA_FAST TSD_DATA_FAST
TSD_DATA_SLOWER TSD_DATA_SLOWER
@@ -104,8 +100,8 @@ tsd_assert_fast(tsd_t *tsd) {
* counters; it's not in general possible to ensure that they won't * counters; it's not in general possible to ensure that they won't
* change asynchronously from underneath us. * change asynchronously from underneath us.
*/ */
assert(!malloc_slow && tsd_tcache_enabled_get(tsd) && assert(!malloc_slow && tsd_tcache_enabled_get(tsd)
tsd_reentrancy_level_get(tsd) == 0); && tsd_reentrancy_level_get(tsd) == 0);
} }
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
@@ -194,8 +190,8 @@ tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) {
static inline bool static inline bool
tsd_state_nocleanup(tsd_t *tsd) { tsd_state_nocleanup(tsd_t *tsd) {
return tsd_state_get(tsd) == tsd_state_reincarnated || return tsd_state_get(tsd) == tsd_state_reincarnated
tsd_state_get(tsd) == tsd_state_minimal_initialized; || tsd_state_get(tsd) == tsd_state_minimal_initialized;
} }
/* /*

21
include/jemalloc/internal/tsd_generic.h
View File

@@ -1,5 +1,5 @@
#ifdef JEMALLOC_INTERNAL_TSD_GENERIC_H #ifdef JEMALLOC_INTERNAL_TSD_GENERIC_H
#error This file should be included only once, by tsd.h. # error This file should be included only once, by tsd.h.
#endif #endif
#define JEMALLOC_INTERNAL_TSD_GENERIC_H #define JEMALLOC_INTERNAL_TSD_GENERIC_H
@@ -23,8 +23,7 @@ typedef struct {
tsd_t val; tsd_t val;
} tsd_wrapper_t; } tsd_wrapper_t;
void *tsd_init_check_recursion(tsd_init_head_t *head, void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block);
tsd_init_block_t *block);
void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
extern pthread_key_t tsd_tsd; extern pthread_key_t tsd_tsd;
@@ -42,8 +41,8 @@ tsd_cleanup_wrapper(void *arg) {
tsd_cleanup(&wrapper->val); tsd_cleanup(&wrapper->val);
if (wrapper->initialized) { if (wrapper->initialized) {
/* Trigger another cleanup round. */ /* Trigger another cleanup round. */
if (pthread_setspecific(tsd_tsd, (void *)wrapper) != 0) if (pthread_setspecific(tsd_tsd, (void *)wrapper)
{ != 0) {
malloc_write("<jemalloc>: Error setting TSD\n"); malloc_write("<jemalloc>: Error setting TSD\n");
if (opt_abort) { if (opt_abort) {
abort(); abort();
@@ -78,13 +77,13 @@ tsd_wrapper_get(bool init) {
if (init && unlikely(wrapper == NULL)) { if (init && unlikely(wrapper == NULL)) {
tsd_init_block_t block; tsd_init_block_t block;
wrapper = (tsd_wrapper_t *) wrapper = (tsd_wrapper_t *)tsd_init_check_recursion(
tsd_init_check_recursion(&tsd_init_head, &block); &tsd_init_head, &block);
if (wrapper) { if (wrapper) {
return wrapper; return wrapper;
} }
wrapper = (tsd_wrapper_t *) wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(
malloc_tsd_malloc(sizeof(tsd_wrapper_t)); sizeof(tsd_wrapper_t));
block.data = (void *)wrapper; block.data = (void *)wrapper;
if (wrapper == NULL) { if (wrapper == NULL) {
malloc_write("<jemalloc>: Error allocating TSD\n"); malloc_write("<jemalloc>: Error allocating TSD\n");
@@ -108,8 +107,8 @@ tsd_boot0(void) {
tsd_wrapper_t *wrapper; tsd_wrapper_t *wrapper;
tsd_init_block_t block; tsd_init_block_t block;
wrapper = (tsd_wrapper_t *) wrapper = (tsd_wrapper_t *)tsd_init_check_recursion(
tsd_init_check_recursion(&tsd_init_head, &block); &tsd_init_head, &block);
if (wrapper) { if (wrapper) {
return false; return false;
} }

43
include/jemalloc/internal/tsd_internals.h
View File

@@ -80,7 +80,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
O(arena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \
O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \
O(sec_shard, uint8_t, uint8_t) \ O(sec_shard, uint8_t, uint8_t) \
O(binshards, tsd_binshards_t, tsd_binshards_t)\ O(binshards, tsd_binshards_t, tsd_binshards_t) \
O(tsd_link, tsd_link_t, tsd_link_t) \ O(tsd_link, tsd_link_t, tsd_link_t) \
O(in_hook, bool, bool) \ O(in_hook, bool, bool) \
O(peak, peak_t, peak_t) \ O(peak, peak_t, peak_t) \
@@ -91,29 +91,21 @@ typedef ql_elm(tsd_t) tsd_link_t;
#define TSD_DATA_SLOW_INITIALIZER \ #define TSD_DATA_SLOW_INITIALIZER \
/* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \ /* tcache_enabled */ TCACHE_ENABLED_ZERO_INITIALIZER, \
/* reentrancy_level */ 0, \ /* reentrancy_level */ 0, /* min_init_state_nfetched */ 0, \
/* min_init_state_nfetched */ 0, \
/* thread_allocated_last_event */ 0, \ /* thread_allocated_last_event */ 0, \
/* thread_allocated_next_event */ 0, \ /* thread_allocated_next_event */ 0, \
/* thread_deallocated_last_event */ 0, \ /* thread_deallocated_last_event */ 0, \
/* thread_deallocated_next_event */ 0, \ /* thread_deallocated_next_event */ 0, \
/* te_data */ TE_DATA_INITIALIZER, \ /* te_data */ TE_DATA_INITIALIZER, /* prof_sample_last_event */ 0, \
/* prof_sample_last_event */ 0, \ /* stats_interval_last_event */ 0, /* prof_tdata */ NULL, \
/* stats_interval_last_event */ 0, \ /* prng_state */ 0, /* san_extents_until_guard_small */ 0, \
/* prof_tdata */ NULL, \ /* san_extents_until_guard_large */ 0, /* iarena */ NULL, \
/* prng_state */ 0, \ /* arena */ NULL, /* arena_decay_ticker */ \
/* san_extents_until_guard_small */ 0, \
/* san_extents_until_guard_large */ 0, \
/* iarena */ NULL, \
/* arena */ NULL, \
/* arena_decay_ticker */ \
TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \ TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \
/* sec_shard */ (uint8_t)-1, \ /* sec_shard */ (uint8_t) - 1, \
/* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \
/* tsd_link */ {NULL}, \ /* tsd_link */ {NULL}, /* in_hook */ false, \
/* in_hook */ false, \ /* peak */ PEAK_INITIALIZER, /* activity_callback_thunk */ \
/* peak */ PEAK_INITIALIZER, \
/* activity_callback_thunk */ \
ACTIVITY_CALLBACK_THUNK_INITIALIZER, \ ACTIVITY_CALLBACK_THUNK_INITIALIZER, \
/* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \ /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER, \
/* rtree_ctx */ RTREE_CTX_INITIALIZER, /* rtree_ctx */ RTREE_CTX_INITIALIZER,
@@ -127,8 +119,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
O(tcache, tcache_t, tcache_t) O(tcache, tcache_t, tcache_t)
#define TSD_DATA_FAST_INITIALIZER \ #define TSD_DATA_FAST_INITIALIZER \
/* thread_allocated */ 0, \ /* thread_allocated */ 0, /* thread_allocated_next_event_fast */ 0, \
/* thread_allocated_next_event_fast */ 0, \
/* thread_deallocated */ 0, \ /* thread_deallocated */ 0, \
/* thread_deallocated_next_event_fast */ 0, \ /* thread_deallocated_next_event_fast */ 0, \
/* tcache */ TCACHE_ZERO_INITIALIZER, /* tcache */ TCACHE_ZERO_INITIALIZER,
@@ -142,13 +133,12 @@ typedef ql_elm(tsd_t) tsd_link_t;
/* witness */ WITNESS_TSD_INITIALIZER \ /* witness */ WITNESS_TSD_INITIALIZER \
/* test data */ MALLOC_TEST_TSD_INITIALIZER /* test data */ MALLOC_TEST_TSD_INITIALIZER
#define TSD_INITIALIZER \
#define TSD_INITIALIZER { \ { \
TSD_DATA_SLOW_INITIALIZER \ TSD_DATA_SLOW_INITIALIZER \
/* state */ ATOMIC_INIT(tsd_state_uninitialized), \ /* state */ ATOMIC_INIT(tsd_state_uninitialized), \
TSD_DATA_FAST_INITIALIZER \ TSD_DATA_FAST_INITIALIZER TSD_DATA_SLOWER_INITIALIZER \
TSD_DATA_SLOWER_INITIALIZER \ }
}
#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
void _malloc_tsd_cleanup_register(bool (*f)(void)); void _malloc_tsd_cleanup_register(bool (*f)(void));
@@ -243,8 +233,7 @@ struct tsd_s {
* setters below. * setters below.
*/ */
#define O(n, t, nt) \ #define O(n, t, nt) t TSD_MANGLE(n);
t TSD_MANGLE(n);
TSD_DATA_SLOW TSD_DATA_SLOW
/* /*

2
include/jemalloc/internal/tsd_malloc_thread_cleanup.h
View File

@@ -1,5 +1,5 @@
#ifdef JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H #ifdef JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H
#error This file should be included only once, by tsd.h. # error This file should be included only once, by tsd.h.
#endif #endif
#define JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H #define JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H

2
include/jemalloc/internal/tsd_tls.h
View File

@@ -1,5 +1,5 @@
#ifdef JEMALLOC_INTERNAL_TSD_TLS_H #ifdef JEMALLOC_INTERNAL_TSD_TLS_H
#error This file should be included only once, by tsd.h. # error This file should be included only once, by tsd.h.
#endif #endif
#define JEMALLOC_INTERNAL_TSD_TLS_H #define JEMALLOC_INTERNAL_TSD_TLS_H

39
include/jemalloc/internal/tsd_win.h
View File

@@ -1,5 +1,5 @@
#ifdef JEMALLOC_INTERNAL_TSD_WIN_H #ifdef JEMALLOC_INTERNAL_TSD_WIN_H
#error This file should be included only once, by tsd.h. # error This file should be included only once, by tsd.h.
#endif #endif
#define JEMALLOC_INTERNAL_TSD_WIN_H #define JEMALLOC_INTERNAL_TSD_WIN_H
@@ -21,16 +21,16 @@ typedef struct {
extern DWORD tsd_tsd; extern DWORD tsd_tsd;
extern tsd_wrapper_t tsd_boot_wrapper; extern tsd_wrapper_t tsd_boot_wrapper;
extern bool tsd_booted; extern bool tsd_booted;
#if defined(_M_ARM64EC) # if defined(_M_ARM64EC)
#define JEMALLOC_WIN32_TLSGETVALUE2 0 # define JEMALLOC_WIN32_TLSGETVALUE2 0
#else # else
#define JEMALLOC_WIN32_TLSGETVALUE2 1 # define JEMALLOC_WIN32_TLSGETVALUE2 1
#endif # endif
#if JEMALLOC_WIN32_TLSGETVALUE2 # if JEMALLOC_WIN32_TLSGETVALUE2
typedef LPVOID (WINAPI *TGV2)(DWORD dwTlsIndex); typedef LPVOID(WINAPI *TGV2)(DWORD dwTlsIndex);
extern TGV2 tls_get_value2; extern TGV2 tls_get_value2;
extern HMODULE tgv2_mod; extern HMODULE tgv2_mod;
#endif # endif
/* Initialization/cleanup. */ /* Initialization/cleanup. */
JEMALLOC_ALWAYS_INLINE bool JEMALLOC_ALWAYS_INLINE bool
@@ -66,20 +66,20 @@ tsd_wrapper_set(tsd_wrapper_t *wrapper) {
JEMALLOC_ALWAYS_INLINE tsd_wrapper_t * JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
tsd_wrapper_get(bool init) { tsd_wrapper_get(bool init) {
tsd_wrapper_t *wrapper; tsd_wrapper_t *wrapper;
#if JEMALLOC_WIN32_TLSGETVALUE2 # if JEMALLOC_WIN32_TLSGETVALUE2
if (tls_get_value2 != NULL) { if (tls_get_value2 != NULL) {
wrapper = (tsd_wrapper_t *) tls_get_value2(tsd_tsd); wrapper = (tsd_wrapper_t *)tls_get_value2(tsd_tsd);
} else } else
#endif # endif
{ {
DWORD error = GetLastError(); DWORD error = GetLastError();
wrapper = (tsd_wrapper_t *) TlsGetValue(tsd_tsd); wrapper = (tsd_wrapper_t *)TlsGetValue(tsd_tsd);
SetLastError(error); SetLastError(error);
} }
if (init && unlikely(wrapper == NULL)) { if (init && unlikely(wrapper == NULL)) {
wrapper = (tsd_wrapper_t *) wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(
malloc_tsd_malloc(sizeof(tsd_wrapper_t)); sizeof(tsd_wrapper_t));
if (wrapper == NULL) { if (wrapper == NULL) {
malloc_write("<jemalloc>: Error allocating TSD\n"); malloc_write("<jemalloc>: Error allocating TSD\n");
abort(); abort();
@@ -102,12 +102,12 @@ tsd_boot0(void) {
} }
_malloc_tsd_cleanup_register(&tsd_cleanup_wrapper); _malloc_tsd_cleanup_register(&tsd_cleanup_wrapper);
tsd_wrapper_set(&tsd_boot_wrapper); tsd_wrapper_set(&tsd_boot_wrapper);
#if JEMALLOC_WIN32_TLSGETVALUE2 # if JEMALLOC_WIN32_TLSGETVALUE2
tgv2_mod = LoadLibraryA("api-ms-win-core-processthreads-l1-1-8.dll"); tgv2_mod = LoadLibraryA("api-ms-win-core-processthreads-l1-1-8.dll");
if (tgv2_mod != NULL) { if (tgv2_mod != NULL) {
tls_get_value2 = (TGV2)GetProcAddress(tgv2_mod, "TlsGetValue2"); tls_get_value2 = (TGV2)GetProcAddress(tgv2_mod, "TlsGetValue2");
} }
#endif # endif
tsd_booted = true; tsd_booted = true;
return false; return false;
} }
@@ -115,8 +115,7 @@ tsd_boot0(void) {
JEMALLOC_ALWAYS_INLINE void JEMALLOC_ALWAYS_INLINE void
tsd_boot1(void) { tsd_boot1(void) {
tsd_wrapper_t *wrapper; tsd_wrapper_t *wrapper;
wrapper = (tsd_wrapper_t *) wrapper = (tsd_wrapper_t *)malloc_tsd_malloc(sizeof(tsd_wrapper_t));
malloc_tsd_malloc(sizeof(tsd_wrapper_t));
if (wrapper == NULL) { if (wrapper == NULL) {
malloc_write("<jemalloc>: Error allocating TSD\n"); malloc_write("<jemalloc>: Error allocating TSD\n");
abort(); abort();
@@ -174,7 +173,7 @@ tsd_set(tsd_t *val) {
#else // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) #else // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER)
#define JEMALLOC_TSD_TYPE_ATTR(type) __declspec(thread) type # define JEMALLOC_TSD_TYPE_ATTR(type) __declspec(thread) type
extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls; extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls;
extern bool tsd_booted; extern bool tsd_booted;

61
include/jemalloc/internal/typed_list.h
View File

@@ -7,53 +7,48 @@
*/ */
#define TYPED_LIST(list_type, el_type, linkage) \ #define TYPED_LIST(list_type, el_type, linkage) \
typedef struct { \ typedef struct { \
ql_head(el_type) head; \ ql_head(el_type) head; \
} list_type##_t; \ } list_type##_t; \
static inline void \ static inline void list_type##_init(list_type##_t *list) { \
list_type##_init(list_type##_t *list) { \
ql_new(&list->head); \ ql_new(&list->head); \
} \ } \
static inline el_type * \ static inline el_type *list_type##_first(const list_type##_t *list) { \
list_type##_first(const list_type##_t *list) { \
return ql_first(&list->head); \ return ql_first(&list->head); \
} \ } \
static inline el_type * \ static inline el_type *list_type##_last(const list_type##_t *list) { \
list_type##_last(const list_type##_t *list) { \
return ql_last(&list->head, linkage); \ return ql_last(&list->head, linkage); \
} \ } \
static inline el_type * \ static inline el_type *list_type##_next( \
list_type##_next(const list_type##_t *list, el_type *item) { \ const list_type##_t *list, el_type *item) { \
return ql_next(&list->head, item, linkage); \ return ql_next(&list->head, item, linkage); \
} \ } \
static inline void \ static inline void list_type##_append( \
list_type##_append(list_type##_t *list, el_type *item) { \ list_type##_t *list, el_type *item) { \
ql_elm_new(item, linkage); \ ql_elm_new(item, linkage); \
ql_tail_insert(&list->head, item, linkage); \ ql_tail_insert(&list->head, item, linkage); \
} \ } \
static inline void \ static inline void list_type##_prepend( \
list_type##_prepend(list_type##_t *list, el_type *item) { \ list_type##_t *list, el_type *item) { \
ql_elm_new(item, linkage); \ ql_elm_new(item, linkage); \
ql_head_insert(&list->head, item, linkage); \ ql_head_insert(&list->head, item, linkage); \
} \ } \
static inline void \ static inline void list_type##_replace( \
list_type##_replace(list_type##_t *list, el_type *to_remove, \ list_type##_t *list, el_type *to_remove, el_type *to_insert) { \
el_type *to_insert) { \
ql_elm_new(to_insert, linkage); \ ql_elm_new(to_insert, linkage); \
ql_after_insert(to_remove, to_insert, linkage); \ ql_after_insert(to_remove, to_insert, linkage); \
ql_remove(&list->head, to_remove, linkage); \ ql_remove(&list->head, to_remove, linkage); \
} \ } \
static inline void \ static inline void list_type##_remove( \
list_type##_remove(list_type##_t *list, el_type *item) { \ list_type##_t *list, el_type *item) { \
ql_remove(&list->head, item, linkage); \ ql_remove(&list->head, item, linkage); \
} \ } \
static inline bool \ static inline bool list_type##_empty(list_type##_t *list) { \
list_type##_empty(list_type##_t *list) { \
return ql_empty(&list->head); \ return ql_empty(&list->head); \
} \ } \
static inline void \ static inline void list_type##_concat( \
list_type##_concat(list_type##_t *list_a, list_type##_t *list_b) { \ list_type##_t *list_a, list_type##_t *list_b) { \
ql_concat(&list_a->head, &list_b->head, linkage); \ ql_concat(&list_a->head, &list_b->head, linkage); \
} }
#endif /* JEMALLOC_INTERNAL_TYPED_LIST_H */ #endif /* JEMALLOC_INTERNAL_TYPED_LIST_H */

25
include/jemalloc/internal/util.h
View File

@@ -40,12 +40,12 @@
#endif #endif
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L
#include <stddef.h> # include <stddef.h>
#else #else
#if !defined(JEMALLOC_INTERNAL_UNREACHABLE) # if !defined(JEMALLOC_INTERNAL_UNREACHABLE)
# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure # error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure
#endif # endif
#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE() # define unreachable() JEMALLOC_INTERNAL_UNREACHABLE()
#endif #endif
/* Set error code. */ /* Set error code. */
@@ -69,17 +69,17 @@ get_errno(void) {
} }
#ifdef _MSC_VER #ifdef _MSC_VER
#define util_assume __assume # define util_assume __assume
#elif defined(__clang__) && (__clang_major__ > 3 || \ #elif defined(__clang__) \
(__clang_major__ == 3 && __clang_minor__ >= 6)) && (__clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6))
#define util_assume __builtin_assume # define util_assume __builtin_assume
#else #else
#define util_assume(expr) \ # define util_assume(expr) \
do { \ do { \
if (!(expr)) { \ if (!(expr)) { \
unreachable(); \ unreachable(); \
} \ } \
} while(0) } while (0)
#endif #endif
/* Allows compiler constant folding on inlined paths. */ /* Allows compiler constant folding on inlined paths. */
@@ -148,7 +148,6 @@ util_prefetch_write_range(void *ptr, size_t sz) {
* key1-key2:value|key3-key4:value|... * key1-key2:value|key3-key4:value|...
* Note it does not handle the ending '\0'. * Note it does not handle the ending '\0'.
*/ */
bool bool multi_setting_parse_next(const char **setting_segment_cur,
multi_setting_parse_next(const char **setting_segment_cur, size_t *len_left, size_t *len_left, size_t *key_start, size_t *key_end, size_t *value);
size_t *key_start, size_t *key_end, size_t *value);
#endif /* JEMALLOC_INTERNAL_UTIL_H */ #endif /* JEMALLOC_INTERNAL_UTIL_H */

54
include/jemalloc/internal/witness.h
View File

@@ -66,8 +66,8 @@ enum witness_rank_e {
WITNESS_RANK_HOOK, WITNESS_RANK_HOOK,
WITNESS_RANK_BIN, WITNESS_RANK_BIN,
WITNESS_RANK_LEAF=0x1000, WITNESS_RANK_LEAF = 0x1000,
WITNESS_RANK_BATCHER=WITNESS_RANK_LEAF, WITNESS_RANK_BATCHER = WITNESS_RANK_LEAF,
WITNESS_RANK_ARENA_STATS = WITNESS_RANK_LEAF, WITNESS_RANK_ARENA_STATS = WITNESS_RANK_LEAF,
WITNESS_RANK_COUNTER_ACCUM = WITNESS_RANK_LEAF, WITNESS_RANK_COUNTER_ACCUM = WITNESS_RANK_LEAF,
WITNESS_RANK_DSS = WITNESS_RANK_LEAF, WITNESS_RANK_DSS = WITNESS_RANK_LEAF,
@@ -86,15 +86,20 @@ typedef enum witness_rank_e witness_rank_t;
/* PER-WITNESS DATA */ /* PER-WITNESS DATA */
/******************************************************************************/ /******************************************************************************/
#if defined(JEMALLOC_DEBUG) #if defined(JEMALLOC_DEBUG)
# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}} # define WITNESS_INITIALIZER(name, rank) \
{ \
name, rank, NULL, NULL, { \
NULL, NULL \
} \
}
#else #else
# define WITNESS_INITIALIZER(name, rank) # define WITNESS_INITIALIZER(name, rank)
#endif #endif
typedef struct witness_s witness_t; typedef struct witness_s witness_t;
typedef ql_head(witness_t) witness_list_t; typedef ql_head(witness_t) witness_list_t;
typedef int witness_comp_t (const witness_t *, void *, const witness_t *, typedef int witness_comp_t(
void *); const witness_t *, void *, const witness_t *, void *);
struct witness_s { struct witness_s {
/* Name, used for printing lock order reversal messages. */ /* Name, used for printing lock order reversal messages. */
@@ -129,7 +134,8 @@ struct witness_tsd_s {
bool forking; bool forking;
}; };
#define WITNESS_TSD_INITIALIZER { ql_head_initializer(witnesses), false } #define WITNESS_TSD_INITIALIZER \
{ ql_head_initializer(witnesses), false }
#define WITNESS_TSDN_NULL ((witness_tsdn_t *)0) #define WITNESS_TSDN_NULL ((witness_tsdn_t *)0)
/******************************************************************************/ /******************************************************************************/
@@ -162,17 +168,17 @@ witness_tsdn_tsd(witness_tsdn_t *witness_tsdn) {
void witness_init(witness_t *witness, const char *name, witness_rank_t rank, void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
witness_comp_t *comp, void *opaque); witness_comp_t *comp, void *opaque);
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); typedef void(witness_lock_error_t)(const witness_list_t *, const witness_t *);
extern witness_lock_error_t *JET_MUTABLE witness_lock_error; extern witness_lock_error_t *JET_MUTABLE witness_lock_error;
typedef void (witness_owner_error_t)(const witness_t *); typedef void(witness_owner_error_t)(const witness_t *);
extern witness_owner_error_t *JET_MUTABLE witness_owner_error; extern witness_owner_error_t *JET_MUTABLE witness_owner_error;
typedef void (witness_not_owner_error_t)(const witness_t *); typedef void(witness_not_owner_error_t)(const witness_t *);
extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error; extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error;
typedef void (witness_depth_error_t)(const witness_list_t *, typedef void(witness_depth_error_t)(
witness_rank_t rank_inclusive, unsigned depth); const witness_list_t *, witness_rank_t rank_inclusive, unsigned depth);
extern witness_depth_error_t *JET_MUTABLE witness_depth_error; extern witness_depth_error_t *JET_MUTABLE witness_depth_error;
void witnesses_cleanup(witness_tsd_t *witness_tsd); void witnesses_cleanup(witness_tsd_t *witness_tsd);
@@ -189,7 +195,7 @@ witness_owner(witness_tsd_t *witness_tsd, const witness_t *witness) {
cassert(config_debug); cassert(config_debug);
witnesses = &witness_tsd->witnesses; witnesses = &witness_tsd->witnesses;
ql_foreach(w, witnesses, link) { ql_foreach (w, witnesses, link) {
if (w == witness) { if (w == witness) {
return true; return true;
} }
@@ -221,8 +227,8 @@ witness_assert_owner(witness_tsdn_t *witness_tsdn, const witness_t *witness) {
} }
static inline void static inline void
witness_assert_not_owner(witness_tsdn_t *witness_tsdn, witness_assert_not_owner(
const witness_t *witness) { witness_tsdn_t *witness_tsdn, const witness_t *witness) {
witness_tsd_t *witness_tsd; witness_tsd_t *witness_tsd;
witness_list_t *witnesses; witness_list_t *witnesses;
witness_t *w; witness_t *w;
@@ -240,7 +246,7 @@ witness_assert_not_owner(witness_tsdn_t *witness_tsdn,
} }
witnesses = &witness_tsd->witnesses; witnesses = &witness_tsd->witnesses;
ql_foreach(w, witnesses, link) { ql_foreach (w, witnesses, link) {
if (w == witness) { if (w == witness) {
witness_not_owner_error(witness); witness_not_owner_error(witness);
} }
@@ -249,8 +255,8 @@ witness_assert_not_owner(witness_tsdn_t *witness_tsdn,
/* Returns depth. Not intended for direct use. */ /* Returns depth. Not intended for direct use. */
static inline unsigned static inline unsigned
witness_depth_to_rank(witness_list_t *witnesses, witness_rank_t rank_inclusive) witness_depth_to_rank(
{ witness_list_t *witnesses, witness_rank_t rank_inclusive) {
unsigned d = 0; unsigned d = 0;
witness_t *w = ql_last(witnesses, link); witness_t *w = ql_last(witnesses, link);
@@ -292,8 +298,8 @@ witness_assert_lockless(witness_tsdn_t *witness_tsdn) {
} }
static inline void static inline void
witness_assert_positive_depth_to_rank(witness_tsdn_t *witness_tsdn, witness_assert_positive_depth_to_rank(
witness_rank_t rank_inclusive) { witness_tsdn_t *witness_tsdn, witness_rank_t rank_inclusive) {
if (!config_debug || witness_tsdn_null(witness_tsdn)) { if (!config_debug || witness_tsdn_null(witness_tsdn)) {
return; return;
} }
@@ -335,9 +341,9 @@ witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
} else if (w->rank > witness->rank) { } else if (w->rank > witness->rank) {
/* Not forking, rank order reversal. */ /* Not forking, rank order reversal. */
witness_lock_error(witnesses, witness); witness_lock_error(witnesses, witness);
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp != } else if (w->rank == witness->rank
witness->comp || w->comp(w, w->opaque, witness, witness->opaque) > && (w->comp == NULL || w->comp != witness->comp
0)) { || w->comp(w, w->opaque, witness, witness->opaque) > 0)) {
/* /*
* Missing/incompatible comparison function, or comparison * Missing/incompatible comparison function, or comparison
* function indicates rank order reversal. * function indicates rank order reversal.
@@ -346,8 +352,8 @@ witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) {
} }
/* Suppress spurious warning from static analysis */ /* Suppress spurious warning from static analysis */
assert(ql_empty(witnesses) || assert(
qr_prev(ql_first(witnesses), link) != NULL); ql_empty(witnesses) || qr_prev(ql_first(witnesses), link) != NULL);
ql_elm_new(witness, link); ql_elm_new(witness, link);
ql_tail_insert(witnesses, witness, link); ql_tail_insert(witnesses, witness, link);
} }

218
include/msvc_compat/C99/stdint.h
View File

@@ -30,39 +30,39 @@
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
#ifndef _MSC_VER // [ #ifndef _MSC_VER // [
#error "Use this header only with Microsoft Visual C++ compilers!" # error "Use this header only with Microsoft Visual C++ compilers!"
#endif // _MSC_VER ] #endif // _MSC_VER ]
#ifndef _MSC_STDINT_H_ // [ #ifndef _MSC_STDINT_H_ // [
#define _MSC_STDINT_H_ # define _MSC_STDINT_H_
#if _MSC_VER > 1000 # if _MSC_VER > 1000
#pragma once # pragma once
#endif # endif
#include <limits.h> # include <limits.h>
// For Visual Studio 6 in C++ mode and for many Visual Studio versions when // For Visual Studio 6 in C++ mode and for many Visual Studio versions when
// compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}' // compiling for ARM we should wrap <wchar.h> include with 'extern "C++" {}'
// or compiler give many errors like this: // or compiler give many errors like this:
// error C2733: second C linkage of overloaded function 'wmemchr' not allowed // error C2733: second C linkage of overloaded function 'wmemchr' not allowed
#ifdef __cplusplus # ifdef __cplusplus
extern "C" { extern "C" {
#endif # endif
# include <wchar.h> # include <wchar.h>
#ifdef __cplusplus # ifdef __cplusplus
} }
#endif # endif
// Define _W64 macros to mark types changing their size, like intptr_t. // Define _W64 macros to mark types changing their size, like intptr_t.
#ifndef _W64 # ifndef _W64
# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 # if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) \
&& _MSC_VER >= 1300
# define _W64 __w64 # define _W64 __w64
# else # else
# define _W64 # define _W64
# endif # endif
#endif # endif
// 7.18.1 Integer types // 7.18.1 Integer types
@@ -71,25 +71,24 @@ extern "C" {
// Visual Studio 6 and Embedded Visual C++ 4 doesn't // Visual Studio 6 and Embedded Visual C++ 4 doesn't
// realize that, e.g. char has the same size as __int8 // realize that, e.g. char has the same size as __int8
// so we give up on __intX for them. // so we give up on __intX for them.
#if (_MSC_VER < 1300) # if (_MSC_VER < 1300)
typedef signed char int8_t; typedef signed char int8_t;
typedef signed short int16_t; typedef signed short int16_t;
typedef signed int int32_t; typedef signed int int32_t;
typedef unsigned char uint8_t; typedef unsigned char uint8_t;
typedef unsigned short uint16_t; typedef unsigned short uint16_t;
typedef unsigned int uint32_t; typedef unsigned int uint32_t;
#else # else
typedef signed __int8 int8_t; typedef signed __int8 int8_t;
typedef signed __int16 int16_t; typedef signed __int16 int16_t;
typedef signed __int32 int32_t; typedef signed __int32 int32_t;
typedef unsigned __int8 uint8_t; typedef unsigned __int8 uint8_t;
typedef unsigned __int16 uint16_t; typedef unsigned __int16 uint16_t;
typedef unsigned __int32 uint32_t; typedef unsigned __int32 uint32_t;
#endif # endif
typedef signed __int64 int64_t; typedef signed __int64 int64_t;
typedef unsigned __int64 uint64_t; typedef unsigned __int64 uint64_t;
// 7.18.1.2 Minimum-width integer types // 7.18.1.2 Minimum-width integer types
typedef int8_t int_least8_t; typedef int8_t int_least8_t;
typedef int16_t int_least16_t; typedef int16_t int_least16_t;
@@ -111,137 +110,138 @@ typedef uint32_t uint_fast32_t;
typedef uint64_t uint_fast64_t; typedef uint64_t uint_fast64_t;
// 7.18.1.4 Integer types capable of holding object pointers // 7.18.1.4 Integer types capable of holding object pointers
#ifdef _WIN64 // [ # ifdef _WIN64 // [
typedef signed __int64 intptr_t; typedef signed __int64 intptr_t;
typedef unsigned __int64 uintptr_t; typedef unsigned __int64 uintptr_t;
#else // _WIN64 ][ # else // _WIN64 ][
typedef _W64 signed int intptr_t; typedef _W64 signed int intptr_t;
typedef _W64 unsigned int uintptr_t; typedef _W64 unsigned int uintptr_t;
#endif // _WIN64 ] # endif // _WIN64 ]
// 7.18.1.5 Greatest-width integer types // 7.18.1.5 Greatest-width integer types
typedef int64_t intmax_t; typedef int64_t intmax_t;
typedef uint64_t uintmax_t; typedef uint64_t uintmax_t;
// 7.18.2 Limits of specified-width integer types // 7.18.2 Limits of specified-width integer types
#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 # if !defined(__cplusplus) \
|| defined( \
__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259
// 7.18.2.1 Limits of exact-width integer types // 7.18.2.1 Limits of exact-width integer types
#define INT8_MIN ((int8_t)_I8_MIN) # define INT8_MIN ((int8_t)_I8_MIN)
#define INT8_MAX _I8_MAX # define INT8_MAX _I8_MAX
#define INT16_MIN ((int16_t)_I16_MIN) # define INT16_MIN ((int16_t)_I16_MIN)
#define INT16_MAX _I16_MAX # define INT16_MAX _I16_MAX
#define INT32_MIN ((int32_t)_I32_MIN) # define INT32_MIN ((int32_t)_I32_MIN)
#define INT32_MAX _I32_MAX # define INT32_MAX _I32_MAX
#define INT64_MIN ((int64_t)_I64_MIN) # define INT64_MIN ((int64_t)_I64_MIN)
#define INT64_MAX _I64_MAX # define INT64_MAX _I64_MAX
#define UINT8_MAX _UI8_MAX # define UINT8_MAX _UI8_MAX
#define UINT16_MAX _UI16_MAX # define UINT16_MAX _UI16_MAX
#define UINT32_MAX _UI32_MAX # define UINT32_MAX _UI32_MAX
#define UINT64_MAX _UI64_MAX # define UINT64_MAX _UI64_MAX
// 7.18.2.2 Limits of minimum-width integer types // 7.18.2.2 Limits of minimum-width integer types
#define INT_LEAST8_MIN INT8_MIN # define INT_LEAST8_MIN INT8_MIN
#define INT_LEAST8_MAX INT8_MAX # define INT_LEAST8_MAX INT8_MAX
#define INT_LEAST16_MIN INT16_MIN # define INT_LEAST16_MIN INT16_MIN
#define INT_LEAST16_MAX INT16_MAX # define INT_LEAST16_MAX INT16_MAX
#define INT_LEAST32_MIN INT32_MIN # define INT_LEAST32_MIN INT32_MIN
#define INT_LEAST32_MAX INT32_MAX # define INT_LEAST32_MAX INT32_MAX
#define INT_LEAST64_MIN INT64_MIN # define INT_LEAST64_MIN INT64_MIN
#define INT_LEAST64_MAX INT64_MAX # define INT_LEAST64_MAX INT64_MAX
#define UINT_LEAST8_MAX UINT8_MAX # define UINT_LEAST8_MAX UINT8_MAX
#define UINT_LEAST16_MAX UINT16_MAX # define UINT_LEAST16_MAX UINT16_MAX
#define UINT_LEAST32_MAX UINT32_MAX # define UINT_LEAST32_MAX UINT32_MAX
#define UINT_LEAST64_MAX UINT64_MAX # define UINT_LEAST64_MAX UINT64_MAX
// 7.18.2.3 Limits of fastest minimum-width integer types // 7.18.2.3 Limits of fastest minimum-width integer types
#define INT_FAST8_MIN INT8_MIN # define INT_FAST8_MIN INT8_MIN
#define INT_FAST8_MAX INT8_MAX # define INT_FAST8_MAX INT8_MAX
#define INT_FAST16_MIN INT16_MIN # define INT_FAST16_MIN INT16_MIN
#define INT_FAST16_MAX INT16_MAX # define INT_FAST16_MAX INT16_MAX
#define INT_FAST32_MIN INT32_MIN # define INT_FAST32_MIN INT32_MIN
#define INT_FAST32_MAX INT32_MAX # define INT_FAST32_MAX INT32_MAX
#define INT_FAST64_MIN INT64_MIN # define INT_FAST64_MIN INT64_MIN
#define INT_FAST64_MAX INT64_MAX # define INT_FAST64_MAX INT64_MAX
#define UINT_FAST8_MAX UINT8_MAX # define UINT_FAST8_MAX UINT8_MAX
#define UINT_FAST16_MAX UINT16_MAX # define UINT_FAST16_MAX UINT16_MAX
#define UINT_FAST32_MAX UINT32_MAX # define UINT_FAST32_MAX UINT32_MAX
#define UINT_FAST64_MAX UINT64_MAX # define UINT_FAST64_MAX UINT64_MAX
// 7.18.2.4 Limits of integer types capable of holding object pointers // 7.18.2.4 Limits of integer types capable of holding object pointers
#ifdef _WIN64 // [ # ifdef _WIN64 // [
# define INTPTR_MIN INT64_MIN # define INTPTR_MIN INT64_MIN
# define INTPTR_MAX INT64_MAX # define INTPTR_MAX INT64_MAX
# define UINTPTR_MAX UINT64_MAX # define UINTPTR_MAX UINT64_MAX
#else // _WIN64 ][ # else // _WIN64 ][
# define INTPTR_MIN INT32_MIN # define INTPTR_MIN INT32_MIN
# define INTPTR_MAX INT32_MAX # define INTPTR_MAX INT32_MAX
# define UINTPTR_MAX UINT32_MAX # define UINTPTR_MAX UINT32_MAX
#endif // _WIN64 ] # endif // _WIN64 ]
// 7.18.2.5 Limits of greatest-width integer types // 7.18.2.5 Limits of greatest-width integer types
#define INTMAX_MIN INT64_MIN # define INTMAX_MIN INT64_MIN
#define INTMAX_MAX INT64_MAX # define INTMAX_MAX INT64_MAX
#define UINTMAX_MAX UINT64_MAX # define UINTMAX_MAX UINT64_MAX
// 7.18.3 Limits of other integer types // 7.18.3 Limits of other integer types
#ifdef _WIN64 // [ # ifdef _WIN64 // [
# define PTRDIFF_MIN _I64_MIN # define PTRDIFF_MIN _I64_MIN
# define PTRDIFF_MAX _I64_MAX # define PTRDIFF_MAX _I64_MAX
#else // _WIN64 ][ # else // _WIN64 ][
# define PTRDIFF_MIN _I32_MIN # define PTRDIFF_MIN _I32_MIN
# define PTRDIFF_MAX _I32_MAX # define PTRDIFF_MAX _I32_MAX
#endif // _WIN64 ] # endif // _WIN64 ]
#define SIG_ATOMIC_MIN INT_MIN # define SIG_ATOMIC_MIN INT_MIN
#define SIG_ATOMIC_MAX INT_MAX # define SIG_ATOMIC_MAX INT_MAX
#ifndef SIZE_MAX // [ # ifndef SIZE_MAX // [
# ifdef _WIN64 // [ # ifdef _WIN64 // [
# define SIZE_MAX _UI64_MAX # define SIZE_MAX _UI64_MAX
# else // _WIN64 ][ # else // _WIN64 ][
# define SIZE_MAX _UI32_MAX # define SIZE_MAX _UI32_MAX
# endif // _WIN64 ] # endif // _WIN64 ]
#endif // SIZE_MAX ] # endif // SIZE_MAX ]
// WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h> // WCHAR_MIN and WCHAR_MAX are also defined in <wchar.h>
#ifndef WCHAR_MIN // [ # ifndef WCHAR_MIN // [
# define WCHAR_MIN 0 # define WCHAR_MIN 0
#endif // WCHAR_MIN ] # endif // WCHAR_MIN ]
#ifndef WCHAR_MAX // [ # ifndef WCHAR_MAX // [
# define WCHAR_MAX _UI16_MAX # define WCHAR_MAX _UI16_MAX
#endif // WCHAR_MAX ] # endif // WCHAR_MAX ]
#define WINT_MIN 0 # define WINT_MIN 0
#define WINT_MAX _UI16_MAX # define WINT_MAX _UI16_MAX
#endif // __STDC_LIMIT_MACROS ]
# endif // __STDC_LIMIT_MACROS ]
// 7.18.4 Limits of other integer types // 7.18.4 Limits of other integer types
#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 # if !defined(__cplusplus) \
|| defined( \
__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260
// 7.18.4.1 Macros for minimum-width integer constants // 7.18.4.1 Macros for minimum-width integer constants
#define INT8_C(val) val##i8 # define INT8_C(val) val##i8
#define INT16_C(val) val##i16 # define INT16_C(val) val##i16
#define INT32_C(val) val##i32 # define INT32_C(val) val##i32
#define INT64_C(val) val##i64 # define INT64_C(val) val##i64
#define UINT8_C(val) val##ui8 # define UINT8_C(val) val##ui8
#define UINT16_C(val) val##ui16 # define UINT16_C(val) val##ui16
#define UINT32_C(val) val##ui32 # define UINT32_C(val) val##ui32
#define UINT64_C(val) val##ui64 # define UINT64_C(val) val##ui64
// 7.18.4.2 Macros for greatest-width integer constants // 7.18.4.2 Macros for greatest-width integer constants
#define INTMAX_C INT64_C # define INTMAX_C INT64_C
#define UINTMAX_C UINT64_C # define UINTMAX_C UINT64_C
#endif // __STDC_CONSTANT_MACROS ]
# endif // __STDC_CONSTANT_MACROS ]
#endif // _MSC_STDINT_H_ ] #endif // _MSC_STDINT_H_ ]

Some files were not shown because too many files have changed in this diff Show More