Expose stats table columns in JSON output and simplify code

The extents and hpa-nonfull "ind" columns are emitter_type_unsigned but were
set via col_ind.size_val -- correct only by little-endian union overlap.
Output is byte-identical.  Pre-existing (predates the refactor).

.github/workflows/linux-ci.yml

@@ -508,6 +508,16 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: force_tls=0
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: "force_tls=0 --enable-debug"
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
           - env:
               CC: gcc
               CXX: g++

INSTALL.md

@@ -139,7 +139,6 @@ any of the following arguments (not a definitive list) to 'configure':
     in the following list that appears to function correctly:
 
     + libunwind      (requires --enable-prof-libunwind)
-    + frame pointer  (requires --enable-prof-frameptr)
     + libgcc         (unless --disable-prof-libgcc)
     + gcc intrinsics (unless --disable-prof-gcc)
 
@@ -148,12 +147,6 @@ any of the following arguments (not a definitive list) to 'configure':
     Use the libunwind library (http://www.nongnu.org/libunwind/) for stack
     backtracing.
 
-* `--enable-prof-frameptr`
-
-    Use the optimized frame pointer unwinder for stack backtracing. Safe
-    to use in mixed code (with and without frame pointers) - but requires
-    frame pointers to produce meaningful stacks. Linux only.
-
 * `--disable-prof-libgcc`
 
     Disable the use of libgcc's backtracing functionality.

Makefile.in

@@ -146,7 +146,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/prof_data.c \
 	$(srcroot)src/prof_log.c \
 	$(srcroot)src/prof_recent.c \
-	$(srcroot)src/prof_stack_range.c \
 	$(srcroot)src/prof_stats.c \
 	$(srcroot)src/prof_sys.c \
 	$(srcroot)src/psset.c \
@@ -204,7 +203,6 @@ C_UTIL_CPP_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c
 endif
 TESTS_UNIT := \
 	$(srcroot)test/unit/a0.c \
-	$(srcroot)test/unit/arena_decay.c \
 	$(srcroot)test/unit/arena_reset.c \
 	$(srcroot)test/unit/arenas_management.c \
 	$(srcroot)test/unit/atomic.c \
@@ -272,10 +270,12 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
 	$(srcroot)test/unit/pac.c \
+	$(srcroot)test/unit/pac_decay.c \
 	$(srcroot)test/unit/pac_sec_integration.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/peak.c \
+	$(srcroot)test/unit/percpu_arena_resume.c \
 	$(srcroot)test/unit/ph.c \
 	$(srcroot)test/unit/prng.c \
 	$(srcroot)test/unit/prof_accum.c \

configure.ac

@@ -533,13 +533,14 @@ esac
 AC_DEFINE_UNQUOTED([HAVE_CPU_SPINWAIT], [$HAVE_CPU_SPINWAIT], [ ])
 AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT], [ ])
 
+dnl LG_VADDR: number of significant virtual address bits. Auto-detected when --with-lg-vaddr is left unset.
 AC_ARG_WITH([lg_vaddr],
   [AS_HELP_STRING([--with-lg-vaddr=<lg-vaddr>], [Number of significant virtual address bits])],
   [LG_VADDR="$with_lg_vaddr"], [LG_VADDR="detect"])
 
-case "${host_cpu}" in
-  aarch64)
-    if test "x$LG_VADDR" = "xdetect"; then
+if test "x$LG_VADDR" = "xdetect"; then
+  case "${host_cpu}" in
+    aarch64)
       AC_MSG_CHECKING([number of significant virtual address bits])
       if test "x${LG_SIZEOF_PTR}" = "x2" ; then
         #aarch64 ILP32
@@ -549,10 +550,8 @@ case "${host_cpu}" in
         LG_VADDR=48
       fi
       AC_MSG_RESULT([$LG_VADDR])
-    fi
-    ;;
-  x86_64)
-    if test "x$LG_VADDR" = "xdetect"; then
+      ;;
+    x86_64)
       AC_CACHE_CHECK([number of significant virtual address bits],
                      [je_cv_lg_vaddr],
                      AC_RUN_IFELSE([AC_LANG_PROGRAM(
@@ -592,15 +591,33 @@ typedef unsigned __int32 uint32_t;
                    [je_cv_lg_vaddr=`cat conftest.out`],
                    [je_cv_lg_vaddr=error],
                    [je_cv_lg_vaddr=57]))
+      if test "x${je_cv_lg_vaddr}" = "xerror" ; then
+        AC_MSG_ERROR([cannot determine number of significant virtual address bits])
+      fi
       if test "x${je_cv_lg_vaddr}" != "x" ; then
         LG_VADDR="${je_cv_lg_vaddr}"
       fi
-      if test "x${LG_VADDR}" != "xerror" ; then
-        AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
+      ;;
+    *)
+      AC_MSG_CHECKING([number of significant virtual address bits])
+      if test "x${LG_SIZEOF_PTR}" = "x3" ; then
+        LG_VADDR=64
+      elif test "x${LG_SIZEOF_PTR}" = "x2" ; then
+        LG_VADDR=32
+      elif test "x${LG_SIZEOF_PTR}" = "xLG_SIZEOF_PTR_WIN" ; then
+        LG_VADDR="(1U << (LG_SIZEOF_PTR_WIN+3))"
       else
-        AC_MSG_ERROR([cannot determine number of significant virtual address bits])
+        AC_MSG_ERROR([Unsupported lg(pointer size): ${LG_SIZEOF_PTR}])
       fi
-    fi
+      AC_MSG_RESULT([$LG_VADDR])
+      ;;
+  esac
+fi
+AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
+
+dnl rdtscp feature probe for x86_64
+case "${host_cpu}" in
+  x86_64)
     AC_CACHE_CHECK([rdtscp support],
 		   [je_cv_rdtscp],
 		   AC_RUN_IFELSE([AC_LANG_PROGRAM(
@@ -619,23 +636,8 @@ typedef unsigned __int32 uint32_t;
       AC_DEFINE([JEMALLOC_HAVE_RDTSCP], [ ], [ ])
     fi
     ;;
-  *)
-    if test "x$LG_VADDR" = "xdetect"; then
-      AC_MSG_CHECKING([number of significant virtual address bits])
-      if test "x${LG_SIZEOF_PTR}" = "x3" ; then
-        LG_VADDR=64
-      elif test "x${LG_SIZEOF_PTR}" = "x2" ; then
-        LG_VADDR=32
-      elif test "x${LG_SIZEOF_PTR}" = "xLG_SIZEOF_PTR_WIN" ; then
-        LG_VADDR="(1U << (LG_SIZEOF_PTR_WIN+3))"
-      else
-        AC_MSG_ERROR([Unsupported lg(pointer size): ${LG_SIZEOF_PTR}])
-      fi
-      AC_MSG_RESULT([$LG_VADDR])
-    fi
-    ;;
 esac
-AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
+
 AC_CACHE_CHECK([asm volatile support],
                [je_cv_asm_volatile],
                AC_RUN_IFELSE([AC_LANG_PROGRAM(
@@ -1555,33 +1557,6 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then
   fi
 fi
 
-if test `uname -s` = "Linux"
-then
-  AC_ARG_ENABLE([prof-frameptr],
-    [AS_HELP_STRING([--enable-prof-frameptr], [Use optimized frame pointer unwinder for backtracing (Linux only)])],
-  [if test "x$enable_prof_frameptr" = "xno" ; then
-    enable_prof_frameptr="0"
-  else
-    enable_prof_frameptr="1"
-    if test "x$enable_prof" = "x0" ; then
-      AC_MSG_ERROR([--enable-prof-frameptr should only be used with --enable-prof])
-    fi
-  fi
-  ],
-  [enable_prof_frameptr="0"]
-  )
-  if test "x$backtrace_method" = "x" -a "x$enable_prof_frameptr" = "x1" \
-      -a "x$GCC" = "xyes" ; then
-    JE_CFLAGS_ADD([-fno-omit-frame-pointer])
-    backtrace_method="frame pointer linux"
-    AC_DEFINE([JEMALLOC_PROF_FRAME_POINTER], [ ], [ ])
-  else
-    enable_prof_frameptr="0"
-  fi
-else
-  enable_prof_frameptr="0"
-fi
-
 AC_ARG_ENABLE([prof-libgcc],
   [AS_HELP_STRING([--disable-prof-libgcc],
   [Do not use libgcc for backtracing])],
@@ -2224,6 +2199,43 @@ dnl Check if we have dlsym support.
   if test "x${je_cv_pthread_get_name_np}" = "xyes" ; then
     AC_DEFINE([JEMALLOC_HAVE_PTHREAD_GET_NAME_NP], [ ], [ ])
   fi
+  dnl Runtime-check pthread_condattr_setclock and clock_gettime with
+  dnl CLOCK_MONOTONIC to confirm the kernel actually supports the
+  dnl monotonic clock.
+  AC_CACHE_CHECK([whether pthread_condattr_setclock with CLOCK_MONOTONIC is usable],
+                 [je_cv_pthread_cond_timedwait_monotonic],
+                 AC_RUN_IFELSE([AC_LANG_PROGRAM(
+[[
+#include <pthread.h>
+#include <time.h>
+]], [[
+  pthread_condattr_t attr;
+  pthread_cond_t cond;
+  struct timespec ts;
+  int ret = pthread_condattr_init(&attr);
+  if (ret != 0) {
+    return 1;
+  }
+  ret = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC);
+  if (ret != 0) {
+    pthread_condattr_destroy(&attr);
+    return 1;
+  }
+  ret = pthread_cond_init(&cond, &attr);
+  pthread_condattr_destroy(&attr);
+  if (ret != 0) {
+    return 1;
+  }
+  ret = clock_gettime(CLOCK_MONOTONIC, &ts);
+  pthread_cond_destroy(&cond);
+  return ret != 0;
+]])],
+                 [je_cv_pthread_cond_timedwait_monotonic=yes],
+                 [je_cv_pthread_cond_timedwait_monotonic=no],
+                 [je_cv_pthread_cond_timedwait_monotonic=no]))
+  if test "x${je_cv_pthread_cond_timedwait_monotonic}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC], [ ], [ ])
+  fi
 fi
 
 JE_APPEND_VS(CPPFLAGS, -D_REENTRANT)
@@ -3147,7 +3159,6 @@ AC_MSG_RESULT([user_config        : ${enable_user_config}])
 AC_MSG_RESULT([experimental_smallocx : ${enable_experimental_smallocx}])
 AC_MSG_RESULT([prof               : ${enable_prof}])
 AC_MSG_RESULT([prof-libunwind     : ${enable_prof_libunwind}])
-AC_MSG_RESULT([prof-frameptr      : ${enable_prof_frameptr}])
 AC_MSG_RESULT([prof-libgcc        : ${enable_prof_libgcc}])
 AC_MSG_RESULT([prof-gcc           : ${enable_prof_gcc}])
 AC_MSG_RESULT([fill               : ${enable_fill}])

include/jemalloc/internal/arena.h

@@ -177,12 +177,6 @@ struct arena_s {
 /* EXTERNS */
 /******************************************************************************/
 
-/*
- * When the amount of pages to be purged exceeds this amount, deferred purge
- * should happen.
- */
-#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
-
 extern ssize_t opt_dirty_decay_ms;
 extern ssize_t opt_muzzy_decay_ms;
 
@@ -213,7 +207,6 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
     hpa_shard_stats_t *hpastats);
-void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
 edata_t *arena_extent_alloc_large(
     tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
 void arena_extent_dalloc_large_prep(
@@ -222,13 +215,6 @@ void arena_extent_ralloc_large_shrink(
     tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
 void arena_extent_ralloc_large_expand(
     tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
-bool arena_decay_ms_set(
-    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
-ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
-void    arena_decay(
-       tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-uint64_t       arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
-void           arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
 void           arena_reset(tsd_t *tsd, arena_t *arena);
 void           arena_destroy(tsd_t *tsd, arena_t *arena);
 cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,

include/jemalloc/internal/arena_inlines.h

@@ -308,7 +308,7 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
 	uint64_t      *prng_state = tsd_prng_statep_get(tsd);
 	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
 	        tsd_reentrancy_level_get(tsd) > 0))) {
-		arena_decay(tsdn, arena, false, false);
+		pa_shard_do_deferred_work(tsdn, &arena->pa_shard, false);
 	}
 }
 

include/jemalloc/internal/background_thread.h

@@ -13,17 +13,6 @@
 #define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT
 #define DEFAULT_NUM_BACKGROUND_THREAD 4
 
-/*
- * These exist only as a transitional state.  Eventually, deferral should be
- * part of the PAI, and each implementation can indicate wait times with more
- * specificity.
- */
-#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_UNINITIALIZED (-2)
-#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000
-
-#define BACKGROUND_THREAD_DEFERRED_MIN UINT64_C(0)
-#define BACKGROUND_THREAD_DEFERRED_MAX UINT64_MAX
-
 typedef enum {
 	background_thread_stopped,
 	background_thread_started,
@@ -87,6 +76,25 @@ bool background_thread_stats_read(
     tsdn_t *tsdn, background_thread_stats_t *stats);
 void background_thread_ctl_init(tsdn_t *tsdn);
 
+/*
+ * Arena-reset lifecycle bracket (owned by the bg-thread module so the
+ * started<->paused state machine is mutated only inside the module).  begin()
+ * acquires the global background_thread_lock and RETURNS WITH IT HELD;
+ * finish() releases it.  The lock intentionally spans the whole arena-reset
+ * body across the two calls.  Pass arena_ind (not an arena_t *): the destroy
+ * path frees the arena before calling finish().
+ */
+void background_thread_arena_reset_begin(tsd_t *tsd, unsigned arena_ind);
+void background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind);
+
+/*
+ * Serialize a rare admin operation against the background thread by holding the
+ * per-arena info mutex (have_background_thread-gated).  Used by
+ * arena_set_extent_hooks to fence pa_shard_disable_hpa.
+ */
+void background_thread_serialize_lock(tsd_t *tsd, unsigned arena_ind);
+void background_thread_serialize_unlock(tsd_t *tsd, unsigned arena_ind);
+
 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
 extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
     void *(*)(void *), void *__restrict);

include/jemalloc/internal/background_thread_inlines.h

@@ -22,12 +22,6 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) {
 	background_thread_enabled_set_impl(state);
 }
 
-JEMALLOC_ALWAYS_INLINE background_thread_info_t *
-arena_background_thread_info_get(arena_t *arena) {
-	unsigned arena_ind = arena_ind_get(arena);
-	return &background_thread_info[arena_ind % max_background_threads];
-}
-
 JEMALLOC_ALWAYS_INLINE background_thread_info_t *
 background_thread_info_get(size_t ind) {
 	return &background_thread_info[ind % max_background_threads];

include/jemalloc/internal/deferral.h

@@ -0,0 +1,30 @@
+#ifndef JEMALLOC_INTERNAL_DEFERRAL_H
+#define JEMALLOC_INTERNAL_DEFERRAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+/*
+ * The deferred-work contract shared by the page allocators that PRODUCE
+ * deferred work (PAC decay-purge, HPA purge/hugify) and the drivers that
+ * CONSUME it (the background thread, or the application-inline fallback).
+ *
+ * There is deliberately no polymorphic interface here: with only two
+ * page allocators, a vtable's indirect calls are needless and inefficient.
+ * Dispatch is direct pac_*()/hpa_*(), so the contract is a convention
+ * documented here, not an enforced type.  The only shared code it needs is the
+ * constants below; each allocator keeps its own policy definition (functions
+ * and constants) in its own header.
+ *
+ * The convention every page allocator follows:
+ *   - <alloc>_time_until_deferred_work(): nanoseconds until its next
+ *     deferred work is due.  DEFERRED_WORK_MIN = "due now",
+ *     DEFERRED_WORK_MAX = "nothing pending"; anything between is a real
+ *     ns deadline.
+ *   - <alloc>_do_deferred_work(): perform whatever work is due now.
+ * The pa layer reports the soonest deadline across its allocators; the
+ * background thread sleeps until then, or indefinitely on DEFERRED_WORK_MAX.
+ */
+#define DEFERRED_WORK_MIN UINT64_C(0)
+#define DEFERRED_WORK_MAX UINT64_MAX
+
+#endif /* JEMALLOC_INTERNAL_DEFERRAL_H */

include/jemalloc/internal/emitter.h

@@ -58,6 +58,11 @@ struct emitter_col_s {
 		ssize_t     ssize_val;
 		const char *str_val;
 	};
+	/*
+	 * Optional JSON key.  When set, emitter_row() emits this column as a
+	 * "json_key": value pair in JSON mode; columns left NULL are table-only.
+	 */
+	const char *json_key;
 
 	/* Filled in by initialization. */
 	ql_elm(emitter_col_t) link;
@@ -79,6 +84,8 @@ struct emitter_s {
 	bool item_at_depth;
 	/* True if we emitted a key and will emit corresponding value next. */
 	bool emitted_key;
+	/* Gap-collapsing state for the emitter_table_sparse_row() bracket. */
+	bool table_sparse_in_gap;
 };
 
 static inline bool
@@ -270,6 +277,7 @@ emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
 	emitter->item_at_depth = false;
 	emitter->emitted_key = false;
 	emitter->nesting_depth = 0;
+	emitter->table_sparse_in_gap = false;
 }
 
 /******************************************************************************/
@@ -447,6 +455,46 @@ static inline void
 emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
 	ql_elm_new(col, link);
 	ql_tail_insert(&row->cols, col, link);
+	col->json_key = NULL;
+}
+
+/*
+ * Sparse table rows.  Some tables have one row per size class but most rows
+ * are all-zero; in table output those runs are collapsed to a single "---"
+ * separator.  Bracket such a table with emitter_table_sparse_begin() /
+ * _end() and emit each row with emitter_table_sparse_row(), passing whether
+ * that row is a gap (all-zero).  The emitter prints the separator on a
+ * gap->non-gap transition and at a trailing gap, and suppresses the gap rows
+ * themselves; callers no longer track gap state.  Table output only -- a
+ * no-op in JSON, where callers emit every row's object separately.
+ */
+static inline void
+emitter_table_sparse_begin(emitter_t *emitter) {
+	emitter->table_sparse_in_gap = false;
+}
+
+static inline void
+emitter_table_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) {
+	if (emitter->output != emitter_output_table) {
+		return;
+	}
+	bool prev_in_gap = emitter->table_sparse_in_gap;
+	emitter->table_sparse_in_gap = is_gap;
+	if (prev_in_gap && !is_gap) {
+		emitter_printf(emitter, "                     ---\n");
+	}
+	if (!is_gap) {
+		emitter_table_row(emitter, row);
+	}
+}
+
+static inline void
+emitter_table_sparse_end(emitter_t *emitter) {
+	if (emitter->output == emitter_output_table
+	    && emitter->table_sparse_in_gap) {
+		emitter_printf(emitter, "                     ---\n");
+	}
+	emitter->table_sparse_in_gap = false;
 }
 
 /******************************************************************************/
@@ -499,6 +547,142 @@ emitter_dict_end(emitter_t *emitter) {
 	}
 }
 
+/* Internal: emit a row's json-keyed columns as JSON kvs, in column order. */
+static inline void
+emitter_row_json(emitter_t *emitter, emitter_row_t *row) {
+	emitter_col_t *col;
+	ql_foreach (col, &row->cols, link) {
+		if (col->json_key != NULL) {
+			emitter_json_kv(emitter, col->json_key, col->type,
+			    (const void *)&col->bool_val);
+		}
+	}
+}
+
+/*
+ * Emit a row of columns to whichever format is active.  In table mode, prints
+ * the aligned table row (see emitter_table_row).  In JSON mode, emits each
+ * column that has a non-NULL json_key as a "json_key": value pair, in column
+ * order (columns without a json_key are table-only).  This lets a caller
+ * describe a row once and render it to either format.  For JSON objects (one
+ * per row), wrap the call in emitter_json_object_begin()/_end().
+ */
+static inline void
+emitter_row(emitter_t *emitter, emitter_row_t *row) {
+	emitter_table_row(emitter, row);
+	if (emitter_outputs_json(emitter)) {
+		emitter_row_json(emitter, row);
+	}
+}
+
+/*
+ * Sparse counterpart of emitter_row() for per-size-class tables: the table
+ * side collapses all-zero ("gap") rows (see emitter_table_sparse_row), while
+ * the JSON side still emits every row's json-keyed columns.  Bracket the loop
+ * with emitter_table_sparse_begin()/_end(), and wrap each row's JSON in
+ * emitter_json_object_begin()/_end() when the rows form an array of objects.
+ */
+static inline void
+emitter_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) {
+	emitter_table_sparse_row(emitter, row, is_gap);
+	if (emitter_outputs_json(emitter)) {
+		emitter_row_json(emitter, row);
+	}
+}
+
+/*
+ * Descriptor-driven column tables.  A descriptor array defines table and JSON
+ * order; columns with a NULL json_key are table-only.  The getter lets one
+ * engine serve different row types while keeping derived values (rates,
+ * utilization, and so on) with the table-specific code.  A column is active
+ * when all its required_flags are present in the active_flags supplied by the
+ * caller; the emitter assigns no domain-specific meaning to those bits.
+ */
+typedef struct emitter_col_desc_s emitter_col_desc_t;
+struct emitter_col_desc_s {
+	const char       *json_key;
+	const char       *table_label;
+	emitter_justify_t justify;
+	int               width;
+	emitter_type_t    type;
+	unsigned          required_flags;
+	void            (*get)(const void *row, emitter_col_t *col);
+};
+
+static inline bool
+emitter_col_desc_active(const emitter_col_desc_t *desc,
+    unsigned active_flags) {
+	return (desc->required_flags & ~active_flags) == 0;
+}
+
+static inline void
+emitter_col_table_build(const emitter_col_desc_t *descs, size_t ndescs,
+    unsigned active_flags, emitter_row_t *row, emitter_col_t *cols,
+    emitter_row_t *header_row, emitter_col_t *header_cols) {
+	emitter_row_init(row);
+	emitter_row_init(header_row);
+	for (size_t i = 0; i < ndescs; i++) {
+		if (!emitter_col_desc_active(&descs[i], active_flags)) {
+			continue;
+		}
+
+		emitter_col_init(&cols[i], row);
+		cols[i].justify = descs[i].justify;
+		cols[i].width = descs[i].width;
+		cols[i].type = descs[i].type;
+
+		emitter_col_init(&header_cols[i], header_row);
+		header_cols[i].justify = descs[i].justify;
+		header_cols[i].width = descs[i].width;
+		header_cols[i].type = emitter_type_title;
+		header_cols[i].str_val = descs[i].table_label != NULL
+		    ? descs[i].table_label : descs[i].json_key;
+	}
+}
+
+static inline void
+emitter_col_table_header(emitter_t *emitter, emitter_row_t *header_row,
+    emitter_col_t *first_header_col, const char *table_prefix,
+    const char *json_key) {
+	first_header_col->width -= (int)strlen(table_prefix);
+	emitter_table_printf(emitter, "%s", table_prefix);
+	emitter_table_row(emitter, header_row);
+	emitter_json_array_kv_begin(emitter, json_key);
+}
+
+static inline void
+emitter_col_table_fill(const emitter_col_desc_t *descs, size_t ndescs,
+    unsigned active_flags, emitter_col_t *cols, const void *row) {
+	for (size_t i = 0; i < ndescs; i++) {
+		if (!emitter_col_desc_active(&descs[i], active_flags)) {
+			continue;
+		}
+		cols[i].type = descs[i].type;
+		descs[i].get(row, &cols[i]);
+	}
+}
+
+static inline void
+emitter_col_table_emit_json(emitter_t *emitter,
+    const emitter_col_desc_t *descs, size_t ndescs, unsigned active_flags,
+    emitter_col_t *cols) {
+	if (!emitter_outputs_json(emitter)) {
+		return;
+	}
+	for (size_t i = 0; i < ndescs; i++) {
+		const emitter_col_desc_t *desc = &descs[i];
+		if (!emitter_col_desc_active(desc, active_flags)) {
+			continue;
+		}
+		if (desc->json_key != NULL) {
+			emitter_type_t json_type = cols[i].type == emitter_type_title
+			    ? emitter_type_string : cols[i].type;
+			emitter_json_kv(emitter, desc->json_key, json_type,
+			    (const void *)&cols[i].bool_val);
+		}
+	}
+}
+
 static inline void
 emitter_begin(emitter_t *emitter) {
 	if (emitter_outputs_json(emitter)) {

include/jemalloc/internal/hpa.h

@@ -13,6 +13,14 @@
 #include "jemalloc/internal/psset.h"
 #include "jemalloc/internal/sec.h"
 
+/*
+ * HPA-specific deferral interval bounds (currently unused).  Deferral tuning
+ * is per-allocator policy, so it lives in the HPA header; see deferral.h for
+ * the shared deferred-work contract.
+ */
+#define HPA_INTERVAL_MAX_UNINITIALIZED (-2)
+#define HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000
+
 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
 struct hpa_shard_nonderived_stats_s {
 	/*

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -97,6 +97,9 @@
 /* Defined if pthread_get_name_np(3) is available. */
 #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP
 
+/* Defined if pthread_condattr_setclock(..., CLOCK_MONOTONIC) is available. */
+#undef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
@@ -172,9 +175,6 @@
 /* Use gcc intrinsics for profile backtracing if defined. */
 #undef JEMALLOC_PROF_GCC
 
-/* Use frame pointer for profile backtracing if defined. Linux only. */
-#undef JEMALLOC_PROF_FRAME_POINTER
-
 /* JEMALLOC_PAGEID enabled page id */
 #undef JEMALLOC_PAGEID
 

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -18,15 +18,34 @@ malloc_getcpu(void) {
 	return GetCurrentProcessorNumber();
 #elif defined(JEMALLOC_HAVE_SCHED_GETCPU)
 	return (malloc_cpuid_t)sched_getcpu();
+#elif defined(__APPLE__)
+	/*
+	 * No sched_getcpu() on macOS; read the CPU number like _os_cpu_number()
+	 * does, from the low 12 bits of tpidr_el0 (arm64) or the IDT base (x86).
+	 * The 0xfff mask is xnu's __TPIDR_CPU_NUM_MASK, kept in sync with
+	 * _os_cpu_number in libsyscall/os/tsd.h (the kernel counterpart is
+	 * MACHDEP_TPIDR_CPUNUM_MASK in osfmk/arm64/machine_machdep.h):
+	 * https://github.com/apple-oss-distributions/xnu/blob/main/libsyscall/os/tsd.h
+	 * This requires macOS 12+: on macOS 11 and earlier arm64 kept the CPU
+	 * number in tpidrro_el0's low 3 bits instead, so it would be misread here.
+	 * Those releases are retired by Apple, so only the current layout is handled.
+	 */
+#  if defined(__aarch64__)
+	uint64_t cpu;
+	__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(cpu));
+	return (malloc_cpuid_t)(cpu & 0xfff);
+#  elif defined(__x86_64__) || defined(__i386__)
+	struct { uintptr_t p1, p2; } idtr;
+	__asm__ __volatile__("sidt %0" : "=m"(idtr));
+	return (malloc_cpuid_t)(idtr.p1 & 0xfff);
+#  else
+	not_reached();
+	return -1;
+#  endif
 #elif defined(JEMALLOC_HAVE_RDTSCP)
 	unsigned int ecx;
 	asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx");
 	return (malloc_cpuid_t)(ecx & 0xfff);
-#elif defined(__aarch64__) && defined(__APPLE__)
-	/* Other oses most likely use tpidr_el0 instead */
-	uintptr_t c;
-	asm volatile("mrs %x0, tpidrro_el0" : "=r"(c)::"memory");
-	return (malloc_cpuid_t)(c & (1 << 3) - 1);
 #else
 	not_reached();
 	return -1;

include/jemalloc/internal/jemalloc_preamble.h.in

@@ -130,13 +130,6 @@ static const bool config_prof_libunwind =
     false
 #endif
     ;
-static const bool config_prof_frameptr =
-#ifdef JEMALLOC_PROF_FRAME_POINTER
-    true
-#else
-    false
-#endif
-    ;
 static const bool maps_coalesce =
 #ifdef JEMALLOC_MAPS_COALESCE
     true

include/jemalloc/internal/malloc_io.h

@@ -89,13 +89,4 @@ malloc_close(int fd) {
 #endif
 }
 
-static inline off_t
-malloc_lseek(int fd, off_t offset, int whence) {
-#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_lseek)
-	return (off_t)syscall(SYS_lseek, fd, offset, whence);
-#else
-	return lseek(fd, offset, whence);
-#endif
-}
-
 #endif /* JEMALLOC_INTERNAL_MALLOC_IO_H */

include/jemalloc/internal/pa.h

@@ -111,12 +111,6 @@ struct pa_shard_s {
 	pac_t pac;
 };
 
-static inline bool
-pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
-	return ecache_npages_get(&shard->pac.ecache_muzzy) == 0
-	    && pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0;
-}
-
 static inline ehooks_t *
 pa_shard_ehooks_get(const pa_shard_t *shard) {
 	return base_ehooks_get(shard->base);
@@ -158,8 +152,12 @@ void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard);
  */
 void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
 
-/* Flush any caches used by shard */
-void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard);
+/*
+ * Flush the shard's front caches (SEC + HPA) back to the ecaches.  If all is
+ * true, also fully decay-purge the PAC dirty (and muzzy, unless skipped) extents
+ * to the OS -- the "save as much memory as possible" path.
+ */
+void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard, bool all);
 
 /* Gets an edata for the given allocation. */
 edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
@@ -184,22 +182,35 @@ bool pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 void    pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
        bool *deferred_work_generated);
 bool    pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
-       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms);
 ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state);
 
-/*
- * Do deferred work on this PA shard.
- *
- * Morally, this should do both PAC decay and the HPA deferred work.  For now,
- * though, the arena, background thread, and PAC modules are tightly interwoven
- * in a way that's tricky to extricate, so we only do the HPA-specific parts.
- */
 void pa_shard_set_deferral_allowed(
     tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed);
-void     pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+/*
+ * Do deferred work on this PA shard: dispatch to PAC (decay-purge) then HPA.
+ * Each allocator owns its own policy -- PAC decides eagerness from
+ * is_background_thread and, on the application path, notifies the background
+ * thread; the bg-thread driver passes is_background_thread=true and is never
+ * self-notified.
+ */
+void     pa_shard_do_deferred_work(
+    tsdn_t *tsdn, pa_shard_t *shard, bool is_background_thread);
 void     pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 
+/*
+ * Called on the application path after a pa operation (alloc/expand/shrink/
+ * dalloc) reports deferred_work_generated -- i.e. it left PAC decay due or
+ * HPA work (hugify/dehugify/purge) pending.  If PAC dirty decay is immediate
+ * (dirty_decay_ms == 0), purge dirty synchronously; then, if a background
+ * thread is enabled but idle (sleeping indefinitely), wake it early so the
+ * pending work runs promptly rather than at the next scheduled wakeup.
+ *
+ * Application thread only; must never be called on the background thread.
+ */
+void pa_shard_handle_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+
 /******************************************************************************/
 /*
  * Various bits of "boring" functionality that are still part of this module,

include/jemalloc/internal/pac.h

@@ -13,7 +13,7 @@
 #include "san_bump.h"
 
 /*
- * Page allocator classic; an implementation of the PAI interface that:
+ * Page allocator classic (PAC), a page-level allocator that:
  * - Can be used for arenas with custom extent hooks.
  * - Can always satisfy any allocation request (including highly-fragmentary
  *   ones).
@@ -28,6 +28,14 @@ enum pac_purge_eagerness_e {
 };
 typedef enum pac_purge_eagerness_e pac_purge_eagerness_t;
 
+/*
+ * When a decay sweep would purge more than this many pages, its purge is
+ * treated as due (worth waking the background thread for) instead of left
+ * to accumulate.  PAC decay policy; the HPA defers on time intervals, not
+ * on a page count, so this constant is PAC-only.
+ */
+#define PAC_DECAY_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
+
 typedef struct pac_decay_stats_s pac_decay_stats_t;
 struct pac_decay_stats_s {
 	/* Total number of purge sweeps. */
@@ -231,6 +239,41 @@ bool pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
     pac_decay_stats_t *decay_stats, ecache_t *ecache,
     pac_purge_eagerness_t eagerness);
 
+/*
+ * Result of a pac_decay_deferred() pass.  Reported per decay state so
+ * pac_do_deferred_work can decide, per state, whether to notify the background
+ * thread.  npages_new is the per-state epoch backlog delta, only meaningful when
+ * the corresponding *_epoch_advanced is true.
+ */
+typedef struct pac_deferred_work_result_s pac_deferred_work_result_t;
+struct pac_deferred_work_result_s {
+	size_t dirty_npages_new;
+	size_t muzzy_npages_new;
+	bool   dirty_epoch_advanced;
+	bool   muzzy_epoch_advanced;
+};
+
+/*
+ * All deferred decay-purge work for a PAC shard: decide the eagerness from the
+ * caller context, run pac_decay_deferred, and (application path only) notify the
+ * background thread for any decay epoch that advanced.  The bg-thread driver
+ * passes is_background_thread=true and is never self-notified.
+ */
+void pac_do_deferred_work(
+    tsdn_t *tsdn, pac_t *pac, bool is_background_thread);
+
+/*
+ * Application-path hook (after deferred_work_generated): wake the background
+ * thread if it is sleeping idle.  Runs the same early-wake logic as the notify
+ * path (pac_maybe_wake_bg), gated on the thread being idle.
+ */
+void pac_wake_bg_on_deferred(tsdn_t *tsdn, pac_t *pac);
+
+/*
+ * Fully decay the extents of the given state, acquiring decay->mtx internally.
+ */
+void pac_decay_all_now(tsdn_t *tsdn, pac_t *pac, extent_state_t state);
+
 /*
  * Gets / sets the maximum amount that we'll grow an arena down the
  * grow-retained pathways (unless forced to by an allocaction request).
@@ -244,9 +287,22 @@ bool pac_retain_grow_limit_get_set(
     tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit);
 
 bool    pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
-       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms);
 ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
+/* Whether the muzzy decay path is relevant (muzzy pages exist, or decay is on). */
+static inline bool
+pac_should_decay_muzzy(pac_t *pac) {
+	return ecache_npages_get(&pac->ecache_muzzy) != 0
+	    || pac_decay_ms_get(pac, extent_state_muzzy) > 0;
+}
+
+/* Whether dirty decay is immediate (dirty_decay_ms == 0). */
+static inline bool
+pac_decay_immediately(pac_t *pac) {
+	return decay_immediately(&pac->decay_dirty);
+}
+
 void pac_destroy(tsdn_t *tsdn, pac_t *pac);
 
 void pac_sec_flush(tsdn_t *tsdn, pac_t *pac);

include/jemalloc/internal/prof_sys.h

@@ -21,7 +21,6 @@ void prof_fdump_impl(tsd_t *tsd);
 void prof_idump_impl(tsd_t *tsd);
 bool prof_mdump_impl(tsd_t *tsd, const char *filename);
 void prof_gdump_impl(tsd_t *tsd);
-int  prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high);
 
 /* Used in unit tests. */
 typedef int(prof_sys_thread_name_read_t)(char *buf, size_t limit);

include/jemalloc/internal/stats_internal.h

@@ -0,0 +1,340 @@
+#ifndef JEMALLOC_INTERNAL_STATS_INTERNAL_H
+#define JEMALLOC_INTERNAL_STATS_INTERNAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena.h" /* ARENA_NAME_LEN */
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/emitter.h"
+#include "jemalloc/internal/prof_stats.h"
+
+/*
+ * Internal implementation support for src/stats.c only -- do NOT include from
+ * other translation units.
+ *
+ * This header holds noisy local scaffolding for malloc_stats_print(): ctl
+ * gather plumbing, gather structs, emitter table-column helpers, and option
+ * list helpers.  It is deliberately not a reusable stats-gather API.
+ */
+
+#define CTL_GET(n, v, t)                                                       \
+	do {                                                                   \
+		size_t sz = sizeof(t);                                         \
+		xmallctl(n, (void *)v, &sz, NULL, 0);                          \
+	} while (0)
+
+#define CTL_LEAF_PREPARE(mib, miblen, name)                                    \
+	do {                                                                   \
+		assert(miblen < CTL_MAX_DEPTH);                                \
+		size_t miblen_new = CTL_MAX_DEPTH;                             \
+		xmallctlmibnametomib(mib, miblen, name, &miblen_new);          \
+		assert(miblen_new > miblen);                                   \
+	} while (0)
+
+#define CTL_LEAF(mib, miblen, leaf, v, t)                                      \
+	do {                                                                   \
+		assert(miblen < CTL_MAX_DEPTH);                                \
+		size_t miblen_new = CTL_MAX_DEPTH;                             \
+		size_t sz = sizeof(t);                                         \
+		xmallctlbymibname(                                             \
+		    mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0);  \
+		assert(miblen_new == miblen + 1);                              \
+	} while (0)
+
+#define CTL_MIB_GET(n, i, v, t, ind)                                           \
+	do {                                                                   \
+		size_t mib[CTL_MAX_DEPTH];                                     \
+		size_t miblen = sizeof(mib) / sizeof(size_t);                  \
+		size_t sz = sizeof(t);                                         \
+		xmallctlnametomib(n, mib, &miblen);                            \
+		mib[(ind)] = (i);                                              \
+		xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0);           \
+	} while (0)
+
+#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1)
+#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2)
+
+/******************************************************************************/
+/*
+ * Per-section gather structs.  The stats_gather_*() functions that fill them
+ * are static in src/stats.c.
+ */
+
+/* One arena's memory-accounting stats (stats.arenas.<i>.*). */
+typedef struct stats_arena_mem_s {
+	size_t mapped;
+	size_t retained;
+	size_t pinned;
+	size_t base;
+	size_t internal;
+	size_t metadata_edata;
+	size_t metadata_rtree;
+	size_t metadata_thp;
+	size_t tcache_bytes;
+	size_t tcache_stashed_bytes;
+	size_t resident;
+	size_t abandoned_vm;
+	size_t extent_avail;
+	size_t page; /* arenas.page, for the active-bytes conversion */
+} stats_arena_mem_t;
+
+/* One arena extents row (stats.arenas.<i>.extents.<pszind>.*). */
+typedef struct stats_arena_extent_s {
+	size_t ndirty;
+	size_t nmuzzy;
+	size_t nretained;
+	size_t npinned;
+	size_t ntotal;
+	size_t dirty_bytes;
+	size_t muzzy_bytes;
+	size_t retained_bytes;
+	size_t pinned_bytes;
+	size_t total_bytes;
+} stats_arena_extent_t;
+
+/* One arena's "basics" (stats.arenas.<i>.{nthreads,uptime,dss}). */
+typedef struct stats_arena_basics_s {
+	unsigned    nthreads;
+	uint64_t    uptime;
+	const char *dss;
+	char        name[ARENA_NAME_LEN];
+	bool        has_name; /* false for the merged/destroyed pseudo-arenas */
+} stats_arena_basics_t;
+
+/* One arena's decay / page stats (stats.arenas.<i>.*). */
+typedef struct stats_arena_decay_s {
+	ssize_t  dirty_decay_ms;
+	ssize_t  muzzy_decay_ms;
+	size_t   pactive;
+	size_t   pdirty;
+	size_t   pmuzzy;
+	uint64_t dirty_npurge;
+	uint64_t dirty_nmadvise;
+	uint64_t dirty_purged;
+	uint64_t muzzy_npurge;
+	uint64_t muzzy_nmadvise;
+	uint64_t muzzy_purged;
+} stats_arena_decay_t;
+
+/* One size class of arena allocation counts (stats.arenas.<i>.{small,large}.*). */
+typedef struct stats_arena_alloc_s {
+	size_t   allocated;
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	uint64_t nfills;
+	uint64_t nflushes;
+} stats_arena_alloc_t;
+
+/*
+ * One arena bin row (stats.arenas.<i>.bins.<j>.* + arenas.bin.<j>.*,
+ * plus prof.stats.bins.<j>.{live,accum} when profiling stats are enabled).
+ */
+typedef struct stats_arena_bin_s {
+	uint64_t nslabs;
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	uint64_t nfills;
+	uint64_t nflushes;
+	uint64_t nreslabs;
+	size_t   reg_size;
+	size_t   slab_size;
+	size_t   curregs;
+	size_t   curslabs;
+	size_t   nonfull_slabs;
+	uint32_t nregs;
+	uint32_t nshards;
+	prof_stats_t prof_live;
+	prof_stats_t prof_accum;
+} stats_arena_bin_t;
+
+/*
+ * One arena large-extent row (stats.arenas.<i>.lextents.<j>.* +
+ * arenas.lextent.<j>.*, plus prof.stats.lextents.<j>.{live,accum} when
+ * profiling stats are enabled).
+ */
+typedef struct stats_arena_lextent_s {
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	size_t   lextent_size;
+	size_t   curlextents;
+	prof_stats_t prof_live;
+	prof_stats_t prof_accum;
+} stats_arena_lextent_t;
+
+/* HPA shard small-extent cache (stats.arenas.<i>.hpa_sec_*). */
+typedef struct stats_arena_hpa_sec_s {
+	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+	size_t sec_overfills;
+} stats_arena_hpa_sec_t;
+
+/* PAC small-extent cache (stats.arenas.<i>.pac_sec_*). */
+typedef struct stats_arena_pac_sec_s {
+	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+} stats_arena_pac_sec_t;
+
+/* HPA shard counters (stats.arenas.<i>.hpa_shard.*). */
+typedef struct stats_arena_hpa_counters_s {
+	size_t npageslabs;
+	size_t nactive;
+	size_t ndirty;
+	size_t npageslabs_nonhuge;
+	size_t nactive_nonhuge;
+	size_t ndirty_nonhuge;
+	size_t nretained_nonhuge;
+	size_t npageslabs_huge;
+	size_t nactive_huge;
+	size_t ndirty_huge;
+	uint64_t npurge_passes;
+	uint64_t npurges;
+	uint64_t nhugifies;
+	uint64_t nhugify_failures;
+	uint64_t ndehugifies;
+} stats_arena_hpa_counters_t;
+
+/* One HPA slab class (full/empty/nonfull) row. */
+typedef struct stats_arena_hpa_slab_s {
+	size_t npageslabs_huge;
+	size_t nactive_huge;
+	size_t ndirty_huge;
+	size_t npageslabs_nonhuge;
+	size_t nactive_nonhuge;
+	size_t ndirty_nonhuge;
+	size_t nretained_nonhuge;
+} stats_arena_hpa_slab_t;
+
+/* Arena configuration / size-class scalars (arenas.*). */
+typedef struct stats_arena_config_s {
+	unsigned narenas;
+	ssize_t  dirty_decay_ms;
+	ssize_t  muzzy_decay_ms;
+	size_t   quantum;
+	size_t   page;
+	size_t   hugepage;
+	bool     have_tcache_max;
+	size_t   tcache_max;
+	unsigned nbins;
+	unsigned nhbins;
+	unsigned nlextents;
+} stats_arena_config_t;
+
+/* Per-size-class geometry (emitted in JSON only). */
+typedef struct stats_arena_bin_meta_s {
+	size_t   size;
+	uint32_t nregs;
+	size_t   slab_size;
+	uint32_t nshards;
+} stats_arena_bin_meta_t;
+
+typedef struct stats_arena_lextent_meta_s {
+	size_t size;
+} stats_arena_lextent_meta_t;
+
+/* Process-wide global stats (stats.* + stats.background_thread.*). */
+typedef struct stats_global_s {
+	size_t   allocated;
+	size_t   active;
+	size_t   metadata;
+	size_t   metadata_edata;
+	size_t   metadata_rtree;
+	size_t   metadata_thp;
+	size_t   resident;
+	size_t   mapped;
+	size_t   retained;
+	size_t   pinned;
+	size_t   zero_reallocs;
+	size_t   num_background_threads;
+	uint64_t background_thread_num_runs;
+	uint64_t background_thread_run_interval;
+} stats_global_t;
+
+/******************************************************************************/
+/*
+ * Emitter table-column helper macros: declare an emitter_col_t (and, for
+ * COL_HDR, its header column) and wire it into a row.
+ */
+
+#define COL_DECLARE(column_name) emitter_col_t col_##column_name;
+
+#define COL_INIT(row_name, column_name, left_or_right, col_width, etype)       \
+	emitter_col_init(&col_##column_name, &row_name);                       \
+	col_##column_name.justify = emitter_justify_##left_or_right;           \
+	col_##column_name.width = col_width;                                   \
+	col_##column_name.type = emitter_type_##etype;
+
+#define COL(row_name, column_name, left_or_right, col_width, etype)            \
+	COL_DECLARE(column_name);                                              \
+	COL_INIT(row_name, column_name, left_or_right, col_width, etype)
+
+#define COL_HDR_DECLARE(column_name)                                           \
+	COL_DECLARE(column_name);                                              \
+	emitter_col_t header_##column_name;
+
+#define COL_HDR_INIT(                                                          \
+    row_name, column_name, human, left_or_right, col_width, etype)             \
+	COL_INIT(row_name, column_name, left_or_right, col_width, etype)       \
+	emitter_col_init(&header_##column_name, &header_##row_name);           \
+	header_##column_name.justify = emitter_justify_##left_or_right;        \
+	header_##column_name.width = col_width;                                \
+	header_##column_name.type = emitter_type_title;                        \
+	header_##column_name.str_val = human ? human : #column_name;
+
+#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \
+	COL_HDR_DECLARE(column_name)                                           \
+	COL_HDR_INIT(                                                          \
+	    row_name, column_name, human, left_or_right, col_width, etype)
+
+/*
+ * stats_general_print() config/option list helpers.  Unhygienic by design:
+ * they assume `emitter` and the standard scratch locals are in scope at the
+ * call site -- bv/bsz (bool), cpv/cpsz (const char *), uv/usz (unsigned),
+ * i64v/i64sz, u64v/u64sz, sv/ssz (size_t), ssv/sssz (ssize_t), plus bv2/ssv2
+ * for the *_MUTABLE variants.  (CONFIG_WRITE_BOOL uses CTL_GET above; the
+ * OPT_WRITE_* variants use je_mallctl directly.)
+ */
+#define CONFIG_WRITE_BOOL(name)                                                \
+	do {                                                                   \
+		CTL_GET("config." #name, &bv, bool);                           \
+		emitter_kv(                                                    \
+		    emitter, #name, "config." #name, emitter_type_bool, &bv);  \
+	} while (0)
+
+#define OPT_WRITE(name, var, size, emitter_type)                               \
+	if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) {      \
+		emitter_kv(emitter, name, "opt." name, emitter_type, &var);    \
+	}
+
+#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname)       \
+	if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0        \
+	    && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) {      \
+		emitter_kv_note(emitter, name, "opt." name, emitter_type,      \
+		    &var1, altname, emitter_type, &var2);                      \
+	}
+
+#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool)
+#define OPT_WRITE_BOOL_MUTABLE(name, altname)                                  \
+	OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname)
+
+#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned)
+
+#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64)
+#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64)
+
+#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size)
+#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize)
+#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname)                               \
+	OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname)
+
+#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string)
+
+#endif /* JEMALLOC_INTERNAL_STATS_INTERNAL_H */

include/jemalloc/internal/test_hooks.h

@@ -4,6 +4,9 @@
 extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(void);
 extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void);
 extern JEMALLOC_EXPORT void (*test_hooks_safety_check_abort)(const char *);
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+extern JEMALLOC_EXPORT void (*test_hooks_tsd_bootstrap_hook)(void);
+#endif
 
 #if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
 #	define JEMALLOC_TEST_HOOK(fn, hook)                                   \

include/jemalloc/internal/tsd.h

@@ -4,7 +4,8 @@
 /*
  * We put the platform-specific data declarations and inlines into their own
  * header files to avoid cluttering this file.  They define tsd_boot0,
- * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates, tsd_get, and tsd_set.
+ * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates,
+ * tsd_teardown_done, tsd_get, and tsd_set.
  */
 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 #	include "jemalloc/internal/jemalloc_preamble.h"

include/jemalloc/internal/tsd_generic.h

@@ -27,6 +27,7 @@ void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block);
 void  tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 
 extern pthread_key_t   tsd_tsd;
+extern pthread_key_t   tsd_thread_initialized_tsd;
 extern tsd_init_head_t tsd_init_head;
 extern tsd_wrapper_t   tsd_boot_wrapper;
 extern bool            tsd_booted;
@@ -51,6 +52,12 @@ tsd_cleanup_wrapper(void *arg) {
 			return;
 		}
 	}
+	/*
+	 * Leave tsd_thread_initialized_tsd set.  It has no destructor and
+	 * intentionally outlives the TSD wrapper, so later jemalloc calls can
+	 * distinguish this fully-torn-down thread from one that has never
+	 * initialized TSD.
+	 */
 	malloc_tsd_dalloc(wrapper);
 }
 
@@ -63,6 +70,11 @@ tsd_wrapper_set(tsd_wrapper_t *wrapper) {
 		malloc_write("<jemalloc>: Error setting TSD\n");
 		abort();
 	}
+	if (pthread_setspecific(tsd_thread_initialized_tsd,
+	    (void *)&tsd_thread_initialized_tsd) != 0) {
+		malloc_write("<jemalloc>: Error setting TSD\n");
+		abort();
+	}
 }
 
 JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
@@ -116,6 +128,15 @@ tsd_boot0(void) {
 	if (pthread_key_create(&tsd_tsd, tsd_cleanup_wrapper) != 0) {
 		return true;
 	}
+	/*
+	 * This key has no destructor.  It records that the current thread once
+	 * had jemalloc TSD, so a later NULL tsd_tsd value means teardown has
+	 * finished rather than initialization not having happened yet.
+	 */
+	if (pthread_key_create(&tsd_thread_initialized_tsd, NULL) != 0) {
+		pthread_key_delete(tsd_tsd);
+		return true;
+	}
 	tsd_booted = true;
 	tsd_wrapper_set(&tsd_boot_wrapper);
 	tsd_init_finish(&tsd_init_head, &block);
@@ -160,6 +181,12 @@ tsd_get_allocates(void) {
 	return true;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return tsd_booted && pthread_getspecific(tsd_tsd) == NULL
+	    && pthread_getspecific(tsd_thread_initialized_tsd) != NULL;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_malloc_thread_cleanup.h

@@ -50,6 +50,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_tls.h

@@ -43,6 +43,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_win.h

@@ -146,6 +146,11 @@ tsd_get_allocates(void) {
 	return true;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
@@ -220,6 +225,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

scripts/README_GH_ACTIONS.md

@@ -27,12 +27,12 @@ The script can generate workflows for different platforms:
 
 ### Linux CI (`linux-ci.yml`)
 - **test-linux** (AMD64): `ubuntu-latest` (x86_64)
-  - ~96 configurations covering GCC, Clang, various flags
+  - ~98 configurations covering GCC, Clang, various flags
 - **test-linux-arm64** (ARM64): `ubuntu-24.04-arm` (aarch64)
   - ~14 configurations including large hugepage tests
   - **Note:** Free ARM64 runners (Public Preview) - may have longer queue times during peak hours
 
-**Total:** 110 configurations
+**Total:** 112 configurations
 
 ### macOS CI (`macos-ci.yml`)
 - **test-macos** (Intel): `macos-15-intel` (x86_64)
@@ -161,7 +161,7 @@ The Windows workflow uses:
 ### Linux Build Process
 - Ubuntu Latest for AMD64, Ubuntu 24.04 for ARM64
 - Installs 32-bit cross-compilation dependencies when needed
-- Most comprehensive test matrix (110 configurations)
+- Most comprehensive test matrix (112 configurations)
 
 ## Relationship to Travis CI
 
@@ -178,4 +178,3 @@ To regenerate all workflows after modifying `gen_gh_actions.py`:
 ```
 
 **Note**: The generated files should not be edited by hand. All changes should be made to `gen_gh_actions.py` and then regenerated.
-

scripts/gen_gh_actions.py

@@ -224,11 +224,8 @@ def generate_linux_job(arch):
     if arch != ARM64:
         exclude += [LARGE_HUGEPAGE]
 
-    linux_configure_flags = list(configure_flag_unusuals)
-    linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))
-
     linux_unusuals = (compilers_unusual + feature_unusuals
-                    + linux_configure_flags + malloc_conf_unusuals)
+                    + configure_flag_unusuals + malloc_conf_unusuals)
 
     matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, linux_unusuals)
 
@@ -273,6 +270,18 @@ def generate_linux_job(arch):
             'CXX': 'g++',
             'CONFIGURE_FLAGS': '--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof',
             'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': 'force_tls=0',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': 'force_tls=0 --enable-debug',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
         }
     ]
 

scripts/gen_travis.py

@@ -264,11 +264,8 @@ def generate_linux(arch):
     if arch != ARM64:
         exclude += [LARGE_HUGEPAGE]
 
-    linux_configure_flags = list(configure_flag_unusuals)
-    linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))
-
     linux_unusuals = (compilers_unusual + feature_unusuals
-                    + linux_configure_flags + malloc_conf_unusuals)
+                    + configure_flag_unusuals + malloc_conf_unusuals)
 
     return generate_jobs(os, arch, exclude, max_unusual_opts, linux_unusuals)
 

src/arena.c

@@ -83,11 +83,6 @@ const arena_config_t arena_config_default = {
  * definition.
  */
 
-static bool arena_decay_dirty(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-static void arena_maybe_do_deferred_work(
-    tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new);
-
 /******************************************************************************/
 
 void
@@ -96,8 +91,8 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
 	*nthreads += arena_nthreads_get(arena, false);
 	*dss = dss_prec_names[arena_dss_prec_get(arena)];
-	*dirty_decay_ms = arena_decay_ms_get(arena, extent_state_dirty);
-	*muzzy_decay_ms = arena_decay_ms_get(arena, extent_state_muzzy);
+	*dirty_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_dirty);
+	*muzzy_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_muzzy);
 	pa_shard_basic_stats_merge(&arena->pa_shard, nactive, ndirty, nmuzzy);
 }
 
@@ -316,34 +311,6 @@ arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
 	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 }
 
-static void
-arena_background_thread_inactivity_check(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
-	if (!background_thread_enabled() || is_background_thread) {
-		return;
-	}
-	background_thread_info_t *info = arena_background_thread_info_get(
-	    arena);
-	if (background_thread_indefinite_sleep(info)) {
-		arena_maybe_do_deferred_work(
-		    tsdn, arena, &arena->pa_shard.pac.decay_dirty, 0);
-	}
-}
-
-/*
- * React to deferred work generated by a PAI function.
- */
-void
-arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) {
-	witness_assert_depth_to_rank(
-	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-
-	if (decay_immediately(&arena->pa_shard.pac.decay_dirty)) {
-		arena_decay_dirty(tsdn, arena, false, true);
-	}
-	arena_background_thread_inactivity_check(tsdn, arena, false);
-}
-
 static void
 arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) {
 	cassert(config_stats);
@@ -476,188 +443,12 @@ arena_extent_ralloc_large_expand(
 	}
 }
 
-/*
- * In situations where we're not forcing a decay (i.e. because the user
- * specifically requested it), should we purge ourselves, or wait for the
- * background thread to get to it.
- */
-static pac_purge_eagerness_t
-arena_decide_unforced_purge_eagerness(bool is_background_thread) {
-	if (is_background_thread) {
-		return PAC_PURGE_ALWAYS;
-	} else if (!is_background_thread && background_thread_enabled()) {
-		return PAC_PURGE_NEVER;
-	} else {
-		return PAC_PURGE_ON_EPOCH_ADVANCE;
-	}
-}
-
-bool
-arena_decay_ms_set(
-    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms) {
-	pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
-	    /* is_background_thread */ false);
-	return pa_decay_ms_set(
-	    tsdn, &arena->pa_shard, state, decay_ms, eagerness);
-}
-
-ssize_t
-arena_decay_ms_get(arena_t *arena, extent_state_t state) {
-	return pa_decay_ms_get(&arena->pa_shard, state);
-}
-
-static bool
-arena_decay_impl(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
-    pac_decay_stats_t *decay_stats, ecache_t *ecache, bool is_background_thread,
-    bool all) {
-	if (all) {
-		malloc_mutex_lock(tsdn, &decay->mtx);
-		pac_decay_all(tsdn, &arena->pa_shard.pac, decay, decay_stats,
-		    ecache, /* fully_decay */ all);
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-
-	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
-		/* No need to wait if another thread is in progress. */
-		return true;
-	}
-	pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
-	    is_background_thread);
-	bool epoch_advanced = pac_maybe_decay_purge(
-	    tsdn, &arena->pa_shard.pac, decay, decay_stats, ecache, eagerness);
-	size_t npages_new JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(0);
-	if (epoch_advanced) {
-		/* Backlog is updated on epoch advance. */
-		npages_new = decay_epoch_npages_delta(decay);
-	}
-	malloc_mutex_unlock(tsdn, &decay->mtx);
-
-	if (have_background_thread && background_thread_enabled()
-	    && epoch_advanced && !is_background_thread) {
-		arena_maybe_do_deferred_work(tsdn, arena, decay, npages_new);
-	}
-
-	return false;
-}
-
-static bool
-arena_decay_dirty(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_dirty,
-	    &arena->pa_shard.pac.stats->decay_dirty,
-	    &arena->pa_shard.pac.ecache_dirty, is_background_thread, all);
-}
-
-static bool
-arena_decay_muzzy(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	if (pa_shard_dont_decay_muzzy(&arena->pa_shard)) {
-		return false;
-	}
-	return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_muzzy,
-	    &arena->pa_shard.pac.stats->decay_muzzy,
-	    &arena->pa_shard.pac.ecache_muzzy, is_background_thread, all);
-}
-
-void
-arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	if (all) {
-		/*
-		 * We should take a purge of "all" to mean "save as much memory
-		 * as possible", including flushing any caches (for situations
-		 * like thread death, or manual purge calls).
-		 */
-		pa_shard_flush(tsdn, &arena->pa_shard);
-	}
-	if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
-		return;
-	}
-	arena_decay_muzzy(tsdn, arena, is_background_thread, all);
-}
-
-static bool
-arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
-    background_thread_info_t *info, nstime_t *remaining_sleep,
-    size_t npages_new) {
-	malloc_mutex_assert_owner(tsdn, &info->mtx);
-
-	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
-		return false;
-	}
-
-	if (!decay_gradually(decay)) {
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-
-	nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
-	if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-	nstime_subtract(remaining_sleep, &decay->epoch);
-	if (npages_new > 0) {
-		uint64_t npurge_new = decay_npages_purge_in(
-		    decay, remaining_sleep, npages_new);
-		info->npages_to_purge_new += npurge_new;
-	}
-	malloc_mutex_unlock(tsdn, &decay->mtx);
-	return info->npages_to_purge_new
-	    > ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD;
-}
-
-/*
- * Check if deferred work needs to be done sooner than planned.
- * For decay we might want to wake up earlier because of an influx of dirty
- * pages. Rather than waiting for previously estimated time, we proactively
- * purge those pages.
- * If background thread sleeps indefinitely, always wake up because some
- * deferred work has been generated.
- */
-static void
-arena_maybe_do_deferred_work(
-    tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new) {
-	background_thread_info_t *info = arena_background_thread_info_get(
-	    arena);
-	if (malloc_mutex_trylock(tsdn, &info->mtx)) {
-		/*
-		 * Background thread may hold the mutex for a long period of
-		 * time.  We'd like to avoid the variance on application
-		 * threads.  So keep this non-blocking, and leave the work to a
-		 * future epoch.
-		 */
-		return;
-	}
-	if (!background_thread_is_started(info)) {
-		goto label_done;
-	}
-
-	nstime_t remaining_sleep;
-	if (background_thread_indefinite_sleep(info)) {
-		background_thread_wakeup_early(info, NULL);
-	} else if (arena_should_decay_early(tsdn, arena, decay, info,
-	               &remaining_sleep, npages_new)) {
-		info->npages_to_purge_new = 0;
-		background_thread_wakeup_early(info, &remaining_sleep);
-	}
-label_done:
-	malloc_mutex_unlock(tsdn, &info->mtx);
-}
-
-/* Called from background threads. */
-void
-arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) {
-	arena_decay(tsdn, arena, true, false);
-	pa_shard_do_deferred_work(tsdn, &arena->pa_shard);
-}
-
 static void
 arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) {
 	bool deferred_work_generated = false;
 	pa_dalloc(tsdn, &arena->pa_shard, slab, &deferred_work_generated);
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 }
 
@@ -934,7 +725,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
 	    /* zero */ false, guarded, &deferred_work_generated);
 
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 
 	if (slab == NULL) {
@@ -1585,17 +1376,17 @@ arena_get_ehooks(const arena_t *arena) {
 extent_hooks_t *
 arena_set_extent_hooks(
     tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks) {
-	background_thread_info_t *info;
-	if (have_background_thread) {
-		info = arena_background_thread_info_get(arena);
-		malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
-	}
+	/*
+	 * Serialize disabling the HPA against the background thread via the
+	 * module-owned bracket (have_background_thread-gated internally) rather
+	 * than reaching into info->mtx directly.
+	 */
+	unsigned arena_ind = arena_ind_get(arena);
+	background_thread_serialize_lock(tsd, arena_ind);
 	/* No using the HPA now that we have the custom hooks. */
 	pa_shard_disable_hpa(tsd_tsdn(tsd), &arena->pa_shard);
 	extent_hooks_t *ret = base_extent_hooks_set(arena->base, extent_hooks);
-	if (have_background_thread) {
-		malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
-	}
+	background_thread_serialize_unlock(tsd, arena_ind);
 
 	return ret;
 }
@@ -1828,13 +1619,13 @@ arena_create_huge_arena(tsd_t *tsd, unsigned ind) {
 	 */
 	if (!background_thread_enabled()
 	    && arena_dirty_decay_ms_default_get() > 0) {
-		arena_decay_ms_set(
-		    tsd_tsdn(tsd), huge_arena, extent_state_dirty, 0);
+		pa_decay_ms_set(
+		    tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_dirty, 0);
 	}
 	if (!background_thread_enabled()
 	    && arena_muzzy_decay_ms_default_get() > 0) {
-		arena_decay_ms_set(
-		    tsd_tsdn(tsd), huge_arena, extent_state_muzzy, 0);
+		pa_decay_ms_set(
+		    tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_muzzy, 0);
 	}
 
 	return huge_arena;

src/arenas_management.c

@@ -202,8 +202,8 @@ arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
 		 * Purge if the old arena has no associated threads anymore and
 		 * no background threads.
 		 */
-		arena_decay(tsd_tsdn(tsd), oldarena,
-		    /* is_background_thread */ false, /* all */ true);
+		pa_shard_flush(tsd_tsdn(tsd), &oldarena->pa_shard,
+		    /* all */ true);
 	}
 }
 

src/background_thread.c

@@ -6,6 +6,7 @@
 #include "jemalloc/internal/background_thread.h"
 #include "jemalloc/internal/background_thread_inlines.h"
 #include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_a.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex.h"
@@ -33,6 +34,72 @@ size_t     max_background_threads;
 /* Thread info per-index. */
 background_thread_info_t *background_thread_info;
 
+/******************************************************************************/
+/*
+ * Config-independent lifecycle/state-ownership helpers.  Defined
+ * unconditionally and gated at runtime on have_background_thread, so callers in
+ * ctl.c / arena.c never touch background_thread_lock or info->state directly;
+ * they compile to runtime no-ops when !have_background_thread.
+ */
+
+void
+background_thread_arena_reset_begin(tsd_t *tsd, unsigned arena_ind) {
+	/* Temporarily disable the background thread during arena reset. */
+	if (have_background_thread) {
+		/*
+		 * Hold background_thread_lock across the whole arena-reset
+		 * body (acquired here, released in _finish) so a concurrent
+		 * background_threads_enable() cannot start a background
+		 * thread mid-reset.  This must happen whenever the feature
+		 * is compiled in (have_background_thread), even when the
+		 * thread is not currently enabled; the state transition
+		 * below is separately gated on background_thread_enabled().
+		 */
+		malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
+		if (background_thread_enabled()) {
+			background_thread_info_t *info =
+			    background_thread_info_get(arena_ind);
+			assert(info->state == background_thread_started);
+			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+			info->state = background_thread_paused;
+			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+		}
+	}
+}
+
+void
+background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		if (background_thread_enabled()) {
+			background_thread_info_t *info =
+			    background_thread_info_get(arena_ind);
+			assert(info->state == background_thread_paused);
+			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+			info->state = background_thread_started;
+			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+		}
+		malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
+	}
+}
+
+void
+background_thread_serialize_lock(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		background_thread_info_t *info =
+		    background_thread_info_get(arena_ind);
+		malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+	}
+}
+
+void
+background_thread_serialize_unlock(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		background_thread_info_t *info =
+		    background_thread_info_get(arena_ind);
+		malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+	}
+}
+
 /******************************************************************************/
 
 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
@@ -191,6 +258,50 @@ background_thread_cond_wait(
 	return ret;
 }
 
+static int
+background_thread_cond_init(pthread_cond_t *cond) {
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	pthread_condattr_t cond_attr;
+	int ret = pthread_condattr_init(&cond_attr);
+	if (ret != 0) {
+		return ret;
+	}
+	ret = pthread_condattr_setclock(&cond_attr, CLOCK_MONOTONIC);
+	if (ret != 0) {
+		pthread_condattr_destroy(&cond_attr);
+		return ret;
+	}
+	ret = pthread_cond_init(cond, &cond_attr);
+	pthread_condattr_destroy(&cond_attr);
+	return ret;
+#else
+	return pthread_cond_init(cond, NULL);
+#endif
+}
+
+/*
+ * Fill in the absolute deadline for pthread_cond_timedwait.  The clock read
+ * here MUST match the clock the condvar was initialized with in
+ * background_thread_cond_init, otherwise the deadline is interpreted against
+ * the wrong epoch.
+ */
+static void
+background_thread_wakeup_ts_init(struct timespec *ts, uint64_t interval) {
+	nstime_t wakeup;
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	struct timespec now;
+	clock_gettime(CLOCK_MONOTONIC, &now);
+	nstime_init2(&wakeup, now.tv_sec, now.tv_nsec);
+#else
+	struct timeval tv;
+	gettimeofday(&tv, NULL);
+	nstime_init2(&wakeup, tv.tv_sec, tv.tv_usec * 1000);
+#endif
+	nstime_iadd(&wakeup, interval);
+	ts->tv_sec = (size_t)nstime_sec(&wakeup);
+	ts->tv_nsec = (size_t)nstime_nsec(&wakeup);
+}
+
 static void
 background_thread_sleep(
     tsdn_t *tsdn, background_thread_info_t *info, uint64_t interval) {
@@ -199,11 +310,8 @@ background_thread_sleep(
 	}
 	info->npages_to_purge_new = 0;
 
-	struct timeval tv;
-	/* Specific clock required by timedwait. */
-	gettimeofday(&tv, NULL);
 	nstime_t before_sleep;
-	nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000);
+	nstime_init_update(&before_sleep);
 
 	int ret;
 	if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
@@ -223,21 +331,16 @@ background_thread_sleep(
 		background_thread_wakeup_time_set(
 		    tsdn, info, nstime_ns(&next_wakeup));
 
-		nstime_t ts_wakeup;
-		nstime_copy(&ts_wakeup, &before_sleep);
-		nstime_iadd(&ts_wakeup, interval);
 		struct timespec ts;
-		ts.tv_sec = (size_t)nstime_sec(&ts_wakeup);
-		ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup);
+		background_thread_wakeup_ts_init(&ts, interval);
 
 		assert(!background_thread_indefinite_sleep(info));
 		ret = background_thread_cond_wait(info, &ts);
 		assert(ret == ETIMEDOUT || ret == 0);
 	}
 	if (config_stats) {
-		gettimeofday(&tv, NULL);
 		nstime_t after_sleep;
-		nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000);
+		nstime_init_update(&after_sleep);
 		if (nstime_compare(&after_sleep, &before_sleep) > 0) {
 			nstime_subtract(&after_sleep, &before_sleep);
 			nstime_add(&info->tot_sleep_time, &after_sleep);
@@ -262,7 +365,7 @@ background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) {
 static inline void
 background_work_sleep_once(
     tsdn_t *tsdn, background_thread_info_t *info, unsigned ind) {
-	uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX;
+	uint64_t ns_until_deferred = DEFERRED_WORK_MAX;
 	unsigned narenas = narenas_total_get();
 	bool     slept_indefinitely = background_thread_indefinite_sleep(info);
 
@@ -277,7 +380,8 @@ background_work_sleep_once(
 		 * work that caused this thread to wake up is scheduled for.
 		 */
 		if (!slept_indefinitely) {
-			arena_do_deferred_work(tsdn, arena);
+			pa_shard_do_deferred_work(tsdn, &arena->pa_shard,
+			    /* is_background_thread */ true);
 		}
 		if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) {
 			/* Min interval will be used. */
@@ -291,7 +395,7 @@ background_work_sleep_once(
 	}
 
 	uint64_t sleep_ns;
-	if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) {
+	if (ns_until_deferred == DEFERRED_WORK_MAX) {
 		sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP;
 	} else {
 		sleep_ns = (ns_until_deferred
@@ -751,7 +855,7 @@ background_thread_postfork_child(tsdn_t *tsdn) {
 		background_thread_info_t *info = &background_thread_info[i];
 		malloc_mutex_lock(tsdn, &info->mtx);
 		info->state = background_thread_stopped;
-		int ret = pthread_cond_init(&info->cond, NULL);
+		int ret = background_thread_cond_init(&info->cond);
 		assert(ret == 0);
 		background_thread_info_init(tsdn, info);
 		malloc_mutex_unlock(tsdn, &info->mtx);
@@ -869,7 +973,7 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) {
 		        malloc_mutex_address_ordered)) {
 			return true;
 		}
-		if (pthread_cond_init(&info->cond, NULL)) {
+		if (background_thread_cond_init(&info->cond)) {
 			return true;
 		}
 		malloc_mutex_lock(tsdn, &info->mtx);

src/ctl.c

@@ -121,7 +121,6 @@ CTL_PROTO(config_opt_safety_checks)
 CTL_PROTO(config_prof)
 CTL_PROTO(config_prof_libgcc)
 CTL_PROTO(config_prof_libunwind)
-CTL_PROTO(config_prof_frameptr)
 CTL_PROTO(config_stats)
 CTL_PROTO(config_utrace)
 CTL_PROTO(config_xmalloc)
@@ -495,7 +494,6 @@ static const ctl_named_node_t config_node[] = {
     {NAME("prof"), CTL(config_prof)},
     {NAME("prof_libgcc"), CTL(config_prof_libgcc)},
     {NAME("prof_libunwind"), CTL(config_prof_libunwind)},
-    {NAME("prof_frameptr"), CTL(config_prof_frameptr)},
     {NAME("stats"), CTL(config_stats)}, {NAME("utrace"), CTL(config_utrace)},
     {NAME("xmalloc"), CTL(config_xmalloc)}};
 
@@ -2210,7 +2208,6 @@ CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool)
 CTL_RO_CONFIG_GEN(config_prof, bool)
 CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
 CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
-CTL_RO_CONFIG_GEN(config_prof_frameptr, bool)
 CTL_RO_CONFIG_GEN(config_stats, bool)
 CTL_RO_CONFIG_GEN(config_utrace, bool)
 CTL_RO_CONFIG_GEN(config_xmalloc, bool)
@@ -2374,10 +2371,18 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
 		if (newind < percpu_arena_ind_limit(opt_percpu_arena)) {
 			/*
-			 * If perCPU arena is enabled, thread_arena control is
-			 * not allowed for the auto arena range.
+			 * Setting thread.arena to an arena in the auto range
+			 * means "resume automatic per-CPU selection" rather than
+			 * pinning to a specific per-CPU arena. This lets a caller
+			 * that temporarily switched to a manually managed arena
+			 * (e.g. a scoped guard) hand the thread back to per-CPU
+			 * management. It is otherwise impossible: a thread bound
+			 * to a manual arena is never reclaimed by percpu (see
+			 * arena_choose_impl), so without this it would stay
+			 * pinned forever.
 			 */
-			return EPERM;
+			percpu_arena_update(tsd, percpu_arena_choose());
+			return 0;
 		}
 	}
 
@@ -2616,7 +2621,7 @@ thread_idle_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 	if (opt_narenas > ncpus * 2) {
 		arena_t *arena = arena_choose(tsd, NULL);
 		if (arena != NULL) {
-			arena_decay(tsd_tsdn(tsd), arena, false, true);
+			pa_shard_flush(tsd_tsdn(tsd), &arena->pa_shard, true);
 		}
 		/*
 		 * The missing arena case is not actually an error; a thread
@@ -2738,7 +2743,13 @@ arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) {
 
 	for (unsigned i = 0; i < count; i++) {
 		if (tarenas[i] != NULL) {
-			arena_decay(tsdn, tarenas[i], false, all);
+			if (all) {
+				pa_shard_flush(tsdn, &tarenas[i]->pa_shard,
+				    true);
+			} else {
+				pa_shard_do_deferred_work(
+				    tsdn, &tarenas[i]->pa_shard, false);
+			}
 		}
 	}
 }
@@ -2790,37 +2801,6 @@ arena_i_reset_destroy_helper(tsd_t *tsd, const size_t *mib, size_t miblen,
 	return ret;
 }
 
-static void
-arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) {
-	/* Temporarily disable the background thread during arena reset. */
-	if (have_background_thread) {
-		malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
-		if (background_thread_enabled()) {
-			background_thread_info_t *info =
-			    background_thread_info_get(arena_ind);
-			assert(info->state == background_thread_started);
-			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
-			info->state = background_thread_paused;
-			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
-		}
-	}
-}
-
-static void
-arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) {
-	if (have_background_thread) {
-		if (background_thread_enabled()) {
-			background_thread_info_t *info =
-			    background_thread_info_get(arena_ind);
-			assert(info->state == background_thread_paused);
-			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
-			info->state = background_thread_started;
-			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
-		}
-		malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
-	}
-}
-
 static int
 arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
     size_t *oldlenp, void *newp, size_t newlen) {
@@ -2834,9 +2814,9 @@ arena_i_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 		return ret;
 	}
 
-	arena_reset_prepare_background_thread(tsd, arena_ind);
+	background_thread_arena_reset_begin(tsd, arena_ind);
 	arena_reset(tsd, arena);
-	arena_reset_finish_background_thread(tsd, arena_ind);
+	background_thread_arena_reset_finish(tsd, arena_ind);
 
 	return ret;
 }
@@ -2863,10 +2843,10 @@ arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 		goto label_return;
 	}
 
-	arena_reset_prepare_background_thread(tsd, arena_ind);
+	background_thread_arena_reset_begin(tsd, arena_ind);
 	/* Merge stats after resetting and purging arena. */
 	arena_reset(tsd, arena);
-	arena_decay(tsd_tsdn(tsd), arena, false, true);
+	pa_shard_flush(tsd_tsdn(tsd), &arena->pa_shard, true);
 	ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED);
 	ctl_darena->initialized = true;
 	ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true);
@@ -2877,7 +2857,7 @@ arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 	/* Record arena index for later recycling via arenas.create. */
 	ql_elm_new(ctl_arena, destroyed_link);
 	ql_tail_insert(&ctl_arenas->destroyed, ctl_arena, destroyed_link);
-	arena_reset_finish_background_thread(tsd, arena_ind);
+	background_thread_arena_reset_finish(tsd, arena_ind);
 
 	assert(ret == 0);
 label_return:
@@ -2997,7 +2977,7 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
 	}
 
 	extent_state_t state = dirty ? extent_state_dirty : extent_state_muzzy;
-	ssize_t oldval = arena_decay_ms_get(arena, state);
+	ssize_t oldval = pa_decay_ms_get(&arena->pa_shard, state);
 	ret = ctl_read(oldp, oldlenp, &oldval, sizeof(oldval));
 	if (ret != 0 || newp == NULL) {
 		return ret;
@@ -3005,7 +2985,8 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
 
 	ssize_t newval;
 	ret = ctl_write(&newval, sizeof(newval), newp, newlen);
-	if (ret == 0 && arena_decay_ms_set(tsd_tsdn(tsd), arena, state, newval)) {
+	if (ret == 0 && pa_decay_ms_set(tsd_tsdn(tsd), &arena->pa_shard,
+	    state, newval)) {
 		ret = EFAULT;
 	}
 	return ret;

src/hpa.c

@@ -1,6 +1,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 
 #include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/hpa_utils.h"
 #include "jemalloc/internal/jemalloc_probe.h"
@@ -1081,7 +1082,7 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
  */
 uint64_t
 hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
-	uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
+	uint64_t time_ns = DEFERRED_WORK_MAX;
 
 	malloc_mutex_lock(tsdn, &shard->mtx);
 
@@ -1101,7 +1102,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
 			time_ns *= 1000 * 1000;
 		} else {
 			malloc_mutex_unlock(tsdn, &shard->mtx);
-			return BACKGROUND_THREAD_DEFERRED_MIN;
+			return DEFERRED_WORK_MIN;
 		}
 	}
 
@@ -1112,7 +1113,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
 		 */
 		if (shard->stats.npurge_passes == 0) {
 			malloc_mutex_unlock(tsdn, &shard->mtx);
-			return BACKGROUND_THREAD_DEFERRED_MIN;
+			return DEFERRED_WORK_MIN;
 		}
 		uint64_t since_last_purge_ms = shard->central->hooks.ms_since(
 		    &shard->last_purge);
@@ -1127,7 +1128,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
 				time_ns = until_purge_ns;
 			}
 		} else {
-			time_ns = BACKGROUND_THREAD_DEFERRED_MIN;
+			time_ns = DEFERRED_WORK_MIN;
 		}
 	}
 	malloc_mutex_unlock(tsdn, &shard->mtx);

src/jemalloc.c

@@ -1025,12 +1025,50 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
 	thread_dalloc_event(tsd, usize);
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+dealloc_no_tsd(void *ptr) {
+	/*
+	 * On the generic pthread_getspecific() TSD path, a deallocation can run
+	 * after the thread's TSD has been torn down.  Using tsd_fetch_min() here
+	 * would allocate and publish a fresh TSD wrapper mid-teardown,
+	 * reincarnating the state teardown just released (the crashes this change
+	 * fixes).  Instead we hand the object straight back to its arena via
+	 * idalloctm() with a NULL tsdn.
+	 *
+	 * This deliberately bypasses the per-thread bookkeeping that the normal
+	 * ifree()/isfree() slow path performs, none of which is available or
+	 * meaningful without a live TSD.  The trade-offs, all acceptable for
+	 * these rare teardown-time frees (the memory itself is still correctly
+	 * returned to the arena):
+	 *   - Profiling: prof_free() is skipped, so a sampled object is not
+	 *     unregistered from its prof context; its bytes stay counted as live
+	 *     in prof stats and in any final heap/leak dump.
+	 *   - Junk filling: opt_junk_free is not applied to the freed region.
+	 *   - Sized dealloc: for sdallocx() the caller-supplied size is ignored
+	 *     (szind is looked up from the extent map), so the sized-dealloc
+	 *     safety check does not run.
+	 *   - Thread events: thread_dalloc_event() does not fire, so this free
+	 *     does not advance decay / tcache GC or dalloc stats.
+	 */
+	if (!tsd_teardown_done()) {
+		return false;
+	}
+
+	idalloctm(TSDN_NULL, ptr, /* tcache */ NULL, /* alloc_ctx */ NULL,
+	    /* is_internal */ false, /* slow_path */ true);
+	return true;
+}
+
 JEMALLOC_NOINLINE
 void
 free_default(void *ptr) {
 	UTRACE(ptr, 0, 0);
 	if (likely(ptr != NULL)) {
 		int saved_errno = get_errno();
+		if (unlikely(dealloc_no_tsd(ptr))) {
+			set_errno(saved_errno);
+			return;
+		}
 		/*
 		 * We avoid setting up tsd fully (e.g. tcache, arena binding)
 		 * based on only free() calls -- other activities trigger the
@@ -1568,6 +1606,9 @@ do_realloc_nonnull_zero(void *ptr) {
 		return do_rallocx(ptr, 1, MALLOCX_TCACHE_NONE, true);
 	} else if (opt_zero_realloc_action == zero_realloc_action_free) {
 		UTRACE(ptr, 0, 0);
+		if (unlikely(dealloc_no_tsd(ptr))) {
+			return NULL;
+		}
 		tsd_t *tsd = tsd_fetch();
 		check_entry_exit_locking(tsd_tsdn(tsd));
 
@@ -1843,6 +1884,12 @@ je_dallocx(void *ptr, int flags) {
 	assert(ptr != NULL);
 	assert(malloc_initialized() || malloc_is_initializer());
 
+	UTRACE(ptr, 0, 0);
+	if (unlikely(dealloc_no_tsd(ptr))) {
+		LOG("core.dallocx.exit", "");
+		return;
+	}
+
 	tsd_t *tsd = tsd_fetch_min();
 	bool   fast = tsd_fast(tsd);
 	check_entry_exit_locking(tsd_tsdn(tsd));
@@ -1851,7 +1898,6 @@ je_dallocx(void *ptr, int flags) {
 	tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
 	    /* is_alloc */ false);
 
-	UTRACE(ptr, 0, 0);
 	if (likely(fast)) {
 		tsd_assert_fast(tsd);
 		ifree(tsd, ptr, tcache, false);
@@ -1879,6 +1925,12 @@ sdallocx_default(void *ptr, size_t size, int flags) {
 	assert(ptr != NULL);
 	assert(malloc_initialized() || malloc_is_initializer());
 
+	UTRACE(ptr, 0, 0);
+	if (unlikely(dealloc_no_tsd(ptr))) {
+		set_errno(saved_errno);
+		return;
+	}
+
 	tsd_t *tsd = tsd_fetch_min();
 	bool   fast = tsd_fast(tsd);
 	size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
@@ -1888,7 +1940,6 @@ sdallocx_default(void *ptr, size_t size, int flags) {
 	tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
 	    /* is_alloc */ false);
 
-	UTRACE(ptr, 0, 0);
 	if (likely(fast)) {
 		tsd_assert_fast(tsd);
 		isfree(tsd, ptr, usize, tcache, false);

src/large.c

@@ -78,7 +78,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
 		return true;
 	}
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 	arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize);
 
@@ -101,7 +101,7 @@ large_ralloc_no_move_expand(
 	    szind, zero, &deferred_work_generated);
 
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 
 	if (err) {
@@ -247,7 +247,7 @@ large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
 	bool deferred_work_generated = false;
 	pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated);
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 }
 

src/pa.c

@@ -1,6 +1,8 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 
 #include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/background_thread_inlines.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/pa.h"
 
@@ -89,15 +91,21 @@ pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
 	atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
-	pa_shard_flush(tsdn, shard);
+	pa_shard_flush(tsdn, shard, /* all */ false);
 }
 
 void
-pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
+pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard, bool all) {
 	pac_sec_flush(tsdn, &shard->pac);
 	if (shard->ever_used_hpa) {
 		hpa_shard_flush(tsdn, &shard->hpa);
 	}
+	if (all) {
+		pac_decay_all_now(tsdn, &shard->pac, extent_state_dirty);
+		if (pac_should_decay_muzzy(&shard->pac)) {
+			pac_decay_all_now(tsdn, &shard->pac, extent_state_muzzy);
+		}
+	}
 }
 
 static bool
@@ -231,8 +239,8 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
 
 bool
 pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
-    ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
-	return pac_decay_ms_set(tsdn, &shard->pac, state, decay_ms, eagerness);
+    ssize_t decay_ms) {
+	return pac_decay_ms_set(tsdn, &shard->pac, state, decay_ms);
 }
 
 ssize_t
@@ -250,8 +258,28 @@ pa_shard_set_deferral_allowed(
 }
 
 void
-pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
-	if (pa_shard_uses_hpa(shard)) {
+pa_shard_handle_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
+	witness_assert_depth_to_rank(
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
+
+	if (pac_decay_immediately(&shard->pac)) {
+		pac_decay_all_now(tsdn, &shard->pac, extent_state_dirty);
+	}
+	if (background_thread_enabled()) {
+		pac_wake_bg_on_deferred(tsdn, &shard->pac);
+	}
+}
+
+void
+pa_shard_do_deferred_work(
+    tsdn_t *tsdn, pa_shard_t *shard, bool is_background_thread) {
+	pac_do_deferred_work(tsdn, &shard->pac, is_background_thread);
+	/*
+	 * Application threads self-throttle HPA deferred work inline from their
+	 * own alloc/dalloc path (hpa_shard_maybe_do_deferred_work, capped), so
+	 * only drive it (forced, uncapped) from here on the background thread.
+	 */
+	if (is_background_thread && pa_shard_uses_hpa(shard)) {
 		hpa_shard_do_deferred_work(tsdn, &shard->hpa);
 	}
 }
@@ -264,7 +292,7 @@ pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
 uint64_t
 pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
 	uint64_t time = pac_time_until_deferred_work(tsdn, &shard->pac);
-	if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
+	if (time == DEFERRED_WORK_MIN) {
 		return time;
 	}
 

src/pac.c

@@ -2,6 +2,8 @@
 
 #include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/background_thread_inlines.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/pac.h"
 #include "jemalloc/internal/san.h"
@@ -482,10 +484,10 @@ static inline uint64_t
 pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) {
 	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
 		/* Use minimal interval if decay is contended. */
-		return BACKGROUND_THREAD_DEFERRED_MIN;
+		return DEFERRED_WORK_MIN;
 	}
 	uint64_t result = decay_ns_until_purge(
-	    decay, npages, ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD);
+	    decay, npages, PAC_DECAY_PURGE_NPAGES_THRESHOLD);
 
 	malloc_mutex_unlock(tsdn, &decay->mtx);
 	return result;
@@ -497,7 +499,7 @@ pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac) {
 
 	time = pac_ns_until_purge(
 	    tsdn, &pac->decay_dirty, ecache_npages_get(&pac->ecache_dirty));
-	if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
+	if (time == DEFERRED_WORK_MIN) {
 		return time;
 	}
 
@@ -779,9 +781,238 @@ pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
 	return epoch_advanced;
 }
 
+/*
+ * Run the deferred (non-forced) decay-purge for a single decay state, taking
+ * decay->mtx via trylock.  Sets *contended when the lock could not be acquired
+ * (in which case the return is false and *npages_new is untouched).  Returns
+ * whether the epoch advanced; when it did, *npages_new is set to the fresh
+ * backlog delta.
+ */
+static bool
+pac_decay_deferred_one(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
+    pac_decay_stats_t *decay_stats, ecache_t *ecache,
+    pac_purge_eagerness_t eagerness, bool *contended, size_t *npages_new) {
+	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
+		/* No need to wait if another thread is in progress. */
+		*contended = true;
+		return false;
+	}
+	*contended = false;
+	bool epoch_advanced = pac_maybe_decay_purge(
+	    tsdn, pac, decay, decay_stats, ecache, eagerness);
+	if (epoch_advanced) {
+		/* Backlog is updated on epoch advance. */
+		*npages_new = decay_epoch_npages_delta(decay);
+	}
+	malloc_mutex_unlock(tsdn, &decay->mtx);
+	return epoch_advanced;
+}
+
+/*
+ * Non-forced deferred decay-purge for both the dirty and muzzy states, at the
+ * given eagerness; corresponding decay->mtx are acquired internally.  Reports
+ * per-state epoch-advance in *result.  JET_EXTERN: exposed for deterministic
+ * unit testing only; production callers use pac_do_deferred_work.
+ */
+JET_EXTERN void
+pac_decay_deferred(tsdn_t *tsdn, pac_t *pac,
+    pac_purge_eagerness_t eagerness, pac_deferred_work_result_t *result) {
+	memset(result, 0, sizeof(*result));
+
+	bool contended;
+	result->dirty_epoch_advanced = pac_decay_deferred_one(tsdn, pac,
+	    &pac->decay_dirty, &pac->stats->decay_dirty, &pac->ecache_dirty,
+	    eagerness, &contended, &result->dirty_npages_new);
+	if (contended) {
+		/* When dirty decay is contended, don't wait on muzzy. */
+		return;
+	}
+
+	if (!pac_should_decay_muzzy(pac)) {
+		return;
+	}
+	result->muzzy_epoch_advanced = pac_decay_deferred_one(tsdn, pac,
+	    &pac->decay_muzzy, &pac->stats->decay_muzzy, &pac->ecache_muzzy,
+	    eagerness, &contended, &result->muzzy_npages_new);
+}
+
+/*
+ * This is the single, deliberate place where PAC reaches jointly into both a
+ * decay_t and a background_thread_info_t.  It is intentionally NOT decoupled
+ * further, for two reasons:
+ *   (a) The early-wake decision is intrinsically a JOINT info+decay
+ *       computation: holding info->mtx (outer) we trylock decay->mtx (inner)
+ *       to read remaining_sleep = wakeup_time - decay->epoch and the
+ *       decay_npages_purge_in() estimate atomically against epoch advance.
+ *       Splitting it would require copying these values across a new API while
+ *       still holding both locks, adding interface surface for no behavioral
+ *       gain.
+ *   (b) It runs on the application FREE path (pac_do_deferred_work ->
+ *       pac_maybe_wake_bg), where perf parity matters most.
+ *
+ * The info->mtx-outer / decay->mtx-inner trylock nesting is load-bearing for
+ * lock ordering; keep it.
+ *
+ * Parity trap: when npages_new == 0 (e.g. an epoch advanced with no new
+ * backlog) the accumulation below is skipped, but the THRESHOLD compare on the
+ * EXISTING info->npages_to_purge_new backlog still runs and can still trigger
+ * an early wakeup.  Do not "simplify" by early-returning when npages_new == 0.
+ */
+static bool
+pac_decay_should_wake_early(tsdn_t *tsdn, decay_t *decay,
+    background_thread_info_t *info, nstime_t *remaining_sleep,
+    size_t npages_new) {
+	malloc_mutex_assert_owner(tsdn, &info->mtx);
+
+	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
+		return false;
+	}
+	if (!decay_gradually(decay)) {
+		malloc_mutex_unlock(tsdn, &decay->mtx);
+		return false;
+	}
+	nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
+	if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
+		malloc_mutex_unlock(tsdn, &decay->mtx);
+		return false;
+	}
+	nstime_subtract(remaining_sleep, &decay->epoch);
+	if (npages_new > 0) {
+		uint64_t npurge_new = decay_npages_purge_in(
+		    decay, remaining_sleep, npages_new);
+		info->npages_to_purge_new += npurge_new;
+	}
+	malloc_mutex_unlock(tsdn, &decay->mtx);
+	return info->npages_to_purge_new
+	    > PAC_DECAY_PURGE_NPAGES_THRESHOLD;
+}
+
+/*
+ * The PAC's base index is, under the current PA/PAC construction contract, the
+ * owning arena index (pa_shard_init asserts base_ind_get(base) == ind).  This
+ * is a current construction invariant, not a permanent PAC guarantee.
+ */
+static unsigned
+pac_ind_get(const pac_t *pac) {
+	return base_ind_get(pac->base);
+}
+
+/*
+ * Notify the background thread that a decay epoch advanced: if it sleeps
+ * indefinitely, wake it now; otherwise wake it early when the projected backlog
+ * crosses the purge threshold before its next scheduled wakeup.  Non-blocking
+ * (trylocks info->mtx) to keep the application free path cheap.
+ */
+static void
+pac_maybe_wake_bg(tsdn_t *tsdn, pac_t *pac, decay_t *decay, size_t npages_new) {
+	background_thread_info_t *info =
+	    background_thread_info_get(pac_ind_get(pac));
+	if (malloc_mutex_trylock(tsdn, &info->mtx)) {
+		/*
+		 * The background thread may hold the mutex for a while; keep this
+		 * non-blocking and leave the work to a future epoch.
+		 */
+		return;
+	}
+	if (!background_thread_is_started(info)) {
+		goto label_done;
+	}
+	nstime_t remaining_sleep;
+	if (background_thread_indefinite_sleep(info)) {
+		background_thread_wakeup_early(info, NULL);
+	} else if (pac_decay_should_wake_early(tsdn, decay, info,
+	               &remaining_sleep, npages_new)) {
+		info->npages_to_purge_new = 0;
+		background_thread_wakeup_early(info, &remaining_sleep);
+	}
+label_done:
+	malloc_mutex_unlock(tsdn, &info->mtx);
+}
+
+/*
+ * A hook prepared for calling in pa: after a deferred work generated, wake the
+ * background thread only when it looks idle (lock-free acquire read of
+ * indefinite_sleep), then run the full early-wake decision under info->mtx.
+ */
+void
+pac_wake_bg_on_deferred(tsdn_t *tsdn, pac_t *pac) {
+	background_thread_info_t *info =
+	    background_thread_info_get(pac_ind_get(pac));
+	if (background_thread_indefinite_sleep(info)) {
+		pac_maybe_wake_bg(
+		    tsdn, pac, &pac->decay_dirty, /* npages_new */ 0);
+	}
+}
+
+static pac_purge_eagerness_t pac_decide_purge_eagerness(
+    bool is_background_thread);
+
+void
+pac_do_deferred_work(tsdn_t *tsdn, pac_t *pac, bool is_background_thread) {
+	/*
+	 * A concurrent background_thread enable/disable (mallctl) can race this
+	 * path: the enable state is read lock-free twice below (for the eagerness
+	 * decision, then the notify guard), so the two reads may disagree.  Worst
+	 * case is benign and self-healing:
+	 *   disabled->enabled: purged immediately, plus a possibly-redundant wake;
+	 *   enabled->disabled: deferred but not notified this pass -- the pages
+	 *     stay in the decay backlog and are reclaimed on the next decay tick or
+	 *     by the bg thread before it stops.
+	 * It stays safe regardless: info is allocated once and never freed; the
+	 * wake is gated by info->mtx + background_thread_is_started(); the bg-thread
+	 * locks here are trylocks; and the purge runs under decay->mtx, which the
+	 * toggle never touches.
+	 */
+	pac_purge_eagerness_t eagerness =
+	    pac_decide_purge_eagerness(is_background_thread);
+	pac_deferred_work_result_t result;
+	pac_decay_deferred(tsdn, pac, eagerness, &result);
+
+	if (have_background_thread && background_thread_enabled()
+	    && !is_background_thread) {
+		if (result.dirty_epoch_advanced) {
+			pac_maybe_wake_bg(tsdn, pac, &pac->decay_dirty,
+			    result.dirty_npages_new);
+		}
+		if (result.muzzy_epoch_advanced) {
+			pac_maybe_wake_bg(tsdn, pac, &pac->decay_muzzy,
+			    result.muzzy_npages_new);
+		}
+	}
+}
+
+void
+pac_decay_all_now(tsdn_t *tsdn, pac_t *pac, extent_state_t state) {
+	decay_t           *decay;
+	pac_decay_stats_t *decay_stats;
+	ecache_t          *ecache;
+	pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache);
+
+	malloc_mutex_lock(tsdn, &decay->mtx);
+	pac_decay_all(
+	    tsdn, pac, decay, decay_stats, ecache, /* fully_decay */ true);
+	malloc_mutex_unlock(tsdn, &decay->mtx);
+}
+
+/*
+ * Decide the unforced decay-purge eagerness.  On the background thread, force
+ * the purge; on an application thread, defer to the background thread when it is
+ * enabled, otherwise purge on the current thread on epoch advance.
+ */
+static pac_purge_eagerness_t
+pac_decide_purge_eagerness(bool is_background_thread) {
+	if (is_background_thread) {
+		return PAC_PURGE_ALWAYS;
+	} else if (background_thread_enabled()) {
+		return PAC_PURGE_NEVER;
+	} else {
+		return PAC_PURGE_ON_EPOCH_ADVANCE;
+	}
+}
+
 bool
 pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
-    ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
+    ssize_t decay_ms) {
 	decay_t           *decay;
 	pac_decay_stats_t *decay_stats;
 	ecache_t          *ecache;
@@ -810,6 +1041,12 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
 	nstime_t cur_time;
 	nstime_init_update(&cur_time);
 	decay_reinit(decay, &cur_time, decay_ms);
+	/*
+	 * decay_ms is only ever set from a non-background thread (mallctl or
+	 * arena init), so decide the eagerness here rather than threading it in.
+	 */
+	pac_purge_eagerness_t eagerness =
+	    pac_decide_purge_eagerness(/* is_background_thread */ false);
 	pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness);
 	malloc_mutex_unlock(tsdn, &decay->mtx);
 

src/pages.c

@@ -826,8 +826,16 @@ init_thp_state(void) {
 	static const char sys_state_never[] = "always madvise [never]\n";
 	char              buf[sizeof(sys_state_madvise)];
 
+#	if defined(O_CLOEXEC)
+	int fd = malloc_open(
+	    "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY | O_CLOEXEC);
+#	else
 	int fd = malloc_open(
 	    "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+	if (fd != -1) {
+		fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
+	}
+#	endif
 	if (fd == -1) {
 		goto label_error;
 	}

src/prof_stack_range.c

@@ -1,174 +0,0 @@
-#include "jemalloc/internal/jemalloc_preamble.h"
-
-#include "jemalloc/internal/malloc_io.h"
-#include "jemalloc/internal/prof_sys.h"
-
-#if defined(__linux__) && defined(JEMALLOC_HAVE_GETTID)
-
-#	include <errno.h>
-#	include <fcntl.h>
-#	include <stdio.h>
-#	include <stdlib.h> // strtoul
-#	include <string.h>
-#	include <unistd.h>
-
-/*
- * Converts a string representing a hexadecimal number to an unsigned long long
- * integer. Functionally equivalent to strtoull() (for base 16) but faster for
- * that case.
- *
- * @param nptr Pointer to the string to be converted.
- * @param endptr Pointer to a pointer to character, which will be set to the
- * character in `nptr` where parsing stopped. Can be NULL.
- * @return The converted unsigned long long integer value.
- */
-static inline unsigned long long int
-strtoull_hex(const char *nptr, char **endptr) {
-	unsigned long long int val = 0;
-	int                    ii = 0;
-	for (; ii < 16; ++ii) {
-		char c = nptr[ii];
-		if (c >= '0' && c <= '9') {
-			val = (val << 4) + (c - '0');
-		} else if (c >= 'a' && c <= 'f') {
-			val = (val << 4) + (c - 'a' + 10);
-		} else {
-			break;
-		}
-	}
-	if (endptr) {
-		*endptr = (char *)(nptr + ii);
-	}
-	return val;
-}
-
-static int
-prof_mapping_containing_addr(uintptr_t addr, const char *maps_path,
-    uintptr_t *mm_start, uintptr_t *mm_end) {
-	int ret = ENOENT; /* not found */
-	*mm_start = *mm_end = 0;
-
-	/*
-     * Each line of /proc/<pid>/maps is:
-     * <start>-<end> <perms> <offset> <dev> <inode> <pathname>
-     *
-     * The fields we care about are always within the first 34 characters so
-     * as long as `buf` contains the start of a mapping line it can always be
-     * parsed.
-     */
-	static const int kMappingFieldsWidth = 34;
-
-	int     fd = -1;
-	char    buf[4096];
-	ssize_t remaining = 0; /* actual number of bytes read to buf */
-	char   *line = NULL;
-
-	while (1) {
-		if (fd < 0) {
-			/* case 0: initial open of maps file */
-			fd = malloc_open(maps_path, O_RDONLY);
-			if (fd < 0) {
-				return errno;
-			}
-
-			remaining = malloc_read_fd(fd, buf, sizeof(buf));
-			if (remaining < 0) {
-				ret = errno;
-				break;
-			} else if (remaining == 0) {
-				break;
-			}
-			line = buf;
-		} else if (line == NULL) {
-			/* case 1: no newline found in buf */
-			remaining = malloc_read_fd(fd, buf, sizeof(buf));
-			if (remaining < 0) {
-				ret = errno;
-				break;
-			} else if (remaining == 0) {
-				break;
-			}
-			line = memchr(buf, '\n', remaining);
-			if (line != NULL) {
-				line++; /* advance to character after newline */
-				remaining -= (line - buf);
-			}
-		} else if (line != NULL && remaining < kMappingFieldsWidth) {
-			/*
-             * case 2: found newline but insufficient characters remaining in
-             * buf
-             */
-			memcpy(buf, line,
-			    remaining); /* copy remaining characters to start of buf */
-			line = buf;
-
-			ssize_t count = malloc_read_fd(
-			    fd, buf + remaining, sizeof(buf) - remaining);
-			if (count < 0) {
-				ret = errno;
-				break;
-			} else if (count == 0) {
-				break;
-			}
-
-			remaining +=
-			    count; /* actual number of bytes read to buf */
-		} else {
-			/* case 3: found newline and sufficient characters to parse */
-
-			/* parse <start>-<end> */
-			char     *tmp = line;
-			uintptr_t start_addr = (uintptr_t)strtoull_hex(
-			    tmp, &tmp);
-			if (addr >= start_addr) {
-				tmp++; /* advance to character after '-' */
-				uintptr_t end_addr = (uintptr_t)strtoull_hex(
-				    tmp, NULL);
-				if (addr < end_addr) {
-					*mm_start = start_addr;
-					*mm_end = end_addr;
-					ret = 0;
-					break;
-				}
-			}
-
-			/* Advance to character after next newline in the current buf. */
-			char *prev_line = line;
-			line = memchr(line, '\n', remaining);
-			if (line != NULL) {
-				line++; /* advance to character after newline */
-				remaining -= (line - prev_line);
-			}
-		}
-	}
-
-	malloc_close(fd);
-	return ret;
-}
-
-int
-prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high) {
-	/*
-     * NOTE: Prior to kernel 4.5 an entry for every thread stack was included in
-     * /proc/<pid>/maps as [STACK:<tid>]. Starting with kernel 4.5 only the main
-     * thread stack remains as the [stack] mapping. For other thread stacks the
-     * mapping is still visible in /proc/<pid>/task/<tid>/maps (though not
-     * labeled as [STACK:tid]).
-     * https://lists.ubuntu.com/archives/kernel-team/2016-March/074681.html
-    */
-	char maps_path[64]; // "/proc/<pid>/task/<tid>/maps"
-	malloc_snprintf(maps_path, sizeof(maps_path), "/proc/%d/task/%d/maps",
-	    getpid(), gettid());
-	return prof_mapping_containing_addr(fp, maps_path, low, high);
-}
-
-#else
-
-int
-prof_thread_stack_range(
-    UNUSED uintptr_t addr, uintptr_t *stack_start, uintptr_t *stack_end) {
-	*stack_start = *stack_end = 0;
-	return ENOENT;
-}
-
-#endif // __linux__

src/prof_sys.c

@@ -28,11 +28,6 @@
 		JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
 #endif
 
-#ifdef JEMALLOC_PROF_FRAME_POINTER
-// execinfo backtrace() as fallback unwinder
-#	include <execinfo.h>
-#endif
-
 /******************************************************************************/
 
 malloc_mutex_t prof_dump_filename_mtx;
@@ -109,103 +104,6 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
 
 	_Unwind_Backtrace(prof_unwind_callback, &data);
 }
-#elif (defined(JEMALLOC_PROF_FRAME_POINTER))
-JEMALLOC_DIAGNOSTIC_PUSH
-JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS
-
-struct stack_range {
-	uintptr_t start;
-	uintptr_t end;
-};
-
-struct thread_unwind_info {
-	struct stack_range stack_range;
-	bool               fallback;
-};
-static __thread struct thread_unwind_info unwind_info = {
-    .stack_range =
-        {
-            .start = 0,
-            .end = 0,
-        },
-    .fallback = false,
-}; /* thread local */
-
-static void
-prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
-	/* fp: 		current stack frame pointer
-	 *
-	 * stack_range:	readable stack memory range for the current thread.
-	 *		Used to validate frame addresses during stack unwinding.
-	 *		For most threads there is a single valid stack range
-	 *		that is fixed at thread creation time.  This may not be
-	 *		the case when folly fibers or boost contexts are used.
-	 *		In those cases fall back to using execinfo backtrace()
-	 *		(DWARF unwind).
-	 */
-
-	/* always safe to get the current stack frame address */
-	uintptr_t fp = (uintptr_t)__builtin_frame_address(0);
-
-	/* new thread - get the stack range */
-	if (!unwind_info.fallback
-	    && unwind_info.stack_range.start == unwind_info.stack_range.end) {
-		if (prof_thread_stack_range(fp, &unwind_info.stack_range.start,
-		        &unwind_info.stack_range.end)
-		    != 0) {
-			unwind_info.fallback = true;
-		} else {
-			assert(fp >= unwind_info.stack_range.start
-			    && fp < unwind_info.stack_range.end);
-		}
-	}
-
-	if (unwind_info.fallback) {
-		goto label_fallback;
-	}
-
-	unsigned ii = 0;
-	while (ii < max_len && fp != 0) {
-		if (fp < unwind_info.stack_range.start
-		    || fp >= unwind_info.stack_range.end) {
-			/*
-			 * Determining the stack range from procfs can be
-			 * relatively expensive especially for programs with
-			 * many threads / shared libraries.  If the stack
-			 * range has changed, it is likely to change again
-			 * in the future (fibers or some other stack
-			 * manipulation).  So fall back to backtrace for this
-			 * thread.
-			 */
-			unwind_info.fallback = true;
-			goto label_fallback;
-		}
-		void *ip = ((void **)fp)[1];
-		if (ip == 0) {
-			break;
-		}
-		vec[ii++] = ip;
-		fp = ((uintptr_t *)fp)[0];
-	}
-	*len = ii;
-	return;
-
-label_fallback:
-	/*
-	 * Using the backtrace from execinfo.h here.  Note that it may get
-	 * redirected to libunwind when a libunwind not built with build-time
-	 * flag --disable-weak-backtrace is linked.
-	 */
-	assert(unwind_info.fallback);
-	int nframes = backtrace(vec, max_len);
-	if (nframes > 0) {
-		*len = nframes;
-	} else {
-		*len = 0;
-	}
-}
-
-JEMALLOC_DIAGNOSTIC_POP
 #elif (defined(JEMALLOC_PROF_GCC))
 JEMALLOC_DIAGNOSTIC_PUSH
 JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS

src/tcache.c

@@ -1044,21 +1044,28 @@ tcache_create_explicit(tsd_t *tsd) {
 bool
 tsd_tcache_enabled_data_init(tsd_t *tsd) {
 	/* Called upon tsd initialization. */
-	tsd_tcache_enabled_set(tsd, opt_tcache);
 	/*
 	 * tcache is not available yet, but we need to set up its tcache_nbins
 	 * in advance.
 	 */
 	tcache_default_settings_init(tsd_tcache_slowp_get(tsd));
-	tsd_slow_update(tsd);
 
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+	if (test_hooks_tsd_bootstrap_hook != NULL) {
+		test_hooks_tsd_bootstrap_hook();
+	}
+#endif
+
+	bool err = false;
 	if (opt_tcache) {
-		/* Trigger tcache init. */
-		return tsd_tcache_data_init(
+		/* Initialize the bins before advertising the tcache. */
+		err = tsd_tcache_data_init(
 			tsd, NULL, tcache_get_default_ncached_max());
 	}
 
-	return false;
+	tsd_tcache_enabled_set(tsd, opt_tcache && !err);
+	tsd_slow_update(tsd);
+	return err;
 }
 
 void
@@ -1227,17 +1234,18 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
 	 * tsd).  Manually trigger decay to avoid pathological cases.  Also
 	 * include arena 0 because the tcache array is allocated from it.
 	 */
-	arena_decay(
-	    tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), false, false);
+	pa_shard_do_deferred_work(tsd_tsdn(tsd),
+	    &arena_get(tsd_tsdn(tsd), 0, false)->pa_shard,
+	    /* is_background_thread */ false);
 
 	if (arena_nthreads_get(arena, false) == 0
 	    && !background_thread_enabled()) {
 		/* Force purging when no threads assigned to the arena anymore. */
-		arena_decay(tsd_tsdn(tsd), arena,
-		    /* is_background_thread */ false, /* all */ true);
+		pa_shard_flush(tsd_tsdn(tsd), &arena->pa_shard,
+		    /* all */ true);
 	} else {
-		arena_decay(tsd_tsdn(tsd), arena,
-		    /* is_background_thread */ false, /* all */ false);
+		pa_shard_do_deferred_work(tsd_tsdn(tsd), &arena->pa_shard,
+		    /* is_background_thread */ false);
 	}
 }
 

src/test_hooks.c

@@ -13,3 +13,8 @@ void (*test_hooks_libc_hook)(void) = NULL;
 
 JEMALLOC_EXPORT
 void (*test_hooks_safety_check_abort)(const char *) = NULL;
+
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+JEMALLOC_EXPORT
+void (*test_hooks_tsd_bootstrap_hook)(void) = NULL;
+#endif

src/tsd.c

@@ -54,6 +54,7 @@ struct tsd_init_head_s {
 };
 
 pthread_key_t   tsd_tsd;
+pthread_key_t   tsd_thread_initialized_tsd;
 tsd_init_head_t tsd_init_head = {
     ql_head_initializer(blocks), MALLOC_MUTEX_INITIALIZER};
 

test/unit/background_thread.c

@@ -107,9 +107,88 @@ TEST_BEGIN(test_background_thread_running) {
 }
 TEST_END
 
+TEST_BEGIN(test_background_thread_arena_reset) {
+	test_skip_if(!have_background_thread);
+
+	test_switch_background_thread_ctl(true);
+
+	unsigned arena_ind;
+	size_t   sz = sizeof(arena_ind);
+	expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Unexpected arenas.create failure");
+	void *p = mallocx(PAGE,
+	    MALLOCX_TCACHE_NONE | MALLOCX_ARENA(arena_ind));
+	expect_ptr_not_null(p, "Unexpected mallocx failure");
+
+	/*
+	 * Resetting an arena takes its background thread through
+	 * started -> paused -> started (background_thread_arena_reset_begin/
+	 * finish).  The reset frees p, so we must not touch it afterwards.
+	 */
+	size_t mib[3];
+	size_t miblen = sizeof(mib) / sizeof(mib[0]);
+	expect_d_eq(mallctlnametomib("arena.0.reset", mib, &miblen), 0,
+	    "Unexpected mallctlnametomib failure");
+	mib[1] = arena_ind;
+	expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
+	    "Unexpected arena reset failure");
+
+#if defined(JEMALLOC_BACKGROUND_THREAD)
+	background_thread_info_t *info = background_thread_info_get(arena_ind);
+	tsd_t                    *tsd = tsd_fetch();
+	malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+	background_thread_state_t st = info->state;
+	malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+	expect_d_eq((int)st, (int)background_thread_started,
+	    "Arena reset must leave the background thread started, not paused");
+#endif
+	expect_zu_gt(n_background_threads, 0,
+	    "Arena reset must not lose background threads");
+
+	test_switch_background_thread_ctl(false);
+}
+TEST_END
+
+TEST_BEGIN(test_background_thread_stats_ctl) {
+	test_skip_if(!have_background_thread);
+	test_skip_if(!config_stats);
+
+	test_switch_background_thread_ctl(true);
+
+	uint64_t epoch = 1;
+	size_t   sz = sizeof(epoch);
+	expect_d_eq(mallctl("epoch", (void *)&epoch, &sz, (void *)&epoch,
+	                sizeof(epoch)),
+	    0, "Unexpected epoch mallctl failure");
+
+	size_t num_threads;
+	sz = sizeof(num_threads);
+	expect_d_eq(mallctl("stats.background_thread.num_threads",
+	                (void *)&num_threads, &sz, NULL, 0),
+	    0, "Unexpected stats.background_thread.num_threads failure");
+	expect_zu_eq(num_threads, n_background_threads,
+	    "Reported num_threads should match n_background_threads");
+
+	uint64_t num_runs;
+	sz = sizeof(num_runs);
+	expect_d_eq(mallctl("stats.background_thread.num_runs",
+	                (void *)&num_runs, &sz, NULL, 0),
+	    0, "Unexpected stats.background_thread.num_runs failure");
+
+	uint64_t run_interval;
+	sz = sizeof(run_interval);
+	expect_d_eq(mallctl("stats.background_thread.run_interval",
+	                (void *)&run_interval, &sz, NULL, 0),
+	    0, "Unexpected stats.background_thread.run_interval failure");
+
+	test_switch_background_thread_ctl(false);
+}
+TEST_END
+
 int
 main(void) {
 	/* Background_thread creation tests reentrancy naturally. */
-	return test_no_reentrancy(
-	    test_background_thread_ctl, test_background_thread_running);
+	return test_no_reentrancy(test_background_thread_ctl,
+	    test_background_thread_running, test_background_thread_arena_reset,
+	    test_background_thread_stats_ctl);
 }

test/unit/background_thread_enable.c

@@ -96,7 +96,52 @@ TEST_BEGIN(test_max_background_threads) {
 }
 TEST_END
 
+static atomic_b_t stress_stop;
+
+static void
+set_background_thread(bool enable) {
+	size_t sz = sizeof(bool);
+	expect_d_eq(mallctl("background_thread", NULL, NULL, &enable, sz), 0,
+	    "Failed to set background_thread");
+}
+
+static void *
+stress_worker(void *arg) {
+	(void)arg;
+	while (!atomic_load_b(&stress_stop, ATOMIC_RELAXED)) {
+		void *p = mallocx(PAGE, MALLOCX_TCACHE_NONE);
+		if (p != NULL) {
+			dallocx(p, MALLOCX_TCACHE_NONE);
+		}
+	}
+	return NULL;
+}
+
+TEST_BEGIN(test_toggle_stress_with_concurrent_alloc) {
+	test_skip_if(!have_background_thread);
+
+	atomic_store_b(&stress_stop, false, ATOMIC_RELAXED);
+	thd_t thd;
+	thd_create(&thd, stress_worker, NULL);
+
+	for (unsigned i = 0; i < 200; i++) {
+		set_background_thread(true);
+		expect_zu_gt(n_background_threads, 0,
+		    "Background threads should be non-zero after enable "
+		    "(cycle %u)", i);
+		set_background_thread(false);
+		expect_zu_eq(n_background_threads, 0,
+		    "Background threads should be zero after disable "
+		    "(cycle %u)", i);
+	}
+
+	atomic_store_b(&stress_stop, true, ATOMIC_RELAXED);
+	thd_join(thd, NULL);
+}
+TEST_END
+
 int
 main(void) {
-	return test_no_reentrancy(test_deferred, test_max_background_threads);
+	return test_no_reentrancy(test_deferred, test_max_background_threads,
+	    test_toggle_stress_with_concurrent_alloc);
 }

test/unit/decay.c

@@ -273,9 +273,67 @@ TEST_BEGIN(test_decay_ns_until_purge) {
 }
 TEST_END
 
+/*
+ * Branch coverage for the decay-level math the early-wake path
+ * (pac_decay_should_wake_early) relies on, exercised here at the public decay
+ * seam with explicit nstime rather than via real-thread timing.
+ */
+TEST_BEGIN(test_decay_early_wake_branches) {
+	decay_t decay;
+	memset(&decay, 0, sizeof(decay));
+
+	nstime_t curtime;
+	nstime_init(&curtime, 0);
+
+	uint64_t decay_ms = 1000;
+	nstime_t decay_nstime;
+	nstime_init(&decay_nstime, decay_ms * 1000 * 1000);
+	expect_false(decay_init(&decay, &curtime, (ssize_t)decay_ms),
+	    "Failed to initialize decay");
+
+	size_t new_pages = 100;
+
+	/*
+	 * Parity trap: with zero new pages the accumulation contributes
+	 * nothing, so decay_npages_purge_in must return 0 regardless of the
+	 * remaining-sleep window.  (The early-wake THRESHOLD compare on the
+	 * EXISTING backlog still runs in pac_decay_should_wake_early; that is
+	 * deliberately kept and documented in src/pac.c.)
+	 */
+	nstime_t window;
+	nstime_copy(&window, &decay_nstime);
+	expect_u64_eq(decay_npages_purge_in(&decay, &window, 0), 0,
+	    "Zero new pages must contribute zero to the purge estimate");
+
+	/*
+	 * A remaining-sleep window at/after the full decay period caps the
+	 * estimate at the new-page count (all of them will have decayed).
+	 */
+	expect_u64_eq(decay_npages_purge_in(&decay, &window, new_pages),
+	    new_pages, "All new pages should be due within the decay window");
+
+	nstime_t beyond;
+	nstime_copy(&beyond, &decay_nstime);
+	nstime_add(&beyond, &decay_nstime);
+	expect_u64_eq(decay_npages_purge_in(&decay, &beyond, new_pages),
+	    new_pages,
+	    "Estimate must cap at the new-page count beyond the decay window");
+
+	/*
+	 * No-backlog branch: with a zero threshold, decay_ns_until_purge is
+	 * unbounded (nothing schedulable).  This mirrors the indefinite-sleep
+	 * case.
+	 */
+	expect_u64_eq(decay_ns_until_purge(&decay, 0, 0),
+	    DECAY_UNBOUNDED_TIME_TO_PURGE,
+	    "Zero threshold must yield an unbounded time-to-purge");
+}
+TEST_END
+
 int
 main(void) {
 	return test(test_decay_init, test_decay_ms_valid,
 	    test_decay_npages_purge_in, test_decay_maybe_advance_epoch,
-	    test_decay_empty, test_decay, test_decay_ns_until_purge);
+	    test_decay_empty, test_decay, test_decay_ns_until_purge,
+	    test_decay_early_wake_branches);
 }

test/unit/emitter.c

@@ -498,13 +498,13 @@ emit_table_row(emitter_t *emitter) {
 	emitter_begin(emitter);
 	emitter_row_t row;
 	emitter_col_t abc = {
-	    emitter_justify_left, 10, emitter_type_title, {0}, {0, 0}};
+	    emitter_justify_left, 10, emitter_type_title};
 	abc.str_val = "ABC title";
 	emitter_col_t def = {
-	    emitter_justify_right, 15, emitter_type_title, {0}, {0, 0}};
+	    emitter_justify_right, 15, emitter_type_title};
 	def.str_val = "DEF title";
 	emitter_col_t ghi = {
-	    emitter_justify_right, 5, emitter_type_title, {0}, {0, 0}};
+	    emitter_justify_right, 5, emitter_type_title};
 	ghi.str_val = "GHI";
 
 	emitter_row_init(&row);
@@ -548,6 +548,230 @@ static const char *table_row_table =
     "789                 false 1011\n"
     "\"a string\"          false  ghi\n";
 
+/*
+ * emitter_row(): columns that carry a json_key render both to the aligned
+ * table (all columns) and, in JSON mode, to "json_key": value pairs in column
+ * order; columns without a json_key are table-only.
+ */
+static void
+emit_row(emitter_t *emitter) {
+	emitter_begin(emitter);
+
+	emitter_row_t row;
+	emitter_row_init(&row);
+
+	/* Table-only column (no json_key set). */
+	emitter_col_t col_label;
+	emitter_col_init(&col_label, &row);
+	col_label.justify = emitter_justify_left;
+	col_label.width = 6;
+	col_label.type = emitter_type_title;
+	col_label.str_val = "row:";
+
+	/* Dual columns: rendered in both table and JSON. */
+	emitter_col_t col_x;
+	emitter_col_init(&col_x, &row);
+	col_x.justify = emitter_justify_right;
+	col_x.width = 6;
+	col_x.type = emitter_type_unsigned;
+	col_x.unsigned_val = 10;
+	col_x.json_key = "x";
+
+	emitter_col_t col_y;
+	emitter_col_init(&col_y, &row);
+	col_y.justify = emitter_justify_right;
+	col_y.width = 6;
+	col_y.type = emitter_type_unsigned;
+	col_y.unsigned_val = 20;
+	col_y.json_key = "y";
+
+	emitter_json_object_kv_begin(emitter, "obj");
+	emitter_row(emitter, &row);
+	emitter_json_object_end(emitter);
+
+	emitter_end(emitter);
+}
+
+static const char *row_json =
+    "{\n"
+    "\t\"obj\": {\n"
+    "\t\t\"x\": 10,\n"
+    "\t\t\"y\": 20\n"
+    "\t}\n"
+    "}\n";
+static const char *row_json_compact = "{\"obj\":{\"x\":10,\"y\":20}}";
+static const char *row_table = "row:      10    20\n";
+
+/*
+ * emitter_sparse_row(): the table side collapses runs of all-zero ("gap") rows
+ * to a single "---" separator (on a gap->non-gap transition and at a trailing
+ * gap), while the JSON side still emits every row's json-keyed columns.
+ */
+static void
+emit_sparse_row(emitter_t *emitter) {
+	emitter_begin(emitter);
+
+	emitter_row_t row;
+	emitter_row_init(&row);
+	emitter_col_t col_v;
+	emitter_col_init(&col_v, &row);
+	col_v.justify = emitter_justify_right;
+	col_v.width = 6;
+	col_v.type = emitter_type_unsigned;
+	col_v.json_key = "v";
+
+	unsigned vals[] = {1, 0, 0, 2, 0, 0};
+	bool     gap[] = {false, true, true, false, true, true};
+
+	emitter_json_array_kv_begin(emitter, "arr");
+	emitter_table_sparse_begin(emitter);
+	for (unsigned i = 0; i < sizeof(vals) / sizeof(vals[0]); i++) {
+		col_v.unsigned_val = vals[i];
+		emitter_json_object_begin(emitter);
+		emitter_sparse_row(emitter, &row, gap[i]);
+		emitter_json_object_end(emitter);
+	}
+	emitter_table_sparse_end(emitter);
+	emitter_json_array_end(emitter);
+
+	emitter_end(emitter);
+}
+
+static const char *sparse_row_json =
+    "{\n"
+    "\t\"arr\": [\n"
+    "\t\t{\n"
+    "\t\t\t\"v\": 1\n"
+    "\t\t},\n"
+    "\t\t{\n"
+    "\t\t\t\"v\": 0\n"
+    "\t\t},\n"
+    "\t\t{\n"
+    "\t\t\t\"v\": 0\n"
+    "\t\t},\n"
+    "\t\t{\n"
+    "\t\t\t\"v\": 2\n"
+    "\t\t},\n"
+    "\t\t{\n"
+    "\t\t\t\"v\": 0\n"
+    "\t\t},\n"
+    "\t\t{\n"
+    "\t\t\t\"v\": 0\n"
+    "\t\t}\n"
+    "\t]\n"
+    "}\n";
+static const char *sparse_row_json_compact =
+    "{\"arr\":[{\"v\":1},{\"v\":0},{\"v\":0},{\"v\":2},{\"v\":0},{\"v\":0}]}";
+static const char *sparse_row_table =
+    "     1\n"
+    "                     ---\n"
+    "     2\n"
+    "                     ---\n";
+
+typedef struct {
+	size_t      size;
+	uint64_t    count;
+	uint64_t    prof_count;
+	const char *label;
+	uint64_t    requests;
+} col_table_test_row_t;
+
+#define COL_TABLE_TEST_GETTER(name, member, value_member)                     \
+	static void                                                            \
+	col_table_test_get_##name(const void *vrow, emitter_col_t *col) {       \
+		const col_table_test_row_t *row = vrow;                          \
+		col->value_member = row->member;                                 \
+	}
+COL_TABLE_TEST_GETTER(size, size, size_val)
+COL_TABLE_TEST_GETTER(count, count, uint64_val)
+COL_TABLE_TEST_GETTER(prof_count, prof_count, uint64_val)
+COL_TABLE_TEST_GETTER(label, label, str_val)
+COL_TABLE_TEST_GETTER(requests, requests, uint64_val)
+#undef COL_TABLE_TEST_GETTER
+
+enum {
+	COL_TABLE_TEST_SIZE,
+	COL_TABLE_TEST_COUNT,
+	COL_TABLE_TEST_PROF_COUNT,
+	COL_TABLE_TEST_LABEL,
+	COL_TABLE_TEST_REQUESTS,
+	COL_TABLE_TEST_NCOLS
+};
+
+#define COL_TABLE_TEST_FLAG_PROF (1U << 0)
+
+static const emitter_col_desc_t col_table_test_descs[] = {
+	{NULL, "size", emitter_justify_right, 8, emitter_type_size, 0,
+	    col_table_test_get_size},
+	{"count", "count", emitter_justify_right, 8, emitter_type_uint64,
+	    0, col_table_test_get_count},
+	{"prof_count", "prof", emitter_justify_right, 6, emitter_type_uint64,
+	    COL_TABLE_TEST_FLAG_PROF, col_table_test_get_prof_count},
+	{"label", "label", emitter_justify_right, 7, emitter_type_title, 0,
+	    col_table_test_get_label},
+	{"requests", "requests", emitter_justify_right, 10,
+	    emitter_type_uint64, 0, col_table_test_get_requests},
+};
+_Static_assert(
+    sizeof(col_table_test_descs) / sizeof(col_table_test_descs[0]) ==
+    COL_TABLE_TEST_NCOLS,
+    "col_table_test_descs must match COL_TABLE_TEST_NCOLS");
+
+TEST_BEGIN(test_col_desc_flags) {
+	expect_true(emitter_col_desc_active(
+	    &col_table_test_descs[COL_TABLE_TEST_COUNT], 0),
+	    "A column without requirements should always be active");
+	expect_false(emitter_col_desc_active(
+	    &col_table_test_descs[COL_TABLE_TEST_PROF_COUNT], 0),
+	    "A column with an unmet requirement should be inactive");
+	expect_true(emitter_col_desc_active(
+	    &col_table_test_descs[COL_TABLE_TEST_PROF_COUNT],
+	    COL_TABLE_TEST_FLAG_PROF),
+	    "A column with a satisfied requirement should be active");
+}
+TEST_END
+
+static void
+emit_col_table(emitter_t *emitter) {
+	col_table_test_row_t value = {42, 7, 100, "row", 9};
+	emitter_row_t row, header_row;
+	emitter_col_t cols[COL_TABLE_TEST_NCOLS];
+	emitter_col_t header_cols[COL_TABLE_TEST_NCOLS];
+
+	emitter_begin(emitter);
+	emitter_col_table_build(col_table_test_descs, COL_TABLE_TEST_NCOLS,
+	    COL_TABLE_TEST_FLAG_PROF, &row, cols, &header_row, header_cols);
+	emitter_col_table_header(
+	    emitter, &header_row, &header_cols[0], "mock:", "rows");
+	emitter_col_table_fill(col_table_test_descs, COL_TABLE_TEST_NCOLS,
+	    COL_TABLE_TEST_FLAG_PROF, cols, &value);
+	emitter_json_object_begin(emitter);
+	emitter_col_table_emit_json(emitter, col_table_test_descs,
+	    COL_TABLE_TEST_NCOLS, COL_TABLE_TEST_FLAG_PROF, cols);
+	emitter_json_object_end(emitter);
+	emitter_table_row(emitter, &row);
+	emitter_json_array_end(emitter);
+	emitter_end(emitter);
+}
+
+static const char *col_table_json =
+    "{\n"
+    "\t\"rows\": [\n"
+    "\t\t{\n"
+    "\t\t\t\"count\": 7,\n"
+    "\t\t\t\"prof_count\": 100,\n"
+    "\t\t\t\"label\": \"row\",\n"
+    "\t\t\t\"requests\": 9\n"
+    "\t\t}\n"
+    "\t]\n"
+    "}\n";
+static const char *col_table_json_compact =
+    "{\"rows\":[{\"count\":7,\"prof_count\":100,\"label\":\"row\","
+    "\"requests\":9}]}";
+static const char *col_table_table =
+    "mock:size   count  prof  label  requests\n"
+    "      42       7   100    row         9\n";
+
 #define GENERATE_TEST(feature)                                                 \
 	TEST_BEGIN(test_##feature) {                                           \
 		expect_emit_output(emit_##feature, feature##_json,             \
@@ -563,10 +787,14 @@ GENERATE_TEST(modal)
 GENERATE_TEST(json_array)
 GENERATE_TEST(json_nested_array)
 GENERATE_TEST(table_row)
+GENERATE_TEST(row)
+GENERATE_TEST(sparse_row)
+GENERATE_TEST(col_table)
 
 int
 main(void) {
 	return test_no_reentrancy(test_dict, test_table_printf,
 	    test_nested_dict, test_types, test_modal, test_json_array,
-	    test_json_nested_array, test_table_row);
+	    test_json_nested_array, test_table_row, test_row, test_sparse_row,
+	    test_col_table, test_col_desc_flags);
 }

test/unit/fork.c

@@ -568,10 +568,98 @@ TEST_BEGIN(test_fork_postfork_double_fork) {
 }
 TEST_END
 
+#ifndef _WIN32
+static bool
+get_background_thread_enabled(void) {
+	bool   enabled = false;
+	size_t sz = sizeof(enabled);
+	if (mallctl("background_thread", (void *)&enabled, &sz, NULL, 0) != 0) {
+		return false;
+	}
+	return enabled;
+}
+
+static bool
+set_background_thread_enabled(bool enabled) {
+	return mallctl("background_thread", NULL, NULL, &enabled,
+	           sizeof(enabled)) == 0;
+}
+#endif
+
+TEST_BEGIN(test_fork_background_thread) {
+#ifndef _WIN32
+	test_skip_if(!have_background_thread);
+
+	/* Enable the background thread in the parent. */
+	bool enabled = true;
+	expect_d_eq(mallctl("background_thread", NULL, NULL, &enabled,
+	                sizeof(enabled)),
+	    0, "Unexpected mallctl() failure");
+	expect_zu_gt(n_background_threads, 0,
+	    "Number of background threads should be non zero after enabling.");
+
+	pid_t pid = fork();
+	if (pid == -1) {
+		test_fail("Unexpected fork() failure");
+	} else if (pid == 0) {
+		/*
+		 * Child: postfork_child disables the background thread, but the
+		 * allocator must remain usable and re-init capable.
+		 */
+		if (get_background_thread_enabled()) {
+			/* Should report disabled in the child. */
+			_exit(1);
+		}
+
+		/* Several malloc/free round-trips to confirm usability. */
+		for (unsigned i = 0; i < 16; i++) {
+			void *p = mallocx(64, MALLOCX_TCACHE_NONE);
+			if (p == NULL) {
+				_exit(2);
+			}
+			dallocx(p, MALLOCX_TCACHE_NONE);
+		}
+
+		void *q = malloc(128);
+		if (q == NULL) {
+			_exit(3);
+		}
+		free(q);
+
+		/* Re-enable the background thread in the child. */
+		if (!set_background_thread_enabled(true)) {
+			_exit(4);
+		}
+		if (!get_background_thread_enabled()) {
+			_exit(5);
+		}
+
+		/* And confirm allocation still works afterwards. */
+		void *r = mallocx(256, MALLOCX_TCACHE_NONE);
+		if (r == NULL) {
+			_exit(6);
+		}
+		dallocx(r, MALLOCX_TCACHE_NONE);
+
+		_exit(0);
+	} else {
+		/* Parent: keeps its background threads across the fork. */
+		wait_for_child_exit(pid);
+		expect_true(get_background_thread_enabled(),
+		    "Parent should still have background thread enabled.");
+		expect_zu_gt(n_background_threads, 0,
+		    "Parent should still have background threads after fork.");
+	}
+#else
+	test_skip("fork(2) is irrelevant to Windows");
+#endif
+}
+TEST_END
+
 int
 main(void) {
 	return test_no_reentrancy(test_fork, test_fork_child_usability,
 	    test_fork_multithreaded, test_fork_postfork_descriptor_relink,
 	    test_fork_postfork_descriptor_relink_multithreaded,
-	    test_fork_postfork_double_fork);
+	    test_fork_postfork_double_fork, test_fork_background_thread);
 }

test/unit/hpa.c

@@ -1,5 +1,6 @@
 #include "test/jemalloc_test.h"
 
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/nstime.h"
 
@@ -1131,14 +1132,14 @@ TEST_BEGIN(test_deferred_until_time) {
 	/* Current time = 900ms, purge_eligible at 1300ms */
 	nstime_init(&defer_curtime, 900UL * 1000 * 1000);
 	uint64_t until_ns = hpa_time_until_deferred_work(tsdn, shard);
-	expect_u64_eq(until_ns, BACKGROUND_THREAD_DEFERRED_MIN,
+	expect_u64_eq(until_ns, DEFERRED_WORK_MIN,
 	    "First pass did not happen");
 
 	/* Fake that first pass happened more than min_purge_interval_ago */
 	nstime_init(&shard->last_purge, 350UL * 1000 * 1000);
 	shard->stats.npurge_passes = 1;
 	until_ns = hpa_time_until_deferred_work(tsdn, shard);
-	expect_u64_eq(until_ns, BACKGROUND_THREAD_DEFERRED_MIN,
+	expect_u64_eq(until_ns, DEFERRED_WORK_MIN,
 	    "No need to heck anything it is more than interval");
 
 	nstime_init(&shard->last_purge, 900UL * 1000 * 1000);

test/unit/mallctl.c

@@ -489,7 +489,6 @@ TEST_BEGIN(test_mallctl_config) {
 	TEST_MALLCTL_CONFIG(prof, bool);
 	TEST_MALLCTL_CONFIG(prof_libgcc, bool);
 	TEST_MALLCTL_CONFIG(prof_libunwind, bool);
-	TEST_MALLCTL_CONFIG(prof_frameptr, bool);
 	TEST_MALLCTL_CONFIG(stats, bool);
 	TEST_MALLCTL_CONFIG(utrace, bool);
 	TEST_MALLCTL_CONFIG(xmalloc, bool);
@@ -764,12 +763,17 @@ TEST_BEGIN(test_thread_arena) {
 		    0, "Unexpected mallctl() failure");
 		new_arena_ind = percpu_arena_ind_limit(opt_percpu_arena) - 1;
 		if (old_arena_ind != new_arena_ind) {
+			/*
+			 * Setting thread.arena to an index within the per-CPU
+			 * range resumes automatic per-CPU selection rather than
+			 * failing (see test/unit/percpu_arena_resume.c).
+			 */
 			expect_d_eq(
 			    mallctl("thread.arena", (void *)&old_arena_ind, &sz,
 			        (void *)&new_arena_ind, sizeof(unsigned)),
-			    EPERM,
-			    "thread.arena ctl "
-			    "should not be allowed with percpu arena");
+			    0,
+			    "thread.arena within the per-CPU range should "
+			    "resume per-CPU selection");
 		}
 	}
 }

test/unit/pac.c

@@ -2,6 +2,13 @@
 
 #include "jemalloc/internal/pa.h"
 
+/*
+ * pac_decay_deferred is JET_EXTERN (test-only; see src/pac.c) and has no public
+ * header declaration, so declare it locally for the deterministic test below.
+ */
+extern void pac_decay_deferred(tsdn_t *tsdn, pac_t *pac,
+    pac_purge_eagerness_t eagerness, pac_deferred_work_result_t *result);
+
 #define PAC_TEST_ARENA_IND 124
 
 static bool pac_test_dalloc_fail;
@@ -114,12 +121,14 @@ pac_test_data_init(bool custom_hooks, ssize_t dirty_decay_ms,
 
 	nstime_t time;
 	nstime_init(&time, 0);
-	assert_false(pa_central_init(&data->central, data->base, /* hpa */ false,
-	    &hpa_hooks_default), "Unexpected pa_central_init failure");
+	assert_false(pa_central_init(&data->central, data->base,
+	    /* hpa */ false, &hpa_hooks_default),
+	    "Unexpected pa_central_init failure");
 	assert_false(pa_shard_init(tsdn, &data->shard, &data->central,
 	    &data->emap, data->base, PAC_TEST_ARENA_IND, &data->stats,
 	    &data->stats_mtx, &time, SC_LARGE_MAXCLASS + PAGE,
-	    dirty_decay_ms, muzzy_decay_ms), "Unexpected pa_shard_init failure");
+	    dirty_decay_ms, muzzy_decay_ms),
+	    "Unexpected pa_shard_init failure");
 
 	return data;
 }
@@ -382,6 +391,45 @@ TEST_BEGIN(test_pac_large_guarded_dalloc_unguards_before_caching) {
 }
 TEST_END
 
+TEST_BEGIN(test_pac_deferred_never_does_not_purge) {
+	/*
+	 * PAC_PURGE_NEVER is the eagerness the application free path selects
+	 * when a background thread is enabled, meaning current thread does not
+	 * purge.  It is only reachable through the exposed pac_decay_deferred
+	 * primitive (the production entry pac_do_deferred_work picks eagerness
+	 * itself), so this is a deliberate internal test of that critical,
+	 * stable edge.
+	 */
+	pac_test_hooks_reset();
+	/*
+	 * Finite dirty decay so a purge is gated by eagerness, not decay_ms<=0.
+	 */
+	pac_test_data_t *data = pac_test_data_init(
+	    /* custom_hooks */ true, /* dirty_decay_ms */ 100 * 1000, -1);
+	pac_t *pac = &data->shard.pac;
+
+	edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false);
+	size_t ndirty_before_dalloc = ecache_npages_get(&pac->ecache_dirty);
+
+	/* Dalloc caches the extent as dirty and requests deferred work. */
+	bool deferred_work_generated = false;
+	pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated);
+	expect_true(deferred_work_generated,
+	    "Non-pinned dalloc should request deferred work");
+	size_t ndirty_after_dalloc = ecache_npages_get(&pac->ecache_dirty);
+	expect_zu_gt(ndirty_after_dalloc, ndirty_before_dalloc,
+	    "Dalloc should add dirty pages");
+
+	/* PAC_PURGE_NEVER must leave the dirty pages untouched. */
+	pac_deferred_work_result_t result;
+	pac_decay_deferred(tsdn_fetch(), pac, PAC_PURGE_NEVER, &result);
+	expect_zu_eq(ecache_npages_get(&pac->ecache_dirty), ndirty_after_dalloc,
+	    "PAC_PURGE_NEVER must not purge any dirty pages");
+
+	pac_test_data_destroy(data);
+}
+TEST_END
+
 int
 main(void) {
 	return test_no_reentrancy(test_pac_dirty_muzzy_alloc_priority,
@@ -391,5 +439,6 @@ main(void) {
 	    test_pac_decay_retains_when_dalloc_fails,
 	    test_pac_non_pinned_dalloc_signals_deferred_work,
 	    test_pac_non_pinned_shrink_signals_deferred_work,
-	    test_pac_large_guarded_dalloc_unguards_before_caching);
+	    test_pac_large_guarded_dalloc_unguards_before_caching,
+	    test_pac_deferred_never_does_not_purge);
 }

test/unit/pac_decay.c

@@ -1,6 +1,8 @@
 #include "test/jemalloc_test.h"
 #include "test/arena_util.h"
 
+#include "jemalloc/internal/deferral.h"
+#include "jemalloc/internal/pac.h"
 #include "jemalloc/internal/ticker.h"
 
 static nstime_monotonic_t *nstime_monotonic_orig;
@@ -439,8 +441,35 @@ TEST_BEGIN(test_decay_never) {
 }
 TEST_END
 
+/*
+ * pac_time_until_deferred_work is the public PAC scheduling entry the
+ * background thread uses to decide its next wakeup.  Exercised here on a real
+ * arena through the public interface (no direct decay-state manipulation).
+ */
+TEST_BEGIN(test_pac_time_until_deferred_work) {
+	test_skip_if(is_background_thread_enabled());
+	test_skip_if(opt_hpa);
+
+	unsigned arena_ind = do_arena_create(1000, -1);
+	tsdn_t  *tsdn = tsd_tsdn(tsd_fetch());
+	pac_t   *pac = &arena_get(tsdn, arena_ind, false)->pa_shard.pac;
+
+	/* Idle: nothing pending -> the scheduler reports the maximum wait. */
+	expect_u64_eq(pac_time_until_deferred_work(tsdn, pac), DEFERRED_WORK_MAX,
+	    "Idle PAC should defer for the maximum interval");
+
+	/* Dirty pages under a finite decay_ms -> a finite wait is scheduled. */
+	generate_dirty(arena_ind, PAGE);
+	expect_u64_lt(pac_time_until_deferred_work(tsdn, pac), DEFERRED_WORK_MAX,
+	    "Pending dirty decay should schedule a finite deferred-work time");
+
+	do_arena_destroy(arena_ind);
+}
+TEST_END
+
 int
 main(void) {
 	return test(test_decay_ticks, test_decay_ticker,
-	    test_decay_nonmonotonic, test_decay_now, test_decay_never);
+	    test_decay_nonmonotonic, test_decay_now, test_decay_never,
+	    test_pac_time_until_deferred_work);
 }

test/unit/pac_decay.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-export MALLOC_CONF="dirty_decay_ms:1000,muzzy_decay_ms:1000,tcache_max:1024"
+export MALLOC_CONF="dirty_decay_ms:1000,muzzy_decay_ms:1000,tcache_max:1024,background_thread:false"

test/unit/percpu_arena_resume.c

@@ -0,0 +1,81 @@
+#include "test/jemalloc_test.h"
+
+/*
+ * Under percpu_arena, binding a thread to a manual arena (an index at or above
+ * the per-CPU auto range) is one-way: percpu never reclaims it (see
+ * arena_choose_impl). Setting thread.arena back to an index in the auto range
+ * resumes per-CPU selection instead of failing with EPERM.
+ */
+TEST_BEGIN(test_thread_arena_resume_percpu) {
+	test_skip_if(!have_percpu_arena
+	    || !PERCPU_ARENA_ENABLED(opt_percpu_arena));
+
+	unsigned limit = percpu_arena_ind_limit(opt_percpu_arena);
+	/* Bypass the tcache so every allocation and free hits the arena. */
+	const int flags = MALLOCX_TCACHE_NONE;
+
+	void *warm = mallocx(1, 0);
+	expect_ptr_not_null(warm, "Unexpected mallocx() failure");
+	dallocx(warm, 0);
+
+	unsigned cur;
+	size_t sz = sizeof(cur);
+	expect_d_eq(mallctl("thread.arena", (void *)&cur, &sz, NULL, 0), 0,
+	    "Unexpected mallctl() failure");
+	expect_u_lt(cur, limit, "Thread should start on a per-CPU arena");
+
+	unsigned manual;
+	sz = sizeof(manual);
+	expect_d_eq(mallctl("arenas.create", (void *)&manual, &sz, NULL, 0), 0,
+	    "Unexpected arenas.create() failure");
+	expect_u_ge(manual, limit, "A manual arena is outside the per-CPU range");
+
+	unsigned old;
+	sz = sizeof(old);
+	expect_d_eq(mallctl("thread.arena", (void *)&old, &sz, (void *)&manual,
+	    sizeof(manual)), 0, "Binding to a manual arena should be allowed");
+	sz = sizeof(cur);
+	expect_d_eq(mallctl("thread.arena", (void *)&cur, &sz, NULL, 0), 0,
+	    "Unexpected mallctl() failure");
+	expect_u_eq(cur, manual, "Thread should be bound to the manual arena");
+
+	void *p_manual = mallocx(1024, flags);
+	expect_ptr_not_null(p_manual, "Unexpected mallocx() failure");
+	unsigned found;
+	sz = sizeof(found);
+	expect_d_eq(mallctl("arenas.lookup", (void *)&found, &sz,
+	    (void *)&p_manual, sizeof(p_manual)), 0,
+	    "Unexpected arenas.lookup() failure");
+	expect_u_eq(found, manual, "Allocation should come from the manual arena");
+
+	void *scratch = mallocx(1024, flags);
+	expect_ptr_not_null(scratch, "Unexpected mallocx() failure");
+	dallocx(scratch, flags);
+
+	unsigned resume = 0;
+	expect_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&resume,
+	    sizeof(resume)), 0, "Should resume per-CPU selection, not fail");
+	sz = sizeof(cur);
+	expect_d_eq(mallctl("thread.arena", (void *)&cur, &sz, NULL, 0), 0,
+	    "Unexpected mallctl() failure");
+	expect_u_lt(cur, limit, "Thread should be back on a per-CPU arena");
+
+	void *p_percpu = mallocx(1024, flags);
+	expect_ptr_not_null(p_percpu, "Unexpected mallocx() failure");
+	sz = sizeof(found);
+	expect_d_eq(mallctl("arenas.lookup", (void *)&found, &sz,
+	    (void *)&p_percpu, sizeof(p_percpu)), 0,
+	    "Unexpected arenas.lookup() failure");
+	expect_u_lt(found, limit, "Allocation should come from a per-CPU arena");
+	dallocx(p_percpu, flags);
+
+	/* Free the manual-arena region while bound to a different arena. */
+	dallocx(p_manual, flags);
+}
+TEST_END
+
+int
+main(void) {
+	return test(
+	    test_thread_arena_resume_percpu);
+}

test/unit/percpu_arena_resume.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+export MALLOC_CONF="percpu_arena:percpu"

test/unit/tsd.c

@@ -130,6 +130,71 @@ TEST_BEGIN(test_tsd_reincarnation) {
 }
 TEST_END
 
+static bool tsd_bootstrap_reentrant_hook_ran;
+static unsigned tsd_bootstrap_reentrant_hook_runs;
+
+static void
+tsd_bootstrap_reentrant_hook(void) {
+	tsd_bootstrap_reentrant_hook_ran = true;
+	tsd_bootstrap_reentrant_hook_runs++;
+	test_hooks_tsd_bootstrap_hook = NULL;
+
+	void *p = malloc(16);
+	expect_ptr_not_null(p, "Unexpected recursive malloc() failure");
+	free(p);
+}
+
+static void *
+thd_start_reentrant_tsd_bootstrap(void *arg) {
+	(void)arg;
+
+	test_hooks_tsd_bootstrap_hook = tsd_bootstrap_reentrant_hook;
+	void *p = malloc(1);
+	expect_ptr_not_null(p, "Unexpected malloc() failure");
+	free(p);
+	test_hooks_tsd_bootstrap_hook = NULL;
+
+	expect_true(tsd_bootstrap_reentrant_hook_ran,
+	    "TSD bootstrap hook should have executed");
+	return NULL;
+}
+
+static void *
+thd_start_reentrant_tsd_bootstrap_minimal(void *arg) {
+	(void)arg;
+
+	tsd_t *tsd = tsd_fetch_min();
+	expect_u_eq(tsd_state_get(tsd), tsd_state_minimal_initialized,
+	    "TSD should be minimal initialized");
+
+	test_hooks_tsd_bootstrap_hook = tsd_bootstrap_reentrant_hook;
+	void *p = malloc(1);
+	expect_ptr_not_null(p, "Unexpected malloc() failure");
+	free(p);
+	test_hooks_tsd_bootstrap_hook = NULL;
+
+	expect_true(tsd_bootstrap_reentrant_hook_ran,
+	    "TSD bootstrap hook should have executed");
+	return NULL;
+}
+
+TEST_BEGIN(test_tsd_reentrant_bootstrap) {
+	thd_t thd;
+
+	tsd_bootstrap_reentrant_hook_ran = false;
+	tsd_bootstrap_reentrant_hook_runs = 0;
+	thd_create(&thd, thd_start_reentrant_tsd_bootstrap, NULL);
+	thd_join(thd, NULL);
+
+	tsd_bootstrap_reentrant_hook_ran = false;
+	thd_create(&thd, thd_start_reentrant_tsd_bootstrap_minimal, NULL);
+	thd_join(thd, NULL);
+
+	expect_u_eq(tsd_bootstrap_reentrant_hook_runs, 2,
+	    "TSD bootstrap hook should have executed once per case");
+}
+TEST_END
+
 static void *
 thd_start_dalloc_only(void *arg) {
 	void **ptrs = (void **)arg;
@@ -185,6 +250,308 @@ TEST_BEGIN(test_tsd_sub_thread_dalloc_only) {
 }
 TEST_END
 
+#if !defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) \
+    && !defined(_WIN32)
+#define TEST_TSD_AFTER_TEARDOWN
+static const bool skip_tsd_after_teardown = false;
+#else
+static const bool skip_tsd_after_teardown = true;
+#endif
+
+static void *
+tsd_raw_get(void) {
+#ifdef TEST_TSD_AFTER_TEARDOWN
+	return pthread_getspecific(tsd_tsd);
+#else
+	return NULL;
+#endif
+}
+
+static bool
+tsd_raw_clear(void) {
+#ifdef TEST_TSD_AFTER_TEARDOWN
+	return pthread_setspecific(tsd_tsd, NULL) == 0;
+#else
+	return true;
+#endif
+}
+
+static void
+tsd_teardown_for_test(void) {
+	tsd_t *tsd = tsd_fetch();
+	void *tsd_wrapper = tsd_raw_get();
+	expect_ptr_not_null(tsd_wrapper, "TSD wrapper should exist after malloc");
+
+	/*
+	 * Emulate a jemalloc call after the pthread-key destructor has finished
+	 * with the generic TSD wrapper.  This is the state seen by cleanup code
+	 * that runs late in thread teardown on pthread_getspecific()-based
+	 * builds: the first cleanup call moves a nominal TSD to purgatory, where
+	 * deallocation is handled by the existing reincarnation path; the second
+	 * cleanup call emulates the final destructor round, after which the
+	 * wrapper is released and no TSD remains to reincarnate.
+	 */
+	tsd_cleanup(tsd);
+	tsd_cleanup(tsd);
+	malloc_tsd_dalloc(tsd_wrapper);
+	expect_true(tsd_raw_clear(), "Unexpected TSD key clear failure");
+	expect_ptr_null(tsd_raw_get(),
+	    "TSD key should be clear before the late jemalloc call");
+}
+
+static void *
+thd_start_free_after_teardown(void *arg) {
+	(void)arg;
+
+	void *keep = malloc(32);
+	expect_ptr_not_null(keep, "Unexpected malloc() failure");
+	void *free_victim = malloc(32);
+	expect_ptr_not_null(free_victim, "Unexpected malloc() failure");
+	void *dallocx_victim = mallocx(32, 0);
+	expect_ptr_not_null(dallocx_victim, "Unexpected mallocx() failure");
+	void *sdallocx_victim = mallocx(32, 0);
+	expect_ptr_not_null(sdallocx_victim, "Unexpected mallocx() failure");
+
+	tsd_teardown_for_test();
+
+	free(free_victim);
+	expect_ptr_null(tsd_raw_get(),
+	    "free() after TSD teardown must not re-create TSD");
+	dallocx(dallocx_victim, 0);
+	expect_ptr_null(tsd_raw_get(),
+	    "dallocx() after TSD teardown must not re-create TSD");
+	sdallocx(sdallocx_victim, 32, 0);
+	expect_ptr_null(tsd_raw_get(),
+	    "sdallocx() after TSD teardown must not re-create TSD");
+
+	free(keep);
+	return NULL;
+}
+
+TEST_BEGIN(test_tsd_free_after_teardown) {
+	test_skip_if(skip_tsd_after_teardown);
+
+	thd_t thd;
+	thd_create(&thd, thd_start_free_after_teardown, NULL);
+	thd_join(thd, NULL);
+}
+TEST_END
+
+static void
+expect_tsd_recreated(void *p, const char *func) {
+	expect_ptr_not_null(p, "%s() after TSD teardown should still allocate",
+	    func);
+	expect_ptr_not_null(tsd_raw_get(),
+	    "%s() after TSD teardown should preserve TSD reincarnation", func);
+}
+
+static void *
+thd_start_malloc_after_teardown(void *arg) {
+	(void)arg;
+
+	void *keep = malloc(32);
+	expect_ptr_not_null(keep, "Unexpected malloc() failure");
+
+	tsd_teardown_for_test();
+
+	void *p = malloc(32);
+	expect_tsd_recreated(p, "malloc");
+	if (p != NULL) {
+		free(p);
+	}
+
+	free(keep);
+	return NULL;
+}
+
+static void *
+thd_start_mallocx_after_teardown(void *arg) {
+	(void)arg;
+
+	void *keep = malloc(32);
+	expect_ptr_not_null(keep, "Unexpected malloc() failure");
+
+	tsd_teardown_for_test();
+
+	void *p = mallocx(32, 0);
+	expect_tsd_recreated(p, "mallocx");
+	if (p != NULL) {
+		dallocx(p, 0);
+	}
+
+	free(keep);
+	return NULL;
+}
+
+static void *
+thd_start_calloc_after_teardown(void *arg) {
+	(void)arg;
+
+	void *keep = malloc(32);
+	expect_ptr_not_null(keep, "Unexpected malloc() failure");
+
+	tsd_teardown_for_test();
+
+	void *p = calloc(1, 32);
+	expect_tsd_recreated(p, "calloc");
+	if (p != NULL) {
+		free(p);
+	}
+
+	free(keep);
+	return NULL;
+}
+
+static void *
+thd_start_aligned_alloc_after_teardown(void *arg) {
+	(void)arg;
+
+	void *keep = malloc(32);
+	expect_ptr_not_null(keep, "Unexpected malloc() failure");
+
+	tsd_teardown_for_test();
+
+	void *p = aligned_alloc(sizeof(void *), 32);
+	expect_tsd_recreated(p, "aligned_alloc");
+	if (p != NULL) {
+		free(p);
+	}
+
+	free(keep);
+	return NULL;
+}
+
+static void *
+thd_start_posix_memalign_after_teardown(void *arg) {
+	(void)arg;
+
+	void *keep = malloc(32);
+	expect_ptr_not_null(keep, "Unexpected malloc() failure");
+
+	tsd_teardown_for_test();
+
+	void *p = NULL;
+	expect_d_eq(posix_memalign(&p, sizeof(void *), 32), 0,
+	    "posix_memalign() after TSD teardown should still allocate");
+	expect_tsd_recreated(p, "posix_memalign");
+	if (p != NULL) {
+		free(p);
+	}
+
+	free(keep);
+	return NULL;
+}
+
+static void *
+thd_start_realloc_after_teardown(void *arg) {
+	(void)arg;
+
+	void *p = malloc(32);
+	expect_ptr_not_null(p, "Unexpected malloc() failure");
+
+	tsd_teardown_for_test();
+
+	void *ret = realloc(p, 64);
+	expect_tsd_recreated(ret, "realloc");
+	if (ret != NULL) {
+		p = ret;
+	}
+	free(p);
+	return NULL;
+}
+
+static void *
+thd_start_rallocx_after_teardown(void *arg) {
+	(void)arg;
+
+	void *p = mallocx(32, 0);
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
+
+	tsd_teardown_for_test();
+
+	void *ret = rallocx(p, 64, 0);
+	expect_tsd_recreated(ret, "rallocx");
+	if (ret != NULL) {
+		p = ret;
+	}
+	dallocx(p, 0);
+	return NULL;
+}
+
+static void *
+thd_start_xallocx_after_teardown(void *arg) {
+	(void)arg;
+
+	void *p = mallocx(32, 0);
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
+	size_t old_usize = sallocx(p, 0);
+
+	tsd_teardown_for_test();
+
+	expect_zu_ge(xallocx(p, 64, 0, 0), old_usize,
+	    "xallocx() after TSD teardown should preserve behavior");
+	expect_ptr_not_null(tsd_raw_get(),
+	    "xallocx() after TSD teardown should preserve TSD reincarnation");
+	dallocx(p, 0);
+	return NULL;
+}
+
+static void *
+thd_start_realloc_zero_after_teardown(void *arg) {
+	(void)arg;
+
+	void *p = malloc(32);
+	expect_ptr_not_null(p, "Unexpected malloc() failure");
+
+	tsd_teardown_for_test();
+
+	if (opt_zero_realloc_action == zero_realloc_action_abort) {
+		free(p);
+		expect_ptr_null(tsd_raw_get(),
+		    "free() after TSD teardown must not re-create TSD");
+		return NULL;
+	}
+
+	void *ret = realloc(p, 0);
+	if (opt_zero_realloc_action == zero_realloc_action_free) {
+		expect_ptr_null(ret, "realloc(ptr, 0) should return NULL");
+		expect_ptr_null(tsd_raw_get(),
+		    "realloc(ptr, 0) after TSD teardown must not re-create TSD");
+	} else {
+		expect_tsd_recreated(ret, "realloc(ptr, 0)");
+		if (ret != NULL) {
+			p = ret;
+		}
+		free(p);
+	}
+	return NULL;
+}
+
+TEST_BEGIN(test_tsd_malloc_after_teardown) {
+	test_skip_if(skip_tsd_after_teardown);
+
+	thd_t thd;
+	thd_create(&thd, thd_start_malloc_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_mallocx_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_calloc_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_aligned_alloc_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_posix_memalign_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_realloc_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_rallocx_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_xallocx_after_teardown, NULL);
+	thd_join(thd, NULL);
+	thd_create(&thd, thd_start_realloc_zero_after_teardown, NULL);
+	thd_join(thd, NULL);
+}
+TEST_END
+
 int
 main(void) {
 	/* Ensure tsd bootstrapped. */
@@ -194,5 +561,7 @@ main(void) {
 	}
 
 	return test_no_reentrancy(test_tsd_main_thread, test_tsd_sub_thread,
-	    test_tsd_sub_thread_dalloc_only, test_tsd_reincarnation);
+	    test_tsd_sub_thread_dalloc_only, test_tsd_reincarnation,
+	    test_tsd_reentrant_bootstrap,
+	    test_tsd_free_after_teardown, test_tsd_malloc_after_teardown);
 }