Expose stats table columns in JSON output and simplify code

The extents and hpa-nonfull "ind" columns are emitter_type_unsigned but were
set via col_ind.size_val -- correct only by little-endian union overlap.
Output is byte-identical.  Pre-existing (predates the refactor).

.github/workflows/linux-ci.yml

@@ -508,6 +508,26 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: force_tls=0
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: "force_tls=0 --enable-debug"
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: "--enable-cxx-infallible-new --enable-debug"
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
     - uses: actions/checkout@v6
@@ -637,6 +657,11 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=background_thread:true"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/macos-ci.yml

@@ -60,6 +60,11 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
     - uses: actions/checkout@v6
@@ -162,6 +167,11 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/windows-ci.yml

@@ -52,6 +52,11 @@ jobs:
               CXX: cl.exe
               CROSS_COMPILE_32BIT: yes
               CONFIGURE_FLAGS: --enable-debug
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: -fcommon
 
     steps:
     - uses: actions/checkout@v6

INSTALL.md

@@ -139,7 +139,6 @@ any of the following arguments (not a definitive list) to 'configure':
     in the following list that appears to function correctly:
 
     + libunwind      (requires --enable-prof-libunwind)
-    + frame pointer  (requires --enable-prof-frameptr)
     + libgcc         (unless --disable-prof-libgcc)
     + gcc intrinsics (unless --disable-prof-gcc)
 
@@ -148,12 +147,6 @@ any of the following arguments (not a definitive list) to 'configure':
     Use the libunwind library (http://www.nongnu.org/libunwind/) for stack
     backtracing.
 
-* `--enable-prof-frameptr`
-
-    Use the optimized frame pointer unwinder for stack backtracing. Safe
-    to use in mixed code (with and without frame pointers) - but requires
-    frame pointers to produce meaningful stacks. Linux only.
-
 * `--disable-prof-libgcc`
 
     Disable the use of libgcc's backtracing functionality.
@@ -217,6 +210,15 @@ any of the following arguments (not a definitive list) to 'configure':
     Disable C++ integration.  This will cause new and delete operator
     implementations to be omitted.
 
+* `--enable-cxx-infallible-new`
+
+    Make the throwing `operator new` abort on allocation failure (logging the
+    requested size) instead of throwing `std::bad_alloc`; the `std::nothrow`
+    overloads still return null.  Disabled by default, and has no effect when
+    C++ integration is disabled (`--disable-cxx`).  When enabled, under LTO
+    this lets the compiler treat `operator new` as non-throwing and elide
+    exception-handling cleanup in callers.
+
 * `--with-xslroot=<path>`
 
     Specify where to find DocBook XSL stylesheets when building the

Makefile.in

@@ -146,7 +146,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/prof_data.c \
 	$(srcroot)src/prof_log.c \
 	$(srcroot)src/prof_recent.c \
-	$(srcroot)src/prof_stack_range.c \
 	$(srcroot)src/prof_stats.c \
 	$(srcroot)src/prof_sys.c \
 	$(srcroot)src/psset.c \
@@ -204,7 +203,6 @@ C_UTIL_CPP_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c
 endif
 TESTS_UNIT := \
 	$(srcroot)test/unit/a0.c \
-	$(srcroot)test/unit/arena_decay.c \
 	$(srcroot)test/unit/arena_reset.c \
 	$(srcroot)test/unit/arenas_management.c \
 	$(srcroot)test/unit/atomic.c \
@@ -272,9 +270,12 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
 	$(srcroot)test/unit/pac.c \
+	$(srcroot)test/unit/pac_decay.c \
+	$(srcroot)test/unit/pac_sec_integration.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/peak.c \
+	$(srcroot)test/unit/percpu_arena_resume.c \
 	$(srcroot)test/unit/ph.c \
 	$(srcroot)test/unit/prng.c \
 	$(srcroot)test/unit/prof_accum.c \
@@ -330,6 +331,7 @@ endif
 TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
 	$(srcroot)test/integration/allocated.c \
 	$(srcroot)test/integration/extent.c \
+	$(srcroot)test/integration/free_sized.c \
 	$(srcroot)test/integration/malloc.c \
 	$(srcroot)test/integration/mallocx.c \
 	$(srcroot)test/integration/MALLOCX_ARENA.c \
@@ -349,8 +351,11 @@ ifeq (@enable_cxx@, 1)
 CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
 TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
 ifeq (@enable_cxx_exceptions@, 1)
-TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new_true.cpp \
-	$(srcroot)test/integration/cpp/infallible_new_false.cpp
+ifeq (@enable_cxx_infallible_new@, 1)
+TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new.cpp
+else
+TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/failing_new.cpp
+endif
 endif
 else
 CPP_SRCS :=

configure.ac

@@ -396,8 +396,31 @@ if test "x$enable_cxx" = "x1"; then
     enable_cxx_exceptions="0"
   fi
 fi
+
+dnl Do not enable infallible-new by default. When enabled, the throwing
+dnl operator new aborts on OOM (with size logged) instead of throwing
+dnl std::bad_alloc; nothrow new still returns null. With LTO this lets the
+dnl compiler prove operator new doesn't throw and elide exception cleanup.
+AC_ARG_ENABLE([cxx-infallible-new],
+  [AS_HELP_STRING([--enable-cxx-infallible-new],
+    [Abort on OOM in throwing operator new instead of throwing std::bad_alloc])],
+[if test "x$enable_cxx_infallible_new" = "xno" ; then
+  enable_cxx_infallible_new="0"
+else
+  enable_cxx_infallible_new="1"
+fi
+],
+[enable_cxx_infallible_new="0"]
+)
+if test "x$enable_cxx" != "x1" ; then
+  enable_cxx_infallible_new="0"
+fi
+if test "x$enable_cxx_infallible_new" = "x1" ; then
+  AC_DEFINE([JEMALLOC_INFALLIBLE_NEW], [ ], [ ])
+fi
 AC_SUBST([enable_cxx])
 AC_SUBST([enable_cxx_exceptions])
+AC_SUBST([enable_cxx_infallible_new])
 AC_SUBST([CONFIGURE_CXXFLAGS])
 AC_SUBST([SPECIFIED_CXXFLAGS])
 AC_SUBST([EXTRA_CXXFLAGS])
@@ -510,13 +533,14 @@ esac
 AC_DEFINE_UNQUOTED([HAVE_CPU_SPINWAIT], [$HAVE_CPU_SPINWAIT], [ ])
 AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT], [ ])
 
+dnl LG_VADDR: number of significant virtual address bits. Auto-detected when --with-lg-vaddr is left unset.
 AC_ARG_WITH([lg_vaddr],
   [AS_HELP_STRING([--with-lg-vaddr=<lg-vaddr>], [Number of significant virtual address bits])],
   [LG_VADDR="$with_lg_vaddr"], [LG_VADDR="detect"])
 
-case "${host_cpu}" in
-  aarch64)
-    if test "x$LG_VADDR" = "xdetect"; then
+if test "x$LG_VADDR" = "xdetect"; then
+  case "${host_cpu}" in
+    aarch64)
       AC_MSG_CHECKING([number of significant virtual address bits])
       if test "x${LG_SIZEOF_PTR}" = "x2" ; then
         #aarch64 ILP32
@@ -526,10 +550,8 @@ case "${host_cpu}" in
         LG_VADDR=48
       fi
       AC_MSG_RESULT([$LG_VADDR])
-    fi
-    ;;
-  x86_64)
-    if test "x$LG_VADDR" = "xdetect"; then
+      ;;
+    x86_64)
       AC_CACHE_CHECK([number of significant virtual address bits],
                      [je_cv_lg_vaddr],
                      AC_RUN_IFELSE([AC_LANG_PROGRAM(
@@ -569,35 +591,14 @@ typedef unsigned __int32 uint32_t;
                    [je_cv_lg_vaddr=`cat conftest.out`],
                    [je_cv_lg_vaddr=error],
                    [je_cv_lg_vaddr=57]))
+      if test "x${je_cv_lg_vaddr}" = "xerror" ; then
+        AC_MSG_ERROR([cannot determine number of significant virtual address bits])
+      fi
       if test "x${je_cv_lg_vaddr}" != "x" ; then
         LG_VADDR="${je_cv_lg_vaddr}"
       fi
-      if test "x${LG_VADDR}" != "xerror" ; then
-        AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
-      else
-        AC_MSG_ERROR([cannot determine number of significant virtual address bits])
-      fi
-      AC_CACHE_CHECK([rdtscp support],
-		     [je_cv_rdtscp],
-		     AC_RUN_IFELSE([AC_LANG_PROGRAM(
-[[
-#include <stdint.h>
-]],
-[[
-      unsigned int dx;
-      asm volatile("rdtscp" : "=d"(dx) ::);
-      return 0;
-]])],
-      [je_cv_rdtscp=yes],
-      [je_cv_rdtscp=no],
-      [je_cv_rdtscp=no]))
-      if test "x${je_cv_rdtscp}" = "xyes"; then
-        AC_DEFINE([JEMALLOC_HAVE_RDTSCP], [ ], [ ])
-      fi
-    fi
-    ;;
-  *)
-    if test "x$LG_VADDR" = "xdetect"; then
+      ;;
+    *)
       AC_MSG_CHECKING([number of significant virtual address bits])
       if test "x${LG_SIZEOF_PTR}" = "x3" ; then
         LG_VADDR=64
@@ -609,10 +610,34 @@ typedef unsigned __int32 uint32_t;
         AC_MSG_ERROR([Unsupported lg(pointer size): ${LG_SIZEOF_PTR}])
       fi
       AC_MSG_RESULT([$LG_VADDR])
+      ;;
+  esac
+fi
+AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
+
+dnl rdtscp feature probe for x86_64
+case "${host_cpu}" in
+  x86_64)
+    AC_CACHE_CHECK([rdtscp support],
+		   [je_cv_rdtscp],
+		   AC_RUN_IFELSE([AC_LANG_PROGRAM(
+[[
+#include <stdint.h>
+]],
+[[
+      unsigned int dx;
+      asm volatile("rdtscp" : "=d"(dx) ::);
+      return 0;
+]])],
+    [je_cv_rdtscp=yes],
+    [je_cv_rdtscp=no],
+    [je_cv_rdtscp=no]))
+    if test "x${je_cv_rdtscp}" = "xyes"; then
+      AC_DEFINE([JEMALLOC_HAVE_RDTSCP], [ ], [ ])
     fi
     ;;
 esac
-AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
+
 AC_CACHE_CHECK([asm volatile support],
                [je_cv_asm_volatile],
                AC_RUN_IFELSE([AC_LANG_PROGRAM(
@@ -1532,33 +1557,6 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then
   fi
 fi
 
-if test `uname -s` = "Linux"
-then
-  AC_ARG_ENABLE([prof-frameptr],
-    [AS_HELP_STRING([--enable-prof-frameptr], [Use optimized frame pointer unwinder for backtracing (Linux only)])],
-  [if test "x$enable_prof_frameptr" = "xno" ; then
-    enable_prof_frameptr="0"
-  else
-    enable_prof_frameptr="1"
-    if test "x$enable_prof" = "x0" ; then
-      AC_MSG_ERROR([--enable-prof-frameptr should only be used with --enable-prof])
-    fi
-  fi
-  ],
-  [enable_prof_frameptr="0"]
-  )
-  if test "x$backtrace_method" = "x" -a "x$enable_prof_frameptr" = "x1" \
-      -a "x$GCC" = "xyes" ; then
-    JE_CFLAGS_ADD([-fno-omit-frame-pointer])
-    backtrace_method="frame pointer linux"
-    AC_DEFINE([JEMALLOC_PROF_FRAME_POINTER], [ ], [ ])
-  else
-    enable_prof_frameptr="0"
-  fi
-else
-  enable_prof_frameptr="0"
-fi
-
 AC_ARG_ENABLE([prof-libgcc],
   [AS_HELP_STRING([--disable-prof-libgcc],
   [Do not use libgcc for backtracing])],
@@ -2201,6 +2199,43 @@ dnl Check if we have dlsym support.
   if test "x${je_cv_pthread_get_name_np}" = "xyes" ; then
     AC_DEFINE([JEMALLOC_HAVE_PTHREAD_GET_NAME_NP], [ ], [ ])
   fi
+  dnl Runtime-check pthread_condattr_setclock and clock_gettime with
+  dnl CLOCK_MONOTONIC to confirm the kernel actually supports the
+  dnl monotonic clock.
+  AC_CACHE_CHECK([whether pthread_condattr_setclock with CLOCK_MONOTONIC is usable],
+                 [je_cv_pthread_cond_timedwait_monotonic],
+                 AC_RUN_IFELSE([AC_LANG_PROGRAM(
+[[
+#include <pthread.h>
+#include <time.h>
+]], [[
+  pthread_condattr_t attr;
+  pthread_cond_t cond;
+  struct timespec ts;
+  int ret = pthread_condattr_init(&attr);
+  if (ret != 0) {
+    return 1;
+  }
+  ret = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC);
+  if (ret != 0) {
+    pthread_condattr_destroy(&attr);
+    return 1;
+  }
+  ret = pthread_cond_init(&cond, &attr);
+  pthread_condattr_destroy(&attr);
+  if (ret != 0) {
+    return 1;
+  }
+  ret = clock_gettime(CLOCK_MONOTONIC, &ts);
+  pthread_cond_destroy(&cond);
+  return ret != 0;
+]])],
+                 [je_cv_pthread_cond_timedwait_monotonic=yes],
+                 [je_cv_pthread_cond_timedwait_monotonic=no],
+                 [je_cv_pthread_cond_timedwait_monotonic=no]))
+  if test "x${je_cv_pthread_cond_timedwait_monotonic}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC], [ ], [ ])
+  fi
 fi
 
 JE_APPEND_VS(CPPFLAGS, -D_REENTRANT)
@@ -3124,7 +3159,6 @@ AC_MSG_RESULT([user_config        : ${enable_user_config}])
 AC_MSG_RESULT([experimental_smallocx : ${enable_experimental_smallocx}])
 AC_MSG_RESULT([prof               : ${enable_prof}])
 AC_MSG_RESULT([prof-libunwind     : ${enable_prof_libunwind}])
-AC_MSG_RESULT([prof-frameptr      : ${enable_prof_frameptr}])
 AC_MSG_RESULT([prof-libgcc        : ${enable_prof_libgcc}])
 AC_MSG_RESULT([prof-gcc           : ${enable_prof_gcc}])
 AC_MSG_RESULT([fill               : ${enable_fill}])

doc/jemalloc.xml.in

@@ -846,6 +846,16 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
         build configuration.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="config.infallible_new">
+        <term>
+          <mallctl>config.infallible_new</mallctl>
+          (<type>bool</type>)
+          <literal>r-</literal>
+        </term>
+        <listitem><para><option>--enable-cxx-infallible-new</option> was
+        specified during build configuration.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="config.lazy_lock">
         <term>
           <mallctl>config.lazy_lock</mallctl>

include/jemalloc/internal/arena.h

@@ -177,12 +177,6 @@ struct arena_s {
 /* EXTERNS */
 /******************************************************************************/
 
-/*
- * When the amount of pages to be purged exceeds this amount, deferred purge
- * should happen.
- */
-#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
-
 extern ssize_t opt_dirty_decay_ms;
 extern ssize_t opt_muzzy_decay_ms;
 
@@ -213,7 +207,6 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
     hpa_shard_stats_t *hpastats);
-void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
 edata_t *arena_extent_alloc_large(
     tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
 void arena_extent_dalloc_large_prep(
@@ -222,13 +215,6 @@ void arena_extent_ralloc_large_shrink(
     tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
 void arena_extent_ralloc_large_expand(
     tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
-bool arena_decay_ms_set(
-    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
-ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
-void    arena_decay(
-       tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-uint64_t       arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
-void           arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
 void           arena_reset(tsd_t *tsd, arena_t *arena);
 void           arena_destroy(tsd_t *tsd, arena_t *arena);
 cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
@@ -240,7 +226,6 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
 void  arena_prof_promote(
      tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
 size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
-void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
 
 void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
 void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
@@ -274,8 +259,6 @@ void   arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
        cache_bin_array_descriptor_t *desc);
 void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
        cache_bin_array_descriptor_t *desc);
-void   arena_cache_bin_array_postfork_child(arena_t *arena,
-       cache_bin_array_descriptor_t *desc_or_null);
 void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);

include/jemalloc/internal/arena_inlines.h

@@ -308,7 +308,7 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
 	uint64_t      *prng_state = tsd_prng_statep_get(tsd);
 	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
 	        tsd_reentrancy_level_get(tsd) > 0))) {
-		arena_decay(tsdn, arena, false, false);
+		pa_shard_do_deferred_work(tsdn, &arena->pa_shard, false);
 	}
 }
 

include/jemalloc/internal/background_thread.h

@@ -13,17 +13,6 @@
 #define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT
 #define DEFAULT_NUM_BACKGROUND_THREAD 4
 
-/*
- * These exist only as a transitional state.  Eventually, deferral should be
- * part of the PAI, and each implementation can indicate wait times with more
- * specificity.
- */
-#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_UNINITIALIZED (-2)
-#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000
-
-#define BACKGROUND_THREAD_DEFERRED_MIN UINT64_C(0)
-#define BACKGROUND_THREAD_DEFERRED_MAX UINT64_MAX
-
 typedef enum {
 	background_thread_stopped,
 	background_thread_started,
@@ -87,6 +76,25 @@ bool background_thread_stats_read(
     tsdn_t *tsdn, background_thread_stats_t *stats);
 void background_thread_ctl_init(tsdn_t *tsdn);
 
+/*
+ * Arena-reset lifecycle bracket (owned by the bg-thread module so the
+ * started<->paused state machine is mutated only inside the module).  begin()
+ * acquires the global background_thread_lock and RETURNS WITH IT HELD;
+ * finish() releases it.  The lock intentionally spans the whole arena-reset
+ * body across the two calls.  Pass arena_ind (not an arena_t *): the destroy
+ * path frees the arena before calling finish().
+ */
+void background_thread_arena_reset_begin(tsd_t *tsd, unsigned arena_ind);
+void background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind);
+
+/*
+ * Serialize a rare admin operation against the background thread by holding the
+ * per-arena info mutex (have_background_thread-gated).  Used by
+ * arena_set_extent_hooks to fence pa_shard_disable_hpa.
+ */
+void background_thread_serialize_lock(tsd_t *tsd, unsigned arena_ind);
+void background_thread_serialize_unlock(tsd_t *tsd, unsigned arena_ind);
+
 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
 extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
     void *(*)(void *), void *__restrict);

include/jemalloc/internal/background_thread_inlines.h

@@ -22,12 +22,6 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) {
 	background_thread_enabled_set_impl(state);
 }
 
-JEMALLOC_ALWAYS_INLINE background_thread_info_t *
-arena_background_thread_info_get(arena_t *arena) {
-	unsigned arena_ind = arena_ind_get(arena);
-	return &background_thread_info[arena_ind % max_background_threads];
-}
-
 JEMALLOC_ALWAYS_INLINE background_thread_info_t *
 background_thread_info_get(size_t ind) {
 	return &background_thread_info[ind % max_background_threads];

include/jemalloc/internal/base.h

@@ -106,7 +106,6 @@ base_t *base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
     bool metadata_use_hooks);
 void    base_delete(tsdn_t *tsdn, base_t *base);
 ehooks_t       *base_ehooks_get(base_t *base);
-ehooks_t       *base_ehooks_get_for_metadata(base_t *base);
 extent_hooks_t *base_extent_hooks_set(
     base_t *base, extent_hooks_t *extent_hooks);
 void    *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);

include/jemalloc/internal/bin.h

@@ -69,23 +69,13 @@ void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin);
 void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
 
 /* Slab region allocation. */
-void *bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info);
 void  bin_slab_reg_alloc_batch(
      edata_t *slab, const bin_info_t *bin_info, unsigned cnt, void **ptrs);
 
 /* Slab list management. */
-void     bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab);
-void     bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab);
-edata_t *bin_slabs_nonfull_tryget(bin_t *bin);
-void     bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab);
 void     bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab);
 
-/* Slab association / demotion. */
-void bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin);
-void bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
-
 /* Deallocation helpers (called under bin lock). */
-void bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_empty(
     tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_nonempty(

include/jemalloc/internal/bitmap.h

@@ -176,9 +176,7 @@ typedef struct bitmap_info_s {
 #endif /* BITMAP_USE_TREE */
 } bitmap_info_t;
 
-void   bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
 void   bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
-size_t bitmap_size(const bitmap_info_t *binfo);
 
 static inline bool
 bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {

include/jemalloc/internal/buf_writer.h

@@ -25,7 +25,6 @@ typedef struct {
 
 bool       buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer,
           write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len);
-void       buf_writer_flush(buf_writer_t *buf_writer);
 write_cb_t buf_writer_cb;
 void       buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer);
 

include/jemalloc/internal/ckh.h

@@ -93,10 +93,4 @@ bool ckh_remove(
     tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data);
 bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
 
-/* Some useful hash and comparison functions for strings and pointers. */
-void ckh_string_hash(const void *key, size_t r_hash[2]);
-bool ckh_string_keycomp(const void *k1, const void *k2);
-void ckh_pointer_hash(const void *key, size_t r_hash[2]);
-bool ckh_pointer_keycomp(const void *k1, const void *k2);
-
 #endif /* JEMALLOC_INTERNAL_CKH_H */

include/jemalloc/internal/conf.h

@@ -7,9 +7,9 @@ void malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
     char readlink_buf[PATH_MAX + 1]);
 void malloc_abort_invalid_conf(void);
 
-#ifdef JEMALLOC_JET
 extern bool had_conf_error;
 
+#ifdef JEMALLOC_JET
 bool conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
     char const **v_p, size_t *vlen_p);
 void conf_error(

include/jemalloc/internal/deferral.h

@@ -0,0 +1,30 @@
+#ifndef JEMALLOC_INTERNAL_DEFERRAL_H
+#define JEMALLOC_INTERNAL_DEFERRAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+/*
+ * The deferred-work contract shared by the page allocators that PRODUCE
+ * deferred work (PAC decay-purge, HPA purge/hugify) and the drivers that
+ * CONSUME it (the background thread, or the application-inline fallback).
+ *
+ * There is deliberately no polymorphic interface here: with only two
+ * page allocators, a vtable's indirect calls are needless and inefficient.
+ * Dispatch is direct pac_*()/hpa_*(), so the contract is a convention
+ * documented here, not an enforced type.  The only shared code it needs is the
+ * constants below; each allocator keeps its own policy definition (functions
+ * and constants) in its own header.
+ *
+ * The convention every page allocator follows:
+ *   - <alloc>_time_until_deferred_work(): nanoseconds until its next
+ *     deferred work is due.  DEFERRED_WORK_MIN = "due now",
+ *     DEFERRED_WORK_MAX = "nothing pending"; anything between is a real
+ *     ns deadline.
+ *   - <alloc>_do_deferred_work(): perform whatever work is due now.
+ * The pa layer reports the soonest deadline across its allocators; the
+ * background thread sleeps until then, or indefinitely on DEFERRED_WORK_MAX.
+ */
+#define DEFERRED_WORK_MIN UINT64_C(0)
+#define DEFERRED_WORK_MAX UINT64_MAX
+
+#endif /* JEMALLOC_INTERNAL_DEFERRAL_H */

include/jemalloc/internal/ehooks.h

@@ -57,13 +57,6 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
 bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
 #endif
 bool ehooks_default_split_impl(void);
-/*
- * Merge is the only default extent hook we declare -- see the comment in
- * ehooks_merge.
- */
-bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a,
-    size_t size_a, void *addr_b, size_t size_b, bool committed,
-    unsigned arena_ind);
 bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b);
 void ehooks_default_zero_impl(void *addr, size_t size);
 void ehooks_default_guard_impl(void *guard1, void *guard2);

include/jemalloc/internal/emitter.h

@@ -58,6 +58,11 @@ struct emitter_col_s {
 		ssize_t     ssize_val;
 		const char *str_val;
 	};
+	/*
+	 * Optional JSON key.  When set, emitter_row() emits this column as a
+	 * "json_key": value pair in JSON mode; columns left NULL are table-only.
+	 */
+	const char *json_key;
 
 	/* Filled in by initialization. */
 	ql_elm(emitter_col_t) link;
@@ -79,6 +84,8 @@ struct emitter_s {
 	bool item_at_depth;
 	/* True if we emitted a key and will emit corresponding value next. */
 	bool emitted_key;
+	/* Gap-collapsing state for the emitter_table_sparse_row() bracket. */
+	bool table_sparse_in_gap;
 };
 
 static inline bool
@@ -270,6 +277,7 @@ emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
 	emitter->item_at_depth = false;
 	emitter->emitted_key = false;
 	emitter->nesting_depth = 0;
+	emitter->table_sparse_in_gap = false;
 }
 
 /******************************************************************************/
@@ -447,6 +455,46 @@ static inline void
 emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
 	ql_elm_new(col, link);
 	ql_tail_insert(&row->cols, col, link);
+	col->json_key = NULL;
+}
+
+/*
+ * Sparse table rows.  Some tables have one row per size class but most rows
+ * are all-zero; in table output those runs are collapsed to a single "---"
+ * separator.  Bracket such a table with emitter_table_sparse_begin() /
+ * _end() and emit each row with emitter_table_sparse_row(), passing whether
+ * that row is a gap (all-zero).  The emitter prints the separator on a
+ * gap->non-gap transition and at a trailing gap, and suppresses the gap rows
+ * themselves; callers no longer track gap state.  Table output only -- a
+ * no-op in JSON, where callers emit every row's object separately.
+ */
+static inline void
+emitter_table_sparse_begin(emitter_t *emitter) {
+	emitter->table_sparse_in_gap = false;
+}
+
+static inline void
+emitter_table_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) {
+	if (emitter->output != emitter_output_table) {
+		return;
+	}
+	bool prev_in_gap = emitter->table_sparse_in_gap;
+	emitter->table_sparse_in_gap = is_gap;
+	if (prev_in_gap && !is_gap) {
+		emitter_printf(emitter, "                     ---\n");
+	}
+	if (!is_gap) {
+		emitter_table_row(emitter, row);
+	}
+}
+
+static inline void
+emitter_table_sparse_end(emitter_t *emitter) {
+	if (emitter->output == emitter_output_table
+	    && emitter->table_sparse_in_gap) {
+		emitter_printf(emitter, "                     ---\n");
+	}
+	emitter->table_sparse_in_gap = false;
 }
 
 /******************************************************************************/
@@ -499,6 +547,142 @@ emitter_dict_end(emitter_t *emitter) {
 	}
 }
 
+/* Internal: emit a row's json-keyed columns as JSON kvs, in column order. */
+static inline void
+emitter_row_json(emitter_t *emitter, emitter_row_t *row) {
+	emitter_col_t *col;
+	ql_foreach (col, &row->cols, link) {
+		if (col->json_key != NULL) {
+			emitter_json_kv(emitter, col->json_key, col->type,
+			    (const void *)&col->bool_val);
+		}
+	}
+}
+
+/*
+ * Emit a row of columns to whichever format is active.  In table mode, prints
+ * the aligned table row (see emitter_table_row).  In JSON mode, emits each
+ * column that has a non-NULL json_key as a "json_key": value pair, in column
+ * order (columns without a json_key are table-only).  This lets a caller
+ * describe a row once and render it to either format.  For JSON objects (one
+ * per row), wrap the call in emitter_json_object_begin()/_end().
+ */
+static inline void
+emitter_row(emitter_t *emitter, emitter_row_t *row) {
+	emitter_table_row(emitter, row);
+	if (emitter_outputs_json(emitter)) {
+		emitter_row_json(emitter, row);
+	}
+}
+
+/*
+ * Sparse counterpart of emitter_row() for per-size-class tables: the table
+ * side collapses all-zero ("gap") rows (see emitter_table_sparse_row), while
+ * the JSON side still emits every row's json-keyed columns.  Bracket the loop
+ * with emitter_table_sparse_begin()/_end(), and wrap each row's JSON in
+ * emitter_json_object_begin()/_end() when the rows form an array of objects.
+ */
+static inline void
+emitter_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) {
+	emitter_table_sparse_row(emitter, row, is_gap);
+	if (emitter_outputs_json(emitter)) {
+		emitter_row_json(emitter, row);
+	}
+}
+
+/*
+ * Descriptor-driven column tables.  A descriptor array defines table and JSON
+ * order; columns with a NULL json_key are table-only.  The getter lets one
+ * engine serve different row types while keeping derived values (rates,
+ * utilization, and so on) with the table-specific code.  A column is active
+ * when all its required_flags are present in the active_flags supplied by the
+ * caller; the emitter assigns no domain-specific meaning to those bits.
+ */
+typedef struct emitter_col_desc_s emitter_col_desc_t;
+struct emitter_col_desc_s {
+	const char       *json_key;
+	const char       *table_label;
+	emitter_justify_t justify;
+	int               width;
+	emitter_type_t    type;
+	unsigned          required_flags;
+	void            (*get)(const void *row, emitter_col_t *col);
+};
+
+static inline bool
+emitter_col_desc_active(const emitter_col_desc_t *desc,
+    unsigned active_flags) {
+	return (desc->required_flags & ~active_flags) == 0;
+}
+
+static inline void
+emitter_col_table_build(const emitter_col_desc_t *descs, size_t ndescs,
+    unsigned active_flags, emitter_row_t *row, emitter_col_t *cols,
+    emitter_row_t *header_row, emitter_col_t *header_cols) {
+	emitter_row_init(row);
+	emitter_row_init(header_row);
+	for (size_t i = 0; i < ndescs; i++) {
+		if (!emitter_col_desc_active(&descs[i], active_flags)) {
+			continue;
+		}
+
+		emitter_col_init(&cols[i], row);
+		cols[i].justify = descs[i].justify;
+		cols[i].width = descs[i].width;
+		cols[i].type = descs[i].type;
+
+		emitter_col_init(&header_cols[i], header_row);
+		header_cols[i].justify = descs[i].justify;
+		header_cols[i].width = descs[i].width;
+		header_cols[i].type = emitter_type_title;
+		header_cols[i].str_val = descs[i].table_label != NULL
+		    ? descs[i].table_label : descs[i].json_key;
+	}
+}
+
+static inline void
+emitter_col_table_header(emitter_t *emitter, emitter_row_t *header_row,
+    emitter_col_t *first_header_col, const char *table_prefix,
+    const char *json_key) {
+	first_header_col->width -= (int)strlen(table_prefix);
+	emitter_table_printf(emitter, "%s", table_prefix);
+	emitter_table_row(emitter, header_row);
+	emitter_json_array_kv_begin(emitter, json_key);
+}
+
+static inline void
+emitter_col_table_fill(const emitter_col_desc_t *descs, size_t ndescs,
+    unsigned active_flags, emitter_col_t *cols, const void *row) {
+	for (size_t i = 0; i < ndescs; i++) {
+		if (!emitter_col_desc_active(&descs[i], active_flags)) {
+			continue;
+		}
+		cols[i].type = descs[i].type;
+		descs[i].get(row, &cols[i]);
+	}
+}
+
+static inline void
+emitter_col_table_emit_json(emitter_t *emitter,
+    const emitter_col_desc_t *descs, size_t ndescs, unsigned active_flags,
+    emitter_col_t *cols) {
+	if (!emitter_outputs_json(emitter)) {
+		return;
+	}
+	for (size_t i = 0; i < ndescs; i++) {
+		const emitter_col_desc_t *desc = &descs[i];
+		if (!emitter_col_desc_active(desc, active_flags)) {
+			continue;
+		}
+		if (desc->json_key != NULL) {
+			emitter_type_t json_type = cols[i].type == emitter_type_title
+			    ? emitter_type_string : cols[i].type;
+			emitter_json_kv(emitter, desc->json_key, json_type,
+			    (const void *)&cols[i].bool_val);
+		}
+	}
+}
+
 static inline void
 emitter_begin(emitter_t *emitter) {
 	if (emitter_outputs_json(emitter)) {

include/jemalloc/internal/hpa.h

@@ -13,6 +13,14 @@
 #include "jemalloc/internal/psset.h"
 #include "jemalloc/internal/sec.h"
 
+/*
+ * HPA-specific deferral interval bounds (currently unused).  Deferral tuning
+ * is per-allocator policy, so it lives in the HPA header; see deferral.h for
+ * the shared deferred-work contract.
+ */
+#define HPA_INTERVAL_MAX_UNINITIALIZED (-2)
+#define HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000
+
 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
 struct hpa_shard_nonderived_stats_s {
 	/*
@@ -190,7 +198,6 @@ struct hpa_shard_s {
 	nstime_t last_time_work_attempted;
 };
 
-bool hpa_hugepage_size_exceeds_limit(void);
 /*
  * Whether or not the HPA can be used given the current configuration.  This
  * is not necessarily a guarantee that it backs its allocations by hugepages,
@@ -204,11 +211,6 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
 edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
     size_t alignment, bool zero, bool guarded, bool frequent_reuse,
     bool *deferred_work_generated);
-bool hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool zero,
-    bool *deferred_work_generated);
-bool hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool *deferred_work_generated);
 void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
     bool *deferred_work_generated);
 uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);

include/jemalloc/internal/hpa_opts.h

@@ -108,11 +108,6 @@ struct hpa_shard_opts_s {
 	 */
 	uint64_t min_purge_interval_ms;
 
-	/*
-	 * Maximum number of hugepages to purge on each purging attempt.
-	 */
-	ssize_t experimental_max_purge_nhp;
-
 	/*
 	 * Minimum number of inactive bytes needed for a non-empty page to be
 	 * considered purgable.
@@ -176,8 +171,6 @@ struct hpa_shard_opts_s {
 	false,								\
 	/* min_purge_interval_ms */					\
 	5 * 1000,							\
-	/* experimental_max_purge_nhp */				\
-	-1,      							\
 	/* size_t purge_threshold */					\
 	PAGE,								\
 	/* min_purge_delay_ms */             				\

include/jemalloc/internal/hpdata.h

@@ -425,11 +425,6 @@ hpdata_full(const hpdata_t *hpdata) {
 
 void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 
-/*
- * Given an hpdata which can serve an allocation request, pick and reserve an
- * offset within that allocation.
- */
-void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
 /*

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -97,6 +97,9 @@
 /* Defined if pthread_get_name_np(3) is available. */
 #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP
 
+/* Defined if pthread_condattr_setclock(..., CLOCK_MONOTONIC) is available. */
+#undef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
@@ -172,9 +175,6 @@
 /* Use gcc intrinsics for profile backtracing if defined. */
 #undef JEMALLOC_PROF_GCC
 
-/* Use frame pointer for profile backtracing if defined. Linux only. */
-#undef JEMALLOC_PROF_FRAME_POINTER
-
 /* JEMALLOC_PAGEID enabled page id */
 #undef JEMALLOC_PAGEID
 
@@ -468,6 +468,13 @@
 /* Are C++ exceptions enabled? */
 #undef JEMALLOC_HAVE_CXX_EXCEPTIONS
 
+/*
+ * If defined, throwing operator new aborts on OOM (with size logged) instead
+ * of throwing std::bad_alloc. Nothrow new still returns null. With LTO this
+ * lets the compiler prove operator new is no-throw and elide exception cleanup.
+ */
+#undef JEMALLOC_INFALLIBLE_NEW
+
 /* Performs additional size checks when defined. */
 #undef JEMALLOC_OPT_SIZE_CHECKS
 

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -21,6 +21,7 @@ extern bool             opt_confirm_conf;
 extern bool             opt_hpa;
 extern hpa_shard_opts_t opt_hpa_opts;
 extern sec_opts_t       opt_hpa_sec_opts;
+extern sec_opts_t       opt_pac_sec_opts;
 
 extern const char *opt_junk;
 extern bool        opt_junk_alloc;
@@ -30,7 +31,6 @@ extern void (*JET_MUTABLE junk_alloc_callback)(void *ptr, size_t size);
 extern void (*JET_MUTABLE invalid_conf_abort)(void);
 extern bool                  opt_utrace;
 extern bool                  opt_xmalloc;
-extern bool                  opt_experimental_infallible_new;
 extern bool                  opt_experimental_tcache_gc;
 extern bool                  opt_zero;
 extern unsigned              opt_narenas;

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -18,15 +18,34 @@ malloc_getcpu(void) {
 	return GetCurrentProcessorNumber();
 #elif defined(JEMALLOC_HAVE_SCHED_GETCPU)
 	return (malloc_cpuid_t)sched_getcpu();
+#elif defined(__APPLE__)
+	/*
+	 * No sched_getcpu() on macOS; read the CPU number like _os_cpu_number()
+	 * does, from the low 12 bits of tpidr_el0 (arm64) or the IDT base (x86).
+	 * The 0xfff mask is xnu's __TPIDR_CPU_NUM_MASK, kept in sync with
+	 * _os_cpu_number in libsyscall/os/tsd.h (the kernel counterpart is
+	 * MACHDEP_TPIDR_CPUNUM_MASK in osfmk/arm64/machine_machdep.h):
+	 * https://github.com/apple-oss-distributions/xnu/blob/main/libsyscall/os/tsd.h
+	 * This requires macOS 12+: on macOS 11 and earlier arm64 kept the CPU
+	 * number in tpidrro_el0's low 3 bits instead, so it would be misread here.
+	 * Those releases are retired by Apple, so only the current layout is handled.
+	 */
+#  if defined(__aarch64__)
+	uint64_t cpu;
+	__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(cpu));
+	return (malloc_cpuid_t)(cpu & 0xfff);
+#  elif defined(__x86_64__) || defined(__i386__)
+	struct { uintptr_t p1, p2; } idtr;
+	__asm__ __volatile__("sidt %0" : "=m"(idtr));
+	return (malloc_cpuid_t)(idtr.p1 & 0xfff);
+#  else
+	not_reached();
+	return -1;
+#  endif
 #elif defined(JEMALLOC_HAVE_RDTSCP)
 	unsigned int ecx;
 	asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx");
 	return (malloc_cpuid_t)(ecx & 0xfff);
-#elif defined(__aarch64__) && defined(__APPLE__)
-	/* Other oses most likely use tpidr_el0 instead */
-	uintptr_t c;
-	asm volatile("mrs %x0, tpidrro_el0" : "=r"(c)::"memory");
-	return (malloc_cpuid_t)(c & (1 << 3) - 1);
 #else
 	not_reached();
 	return -1;

include/jemalloc/internal/jemalloc_internal_macros.h

@@ -39,9 +39,11 @@
 #ifdef JEMALLOC_JET
 #	define JET_MUTABLE
 #	define JET_EXTERN extern
+#	define JET_EXTERN_INLINE extern
 #else
 #	define JET_MUTABLE const
 #	define JET_EXTERN static
+#	define JET_EXTERN_INLINE static inline
 #endif
 
 #define JEMALLOC_VA_ARGS_HEAD(head, ...) head

include/jemalloc/internal/jemalloc_preamble.h.in

@@ -130,13 +130,6 @@ static const bool config_prof_libunwind =
     false
 #endif
     ;
-static const bool config_prof_frameptr =
-#ifdef JEMALLOC_PROF_FRAME_POINTER
-    true
-#else
-    false
-#endif
-    ;
 static const bool maps_coalesce =
 #ifdef JEMALLOC_MAPS_COALESCE
     true
@@ -238,6 +231,18 @@ static const bool config_enable_cxx =
 #endif
 ;
 
+/*
+ * Whether the throwing operator new aborts on OOM instead of throwing
+ * std::bad_alloc (--enable-cxx-infallible-new).
+ */
+static const bool config_infallible_new =
+#ifdef JEMALLOC_INFALLIBLE_NEW
+    true
+#else
+    false
+#endif
+;
+
 #if defined(_WIN32) || defined(__APPLE__) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
 /* Currently percpu_arena depends on sched_getcpu. */
 #define JEMALLOC_PERCPU_ARENA

include/jemalloc/internal/malloc_io.h

@@ -89,13 +89,4 @@ malloc_close(int fd) {
 #endif
 }
 
-static inline off_t
-malloc_lseek(int fd, off_t offset, int whence) {
-#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_lseek)
-	return (off_t)syscall(SYS_lseek, fd, offset, whence);
-#else
-	return lseek(fd, offset, whence);
-#endif
-}
-
 #endif /* JEMALLOC_INTERNAL_MALLOC_IO_H */

include/jemalloc/internal/mutex_prof.h

@@ -37,7 +37,8 @@ typedef enum {
 	OP(tcache_list)                                                        \
 	OP(hpa_shard)                                                          \
 	OP(hpa_shard_grow)                                                     \
-	OP(hpa_sec)
+	OP(hpa_sec)                                                            \
+	OP(pac_sec)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,

include/jemalloc/internal/nstime.h

@@ -28,7 +28,6 @@ static const nstime_t nstime_zero = NSTIME_ZERO_INITIALIZER;
 void     nstime_init(nstime_t *time, uint64_t ns);
 void     nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
 uint64_t nstime_ns(const nstime_t *time);
-uint64_t nstime_ms(const nstime_t *time);
 uint64_t nstime_sec(const nstime_t *time);
 uint64_t nstime_nsec(const nstime_t *time);
 void     nstime_copy(nstime_t *time, const nstime_t *source);
@@ -36,11 +35,9 @@ int      nstime_compare(const nstime_t *a, const nstime_t *b);
 void     nstime_add(nstime_t *time, const nstime_t *addend);
 void     nstime_iadd(nstime_t *time, uint64_t addend);
 void     nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
-void     nstime_isubtract(nstime_t *time, uint64_t subtrahend);
 void     nstime_imultiply(nstime_t *time, uint64_t multiplier);
 void     nstime_idivide(nstime_t *time, uint64_t divisor);
 uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
-uint64_t nstime_ns_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ms_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ns_since(const nstime_t *past);
 uint64_t nstime_ms_since(const nstime_t *past);

include/jemalloc/internal/pa.h

@@ -91,9 +91,6 @@ struct pa_shard_s {
 	 */
 	bool ever_used_hpa;
 
-	/* Allocates from a PAC. */
-	pac_t pac;
-
 	hpa_shard_t hpa;
 
 	/* The source of edata_t objects. */
@@ -109,13 +106,10 @@ struct pa_shard_s {
 
 	/* The base from which we get the ehooks and allocate metadat. */
 	base_t *base;
-};
 
-static inline bool
-pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
-	return ecache_npages_get(&shard->pac.ecache_muzzy) == 0
-	    && pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0;
-}
+	/* Allocates from a PAC. */
+	pac_t pac;
+};
 
 static inline ehooks_t *
 pa_shard_ehooks_get(const pa_shard_t *shard) {
@@ -158,8 +152,12 @@ void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard);
  */
 void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
 
-/* Flush any caches used by shard */
-void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard);
+/*
+ * Flush the shard's front caches (SEC + HPA) back to the ecaches.  If all is
+ * true, also fully decay-purge the PAC dirty (and muzzy, unless skipped) extents
+ * to the OS -- the "save as much memory as possible" path.
+ */
+void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard, bool all);
 
 /* Gets an edata for the given allocation. */
 edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
@@ -184,22 +182,35 @@ bool pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 void    pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
        bool *deferred_work_generated);
 bool    pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
-       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms);
 ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state);
 
-/*
- * Do deferred work on this PA shard.
- *
- * Morally, this should do both PAC decay and the HPA deferred work.  For now,
- * though, the arena, background thread, and PAC modules are tightly interwoven
- * in a way that's tricky to extricate, so we only do the HPA-specific parts.
- */
 void pa_shard_set_deferral_allowed(
     tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed);
-void     pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+/*
+ * Do deferred work on this PA shard: dispatch to PAC (decay-purge) then HPA.
+ * Each allocator owns its own policy -- PAC decides eagerness from
+ * is_background_thread and, on the application path, notifies the background
+ * thread; the bg-thread driver passes is_background_thread=true and is never
+ * self-notified.
+ */
+void     pa_shard_do_deferred_work(
+    tsdn_t *tsdn, pa_shard_t *shard, bool is_background_thread);
 void     pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 
+/*
+ * Called on the application path after a pa operation (alloc/expand/shrink/
+ * dalloc) reports deferred_work_generated -- i.e. it left PAC decay due or
+ * HPA work (hugify/dehugify/purge) pending.  If PAC dirty decay is immediate
+ * (dirty_decay_ms == 0), purge dirty synchronously; then, if a background
+ * thread is enabled but idle (sleeping indefinitely), wake it early so the
+ * pending work runs promptly rather than at the next scheduled wakeup.
+ *
+ * Application thread only; must never be called on the background thread.
+ */
+void pa_shard_handle_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+
 /******************************************************************************/
 /*
  * Various bits of "boring" functionality that are still part of this module,
@@ -220,8 +231,6 @@ void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
 
 size_t pa_shard_nactive(const pa_shard_t *shard);
-size_t pa_shard_ndirty(const pa_shard_t *shard);
-size_t pa_shard_nmuzzy(const pa_shard_t *shard);
 
 void pa_shard_basic_stats_merge(
     const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);

include/jemalloc/internal/pac.h

@@ -8,11 +8,12 @@
 #include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
 #include "jemalloc/internal/lockedint.h"
+#include "jemalloc/internal/sec.h"
 #include "jemalloc/internal/tsd_types.h"
 #include "san_bump.h"
 
 /*
- * Page allocator classic; an implementation of the PAI interface that:
+ * Page allocator classic (PAC), a page-level allocator that:
  * - Can be used for arenas with custom extent hooks.
  * - Can always satisfy any allocation request (including highly-fragmentary
  *   ones).
@@ -27,6 +28,14 @@ enum pac_purge_eagerness_e {
 };
 typedef enum pac_purge_eagerness_e pac_purge_eagerness_t;
 
+/*
+ * When a decay sweep would purge more than this many pages, its purge is
+ * treated as due (worth waking the background thread for) instead of left
+ * to accumulate.  PAC decay policy; the HPA defers on time intervals, not
+ * on a page count, so this constant is PAC-only.
+ */
+#define PAC_DECAY_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
+
 typedef struct pac_decay_stats_s pac_decay_stats_t;
 struct pac_decay_stats_s {
 	/* Total number of purge sweeps. */
@@ -84,12 +93,24 @@ struct pac_stats_s {
 
 	/* VM space had to be leaked (undocumented).  Normally 0. */
 	atomic_zu_t abandoned_vm;
+
+	/* PAC SEC stats.  Derived. */
+	sec_stats_t pac_sec_stats;
 };
 
 typedef struct pac_s pac_t;
 struct pac_s {
+	/* Small extent cache in front of PAC ecaches to reduce contention. */
+	sec_t sec;
+	/*
+	 * Runtime gate for PAC SEC.  0 disables (when SEC is not configured or
+	 * dirty_decay_ms == 0); otherwise mirrors sec.opts.max_alloc.
+	 */
+	atomic_zu_t sec_max_alloc;
+
 	/* True once pinned memory has been seen. */
 	atomic_b_t has_pinned;
+
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.
@@ -218,6 +239,41 @@ bool pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
     pac_decay_stats_t *decay_stats, ecache_t *ecache,
     pac_purge_eagerness_t eagerness);
 
+/*
+ * Result of a pac_decay_deferred() pass.  Reported per decay state so
+ * pac_do_deferred_work can decide, per state, whether to notify the background
+ * thread.  npages_new is the per-state epoch backlog delta, only meaningful when
+ * the corresponding *_epoch_advanced is true.
+ */
+typedef struct pac_deferred_work_result_s pac_deferred_work_result_t;
+struct pac_deferred_work_result_s {
+	size_t dirty_npages_new;
+	size_t muzzy_npages_new;
+	bool   dirty_epoch_advanced;
+	bool   muzzy_epoch_advanced;
+};
+
+/*
+ * All deferred decay-purge work for a PAC shard: decide the eagerness from the
+ * caller context, run pac_decay_deferred, and (application path only) notify the
+ * background thread for any decay epoch that advanced.  The bg-thread driver
+ * passes is_background_thread=true and is never self-notified.
+ */
+void pac_do_deferred_work(
+    tsdn_t *tsdn, pac_t *pac, bool is_background_thread);
+
+/*
+ * Application-path hook (after deferred_work_generated): wake the background
+ * thread if it is sleeping idle.  Runs the same early-wake logic as the notify
+ * path (pac_maybe_wake_bg), gated on the thread being idle.
+ */
+void pac_wake_bg_on_deferred(tsdn_t *tsdn, pac_t *pac);
+
+/*
+ * Fully decay the extents of the given state, acquiring decay->mtx internally.
+ */
+void pac_decay_all_now(tsdn_t *tsdn, pac_t *pac, extent_state_t state);
+
 /*
  * Gets / sets the maximum amount that we'll grow an arena down the
  * grow-retained pathways (unless forced to by an allocaction request).
@@ -231,10 +287,24 @@ bool pac_retain_grow_limit_get_set(
     tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit);
 
 bool    pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
-       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms);
 ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
-void pac_reset(tsdn_t *tsdn, pac_t *pac);
+/* Whether the muzzy decay path is relevant (muzzy pages exist, or decay is on). */
+static inline bool
+pac_should_decay_muzzy(pac_t *pac) {
+	return ecache_npages_get(&pac->ecache_muzzy) != 0
+	    || pac_decay_ms_get(pac, extent_state_muzzy) > 0;
+}
+
+/* Whether dirty decay is immediate (dirty_decay_ms == 0). */
+static inline bool
+pac_decay_immediately(pac_t *pac) {
+	return decay_immediately(&pac->decay_dirty);
+}
+
 void pac_destroy(tsdn_t *tsdn, pac_t *pac);
 
+void pac_sec_flush(tsdn_t *tsdn, pac_t *pac);
+
 #endif /* JEMALLOC_INTERNAL_PAC_H */

include/jemalloc/internal/prof.h

@@ -419,7 +419,6 @@ void        prof_prefork1(tsdn_t *tsdn);
 void        prof_postfork_parent(tsdn_t *tsdn);
 void        prof_postfork_child(tsdn_t *tsdn);
 
-uint64_t prof_sample_new_event_wait(tsd_t *tsd);
 uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
 
 extern te_base_cb_t prof_sample_te_handler;

include/jemalloc/internal/prof_data.h

@@ -29,9 +29,4 @@ void          prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
 void          prof_reset(tsd_t *tsd, size_t lg_sample);
 void          prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx);
 
-/* Used in unit tests. */
-size_t prof_tdata_count(void);
-size_t prof_bt_count(void);
-void   prof_cnt_all(prof_cnt_t *cnt_all);
-
 #endif /* JEMALLOC_INTERNAL_PROF_DATA_H */

include/jemalloc/internal/prof_log.h

@@ -9,14 +9,6 @@ extern malloc_mutex_t log_mtx;
 void prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
 bool prof_log_init(tsd_t *tsdn);
 
-/* Used in unit tests. */
-size_t prof_log_bt_count(void);
-size_t prof_log_alloc_count(void);
-size_t prof_log_thr_count(void);
-bool   prof_log_is_logging(void);
-bool   prof_log_rep_check(void);
-void   prof_log_dummy_set(bool new_value);
-
 bool prof_log_start(tsdn_t *tsdn, const char *filename);
 bool prof_log_stop(tsdn_t *tsdn);
 

include/jemalloc/internal/prof_recent.h

@@ -17,8 +17,6 @@ void edata_prof_recent_alloc_init(edata_t *edata);
 /* Used in unit tests. */
 typedef ql_head(prof_recent_t) prof_recent_list_t;
 extern prof_recent_list_t prof_recent_alloc_list;
-edata_t *prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *node);
-prof_recent_t *edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata);
 
 ssize_t prof_recent_alloc_max_ctl_read(void);
 ssize_t prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max);

include/jemalloc/internal/prof_sys.h

@@ -21,7 +21,6 @@ void prof_fdump_impl(tsd_t *tsd);
 void prof_idump_impl(tsd_t *tsd);
 bool prof_mdump_impl(tsd_t *tsd, const char *filename);
 void prof_gdump_impl(tsd_t *tsd);
-int  prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high);
 
 /* Used in unit tests. */
 typedef int(prof_sys_thread_name_read_t)(char *buf, size_t limit);

include/jemalloc/internal/sc.h

@@ -368,7 +368,6 @@ struct sc_data_s {
 	sc_t sc[SC_NSIZES];
 };
 
-size_t reg_size_compute(int lg_base, int lg_delta, int ndelta);
 void   sc_data_init(sc_data_t *data);
 /*
  * Updates slab sizes in [begin, end] to be pgs pages in length, if possible.

include/jemalloc/internal/sec.h

@@ -35,19 +35,20 @@ typedef struct sec_stats_s sec_stats_t;
 struct sec_stats_s {
 	/* Sum of bytes_cur across all shards. */
 	size_t bytes;
+	/* Subset of bytes that are pinned. */
+	size_t bytes_pinned;
 
 	/* Totals of bin_stats. */
 	sec_bin_stats_t total;
 };
 
-static inline void
-sec_bin_stats_init(sec_bin_stats_t *stats) {
-	stats->ndalloc_flush = 0;
-	stats->nmisses = 0;
-	stats->nhits = 0;
-	stats->ndalloc_noflush = 0;
-	stats->noverfills = 0;
-}
+typedef struct sec_pszind_stats_s sec_pszind_stats_t;
+struct sec_pszind_stats_s {
+	size_t nextents;
+	size_t bytes;
+	size_t nextents_pinned;
+	size_t bytes_pinned;
+};
 
 static inline void
 sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
@@ -61,6 +62,7 @@ sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
 static inline void
 sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 	dst->bytes += src->bytes;
+	dst->bytes_pinned += src->bytes_pinned;
 	sec_bin_stats_accum(&dst->total, &src->total);
 }
 
@@ -68,16 +70,21 @@ sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 typedef struct sec_bin_s sec_bin_t;
 struct sec_bin_s {
 	/*
-	 * Protects the data members of the bin.
+	 * Protects the freelist and synchronizes counter updates.
 	 */
 	malloc_mutex_t mtx;
 
 	/*
 	 * Number of bytes in this particular bin.
 	 */
-	size_t              bytes_cur;
+	atomic_zu_t         bytes_cur;
+	atomic_zu_t         bytes_pinned_cur;
 	edata_list_active_t freelist;
-	sec_bin_stats_t     stats;
+	atomic_zu_t         nmisses;
+	atomic_zu_t         nhits;
+	atomic_zu_t         ndalloc_flush;
+	atomic_zu_t         ndalloc_noflush;
+	atomic_zu_t         noverfills;
 };
 
 typedef struct sec_s sec_t;
@@ -88,13 +95,13 @@ struct sec_s {
 };
 
 static inline bool
-sec_is_used(sec_t *sec) {
+sec_is_used(const sec_t *sec) {
 	return sec->opts.nshards != 0;
 }
 
 static inline bool
-sec_size_supported(sec_t *sec, size_t size) {
-	return sec_is_used(sec) && size <= sec->opts.max_alloc;
+sec_size_supported(const sec_t *sec, size_t size) {
+	return size <= sec->opts.max_alloc;
 }
 
 /* Min number of extents we will allocate when expanding the SEC. */
@@ -116,12 +123,21 @@ sec_size_supported(sec_t *sec, size_t size) {
 void sec_calc_nallocs_for_size(
     sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
 
-/* If sec does not have extent available, it will return NULL. */
-edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
+/*
+ * Lazily picks (and caches in *idxp) a shard for the calling thread.  Different
+ * SEC instances pass independent per-thread uint8_t slots, initialized to
+ * (uint8_t)-1.
+ */
+uint8_t sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp);
+
+/* Callers must ensure sec_size_supported(sec, size). */
+edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard);
 void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
-        edata_list_active_t *result, size_t nallocs);
+        edata_list_active_t *result, size_t nallocs, uint8_t shard);
 
 /*
+ * Callers must ensure sec_size_supported(sec, edata_size).
+ *
  * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty
  * if there are extents that need to be flushed from cache.  Please note, that
  * if we need to flush, extent(s) returned in the list to be deallocated
@@ -129,7 +145,8 @@ void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
  * considered "hot" and preserved in the cache, while "colder" ones are
  * returned).
  */
-void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list);
+void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
+    uint8_t shard);
 
 bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts);
 
@@ -142,7 +159,10 @@ void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
  * lets them fit easily into the pa_shard stats framework (which also has this
  * split), which simplifies the stats management.
  */
-void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge_pszind(
+    tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
+    sec_pszind_stats_t *stats);
 void sec_mutex_stats_read(
     tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data);
 

include/jemalloc/internal/stats.h

@@ -43,6 +43,9 @@ extern char    opt_stats_interval_opts[stats_print_tot_num_options + 1];
 #define STATS_INTERVAL_ACCUM_LG_BATCH_SIZE 6
 #define STATS_INTERVAL_ACCUM_BATCH_MAX (4 << 20)
 
+/* Per thread batch accum size for stats_interval; read by thread event. */
+extern uint64_t stats_interval_accum_batch;
+
 /* Only accessed by thread event. */
 extern te_base_cb_t stats_interval_te_handler;
 

include/jemalloc/internal/stats_internal.h

@@ -0,0 +1,340 @@
+#ifndef JEMALLOC_INTERNAL_STATS_INTERNAL_H
+#define JEMALLOC_INTERNAL_STATS_INTERNAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena.h" /* ARENA_NAME_LEN */
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/emitter.h"
+#include "jemalloc/internal/prof_stats.h"
+
+/*
+ * Internal implementation support for src/stats.c only -- do NOT include from
+ * other translation units.
+ *
+ * This header holds noisy local scaffolding for malloc_stats_print(): ctl
+ * gather plumbing, gather structs, emitter table-column helpers, and option
+ * list helpers.  It is deliberately not a reusable stats-gather API.
+ */
+
+#define CTL_GET(n, v, t)                                                       \
+	do {                                                                   \
+		size_t sz = sizeof(t);                                         \
+		xmallctl(n, (void *)v, &sz, NULL, 0);                          \
+	} while (0)
+
+#define CTL_LEAF_PREPARE(mib, miblen, name)                                    \
+	do {                                                                   \
+		assert(miblen < CTL_MAX_DEPTH);                                \
+		size_t miblen_new = CTL_MAX_DEPTH;                             \
+		xmallctlmibnametomib(mib, miblen, name, &miblen_new);          \
+		assert(miblen_new > miblen);                                   \
+	} while (0)
+
+#define CTL_LEAF(mib, miblen, leaf, v, t)                                      \
+	do {                                                                   \
+		assert(miblen < CTL_MAX_DEPTH);                                \
+		size_t miblen_new = CTL_MAX_DEPTH;                             \
+		size_t sz = sizeof(t);                                         \
+		xmallctlbymibname(                                             \
+		    mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0);  \
+		assert(miblen_new == miblen + 1);                              \
+	} while (0)
+
+#define CTL_MIB_GET(n, i, v, t, ind)                                           \
+	do {                                                                   \
+		size_t mib[CTL_MAX_DEPTH];                                     \
+		size_t miblen = sizeof(mib) / sizeof(size_t);                  \
+		size_t sz = sizeof(t);                                         \
+		xmallctlnametomib(n, mib, &miblen);                            \
+		mib[(ind)] = (i);                                              \
+		xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0);           \
+	} while (0)
+
+#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1)
+#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2)
+
+/******************************************************************************/
+/*
+ * Per-section gather structs.  The stats_gather_*() functions that fill them
+ * are static in src/stats.c.
+ */
+
+/* One arena's memory-accounting stats (stats.arenas.<i>.*). */
+typedef struct stats_arena_mem_s {
+	size_t mapped;
+	size_t retained;
+	size_t pinned;
+	size_t base;
+	size_t internal;
+	size_t metadata_edata;
+	size_t metadata_rtree;
+	size_t metadata_thp;
+	size_t tcache_bytes;
+	size_t tcache_stashed_bytes;
+	size_t resident;
+	size_t abandoned_vm;
+	size_t extent_avail;
+	size_t page; /* arenas.page, for the active-bytes conversion */
+} stats_arena_mem_t;
+
+/* One arena extents row (stats.arenas.<i>.extents.<pszind>.*). */
+typedef struct stats_arena_extent_s {
+	size_t ndirty;
+	size_t nmuzzy;
+	size_t nretained;
+	size_t npinned;
+	size_t ntotal;
+	size_t dirty_bytes;
+	size_t muzzy_bytes;
+	size_t retained_bytes;
+	size_t pinned_bytes;
+	size_t total_bytes;
+} stats_arena_extent_t;
+
+/* One arena's "basics" (stats.arenas.<i>.{nthreads,uptime,dss}). */
+typedef struct stats_arena_basics_s {
+	unsigned    nthreads;
+	uint64_t    uptime;
+	const char *dss;
+	char        name[ARENA_NAME_LEN];
+	bool        has_name; /* false for the merged/destroyed pseudo-arenas */
+} stats_arena_basics_t;
+
+/* One arena's decay / page stats (stats.arenas.<i>.*). */
+typedef struct stats_arena_decay_s {
+	ssize_t  dirty_decay_ms;
+	ssize_t  muzzy_decay_ms;
+	size_t   pactive;
+	size_t   pdirty;
+	size_t   pmuzzy;
+	uint64_t dirty_npurge;
+	uint64_t dirty_nmadvise;
+	uint64_t dirty_purged;
+	uint64_t muzzy_npurge;
+	uint64_t muzzy_nmadvise;
+	uint64_t muzzy_purged;
+} stats_arena_decay_t;
+
+/* One size class of arena allocation counts (stats.arenas.<i>.{small,large}.*). */
+typedef struct stats_arena_alloc_s {
+	size_t   allocated;
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	uint64_t nfills;
+	uint64_t nflushes;
+} stats_arena_alloc_t;
+
+/*
+ * One arena bin row (stats.arenas.<i>.bins.<j>.* + arenas.bin.<j>.*,
+ * plus prof.stats.bins.<j>.{live,accum} when profiling stats are enabled).
+ */
+typedef struct stats_arena_bin_s {
+	uint64_t nslabs;
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	uint64_t nfills;
+	uint64_t nflushes;
+	uint64_t nreslabs;
+	size_t   reg_size;
+	size_t   slab_size;
+	size_t   curregs;
+	size_t   curslabs;
+	size_t   nonfull_slabs;
+	uint32_t nregs;
+	uint32_t nshards;
+	prof_stats_t prof_live;
+	prof_stats_t prof_accum;
+} stats_arena_bin_t;
+
+/*
+ * One arena large-extent row (stats.arenas.<i>.lextents.<j>.* +
+ * arenas.lextent.<j>.*, plus prof.stats.lextents.<j>.{live,accum} when
+ * profiling stats are enabled).
+ */
+typedef struct stats_arena_lextent_s {
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	size_t   lextent_size;
+	size_t   curlextents;
+	prof_stats_t prof_live;
+	prof_stats_t prof_accum;
+} stats_arena_lextent_t;
+
+/* HPA shard small-extent cache (stats.arenas.<i>.hpa_sec_*). */
+typedef struct stats_arena_hpa_sec_s {
+	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+	size_t sec_overfills;
+} stats_arena_hpa_sec_t;
+
+/* PAC small-extent cache (stats.arenas.<i>.pac_sec_*). */
+typedef struct stats_arena_pac_sec_s {
+	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+} stats_arena_pac_sec_t;
+
+/* HPA shard counters (stats.arenas.<i>.hpa_shard.*). */
+typedef struct stats_arena_hpa_counters_s {
+	size_t npageslabs;
+	size_t nactive;
+	size_t ndirty;
+	size_t npageslabs_nonhuge;
+	size_t nactive_nonhuge;
+	size_t ndirty_nonhuge;
+	size_t nretained_nonhuge;
+	size_t npageslabs_huge;
+	size_t nactive_huge;
+	size_t ndirty_huge;
+	uint64_t npurge_passes;
+	uint64_t npurges;
+	uint64_t nhugifies;
+	uint64_t nhugify_failures;
+	uint64_t ndehugifies;
+} stats_arena_hpa_counters_t;
+
+/* One HPA slab class (full/empty/nonfull) row. */
+typedef struct stats_arena_hpa_slab_s {
+	size_t npageslabs_huge;
+	size_t nactive_huge;
+	size_t ndirty_huge;
+	size_t npageslabs_nonhuge;
+	size_t nactive_nonhuge;
+	size_t ndirty_nonhuge;
+	size_t nretained_nonhuge;
+} stats_arena_hpa_slab_t;
+
+/* Arena configuration / size-class scalars (arenas.*). */
+typedef struct stats_arena_config_s {
+	unsigned narenas;
+	ssize_t  dirty_decay_ms;
+	ssize_t  muzzy_decay_ms;
+	size_t   quantum;
+	size_t   page;
+	size_t   hugepage;
+	bool     have_tcache_max;
+	size_t   tcache_max;
+	unsigned nbins;
+	unsigned nhbins;
+	unsigned nlextents;
+} stats_arena_config_t;
+
+/* Per-size-class geometry (emitted in JSON only). */
+typedef struct stats_arena_bin_meta_s {
+	size_t   size;
+	uint32_t nregs;
+	size_t   slab_size;
+	uint32_t nshards;
+} stats_arena_bin_meta_t;
+
+typedef struct stats_arena_lextent_meta_s {
+	size_t size;
+} stats_arena_lextent_meta_t;
+
+/* Process-wide global stats (stats.* + stats.background_thread.*). */
+typedef struct stats_global_s {
+	size_t   allocated;
+	size_t   active;
+	size_t   metadata;
+	size_t   metadata_edata;
+	size_t   metadata_rtree;
+	size_t   metadata_thp;
+	size_t   resident;
+	size_t   mapped;
+	size_t   retained;
+	size_t   pinned;
+	size_t   zero_reallocs;
+	size_t   num_background_threads;
+	uint64_t background_thread_num_runs;
+	uint64_t background_thread_run_interval;
+} stats_global_t;
+
+/******************************************************************************/
+/*
+ * Emitter table-column helper macros: declare an emitter_col_t (and, for
+ * COL_HDR, its header column) and wire it into a row.
+ */
+
+#define COL_DECLARE(column_name) emitter_col_t col_##column_name;
+
+#define COL_INIT(row_name, column_name, left_or_right, col_width, etype)       \
+	emitter_col_init(&col_##column_name, &row_name);                       \
+	col_##column_name.justify = emitter_justify_##left_or_right;           \
+	col_##column_name.width = col_width;                                   \
+	col_##column_name.type = emitter_type_##etype;
+
+#define COL(row_name, column_name, left_or_right, col_width, etype)            \
+	COL_DECLARE(column_name);                                              \
+	COL_INIT(row_name, column_name, left_or_right, col_width, etype)
+
+#define COL_HDR_DECLARE(column_name)                                           \
+	COL_DECLARE(column_name);                                              \
+	emitter_col_t header_##column_name;
+
+#define COL_HDR_INIT(                                                          \
+    row_name, column_name, human, left_or_right, col_width, etype)             \
+	COL_INIT(row_name, column_name, left_or_right, col_width, etype)       \
+	emitter_col_init(&header_##column_name, &header_##row_name);           \
+	header_##column_name.justify = emitter_justify_##left_or_right;        \
+	header_##column_name.width = col_width;                                \
+	header_##column_name.type = emitter_type_title;                        \
+	header_##column_name.str_val = human ? human : #column_name;
+
+#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \
+	COL_HDR_DECLARE(column_name)                                           \
+	COL_HDR_INIT(                                                          \
+	    row_name, column_name, human, left_or_right, col_width, etype)
+
+/*
+ * stats_general_print() config/option list helpers.  Unhygienic by design:
+ * they assume `emitter` and the standard scratch locals are in scope at the
+ * call site -- bv/bsz (bool), cpv/cpsz (const char *), uv/usz (unsigned),
+ * i64v/i64sz, u64v/u64sz, sv/ssz (size_t), ssv/sssz (ssize_t), plus bv2/ssv2
+ * for the *_MUTABLE variants.  (CONFIG_WRITE_BOOL uses CTL_GET above; the
+ * OPT_WRITE_* variants use je_mallctl directly.)
+ */
+#define CONFIG_WRITE_BOOL(name)                                                \
+	do {                                                                   \
+		CTL_GET("config." #name, &bv, bool);                           \
+		emitter_kv(                                                    \
+		    emitter, #name, "config." #name, emitter_type_bool, &bv);  \
+	} while (0)
+
+#define OPT_WRITE(name, var, size, emitter_type)                               \
+	if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) {      \
+		emitter_kv(emitter, name, "opt." name, emitter_type, &var);    \
+	}
+
+#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname)       \
+	if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0        \
+	    && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) {      \
+		emitter_kv_note(emitter, name, "opt." name, emitter_type,      \
+		    &var1, altname, emitter_type, &var2);                      \
+	}
+
+#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool)
+#define OPT_WRITE_BOOL_MUTABLE(name, altname)                                  \
+	OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname)
+
+#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned)
+
+#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64)
+#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64)
+
+#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size)
+#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize)
+#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname)                               \
+	OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname)
+
+#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string)
+
+#endif /* JEMALLOC_INTERNAL_STATS_INTERNAL_H */

include/jemalloc/internal/sz.h

@@ -325,11 +325,13 @@ sz_s2u_compute_using_delta(size_t size) {
 	    : x - SC_LG_NGROUP - 1;
 	size_t delta = ZU(1) << lg_delta;
 	size_t delta_mask = delta - 1;
-	if (unlikely(size > SIZE_T_MAX - delta_mask)) {
+
+	size_t usize = size + delta_mask;
+	if (unlikely(usize < size)) {
+		/* size_t overflow. */
 		return 0;
 	}
-
-	size_t usize = (size + delta_mask) & ~delta_mask;
+	usize &= ~delta_mask;
 	return usize;
 }
 
@@ -359,7 +361,12 @@ sz_s2u_compute(size_t size) {
 		 * multiple of PAGE to minimize the memory overhead, especially
 		 * when using hugepages.
 		 */
-		size_t usize = PAGE_CEILING(size);
+		size_t usize = size + PAGE_MASK;
+		if (unlikely(usize < size)) {
+			/* size_t overflow. */
+			return 0;
+		}
+		usize &= ~PAGE_MASK;
 		assert(usize - size < PAGE);
 		return usize;
 	}
@@ -416,10 +423,12 @@ sz_sa2u(size_t size, size_t alignment) {
 		 *    144 | 10100000 |  32
 		 *    192 | 11000000 |  64
 		 */
-		if (unlikely(size > SIZE_T_MAX - alignment_mask)) {
+		size_t rounded = size + alignment_mask;
+		if (unlikely(rounded < size)) {
+			/* size_t overflow. */
 			return 0;
 		}
-		usize = sz_s2u((size + alignment_mask) & ~alignment_mask);
+		usize = sz_s2u(rounded & ~alignment_mask);
 		if (usize < SC_LARGE_MINCLASS) {
 			return usize;
 		}

include/jemalloc/internal/test_hooks.h

@@ -4,6 +4,9 @@
 extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(void);
 extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void);
 extern JEMALLOC_EXPORT void (*test_hooks_safety_check_abort)(const char *);
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+extern JEMALLOC_EXPORT void (*test_hooks_tsd_bootstrap_hook)(void);
+#endif
 
 #if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
 #	define JEMALLOC_TEST_HOOK(fn, hook)                                   \

include/jemalloc/internal/tsd.h

@@ -4,7 +4,8 @@
 /*
  * We put the platform-specific data declarations and inlines into their own
  * header files to avoid cluttering this file.  They define tsd_boot0,
- * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates, tsd_get, and tsd_set.
+ * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates,
+ * tsd_teardown_done, tsd_get, and tsd_set.
  */
 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 #	include "jemalloc/internal/jemalloc_preamble.h"

include/jemalloc/internal/tsd_generic.h

@@ -27,6 +27,7 @@ void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block);
 void  tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 
 extern pthread_key_t   tsd_tsd;
+extern pthread_key_t   tsd_thread_initialized_tsd;
 extern tsd_init_head_t tsd_init_head;
 extern tsd_wrapper_t   tsd_boot_wrapper;
 extern bool            tsd_booted;
@@ -51,6 +52,12 @@ tsd_cleanup_wrapper(void *arg) {
 			return;
 		}
 	}
+	/*
+	 * Leave tsd_thread_initialized_tsd set.  It has no destructor and
+	 * intentionally outlives the TSD wrapper, so later jemalloc calls can
+	 * distinguish this fully-torn-down thread from one that has never
+	 * initialized TSD.
+	 */
 	malloc_tsd_dalloc(wrapper);
 }
 
@@ -63,6 +70,11 @@ tsd_wrapper_set(tsd_wrapper_t *wrapper) {
 		malloc_write("<jemalloc>: Error setting TSD\n");
 		abort();
 	}
+	if (pthread_setspecific(tsd_thread_initialized_tsd,
+	    (void *)&tsd_thread_initialized_tsd) != 0) {
+		malloc_write("<jemalloc>: Error setting TSD\n");
+		abort();
+	}
 }
 
 JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
@@ -116,6 +128,15 @@ tsd_boot0(void) {
 	if (pthread_key_create(&tsd_tsd, tsd_cleanup_wrapper) != 0) {
 		return true;
 	}
+	/*
+	 * This key has no destructor.  It records that the current thread once
+	 * had jemalloc TSD, so a later NULL tsd_tsd value means teardown has
+	 * finished rather than initialization not having happened yet.
+	 */
+	if (pthread_key_create(&tsd_thread_initialized_tsd, NULL) != 0) {
+		pthread_key_delete(tsd_tsd);
+		return true;
+	}
 	tsd_booted = true;
 	tsd_wrapper_set(&tsd_boot_wrapper);
 	tsd_init_finish(&tsd_init_head, &block);
@@ -160,6 +181,12 @@ tsd_get_allocates(void) {
 	return true;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return tsd_booted && pthread_getspecific(tsd_tsd) == NULL
+	    && pthread_getspecific(tsd_thread_initialized_tsd) != NULL;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_internals.h

@@ -83,6 +83,7 @@ typedef void (*test_callback_t)(int *);
 	O(arena, arena_t *, arena_t *)                                         \
 	O(arena_decay_ticker, ticker_geom_t, ticker_geom_t)                    \
 	O(sec_shard, uint8_t, uint8_t)                                         \
+	O(pac_sec_shard, uint8_t, uint8_t)                                     \
 	O(binshards, tsd_binshards_t, tsd_binshards_t)                         \
 	O(peak, peak_t, peak_t)                                                \
 	O(tcache_slow, tcache_slow_t, tcache_slow_t)                           \
@@ -102,6 +103,7 @@ typedef void (*test_callback_t)(int *);
 	    /* arena */ NULL, /* arena_decay_ticker */                         \
 	    TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),                   \
 	    /* sec_shard */ (uint8_t) - 1,                                     \
+	    /* pac_sec_shard */ (uint8_t) - 1,                                 \
 	    /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER,                    \
 	    /* peak */ PEAK_INITIALIZER, /* tcache_slow */                     \
 	    TCACHE_SLOW_ZERO_INITIALIZER,                                      \

include/jemalloc/internal/tsd_malloc_thread_cleanup.h

@@ -50,6 +50,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_tls.h

@@ -43,6 +43,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_win.h

@@ -146,6 +146,11 @@ tsd_get_allocates(void) {
 	return true;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
@@ -220,6 +225,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

scripts/README_GH_ACTIONS.md

@@ -27,12 +27,12 @@ The script can generate workflows for different platforms:
 
 ### Linux CI (`linux-ci.yml`)
 - **test-linux** (AMD64): `ubuntu-latest` (x86_64)
-  - ~96 configurations covering GCC, Clang, various flags
+  - ~98 configurations covering GCC, Clang, various flags
 - **test-linux-arm64** (ARM64): `ubuntu-24.04-arm` (aarch64)
   - ~14 configurations including large hugepage tests
   - **Note:** Free ARM64 runners (Public Preview) - may have longer queue times during peak hours
 
-**Total:** 110 configurations
+**Total:** 112 configurations
 
 ### macOS CI (`macos-ci.yml`)
 - **test-macos** (Intel): `macos-15-intel` (x86_64)
@@ -161,7 +161,7 @@ The Windows workflow uses:
 ### Linux Build Process
 - Ubuntu Latest for AMD64, Ubuntu 24.04 for ARM64
 - Installs 32-bit cross-compilation dependencies when needed
-- Most comprehensive test matrix (110 configurations)
+- Most comprehensive test matrix (112 configurations)
 
 ## Relationship to Travis CI
 
@@ -178,4 +178,3 @@ To regenerate all workflows after modifying `gen_gh_actions.py`:
 ```
 
 **Note**: The generated files should not be edited by hand. All changes should be made to `gen_gh_actions.py` and then regenerated.
-

scripts/gen_gh_actions.py

@@ -224,11 +224,8 @@ def generate_linux_job(arch):
     if arch != ARM64:
         exclude += [LARGE_HUGEPAGE]
 
-    linux_configure_flags = list(configure_flag_unusuals)
-    linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))
-
     linux_unusuals = (compilers_unusual + feature_unusuals
-                    + linux_configure_flags + malloc_conf_unusuals)
+                    + configure_flag_unusuals + malloc_conf_unusuals)
 
     matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, linux_unusuals)
 
@@ -273,9 +270,39 @@ def generate_linux_job(arch):
             'CXX': 'g++',
             'CONFIGURE_FLAGS': '--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof',
             'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': 'force_tls=0',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': 'force_tls=0 --enable-debug',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
         }
     ]
 
+    # --enable-cxx-infallible-new coverage. Plain variant runs on all arches;
+    # debug-combined variant runs only on AMD64 to bound matrix size.
+    infallible_new_entries = [
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+    ]
+    if arch == AMD64:
+        infallible_new_entries.append({
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new --enable-debug',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        })
+
     if arch == AMD64:
         for entry in manual_entries:
             job += "          - env:\n"
@@ -285,6 +312,14 @@ def generate_linux_job(arch):
                 else:
                     job += f"              {key}: {value}\n"
 
+    for entry in infallible_new_entries:
+        job += "          - env:\n"
+        for key, value in entry.items():
+            if ' ' in str(value):
+                job += f'              {key}: "{value}"\n'
+            else:
+                job += f"              {key}: {value}\n"
+
     job += f"""
     steps:
     - uses: actions/checkout@v6
@@ -385,6 +420,24 @@ def generate_macos_job(arch):
             else:
                 job += f"              {key}: {value}\n"
 
+    # --enable-cxx-infallible-new coverage on macOS (both arches).
+    macos_extra_cflags = ' '.join(get_extra_cflags(OSX, GCC.value))
+    infallible_new_entries = [
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
+            'EXTRA_CFLAGS': macos_extra_cflags
+        },
+    ]
+    for entry in infallible_new_entries:
+        job += "          - env:\n"
+        for key, value in entry.items():
+            if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
+                job += f'              {key}: "{value}"\n'
+            else:
+                job += f"              {key}: {value}\n"
+
     job += f"""
     steps:
     - uses: actions/checkout@v6
@@ -467,6 +520,24 @@ def generate_windows_job(arch):
             else:
                 job += f"              {key}: {value}\n"
 
+    # --enable-cxx-infallible-new coverage on Windows (MinGW-GCC only).
+    windows_mingw_cflags = ' '.join(get_extra_cflags(WINDOWS, GCC.value))
+    infallible_new_entries = [
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
+            'EXTRA_CFLAGS': windows_mingw_cflags
+        },
+    ]
+    for entry in infallible_new_entries:
+        job += "          - env:\n"
+        for key, value in entry.items():
+            if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
+                job += f'              {key}: "{value}"\n'
+            else:
+                job += f"              {key}: {value}\n"
+
     job += f"""
     steps:
     - uses: actions/checkout@v6

scripts/gen_travis.py

@@ -264,11 +264,8 @@ def generate_linux(arch):
     if arch != ARM64:
         exclude += [LARGE_HUGEPAGE]
 
-    linux_configure_flags = list(configure_flag_unusuals)
-    linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))
-
     linux_unusuals = (compilers_unusual + feature_unusuals
-                    + linux_configure_flags + malloc_conf_unusuals)
+                    + configure_flag_unusuals + malloc_conf_unusuals)
 
     return generate_jobs(os, arch, exclude, max_unusual_opts, linux_unusuals)
 

src/arena.c

@@ -83,11 +83,6 @@ const arena_config_t arena_config_default = {
  * definition.
  */
 
-static bool arena_decay_dirty(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-static void arena_maybe_do_deferred_work(
-    tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new);
-
 /******************************************************************************/
 
 void
@@ -96,8 +91,8 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
 	*nthreads += arena_nthreads_get(arena, false);
 	*dss = dss_prec_names[arena_dss_prec_get(arena)];
-	*dirty_decay_ms = arena_decay_ms_get(arena, extent_state_dirty);
-	*muzzy_decay_ms = arena_decay_ms_get(arena, extent_state_muzzy);
+	*dirty_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_dirty);
+	*muzzy_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_muzzy);
 	pa_shard_basic_stats_merge(&arena->pa_shard, nactive, ndirty, nmuzzy);
 }
 
@@ -294,7 +289,7 @@ arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
  * from scratch (descriptors held by other threads at fork time are gone)
  * without locking.
  */
-void
+static void
 arena_cache_bin_array_postfork_child(arena_t *arena,
     cache_bin_array_descriptor_t *desc_or_null) {
 	cassert(config_stats);
@@ -316,34 +311,6 @@ arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
 	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 }
 
-static void
-arena_background_thread_inactivity_check(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
-	if (!background_thread_enabled() || is_background_thread) {
-		return;
-	}
-	background_thread_info_t *info = arena_background_thread_info_get(
-	    arena);
-	if (background_thread_indefinite_sleep(info)) {
-		arena_maybe_do_deferred_work(
-		    tsdn, arena, &arena->pa_shard.pac.decay_dirty, 0);
-	}
-}
-
-/*
- * React to deferred work generated by a PAI function.
- */
-void
-arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) {
-	witness_assert_depth_to_rank(
-	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-
-	if (decay_immediately(&arena->pa_shard.pac.decay_dirty)) {
-		arena_decay_dirty(tsdn, arena, false, true);
-	}
-	arena_background_thread_inactivity_check(tsdn, arena, false);
-}
-
 static void
 arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) {
 	cassert(config_stats);
@@ -476,188 +443,12 @@ arena_extent_ralloc_large_expand(
 	}
 }
 
-/*
- * In situations where we're not forcing a decay (i.e. because the user
- * specifically requested it), should we purge ourselves, or wait for the
- * background thread to get to it.
- */
-static pac_purge_eagerness_t
-arena_decide_unforced_purge_eagerness(bool is_background_thread) {
-	if (is_background_thread) {
-		return PAC_PURGE_ALWAYS;
-	} else if (!is_background_thread && background_thread_enabled()) {
-		return PAC_PURGE_NEVER;
-	} else {
-		return PAC_PURGE_ON_EPOCH_ADVANCE;
-	}
-}
-
-bool
-arena_decay_ms_set(
-    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms) {
-	pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
-	    /* is_background_thread */ false);
-	return pa_decay_ms_set(
-	    tsdn, &arena->pa_shard, state, decay_ms, eagerness);
-}
-
-ssize_t
-arena_decay_ms_get(arena_t *arena, extent_state_t state) {
-	return pa_decay_ms_get(&arena->pa_shard, state);
-}
-
-static bool
-arena_decay_impl(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
-    pac_decay_stats_t *decay_stats, ecache_t *ecache, bool is_background_thread,
-    bool all) {
-	if (all) {
-		malloc_mutex_lock(tsdn, &decay->mtx);
-		pac_decay_all(tsdn, &arena->pa_shard.pac, decay, decay_stats,
-		    ecache, /* fully_decay */ all);
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-
-	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
-		/* No need to wait if another thread is in progress. */
-		return true;
-	}
-	pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
-	    is_background_thread);
-	bool epoch_advanced = pac_maybe_decay_purge(
-	    tsdn, &arena->pa_shard.pac, decay, decay_stats, ecache, eagerness);
-	size_t npages_new JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(0);
-	if (epoch_advanced) {
-		/* Backlog is updated on epoch advance. */
-		npages_new = decay_epoch_npages_delta(decay);
-	}
-	malloc_mutex_unlock(tsdn, &decay->mtx);
-
-	if (have_background_thread && background_thread_enabled()
-	    && epoch_advanced && !is_background_thread) {
-		arena_maybe_do_deferred_work(tsdn, arena, decay, npages_new);
-	}
-
-	return false;
-}
-
-static bool
-arena_decay_dirty(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_dirty,
-	    &arena->pa_shard.pac.stats->decay_dirty,
-	    &arena->pa_shard.pac.ecache_dirty, is_background_thread, all);
-}
-
-static bool
-arena_decay_muzzy(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	if (pa_shard_dont_decay_muzzy(&arena->pa_shard)) {
-		return false;
-	}
-	return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_muzzy,
-	    &arena->pa_shard.pac.stats->decay_muzzy,
-	    &arena->pa_shard.pac.ecache_muzzy, is_background_thread, all);
-}
-
-void
-arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	if (all) {
-		/*
-		 * We should take a purge of "all" to mean "save as much memory
-		 * as possible", including flushing any caches (for situations
-		 * like thread death, or manual purge calls).
-		 */
-		pa_shard_flush(tsdn, &arena->pa_shard);
-	}
-	if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
-		return;
-	}
-	arena_decay_muzzy(tsdn, arena, is_background_thread, all);
-}
-
-static bool
-arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
-    background_thread_info_t *info, nstime_t *remaining_sleep,
-    size_t npages_new) {
-	malloc_mutex_assert_owner(tsdn, &info->mtx);
-
-	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
-		return false;
-	}
-
-	if (!decay_gradually(decay)) {
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-
-	nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
-	if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-	nstime_subtract(remaining_sleep, &decay->epoch);
-	if (npages_new > 0) {
-		uint64_t npurge_new = decay_npages_purge_in(
-		    decay, remaining_sleep, npages_new);
-		info->npages_to_purge_new += npurge_new;
-	}
-	malloc_mutex_unlock(tsdn, &decay->mtx);
-	return info->npages_to_purge_new
-	    > ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD;
-}
-
-/*
- * Check if deferred work needs to be done sooner than planned.
- * For decay we might want to wake up earlier because of an influx of dirty
- * pages. Rather than waiting for previously estimated time, we proactively
- * purge those pages.
- * If background thread sleeps indefinitely, always wake up because some
- * deferred work has been generated.
- */
 static void
-arena_maybe_do_deferred_work(
-    tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new) {
-	background_thread_info_t *info = arena_background_thread_info_get(
-	    arena);
-	if (malloc_mutex_trylock(tsdn, &info->mtx)) {
-		/*
-		 * Background thread may hold the mutex for a long period of
-		 * time.  We'd like to avoid the variance on application
-		 * threads.  So keep this non-blocking, and leave the work to a
-		 * future epoch.
-		 */
-		return;
-	}
-	if (!background_thread_is_started(info)) {
-		goto label_done;
-	}
-
-	nstime_t remaining_sleep;
-	if (background_thread_indefinite_sleep(info)) {
-		background_thread_wakeup_early(info, NULL);
-	} else if (arena_should_decay_early(tsdn, arena, decay, info,
-	               &remaining_sleep, npages_new)) {
-		info->npages_to_purge_new = 0;
-		background_thread_wakeup_early(info, &remaining_sleep);
-	}
-label_done:
-	malloc_mutex_unlock(tsdn, &info->mtx);
-}
-
-/* Called from background threads. */
-void
-arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) {
-	arena_decay(tsdn, arena, true, false);
-	pa_shard_do_deferred_work(tsdn, &arena->pa_shard);
-}
-
-void
 arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) {
 	bool deferred_work_generated = false;
 	pa_dalloc(tsdn, &arena->pa_shard, slab, &deferred_work_generated);
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 }
 
@@ -934,7 +725,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
 	    /* zero */ false, guarded, &deferred_work_generated);
 
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 
 	if (slab == NULL) {
@@ -1585,17 +1376,17 @@ arena_get_ehooks(const arena_t *arena) {
 extent_hooks_t *
 arena_set_extent_hooks(
     tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks) {
-	background_thread_info_t *info;
-	if (have_background_thread) {
-		info = arena_background_thread_info_get(arena);
-		malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
-	}
+	/*
+	 * Serialize disabling the HPA against the background thread via the
+	 * module-owned bracket (have_background_thread-gated internally) rather
+	 * than reaching into info->mtx directly.
+	 */
+	unsigned arena_ind = arena_ind_get(arena);
+	background_thread_serialize_lock(tsd, arena_ind);
 	/* No using the HPA now that we have the custom hooks. */
 	pa_shard_disable_hpa(tsd_tsdn(tsd), &arena->pa_shard);
 	extent_hooks_t *ret = base_extent_hooks_set(arena->base, extent_hooks);
-	if (have_background_thread) {
-		malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
-	}
+	background_thread_serialize_unlock(tsd, arena_ind);
 
 	return ret;
 }
@@ -1828,13 +1619,13 @@ arena_create_huge_arena(tsd_t *tsd, unsigned ind) {
 	 */
 	if (!background_thread_enabled()
 	    && arena_dirty_decay_ms_default_get() > 0) {
-		arena_decay_ms_set(
-		    tsd_tsdn(tsd), huge_arena, extent_state_dirty, 0);
+		pa_decay_ms_set(
+		    tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_dirty, 0);
 	}
 	if (!background_thread_enabled()
 	    && arena_muzzy_decay_ms_default_get() > 0) {
-		arena_decay_ms_set(
-		    tsd_tsdn(tsd), huge_arena, extent_state_muzzy, 0);
+		pa_decay_ms_set(
+		    tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_muzzy, 0);
 	}
 
 	return huge_arena;

src/arenas_management.c

@@ -202,8 +202,8 @@ arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
 		 * Purge if the old arena has no associated threads anymore and
 		 * no background threads.
 		 */
-		arena_decay(tsd_tsdn(tsd), oldarena,
-		    /* is_background_thread */ false, /* all */ true);
+		pa_shard_flush(tsd_tsdn(tsd), &oldarena->pa_shard,
+		    /* all */ true);
 	}
 }
 

src/background_thread.c

@@ -6,6 +6,7 @@
 #include "jemalloc/internal/background_thread.h"
 #include "jemalloc/internal/background_thread_inlines.h"
 #include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_a.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex.h"
@@ -33,6 +34,72 @@ size_t     max_background_threads;
 /* Thread info per-index. */
 background_thread_info_t *background_thread_info;
 
+/******************************************************************************/
+/*
+ * Config-independent lifecycle/state-ownership helpers.  Defined
+ * unconditionally and gated at runtime on have_background_thread, so callers in
+ * ctl.c / arena.c never touch background_thread_lock or info->state directly;
+ * they compile to runtime no-ops when !have_background_thread.
+ */
+
+void
+background_thread_arena_reset_begin(tsd_t *tsd, unsigned arena_ind) {
+	/* Temporarily disable the background thread during arena reset. */
+	if (have_background_thread) {
+		/*
+		 * Hold background_thread_lock across the whole arena-reset
+		 * body (acquired here, released in _finish) so a concurrent
+		 * background_threads_enable() cannot start a background
+		 * thread mid-reset.  This must happen whenever the feature
+		 * is compiled in (have_background_thread), even when the
+		 * thread is not currently enabled; the state transition
+		 * below is separately gated on background_thread_enabled().
+		 */
+		malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
+		if (background_thread_enabled()) {
+			background_thread_info_t *info =
+			    background_thread_info_get(arena_ind);
+			assert(info->state == background_thread_started);
+			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+			info->state = background_thread_paused;
+			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+		}
+	}
+}
+
+void
+background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		if (background_thread_enabled()) {
+			background_thread_info_t *info =
+			    background_thread_info_get(arena_ind);
+			assert(info->state == background_thread_paused);
+			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+			info->state = background_thread_started;
+			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+		}
+		malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
+	}
+}
+
+void
+background_thread_serialize_lock(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		background_thread_info_t *info =
+		    background_thread_info_get(arena_ind);
+		malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+	}
+}
+
+void
+background_thread_serialize_unlock(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		background_thread_info_t *info =
+		    background_thread_info_get(arena_ind);
+		malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+	}
+}
+
 /******************************************************************************/
 
 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
@@ -191,6 +258,50 @@ background_thread_cond_wait(
 	return ret;
 }
 
+static int
+background_thread_cond_init(pthread_cond_t *cond) {
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	pthread_condattr_t cond_attr;
+	int ret = pthread_condattr_init(&cond_attr);
+	if (ret != 0) {
+		return ret;
+	}
+	ret = pthread_condattr_setclock(&cond_attr, CLOCK_MONOTONIC);
+	if (ret != 0) {
+		pthread_condattr_destroy(&cond_attr);
+		return ret;
+	}
+	ret = pthread_cond_init(cond, &cond_attr);
+	pthread_condattr_destroy(&cond_attr);
+	return ret;
+#else
+	return pthread_cond_init(cond, NULL);
+#endif
+}
+
+/*
+ * Fill in the absolute deadline for pthread_cond_timedwait.  The clock read
+ * here MUST match the clock the condvar was initialized with in
+ * background_thread_cond_init, otherwise the deadline is interpreted against
+ * the wrong epoch.
+ */
+static void
+background_thread_wakeup_ts_init(struct timespec *ts, uint64_t interval) {
+	nstime_t wakeup;
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	struct timespec now;
+	clock_gettime(CLOCK_MONOTONIC, &now);
+	nstime_init2(&wakeup, now.tv_sec, now.tv_nsec);
+#else
+	struct timeval tv;
+	gettimeofday(&tv, NULL);
+	nstime_init2(&wakeup, tv.tv_sec, tv.tv_usec * 1000);
+#endif
+	nstime_iadd(&wakeup, interval);
+	ts->tv_sec = (size_t)nstime_sec(&wakeup);
+	ts->tv_nsec = (size_t)nstime_nsec(&wakeup);
+}
+
 static void
 background_thread_sleep(
     tsdn_t *tsdn, background_thread_info_t *info, uint64_t interval) {
@@ -199,11 +310,8 @@ background_thread_sleep(
 	}
 	info->npages_to_purge_new = 0;
 
-	struct timeval tv;
-	/* Specific clock required by timedwait. */
-	gettimeofday(&tv, NULL);
 	nstime_t before_sleep;
-	nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000);
+	nstime_init_update(&before_sleep);
 
 	int ret;
 	if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
@@ -223,21 +331,16 @@ background_thread_sleep(
 		background_thread_wakeup_time_set(
 		    tsdn, info, nstime_ns(&next_wakeup));
 
-		nstime_t ts_wakeup;
-		nstime_copy(&ts_wakeup, &before_sleep);
-		nstime_iadd(&ts_wakeup, interval);
 		struct timespec ts;
-		ts.tv_sec = (size_t)nstime_sec(&ts_wakeup);
-		ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup);
+		background_thread_wakeup_ts_init(&ts, interval);
 
 		assert(!background_thread_indefinite_sleep(info));
 		ret = background_thread_cond_wait(info, &ts);
 		assert(ret == ETIMEDOUT || ret == 0);
 	}
 	if (config_stats) {
-		gettimeofday(&tv, NULL);
 		nstime_t after_sleep;
-		nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000);
+		nstime_init_update(&after_sleep);
 		if (nstime_compare(&after_sleep, &before_sleep) > 0) {
 			nstime_subtract(&after_sleep, &before_sleep);
 			nstime_add(&info->tot_sleep_time, &after_sleep);
@@ -262,7 +365,7 @@ background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) {
 static inline void
 background_work_sleep_once(
     tsdn_t *tsdn, background_thread_info_t *info, unsigned ind) {
-	uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX;
+	uint64_t ns_until_deferred = DEFERRED_WORK_MAX;
 	unsigned narenas = narenas_total_get();
 	bool     slept_indefinitely = background_thread_indefinite_sleep(info);
 
@@ -277,7 +380,8 @@ background_work_sleep_once(
 		 * work that caused this thread to wake up is scheduled for.
 		 */
 		if (!slept_indefinitely) {
-			arena_do_deferred_work(tsdn, arena);
+			pa_shard_do_deferred_work(tsdn, &arena->pa_shard,
+			    /* is_background_thread */ true);
 		}
 		if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) {
 			/* Min interval will be used. */
@@ -291,7 +395,7 @@ background_work_sleep_once(
 	}
 
 	uint64_t sleep_ns;
-	if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) {
+	if (ns_until_deferred == DEFERRED_WORK_MAX) {
 		sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP;
 	} else {
 		sleep_ns = (ns_until_deferred
@@ -751,7 +855,7 @@ background_thread_postfork_child(tsdn_t *tsdn) {
 		background_thread_info_t *info = &background_thread_info[i];
 		malloc_mutex_lock(tsdn, &info->mtx);
 		info->state = background_thread_stopped;
-		int ret = pthread_cond_init(&info->cond, NULL);
+		int ret = background_thread_cond_init(&info->cond);
 		assert(ret == 0);
 		background_thread_info_init(tsdn, info);
 		malloc_mutex_unlock(tsdn, &info->mtx);
@@ -869,7 +973,7 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) {
 		        malloc_mutex_address_ordered)) {
 			return true;
 		}
-		if (pthread_cond_init(&info->cond, NULL)) {
+		if (background_thread_cond_init(&info->cond)) {
 			return true;
 		}
 		malloc_mutex_lock(tsdn, &info->mtx);

src/base.c

@@ -8,6 +8,8 @@
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/sz.h"
 
+JET_EXTERN ehooks_t *base_ehooks_get_for_metadata(base_t *base);
+
 /*
  * In auto mode, arenas switch to huge pages for the base allocator on the
  * second base block.  a0 switches to thp on the 5th block (after 20 megabytes
@@ -17,6 +19,17 @@
 #define BASE_AUTO_THP_THRESHOLD 2
 #define BASE_AUTO_THP_THRESHOLD_A0 5
 
+/*
+ * Cap the base-block growth heuristic in base_block_alloc().  The
+ * growth heuristic reduces the number of disjoint VM ranges when new base
+ * blocks are rare, but high thread churn can cause many parallel misses for
+ * metadata allocations.  Without a cap, those misses can advance
+ * base->pind_last causing small requests to mmap multi-TiB blocks and exhaust
+ * the address space.  Large individual requests still use min_block_size and
+ * can exceed this cap.
+ */
+#define BASE_BLOCK_GROWTH_MAX ((size_t)128 << 20) /* 128 MiB */
+
 /******************************************************************************/
 /* Data. */
 
@@ -374,7 +387,11 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind,
 	pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS))
 	    ? *pind_last + 1
 	    : *pind_last;
-	size_t   next_block_size = base_block_size_ceil(sz_pind2sz(pind_next));
+	size_t next_block_size = base_block_size_ceil(sz_pind2sz(pind_next));
+	size_t max_block_size = base_block_size_ceil(BASE_BLOCK_GROWTH_MAX);
+	next_block_size = (next_block_size < max_block_size)
+	    ? next_block_size
+	    : max_block_size;
 	size_t   block_size = (min_block_size > next_block_size)
 	      ? min_block_size
 	      : next_block_size;
@@ -545,7 +562,7 @@ base_ehooks_get(base_t *base) {
 	return &base->ehooks;
 }
 
-ehooks_t *
+JET_EXTERN ehooks_t *
 base_ehooks_get_for_metadata(base_t *base) {
 	return &base->ehooks_base;
 }

src/bin.c

@@ -69,7 +69,7 @@ bin_postfork_child(tsdn_t *tsdn, bin_t *bin) {
 	malloc_mutex_postfork_child(tsdn, &bin->lock);
 }
 
-void *
+JET_EXTERN void *
 bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info) {
 	void        *ret;
 	slab_data_t *slab_data = edata_slab_data_get(slab);
@@ -134,7 +134,7 @@ bin_slab_reg_alloc_batch(
 	edata_nfree_sub(slab, cnt);
 }
 
-void
+JET_EXTERN void
 bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) {
 	assert(edata_nfree_get(slab) > 0);
 	edata_heap_insert(&bin->slabs_nonfull, slab);
@@ -143,7 +143,7 @@ bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) {
 	edata_heap_remove(&bin->slabs_nonfull, slab);
 	if (config_stats) {
@@ -151,7 +151,7 @@ bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) {
 	}
 }
 
-edata_t *
+JET_EXTERN edata_t *
 bin_slabs_nonfull_tryget(bin_t *bin) {
 	edata_t *slab = edata_heap_remove_first(&bin->slabs_nonfull);
 	if (slab == NULL) {
@@ -164,7 +164,7 @@ bin_slabs_nonfull_tryget(bin_t *bin) {
 	return slab;
 }
 
-void
+JET_EXTERN void
 bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab) {
 	assert(edata_nfree_get(slab) == 0);
 	/*
@@ -186,7 +186,7 @@ bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab) {
 	edata_list_active_remove(&bin->slabs_full, slab);
 }
 
-void
+JET_EXTERN void
 bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin) {
 	/* Dissociate slab from bin. */
 	if (slab == bin->slabcur) {
@@ -208,7 +208,7 @@ bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) {
 	assert(edata_nfree_get(slab) > 0);
 
@@ -234,7 +234,7 @@ bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin) {
 	malloc_mutex_assert_owner(tsdn, &bin->lock);
 

src/bitmap.c

@@ -5,8 +5,11 @@
 
 /******************************************************************************/
 
+JET_EXTERN size_t bitmap_size(const bitmap_info_t *binfo);
+
 #ifdef BITMAP_USE_TREE
 
+#ifdef JEMALLOC_JET
 void
 bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	unsigned i;
@@ -34,6 +37,7 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	binfo->nlevels = i;
 	binfo->nbits = nbits;
 }
+#endif
 
 static size_t
 bitmap_info_ngroups(const bitmap_info_t *binfo) {
@@ -82,6 +86,7 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) {
 
 #else /* BITMAP_USE_TREE */
 
+#ifdef JEMALLOC_JET
 void
 bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	assert(nbits > 0);
@@ -90,6 +95,7 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits) {
 	binfo->ngroups = BITMAP_BITS2GROUPS(nbits);
 	binfo->nbits = nbits;
 }
+#endif
 
 static size_t
 bitmap_info_ngroups(const bitmap_info_t *binfo) {
@@ -115,7 +121,7 @@ bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill) {
 
 #endif /* BITMAP_USE_TREE */
 
-size_t
+JET_EXTERN size_t
 bitmap_size(const bitmap_info_t *binfo) {
 	return (bitmap_info_ngroups(binfo) << LG_SIZEOF_BITMAP);
 }

src/buf_writer.c

@@ -68,7 +68,7 @@ buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer, write_cb_t *write_cb,
 	return buf_writer->buf == NULL;
 }
 
-void
+JET_EXTERN void
 buf_writer_flush(buf_writer_t *buf_writer) {
 	buf_writer_assert(buf_writer);
 	if (buf_writer->buf == NULL) {

src/ckh.c

@@ -541,6 +541,7 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) {
 	return true;
 }
 
+#ifdef JEMALLOC_JET
 void
 ckh_string_hash(const void *key, size_t r_hash[2]) {
 	hash(key, strlen((const char *)key), 0x94122f33U, r_hash);
@@ -570,3 +571,4 @@ bool
 ckh_pointer_keycomp(const void *k1, const void *k2) {
 	return (k1 == k2);
 }
+#endif

src/conf.c

@@ -731,11 +731,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			if (config_xmalloc) {
 				CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
 			}
-			if (config_enable_cxx) {
-				CONF_HANDLE_BOOL(
-				    opt_experimental_infallible_new,
-				    "experimental_infallible_new")
-			}
 
 			CONF_HANDLE_BOOL(opt_experimental_tcache_gc,
 			    "experimental_tcache_gc")
@@ -880,10 +875,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			    "hpa_min_purge_interval_ms", 0, 0,
 			    CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false);
 
-			CONF_HANDLE_SSIZE_T(
-			    opt_hpa_opts.experimental_max_purge_nhp,
-			    "experimental_hpa_max_purge_nhp", -1, SSIZE_MAX);
-
 			/*
 			 * Accept either a ratio-based or an exact purge
 			 * threshold.
@@ -955,6 +946,17 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
 			    "hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
 			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
+			CONF_HANDLE_SIZE_T(opt_pac_sec_opts.nshards,
+			    "experimental_pac_sec_nshards", 0, 0,
+			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
+			CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_alloc,
+			    "experimental_pac_sec_max_alloc", PAGE,
+			    USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN,
+			    CONF_CHECK_MAX, true);
+			CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_bytes,
+			    "experimental_pac_sec_max_bytes",
+			    SEC_OPTS_MAX_BYTES_DEFAULT, 0,
+			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
 
 			if (CONF_MATCH("slab_sizes")) {
 				if (CONF_MATCH_VALUE("default")) {

src/ehooks.c

@@ -228,7 +228,7 @@ ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b) {
 	return false;
 }
 
-bool
+static bool
 ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
     void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
 	tsdn_t *tsdn = tsdn_fetch();

src/hpa.c

@@ -1,6 +1,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 
 #include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/hpa_utils.h"
 #include "jemalloc/internal/jemalloc_probe.h"
@@ -14,7 +15,19 @@ const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
 bool opt_experimental_hpa_start_huge_if_thp_always = true;
 bool opt_experimental_hpa_enforce_hugify = false;
 
-bool
+static inline uint8_t
+hpa_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
+	if (sec->opts.nshards <= 1) {
+		return 0;
+	}
+	if (tsdn_null(tsdn)) {
+		return 0;
+	}
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd));
+}
+
+JET_EXTERN bool
 hpa_hugepage_size_exceeds_limit(void) {
 	return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE;
 }
@@ -622,22 +635,8 @@ hpa_shard_maybe_do_deferred_work(
 	 * too frequently.
 	 */
 	if (hpa_min_purge_interval_passed(tsdn, shard)) {
-		size_t max_purges = max_ops;
-		/*
-		 * Limit number of hugepages (slabs) to purge.
-		 * When experimental_max_purge_nhp option is used, there is no
-		 * guarantee we'll always respect dirty_mult option.  Option
-		 * experimental_max_purge_nhp provides a way to configure same
-		 * behavior as was possible before, with buggy implementation
-		 * of purging algorithm.
-		 */
-		ssize_t max_purge_nhp = shard->opts.experimental_max_purge_nhp;
-		if (max_purge_nhp != -1 && max_purges > (size_t)max_purge_nhp) {
-			max_purges = max_purge_nhp;
-		}
-
 		malloc_mutex_assert_owner(tsdn, &shard->mtx);
-		nops += hpa_purge(tsdn, shard, max_purges);
+		nops += hpa_purge(tsdn, shard, max_ops);
 		malloc_mutex_assert_owner(tsdn, &shard->mtx);
 	}
 
@@ -945,9 +944,13 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	    && (size > shard->opts.slab_max_alloc)) {
 		return NULL;
 	}
-	edata_t *edata = sec_alloc(tsdn, &shard->sec, size);
-	if (edata != NULL) {
-		return edata;
+	edata_t *edata = NULL;
+	if (sec_size_supported(&shard->sec, size)) {
+		edata = sec_alloc(tsdn, &shard->sec, size,
+		    hpa_sec_shard_pick(tsdn, &shard->sec));
+		if (edata != NULL) {
+			return edata;
+		}
 	}
 	edata_list_active_t results;
 	edata_list_active_init(&results);
@@ -966,7 +969,8 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	}
 	if (nsuccess > 0) {
 		assert(sec_size_supported(&shard->sec, size));
-		sec_fill(tsdn, &shard->sec, size, &results, nsuccess);
+		sec_fill(tsdn, &shard->sec, size, &results, nsuccess,
+		    hpa_sec_shard_pick(tsdn, &shard->sec));
 		/* Unlikely rollback in case of overfill */
 		if (!edata_list_active_empty(&results)) {
 			hpa_dalloc_batch(
@@ -978,20 +982,6 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	return edata;
 }
 
-bool
-hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
-    size_t new_size, bool zero, bool *deferred_work_generated) {
-	/* Expand not yet supported. */
-	return true;
-}
-
-bool
-hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
-    size_t new_size, bool *deferred_work_generated) {
-	/* Shrink not yet supported. */
-	return true;
-}
-
 static void
 hpa_dalloc_prepare_unlocked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
 	malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
@@ -1073,11 +1063,14 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
 	edata_list_active_init(&dalloc_list);
 	edata_list_active_append(&dalloc_list, edata);
 
-	sec_dalloc(tsdn, &shard->sec, &dalloc_list);
-	if (edata_list_active_empty(&dalloc_list)) {
-		/* sec consumed the pointer */
-		*deferred_work_generated = false;
-		return;
+	if (sec_size_supported(&shard->sec, edata_size_get(edata))) {
+		sec_dalloc(tsdn, &shard->sec, &dalloc_list,
+		    hpa_sec_shard_pick(tsdn, &shard->sec));
+		if (edata_list_active_empty(&dalloc_list)) {
+			/* sec consumed the pointer */
+			*deferred_work_generated = false;
+			return;
+		}
 	}
 	/* We may have more than one pointer to flush now */
 	hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated);
@@ -1089,7 +1082,7 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
  */
 uint64_t
 hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
-	uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
+	uint64_t time_ns = DEFERRED_WORK_MAX;
 
 	malloc_mutex_lock(tsdn, &shard->mtx);
 
@@ -1109,7 +1102,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
 			time_ns *= 1000 * 1000;
 		} else {
 			malloc_mutex_unlock(tsdn, &shard->mtx);
-			return BACKGROUND_THREAD_DEFERRED_MIN;
+			return DEFERRED_WORK_MIN;
 		}
 	}
 
@@ -1120,7 +1113,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
 		 */
 		if (shard->stats.npurge_passes == 0) {
 			malloc_mutex_unlock(tsdn, &shard->mtx);
-			return BACKGROUND_THREAD_DEFERRED_MIN;
+			return DEFERRED_WORK_MIN;
 		}
 		uint64_t since_last_purge_ms = shard->central->hooks.ms_since(
 		    &shard->last_purge);
@@ -1135,7 +1128,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
 				time_ns = until_purge_ns;
 			}
 		} else {
-			time_ns = BACKGROUND_THREAD_DEFERRED_MIN;
+			time_ns = DEFERRED_WORK_MIN;
 		}
 	}
 	malloc_mutex_unlock(tsdn, &shard->mtx);

src/hpdata.c

@@ -46,6 +46,7 @@ ph_gen(, hpdata_age_heap, hpdata_t, age_link, hpdata_age_comp)
 	hpdata_assert_consistent(hpdata);
 }
 
+#ifdef JEMALLOC_JET
 void *
 hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
 	hpdata_assert_consistent(hpdata);
@@ -138,6 +139,7 @@ hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz) {
 	return (
 	    void *)((byte_t *)hpdata_addr_get(hpdata) + (result << LG_PAGE));
 }
+#endif
 
 void
 hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {

src/jemalloc.c

@@ -182,7 +182,6 @@ void (*JET_MUTABLE invalid_conf_abort)(void) = &abort;
 
 bool     opt_utrace = false;
 bool     opt_xmalloc = false;
-bool     opt_experimental_infallible_new = false;
 bool     opt_experimental_tcache_gc = true;
 bool     opt_zero = false;
 unsigned opt_narenas = 0;
@@ -199,6 +198,9 @@ size_t opt_calloc_madvise_threshold = CALLOC_MADVISE_THRESHOLD_DEFAULT;
 bool             opt_hpa = false;
 hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT;
 sec_opts_t       opt_hpa_sec_opts = SEC_OPTS_DEFAULT;
+sec_opts_t       opt_pac_sec_opts = {0,
+    (32 * 1024) > (PAGE * 2) ? (32 * 1024) : (PAGE * 2),
+    SEC_OPTS_MAX_BYTES_DEFAULT};
 
 /* False should be the common case.  Set to true to trigger initialization. */
 bool malloc_slow = true;
@@ -1023,12 +1025,50 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) {
 	thread_dalloc_event(tsd, usize);
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+dealloc_no_tsd(void *ptr) {
+	/*
+	 * On the generic pthread_getspecific() TSD path, a deallocation can run
+	 * after the thread's TSD has been torn down.  Using tsd_fetch_min() here
+	 * would allocate and publish a fresh TSD wrapper mid-teardown,
+	 * reincarnating the state teardown just released (the crashes this change
+	 * fixes).  Instead we hand the object straight back to its arena via
+	 * idalloctm() with a NULL tsdn.
+	 *
+	 * This deliberately bypasses the per-thread bookkeeping that the normal
+	 * ifree()/isfree() slow path performs, none of which is available or
+	 * meaningful without a live TSD.  The trade-offs, all acceptable for
+	 * these rare teardown-time frees (the memory itself is still correctly
+	 * returned to the arena):
+	 *   - Profiling: prof_free() is skipped, so a sampled object is not
+	 *     unregistered from its prof context; its bytes stay counted as live
+	 *     in prof stats and in any final heap/leak dump.
+	 *   - Junk filling: opt_junk_free is not applied to the freed region.
+	 *   - Sized dealloc: for sdallocx() the caller-supplied size is ignored
+	 *     (szind is looked up from the extent map), so the sized-dealloc
+	 *     safety check does not run.
+	 *   - Thread events: thread_dalloc_event() does not fire, so this free
+	 *     does not advance decay / tcache GC or dalloc stats.
+	 */
+	if (!tsd_teardown_done()) {
+		return false;
+	}
+
+	idalloctm(TSDN_NULL, ptr, /* tcache */ NULL, /* alloc_ctx */ NULL,
+	    /* is_internal */ false, /* slow_path */ true);
+	return true;
+}
+
 JEMALLOC_NOINLINE
 void
 free_default(void *ptr) {
 	UTRACE(ptr, 0, 0);
 	if (likely(ptr != NULL)) {
 		int saved_errno = get_errno();
+		if (unlikely(dealloc_no_tsd(ptr))) {
+			set_errno(saved_errno);
+			return;
+		}
 		/*
 		 * We avoid setting up tsd fully (e.g. tcache, arena binding)
 		 * based on only free() calls -- other activities trigger the
@@ -1070,14 +1110,35 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW
 je_free_sized(void *ptr, size_t size) {
 	LOG("core.free_sized.entry", "ptr: %p, size: %zu", ptr, size);
 
-	je_sdallocx_noflags(ptr, size);
+	/*
+	 * free_sized(NULL, size) is a no-op, like free(NULL).  Do the NULL
+	 * check here rather than deferring to the je_free_impl() fallback:
+	 * without a size hint, free_fastpath() does an emap lookup that NULL
+	 * fails, so it reaches free_default(), which handles NULL.  With a
+	 * size hint, the fast path skips that lookup and falls through to
+	 * sdallocx_default(), which expects a non-NULL pointer.
+	 */
+	if (likely(ptr != NULL)) {
+		je_sdallocx_noflags(ptr, size);
+	}
 
 	LOG("core.free_sized.exit", "");
 }
 
 JEMALLOC_EXPORT void JEMALLOC_NOTHROW
 je_free_aligned_sized(void *ptr, size_t alignment, size_t size) {
-	je_sdallocx(ptr, size, /* flags */ MALLOCX_ALIGN(alignment));
+	LOG("core.free_aligned_sized.entry",
+	    "ptr: %p, alignment: %zu, size: %zu", ptr, alignment, size);
+
+	/*
+	 * Same as je_free_sized() above: the sdallocx path expects a non-NULL
+	 * pointer, so handle the C23 free_aligned_sized(NULL, ...) no-op here.
+	 */
+	if (likely(ptr != NULL)) {
+		je_sdallocx_impl(ptr, size, /* flags */ MALLOCX_ALIGN(alignment));
+	}
+
+	LOG("core.free_aligned_sized.exit", "");
 }
 
 /*
@@ -1545,6 +1606,9 @@ do_realloc_nonnull_zero(void *ptr) {
 		return do_rallocx(ptr, 1, MALLOCX_TCACHE_NONE, true);
 	} else if (opt_zero_realloc_action == zero_realloc_action_free) {
 		UTRACE(ptr, 0, 0);
+		if (unlikely(dealloc_no_tsd(ptr))) {
+			return NULL;
+		}
 		tsd_t *tsd = tsd_fetch();
 		check_entry_exit_locking(tsd_tsdn(tsd));
 
@@ -1820,6 +1884,12 @@ je_dallocx(void *ptr, int flags) {
 	assert(ptr != NULL);
 	assert(malloc_initialized() || malloc_is_initializer());
 
+	UTRACE(ptr, 0, 0);
+	if (unlikely(dealloc_no_tsd(ptr))) {
+		LOG("core.dallocx.exit", "");
+		return;
+	}
+
 	tsd_t *tsd = tsd_fetch_min();
 	bool   fast = tsd_fast(tsd);
 	check_entry_exit_locking(tsd_tsdn(tsd));
@@ -1828,7 +1898,6 @@ je_dallocx(void *ptr, int flags) {
 	tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
 	    /* is_alloc */ false);
 
-	UTRACE(ptr, 0, 0);
 	if (likely(fast)) {
 		tsd_assert_fast(tsd);
 		ifree(tsd, ptr, tcache, false);
@@ -1856,6 +1925,12 @@ sdallocx_default(void *ptr, size_t size, int flags) {
 	assert(ptr != NULL);
 	assert(malloc_initialized() || malloc_is_initializer());
 
+	UTRACE(ptr, 0, 0);
+	if (unlikely(dealloc_no_tsd(ptr))) {
+		set_errno(saved_errno);
+		return;
+	}
+
 	tsd_t *tsd = tsd_fetch_min();
 	bool   fast = tsd_fast(tsd);
 	size_t usize = inallocx(tsd_tsdn(tsd), size, flags);
@@ -1865,7 +1940,6 @@ sdallocx_default(void *ptr, size_t size, int flags) {
 	tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind, !fast,
 	    /* is_alloc */ false);
 
-	UTRACE(ptr, 0, 0);
 	if (likely(fast)) {
 		tsd_assert_fast(tsd);
 		isfree(tsd, ptr, usize, tcache, false);

src/jemalloc_cpp.cpp

@@ -66,20 +66,20 @@ void operator delete[](
 JEMALLOC_NOINLINE
 static void *
 handleOOM(std::size_t size, bool nothrow) {
-	if (opt_experimental_infallible_new) {
-		const char *huge_warning = (size >= ((std::size_t)1 << 30))
-		    ? "This may be caused by heap corruption, if the large size "
-		      "is unexpected (suggest building with sanitizers for "
-		      "debugging)."
-		    : "";
-
-		safety_check_fail(
-		    "<jemalloc>: Allocation of size %zu failed. "
-		    "%s opt.experimental_infallible_new is true. Aborting.\n",
-		    size, huge_warning);
+#ifdef JEMALLOC_INFALLIBLE_NEW
+	if (nothrow) {
 		return nullptr;
 	}
-
+	const char *huge_warning = (size >= ((std::size_t)1 << 30))
+	    ? "This may be caused by heap corruption, if the large size "
+	      "is unexpected (suggest building with sanitizers for "
+	      "debugging). "
+	    : "";
+	safety_check_fail(
+	    "<jemalloc>: Allocation of size %zu failed. %sAborting.\n",
+	    size, huge_warning);
+	return nullptr;
+#else
 	void *ptr = nullptr;
 
 	while (ptr == nullptr) {
@@ -108,6 +108,7 @@ handleOOM(std::size_t size, bool nothrow) {
 #endif
 	}
 	return ptr;
+#endif
 }
 
 template <bool IsNoExcept>
@@ -230,12 +231,11 @@ operator delete[](void *ptr, const std::nothrow_t &) noexcept {
 JEMALLOC_ALWAYS_INLINE
 void
 sizedDeleteImpl(void *ptr, std::size_t size) noexcept {
-	if (unlikely(ptr == nullptr)) {
-		return;
-	}
 	LOG("core.operator_delete.entry", "ptr: %p, size: %zu", ptr, size);
 
-	je_sdallocx_noflags(ptr, size);
+	if (likely(ptr != nullptr)) {
+		je_sdallocx_noflags(ptr, size);
+	}
 
 	LOG("core.operator_delete.exit", "");
 }
@@ -261,13 +261,12 @@ alignedSizedDeleteImpl(
 	if (config_debug) {
 		assert(((size_t)alignment & ((size_t)alignment - 1)) == 0);
 	}
-	if (unlikely(ptr == nullptr)) {
-		return;
-	}
 	LOG("core.operator_delete.entry", "ptr: %p, size: %zu, alignment: %zu",
 	    ptr, size, alignment);
 
-	je_sdallocx_impl(ptr, size, MALLOCX_ALIGN(alignment));
+	if (likely(ptr != nullptr)) {
+		je_sdallocx_impl(ptr, size, MALLOCX_ALIGN(alignment));
+	}
 
 	LOG("core.operator_delete.exit", "");
 }

src/large.c

@@ -78,7 +78,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) {
 		return true;
 	}
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 	arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize);
 
@@ -101,7 +101,7 @@ large_ralloc_no_move_expand(
 	    szind, zero, &deferred_work_generated);
 
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 
 	if (err) {
@@ -247,7 +247,7 @@ large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
 	bool deferred_work_generated = false;
 	pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated);
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 }
 

src/nstime.c

@@ -60,11 +60,13 @@ nstime_ns(const nstime_t *time) {
 	return time->ns;
 }
 
+#ifdef JEMALLOC_JET
 uint64_t
 nstime_ms(const nstime_t *time) {
 	nstime_assert_initialized(time);
 	return time->ns / MILLION;
 }
+#endif
 
 uint64_t
 nstime_sec(const nstime_t *time) {
@@ -119,6 +121,7 @@ nstime_subtract(nstime_t *time, const nstime_t *subtrahend) {
 	time->ns -= subtrahend->ns;
 }
 
+#ifdef JEMALLOC_JET
 void
 nstime_isubtract(nstime_t *time, uint64_t subtrahend) {
 	nstime_assert_initialized(time);
@@ -127,6 +130,7 @@ nstime_isubtract(nstime_t *time, uint64_t subtrahend) {
 	/* No initialize operand -- subtraction must be initialized. */
 	time->ns -= subtrahend;
 }
+#endif
 
 void
 nstime_imultiply(nstime_t *time, uint64_t multiplier) {
@@ -158,7 +162,7 @@ nstime_divide(const nstime_t *time, const nstime_t *divisor) {
 	return time->ns / divisor->ns;
 }
 
-uint64_t
+static uint64_t
 nstime_ns_between(const nstime_t *earlier, const nstime_t *later) {
 	nstime_assert_initialized(earlier);
 	nstime_assert_initialized(later);

src/pa.c

@@ -1,6 +1,8 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 
 #include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/background_thread_inlines.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/pa.h"
 
@@ -89,14 +91,21 @@ pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
 	atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
-	pa_shard_flush(tsdn, shard);
+	pa_shard_flush(tsdn, shard, /* all */ false);
 }
 
 void
-pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
+pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard, bool all) {
+	pac_sec_flush(tsdn, &shard->pac);
 	if (shard->ever_used_hpa) {
 		hpa_shard_flush(tsdn, &shard->hpa);
 	}
+	if (all) {
+		pac_decay_all_now(tsdn, &shard->pac, extent_state_dirty);
+		if (pac_should_decay_muzzy(&shard->pac)) {
+			pac_decay_all_now(tsdn, &shard->pac, extent_state_muzzy);
+		}
+	}
 }
 
 static bool
@@ -230,8 +239,8 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
 
 bool
 pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
-    ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
-	return pac_decay_ms_set(tsdn, &shard->pac, state, decay_ms, eagerness);
+    ssize_t decay_ms) {
+	return pac_decay_ms_set(tsdn, &shard->pac, state, decay_ms);
 }
 
 ssize_t
@@ -249,8 +258,28 @@ pa_shard_set_deferral_allowed(
 }
 
 void
-pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
-	if (pa_shard_uses_hpa(shard)) {
+pa_shard_handle_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
+	witness_assert_depth_to_rank(
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
+
+	if (pac_decay_immediately(&shard->pac)) {
+		pac_decay_all_now(tsdn, &shard->pac, extent_state_dirty);
+	}
+	if (background_thread_enabled()) {
+		pac_wake_bg_on_deferred(tsdn, &shard->pac);
+	}
+}
+
+void
+pa_shard_do_deferred_work(
+    tsdn_t *tsdn, pa_shard_t *shard, bool is_background_thread) {
+	pac_do_deferred_work(tsdn, &shard->pac, is_background_thread);
+	/*
+	 * Application threads self-throttle HPA deferred work inline from their
+	 * own alloc/dalloc path (hpa_shard_maybe_do_deferred_work, capped), so
+	 * only drive it (forced, uncapped) from here on the background thread.
+	 */
+	if (is_background_thread && pa_shard_uses_hpa(shard)) {
 		hpa_shard_do_deferred_work(tsdn, &shard->hpa);
 	}
 }
@@ -263,7 +292,7 @@ pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
 uint64_t
 pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
 	uint64_t time = pac_time_until_deferred_work(tsdn, &shard->pac);
-	if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
+	if (time == DEFERRED_WORK_MIN) {
 		return time;
 	}
 

src/pa_extra.c

@@ -17,6 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
 
 void
 pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
+	sec_prefork2(tsdn, &shard->pac.sec);
 	if (shard->ever_used_hpa) {
 		hpa_shard_prefork2(tsdn, &shard->hpa);
 	}
@@ -54,6 +55,7 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
+	sec_postfork_parent(tsdn, &shard->pac.sec);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
@@ -69,6 +71,7 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
 	ecache_postfork_child(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
+	sec_postfork_child(tsdn, &shard->pac.sec);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
@@ -81,8 +84,18 @@ pa_shard_nactive(const pa_shard_t *shard) {
 	return atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
 }
 
-size_t
-pa_shard_ndirty(const pa_shard_t *shard) {
+static size_t
+pac_sec_pinned_bytes_get(const sec_stats_t *stats) {
+	return min_zu(stats->bytes_pinned, stats->bytes);
+}
+
+static size_t
+pac_sec_dirty_npages_get(const sec_stats_t *stats) {
+	return (stats->bytes - pac_sec_pinned_bytes_get(stats)) >> LG_PAGE;
+}
+
+static size_t
+pa_shard_ndirty_no_pac_sec(const pa_shard_t *shard) {
 	size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty);
 	if (shard->ever_used_hpa) {
 		ndirty += psset_ndirty(&shard->hpa.psset);
@@ -90,7 +103,15 @@ pa_shard_ndirty(const pa_shard_t *shard) {
 	return ndirty;
 }
 
-size_t
+static size_t
+pa_shard_ndirty(const pa_shard_t *shard) {
+	sec_stats_t pac_sec_stats = {0};
+	sec_stats_merge(TSDN_NULL, &shard->pac.sec, &pac_sec_stats);
+	return pa_shard_ndirty_no_pac_sec(shard)
+	    + pac_sec_dirty_npages_get(&pac_sec_stats);
+}
+
+static size_t
 pa_shard_nmuzzy(const pa_shard_t *shard) {
 	return ecache_npages_get(&shard->pac.ecache_muzzy);
 }
@@ -109,17 +130,29 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     hpa_shard_stats_t *hpa_stats_out, size_t *resident) {
 	cassert(config_stats);
 
+	sec_stats_t pac_sec_stats = {0};
+	sec_stats_merge(tsdn, &shard->pac.sec, &pac_sec_stats);
+	sec_stats_accum(&pa_shard_stats_out->pac_stats.pac_sec_stats,
+	    &pac_sec_stats);
+	size_t pac_sec_pinned_bytes = pac_sec_pinned_bytes_get(&pac_sec_stats);
+	size_t pac_sec_pinned_npages = pac_sec_pinned_bytes >> LG_PAGE;
+	size_t pac_sec_dirty_npages =
+	    pac_sec_dirty_npages_get(&pac_sec_stats);
+
 	pa_shard_stats_out->pac_stats.retained +=
 	    ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
 	pa_shard_stats_out->pac_stats.pinned +=
-	    ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE;
+	    (ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE)
+	    + pac_sec_pinned_bytes;
 	pa_shard_stats_out->edata_avail += atomic_load_zu(
 	    &shard->edata_cache.count, ATOMIC_RELAXED);
 
 	size_t resident_pgs = 0;
 	resident_pgs += pa_shard_nactive(shard);
-	resident_pgs += pa_shard_ndirty(shard);
+	resident_pgs += pa_shard_ndirty_no_pac_sec(shard);
+	resident_pgs += pac_sec_dirty_npages;
 	resident_pgs += ecache_npages_get(&shard->pac.ecache_pinned);
+	resident_pgs += pac_sec_pinned_npages;
 	*resident += (resident_pgs << LG_PAGE);
 
 	/* Dirty decay stats */
@@ -167,6 +200,16 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 		pinned_bytes = ecache_nbytes_get(
 		    &shard->pac.ecache_pinned, i);
 
+		sec_pszind_stats_t pac_sec_pszind_stats = {0};
+		sec_stats_merge_pszind(
+		    tsdn, &shard->pac.sec, i, &pac_sec_pszind_stats);
+		dirty += pac_sec_pszind_stats.nextents
+		    - pac_sec_pszind_stats.nextents_pinned;
+		dirty_bytes += pac_sec_pszind_stats.bytes
+		    - pac_sec_pszind_stats.bytes_pinned;
+		pinned += pac_sec_pszind_stats.nextents_pinned;
+		pinned_bytes += pac_sec_pszind_stats.bytes_pinned;
+
 		estats_out[i].ndirty = dirty;
 		estats_out[i].nmuzzy = muzzy;
 		estats_out[i].nretained = retained;
@@ -208,6 +251,9 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 	    &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);
 
+	sec_mutex_stats_read(tsdn, &shard->pac.sec,
+	    &mutex_prof_data[arena_prof_mutex_pac_sec]);
+
 	if (shard->ever_used_hpa) {
 		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 		    &shard->hpa.mtx, arena_prof_mutex_hpa_shard);

src/pac.c

@@ -2,11 +2,25 @@
 
 #include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/background_thread_inlines.h"
+#include "jemalloc/internal/deferral.h"
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/pac.h"
 #include "jemalloc/internal/san.h"
 #include "jemalloc/internal/witness.h"
 
+static inline uint8_t
+pac_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
+	if (sec->opts.nshards <= 1) {
+		return 0;
+	}
+	if (tsdn_null(tsdn)) {
+		return 0;
+	}
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	return sec_shard_pick(tsd, sec, tsd_pac_sec_shardp_get(tsd));
+}
+
 static inline void
 pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay,
     pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) {
@@ -95,6 +109,16 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	pac->stats_mtx = stats_mtx;
 	atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
 
+	if (sec_init(tsdn, &pac->sec, base, &opt_pac_sec_opts)) {
+		/* sec_init already zeroed nshards and max_alloc. */
+	}
+	if (!sec_is_used(&pac->sec) || dirty_decay_ms == 0) {
+		atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED);
+	} else {
+		atomic_store_zu(&pac->sec_max_alloc,
+		    pac->sec.opts.max_alloc, ATOMIC_RELAXED);
+	}
+
 	return false;
 }
 
@@ -136,6 +160,24 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 
 	edata_t *edata = NULL;
 
+	if (!guarded && !zero && alignment <= PAGE
+	    && size <= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) {
+		/*
+		 * sec_max_alloc mirrors sec.opts.max_alloc when SEC is
+		 * enabled, 0 when dirty decay is disabled.
+		 */
+		edata = sec_alloc(tsdn, &pac->sec, size,
+		    pac_sec_shard_pick(tsdn, &pac->sec));
+		if (edata != NULL) {
+			return edata;
+		}
+		/*
+		 * Unlike HPA, PAC has no batch allocation primitive; we
+		 * intentionally do not refill SEC on a miss.  Extents enter the
+		 * cache only through the dalloc path.
+		 */
+	}
+
 	/*
 	 * Guarded allocations need surrounding guard pages, which the pinned
 	 * pool does not maintain; skip ecache_pinned in that case.
@@ -398,6 +440,37 @@ pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
 			san_unguard_pages_two_sided(
 			    tsdn, ehooks, edata, pac->emap);
 		}
+	} else if (edata_size_get(edata)
+	    <= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) {
+		/*
+		 * A dalloc can race with disabling SEC and cache an extent after
+		 * the flush.  Avoid a hot-path gate lock; such extents remain
+		 * stats-tracked and are flushed by reset/destroy or a later
+		 * disable.
+		 */
+		edata_list_active_t dalloc_list;
+		edata_list_active_init(&dalloc_list);
+		edata_list_active_append(&dalloc_list, edata);
+		sec_dalloc(tsdn, &pac->sec, &dalloc_list,
+		    pac_sec_shard_pick(tsdn, &pac->sec));
+		if (edata_list_active_empty(&dalloc_list)) {
+			*deferred_work_generated = false;
+			return;
+		}
+		/* Flush overflow extents to their backing ecaches. */
+		bool any_deferred_work = false;
+		edata_t *flush_edata;
+		while ((flush_edata =
+		    edata_list_active_first(&dalloc_list)) != NULL) {
+			edata_list_active_remove(&dalloc_list,
+			    flush_edata);
+			if (!edata_pinned_get(flush_edata)) {
+				any_deferred_work = true;
+			}
+			pac_ecache_dalloc(tsdn, pac, ehooks, flush_edata);
+		}
+		*deferred_work_generated = any_deferred_work;
+		return;
 	}
 
 	bool pinned = edata_pinned_get(edata);
@@ -411,10 +484,10 @@ static inline uint64_t
 pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) {
 	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
 		/* Use minimal interval if decay is contended. */
-		return BACKGROUND_THREAD_DEFERRED_MIN;
+		return DEFERRED_WORK_MIN;
 	}
 	uint64_t result = decay_ns_until_purge(
-	    decay, npages, ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD);
+	    decay, npages, PAC_DECAY_PURGE_NPAGES_THRESHOLD);
 
 	malloc_mutex_unlock(tsdn, &decay->mtx);
 	return result;
@@ -426,7 +499,7 @@ pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac) {
 
 	time = pac_ns_until_purge(
 	    tsdn, &pac->decay_dirty, ecache_npages_get(&pac->ecache_dirty));
-	if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
+	if (time == DEFERRED_WORK_MIN) {
 		return time;
 	}
 
@@ -708,9 +781,238 @@ pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
 	return epoch_advanced;
 }
 
+/*
+ * Run the deferred (non-forced) decay-purge for a single decay state, taking
+ * decay->mtx via trylock.  Sets *contended when the lock could not be acquired
+ * (in which case the return is false and *npages_new is untouched).  Returns
+ * whether the epoch advanced; when it did, *npages_new is set to the fresh
+ * backlog delta.
+ */
+static bool
+pac_decay_deferred_one(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
+    pac_decay_stats_t *decay_stats, ecache_t *ecache,
+    pac_purge_eagerness_t eagerness, bool *contended, size_t *npages_new) {
+	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
+		/* No need to wait if another thread is in progress. */
+		*contended = true;
+		return false;
+	}
+	*contended = false;
+	bool epoch_advanced = pac_maybe_decay_purge(
+	    tsdn, pac, decay, decay_stats, ecache, eagerness);
+	if (epoch_advanced) {
+		/* Backlog is updated on epoch advance. */
+		*npages_new = decay_epoch_npages_delta(decay);
+	}
+	malloc_mutex_unlock(tsdn, &decay->mtx);
+	return epoch_advanced;
+}
+
+/*
+ * Non-forced deferred decay-purge for both the dirty and muzzy states, at the
+ * given eagerness; corresponding decay->mtx are acquired internally.  Reports
+ * per-state epoch-advance in *result.  JET_EXTERN: exposed for deterministic
+ * unit testing only; production callers use pac_do_deferred_work.
+ */
+JET_EXTERN void
+pac_decay_deferred(tsdn_t *tsdn, pac_t *pac,
+    pac_purge_eagerness_t eagerness, pac_deferred_work_result_t *result) {
+	memset(result, 0, sizeof(*result));
+
+	bool contended;
+	result->dirty_epoch_advanced = pac_decay_deferred_one(tsdn, pac,
+	    &pac->decay_dirty, &pac->stats->decay_dirty, &pac->ecache_dirty,
+	    eagerness, &contended, &result->dirty_npages_new);
+	if (contended) {
+		/* When dirty decay is contended, don't wait on muzzy. */
+		return;
+	}
+
+	if (!pac_should_decay_muzzy(pac)) {
+		return;
+	}
+	result->muzzy_epoch_advanced = pac_decay_deferred_one(tsdn, pac,
+	    &pac->decay_muzzy, &pac->stats->decay_muzzy, &pac->ecache_muzzy,
+	    eagerness, &contended, &result->muzzy_npages_new);
+}
+
+/*
+ * This is the single, deliberate place where PAC reaches jointly into both a
+ * decay_t and a background_thread_info_t.  It is intentionally NOT decoupled
+ * further, for two reasons:
+ *   (a) The early-wake decision is intrinsically a JOINT info+decay
+ *       computation: holding info->mtx (outer) we trylock decay->mtx (inner)
+ *       to read remaining_sleep = wakeup_time - decay->epoch and the
+ *       decay_npages_purge_in() estimate atomically against epoch advance.
+ *       Splitting it would require copying these values across a new API while
+ *       still holding both locks, adding interface surface for no behavioral
+ *       gain.
+ *   (b) It runs on the application FREE path (pac_do_deferred_work ->
+ *       pac_maybe_wake_bg), where perf parity matters most.
+ *
+ * The info->mtx-outer / decay->mtx-inner trylock nesting is load-bearing for
+ * lock ordering; keep it.
+ *
+ * Parity trap: when npages_new == 0 (e.g. an epoch advanced with no new
+ * backlog) the accumulation below is skipped, but the THRESHOLD compare on the
+ * EXISTING info->npages_to_purge_new backlog still runs and can still trigger
+ * an early wakeup.  Do not "simplify" by early-returning when npages_new == 0.
+ */
+static bool
+pac_decay_should_wake_early(tsdn_t *tsdn, decay_t *decay,
+    background_thread_info_t *info, nstime_t *remaining_sleep,
+    size_t npages_new) {
+	malloc_mutex_assert_owner(tsdn, &info->mtx);
+
+	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
+		return false;
+	}
+	if (!decay_gradually(decay)) {
+		malloc_mutex_unlock(tsdn, &decay->mtx);
+		return false;
+	}
+	nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
+	if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
+		malloc_mutex_unlock(tsdn, &decay->mtx);
+		return false;
+	}
+	nstime_subtract(remaining_sleep, &decay->epoch);
+	if (npages_new > 0) {
+		uint64_t npurge_new = decay_npages_purge_in(
+		    decay, remaining_sleep, npages_new);
+		info->npages_to_purge_new += npurge_new;
+	}
+	malloc_mutex_unlock(tsdn, &decay->mtx);
+	return info->npages_to_purge_new
+	    > PAC_DECAY_PURGE_NPAGES_THRESHOLD;
+}
+
+/*
+ * The PAC's base index is, under the current PA/PAC construction contract, the
+ * owning arena index (pa_shard_init asserts base_ind_get(base) == ind).  This
+ * is a current construction invariant, not a permanent PAC guarantee.
+ */
+static unsigned
+pac_ind_get(const pac_t *pac) {
+	return base_ind_get(pac->base);
+}
+
+/*
+ * Notify the background thread that a decay epoch advanced: if it sleeps
+ * indefinitely, wake it now; otherwise wake it early when the projected backlog
+ * crosses the purge threshold before its next scheduled wakeup.  Non-blocking
+ * (trylocks info->mtx) to keep the application free path cheap.
+ */
+static void
+pac_maybe_wake_bg(tsdn_t *tsdn, pac_t *pac, decay_t *decay, size_t npages_new) {
+	background_thread_info_t *info =
+	    background_thread_info_get(pac_ind_get(pac));
+	if (malloc_mutex_trylock(tsdn, &info->mtx)) {
+		/*
+		 * The background thread may hold the mutex for a while; keep this
+		 * non-blocking and leave the work to a future epoch.
+		 */
+		return;
+	}
+	if (!background_thread_is_started(info)) {
+		goto label_done;
+	}
+	nstime_t remaining_sleep;
+	if (background_thread_indefinite_sleep(info)) {
+		background_thread_wakeup_early(info, NULL);
+	} else if (pac_decay_should_wake_early(tsdn, decay, info,
+	               &remaining_sleep, npages_new)) {
+		info->npages_to_purge_new = 0;
+		background_thread_wakeup_early(info, &remaining_sleep);
+	}
+label_done:
+	malloc_mutex_unlock(tsdn, &info->mtx);
+}
+
+/*
+ * A hook prepared for calling in pa: after a deferred work generated, wake the
+ * background thread only when it looks idle (lock-free acquire read of
+ * indefinite_sleep), then run the full early-wake decision under info->mtx.
+ */
+void
+pac_wake_bg_on_deferred(tsdn_t *tsdn, pac_t *pac) {
+	background_thread_info_t *info =
+	    background_thread_info_get(pac_ind_get(pac));
+	if (background_thread_indefinite_sleep(info)) {
+		pac_maybe_wake_bg(
+		    tsdn, pac, &pac->decay_dirty, /* npages_new */ 0);
+	}
+}
+
+static pac_purge_eagerness_t pac_decide_purge_eagerness(
+    bool is_background_thread);
+
+void
+pac_do_deferred_work(tsdn_t *tsdn, pac_t *pac, bool is_background_thread) {
+	/*
+	 * A concurrent background_thread enable/disable (mallctl) can race this
+	 * path: the enable state is read lock-free twice below (for the eagerness
+	 * decision, then the notify guard), so the two reads may disagree.  Worst
+	 * case is benign and self-healing:
+	 *   disabled->enabled: purged immediately, plus a possibly-redundant wake;
+	 *   enabled->disabled: deferred but not notified this pass -- the pages
+	 *     stay in the decay backlog and are reclaimed on the next decay tick or
+	 *     by the bg thread before it stops.
+	 * It stays safe regardless: info is allocated once and never freed; the
+	 * wake is gated by info->mtx + background_thread_is_started(); the bg-thread
+	 * locks here are trylocks; and the purge runs under decay->mtx, which the
+	 * toggle never touches.
+	 */
+	pac_purge_eagerness_t eagerness =
+	    pac_decide_purge_eagerness(is_background_thread);
+	pac_deferred_work_result_t result;
+	pac_decay_deferred(tsdn, pac, eagerness, &result);
+
+	if (have_background_thread && background_thread_enabled()
+	    && !is_background_thread) {
+		if (result.dirty_epoch_advanced) {
+			pac_maybe_wake_bg(tsdn, pac, &pac->decay_dirty,
+			    result.dirty_npages_new);
+		}
+		if (result.muzzy_epoch_advanced) {
+			pac_maybe_wake_bg(tsdn, pac, &pac->decay_muzzy,
+			    result.muzzy_npages_new);
+		}
+	}
+}
+
+void
+pac_decay_all_now(tsdn_t *tsdn, pac_t *pac, extent_state_t state) {
+	decay_t           *decay;
+	pac_decay_stats_t *decay_stats;
+	ecache_t          *ecache;
+	pac_decay_data_get(pac, state, &decay, &decay_stats, &ecache);
+
+	malloc_mutex_lock(tsdn, &decay->mtx);
+	pac_decay_all(
+	    tsdn, pac, decay, decay_stats, ecache, /* fully_decay */ true);
+	malloc_mutex_unlock(tsdn, &decay->mtx);
+}
+
+/*
+ * Decide the unforced decay-purge eagerness.  On the background thread, force
+ * the purge; on an application thread, defer to the background thread when it is
+ * enabled, otherwise purge on the current thread on epoch advance.
+ */
+static pac_purge_eagerness_t
+pac_decide_purge_eagerness(bool is_background_thread) {
+	if (is_background_thread) {
+		return PAC_PURGE_ALWAYS;
+	} else if (background_thread_enabled()) {
+		return PAC_PURGE_NEVER;
+	} else {
+		return PAC_PURGE_ON_EPOCH_ADVANCE;
+	}
+}
+
 bool
 pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
-    ssize_t decay_ms, pac_purge_eagerness_t eagerness) {
+    ssize_t decay_ms) {
 	decay_t           *decay;
 	pac_decay_stats_t *decay_stats;
 	ecache_t          *ecache;
@@ -720,6 +1022,13 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
 		return true;
 	}
 
+	bool update_pac_sec = (state == extent_state_dirty)
+	    && sec_is_used(&pac->sec);
+	if (update_pac_sec && decay_ms == 0) {
+		atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED);
+		pac_sec_flush(tsdn, pac);
+	}
+
 	malloc_mutex_lock(tsdn, &decay->mtx);
 	/*
 	 * Restart decay backlog from scratch, which may cause many dirty pages
@@ -732,9 +1041,20 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
 	nstime_t cur_time;
 	nstime_init_update(&cur_time);
 	decay_reinit(decay, &cur_time, decay_ms);
+	/*
+	 * decay_ms is only ever set from a non-background thread (mallctl or
+	 * arena init), so decide the eagerness here rather than threading it in.
+	 */
+	pac_purge_eagerness_t eagerness =
+	    pac_decide_purge_eagerness(/* is_background_thread */ false);
 	pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness);
 	malloc_mutex_unlock(tsdn, &decay->mtx);
 
+	if (update_pac_sec && decay_ms != 0) {
+		atomic_store_zu(&pac->sec_max_alloc,
+		    pac->sec.opts.max_alloc, ATOMIC_RELAXED);
+	}
+
 	return false;
 }
 
@@ -747,16 +1067,6 @@ pac_decay_ms_get(pac_t *pac, extent_state_t state) {
 	return decay_ms_read(decay);
 }
 
-void
-pac_reset(tsdn_t *tsdn, pac_t *pac) {
-	/*
-	 * No-op for now; purging is still done at the arena-level.  It should
-	 * get moved in here, though.
-	 */
-	(void)tsdn;
-	(void)pac;
-}
-
 void
 pac_destroy(tsdn_t *tsdn, pac_t *pac) {
 	assert(ecache_npages_get(&pac->ecache_dirty) == 0);
@@ -819,3 +1129,16 @@ pac_destroy(tsdn_t *tsdn, pac_t *pac) {
 		extent_destroy_wrapper(tsdn, pac, ehooks, edata);
 	}
 }
+
+void
+pac_sec_flush(tsdn_t *tsdn, pac_t *pac) {
+	ehooks_t *ehooks = pac_ehooks_get(pac);
+	edata_list_active_t to_flush;
+	edata_list_active_init(&to_flush);
+	sec_flush(tsdn, &pac->sec, &to_flush);
+	edata_t *edata;
+	while ((edata = edata_list_active_first(&to_flush)) != NULL) {
+		edata_list_active_remove(&to_flush, edata);
+		pac_ecache_dalloc(tsdn, pac, ehooks, edata);
+	}
+}

src/pages.c

@@ -783,7 +783,7 @@ os_overcommits_proc(void) {
 #endif
 
 static bool
-pages_should_skip_set_thp_state() {
+pages_should_skip_set_thp_state(void) {
 	if (opt_thp == thp_mode_do_nothing
 	    || (opt_thp == thp_mode_always
 	        && init_system_thp_mode == system_thp_mode_always)
@@ -826,8 +826,16 @@ init_thp_state(void) {
 	static const char sys_state_never[] = "always madvise [never]\n";
 	char              buf[sizeof(sys_state_madvise)];
 
+#	if defined(O_CLOEXEC)
+	int fd = malloc_open(
+	    "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY | O_CLOEXEC);
+#	else
 	int fd = malloc_open(
 	    "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+	if (fd != -1) {
+		fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
+	}
+#	endif
 	if (fd == -1) {
 		goto label_error;
 	}

src/prof.c

@@ -249,7 +249,7 @@ prof_tctx_create(tsd_t *tsd) {
  * (e.g.
  * -mno-sse) in order for the workaround to be complete.
  */
-uint64_t
+JET_EXTERN uint64_t
 prof_sample_new_event_wait(tsd_t *tsd) {
 #ifdef JEMALLOC_PROF
 	if (lg_prof_sample == 0) {
@@ -295,7 +295,7 @@ prof_sample_new_event_wait(tsd_t *tsd) {
 #endif
 }
 
-void
+static void
 prof_sample_event_handler(tsd_t *tsd) {
 	cassert(config_prof);
 	if (prof_interval == 0 || !prof_active_get_unlocked()) {

src/prof_data.c

@@ -404,6 +404,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) {
 	return ret.p;
 }
 
+#ifdef JEMALLOC_JET
 /* Used in unit tests. */
 static prof_tdata_t *
 prof_tdata_count_iter(
@@ -449,6 +450,7 @@ prof_bt_count(void) {
 
 	return bt_count;
 }
+#endif
 
 static void
 prof_thread_name_write_tdata(prof_tdata_t *tdata, const char *thread_name) {
@@ -1105,6 +1107,7 @@ prof_dump_impl(tsd_t *tsd, write_cb_t *prof_dump_write, void *cbopaque,
 	}
 }
 
+#ifdef JEMALLOC_JET
 /* Used in unit tests. */
 void
 prof_cnt_all(prof_cnt_t *cnt_all) {
@@ -1119,6 +1122,7 @@ prof_cnt_all(prof_cnt_t *cnt_all) {
 		prof_gctx_finish(tsd, &gctxs);
 	}
 }
+#endif
 
 void
 prof_bt_hash(const void *key, size_t r_hash[2]) {

src/prof_log.c

@@ -309,6 +309,7 @@ prof_thr_node_keycomp(const void *k1, const void *k2) {
 	return thr_node1->thr_uid == thr_node2->thr_uid;
 }
 
+#ifdef JEMALLOC_JET
 /* Used in unit tests. */
 size_t
 prof_log_bt_count(void) {
@@ -413,6 +414,7 @@ prof_log_dummy_set(bool new_value) {
 	cassert(config_prof);
 	prof_log_dummy = new_value;
 }
+#endif
 
 /* Used as an atexit function to stop logging on exit. */
 static void

src/prof_recent.c

@@ -108,11 +108,13 @@ prof_recent_alloc_edata_get_no_lock(const prof_recent_t *n) {
 	return (edata_t *)atomic_load_p(&n->alloc_edata, ATOMIC_ACQUIRE);
 }
 
+#ifdef JEMALLOC_JET
 edata_t *
 prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *n) {
 	cassert(config_prof);
 	return prof_recent_alloc_edata_get_no_lock(n);
 }
+#endif
 
 static inline edata_t *
 prof_recent_alloc_edata_get(tsd_t *tsd, const prof_recent_t *n) {
@@ -138,11 +140,13 @@ edata_prof_recent_alloc_get_no_lock(const edata_t *edata) {
 	return edata_prof_recent_alloc_get_dont_call_directly(edata);
 }
 
+#ifdef JEMALLOC_JET
 prof_recent_t *
 edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata) {
 	cassert(config_prof);
 	return edata_prof_recent_alloc_get_no_lock(edata);
 }
+#endif
 
 static inline prof_recent_t *
 edata_prof_recent_alloc_get(tsd_t *tsd, const edata_t *edata) {

src/prof_stack_range.c

@@ -1,174 +0,0 @@
-#include "jemalloc/internal/jemalloc_preamble.h"
-
-#include "jemalloc/internal/malloc_io.h"
-#include "jemalloc/internal/prof_sys.h"
-
-#if defined(__linux__) && defined(JEMALLOC_HAVE_GETTID)
-
-#	include <errno.h>
-#	include <fcntl.h>
-#	include <stdio.h>
-#	include <stdlib.h> // strtoul
-#	include <string.h>
-#	include <unistd.h>
-
-/*
- * Converts a string representing a hexadecimal number to an unsigned long long
- * integer. Functionally equivalent to strtoull() (for base 16) but faster for
- * that case.
- *
- * @param nptr Pointer to the string to be converted.
- * @param endptr Pointer to a pointer to character, which will be set to the
- * character in `nptr` where parsing stopped. Can be NULL.
- * @return The converted unsigned long long integer value.
- */
-static inline unsigned long long int
-strtoull_hex(const char *nptr, char **endptr) {
-	unsigned long long int val = 0;
-	int                    ii = 0;
-	for (; ii < 16; ++ii) {
-		char c = nptr[ii];
-		if (c >= '0' && c <= '9') {
-			val = (val << 4) + (c - '0');
-		} else if (c >= 'a' && c <= 'f') {
-			val = (val << 4) + (c - 'a' + 10);
-		} else {
-			break;
-		}
-	}
-	if (endptr) {
-		*endptr = (char *)(nptr + ii);
-	}
-	return val;
-}
-
-static int
-prof_mapping_containing_addr(uintptr_t addr, const char *maps_path,
-    uintptr_t *mm_start, uintptr_t *mm_end) {
-	int ret = ENOENT; /* not found */
-	*mm_start = *mm_end = 0;
-
-	/*
-     * Each line of /proc/<pid>/maps is:
-     * <start>-<end> <perms> <offset> <dev> <inode> <pathname>
-     *
-     * The fields we care about are always within the first 34 characters so
-     * as long as `buf` contains the start of a mapping line it can always be
-     * parsed.
-     */
-	static const int kMappingFieldsWidth = 34;
-
-	int     fd = -1;
-	char    buf[4096];
-	ssize_t remaining = 0; /* actual number of bytes read to buf */
-	char   *line = NULL;
-
-	while (1) {
-		if (fd < 0) {
-			/* case 0: initial open of maps file */
-			fd = malloc_open(maps_path, O_RDONLY);
-			if (fd < 0) {
-				return errno;
-			}
-
-			remaining = malloc_read_fd(fd, buf, sizeof(buf));
-			if (remaining < 0) {
-				ret = errno;
-				break;
-			} else if (remaining == 0) {
-				break;
-			}
-			line = buf;
-		} else if (line == NULL) {
-			/* case 1: no newline found in buf */
-			remaining = malloc_read_fd(fd, buf, sizeof(buf));
-			if (remaining < 0) {
-				ret = errno;
-				break;
-			} else if (remaining == 0) {
-				break;
-			}
-			line = memchr(buf, '\n', remaining);
-			if (line != NULL) {
-				line++; /* advance to character after newline */
-				remaining -= (line - buf);
-			}
-		} else if (line != NULL && remaining < kMappingFieldsWidth) {
-			/*
-             * case 2: found newline but insufficient characters remaining in
-             * buf
-             */
-			memcpy(buf, line,
-			    remaining); /* copy remaining characters to start of buf */
-			line = buf;
-
-			ssize_t count = malloc_read_fd(
-			    fd, buf + remaining, sizeof(buf) - remaining);
-			if (count < 0) {
-				ret = errno;
-				break;
-			} else if (count == 0) {
-				break;
-			}
-
-			remaining +=
-			    count; /* actual number of bytes read to buf */
-		} else {
-			/* case 3: found newline and sufficient characters to parse */
-
-			/* parse <start>-<end> */
-			char     *tmp = line;
-			uintptr_t start_addr = (uintptr_t)strtoull_hex(
-			    tmp, &tmp);
-			if (addr >= start_addr) {
-				tmp++; /* advance to character after '-' */
-				uintptr_t end_addr = (uintptr_t)strtoull_hex(
-				    tmp, NULL);
-				if (addr < end_addr) {
-					*mm_start = start_addr;
-					*mm_end = end_addr;
-					ret = 0;
-					break;
-				}
-			}
-
-			/* Advance to character after next newline in the current buf. */
-			char *prev_line = line;
-			line = memchr(line, '\n', remaining);
-			if (line != NULL) {
-				line++; /* advance to character after newline */
-				remaining -= (line - prev_line);
-			}
-		}
-	}
-
-	malloc_close(fd);
-	return ret;
-}
-
-int
-prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high) {
-	/*
-     * NOTE: Prior to kernel 4.5 an entry for every thread stack was included in
-     * /proc/<pid>/maps as [STACK:<tid>]. Starting with kernel 4.5 only the main
-     * thread stack remains as the [stack] mapping. For other thread stacks the
-     * mapping is still visible in /proc/<pid>/task/<tid>/maps (though not
-     * labeled as [STACK:tid]).
-     * https://lists.ubuntu.com/archives/kernel-team/2016-March/074681.html
-    */
-	char maps_path[64]; // "/proc/<pid>/task/<tid>/maps"
-	malloc_snprintf(maps_path, sizeof(maps_path), "/proc/%d/task/%d/maps",
-	    getpid(), gettid());
-	return prof_mapping_containing_addr(fp, maps_path, low, high);
-}
-
-#else
-
-int
-prof_thread_stack_range(
-    UNUSED uintptr_t addr, uintptr_t *stack_start, uintptr_t *stack_end) {
-	*stack_start = *stack_end = 0;
-	return ENOENT;
-}
-
-#endif // __linux__

src/prof_sys.c

@@ -28,11 +28,6 @@
 		JEMALLOC_TEST_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
 #endif
 
-#ifdef JEMALLOC_PROF_FRAME_POINTER
-// execinfo backtrace() as fallback unwinder
-#	include <execinfo.h>
-#endif
-
 /******************************************************************************/
 
 malloc_mutex_t prof_dump_filename_mtx;
@@ -109,103 +104,6 @@ prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
 
 	_Unwind_Backtrace(prof_unwind_callback, &data);
 }
-#elif (defined(JEMALLOC_PROF_FRAME_POINTER))
-JEMALLOC_DIAGNOSTIC_PUSH
-JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS
-
-struct stack_range {
-	uintptr_t start;
-	uintptr_t end;
-};
-
-struct thread_unwind_info {
-	struct stack_range stack_range;
-	bool               fallback;
-};
-static __thread struct thread_unwind_info unwind_info = {
-    .stack_range =
-        {
-            .start = 0,
-            .end = 0,
-        },
-    .fallback = false,
-}; /* thread local */
-
-static void
-prof_backtrace_impl(void **vec, unsigned *len, unsigned max_len) {
-	/* fp: 		current stack frame pointer
-	 *
-	 * stack_range:	readable stack memory range for the current thread.
-	 *		Used to validate frame addresses during stack unwinding.
-	 *		For most threads there is a single valid stack range
-	 *		that is fixed at thread creation time.  This may not be
-	 *		the case when folly fibers or boost contexts are used.
-	 *		In those cases fall back to using execinfo backtrace()
-	 *		(DWARF unwind).
-	 */
-
-	/* always safe to get the current stack frame address */
-	uintptr_t fp = (uintptr_t)__builtin_frame_address(0);
-
-	/* new thread - get the stack range */
-	if (!unwind_info.fallback
-	    && unwind_info.stack_range.start == unwind_info.stack_range.end) {
-		if (prof_thread_stack_range(fp, &unwind_info.stack_range.start,
-		        &unwind_info.stack_range.end)
-		    != 0) {
-			unwind_info.fallback = true;
-		} else {
-			assert(fp >= unwind_info.stack_range.start
-			    && fp < unwind_info.stack_range.end);
-		}
-	}
-
-	if (unwind_info.fallback) {
-		goto label_fallback;
-	}
-
-	unsigned ii = 0;
-	while (ii < max_len && fp != 0) {
-		if (fp < unwind_info.stack_range.start
-		    || fp >= unwind_info.stack_range.end) {
-			/*
-			 * Determining the stack range from procfs can be
-			 * relatively expensive especially for programs with
-			 * many threads / shared libraries.  If the stack
-			 * range has changed, it is likely to change again
-			 * in the future (fibers or some other stack
-			 * manipulation).  So fall back to backtrace for this
-			 * thread.
-			 */
-			unwind_info.fallback = true;
-			goto label_fallback;
-		}
-		void *ip = ((void **)fp)[1];
-		if (ip == 0) {
-			break;
-		}
-		vec[ii++] = ip;
-		fp = ((uintptr_t *)fp)[0];
-	}
-	*len = ii;
-	return;
-
-label_fallback:
-	/*
-	 * Using the backtrace from execinfo.h here.  Note that it may get
-	 * redirected to libunwind when a libunwind not built with build-time
-	 * flag --disable-weak-backtrace is linked.
-	 */
-	assert(unwind_info.fallback);
-	int nframes = backtrace(vec, max_len);
-	if (nframes > 0) {
-		*len = nframes;
-	} else {
-		*len = 0;
-	}
-}
-
-JEMALLOC_DIAGNOSTIC_POP
 #elif (defined(JEMALLOC_PROF_GCC))
 JEMALLOC_DIAGNOSTIC_PUSH
 JEMALLOC_DIAGNOSTIC_IGNORE_FRAME_ADDRESS

src/sc.c

@@ -13,7 +13,7 @@
  * at least the damage is compartmentalized to this file.
  */
 
-size_t
+JET_EXTERN size_t
 reg_size_compute(int lg_base, int lg_delta, int ndelta) {
 	return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta);
 }

src/sec.c

@@ -6,8 +6,13 @@
 
 static bool
 sec_bin_init(sec_bin_t *bin) {
-	bin->bytes_cur = 0;
-	sec_bin_stats_init(&bin->stats);
+	atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
+	atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
+	atomic_store_zu(&bin->ndalloc_flush, 0, ATOMIC_RELAXED);
+	atomic_store_zu(&bin->nmisses, 0, ATOMIC_RELAXED);
+	atomic_store_zu(&bin->nhits, 0, ATOMIC_RELAXED);
+	atomic_store_zu(&bin->ndalloc_noflush, 0, ATOMIC_RELAXED);
+	atomic_store_zu(&bin->noverfills, 0, ATOMIC_RELAXED);
 	edata_list_active_init(&bin->freelist);
 	bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN,
 	    malloc_mutex_rank_exclusive);
@@ -20,8 +25,15 @@ sec_bin_init(sec_bin_t *bin) {
 
 bool
 sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
+	/*
+	 * Invariant: max_alloc == 0 whenever nshards == 0.  This lets
+	 * sec_size_supported() collapse to a single comparison.
+	 */
 	sec->opts = *opts;
+	sec->bins = NULL;
+	sec->npsizes = 0;
 	if (opts->nshards == 0) {
+		sec->opts.max_alloc = 0;
 		return false;
 	}
 	assert(opts->max_alloc >= PAGE);
@@ -40,11 +52,15 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
 	size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
 	void  *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
 	if (dynalloc == NULL) {
+		sec->opts.nshards = 0;
+		sec->opts.max_alloc = 0;
 		return true;
 	}
 	sec->bins = (sec_bin_t *)dynalloc;
 	for (pszind_t j = 0; j < ntotal_bins; j++) {
 		if (sec_bin_init(&sec->bins[j])) {
+			sec->opts.nshards = 0;
+			sec->opts.max_alloc = 0;
 			return true;
 		}
 	}
@@ -53,18 +69,16 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
 	return false;
 }
 
-static uint8_t
-sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
+uint8_t
+sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp) {
 	/*
 	 * Eventually, we should implement affinity, tracking source shard using
 	 * the edata_t's newly freed up fields.  For now, just randomly
 	 * distribute across all shards.
+	 *
+	 * Callers must ensure sec->opts.nshards > 1.
 	 */
-	if (tsdn_null(tsdn)) {
-		return 0;
-	}
-	tsd_t   *tsd = tsdn_tsd(tsdn);
-	uint8_t *idxp = tsd_sec_shardp_get(tsd);
+	assert(sec->opts.nshards > 1);
 	if (*idxp == (uint8_t)-1) {
 		/*
 		 * First use; initialize using the trick from Daniel Lemire's
@@ -128,19 +142,29 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
 		assert(!edata_list_active_empty(&bin->freelist));
 		edata_list_active_remove(&bin->freelist, edata);
 		size_t sz = edata_size_get(edata);
-		assert(sz <= bin->bytes_cur && sz > 0);
-		bin->bytes_cur -= sz;
-		bin->stats.nhits++;
+		size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+		assert(sz <= bytes_cur && sz > 0);
+		bytes_cur -= sz;
+		if (edata_pinned_get(edata)) {
+			size_t bytes_pinned_cur = atomic_load_zu(
+			    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
+			assert(sz <= bytes_pinned_cur);
+			bytes_pinned_cur -= sz;
+			atomic_store_zu(&bin->bytes_pinned_cur,
+			    bytes_pinned_cur, ATOMIC_RELAXED);
+		}
+		atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
+		atomic_load_add_store_zu(&bin->nhits, 1);
 	}
 	return edata;
 }
 
 static edata_t *
 sec_multishard_trylock_alloc(
-    tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
+    tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind, uint8_t shard) {
 	assert(sec->opts.nshards > 0);
 
-	uint8_t    cur_shard = sec_shard_pick(tsdn, sec);
+	uint8_t    cur_shard = shard;
 	sec_bin_t *bin;
 	for (size_t i = 0; i < sec->opts.nshards; ++i) {
 		bin = sec_bin_pick(sec, cur_shard, pszind);
@@ -164,13 +188,13 @@ sec_multishard_trylock_alloc(
 	 * declaring a miss.  That could recover more remote-shard hits under
 	 * contention, but it also changes the allocation latency policy.
 	 */
-	assert(cur_shard == sec_shard_pick(tsdn, sec));
+	assert(cur_shard == shard);
 	bin = sec_bin_pick(sec, cur_shard, pszind);
 	malloc_mutex_lock(tsdn, &bin->mtx);
 	edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
 	if (edata == NULL) {
 		/* Only now we know it is a miss. */
-		bin->stats.nmisses++;
+		atomic_load_add_store_zu(&bin->nmisses, 1);
 	}
 	malloc_mutex_unlock(tsdn, &bin->mtx);
 	JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1);
@@ -178,10 +202,8 @@ sec_multishard_trylock_alloc(
 }
 
 edata_t *
-sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
-	if (!sec_size_supported(sec, size)) {
-		return NULL;
-	}
+sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard) {
+	assert(sec_size_supported(sec, size));
 	assert((size & PAGE_MASK) == 0);
 	pszind_t pszind = sz_psz2ind(size);
 	assert(pszind < sec->npsizes);
@@ -195,14 +217,14 @@ sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
 		malloc_mutex_lock(tsdn, &bin->mtx);
 		edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
 		if (edata == NULL) {
-			bin->stats.nmisses++;
+			atomic_load_add_store_zu(&bin->nmisses, 1);
 		}
 		malloc_mutex_unlock(tsdn, &bin->mtx);
 		JE_USDT(sec_alloc, 5, sec, bin, edata, size,
 		    /* frequent_reuse */ 1);
 		return edata;
 	}
-	return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
+	return sec_multishard_trylock_alloc(tsdn, sec, size, pszind, shard);
 }
 
 static void
@@ -210,40 +232,56 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
     edata_list_active_t *dalloc_list) {
 	malloc_mutex_assert_owner(tsdn, &bin->mtx);
 
-	bin->bytes_cur += size;
+	size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+	size_t bytes_pinned_cur = atomic_load_zu(
+	    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
+	bytes_cur += size;
 	edata_t *edata = edata_list_active_first(dalloc_list);
 	assert(edata != NULL);
 	edata_list_active_remove(dalloc_list, edata);
 	JE_USDT(sec_dalloc, 3, sec, bin, edata);
 	edata_list_active_prepend(&bin->freelist, edata);
+	if (edata_pinned_get(edata)) {
+		bytes_pinned_cur += size;
+	}
 	/* Single extent can be returned to SEC */
 	assert(edata_list_active_empty(dalloc_list));
 
-	if (bin->bytes_cur <= sec->opts.max_bytes) {
-		bin->stats.ndalloc_noflush++;
+	if (bytes_cur <= sec->opts.max_bytes) {
+		atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+		    ATOMIC_RELAXED);
+		atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
+		atomic_load_add_store_zu(&bin->ndalloc_noflush, 1);
 		return;
 	}
-	bin->stats.ndalloc_flush++;
+	atomic_load_add_store_zu(&bin->ndalloc_flush, 1);
 	/* we want to flush 1/4 of max_bytes */
 	size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2);
-	while (bin->bytes_cur > bytes_target
+	while (bytes_cur > bytes_target
 	    && !edata_list_active_empty(&bin->freelist)) {
 		edata_t *cur = edata_list_active_last(&bin->freelist);
 		size_t   sz = edata_size_get(cur);
-		assert(sz <= bin->bytes_cur && sz > 0);
-		bin->bytes_cur -= sz;
+		assert(sz <= bytes_cur && sz > 0);
+		bytes_cur -= sz;
+		if (edata_pinned_get(cur)) {
+			assert(sz <= bytes_pinned_cur);
+			bytes_pinned_cur -= sz;
+		}
 		edata_list_active_remove(&bin->freelist, cur);
 		edata_list_active_append(dalloc_list, cur);
 	}
+	atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+	    ATOMIC_RELAXED);
+	atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
 }
 
 static void
 sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
-    pszind_t pszind, edata_list_active_t *dalloc_list) {
+    pszind_t pszind, edata_list_active_t *dalloc_list, uint8_t shard) {
 	assert(sec->opts.nshards > 0);
 
 	/* Try to dalloc in this threads bin first */
-	uint8_t cur_shard = sec_shard_pick(tsdn, sec);
+	uint8_t cur_shard = shard;
 	for (size_t i = 0; i < sec->opts.nshards; ++i) {
 		sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
 		if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
@@ -258,7 +296,7 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
 		}
 	}
 	/* No bin had alloc or had the extent */
-	assert(cur_shard == sec_shard_pick(tsdn, sec));
+	assert(cur_shard == shard);
 	sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
 	malloc_mutex_lock(tsdn, &bin->mtx);
 	sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
@@ -266,20 +304,16 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
 }
 
 void
-sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
-	if (!sec_is_used(sec)) {
-		return;
-	}
+sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
+    uint8_t shard) {
 	edata_t *edata = edata_list_active_first(dalloc_list);
 	size_t   size = edata_size_get(edata);
-	if (size > sec->opts.max_alloc) {
-		return;
-	}
+	assert(sec_size_supported(sec, size));
 	pszind_t pszind = sz_psz2ind(size);
 	assert(pszind < sec->npsizes);
 
 	/*
-         * If there's only one shard, skip the trylock optimization and
+	 * If there's only one shard, skip the trylock optimization and
 	 * go straight to the blocking lock.
 	 */
 	if (sec->opts.nshards == 1) {
@@ -289,34 +323,58 @@ sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
 		malloc_mutex_unlock(tsdn, &bin->mtx);
 		return;
 	}
-	sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
+	sec_multishard_trylock_dalloc(
+	    tsdn, sec, size, pszind, dalloc_list, shard);
+}
+
+static void
+sec_list_pinned_bytes_get(
+    edata_list_active_t *list, size_t size, size_t *pinned_bytes) {
+	*pinned_bytes = 0;
+	for (edata_t *edata = edata_list_active_first(list); edata != NULL;
+	    edata = edata_list_active_next(list, edata)) {
+		assert(edata_size_get(edata) == size);
+		if (edata_pinned_get(edata)) {
+			*pinned_bytes += size;
+		}
+	}
 }
 
 void
 sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
-    size_t nallocs) {
+    size_t nallocs, uint8_t shard) {
 	assert((size & PAGE_MASK) == 0);
-	assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
+	assert(sec_size_supported(sec, size));
 	assert(nallocs > 0);
 
 	pszind_t pszind = sz_psz2ind(size);
 	assert(pszind < sec->npsizes);
 
-	sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
+	sec_bin_t *bin = sec_bin_pick(sec, shard, pszind);
 	malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
 	malloc_mutex_lock(tsdn, &bin->mtx);
+	size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+	size_t bytes_pinned_cur = atomic_load_zu(
+	    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
 	size_t new_cached_bytes = nallocs * size;
-	if (bin->bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
+	if (bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
 		assert(!edata_list_active_empty(result));
+		size_t new_cached_pinned_bytes;
+		sec_list_pinned_bytes_get(
+		    result, size, &new_cached_pinned_bytes);
+		bytes_pinned_cur += new_cached_pinned_bytes;
 		edata_list_active_concat(&bin->freelist, result);
-		bin->bytes_cur += new_cached_bytes;
+		atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+		    ATOMIC_RELAXED);
+		atomic_store_zu(&bin->bytes_cur, bytes_cur + new_cached_bytes,
+		    ATOMIC_RELAXED);
 	} else {
 		/*
 		 * Unlikely case of many threads filling at the same time and
 		 * going above max.
 		 */
-		bin->stats.noverfills++;
-		while (bin->bytes_cur + size <= sec->opts.max_bytes) {
+		atomic_load_add_store_zu(&bin->noverfills, 1);
+		while (bytes_cur + size <= sec->opts.max_bytes) {
 			edata_t *edata = edata_list_active_first(result);
 			if (edata == NULL) {
 				break;
@@ -324,8 +382,14 @@ sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
 			edata_list_active_remove(result, edata);
 			assert(size == edata_size_get(edata));
 			edata_list_active_append(&bin->freelist, edata);
-			bin->bytes_cur += size;
+			bytes_cur += size;
+			if (edata_pinned_get(edata)) {
+				bytes_pinned_cur += size;
+			}
 		}
+		atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+		    ATOMIC_RELAXED);
+		atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
 	}
 	malloc_mutex_unlock(tsdn, &bin->mtx);
 }
@@ -339,27 +403,73 @@ sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
 	for (pszind_t i = 0; i < ntotal_bins; i++) {
 		sec_bin_t *bin = &sec->bins[i];
 		malloc_mutex_lock(tsdn, &bin->mtx);
-		bin->bytes_cur = 0;
+		atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
+		atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
 		edata_list_active_concat(to_flush, &bin->freelist);
 		malloc_mutex_unlock(tsdn, &bin->mtx);
 	}
 }
 
+static void
+sec_bin_bytes_get(const sec_bin_t *bin, size_t *bytes, size_t *bytes_pinned) {
+	*bytes = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+	*bytes_pinned = atomic_load_zu(
+	    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
+	*bytes_pinned = min_zu(*bytes_pinned, *bytes);
+}
+
+static void
+sec_stats_merge_bin(
+    const sec_bin_t *bin, pszind_t pszind, sec_pszind_stats_t *stats) {
+	size_t bytes;
+	size_t bytes_pinned;
+	sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
+
+	stats->bytes += bytes;
+	stats->bytes_pinned += bytes_pinned;
+	size_t size = sz_pind2sz(pszind);
+	stats->nextents += bytes / size;
+	stats->nextents_pinned += bytes_pinned / size;
+}
+
 void
-sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
+sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
 	if (!sec_is_used(sec)) {
 		return;
 	}
-	size_t sum = 0;
 	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
 	for (pszind_t i = 0; i < ntotal_bins; i++) {
 		sec_bin_t *bin = &sec->bins[i];
-		malloc_mutex_lock(tsdn, &bin->mtx);
-		sum += bin->bytes_cur;
-		sec_bin_stats_accum(&stats->total, &bin->stats);
-		malloc_mutex_unlock(tsdn, &bin->mtx);
+		size_t bytes;
+		size_t bytes_pinned;
+		sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
+		stats->bytes += bytes;
+		stats->bytes_pinned += bytes_pinned;
+		stats->total.nmisses +=
+		    atomic_load_zu(&bin->nmisses, ATOMIC_RELAXED);
+		stats->total.nhits +=
+		    atomic_load_zu(&bin->nhits, ATOMIC_RELAXED);
+		stats->total.ndalloc_flush +=
+		    atomic_load_zu(&bin->ndalloc_flush, ATOMIC_RELAXED);
+		stats->total.ndalloc_noflush +=
+		    atomic_load_zu(&bin->ndalloc_noflush, ATOMIC_RELAXED);
+		stats->total.noverfills +=
+		    atomic_load_zu(&bin->noverfills, ATOMIC_RELAXED);
+	}
+}
+
+void
+sec_stats_merge_pszind(tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
+    sec_pszind_stats_t *stats) {
+	if (!sec_is_used(sec) || pszind >= sec->npsizes) {
+		return;
+	}
+
+	for (size_t shard = 0; shard < sec->opts.nshards; shard++) {
+		size_t ind = shard * sec->npsizes + pszind;
+		assert(ind < sec->npsizes * sec->opts.nshards);
+		sec_stats_merge_bin(&sec->bins[ind], pszind, stats);
 	}
-	stats->bytes += sum;
 }
 
 void

src/tcache.c

@@ -112,12 +112,12 @@ tcache_salloc(tsdn_t *tsdn, const void *ptr) {
 	return arena_salloc(tsdn, ptr);
 }
 
-uint64_t
+static uint64_t
 tcache_gc_new_event_wait(tsd_t *tsd) {
 	return opt_tcache_gc_incr_bytes;
 }
 
-uint64_t
+static uint64_t
 tcache_gc_postponed_event_wait(tsd_t *tsd) {
 	return TE_MIN_START_WAIT;
 }
@@ -148,7 +148,7 @@ tcache_bin_fill_ctl_get(tcache_slow_t *tcache_slow, szind_t szind) {
  * The base is adjusted during GC based on the traffic within a period of time,
  * while the offset is updated in real time to handle the immediate traffic.
  */
-static inline uint8_t
+JET_EXTERN_INLINE uint8_t
 tcache_nfill_small_lg_div_get(tcache_slow_t *tcache_slow, szind_t szind) {
 	cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
 	return (ctl->base - (opt_experimental_tcache_gc ? ctl->offset : 0));
@@ -159,7 +159,7 @@ tcache_nfill_small_lg_div_get(tcache_slow_t *tcache_slow, szind_t szind) {
  * offset is increased so that (base - offset) is decreased,
  * which in return increases the number of items to be filled.
  */
-static inline void
+JET_EXTERN_INLINE void
 tcache_nfill_small_burst_prepare(tcache_slow_t *tcache_slow, szind_t szind) {
 	cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
 	if (ctl->offset + 1 < ctl->base) {
@@ -167,7 +167,7 @@ tcache_nfill_small_burst_prepare(tcache_slow_t *tcache_slow, szind_t szind) {
 	}
 }
 
-static inline void
+JET_EXTERN_INLINE void
 tcache_nfill_small_burst_reset(tcache_slow_t *tcache_slow, szind_t szind) {
 	cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
 	ctl->offset = 0;
@@ -180,7 +180,7 @@ tcache_nfill_small_burst_reset(tcache_slow_t *tcache_slow, szind_t szind) {
  * limit != 0: limit is set to ncached_max, indicating that the fill
  * count should be decreased, i.e. lg_div(base) should be increased.
  */
-static inline void
+JET_EXTERN_INLINE void
 tcache_nfill_small_gc_update(
     tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit) {
 	cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
@@ -201,7 +201,7 @@ tcache_nfill_small_gc_update(
 	ctl->offset = 0;
 }
 
-static uint8_t
+JET_EXTERN uint8_t
 tcache_gc_item_delay_compute(szind_t szind) {
 	assert(szind < SC_NBINS);
 	size_t sz = sz_index2size(szind);
@@ -220,7 +220,7 @@ tcache_gc_is_addr_remote(void *addr, uintptr_t min, uintptr_t max) {
 	return ((uintptr_t)addr < min || (uintptr_t)addr >= max);
 }
 
-static inline cache_bin_sz_t
+JET_EXTERN_INLINE cache_bin_sz_t
 tcache_gc_small_nremote_get(cache_bin_t *cache_bin, void *addr,
     uintptr_t *addr_min, uintptr_t *addr_max, szind_t szind, size_t nflush) {
 	assert(addr != NULL && addr_min != NULL && addr_max != NULL);
@@ -277,7 +277,7 @@ tcache_gc_small_nremote_get(cache_bin_t *cache_bin, void *addr,
 }
 
 /* Shuffle the ptrs in the bin to put the remote pointers at the bottom. */
-static inline void
+JET_EXTERN_INLINE void
 tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
     uintptr_t addr_min, uintptr_t addr_max) {
 	void         **swap = NULL;
@@ -350,55 +350,6 @@ tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
 	}
 }
 
-#ifdef JEMALLOC_JET
-/*
- * The GC helpers above are static inline so they cannot be linked from a
- * separate translation unit.  In JET builds we expose thin wrappers with a
- * `_test` suffix so test/unit/tcache_gc.c can exercise them directly.  These
- * symbols are absent from the production library.
- */
-#define JET_WRAP_RET(ret, fn, params, args)	\
-	ret fn##_test params {			\
-		return fn args;			\
-	}
-#define JET_WRAP_VOID(fn, params, args)		\
-	void fn##_test params {			\
-		fn args;			\
-	}
-
-JET_WRAP_RET(cache_bin_sz_t, tcache_gc_small_nremote_get,
-    (cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min,
-     uintptr_t *addr_max, szind_t szind, size_t nflush),
-    (cache_bin, addr, addr_min, addr_max, szind, nflush))
-
-JET_WRAP_VOID(tcache_gc_small_bin_shuffle,
-    (cache_bin_t *cache_bin, cache_bin_sz_t nremote,
-     uintptr_t addr_min, uintptr_t addr_max),
-    (cache_bin, nremote, addr_min, addr_max))
-
-JET_WRAP_RET(uint8_t, tcache_nfill_small_lg_div_get,
-    (tcache_slow_t *tcache_slow, szind_t szind),
-    (tcache_slow, szind))
-
-JET_WRAP_VOID(tcache_nfill_small_burst_prepare,
-    (tcache_slow_t *tcache_slow, szind_t szind),
-    (tcache_slow, szind))
-
-JET_WRAP_VOID(tcache_nfill_small_burst_reset,
-    (tcache_slow_t *tcache_slow, szind_t szind),
-    (tcache_slow, szind))
-
-JET_WRAP_VOID(tcache_nfill_small_gc_update,
-    (tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit),
-    (tcache_slow, szind, limit))
-
-JET_WRAP_RET(uint8_t, tcache_gc_item_delay_compute,
-    (szind_t szind), (szind))
-
-#undef JET_WRAP_RET
-#undef JET_WRAP_VOID
-#endif
-
 static bool
 tcache_gc_small(
     tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) {
@@ -1093,21 +1044,28 @@ tcache_create_explicit(tsd_t *tsd) {
 bool
 tsd_tcache_enabled_data_init(tsd_t *tsd) {
 	/* Called upon tsd initialization. */
-	tsd_tcache_enabled_set(tsd, opt_tcache);
 	/*
 	 * tcache is not available yet, but we need to set up its tcache_nbins
 	 * in advance.
 	 */
 	tcache_default_settings_init(tsd_tcache_slowp_get(tsd));
-	tsd_slow_update(tsd);
 
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+	if (test_hooks_tsd_bootstrap_hook != NULL) {
+		test_hooks_tsd_bootstrap_hook();
+	}
+#endif
+
+	bool err = false;
 	if (opt_tcache) {
-		/* Trigger tcache init. */
-		return tsd_tcache_data_init(
+		/* Initialize the bins before advertising the tcache. */
+		err = tsd_tcache_data_init(
 			tsd, NULL, tcache_get_default_ncached_max());
 	}
 
-	return false;
+	tsd_tcache_enabled_set(tsd, opt_tcache && !err);
+	tsd_slow_update(tsd);
+	return err;
 }
 
 void
@@ -1276,17 +1234,18 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
 	 * tsd).  Manually trigger decay to avoid pathological cases.  Also
 	 * include arena 0 because the tcache array is allocated from it.
 	 */
-	arena_decay(
-	    tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), false, false);
+	pa_shard_do_deferred_work(tsd_tsdn(tsd),
+	    &arena_get(tsd_tsdn(tsd), 0, false)->pa_shard,
+	    /* is_background_thread */ false);
 
 	if (arena_nthreads_get(arena, false) == 0
 	    && !background_thread_enabled()) {
 		/* Force purging when no threads assigned to the arena anymore. */
-		arena_decay(tsd_tsdn(tsd), arena,
-		    /* is_background_thread */ false, /* all */ true);
+		pa_shard_flush(tsd_tsdn(tsd), &arena->pa_shard,
+		    /* all */ true);
 	} else {
-		arena_decay(tsd_tsdn(tsd), arena,
-		    /* is_background_thread */ false, /* all */ false);
+		pa_shard_do_deferred_work(tsd_tsdn(tsd), &arena->pa_shard,
+		    /* is_background_thread */ false);
 	}
 }
 

src/test_hooks.c

@@ -13,3 +13,8 @@ void (*test_hooks_libc_hook)(void) = NULL;
 
 JEMALLOC_EXPORT
 void (*test_hooks_safety_check_abort)(const char *) = NULL;
+
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+JEMALLOC_EXPORT
+void (*test_hooks_tsd_bootstrap_hook)(void) = NULL;
+#endif

src/thread_event.c

@@ -3,6 +3,7 @@
 #include "jemalloc/internal/jemalloc_internal_inlines_a.h"
 #include "jemalloc/internal/peak_event.h"
 #include "jemalloc/internal/prof.h"
+#include "jemalloc/internal/stats.h"
 #include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/thread_event.h"
 #include "jemalloc/internal/thread_event_registry.h"
@@ -185,7 +186,6 @@ te_update_wait(tsd_t *tsd, uint64_t accumbytes, bool allow, uint64_t *ev_wait,
 	return ret;
 }
 
-extern uint64_t stats_interval_accum_batch;
 /* Return number of handlers enqueued into to_trigger array */
 static inline size_t
 te_update_alloc_events(tsd_t *tsd, te_base_cb_t **to_trigger,

src/tsd.c

@@ -54,6 +54,7 @@ struct tsd_init_head_s {
 };
 
 pthread_key_t   tsd_tsd;
+pthread_key_t   tsd_thread_initialized_tsd;
 tsd_init_head_t tsd_init_head = {
     ql_head_initializer(blocks), MALLOC_MUTEX_INITIALIZER};
 

test/analyze/rand.c

@@ -1,5 +1,7 @@
 #include "test/jemalloc_test.h"
 
+extern uint64_t prof_sample_new_event_wait(tsd_t *tsd);
+
 /******************************************************************************/
 
 /*

test/include/test/hpa.h

@@ -0,0 +1,10 @@
+#ifndef TEST_HPA_H
+#define TEST_HPA_H
+
+/*
+ * Static in production builds (JET_EXTERN); exported only for the unit tests
+ * that share it (hpa_central, psset).
+ */
+extern bool hpa_hugepage_size_exceeds_limit(void);
+
+#endif /* TEST_HPA_H */

test/include/test/hpdata.h

@@ -0,0 +1,10 @@
+#ifndef TEST_HPDATA_H
+#define TEST_HPDATA_H
+
+/*
+ * Static in production builds (JET_EXTERN); exported only for the unit tests
+ * that share it (hpdata, psset).
+ */
+extern void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
+
+#endif /* TEST_HPDATA_H */

test/include/test/prof_data.h

@@ -0,0 +1,12 @@
+#ifndef TEST_PROF_DATA_H
+#define TEST_PROF_DATA_H
+
+/*
+ * Static in production builds (#ifdef JEMALLOC_JET); exported only for the unit
+ * tests that share them (prof_reset, prof_accum, prof_active, prof_tctx).
+ */
+extern size_t prof_tdata_count(void);
+extern size_t prof_bt_count(void);
+extern void   prof_cnt_all(prof_cnt_t *cnt_all);
+
+#endif /* TEST_PROF_DATA_H */

test/integration/cpp/failing_new.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+XMALLOC_STR=""
+if [ "x${enable_xmalloc}" = "x1" ] ; then
+  XMALLOC_STR="xmalloc:false"
+fi
+
+export MALLOC_CONF="${XMALLOC_STR}"

test/integration/cpp/infallible_new.cpp

@@ -3,8 +3,9 @@
 #include "test/jemalloc_test.h"
 
 /*
- * We can't test C++ in unit tests.  In order to intercept abort, use the
- * internal test hook in integration tests.
+ * Verifies that, when jemalloc is built with --enable-cxx-infallible-new,
+ * throwing operator new on OOM aborts via safety_check_fail. The test hook
+ * intercepts the abort and asserts the size-bearing message prefix.
  */
 bool fake_abort_called;
 void

test/integration/cpp/infallible_new.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+XMALLOC_STR=""
+if [ "x${enable_xmalloc}" = "x1" ] ; then
+  XMALLOC_STR="xmalloc:false"
+fi
+
+export MALLOC_CONF="${XMALLOC_STR}"

test/integration/cpp/infallible_new_false.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-XMALLOC_STR=""
-if [ "x${enable_xmalloc}" = "x1" ] ; then
-  XMALLOC_STR="xmalloc:false,"
-fi
-
-export MALLOC_CONF="${XMALLOC_STR}experimental_infallible_new:false"

test/integration/cpp/infallible_new_true.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-XMALLOC_STR=""
-if [ "x${enable_xmalloc}" = "x1" ] ; then
-  XMALLOC_STR="xmalloc:false,"
-fi
-
-export MALLOC_CONF="${XMALLOC_STR}experimental_infallible_new:true"

test/integration/free_sized.c

@@ -0,0 +1,37 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_free_sized) {
+	void *p = mallocx(42, 0);
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
+	free_sized(p, 42);
+}
+TEST_END
+
+TEST_BEGIN(test_free_aligned_sized) {
+	size_t alignment = 0x100;
+	void  *p = mallocx(42, MALLOCX_ALIGN(alignment));
+	expect_ptr_not_null(p, "Unexpected mallocx() failure");
+	free_aligned_sized(p, alignment, 42);
+}
+TEST_END
+
+TEST_BEGIN(test_free_sized_null) {
+	/*
+	 * C23 specifies that free_sized(NULL, size) and
+	 * free_aligned_sized(NULL, alignment, size) do nothing, just as
+	 * free(NULL) does.  The size argument is ignored for a NULL pointer.
+	 */
+	free_sized(NULL, 0);
+	free_sized(NULL, 42);
+	free_aligned_sized(NULL, 0x100, 0);
+	free_aligned_sized(NULL, 0x100, 42);
+}
+TEST_END
+
+int
+main(void) {
+	return test(
+	    test_free_sized,
+	    test_free_aligned_sized,
+	    test_free_sized_null);
+}

test/stress/pa/pa_microbench.c

@@ -101,7 +101,6 @@ static hpa_shard_opts_t g_hpa_opts = {
 	/* hugify_delay_ms */           7804,
 	/* hugify_sync */               false,
 	/* min_purge_interval_ms */     5 * 1000,
-	/* experimental_max_purge_nhp */ -1,
 	/* purge_threshold */           HUGEPAGE,
 	/* min_purge_delay_ms */        0,
 	/* hugify_style */              hpa_hugify_style_eager

test/unit/background_thread.c

@@ -107,9 +107,88 @@ TEST_BEGIN(test_background_thread_running) {
 }
 TEST_END
 
+TEST_BEGIN(test_background_thread_arena_reset) {
+	test_skip_if(!have_background_thread);
+
+	test_switch_background_thread_ctl(true);
+
+	unsigned arena_ind;
+	size_t   sz = sizeof(arena_ind);
+	expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Unexpected arenas.create failure");
+	void *p = mallocx(PAGE,
+	    MALLOCX_TCACHE_NONE | MALLOCX_ARENA(arena_ind));
+	expect_ptr_not_null(p, "Unexpected mallocx failure");
+
+	/*
+	 * Resetting an arena takes its background thread through
+	 * started -> paused -> started (background_thread_arena_reset_begin/
+	 * finish).  The reset frees p, so we must not touch it afterwards.
+	 */
+	size_t mib[3];
+	size_t miblen = sizeof(mib) / sizeof(mib[0]);
+	expect_d_eq(mallctlnametomib("arena.0.reset", mib, &miblen), 0,
+	    "Unexpected mallctlnametomib failure");
+	mib[1] = arena_ind;
+	expect_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
+	    "Unexpected arena reset failure");
+
+#if defined(JEMALLOC_BACKGROUND_THREAD)
+	background_thread_info_t *info = background_thread_info_get(arena_ind);
+	tsd_t                    *tsd = tsd_fetch();
+	malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+	background_thread_state_t st = info->state;
+	malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+	expect_d_eq((int)st, (int)background_thread_started,
+	    "Arena reset must leave the background thread started, not paused");
+#endif
+	expect_zu_gt(n_background_threads, 0,
+	    "Arena reset must not lose background threads");
+
+	test_switch_background_thread_ctl(false);
+}
+TEST_END
+
+TEST_BEGIN(test_background_thread_stats_ctl) {
+	test_skip_if(!have_background_thread);
+	test_skip_if(!config_stats);
+
+	test_switch_background_thread_ctl(true);
+
+	uint64_t epoch = 1;
+	size_t   sz = sizeof(epoch);
+	expect_d_eq(mallctl("epoch", (void *)&epoch, &sz, (void *)&epoch,
+	                sizeof(epoch)),
+	    0, "Unexpected epoch mallctl failure");
+
+	size_t num_threads;
+	sz = sizeof(num_threads);
+	expect_d_eq(mallctl("stats.background_thread.num_threads",
+	                (void *)&num_threads, &sz, NULL, 0),
+	    0, "Unexpected stats.background_thread.num_threads failure");
+	expect_zu_eq(num_threads, n_background_threads,
+	    "Reported num_threads should match n_background_threads");
+
+	uint64_t num_runs;
+	sz = sizeof(num_runs);
+	expect_d_eq(mallctl("stats.background_thread.num_runs",
+	                (void *)&num_runs, &sz, NULL, 0),
+	    0, "Unexpected stats.background_thread.num_runs failure");
+
+	uint64_t run_interval;
+	sz = sizeof(run_interval);
+	expect_d_eq(mallctl("stats.background_thread.run_interval",
+	                (void *)&run_interval, &sz, NULL, 0),
+	    0, "Unexpected stats.background_thread.run_interval failure");
+
+	test_switch_background_thread_ctl(false);
+}
+TEST_END
+
 int
 main(void) {
 	/* Background_thread creation tests reentrancy naturally. */
-	return test_no_reentrancy(
-	    test_background_thread_ctl, test_background_thread_running);
+	return test_no_reentrancy(test_background_thread_ctl,
+	    test_background_thread_running, test_background_thread_arena_reset,
+	    test_background_thread_stats_ctl);
 }