Expose stats table columns in JSON output and simplify code

The extents and hpa-nonfull "ind" columns are emitter_type_unsigned but were
set via col_ind.size_val -- correct only by little-endian union overlap.
Output is byte-identical.  Pre-existing (predates the refactor).

.github/workflows/linux-ci.yml

@@ -508,6 +508,26 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: force_tls=0
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: "force_tls=0 --enable-debug"
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: "--enable-cxx-infallible-new --enable-debug"
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
     - uses: actions/checkout@v6
@@ -637,6 +657,11 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=background_thread:true"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/macos-ci.yml

@@ -60,6 +60,11 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
     - uses: actions/checkout@v6
@@ -162,6 +167,11 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
     - uses: actions/checkout@v6

.github/workflows/windows-ci.yml

@@ -52,6 +52,11 @@ jobs:
               CXX: cl.exe
               CROSS_COMPILE_32BIT: yes
               CONFIGURE_FLAGS: --enable-debug
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: -fcommon
 
     steps:
     - uses: actions/checkout@v6

INSTALL.md

@@ -139,7 +139,6 @@ any of the following arguments (not a definitive list) to 'configure':
     in the following list that appears to function correctly:
 
     + libunwind      (requires --enable-prof-libunwind)
-    + frame pointer  (requires --enable-prof-frameptr)
     + libgcc         (unless --disable-prof-libgcc)
     + gcc intrinsics (unless --disable-prof-gcc)
 
@@ -148,12 +147,6 @@ any of the following arguments (not a definitive list) to 'configure':
     Use the libunwind library (http://www.nongnu.org/libunwind/) for stack
     backtracing.
 
-* `--enable-prof-frameptr`
-
-    Use the optimized frame pointer unwinder for stack backtracing. Safe
-    to use in mixed code (with and without frame pointers) - but requires
-    frame pointers to produce meaningful stacks. Linux only.
-
 * `--disable-prof-libgcc`
 
     Disable the use of libgcc's backtracing functionality.
@@ -217,6 +210,15 @@ any of the following arguments (not a definitive list) to 'configure':
     Disable C++ integration.  This will cause new and delete operator
     implementations to be omitted.
 
+* `--enable-cxx-infallible-new`
+
+    Make the throwing `operator new` abort on allocation failure (logging the
+    requested size) instead of throwing `std::bad_alloc`; the `std::nothrow`
+    overloads still return null.  Disabled by default, and has no effect when
+    C++ integration is disabled (`--disable-cxx`).  When enabled, under LTO
+    this lets the compiler treat `operator new` as non-throwing and elide
+    exception-handling cleanup in callers.
+
 * `--with-xslroot=<path>`
 
     Specify where to find DocBook XSL stylesheets when building the

Makefile.in

@@ -132,6 +132,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/inspect.c \
 	$(srcroot)src/large.c \
 	$(srcroot)src/log.c \
+	$(srcroot)src/malloc_dispatch.c \
 	$(srcroot)src/malloc_io.c \
 	$(srcroot)src/conf.c \
 	$(srcroot)src/mutex.c \
@@ -145,7 +146,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/prof_data.c \
 	$(srcroot)src/prof_log.c \
 	$(srcroot)src/prof_recent.c \
-	$(srcroot)src/prof_stack_range.c \
 	$(srcroot)src/prof_stats.c \
 	$(srcroot)src/prof_sys.c \
 	$(srcroot)src/psset.c \
@@ -203,7 +203,6 @@ C_UTIL_CPP_SRCS := $(srcroot)src/nstime.c $(srcroot)src/malloc_io.c
 endif
 TESTS_UNIT := \
 	$(srcroot)test/unit/a0.c \
-	$(srcroot)test/unit/arena_decay.c \
 	$(srcroot)test/unit/arena_reset.c \
 	$(srcroot)test/unit/arenas_management.c \
 	$(srcroot)test/unit/atomic.c \
@@ -260,6 +259,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/log.c \
 	$(srcroot)test/unit/mallctl.c \
 	$(srcroot)test/unit/malloc_conf_2.c \
+	$(srcroot)test/unit/malloc_dispatch.c \
 	$(srcroot)test/unit/malloc_io.c \
 	$(srcroot)test/unit/math.c \
 	$(srcroot)test/unit/mpsc_queue.c \
@@ -270,9 +270,12 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
 	$(srcroot)test/unit/pac.c \
+	$(srcroot)test/unit/pac_decay.c \
+	$(srcroot)test/unit/pac_sec_integration.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/peak.c \
+	$(srcroot)test/unit/percpu_arena_resume.c \
 	$(srcroot)test/unit/ph.c \
 	$(srcroot)test/unit/prng.c \
 	$(srcroot)test/unit/prof_accum.c \
@@ -328,6 +331,7 @@ endif
 TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
 	$(srcroot)test/integration/allocated.c \
 	$(srcroot)test/integration/extent.c \
+	$(srcroot)test/integration/free_sized.c \
 	$(srcroot)test/integration/malloc.c \
 	$(srcroot)test/integration/mallocx.c \
 	$(srcroot)test/integration/MALLOCX_ARENA.c \
@@ -347,8 +351,11 @@ ifeq (@enable_cxx@, 1)
 CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
 TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
 ifeq (@enable_cxx_exceptions@, 1)
-TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new_true.cpp \
-	$(srcroot)test/integration/cpp/infallible_new_false.cpp
+ifeq (@enable_cxx_infallible_new@, 1)
+TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new.cpp
+else
+TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/failing_new.cpp
+endif
 endif
 else
 CPP_SRCS :=

configure.ac

@@ -396,8 +396,31 @@ if test "x$enable_cxx" = "x1"; then
     enable_cxx_exceptions="0"
   fi
 fi
+
+dnl Do not enable infallible-new by default. When enabled, the throwing
+dnl operator new aborts on OOM (with size logged) instead of throwing
+dnl std::bad_alloc; nothrow new still returns null. With LTO this lets the
+dnl compiler prove operator new doesn't throw and elide exception cleanup.
+AC_ARG_ENABLE([cxx-infallible-new],
+  [AS_HELP_STRING([--enable-cxx-infallible-new],
+    [Abort on OOM in throwing operator new instead of throwing std::bad_alloc])],
+[if test "x$enable_cxx_infallible_new" = "xno" ; then
+  enable_cxx_infallible_new="0"
+else
+  enable_cxx_infallible_new="1"
+fi
+],
+[enable_cxx_infallible_new="0"]
+)
+if test "x$enable_cxx" != "x1" ; then
+  enable_cxx_infallible_new="0"
+fi
+if test "x$enable_cxx_infallible_new" = "x1" ; then
+  AC_DEFINE([JEMALLOC_INFALLIBLE_NEW], [ ], [ ])
+fi
 AC_SUBST([enable_cxx])
 AC_SUBST([enable_cxx_exceptions])
+AC_SUBST([enable_cxx_infallible_new])
 AC_SUBST([CONFIGURE_CXXFLAGS])
 AC_SUBST([SPECIFIED_CXXFLAGS])
 AC_SUBST([EXTRA_CXXFLAGS])
@@ -510,13 +533,14 @@ esac
 AC_DEFINE_UNQUOTED([HAVE_CPU_SPINWAIT], [$HAVE_CPU_SPINWAIT], [ ])
 AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT], [ ])
 
+dnl LG_VADDR: number of significant virtual address bits. Auto-detected when --with-lg-vaddr is left unset.
 AC_ARG_WITH([lg_vaddr],
   [AS_HELP_STRING([--with-lg-vaddr=<lg-vaddr>], [Number of significant virtual address bits])],
   [LG_VADDR="$with_lg_vaddr"], [LG_VADDR="detect"])
 
-case "${host_cpu}" in
-  aarch64)
-    if test "x$LG_VADDR" = "xdetect"; then
+if test "x$LG_VADDR" = "xdetect"; then
+  case "${host_cpu}" in
+    aarch64)
       AC_MSG_CHECKING([number of significant virtual address bits])
       if test "x${LG_SIZEOF_PTR}" = "x2" ; then
         #aarch64 ILP32
@@ -526,10 +550,8 @@ case "${host_cpu}" in
         LG_VADDR=48
       fi
       AC_MSG_RESULT([$LG_VADDR])
-    fi
-    ;;
-  x86_64)
-    if test "x$LG_VADDR" = "xdetect"; then
+      ;;
+    x86_64)
       AC_CACHE_CHECK([number of significant virtual address bits],
                      [je_cv_lg_vaddr],
                      AC_RUN_IFELSE([AC_LANG_PROGRAM(
@@ -569,35 +591,14 @@ typedef unsigned __int32 uint32_t;
                    [je_cv_lg_vaddr=`cat conftest.out`],
                    [je_cv_lg_vaddr=error],
                    [je_cv_lg_vaddr=57]))
+      if test "x${je_cv_lg_vaddr}" = "xerror" ; then
+        AC_MSG_ERROR([cannot determine number of significant virtual address bits])
+      fi
       if test "x${je_cv_lg_vaddr}" != "x" ; then
         LG_VADDR="${je_cv_lg_vaddr}"
       fi
-      if test "x${LG_VADDR}" != "xerror" ; then
-        AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
-      else
-        AC_MSG_ERROR([cannot determine number of significant virtual address bits])
-      fi
-      AC_CACHE_CHECK([rdtscp support],
-		     [je_cv_rdtscp],
-		     AC_RUN_IFELSE([AC_LANG_PROGRAM(
-[[
-#include <stdint.h>
-]],
-[[
-      unsigned int dx;
-      asm volatile("rdtscp" : "=d"(dx) ::);
-      return 0;
-]])],
-      [je_cv_rdtscp=yes],
-      [je_cv_rdtscp=no],
-      [je_cv_rdtscp=no]))
-      if test "x${je_cv_rdtscp}" = "xyes"; then
-        AC_DEFINE([JEMALLOC_HAVE_RDTSCP], [ ], [ ])
-      fi
-    fi
-    ;;
-  *)
-    if test "x$LG_VADDR" = "xdetect"; then
+      ;;
+    *)
       AC_MSG_CHECKING([number of significant virtual address bits])
       if test "x${LG_SIZEOF_PTR}" = "x3" ; then
         LG_VADDR=64
@@ -609,10 +610,34 @@ typedef unsigned __int32 uint32_t;
         AC_MSG_ERROR([Unsupported lg(pointer size): ${LG_SIZEOF_PTR}])
       fi
       AC_MSG_RESULT([$LG_VADDR])
+      ;;
+  esac
+fi
+AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
+
+dnl rdtscp feature probe for x86_64
+case "${host_cpu}" in
+  x86_64)
+    AC_CACHE_CHECK([rdtscp support],
+		   [je_cv_rdtscp],
+		   AC_RUN_IFELSE([AC_LANG_PROGRAM(
+[[
+#include <stdint.h>
+]],
+[[
+      unsigned int dx;
+      asm volatile("rdtscp" : "=d"(dx) ::);
+      return 0;
+]])],
+    [je_cv_rdtscp=yes],
+    [je_cv_rdtscp=no],
+    [je_cv_rdtscp=no]))
+    if test "x${je_cv_rdtscp}" = "xyes"; then
+      AC_DEFINE([JEMALLOC_HAVE_RDTSCP], [ ], [ ])
     fi
     ;;
 esac
-AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR], [ ])
+
 AC_CACHE_CHECK([asm volatile support],
                [je_cv_asm_volatile],
                AC_RUN_IFELSE([AC_LANG_PROGRAM(
@@ -1532,33 +1557,6 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then
   fi
 fi
 
-if test `uname -s` = "Linux"
-then
-  AC_ARG_ENABLE([prof-frameptr],
-    [AS_HELP_STRING([--enable-prof-frameptr], [Use optimized frame pointer unwinder for backtracing (Linux only)])],
-  [if test "x$enable_prof_frameptr" = "xno" ; then
-    enable_prof_frameptr="0"
-  else
-    enable_prof_frameptr="1"
-    if test "x$enable_prof" = "x0" ; then
-      AC_MSG_ERROR([--enable-prof-frameptr should only be used with --enable-prof])
-    fi
-  fi
-  ],
-  [enable_prof_frameptr="0"]
-  )
-  if test "x$backtrace_method" = "x" -a "x$enable_prof_frameptr" = "x1" \
-      -a "x$GCC" = "xyes" ; then
-    JE_CFLAGS_ADD([-fno-omit-frame-pointer])
-    backtrace_method="frame pointer linux"
-    AC_DEFINE([JEMALLOC_PROF_FRAME_POINTER], [ ], [ ])
-  else
-    enable_prof_frameptr="0"
-  fi
-else
-  enable_prof_frameptr="0"
-fi
-
 AC_ARG_ENABLE([prof-libgcc],
   [AS_HELP_STRING([--disable-prof-libgcc],
   [Do not use libgcc for backtracing])],
@@ -2201,6 +2199,43 @@ dnl Check if we have dlsym support.
   if test "x${je_cv_pthread_get_name_np}" = "xyes" ; then
     AC_DEFINE([JEMALLOC_HAVE_PTHREAD_GET_NAME_NP], [ ], [ ])
   fi
+  dnl Runtime-check pthread_condattr_setclock and clock_gettime with
+  dnl CLOCK_MONOTONIC to confirm the kernel actually supports the
+  dnl monotonic clock.
+  AC_CACHE_CHECK([whether pthread_condattr_setclock with CLOCK_MONOTONIC is usable],
+                 [je_cv_pthread_cond_timedwait_monotonic],
+                 AC_RUN_IFELSE([AC_LANG_PROGRAM(
+[[
+#include <pthread.h>
+#include <time.h>
+]], [[
+  pthread_condattr_t attr;
+  pthread_cond_t cond;
+  struct timespec ts;
+  int ret = pthread_condattr_init(&attr);
+  if (ret != 0) {
+    return 1;
+  }
+  ret = pthread_condattr_setclock(&attr, CLOCK_MONOTONIC);
+  if (ret != 0) {
+    pthread_condattr_destroy(&attr);
+    return 1;
+  }
+  ret = pthread_cond_init(&cond, &attr);
+  pthread_condattr_destroy(&attr);
+  if (ret != 0) {
+    return 1;
+  }
+  ret = clock_gettime(CLOCK_MONOTONIC, &ts);
+  pthread_cond_destroy(&cond);
+  return ret != 0;
+]])],
+                 [je_cv_pthread_cond_timedwait_monotonic=yes],
+                 [je_cv_pthread_cond_timedwait_monotonic=no],
+                 [je_cv_pthread_cond_timedwait_monotonic=no]))
+  if test "x${je_cv_pthread_cond_timedwait_monotonic}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC], [ ], [ ])
+  fi
 fi
 
 JE_APPEND_VS(CPPFLAGS, -D_REENTRANT)
@@ -3124,7 +3159,6 @@ AC_MSG_RESULT([user_config        : ${enable_user_config}])
 AC_MSG_RESULT([experimental_smallocx : ${enable_experimental_smallocx}])
 AC_MSG_RESULT([prof               : ${enable_prof}])
 AC_MSG_RESULT([prof-libunwind     : ${enable_prof_libunwind}])
-AC_MSG_RESULT([prof-frameptr      : ${enable_prof_frameptr}])
 AC_MSG_RESULT([prof-libgcc        : ${enable_prof_libgcc}])
 AC_MSG_RESULT([prof-gcc           : ${enable_prof_gcc}])
 AC_MSG_RESULT([fill               : ${enable_fill}])

doc/jemalloc.xml.in

@@ -846,6 +846,16 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
         build configuration.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="config.infallible_new">
+        <term>
+          <mallctl>config.infallible_new</mallctl>
+          (<type>bool</type>)
+          <literal>r-</literal>
+        </term>
+        <listitem><para><option>--enable-cxx-infallible-new</option> was
+        specified during build configuration.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="config.lazy_lock">
         <term>
           <mallctl>config.lazy_lock</mallctl>

include/jemalloc/internal/arena.h

@@ -0,0 +1,276 @@
+#ifndef JEMALLOC_INTERNAL_ARENA_H
+#define JEMALLOC_INTERNAL_ARENA_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_decay_constants.h"
+#include "jemalloc/internal/arena_stats.h"
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/bin.h"
+#include "jemalloc/internal/bitmap.h"
+#include "jemalloc/internal/counter.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/ecache.h"
+#include "jemalloc/internal/edata_cache.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/extent_dss.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/nstime.h"
+#include "jemalloc/internal/pa.h"
+#include "jemalloc/internal/pages.h"
+#include "jemalloc/internal/ql.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/stats.h"
+#include "jemalloc/internal/ticker.h"
+
+/******************************************************************************/
+/* TYPES */
+/******************************************************************************/
+
+/* Default decay times in milliseconds. */
+#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
+#define MUZZY_DECAY_MS_DEFAULT (0)
+/* Maximum length of the arena name. */
+#define ARENA_NAME_LEN 32
+
+typedef enum {
+	percpu_arena_mode_names_base = 0, /* Used for options processing. */
+
+	/*
+	 * *_uninit are used only during bootstrapping, and must correspond
+	 * to initialized variant plus percpu_arena_mode_enabled_base.
+	 */
+	percpu_arena_uninit = 0,
+	per_phycpu_arena_uninit = 1,
+
+	/* All non-disabled modes must come after percpu_arena_disabled. */
+	percpu_arena_disabled = 2,
+
+	percpu_arena_mode_names_limit = 3, /* Used for options processing. */
+	percpu_arena_mode_enabled_base = 3,
+
+	percpu_arena = 3,
+	per_phycpu_arena = 4 /* Hyper threads share arena. */
+} percpu_arena_mode_t;
+
+#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
+#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
+
+/*
+ * When allocation_size >= oversize_threshold, use the dedicated huge arena
+ * (unless have explicitly spicified arena index).  0 disables the feature.
+ */
+#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
+
+struct arena_config_s {
+	/* extent hooks to be used for the arena */
+	extent_hooks_t *extent_hooks;
+
+	/*
+	 * Use extent hooks for metadata (base) allocations when true.
+	 */
+	bool metadata_use_hooks;
+};
+
+typedef struct arena_config_s arena_config_t;
+
+extern const arena_config_t arena_config_default;
+
+/******************************************************************************/
+/* STRUCTS */
+/******************************************************************************/
+
+struct arena_s {
+	/*
+	 * Number of threads currently assigned to this arena.  Each thread has
+	 * two distinct assignments, one for application-serving allocation, and
+	 * the other for internal metadata allocation.  Internal metadata must
+	 * not be allocated from arenas explicitly created via the arenas.create
+	 * mallctl, because the arena.<i>.reset mallctl indiscriminately
+	 * discards all allocations for the affected arena.
+	 *
+	 *   0: Application allocation.
+	 *   1: Internal metadata allocation.
+	 *
+	 * Synchronization: atomic.
+	 */
+	atomic_u_t nthreads[2];
+
+	/* Next bin shard for binding new threads. Synchronization: atomic. */
+	atomic_u_t binshard_next;
+
+	/*
+	 * When percpu_arena is enabled, to amortize the cost of reading /
+	 * updating the current CPU id, track the most recent thread accessing
+	 * this arena, and only read CPU if there is a mismatch.
+	 */
+	tsdn_t *last_thd;
+
+	/* Synchronization: internal. */
+	arena_stats_t stats;
+
+	/*
+	 * List of cache_bin_array_descriptors for extant threads associated
+	 * with this arena.  Stats from these are merged incrementally, and at
+	 * exit if opt_stats_print is enabled.
+	 *
+	 * Synchronization: cache_bin_array_descriptor_ql_mtx.
+	 */
+	ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
+	malloc_mutex_t cache_bin_array_descriptor_ql_mtx;
+
+	/*
+	 * Represents a dss_prec_t, but atomically.
+	 *
+	 * Synchronization: atomic.
+	 */
+	atomic_u_t dss_prec;
+
+	/*
+	 * Extant large allocations.
+	 *
+	 * Synchronization: large_mtx.
+	 */
+	edata_list_active_t large;
+	/* Synchronizes all large allocation/update/deallocation. */
+	malloc_mutex_t large_mtx;
+
+	/* The page-level allocator shard this arena uses. */
+	pa_shard_t pa_shard;
+
+	/*
+	 * A cached copy of base->ind.  This can get accessed on hot paths;
+	 * looking it up in base requires an extra pointer hop / cache miss.
+	 */
+	unsigned ind;
+
+	/*
+	 * Base allocator, from which arena metadata are allocated.
+	 *
+	 * Synchronization: internal.
+	 */
+	base_t *base;
+	/* Used to determine uptime.  Read-only after initialization. */
+	nstime_t create_time;
+
+	/* The name of the arena. */
+	char name[ARENA_NAME_LEN];
+
+	/*
+	 * The arena is allocated alongside its bins; really this is a
+	 * dynamically sized array determined by the binshard settings.
+	 * Enforcing cacheline-alignment to minimize the number of cachelines
+	 * touched on the hot paths.
+	 */
+	JEMALLOC_WARN_ON_USAGE(
+	    "Do not use this field directly. "
+	    "Use `arena_get_bin` instead.")
+	JEMALLOC_ALIGNED(CACHELINE)
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+	bin_t all_bins[];
+#else
+	bin_t all_bins[0];
+#endif
+};
+
+/******************************************************************************/
+/* EXTERNS */
+/******************************************************************************/
+
+extern ssize_t opt_dirty_decay_ms;
+extern ssize_t opt_muzzy_decay_ms;
+
+extern percpu_arena_mode_t opt_percpu_arena;
+extern const char *const   percpu_arena_mode_names[];
+
+extern div_info_t arena_binind_div_info[SC_NBINS];
+
+extern emap_t arena_emap_global;
+
+extern size_t opt_oversize_threshold;
+extern size_t oversize_threshold;
+
+extern bool      opt_huge_arena_pac_thp;
+extern pac_thp_t huge_arena_pac_thp;
+
+/*
+ * arena_bin_offsets[binind] is the offset of the first bin shard for size class
+ * binind.
+ */
+extern uint32_t arena_bin_offsets[SC_NBINS];
+
+void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
+    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
+    size_t *nactive, size_t *ndirty, size_t *nmuzzy);
+void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
+    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
+    size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
+    bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
+    hpa_shard_stats_t *hpastats);
+edata_t *arena_extent_alloc_large(
+    tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
+void arena_extent_dalloc_large_prep(
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata);
+void arena_extent_ralloc_large_shrink(
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
+void arena_extent_ralloc_large_expand(
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
+void           arena_reset(tsd_t *tsd, arena_t *arena);
+void           arena_destroy(tsd_t *tsd, arena_t *arena);
+cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
+    szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min,
+    const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats);
+
+void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
+    bool zero, bool slab);
+void  arena_prof_promote(
+     tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
+size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
+
+void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
+void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
+     cache_bin_ptr_array_t *arr, unsigned nflush, bool small,
+     arena_t *stats_arena, cache_bin_stats_t merge_stats);
+bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
+     size_t extra, bool zero, size_t *newsize);
+dss_prec_t      arena_dss_prec_get(const arena_t *arena);
+ehooks_t       *arena_get_ehooks(const arena_t *arena);
+extent_hooks_t *arena_set_extent_hooks(
+    tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
+bool    arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
+void    arena_name_get(const arena_t *arena, char *name);
+void    arena_name_set(arena_t *arena, const char *name);
+ssize_t arena_dirty_decay_ms_default_get(void);
+bool    arena_dirty_decay_ms_default_set(ssize_t decay_ms);
+ssize_t arena_muzzy_decay_ms_default_get(void);
+bool    arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
+bool    arena_retain_grow_limit_get_set(
+       tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
+unsigned arena_nthreads_get(const arena_t *arena, bool internal);
+void     arena_nthreads_inc(arena_t *arena, bool internal);
+void     arena_nthreads_dec(arena_t *arena, bool internal);
+arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
+bool     arena_init_huge(tsdn_t *tsdn, arena_t *a0);
+bool     arena_ind_is_huge(unsigned ind);
+arena_t *arena_choose_huge(tsd_t *tsd);
+bool   arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
+void  *arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind);
+void   arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *desc);
+void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *desc);
+void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork3(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork4(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork5(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
+void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
+void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *surviving_desc);
+
+#endif /* JEMALLOC_INTERNAL_ARENA_H */

include/jemalloc/internal/arena_decay_constants.h

@@ -0,0 +1,13 @@
+#ifndef JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H
+#define JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H
+
+/*
+ * Minimal header so both arena.h and tsd_internals.h can share decay-related
+ * constants without dragging the full arena types into the tsd parse chain
+ * (which is loaded long before arena.h via ckh.h -> tsd.h).
+ */
+
+/* Number of event ticks between time checks. */
+#define ARENA_DECAY_NTICKS_PER_UPDATE 1000
+
+#endif /* JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H */

include/jemalloc/internal/arena_externs.h

@@ -1,131 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_EXTERNS_H
-#define JEMALLOC_INTERNAL_ARENA_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_stats.h"
-#include "jemalloc/internal/bin.h"
-#include "jemalloc/internal/div.h"
-#include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/extent_dss.h"
-#include "jemalloc/internal/pages.h"
-#include "jemalloc/internal/stats.h"
-
-/*
- * When the amount of pages to be purged exceeds this amount, deferred purge
- * should happen.
- */
-#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
-
-extern ssize_t opt_dirty_decay_ms;
-extern ssize_t opt_muzzy_decay_ms;
-
-extern percpu_arena_mode_t opt_percpu_arena;
-extern const char *const   percpu_arena_mode_names[];
-
-extern div_info_t arena_binind_div_info[SC_NBINS];
-
-extern emap_t arena_emap_global;
-
-extern size_t opt_oversize_threshold;
-extern size_t oversize_threshold;
-
-extern bool      opt_huge_arena_pac_thp;
-extern pac_thp_t huge_arena_pac_thp;
-
-/*
- * arena_bin_offsets[binind] is the offset of the first bin shard for size class
- * binind.
- */
-extern uint32_t arena_bin_offsets[SC_NBINS];
-
-void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
-    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
-    size_t *nactive, size_t *ndirty, size_t *nmuzzy);
-void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
-    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
-    size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
-    bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
-    hpa_shard_stats_t *hpastats);
-void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
-edata_t *arena_extent_alloc_large(
-    tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
-void arena_extent_dalloc_large_prep(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata);
-void arena_extent_ralloc_large_shrink(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
-void arena_extent_ralloc_large_expand(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
-bool arena_decay_ms_set(
-    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
-ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
-void    arena_decay(
-       tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-uint64_t       arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
-void           arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
-void           arena_reset(tsd_t *tsd, arena_t *arena);
-void           arena_destroy(tsd_t *tsd, arena_t *arena);
-cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
-    szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min,
-    const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats);
-
-void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
-    bool zero, bool slab);
-void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
-    bool zero, bool slab, tcache_t *tcache);
-void  arena_prof_promote(
-     tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
-void arena_dalloc_promoted(
-    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
-void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
-
-void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
-void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
-     cache_bin_ptr_array_t *arr, unsigned nflush, bool small,
-     arena_t *stats_arena, cache_bin_stats_t merge_stats);
-bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-     size_t extra, bool zero, size_t *newsize);
-void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache);
-dss_prec_t      arena_dss_prec_get(const arena_t *arena);
-ehooks_t       *arena_get_ehooks(const arena_t *arena);
-extent_hooks_t *arena_set_extent_hooks(
-    tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
-bool    arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
-void    arena_name_get(const arena_t *arena, char *name);
-void    arena_name_set(arena_t *arena, const char *name);
-ssize_t arena_dirty_decay_ms_default_get(void);
-bool    arena_dirty_decay_ms_default_set(ssize_t decay_ms);
-ssize_t arena_muzzy_decay_ms_default_get(void);
-bool    arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
-bool    arena_retain_grow_limit_get_set(
-       tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
-unsigned arena_nthreads_get(const arena_t *arena, bool internal);
-void     arena_nthreads_inc(arena_t *arena, bool internal);
-void     arena_nthreads_dec(arena_t *arena, bool internal);
-arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
-bool     arena_init_huge(tsdn_t *tsdn, arena_t *a0);
-bool     arena_ind_is_huge(unsigned ind);
-arena_t *arena_choose_huge(tsd_t *tsd);
-bool   arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
-void  *arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind);
-void   arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
-       cache_bin_array_descriptor_t *desc);
-void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
-       cache_bin_array_descriptor_t *desc);
-void   arena_cache_bin_array_postfork_child(arena_t *arena,
-       cache_bin_array_descriptor_t *desc_or_null);
-void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork3(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork4(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork5(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
-void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
-void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
-       cache_bin_array_descriptor_t *surviving_desc);
-
-#endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */

include/jemalloc/internal/arena_inlines.h

@@ -0,0 +1,400 @@
+#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_H
+#define JEMALLOC_INTERNAL_ARENA_INLINES_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arenas_management.h"
+#include "jemalloc/internal/bin_inlines.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/extent.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/large.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prof.h"
+#include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache.h"
+#include "jemalloc/internal/ticker.h"
+
+/* Cheap field accessors. */
+
+static inline unsigned
+arena_ind_get(const arena_t *arena) {
+	return arena->ind;
+}
+
+static inline void
+arena_internal_add(arena_t *arena, size_t size) {
+	atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
+}
+
+static inline void
+arena_internal_sub(arena_t *arena, size_t size) {
+	atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
+}
+
+static inline size_t
+arena_internal_get(const arena_t *arena) {
+	return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
+}
+
+static inline bool
+arena_is_auto(const arena_t *arena) {
+	assert(narenas_auto > 0);
+
+	return (arena_ind_get(arena) < manual_arena_base);
+}
+
+static inline arena_t *
+arena_get_from_edata(const edata_t *edata) {
+	return (arena_t *)atomic_load_p(
+	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
+}
+
+/* Arena selection and migration. */
+
+static inline void
+thread_migrate_arena(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
+	assert(oldarena != NULL);
+	assert(newarena != NULL);
+
+	arena_migrate(tsd, oldarena, newarena);
+	if (tcache_available(tsd)) {
+		tcache_arena_reassociate(tsd_tsdn(tsd),
+		    tsd_tcache_slowp_get(tsd), newarena);
+	}
+}
+
+static inline void
+percpu_arena_update(tsd_t *tsd, unsigned cpu) {
+	assert(have_percpu_arena);
+	arena_t *oldarena = tsd_arena_get(tsd);
+	assert(oldarena != NULL);
+	unsigned oldind = arena_ind_get(oldarena);
+
+	if (oldind != cpu) {
+		unsigned newind = cpu;
+		arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
+		assert(newarena != NULL);
+
+		thread_migrate_arena(tsd, oldarena, newarena);
+	}
+}
+
+/* Choose an arena based on a per-thread value. */
+static inline arena_t *
+arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
+	arena_t *ret;
+
+	if (arena != NULL) {
+		return arena;
+	}
+
+	/* During reentrancy, arena 0 is the safest bet. */
+	if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
+		return arena_get(tsd_tsdn(tsd), 0, true);
+	}
+
+	ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
+	if (unlikely(ret == NULL)) {
+		ret = arena_choose_hard(tsd, internal);
+		assert(ret);
+		if (tcache_available(tsd)) {
+			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
+			if (tcache_slow->arena != NULL) {
+				/* See comments in tsd_tcache_data_init().*/
+				assert(tcache_slow->arena
+				    == arena_get(tsd_tsdn(tsd), 0, false));
+				if (tcache_slow->arena != ret) {
+					tcache_arena_reassociate(tsd_tsdn(tsd),
+					    tcache_slow, ret);
+				}
+			} else {
+				tcache_arena_associate(
+				    tsd_tsdn(tsd), tcache_slow, ret);
+			}
+		}
+	}
+
+	/*
+	 * Note that for percpu arena, if the current arena is outside of the
+	 * auto percpu arena range, (i.e. thread is assigned to a manually
+	 * managed arena), then percpu arena is skipped.
+	 */
+	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
+	    && !internal
+	    && (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
+	    && (ret->last_thd != tsd_tsdn(tsd))) {
+		unsigned ind = percpu_arena_choose();
+		if (arena_ind_get(ret) != ind) {
+			percpu_arena_update(tsd, ind);
+			ret = tsd_arena_get(tsd);
+		}
+		ret->last_thd = tsd_tsdn(tsd);
+	}
+
+	return ret;
+}
+
+static inline arena_t *
+arena_choose(tsd_t *tsd, arena_t *arena) {
+	return arena_choose_impl(tsd, arena, false);
+}
+
+static inline arena_t *
+arena_ichoose(tsd_t *tsd, arena_t *arena) {
+	return arena_choose_impl(tsd, arena, true);
+}
+
+JEMALLOC_ALWAYS_INLINE arena_t *
+arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
+	if (arena != NULL) {
+		return arena;
+	}
+
+	/*
+	 * For huge allocations, use the dedicated huge arena if both are true:
+	 * 1) is using auto arena selection (i.e. arena == NULL), and 2) the
+	 * thread is not assigned to a manual arena.
+	 */
+	arena_t *tsd_arena = tsd_arena_get(tsd);
+	if (tsd_arena == NULL) {
+		tsd_arena = arena_choose(tsd, NULL);
+	}
+
+	size_t threshold = atomic_load_zu(
+	    &tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
+	if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) {
+		return arena_choose_huge(tsd);
+	}
+
+	return tsd_arena;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
+	if (!config_opt_safety_checks) {
+		return false;
+	}
+
+	/*
+	 * Eagerly detect double free and sized dealloc bugs for large sizes.
+	 * The cost is low enough (as edata will be accessed anyway) to be
+	 * enabled all the time.
+	 */
+	if (unlikely(edata == NULL
+	        || edata_state_get(edata) != extent_state_active)) {
+		safety_check_fail(
+		    "Invalid deallocation detected: "
+		    "pages being freed (%p) not currently active, "
+		    "possibly caused by double free bugs.",
+		    ptr);
+		return true;
+	}
+	if (unlikely(input_size != edata_usize_get(edata)
+	        || input_size > SC_LARGE_MAXCLASS)) {
+		safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
+		    /* true_size */ edata_usize_get(edata), input_size);
+		return true;
+	}
+
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
+    prof_info_t *prof_info, bool reset_recent) {
+	cassert(config_prof);
+	assert(ptr != NULL);
+	assert(prof_info != NULL);
+
+	edata_t *edata = NULL;
+	bool     is_slab;
+
+	/* Static check. */
+	if (alloc_ctx == NULL) {
+		edata = emap_edata_lookup(
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
+		is_slab = edata_slab_get(edata);
+	} else if (unlikely(!(is_slab = alloc_ctx->slab))) {
+		edata = emap_edata_lookup(
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
+	}
+
+	if (unlikely(!is_slab)) {
+		/* edata must have been initialized at this point. */
+		assert(edata != NULL);
+		size_t usize = (alloc_ctx == NULL)
+		    ? edata_usize_get(edata)
+		    : emap_alloc_ctx_usize_get(alloc_ctx);
+		if (reset_recent
+		    && large_dalloc_safety_checks(edata, ptr, usize)) {
+			prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
+			return;
+		}
+		large_prof_info_get(tsd, edata, prof_info, reset_recent);
+	} else {
+		prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
+		/*
+		 * No need to set other fields in prof_info; they will never be
+		 * accessed if alloc_tctx == PROF_TCTX_SENTINEL.
+		 */
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_tctx_reset(
+    tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
+	cassert(config_prof);
+	assert(ptr != NULL);
+
+	/* Static check. */
+	if (alloc_ctx == NULL) {
+		edata_t *edata = emap_edata_lookup(
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
+		if (unlikely(!edata_slab_get(edata))) {
+			large_prof_tctx_reset(edata);
+		}
+	} else {
+		if (unlikely(!alloc_ctx->slab)) {
+			edata_t *edata = emap_edata_lookup(
+			    tsd_tsdn(tsd), &arena_emap_global, ptr);
+			large_prof_tctx_reset(edata);
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
+	cassert(config_prof);
+	assert(ptr != NULL);
+
+	edata_t *edata = emap_edata_lookup(
+	    tsd_tsdn(tsd), &arena_emap_global, ptr);
+	assert(!edata_slab_get(edata));
+
+	large_prof_tctx_reset(edata);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_info_set(
+    tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
+	cassert(config_prof);
+
+	assert(!edata_slab_get(edata));
+	large_prof_info_set(edata, tctx, size);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
+	if (unlikely(tsdn_null(tsdn))) {
+		return;
+	}
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	/*
+	 * We use the ticker_geom_t to avoid having per-arena state in the tsd.
+	 * Instead of having a countdown-until-decay timer running for every
+	 * arena in every thread, we flip a coin once per tick, whose
+	 * probability of coming up heads is 1/nticks; this is effectively the
+	 * operation of the ticker_geom_t.  Each arena has the same chance of a
+	 * coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can
+	 * use a single ticker for all of them.
+	 */
+	ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
+	uint64_t      *prng_state = tsd_prng_statep_get(tsd);
+	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
+	        tsd_reentrancy_level_get(tsd) > 0))) {
+		pa_shard_do_deferred_work(tsdn, &arena->pa_shard, false);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
+	arena_decay_ticks(tsdn, arena, 1);
+}
+
+JEMALLOC_ALWAYS_INLINE arena_t *
+arena_aalloc(tsdn_t *tsdn, const void *ptr) {
+	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	unsigned arena_ind = edata_arena_ind_get(edata);
+	return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_RELAXED);
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
+arena_salloc(tsdn_t *tsdn, const void *ptr) {
+	assert(ptr != NULL);
+	emap_alloc_ctx_t alloc_ctx;
+	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
+	assert(alloc_ctx.szind != SC_NSIZES);
+
+	return emap_alloc_ctx_usize_get(&alloc_ctx);
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
+arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
+	/*
+	 * Return 0 if ptr is not within an extent managed by jemalloc.  This
+	 * function has two extra costs relative to isalloc():
+	 * - The rtree calls cannot claim to be dependent lookups, which induces
+	 *   rtree lookup load dependencies.
+	 * - The lookup may fail, so there is an extra branch to check for
+	 *   failure.
+	 */
+
+	emap_full_alloc_ctx_t full_alloc_ctx;
+	bool                  missing = emap_full_alloc_ctx_try_lookup(
+            tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
+	if (missing) {
+		return 0;
+	}
+
+	if (full_alloc_ctx.edata == NULL) {
+		return 0;
+	}
+	assert(edata_state_get(full_alloc_ctx.edata) == extent_state_active);
+	/* Only slab members should be looked up via interior pointers. */
+	assert(edata_addr_get(full_alloc_ctx.edata) == ptr
+	    || edata_slab_get(full_alloc_ctx.edata));
+
+	assert(full_alloc_ctx.szind != SC_NSIZES);
+
+	return edata_usize_get(full_alloc_ctx.edata);
+}
+
+static inline void
+arena_cache_oblivious_randomize(
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
+	assert(edata_base_get(edata) == edata_addr_get(edata));
+
+	if (alignment < PAGE) {
+		unsigned lg_range = LG_PAGE
+		    - lg_floor(CACHELINE_CEILING(alignment));
+		size_t r;
+		if (!tsdn_null(tsdn)) {
+			tsd_t *tsd = tsdn_tsd(tsdn);
+			r = (size_t)prng_lg_range_u64(
+			    tsd_prng_statep_get(tsd), lg_range);
+		} else {
+			uint64_t stack_value = (uint64_t)(uintptr_t)&r;
+			r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
+		}
+		uintptr_t random_offset = ((uintptr_t)r)
+		    << (LG_PAGE - lg_range);
+		edata->e_addr = (void *)((byte_t *)edata->e_addr
+		    + random_offset);
+		assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
+		    == edata->e_addr);
+	}
+}
+
+static inline bin_t *
+arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
+	bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]);
+	return shard0 + binshard;
+}
+
+#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_H */

include/jemalloc/internal/arena_inlines_a.h

@@ -1,27 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H
-#define JEMALLOC_INTERNAL_ARENA_INLINES_A_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_structs.h"
-
-static inline unsigned
-arena_ind_get(const arena_t *arena) {
-	return arena->ind;
-}
-
-static inline void
-arena_internal_add(arena_t *arena, size_t size) {
-	atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
-}
-
-static inline void
-arena_internal_sub(arena_t *arena, size_t size) {
-	atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
-}
-
-static inline size_t
-arena_internal_get(const arena_t *arena) {
-	return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
-}
-
-#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */

include/jemalloc/internal/arena_inlines_b.h

@@ -1,534 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
-#define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/arena_structs.h"
-#include "jemalloc/internal/bin_inlines.h"
-#include "jemalloc/internal/div.h"
-#include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
-#include "jemalloc/internal/jemalloc_internal_types.h"
-#include "jemalloc/internal/large_externs.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/prof_externs.h"
-#include "jemalloc/internal/prof_structs.h"
-#include "jemalloc/internal/rtree.h"
-#include "jemalloc/internal/safety_check.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_inlines.h"
-#include "jemalloc/internal/ticker.h"
-
-static inline arena_t *
-arena_get_from_edata(const edata_t *edata) {
-	return (arena_t *)atomic_load_p(
-	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
-}
-
-JEMALLOC_ALWAYS_INLINE arena_t *
-arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
-	if (arena != NULL) {
-		return arena;
-	}
-
-	/*
-	 * For huge allocations, use the dedicated huge arena if both are true:
-	 * 1) is using auto arena selection (i.e. arena == NULL), and 2) the
-	 * thread is not assigned to a manual arena.
-	 */
-	arena_t *tsd_arena = tsd_arena_get(tsd);
-	if (tsd_arena == NULL) {
-		tsd_arena = arena_choose(tsd, NULL);
-	}
-
-	size_t threshold = atomic_load_zu(
-	    &tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
-	if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) {
-		return arena_choose_huge(tsd);
-	}
-
-	return tsd_arena;
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
-	if (!config_opt_safety_checks) {
-		return false;
-	}
-
-	/*
-	 * Eagerly detect double free and sized dealloc bugs for large sizes.
-	 * The cost is low enough (as edata will be accessed anyway) to be
-	 * enabled all the time.
-	 */
-	if (unlikely(edata == NULL
-	        || edata_state_get(edata) != extent_state_active)) {
-		safety_check_fail(
-		    "Invalid deallocation detected: "
-		    "pages being freed (%p) not currently active, "
-		    "possibly caused by double free bugs.",
-		    ptr);
-		return true;
-	}
-	if (unlikely(input_size != edata_usize_get(edata)
-	        || input_size > SC_LARGE_MAXCLASS)) {
-		safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
-		    /* true_size */ edata_usize_get(edata), input_size);
-		return true;
-	}
-
-	return false;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
-    prof_info_t *prof_info, bool reset_recent) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-	assert(prof_info != NULL);
-
-	edata_t *edata = NULL;
-	bool     is_slab;
-
-	/* Static check. */
-	if (alloc_ctx == NULL) {
-		edata = emap_edata_lookup(
-		    tsd_tsdn(tsd), &arena_emap_global, ptr);
-		is_slab = edata_slab_get(edata);
-	} else if (unlikely(!(is_slab = alloc_ctx->slab))) {
-		edata = emap_edata_lookup(
-		    tsd_tsdn(tsd), &arena_emap_global, ptr);
-	}
-
-	if (unlikely(!is_slab)) {
-		/* edata must have been initialized at this point. */
-		assert(edata != NULL);
-		size_t usize = (alloc_ctx == NULL)
-		    ? edata_usize_get(edata)
-		    : emap_alloc_ctx_usize_get(alloc_ctx);
-		if (reset_recent
-		    && large_dalloc_safety_checks(edata, ptr, usize)) {
-			prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
-			return;
-		}
-		large_prof_info_get(tsd, edata, prof_info, reset_recent);
-	} else {
-		prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
-		/*
-		 * No need to set other fields in prof_info; they will never be
-		 * accessed if alloc_tctx == PROF_TCTX_SENTINEL.
-		 */
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_tctx_reset(
-    tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-
-	/* Static check. */
-	if (alloc_ctx == NULL) {
-		edata_t *edata = emap_edata_lookup(
-		    tsd_tsdn(tsd), &arena_emap_global, ptr);
-		if (unlikely(!edata_slab_get(edata))) {
-			large_prof_tctx_reset(edata);
-		}
-	} else {
-		if (unlikely(!alloc_ctx->slab)) {
-			edata_t *edata = emap_edata_lookup(
-			    tsd_tsdn(tsd), &arena_emap_global, ptr);
-			large_prof_tctx_reset(edata);
-		}
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-
-	edata_t *edata = emap_edata_lookup(
-	    tsd_tsdn(tsd), &arena_emap_global, ptr);
-	assert(!edata_slab_get(edata));
-
-	large_prof_tctx_reset(edata);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_info_set(
-    tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
-	cassert(config_prof);
-
-	assert(!edata_slab_get(edata));
-	large_prof_info_set(edata, tctx, size);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
-	if (unlikely(tsdn_null(tsdn))) {
-		return;
-	}
-	tsd_t *tsd = tsdn_tsd(tsdn);
-	/*
-	 * We use the ticker_geom_t to avoid having per-arena state in the tsd.
-	 * Instead of having a countdown-until-decay timer running for every
-	 * arena in every thread, we flip a coin once per tick, whose
-	 * probability of coming up heads is 1/nticks; this is effectively the
-	 * operation of the ticker_geom_t.  Each arena has the same chance of a
-	 * coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can
-	 * use a single ticker for all of them.
-	 */
-	ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
-	uint64_t      *prng_state = tsd_prng_statep_get(tsd);
-	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
-	        tsd_reentrancy_level_get(tsd) > 0))) {
-		arena_decay(tsdn, arena, false, false);
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
-	arena_decay_ticks(tsdn, arena, 1);
-}
-
-JEMALLOC_ALWAYS_INLINE void *
-arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
-    bool slab, tcache_t *tcache, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-
-	if (likely(tcache != NULL)) {
-		if (likely(slab)) {
-			assert(sz_can_use_slab(size));
-			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
-			    size, ind, zero, slow_path);
-		} else if (likely(tcache_can_cache_large(tcache, ind))) {
-			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
-			    size, ind, zero, slow_path);
-		}
-		/* (size > tcache_max) case falls through. */
-	}
-
-	return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
-}
-
-JEMALLOC_ALWAYS_INLINE arena_t *
-arena_aalloc(tsdn_t *tsdn, const void *ptr) {
-	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
-	unsigned arena_ind = edata_arena_ind_get(edata);
-	return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_RELAXED);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_salloc(tsdn_t *tsdn, const void *ptr) {
-	assert(ptr != NULL);
-	emap_alloc_ctx_t alloc_ctx;
-	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
-	assert(alloc_ctx.szind != SC_NSIZES);
-
-	return emap_alloc_ctx_usize_get(&alloc_ctx);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
-	/*
-	 * Return 0 if ptr is not within an extent managed by jemalloc.  This
-	 * function has two extra costs relative to isalloc():
-	 * - The rtree calls cannot claim to be dependent lookups, which induces
-	 *   rtree lookup load dependencies.
-	 * - The lookup may fail, so there is an extra branch to check for
-	 *   failure.
-	 */
-
-	emap_full_alloc_ctx_t full_alloc_ctx;
-	bool                  missing = emap_full_alloc_ctx_try_lookup(
-            tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
-	if (missing) {
-		return 0;
-	}
-
-	if (full_alloc_ctx.edata == NULL) {
-		return 0;
-	}
-	assert(edata_state_get(full_alloc_ctx.edata) == extent_state_active);
-	/* Only slab members should be looked up via interior pointers. */
-	assert(edata_addr_get(full_alloc_ctx.edata) == ptr
-	    || edata_slab_get(full_alloc_ctx.edata));
-
-	assert(full_alloc_ctx.szind != SC_NSIZES);
-
-	return edata_usize_get(full_alloc_ctx.edata);
-}
-
-static inline void
-arena_dalloc_large_no_tcache(
-    tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
-	/*
-	 * szind is still needed in this function mainly becuase
-	 * szind < SC_NBINS determines not only if this is a small alloc,
-	 * but also if szind is valid (an inactive extent would have
-	 * szind == SC_NSIZES).
-	 */
-	if (config_prof && unlikely(szind < SC_NBINS)) {
-		arena_dalloc_promoted(tsdn, ptr, NULL, true);
-	} else {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		if (large_dalloc_safety_checks(edata, ptr, usize)) {
-			/* See the comment in isfree. */
-			return;
-		}
-		large_dalloc(tsdn, edata);
-	}
-}
-
-static inline void
-arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
-	assert(ptr != NULL);
-
-	emap_alloc_ctx_t alloc_ctx;
-	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.szind < SC_NSIZES);
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		arena_dalloc_small(tsdn, ptr);
-	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx));
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
-    size_t usize, bool slow_path) {
-	assert(!tsdn_null(tsdn) && tcache != NULL);
-	bool is_sample_promoted = config_prof && szind < SC_NBINS;
-	if (unlikely(is_sample_promoted)) {
-		arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
-	} else {
-		if (tcache_can_cache_large(tcache, szind)) {
-			tcache_dalloc_large(
-			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
-		} else {
-			edata_t *edata = emap_edata_lookup(
-			    tsdn, &arena_emap_global, ptr);
-			if (large_dalloc_safety_checks(edata, ptr, usize)) {
-				/* See the comment in isfree. */
-				return;
-			}
-			large_dalloc(tsdn, edata);
-		}
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
-	if (!config_debug) {
-		return false;
-	}
-	edata_t   *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
-	szind_t    binind = edata_szind_get(edata);
-	div_info_t div_info = arena_binind_div_info[binind];
-	/*
-	 * Calls the internal function bin_slab_regind_impl because the
-	 * safety check does not require a lock.
-	 */
-	size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
-	slab_data_t      *slab_data = edata_slab_data_get(edata);
-	const bin_info_t *bin_info = &bin_infos[binind];
-	assert(edata_nfree_get(edata) < bin_info->nregs);
-	if (unlikely(!bitmap_get(
-	        slab_data->bitmap, &bin_info->bitmap_info, regind))) {
-		safety_check_fail(
-		    "Invalid deallocation detected: the pointer being freed (%p) not "
-		    "currently active, possibly caused by double free bugs.\n",
-		    ptr);
-		return true;
-	}
-	return false;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
-    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-	assert(ptr != NULL);
-
-	if (unlikely(tcache == NULL)) {
-		arena_dalloc_no_tcache(tsdn, ptr);
-		return;
-	}
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (caller_alloc_ctx != NULL) {
-		alloc_ctx = *caller_alloc_ctx;
-	} else {
-		util_assume(tsdn != NULL);
-		emap_alloc_ctx_lookup(
-		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-	}
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.szind < SC_NSIZES);
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-			return;
-		}
-		tcache_dalloc_small(
-		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
-	} else {
-		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
-	}
-}
-
-static inline void
-arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
-	assert(ptr != NULL);
-	assert(size <= SC_LARGE_MAXCLASS);
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (!config_prof || !opt_prof) {
-		/*
-		 * There is no risk of being confused by a promoted sampled
-		 * object, so base szind and slab on the given size.
-		 */
-		szind_t szind = sz_size2index(size);
-		emap_alloc_ctx_init(
-		    &alloc_ctx, szind, (szind < SC_NBINS), size);
-	}
-
-	if ((config_prof && opt_prof) || config_debug) {
-		emap_alloc_ctx_lookup(
-		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
-		assert(alloc_ctx.szind == sz_size2index(size));
-		assert((config_prof && opt_prof)
-		    || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
-
-		if (config_debug) {
-			edata_t *edata = emap_edata_lookup(
-			    tsdn, &arena_emap_global, ptr);
-			assert(alloc_ctx.szind == edata_szind_get(edata));
-			assert(alloc_ctx.slab == edata_slab_get(edata));
-		}
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		arena_dalloc_small(tsdn, ptr);
-	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx));
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
-    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-	assert(ptr != NULL);
-	assert(size <= SC_LARGE_MAXCLASS);
-
-	if (unlikely(tcache == NULL)) {
-		arena_sdalloc_no_tcache(tsdn, ptr, size);
-		return;
-	}
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (config_prof && opt_prof) {
-		if (caller_alloc_ctx == NULL) {
-			/* Uncommon case and should be a static check. */
-			emap_alloc_ctx_lookup(
-			    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-			assert(alloc_ctx.szind == sz_size2index(size));
-			assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
-		} else {
-			alloc_ctx = *caller_alloc_ctx;
-		}
-	} else {
-		/*
-		 * There is no risk of being confused by a promoted sampled
-		 * object, so base szind and slab on the given size.
-		 */
-		alloc_ctx.szind = sz_size2index(size);
-		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
-	}
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		emap_alloc_ctx_init(
-		    &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-			return;
-		}
-		tcache_dalloc_small(
-		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
-	} else {
-		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    sz_s2u(size), slow_path);
-	}
-}
-
-static inline void
-arena_cache_oblivious_randomize(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
-	assert(edata_base_get(edata) == edata_addr_get(edata));
-
-	if (alignment < PAGE) {
-		unsigned lg_range = LG_PAGE
-		    - lg_floor(CACHELINE_CEILING(alignment));
-		size_t r;
-		if (!tsdn_null(tsdn)) {
-			tsd_t *tsd = tsdn_tsd(tsdn);
-			r = (size_t)prng_lg_range_u64(
-			    tsd_prng_statep_get(tsd), lg_range);
-		} else {
-			uint64_t stack_value = (uint64_t)(uintptr_t)&r;
-			r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
-		}
-		uintptr_t random_offset = ((uintptr_t)r)
-		    << (LG_PAGE - lg_range);
-		edata->e_addr = (void *)((byte_t *)edata->e_addr
-		    + random_offset);
-		assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
-		    == edata->e_addr);
-	}
-}
-
-static inline bin_t *
-arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
-	bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]);
-	return shard0 + binshard;
-}
-
-#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */

include/jemalloc/internal/arena_structs.h

@@ -1,114 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_H
-#define JEMALLOC_INTERNAL_ARENA_STRUCTS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_stats.h"
-#include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/bin.h"
-#include "jemalloc/internal/bitmap.h"
-#include "jemalloc/internal/counter.h"
-#include "jemalloc/internal/ecache.h"
-#include "jemalloc/internal/edata_cache.h"
-#include "jemalloc/internal/extent_dss.h"
-#include "jemalloc/internal/jemalloc_internal_types.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/nstime.h"
-#include "jemalloc/internal/pa.h"
-#include "jemalloc/internal/ql.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/ticker.h"
-
-struct arena_s {
-	/*
-	 * Number of threads currently assigned to this arena.  Each thread has
-	 * two distinct assignments, one for application-serving allocation, and
-	 * the other for internal metadata allocation.  Internal metadata must
-	 * not be allocated from arenas explicitly created via the arenas.create
-	 * mallctl, because the arena.<i>.reset mallctl indiscriminately
-	 * discards all allocations for the affected arena.
-	 *
-	 *   0: Application allocation.
-	 *   1: Internal metadata allocation.
-	 *
-	 * Synchronization: atomic.
-	 */
-	atomic_u_t nthreads[2];
-
-	/* Next bin shard for binding new threads. Synchronization: atomic. */
-	atomic_u_t binshard_next;
-
-	/*
-	 * When percpu_arena is enabled, to amortize the cost of reading /
-	 * updating the current CPU id, track the most recent thread accessing
-	 * this arena, and only read CPU if there is a mismatch.
-	 */
-	tsdn_t *last_thd;
-
-	/* Synchronization: internal. */
-	arena_stats_t stats;
-
-	/*
-	 * List of cache_bin_array_descriptors for extant threads associated
-	 * with this arena.  Stats from these are merged incrementally, and at
-	 * exit if opt_stats_print is enabled.
-	 *
-	 * Synchronization: cache_bin_array_descriptor_ql_mtx.
-	 */
-	ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
-	malloc_mutex_t cache_bin_array_descriptor_ql_mtx;
-
-	/*
-	 * Represents a dss_prec_t, but atomically.
-	 *
-	 * Synchronization: atomic.
-	 */
-	atomic_u_t dss_prec;
-
-	/*
-	 * Extant large allocations.
-	 *
-	 * Synchronization: large_mtx.
-	 */
-	edata_list_active_t large;
-	/* Synchronizes all large allocation/update/deallocation. */
-	malloc_mutex_t large_mtx;
-
-	/* The page-level allocator shard this arena uses. */
-	pa_shard_t pa_shard;
-
-	/*
-	 * A cached copy of base->ind.  This can get accessed on hot paths;
-	 * looking it up in base requires an extra pointer hop / cache miss.
-	 */
-	unsigned ind;
-
-	/*
-	 * Base allocator, from which arena metadata are allocated.
-	 *
-	 * Synchronization: internal.
-	 */
-	base_t *base;
-	/* Used to determine uptime.  Read-only after initialization. */
-	nstime_t create_time;
-
-	/* The name of the arena. */
-	char name[ARENA_NAME_LEN];
-
-	/*
-	 * The arena is allocated alongside its bins; really this is a
-	 * dynamically sized array determined by the binshard settings.
-	 * Enforcing cacheline-alignment to minimize the number of cachelines
-	 * touched on the hot paths.
-	 */
-	JEMALLOC_WARN_ON_USAGE(
-	    "Do not use this field directly. "
-	    "Use `arena_get_bin` instead.")
-	JEMALLOC_ALIGNED(CACHELINE)
-#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
-	bin_t all_bins[];
-#else
-	bin_t all_bins[0];
-#endif
-};
-
-#endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */

include/jemalloc/internal/arena_types.h

@@ -1,60 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H
-#define JEMALLOC_INTERNAL_ARENA_TYPES_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/sc.h"
-
-/* Default decay times in milliseconds. */
-#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
-#define MUZZY_DECAY_MS_DEFAULT (0)
-/* Number of event ticks between time checks. */
-#define ARENA_DECAY_NTICKS_PER_UPDATE 1000
-/* Maximum length of the arena name. */
-#define ARENA_NAME_LEN 32
-
-typedef struct arena_s arena_t;
-
-typedef enum {
-	percpu_arena_mode_names_base = 0, /* Used for options processing. */
-
-	/*
-	 * *_uninit are used only during bootstrapping, and must correspond
-	 * to initialized variant plus percpu_arena_mode_enabled_base.
-	 */
-	percpu_arena_uninit = 0,
-	per_phycpu_arena_uninit = 1,
-
-	/* All non-disabled modes must come after percpu_arena_disabled. */
-	percpu_arena_disabled = 2,
-
-	percpu_arena_mode_names_limit = 3, /* Used for options processing. */
-	percpu_arena_mode_enabled_base = 3,
-
-	percpu_arena = 3,
-	per_phycpu_arena = 4 /* Hyper threads share arena. */
-} percpu_arena_mode_t;
-
-#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
-#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
-
-/*
- * When allocation_size >= oversize_threshold, use the dedicated huge arena
- * (unless have explicitly spicified arena index).  0 disables the feature.
- */
-#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
-
-struct arena_config_s {
-	/* extent hooks to be used for the arena */
-	extent_hooks_t *extent_hooks;
-
-	/*
-	 * Use extent hooks for metadata (base) allocations when true.
-	 */
-	bool metadata_use_hooks;
-};
-
-typedef struct arena_config_s arena_config_t;
-
-extern const arena_config_t arena_config_default;
-
-#endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */

include/jemalloc/internal/arenas_management.h

@@ -1,7 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
 #define JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
 
-#include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/tsd_types.h"
 

include/jemalloc/internal/background_thread.h

@@ -0,0 +1,105 @@
+#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_H
+#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/base.h"
+#include "jemalloc/internal/mutex.h"
+
+#if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
+#	define JEMALLOC_PTHREAD_CREATE_WRAPPER
+#endif
+
+#define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX
+#define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT
+#define DEFAULT_NUM_BACKGROUND_THREAD 4
+
+typedef enum {
+	background_thread_stopped,
+	background_thread_started,
+	/* Thread waits on the global lock when paused (for arena_reset). */
+	background_thread_paused,
+} background_thread_state_t;
+
+struct background_thread_info_s {
+#ifdef JEMALLOC_BACKGROUND_THREAD
+	/* Background thread is pthread specific. */
+	pthread_t      thread;
+	pthread_cond_t cond;
+#endif
+	malloc_mutex_t            mtx;
+	background_thread_state_t state;
+	/* When true, it means no wakeup scheduled. */
+	atomic_b_t indefinite_sleep;
+	/* Next scheduled wakeup time (absolute time in ns). */
+	nstime_t next_wakeup;
+	/*
+	 *  Since the last background thread run, newly added number of pages
+	 *  that need to be purged by the next wakeup.  This is adjusted on
+	 *  epoch advance, and is used to determine whether we should signal the
+	 *  background thread to wake up earlier.
+	 */
+	size_t npages_to_purge_new;
+	/* Stats: total number of runs since started. */
+	uint64_t tot_n_runs;
+	/* Stats: total sleep time since started. */
+	nstime_t tot_sleep_time;
+};
+typedef struct background_thread_info_s background_thread_info_t;
+
+struct background_thread_stats_s {
+	size_t            num_threads;
+	uint64_t          num_runs;
+	nstime_t          run_interval;
+	mutex_prof_data_t max_counter_per_bg_thd;
+};
+typedef struct background_thread_stats_s background_thread_stats_t;
+
+extern bool                      opt_background_thread;
+extern size_t                    opt_max_background_threads;
+extern malloc_mutex_t            background_thread_lock;
+extern atomic_b_t                background_thread_enabled_state;
+extern size_t                    n_background_threads;
+extern size_t                    max_background_threads;
+extern background_thread_info_t *background_thread_info;
+
+bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
+bool background_threads_enable(tsd_t *tsd);
+bool background_threads_disable(tsd_t *tsd);
+bool background_thread_is_started(background_thread_info_t *info);
+void background_thread_wakeup_early(
+    background_thread_info_t *info, nstime_t *remaining_sleep);
+void background_thread_prefork0(tsdn_t *tsdn);
+void background_thread_prefork1(tsdn_t *tsdn);
+void background_thread_postfork_parent(tsdn_t *tsdn);
+void background_thread_postfork_child(tsdn_t *tsdn);
+bool background_thread_stats_read(
+    tsdn_t *tsdn, background_thread_stats_t *stats);
+void background_thread_ctl_init(tsdn_t *tsdn);
+
+/*
+ * Arena-reset lifecycle bracket (owned by the bg-thread module so the
+ * started<->paused state machine is mutated only inside the module).  begin()
+ * acquires the global background_thread_lock and RETURNS WITH IT HELD;
+ * finish() releases it.  The lock intentionally spans the whole arena-reset
+ * body across the two calls.  Pass arena_ind (not an arena_t *): the destroy
+ * path frees the arena before calling finish().
+ */
+void background_thread_arena_reset_begin(tsd_t *tsd, unsigned arena_ind);
+void background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind);
+
+/*
+ * Serialize a rare admin operation against the background thread by holding the
+ * per-arena info mutex (have_background_thread-gated).  Used by
+ * arena_set_extent_hooks to fence pa_shard_disable_hpa.
+ */
+void background_thread_serialize_lock(tsd_t *tsd, unsigned arena_ind);
+void background_thread_serialize_unlock(tsd_t *tsd, unsigned arena_ind);
+
+#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
+extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
+    void *(*)(void *), void *__restrict);
+#endif
+bool background_thread_boot0(void);
+bool background_thread_boot1(tsdn_t *tsdn, base_t *base);
+
+#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_H */

include/jemalloc/internal/background_thread_externs.h

@@ -1,38 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
-#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/background_thread_structs.h"
-#include "jemalloc/internal/base.h"
-#include "jemalloc/internal/mutex.h"
-
-extern bool                      opt_background_thread;
-extern size_t                    opt_max_background_threads;
-extern malloc_mutex_t            background_thread_lock;
-extern atomic_b_t                background_thread_enabled_state;
-extern size_t                    n_background_threads;
-extern size_t                    max_background_threads;
-extern background_thread_info_t *background_thread_info;
-
-bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
-bool background_threads_enable(tsd_t *tsd);
-bool background_threads_disable(tsd_t *tsd);
-bool background_thread_is_started(background_thread_info_t *info);
-void background_thread_wakeup_early(
-    background_thread_info_t *info, nstime_t *remaining_sleep);
-void background_thread_prefork0(tsdn_t *tsdn);
-void background_thread_prefork1(tsdn_t *tsdn);
-void background_thread_postfork_parent(tsdn_t *tsdn);
-void background_thread_postfork_child(tsdn_t *tsdn);
-bool background_thread_stats_read(
-    tsdn_t *tsdn, background_thread_stats_t *stats);
-void background_thread_ctl_init(tsdn_t *tsdn);
-
-#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
-extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
-    void *(*)(void *), void *__restrict);
-#endif
-bool background_thread_boot0(void);
-bool background_thread_boot1(tsdn_t *tsdn, base_t *base);
-
-#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H */

include/jemalloc/internal/background_thread_inlines.h

@@ -2,9 +2,9 @@
 #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_inlines_a.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/background_thread_externs.h"
+#include "jemalloc/internal/background_thread.h"
 
 JEMALLOC_ALWAYS_INLINE bool
 background_thread_enabled(void) {
@@ -22,12 +22,6 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) {
 	background_thread_enabled_set_impl(state);
 }
 
-JEMALLOC_ALWAYS_INLINE background_thread_info_t *
-arena_background_thread_info_get(arena_t *arena) {
-	unsigned arena_ind = arena_ind_get(arena);
-	return &background_thread_info[arena_ind % max_background_threads];
-}
-
 JEMALLOC_ALWAYS_INLINE background_thread_info_t *
 background_thread_info_get(size_t ind) {
 	return &background_thread_info[ind % max_background_threads];

include/jemalloc/internal/background_thread_structs.h

@@ -1,69 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
-#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/mutex.h"
-
-/* This file really combines "structs" and "types", but only transitionally. */
-
-#if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
-#	define JEMALLOC_PTHREAD_CREATE_WRAPPER
-#endif
-
-#define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX
-#define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT
-#define DEFAULT_NUM_BACKGROUND_THREAD 4
-
-/*
- * These exist only as a transitional state.  Eventually, deferral should be
- * part of the PAI, and each implementation can indicate wait times with more
- * specificity.
- */
-#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_UNINITIALIZED (-2)
-#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000
-
-#define BACKGROUND_THREAD_DEFERRED_MIN UINT64_C(0)
-#define BACKGROUND_THREAD_DEFERRED_MAX UINT64_MAX
-
-typedef enum {
-	background_thread_stopped,
-	background_thread_started,
-	/* Thread waits on the global lock when paused (for arena_reset). */
-	background_thread_paused,
-} background_thread_state_t;
-
-struct background_thread_info_s {
-#ifdef JEMALLOC_BACKGROUND_THREAD
-	/* Background thread is pthread specific. */
-	pthread_t      thread;
-	pthread_cond_t cond;
-#endif
-	malloc_mutex_t            mtx;
-	background_thread_state_t state;
-	/* When true, it means no wakeup scheduled. */
-	atomic_b_t indefinite_sleep;
-	/* Next scheduled wakeup time (absolute time in ns). */
-	nstime_t next_wakeup;
-	/*
-	 *  Since the last background thread run, newly added number of pages
-	 *  that need to be purged by the next wakeup.  This is adjusted on
-	 *  epoch advance, and is used to determine whether we should signal the
-	 *  background thread to wake up earlier.
-	 */
-	size_t npages_to_purge_new;
-	/* Stats: total number of runs since started. */
-	uint64_t tot_n_runs;
-	/* Stats: total sleep time since started. */
-	nstime_t tot_sleep_time;
-};
-typedef struct background_thread_info_s background_thread_info_t;
-
-struct background_thread_stats_s {
-	size_t            num_threads;
-	uint64_t          num_runs;
-	nstime_t          run_interval;
-	mutex_prof_data_t max_counter_per_bg_thd;
-};
-typedef struct background_thread_stats_s background_thread_stats_t;
-
-#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H */

include/jemalloc/internal/base.h

@@ -106,7 +106,6 @@ base_t *base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
     bool metadata_use_hooks);
 void    base_delete(tsdn_t *tsdn, base_t *base);
 ehooks_t       *base_ehooks_get(base_t *base);
-ehooks_t       *base_ehooks_get_for_metadata(base_t *base);
 extent_hooks_t *base_extent_hooks_set(
     base_t *base, extent_hooks_t *extent_hooks);
 void    *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);

include/jemalloc/internal/bin.h

@@ -4,11 +4,17 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/bin_info.h"
 #include "jemalloc/internal/bin_stats.h"
-#include "jemalloc/internal/bin_types.h"
+#include "jemalloc/internal/div.h"
 #include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/sc.h"
 
+/* Forward decl; only used as a pointer type in bin_choose. */
+typedef struct arena_s arena_t;
+
+#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH)
+#define N_BIN_SHARDS_DEFAULT 1
+
 /*
  * A bin contains a set of extents that are currently being used for slab
  * allocations.
@@ -63,23 +69,13 @@ void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin);
 void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
 
 /* Slab region allocation. */
-void *bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info);
 void  bin_slab_reg_alloc_batch(
      edata_t *slab, const bin_info_t *bin_info, unsigned cnt, void **ptrs);
 
 /* Slab list management. */
-void     bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab);
-void     bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab);
-edata_t *bin_slabs_nonfull_tryget(bin_t *bin);
-void     bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab);
 void     bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab);
 
-/* Slab association / demotion. */
-void bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin);
-void bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
-
 /* Deallocation helpers (called under bin lock). */
-void bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_empty(
     tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_nonempty(
@@ -102,30 +98,11 @@ void *bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin);
 bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     unsigned *binshard_p);
 
-/* Stats. */
-static inline void
-bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) {
-	malloc_mutex_lock(tsdn, &bin->lock);
-	bin->stats.nrequests += n;
-	malloc_mutex_unlock(tsdn, &bin->lock);
-}
-
-static inline void
-bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
-	malloc_mutex_lock(tsdn, &bin->lock);
-	malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
-	bin_stats_t *stats = &dst_bin_stats->stats_data;
-	stats->nmalloc += bin->stats.nmalloc;
-	stats->ndalloc += bin->stats.ndalloc;
-	stats->nrequests += bin->stats.nrequests;
-	stats->curregs += bin->stats.curregs;
-	stats->nfills += bin->stats.nfills;
-	stats->nflushes += bin->stats.nflushes;
-	stats->nslabs += bin->stats.nslabs;
-	stats->reslabs += bin->stats.reslabs;
-	stats->curslabs += bin->stats.curslabs;
-	stats->nonfull_slabs += bin->stats.nonfull_slabs;
-	malloc_mutex_unlock(tsdn, &bin->lock);
-}
+typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
+struct bin_dalloc_locked_info_s {
+	div_info_t div_info;
+	uint32_t   nregs;
+	uint64_t   ndalloc;
+};
 
 #endif /* JEMALLOC_INTERNAL_BIN_H */

include/jemalloc/internal/bin_inlines.h

@@ -7,20 +7,33 @@
 #include "jemalloc/internal/bitmap.h"
 #include "jemalloc/internal/div.h"
 #include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/mutex.h"
 
-/*
- * The dalloc bin info contains just the information that the common paths need
- * during tcache flushes.  By force-inlining these paths, and using local copies
- * of data (so that the compiler knows it's constant), we avoid a whole bunch of
- * redundant loads and stores by leaving this information in registers.
- */
-typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
-struct bin_dalloc_locked_info_s {
-	div_info_t div_info;
-	uint32_t   nregs;
-	uint64_t   ndalloc;
-};
+/* Stats. */
+static inline void
+bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) {
+	malloc_mutex_lock(tsdn, &bin->lock);
+	bin->stats.nrequests += n;
+	malloc_mutex_unlock(tsdn, &bin->lock);
+}
+
+static inline void
+bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
+	malloc_mutex_lock(tsdn, &bin->lock);
+	malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
+	bin_stats_t *stats = &dst_bin_stats->stats_data;
+	stats->nmalloc += bin->stats.nmalloc;
+	stats->ndalloc += bin->stats.ndalloc;
+	stats->nrequests += bin->stats.nrequests;
+	stats->curregs += bin->stats.curregs;
+	stats->nfills += bin->stats.nfills;
+	stats->nflushes += bin->stats.nflushes;
+	stats->nslabs += bin->stats.nslabs;
+	stats->reslabs += bin->stats.reslabs;
+	stats->curslabs += bin->stats.curslabs;
+	stats->nonfull_slabs += bin->stats.nonfull_slabs;
+	malloc_mutex_unlock(tsdn, &bin->lock);
+}
 
 /* Find the region index of a pointer within a slab. */
 JEMALLOC_ALWAYS_INLINE size_t
@@ -52,14 +65,6 @@ bin_slab_regind(const bin_dalloc_locked_info_t *info, szind_t binind,
 	return regind;
 }
 
-JEMALLOC_ALWAYS_INLINE void
-bin_dalloc_locked_begin(
-    bin_dalloc_locked_info_t *info, szind_t binind) {
-	info->div_info = arena_binind_div_info[binind];
-	info->nregs = bin_infos[binind].nregs;
-	info->ndalloc = 0;
-}
-
 /*
  * Does the deallocation work associated with freeing a single pointer (a
  * "step") in between a bin_dalloc_locked begin and end call.

include/jemalloc/internal/bin_types.h

@@ -1,21 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H
-#define JEMALLOC_INTERNAL_BIN_TYPES_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/sc.h"
-
-#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH)
-#define N_BIN_SHARDS_DEFAULT 1
-
-/* Used in TSD static initializer only. Real init in arena_bind(). */
-#define TSD_BINSHARDS_ZERO_INITIALIZER                                         \
-	{                                                                      \
-		{ UINT8_MAX }                                                  \
-	}
-
-typedef struct tsd_binshards_s tsd_binshards_t;
-struct tsd_binshards_s {
-	uint8_t binshard[SC_NBINS];
-};
-
-#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */

include/jemalloc/internal/bitmap.h

@@ -176,9 +176,7 @@ typedef struct bitmap_info_s {
 #endif /* BITMAP_USE_TREE */
 } bitmap_info_t;
 
-void   bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
 void   bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
-size_t bitmap_size(const bitmap_info_t *binfo);
 
 static inline bool
 bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {

include/jemalloc/internal/buf_writer.h

@@ -25,7 +25,6 @@ typedef struct {
 
 bool       buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer,
           write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len);
-void       buf_writer_flush(buf_writer_t *buf_writer);
 write_cb_t buf_writer_cb;
 void       buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer);
 

include/jemalloc/internal/ckh.h

@@ -93,10 +93,4 @@ bool ckh_remove(
     tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data);
 bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
 
-/* Some useful hash and comparison functions for strings and pointers. */
-void ckh_string_hash(const void *key, size_t r_hash[2]);
-bool ckh_string_keycomp(const void *k1, const void *k2);
-void ckh_pointer_hash(const void *key, size_t r_hash[2]);
-bool ckh_pointer_keycomp(const void *k1, const void *k2);
-
 #endif /* JEMALLOC_INTERNAL_CKH_H */

include/jemalloc/internal/conf.h

@@ -7,9 +7,9 @@ void malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
     char readlink_buf[PATH_MAX + 1]);
 void malloc_abort_invalid_conf(void);
 
-#ifdef JEMALLOC_JET
 extern bool had_conf_error;
 
+#ifdef JEMALLOC_JET
 bool conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
     char const **v_p, size_t *vlen_p);
 void conf_error(

include/jemalloc/internal/ctl.h

@@ -3,7 +3,7 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/arena_stats.h"
-#include "jemalloc/internal/background_thread_structs.h"
+#include "jemalloc/internal/background_thread.h"
 #include "jemalloc/internal/bin_stats.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/malloc_io.h"

include/jemalloc/internal/deferral.h

@@ -0,0 +1,30 @@
+#ifndef JEMALLOC_INTERNAL_DEFERRAL_H
+#define JEMALLOC_INTERNAL_DEFERRAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+/*
+ * The deferred-work contract shared by the page allocators that PRODUCE
+ * deferred work (PAC decay-purge, HPA purge/hugify) and the drivers that
+ * CONSUME it (the background thread, or the application-inline fallback).
+ *
+ * There is deliberately no polymorphic interface here: with only two
+ * page allocators, a vtable's indirect calls are needless and inefficient.
+ * Dispatch is direct pac_*()/hpa_*(), so the contract is a convention
+ * documented here, not an enforced type.  The only shared code it needs is the
+ * constants below; each allocator keeps its own policy definition (functions
+ * and constants) in its own header.
+ *
+ * The convention every page allocator follows:
+ *   - <alloc>_time_until_deferred_work(): nanoseconds until its next
+ *     deferred work is due.  DEFERRED_WORK_MIN = "due now",
+ *     DEFERRED_WORK_MAX = "nothing pending"; anything between is a real
+ *     ns deadline.
+ *   - <alloc>_do_deferred_work(): perform whatever work is due now.
+ * The pa layer reports the soonest deadline across its allocators; the
+ * background thread sleeps until then, or indefinitely on DEFERRED_WORK_MAX.
+ */
+#define DEFERRED_WORK_MIN UINT64_C(0)
+#define DEFERRED_WORK_MAX UINT64_MAX
+
+#endif /* JEMALLOC_INTERNAL_DEFERRAL_H */

include/jemalloc/internal/edata.h

@@ -8,13 +8,16 @@
 #include "jemalloc/internal/hpdata.h"
 #include "jemalloc/internal/nstime.h"
 #include "jemalloc/internal/ph.h"
-#include "jemalloc/internal/prof_types.h"
 #include "jemalloc/internal/ql.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/slab_data.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/typed_list.h"
 
+/* Opaque to edata; only stored as pointers in e_prof_info_t. */
+typedef struct prof_tctx_s   prof_tctx_t;
+typedef struct prof_recent_s prof_recent_t;
+
 /*
  * sizeof(edata_t) is 128 bytes on 64-bit architectures.  Ensure the alignment
  * to free up the low bits in the rtree leaf.

include/jemalloc/internal/ehooks.h

@@ -57,13 +57,6 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
 bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
 #endif
 bool ehooks_default_split_impl(void);
-/*
- * Merge is the only default extent hook we declare -- see the comment in
- * ehooks_merge.
- */
-bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a,
-    size_t size_a, void *addr_b, size_t size_b, bool committed,
-    unsigned arena_ind);
 bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b);
 void ehooks_default_zero_impl(void *addr, size_t size);
 void ehooks_default_guard_impl(void *guard1, void *guard2);

include/jemalloc/internal/emitter.h

@@ -58,6 +58,11 @@ struct emitter_col_s {
 		ssize_t     ssize_val;
 		const char *str_val;
 	};
+	/*
+	 * Optional JSON key.  When set, emitter_row() emits this column as a
+	 * "json_key": value pair in JSON mode; columns left NULL are table-only.
+	 */
+	const char *json_key;
 
 	/* Filled in by initialization. */
 	ql_elm(emitter_col_t) link;
@@ -79,6 +84,8 @@ struct emitter_s {
 	bool item_at_depth;
 	/* True if we emitted a key and will emit corresponding value next. */
 	bool emitted_key;
+	/* Gap-collapsing state for the emitter_table_sparse_row() bracket. */
+	bool table_sparse_in_gap;
 };
 
 static inline bool
@@ -270,6 +277,7 @@ emitter_init(emitter_t *emitter, emitter_output_t emitter_output,
 	emitter->item_at_depth = false;
 	emitter->emitted_key = false;
 	emitter->nesting_depth = 0;
+	emitter->table_sparse_in_gap = false;
 }
 
 /******************************************************************************/
@@ -447,6 +455,46 @@ static inline void
 emitter_col_init(emitter_col_t *col, emitter_row_t *row) {
 	ql_elm_new(col, link);
 	ql_tail_insert(&row->cols, col, link);
+	col->json_key = NULL;
+}
+
+/*
+ * Sparse table rows.  Some tables have one row per size class but most rows
+ * are all-zero; in table output those runs are collapsed to a single "---"
+ * separator.  Bracket such a table with emitter_table_sparse_begin() /
+ * _end() and emit each row with emitter_table_sparse_row(), passing whether
+ * that row is a gap (all-zero).  The emitter prints the separator on a
+ * gap->non-gap transition and at a trailing gap, and suppresses the gap rows
+ * themselves; callers no longer track gap state.  Table output only -- a
+ * no-op in JSON, where callers emit every row's object separately.
+ */
+static inline void
+emitter_table_sparse_begin(emitter_t *emitter) {
+	emitter->table_sparse_in_gap = false;
+}
+
+static inline void
+emitter_table_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) {
+	if (emitter->output != emitter_output_table) {
+		return;
+	}
+	bool prev_in_gap = emitter->table_sparse_in_gap;
+	emitter->table_sparse_in_gap = is_gap;
+	if (prev_in_gap && !is_gap) {
+		emitter_printf(emitter, "                     ---\n");
+	}
+	if (!is_gap) {
+		emitter_table_row(emitter, row);
+	}
+}
+
+static inline void
+emitter_table_sparse_end(emitter_t *emitter) {
+	if (emitter->output == emitter_output_table
+	    && emitter->table_sparse_in_gap) {
+		emitter_printf(emitter, "                     ---\n");
+	}
+	emitter->table_sparse_in_gap = false;
 }
 
 /******************************************************************************/
@@ -499,6 +547,142 @@ emitter_dict_end(emitter_t *emitter) {
 	}
 }
 
+/* Internal: emit a row's json-keyed columns as JSON kvs, in column order. */
+static inline void
+emitter_row_json(emitter_t *emitter, emitter_row_t *row) {
+	emitter_col_t *col;
+	ql_foreach (col, &row->cols, link) {
+		if (col->json_key != NULL) {
+			emitter_json_kv(emitter, col->json_key, col->type,
+			    (const void *)&col->bool_val);
+		}
+	}
+}
+
+/*
+ * Emit a row of columns to whichever format is active.  In table mode, prints
+ * the aligned table row (see emitter_table_row).  In JSON mode, emits each
+ * column that has a non-NULL json_key as a "json_key": value pair, in column
+ * order (columns without a json_key are table-only).  This lets a caller
+ * describe a row once and render it to either format.  For JSON objects (one
+ * per row), wrap the call in emitter_json_object_begin()/_end().
+ */
+static inline void
+emitter_row(emitter_t *emitter, emitter_row_t *row) {
+	emitter_table_row(emitter, row);
+	if (emitter_outputs_json(emitter)) {
+		emitter_row_json(emitter, row);
+	}
+}
+
+/*
+ * Sparse counterpart of emitter_row() for per-size-class tables: the table
+ * side collapses all-zero ("gap") rows (see emitter_table_sparse_row), while
+ * the JSON side still emits every row's json-keyed columns.  Bracket the loop
+ * with emitter_table_sparse_begin()/_end(), and wrap each row's JSON in
+ * emitter_json_object_begin()/_end() when the rows form an array of objects.
+ */
+static inline void
+emitter_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) {
+	emitter_table_sparse_row(emitter, row, is_gap);
+	if (emitter_outputs_json(emitter)) {
+		emitter_row_json(emitter, row);
+	}
+}
+
+/*
+ * Descriptor-driven column tables.  A descriptor array defines table and JSON
+ * order; columns with a NULL json_key are table-only.  The getter lets one
+ * engine serve different row types while keeping derived values (rates,
+ * utilization, and so on) with the table-specific code.  A column is active
+ * when all its required_flags are present in the active_flags supplied by the
+ * caller; the emitter assigns no domain-specific meaning to those bits.
+ */
+typedef struct emitter_col_desc_s emitter_col_desc_t;
+struct emitter_col_desc_s {
+	const char       *json_key;
+	const char       *table_label;
+	emitter_justify_t justify;
+	int               width;
+	emitter_type_t    type;
+	unsigned          required_flags;
+	void            (*get)(const void *row, emitter_col_t *col);
+};
+
+static inline bool
+emitter_col_desc_active(const emitter_col_desc_t *desc,
+    unsigned active_flags) {
+	return (desc->required_flags & ~active_flags) == 0;
+}
+
+static inline void
+emitter_col_table_build(const emitter_col_desc_t *descs, size_t ndescs,
+    unsigned active_flags, emitter_row_t *row, emitter_col_t *cols,
+    emitter_row_t *header_row, emitter_col_t *header_cols) {
+	emitter_row_init(row);
+	emitter_row_init(header_row);
+	for (size_t i = 0; i < ndescs; i++) {
+		if (!emitter_col_desc_active(&descs[i], active_flags)) {
+			continue;
+		}
+
+		emitter_col_init(&cols[i], row);
+		cols[i].justify = descs[i].justify;
+		cols[i].width = descs[i].width;
+		cols[i].type = descs[i].type;
+
+		emitter_col_init(&header_cols[i], header_row);
+		header_cols[i].justify = descs[i].justify;
+		header_cols[i].width = descs[i].width;
+		header_cols[i].type = emitter_type_title;
+		header_cols[i].str_val = descs[i].table_label != NULL
+		    ? descs[i].table_label : descs[i].json_key;
+	}
+}
+
+static inline void
+emitter_col_table_header(emitter_t *emitter, emitter_row_t *header_row,
+    emitter_col_t *first_header_col, const char *table_prefix,
+    const char *json_key) {
+	first_header_col->width -= (int)strlen(table_prefix);
+	emitter_table_printf(emitter, "%s", table_prefix);
+	emitter_table_row(emitter, header_row);
+	emitter_json_array_kv_begin(emitter, json_key);
+}
+
+static inline void
+emitter_col_table_fill(const emitter_col_desc_t *descs, size_t ndescs,
+    unsigned active_flags, emitter_col_t *cols, const void *row) {
+	for (size_t i = 0; i < ndescs; i++) {
+		if (!emitter_col_desc_active(&descs[i], active_flags)) {
+			continue;
+		}
+		cols[i].type = descs[i].type;
+		descs[i].get(row, &cols[i]);
+	}
+}
+
+static inline void
+emitter_col_table_emit_json(emitter_t *emitter,
+    const emitter_col_desc_t *descs, size_t ndescs, unsigned active_flags,
+    emitter_col_t *cols) {
+	if (!emitter_outputs_json(emitter)) {
+		return;
+	}
+	for (size_t i = 0; i < ndescs; i++) {
+		const emitter_col_desc_t *desc = &descs[i];
+		if (!emitter_col_desc_active(desc, active_flags)) {
+			continue;
+		}
+		if (desc->json_key != NULL) {
+			emitter_type_t json_type = cols[i].type == emitter_type_title
+			    ? emitter_type_string : cols[i].type;
+			emitter_json_kv(emitter, desc->json_key, json_type,
+			    (const void *)&cols[i].bool_val);
+		}
+	}
+}
+
 static inline void
 emitter_begin(emitter_t *emitter) {
 	if (emitter_outputs_json(emitter)) {

include/jemalloc/internal/extent_dss.h

@@ -2,9 +2,11 @@
 #define JEMALLOC_INTERNAL_EXTENT_DSS_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_types.h"
 #include "jemalloc/internal/tsd_types.h"
 
+/* Forward decl; arena.h includes us, so we can't include arena.h back. */
+typedef struct arena_s arena_t;
+
 typedef enum {
 	dss_prec_disabled = 0,
 	dss_prec_primary = 1,

include/jemalloc/internal/hpa.h

@@ -13,6 +13,14 @@
 #include "jemalloc/internal/psset.h"
 #include "jemalloc/internal/sec.h"
 
+/*
+ * HPA-specific deferral interval bounds (currently unused).  Deferral tuning
+ * is per-allocator policy, so it lives in the HPA header; see deferral.h for
+ * the shared deferred-work contract.
+ */
+#define HPA_INTERVAL_MAX_UNINITIALIZED (-2)
+#define HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000
+
 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
 struct hpa_shard_nonderived_stats_s {
 	/*
@@ -190,7 +198,6 @@ struct hpa_shard_s {
 	nstime_t last_time_work_attempted;
 };
 
-bool hpa_hugepage_size_exceeds_limit(void);
 /*
  * Whether or not the HPA can be used given the current configuration.  This
  * is not necessarily a guarantee that it backs its allocations by hugepages,
@@ -204,11 +211,6 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
 edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
     size_t alignment, bool zero, bool guarded, bool frequent_reuse,
     bool *deferred_work_generated);
-bool hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool zero,
-    bool *deferred_work_generated);
-bool hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool *deferred_work_generated);
 void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
     bool *deferred_work_generated);
 uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);

include/jemalloc/internal/hpa_opts.h

@@ -108,11 +108,6 @@ struct hpa_shard_opts_s {
 	 */
 	uint64_t min_purge_interval_ms;
 
-	/*
-	 * Maximum number of hugepages to purge on each purging attempt.
-	 */
-	ssize_t experimental_max_purge_nhp;
-
 	/*
 	 * Minimum number of inactive bytes needed for a non-empty page to be
 	 * considered purgable.
@@ -176,8 +171,6 @@ struct hpa_shard_opts_s {
 	false,								\
 	/* min_purge_interval_ms */					\
 	5 * 1000,							\
-	/* experimental_max_purge_nhp */				\
-	-1,      							\
 	/* size_t purge_threshold */					\
 	PAGE,								\
 	/* min_purge_delay_ms */             				\

include/jemalloc/internal/hpdata.h

@@ -425,11 +425,6 @@ hpdata_full(const hpdata_t *hpdata) {
 
 void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 
-/*
- * Given an hpdata which can serve an allocation request, pick and reserve an
- * offset within that allocation.
- */
-void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
 /*

include/jemalloc/internal/inspect.h

@@ -11,8 +11,7 @@
  */
 
 /*
- * The following two structs are for experimental purposes. See
- * experimental_utilization_query_ctl and
+ * The following struct is for experimental purposes. See
  * experimental_utilization_batch_query_ctl in src/ctl.c.
  */
 typedef struct inspect_extent_util_stats_s inspect_extent_util_stats_t;
@@ -22,22 +21,7 @@ struct inspect_extent_util_stats_s {
 	size_t size;
 };
 
-typedef struct inspect_extent_util_stats_verbose_s
-    inspect_extent_util_stats_verbose_t;
-
-struct inspect_extent_util_stats_verbose_s {
-	void  *slabcur_addr;
-	size_t nfree;
-	size_t nregs;
-	size_t size;
-	size_t bin_nfree;
-	size_t bin_nregs;
-};
-
 void inspect_extent_util_stats_get(
     tsdn_t *tsdn, const void *ptr, size_t *nfree, size_t *nregs, size_t *size);
-void inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
-    size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree,
-    size_t *bin_nregs, void **slabcur_addr);
 
 #endif /* JEMALLOC_INTERNAL_INSPECT_H */

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -97,6 +97,9 @@
 /* Defined if pthread_get_name_np(3) is available. */
 #undef JEMALLOC_HAVE_PTHREAD_GET_NAME_NP
 
+/* Defined if pthread_condattr_setclock(..., CLOCK_MONOTONIC) is available. */
+#undef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
@@ -172,9 +175,6 @@
 /* Use gcc intrinsics for profile backtracing if defined. */
 #undef JEMALLOC_PROF_GCC
 
-/* Use frame pointer for profile backtracing if defined. Linux only. */
-#undef JEMALLOC_PROF_FRAME_POINTER
-
 /* JEMALLOC_PAGEID enabled page id */
 #undef JEMALLOC_PAGEID
 
@@ -468,6 +468,13 @@
 /* Are C++ exceptions enabled? */
 #undef JEMALLOC_HAVE_CXX_EXCEPTIONS
 
+/*
+ * If defined, throwing operator new aborts on OOM (with size logged) instead
+ * of throwing std::bad_alloc. Nothrow new still returns null. With LTO this
+ * lets the compiler prove operator new is no-throw and elide exception cleanup.
+ */
+#undef JEMALLOC_INFALLIBLE_NEW
+
 /* Performs additional size checks when defined. */
 #undef JEMALLOC_OPT_SIZE_CHECKS
 

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -1,7 +1,6 @@
 #ifndef JEMALLOC_INTERNAL_EXTERNS_H
 #define JEMALLOC_INTERNAL_EXTERNS_H
 
-#include "jemalloc/internal/arena_types.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/fxp.h"
 #include "jemalloc/internal/hpa_opts.h"
@@ -22,6 +21,7 @@ extern bool             opt_confirm_conf;
 extern bool             opt_hpa;
 extern hpa_shard_opts_t opt_hpa_opts;
 extern sec_opts_t       opt_hpa_sec_opts;
+extern sec_opts_t       opt_pac_sec_opts;
 
 extern const char *opt_junk;
 extern bool        opt_junk_alloc;
@@ -31,7 +31,6 @@ extern void (*JET_MUTABLE junk_alloc_callback)(void *ptr, size_t size);
 extern void (*JET_MUTABLE invalid_conf_abort)(void);
 extern bool                  opt_utrace;
 extern bool                  opt_xmalloc;
-extern bool                  opt_experimental_infallible_new;
 extern bool                  opt_experimental_tcache_gc;
 extern bool                  opt_zero;
 extern unsigned              opt_narenas;
@@ -46,6 +45,7 @@ extern bool                  opt_disable_large_size_classes;
 
 extern const char *opt_malloc_conf_symlink;
 extern const char *opt_malloc_conf_env_var;
+extern const char *je_malloc_conf_2_conf_harder;
 
 /* Escape free-fastpath when ptr & mask == 0 (for sanitization purpose). */
 extern uintptr_t san_cache_bin_nonfast_mask;

include/jemalloc/internal/jemalloc_internal_includes.h

@@ -1,84 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_INCLUDES_H
-#define JEMALLOC_INTERNAL_INCLUDES_H
-
-/*
- * jemalloc can conceptually be broken into components (arena, tcache, etc.),
- * but there are circular dependencies that cannot be broken without
- * substantial performance degradation.
- *
- * Historically, we dealt with this by each header into four sections (types,
- * structs, externs, and inlines), and included each header file multiple times
- * in this file, picking out the portion we want on each pass using the
- * following #defines:
- *   JEMALLOC_H_TYPES   : Preprocessor-defined constants and pseudo-opaque data
- *                        types.
- *   JEMALLOC_H_STRUCTS : Data structures.
- *   JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
- *   JEMALLOC_H_INLINES : Inline functions.
- *
- * We're moving toward a world in which the dependencies are explicit; each file
- * will #include the headers it depends on (rather than relying on them being
- * implicitly available via this file including every header file in the
- * project).
- *
- * We're now in an intermediate state: we've broken up the header files to avoid
- * having to include each one multiple times, but have not yet moved the
- * dependency information into the header files (i.e. we still rely on the
- * ordering in this file to ensure all a header's dependencies are available in
- * its translation unit).  Each component is now broken up into multiple header
- * files, corresponding to the sections above (e.g. instead of "foo.h", we now
- * have "foo_types.h", "foo_structs.h", "foo_externs.h", "foo_inlines.h").
- *
- * Those files which have been converted to explicitly include their
- * inter-component dependencies are now in the initial HERMETIC HEADERS
- * section.  All headers may still rely on jemalloc_preamble.h (which, by fiat,
- * must be included first in every translation unit) for system headers and
- * global jemalloc definitions, however.
- */
-
-/******************************************************************************/
-/* TYPES */
-/******************************************************************************/
-
-#include "jemalloc/internal/arena_types.h"
-#include "jemalloc/internal/tcache_types.h"
-#include "jemalloc/internal/prof_types.h"
-
-/******************************************************************************/
-/* STRUCTS */
-/******************************************************************************/
-
-#include "jemalloc/internal/prof_structs.h"
-#include "jemalloc/internal/arena_structs.h"
-#include "jemalloc/internal/tcache_structs.h"
-#include "jemalloc/internal/background_thread_structs.h"
-
-/******************************************************************************/
-/* EXTERNS */
-/******************************************************************************/
-
-#include "jemalloc/internal/jemalloc_internal_externs.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/large_externs.h"
-#include "jemalloc/internal/tcache_externs.h"
-#include "jemalloc/internal/prof_externs.h"
-#include "jemalloc/internal/background_thread_externs.h"
-
-/******************************************************************************/
-/* INLINES */
-/******************************************************************************/
-
-#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
-/*
- * Include portions of arena code interleaved with tcache code in order to
- * resolve circular dependencies.
- */
-#include "jemalloc/internal/arena_inlines_a.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
-#include "jemalloc/internal/tcache_inlines.h"
-#include "jemalloc/internal/arena_inlines_b.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
-#include "jemalloc/internal/prof_inlines.h"
-#include "jemalloc/internal/background_thread_inlines.h"
-
-#endif /* JEMALLOC_INTERNAL_INCLUDES_H */

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -2,14 +2,13 @@
 #define JEMALLOC_INTERNAL_INLINES_A_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/arenas_management.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bit_util.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/tcache_externs.h"
+#include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/ticker.h"
 
 JEMALLOC_ALWAYS_INLINE malloc_cpuid_t
@@ -19,15 +18,34 @@ malloc_getcpu(void) {
 	return GetCurrentProcessorNumber();
 #elif defined(JEMALLOC_HAVE_SCHED_GETCPU)
 	return (malloc_cpuid_t)sched_getcpu();
+#elif defined(__APPLE__)
+	/*
+	 * No sched_getcpu() on macOS; read the CPU number like _os_cpu_number()
+	 * does, from the low 12 bits of tpidr_el0 (arm64) or the IDT base (x86).
+	 * The 0xfff mask is xnu's __TPIDR_CPU_NUM_MASK, kept in sync with
+	 * _os_cpu_number in libsyscall/os/tsd.h (the kernel counterpart is
+	 * MACHDEP_TPIDR_CPUNUM_MASK in osfmk/arm64/machine_machdep.h):
+	 * https://github.com/apple-oss-distributions/xnu/blob/main/libsyscall/os/tsd.h
+	 * This requires macOS 12+: on macOS 11 and earlier arm64 kept the CPU
+	 * number in tpidrro_el0's low 3 bits instead, so it would be misread here.
+	 * Those releases are retired by Apple, so only the current layout is handled.
+	 */
+#  if defined(__aarch64__)
+	uint64_t cpu;
+	__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(cpu));
+	return (malloc_cpuid_t)(cpu & 0xfff);
+#  elif defined(__x86_64__) || defined(__i386__)
+	struct { uintptr_t p1, p2; } idtr;
+	__asm__ __volatile__("sidt %0" : "=m"(idtr));
+	return (malloc_cpuid_t)(idtr.p1 & 0xfff);
+#  else
+	not_reached();
+	return -1;
+#  endif
 #elif defined(JEMALLOC_HAVE_RDTSCP)
 	unsigned int ecx;
 	asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx");
 	return (malloc_cpuid_t)(ecx & 0xfff);
-#elif defined(__aarch64__) && defined(__APPLE__)
-	/* Other oses most likely use tpidr_el0 instead */
-	uintptr_t c;
-	asm volatile("mrs %x0, tpidrro_el0" : "=r"(c)::"memory");
-	return (malloc_cpuid_t)(c & (1 << 3) - 1);
 #else
 	not_reached();
 	return -1;

include/jemalloc/internal/jemalloc_internal_inlines_b.h

@@ -1,110 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_INLINES_B_H
-#define JEMALLOC_INTERNAL_INLINES_B_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_inlines_a.h"
-#include "jemalloc/internal/arenas_management.h"
-#include "jemalloc/internal/extent.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
-
-static inline void
-thread_migrate_arena(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
-	assert(oldarena != NULL);
-	assert(newarena != NULL);
-
-	arena_migrate(tsd, oldarena, newarena);
-	if (tcache_available(tsd)) {
-		tcache_arena_reassociate(tsd_tsdn(tsd),
-		    tsd_tcache_slowp_get(tsd), newarena);
-	}
-}
-
-static inline void
-percpu_arena_update(tsd_t *tsd, unsigned cpu) {
-	assert(have_percpu_arena);
-	arena_t *oldarena = tsd_arena_get(tsd);
-	assert(oldarena != NULL);
-	unsigned oldind = arena_ind_get(oldarena);
-
-	if (oldind != cpu) {
-		unsigned newind = cpu;
-		arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
-		assert(newarena != NULL);
-
-		thread_migrate_arena(tsd, oldarena, newarena);
-	}
-}
-
-/* Choose an arena based on a per-thread value. */
-static inline arena_t *
-arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
-	arena_t *ret;
-
-	if (arena != NULL) {
-		return arena;
-	}
-
-	/* During reentrancy, arena 0 is the safest bet. */
-	if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
-		return arena_get(tsd_tsdn(tsd), 0, true);
-	}
-
-	ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
-	if (unlikely(ret == NULL)) {
-		ret = arena_choose_hard(tsd, internal);
-		assert(ret);
-		if (tcache_available(tsd)) {
-			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
-			if (tcache_slow->arena != NULL) {
-				/* See comments in tsd_tcache_data_init().*/
-				assert(tcache_slow->arena
-				    == arena_get(tsd_tsdn(tsd), 0, false));
-				if (tcache_slow->arena != ret) {
-					tcache_arena_reassociate(tsd_tsdn(tsd),
-					    tcache_slow, ret);
-				}
-			} else {
-				tcache_arena_associate(
-				    tsd_tsdn(tsd), tcache_slow, ret);
-			}
-		}
-	}
-
-	/*
-	 * Note that for percpu arena, if the current arena is outside of the
-	 * auto percpu arena range, (i.e. thread is assigned to a manually
-	 * managed arena), then percpu arena is skipped.
-	 */
-	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
-	    && !internal
-	    && (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
-	    && (ret->last_thd != tsd_tsdn(tsd))) {
-		unsigned ind = percpu_arena_choose();
-		if (arena_ind_get(ret) != ind) {
-			percpu_arena_update(tsd, ind);
-			ret = tsd_arena_get(tsd);
-		}
-		ret->last_thd = tsd_tsdn(tsd);
-	}
-
-	return ret;
-}
-
-static inline arena_t *
-arena_choose(tsd_t *tsd, arena_t *arena) {
-	return arena_choose_impl(tsd, arena, false);
-}
-
-static inline arena_t *
-arena_ichoose(tsd_t *tsd, arena_t *arena) {
-	return arena_choose_impl(tsd, arena, true);
-}
-
-static inline bool
-arena_is_auto(const arena_t *arena) {
-	assert(narenas_auto > 0);
-
-	return (arena_ind_get(arena) < manual_arena_base);
-}
-
-#endif /* JEMALLOC_INTERNAL_INLINES_B_H */

include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -2,12 +2,13 @@
 #define JEMALLOC_INTERNAL_INLINES_C_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/emap.h"
 #include "jemalloc/internal/jemalloc_init.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/log.h"
+#include "jemalloc/internal/malloc_dispatch_inlines.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/thread_event.h"
 #include "jemalloc/internal/witness.h"
@@ -67,7 +68,7 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
 		    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	}
 
-	ret = arena_malloc(
+	ret = malloc_dispatch_malloc(
 	    tsdn, arena, size, ind, zero, slab, tcache, slow_path);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -102,7 +103,8 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 
-	ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
+	ret = malloc_dispatch_palloc(
+	    tsdn, arena, usize, alignment, zero, slab, tcache);
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -156,7 +158,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
 	    && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
 		assert(tcache == NULL);
 	}
-	arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
+	malloc_dispatch_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -169,7 +171,7 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-	arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
+	malloc_dispatch_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -217,8 +219,8 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 		    zero, slab, tcache, arena);
 	}
 
-	return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
-	    slab, tcache);
+	return malloc_dispatch_ralloc(
+	    tsdn, arena, ptr, oldsize, size, alignment, zero, slab, tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void *

include/jemalloc/internal/jemalloc_internal_macros.h

@@ -39,9 +39,11 @@
 #ifdef JEMALLOC_JET
 #	define JET_MUTABLE
 #	define JET_EXTERN extern
+#	define JET_EXTERN_INLINE extern
 #else
 #	define JET_MUTABLE const
 #	define JET_EXTERN static
+#	define JET_EXTERN_INLINE static inline
 #endif
 
 #define JEMALLOC_VA_ARGS_HEAD(head, ...) head

include/jemalloc/internal/jemalloc_preamble.h.in

@@ -130,13 +130,6 @@ static const bool config_prof_libunwind =
     false
 #endif
     ;
-static const bool config_prof_frameptr =
-#ifdef JEMALLOC_PROF_FRAME_POINTER
-    true
-#else
-    false
-#endif
-    ;
 static const bool maps_coalesce =
 #ifdef JEMALLOC_MAPS_COALESCE
     true
@@ -238,6 +231,18 @@ static const bool config_enable_cxx =
 #endif
 ;
 
+/*
+ * Whether the throwing operator new aborts on OOM instead of throwing
+ * std::bad_alloc (--enable-cxx-infallible-new).
+ */
+static const bool config_infallible_new =
+#ifdef JEMALLOC_INFALLIBLE_NEW
+    true
+#else
+    false
+#endif
+;
+
 #if defined(_WIN32) || defined(__APPLE__) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
 /* Currently percpu_arena depends on sched_getcpu. */
 #define JEMALLOC_PERCPU_ARENA

include/jemalloc/internal/large.h

@@ -1,9 +1,13 @@
-#ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H
-#define JEMALLOC_INTERNAL_LARGE_EXTERNS_H
+#ifndef JEMALLOC_INTERNAL_LARGE_H
+#define JEMALLOC_INTERNAL_LARGE_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/edata.h"
 
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s     arena_t;
+typedef struct prof_info_s prof_info_t;
+
 void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
 void *large_palloc(
     tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
@@ -20,4 +24,4 @@ void   large_prof_info_get(
 void large_prof_tctx_reset(edata_t *edata);
 void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size);
 
-#endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */
+#endif /* JEMALLOC_INTERNAL_LARGE_H */

include/jemalloc/internal/malloc_dispatch.h

@@ -0,0 +1,19 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s  arena_t;
+typedef struct tcache_s tcache_t;
+
+void *malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache);
+void  malloc_dispatch_dalloc_promoted(
+     tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
+void *malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr,
+    size_t oldsize, size_t size, size_t alignment, bool zero, bool slab,
+    tcache_t *tcache);
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_H */

include/jemalloc/internal/malloc_dispatch_inlines.h

@@ -0,0 +1,276 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arena_inlines.h"
+#include "jemalloc/internal/bin_inlines.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/large.h"
+#include "jemalloc/internal/malloc_dispatch.h"
+#include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache_inlines.h"
+
+JEMALLOC_ALWAYS_INLINE void *
+malloc_dispatch_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
+    bool zero, bool slab, tcache_t *tcache, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+
+	if (likely(tcache != NULL)) {
+		if (likely(slab)) {
+			assert(sz_can_use_slab(size));
+			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
+		} else if (likely(tcache_can_cache_large(tcache, ind))) {
+			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
+		}
+		/* (size > tcache_max) case falls through. */
+	}
+
+	return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
+}
+
+static inline void
+malloc_dispatch_dalloc_large_no_tcache(
+    tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
+	/*
+	 * szind both classifies small vs large and validates the extent --
+	 * inactive extents have szind == SC_NSIZES.
+	 */
+	if (config_prof && unlikely(szind < SC_NBINS)) {
+		malloc_dispatch_dalloc_promoted(tsdn, ptr, NULL, true);
+	} else {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		if (large_dalloc_safety_checks(edata, ptr, usize)) {
+			/* See the comment in isfree. */
+			return;
+		}
+		large_dalloc(tsdn, edata);
+	}
+}
+
+static inline void
+malloc_dispatch_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
+	assert(ptr != NULL);
+
+	emap_alloc_ctx_t alloc_ctx;
+	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.szind < SC_NSIZES);
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		malloc_dispatch_dalloc_large_no_tcache(
+		    tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    szind_t szind, size_t usize, bool slow_path) {
+	assert(!tsdn_null(tsdn) && tcache != NULL);
+	bool is_sample_promoted = config_prof && szind < SC_NBINS;
+	if (unlikely(is_sample_promoted)) {
+		malloc_dispatch_dalloc_promoted(tsdn, ptr, tcache, slow_path);
+	} else {
+		if (tcache_can_cache_large(tcache, szind)) {
+			tcache_dalloc_large(
+			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
+		} else {
+			edata_t *edata = emap_edata_lookup(
+			    tsdn, &arena_emap_global, ptr);
+			if (large_dalloc_safety_checks(edata, ptr, usize)) {
+				/* See the comment in isfree. */
+				return;
+			}
+			large_dalloc(tsdn, edata);
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_dispatch_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
+	if (!config_debug) {
+		return false;
+	}
+	edata_t   *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	szind_t    binind = edata_szind_get(edata);
+	div_info_t div_info = arena_binind_div_info[binind];
+	/*
+	 * Calls the internal function bin_slab_regind_impl because the
+	 * safety check does not require a lock.
+	 */
+	size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
+	slab_data_t      *slab_data = edata_slab_data_get(edata);
+	const bin_info_t *bin_info = &bin_infos[binind];
+	assert(edata_nfree_get(edata) < bin_info->nregs);
+	if (unlikely(!bitmap_get(
+	        slab_data->bitmap, &bin_info->bitmap_info, regind))) {
+		safety_check_fail(
+		    "Invalid deallocation detected: the pointer being freed (%p) not "
+		    "currently active, possibly caused by double free bugs.\n",
+		    ptr);
+		return true;
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+
+	if (unlikely(tcache == NULL)) {
+		malloc_dispatch_dalloc_no_tcache(tsdn, ptr);
+		return;
+	}
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (caller_alloc_ctx != NULL) {
+		alloc_ctx = *caller_alloc_ctx;
+	} else {
+		util_assume(tsdn != NULL);
+		emap_alloc_ctx_lookup(
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+	}
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.szind < SC_NSIZES);
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+	} else {
+		malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
+	}
+}
+
+static inline void
+malloc_dispatch_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
+	assert(ptr != NULL);
+	assert(size <= SC_LARGE_MAXCLASS);
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (!config_prof || !opt_prof) {
+		/*
+		 * There is no risk of being confused by a promoted sampled
+		 * object, so base szind and slab on the given size.
+		 */
+		szind_t szind = sz_size2index(size);
+		emap_alloc_ctx_init(
+		    &alloc_ctx, szind, (szind < SC_NBINS), size);
+	}
+
+	if ((config_prof && opt_prof) || config_debug) {
+		emap_alloc_ctx_lookup(
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+		assert(alloc_ctx.szind == sz_size2index(size));
+		assert((config_prof && opt_prof)
+		    || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
+
+		if (config_debug) {
+			edata_t *edata = emap_edata_lookup(
+			    tsdn, &arena_emap_global, ptr);
+			assert(alloc_ctx.szind == edata_szind_get(edata));
+			assert(alloc_ctx.slab == edata_slab_get(edata));
+		}
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		malloc_dispatch_dalloc_large_no_tcache(
+		    tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+	assert(size <= SC_LARGE_MAXCLASS);
+
+	if (unlikely(tcache == NULL)) {
+		malloc_dispatch_sdalloc_no_tcache(tsdn, ptr, size);
+		return;
+	}
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (config_prof && opt_prof) {
+		if (caller_alloc_ctx == NULL) {
+			/* Uncommon case and should be a static check. */
+			emap_alloc_ctx_lookup(
+			    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+			assert(alloc_ctx.szind == sz_size2index(size));
+			assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
+		} else {
+			alloc_ctx = *caller_alloc_ctx;
+		}
+	} else {
+		/*
+		 * There is no risk of being confused by a promoted sampled
+		 * object, so base szind and slab on the given size.
+		 */
+		alloc_ctx.szind = sz_size2index(size);
+		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
+	}
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		emap_alloc_ctx_init(
+		    &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+	} else {
+		malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+		    sz_s2u(size), slow_path);
+	}
+}
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H */

include/jemalloc/internal/malloc_io.h

@@ -89,13 +89,4 @@ malloc_close(int fd) {
 #endif
 }
 
-static inline off_t
-malloc_lseek(int fd, off_t offset, int whence) {
-#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_lseek)
-	return (off_t)syscall(SYS_lseek, fd, offset, whence);
-#else
-	return lseek(fd, offset, whence);
-#endif
-}
-
 #endif /* JEMALLOC_INTERNAL_MALLOC_IO_H */

include/jemalloc/internal/mutex_prof.h

@@ -37,7 +37,8 @@ typedef enum {
 	OP(tcache_list)                                                        \
 	OP(hpa_shard)                                                          \
 	OP(hpa_shard_grow)                                                     \
-	OP(hpa_sec)
+	OP(hpa_sec)                                                            \
+	OP(pac_sec)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,

include/jemalloc/internal/nstime.h

@@ -28,7 +28,6 @@ static const nstime_t nstime_zero = NSTIME_ZERO_INITIALIZER;
 void     nstime_init(nstime_t *time, uint64_t ns);
 void     nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
 uint64_t nstime_ns(const nstime_t *time);
-uint64_t nstime_ms(const nstime_t *time);
 uint64_t nstime_sec(const nstime_t *time);
 uint64_t nstime_nsec(const nstime_t *time);
 void     nstime_copy(nstime_t *time, const nstime_t *source);
@@ -36,11 +35,9 @@ int      nstime_compare(const nstime_t *a, const nstime_t *b);
 void     nstime_add(nstime_t *time, const nstime_t *addend);
 void     nstime_iadd(nstime_t *time, uint64_t addend);
 void     nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
-void     nstime_isubtract(nstime_t *time, uint64_t subtrahend);
 void     nstime_imultiply(nstime_t *time, uint64_t multiplier);
 void     nstime_idivide(nstime_t *time, uint64_t divisor);
 uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
-uint64_t nstime_ns_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ms_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ns_since(const nstime_t *past);
 uint64_t nstime_ms_since(const nstime_t *past);

include/jemalloc/internal/pa.h

@@ -91,9 +91,6 @@ struct pa_shard_s {
 	 */
 	bool ever_used_hpa;
 
-	/* Allocates from a PAC. */
-	pac_t pac;
-
 	hpa_shard_t hpa;
 
 	/* The source of edata_t objects. */
@@ -109,13 +106,10 @@ struct pa_shard_s {
 
 	/* The base from which we get the ehooks and allocate metadat. */
 	base_t *base;
-};
 
-static inline bool
-pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
-	return ecache_npages_get(&shard->pac.ecache_muzzy) == 0
-	    && pac_decay_ms_get(&shard->pac, extent_state_muzzy) <= 0;
-}
+	/* Allocates from a PAC. */
+	pac_t pac;
+};
 
 static inline ehooks_t *
 pa_shard_ehooks_get(const pa_shard_t *shard) {
@@ -158,8 +152,12 @@ void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard);
  */
 void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
 
-/* Flush any caches used by shard */
-void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard);
+/*
+ * Flush the shard's front caches (SEC + HPA) back to the ecaches.  If all is
+ * true, also fully decay-purge the PAC dirty (and muzzy, unless skipped) extents
+ * to the OS -- the "save as much memory as possible" path.
+ */
+void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard, bool all);
 
 /* Gets an edata for the given allocation. */
 edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
@@ -184,22 +182,35 @@ bool pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 void    pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
        bool *deferred_work_generated);
 bool    pa_decay_ms_set(tsdn_t *tsdn, pa_shard_t *shard, extent_state_t state,
-       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms);
 ssize_t pa_decay_ms_get(pa_shard_t *shard, extent_state_t state);
 
-/*
- * Do deferred work on this PA shard.
- *
- * Morally, this should do both PAC decay and the HPA deferred work.  For now,
- * though, the arena, background thread, and PAC modules are tightly interwoven
- * in a way that's tricky to extricate, so we only do the HPA-specific parts.
- */
 void pa_shard_set_deferral_allowed(
     tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed);
-void     pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+/*
+ * Do deferred work on this PA shard: dispatch to PAC (decay-purge) then HPA.
+ * Each allocator owns its own policy -- PAC decides eagerness from
+ * is_background_thread and, on the application path, notifies the background
+ * thread; the bg-thread driver passes is_background_thread=true and is never
+ * self-notified.
+ */
+void     pa_shard_do_deferred_work(
+    tsdn_t *tsdn, pa_shard_t *shard, bool is_background_thread);
 void     pa_shard_try_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
 
+/*
+ * Called on the application path after a pa operation (alloc/expand/shrink/
+ * dalloc) reports deferred_work_generated -- i.e. it left PAC decay due or
+ * HPA work (hugify/dehugify/purge) pending.  If PAC dirty decay is immediate
+ * (dirty_decay_ms == 0), purge dirty synchronously; then, if a background
+ * thread is enabled but idle (sleeping indefinitely), wake it early so the
+ * pending work runs promptly rather than at the next scheduled wakeup.
+ *
+ * Application thread only; must never be called on the background thread.
+ */
+void pa_shard_handle_deferred_work(tsdn_t *tsdn, pa_shard_t *shard);
+
 /******************************************************************************/
 /*
  * Various bits of "boring" functionality that are still part of this module,
@@ -220,8 +231,6 @@ void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
 
 size_t pa_shard_nactive(const pa_shard_t *shard);
-size_t pa_shard_ndirty(const pa_shard_t *shard);
-size_t pa_shard_nmuzzy(const pa_shard_t *shard);
 
 void pa_shard_basic_stats_merge(
     const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);

include/jemalloc/internal/pac.h

@@ -8,11 +8,12 @@
 #include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
 #include "jemalloc/internal/lockedint.h"
+#include "jemalloc/internal/sec.h"
 #include "jemalloc/internal/tsd_types.h"
 #include "san_bump.h"
 
 /*
- * Page allocator classic; an implementation of the PAI interface that:
+ * Page allocator classic (PAC), a page-level allocator that:
  * - Can be used for arenas with custom extent hooks.
  * - Can always satisfy any allocation request (including highly-fragmentary
  *   ones).
@@ -27,6 +28,14 @@ enum pac_purge_eagerness_e {
 };
 typedef enum pac_purge_eagerness_e pac_purge_eagerness_t;
 
+/*
+ * When a decay sweep would purge more than this many pages, its purge is
+ * treated as due (worth waking the background thread for) instead of left
+ * to accumulate.  PAC decay policy; the HPA defers on time intervals, not
+ * on a page count, so this constant is PAC-only.
+ */
+#define PAC_DECAY_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
+
 typedef struct pac_decay_stats_s pac_decay_stats_t;
 struct pac_decay_stats_s {
 	/* Total number of purge sweeps. */
@@ -84,12 +93,24 @@ struct pac_stats_s {
 
 	/* VM space had to be leaked (undocumented).  Normally 0. */
 	atomic_zu_t abandoned_vm;
+
+	/* PAC SEC stats.  Derived. */
+	sec_stats_t pac_sec_stats;
 };
 
 typedef struct pac_s pac_t;
 struct pac_s {
+	/* Small extent cache in front of PAC ecaches to reduce contention. */
+	sec_t sec;
+	/*
+	 * Runtime gate for PAC SEC.  0 disables (when SEC is not configured or
+	 * dirty_decay_ms == 0); otherwise mirrors sec.opts.max_alloc.
+	 */
+	atomic_zu_t sec_max_alloc;
+
 	/* True once pinned memory has been seen. */
 	atomic_b_t has_pinned;
+
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.
@@ -218,6 +239,41 @@ bool pac_maybe_decay_purge(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
     pac_decay_stats_t *decay_stats, ecache_t *ecache,
     pac_purge_eagerness_t eagerness);
 
+/*
+ * Result of a pac_decay_deferred() pass.  Reported per decay state so
+ * pac_do_deferred_work can decide, per state, whether to notify the background
+ * thread.  npages_new is the per-state epoch backlog delta, only meaningful when
+ * the corresponding *_epoch_advanced is true.
+ */
+typedef struct pac_deferred_work_result_s pac_deferred_work_result_t;
+struct pac_deferred_work_result_s {
+	size_t dirty_npages_new;
+	size_t muzzy_npages_new;
+	bool   dirty_epoch_advanced;
+	bool   muzzy_epoch_advanced;
+};
+
+/*
+ * All deferred decay-purge work for a PAC shard: decide the eagerness from the
+ * caller context, run pac_decay_deferred, and (application path only) notify the
+ * background thread for any decay epoch that advanced.  The bg-thread driver
+ * passes is_background_thread=true and is never self-notified.
+ */
+void pac_do_deferred_work(
+    tsdn_t *tsdn, pac_t *pac, bool is_background_thread);
+
+/*
+ * Application-path hook (after deferred_work_generated): wake the background
+ * thread if it is sleeping idle.  Runs the same early-wake logic as the notify
+ * path (pac_maybe_wake_bg), gated on the thread being idle.
+ */
+void pac_wake_bg_on_deferred(tsdn_t *tsdn, pac_t *pac);
+
+/*
+ * Fully decay the extents of the given state, acquiring decay->mtx internally.
+ */
+void pac_decay_all_now(tsdn_t *tsdn, pac_t *pac, extent_state_t state);
+
 /*
  * Gets / sets the maximum amount that we'll grow an arena down the
  * grow-retained pathways (unless forced to by an allocaction request).
@@ -231,10 +287,24 @@ bool pac_retain_grow_limit_get_set(
     tsdn_t *tsdn, pac_t *pac, size_t *old_limit, size_t *new_limit);
 
 bool    pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
-       ssize_t decay_ms, pac_purge_eagerness_t eagerness);
+       ssize_t decay_ms);
 ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
-void pac_reset(tsdn_t *tsdn, pac_t *pac);
+/* Whether the muzzy decay path is relevant (muzzy pages exist, or decay is on). */
+static inline bool
+pac_should_decay_muzzy(pac_t *pac) {
+	return ecache_npages_get(&pac->ecache_muzzy) != 0
+	    || pac_decay_ms_get(pac, extent_state_muzzy) > 0;
+}
+
+/* Whether dirty decay is immediate (dirty_decay_ms == 0). */
+static inline bool
+pac_decay_immediately(pac_t *pac) {
+	return decay_immediately(&pac->decay_dirty);
+}
+
 void pac_destroy(tsdn_t *tsdn, pac_t *pac);
 
+void pac_sec_flush(tsdn_t *tsdn, pac_t *pac);
+
 #endif /* JEMALLOC_INTERNAL_PAC_H */

include/jemalloc/internal/peak_event.h

@@ -2,6 +2,7 @@
 #define JEMALLOC_INTERNAL_PEAK_EVENT_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/thread_event_registry.h"
 #include "jemalloc/internal/tsd_types.h"
 
 /*

include/jemalloc/internal/prof.h

@@ -0,0 +1,426 @@
+#ifndef JEMALLOC_INTERNAL_PROF_H
+#define JEMALLOC_INTERNAL_PROF_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/ckh.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prng.h"
+#include "jemalloc/internal/prof_hook.h"
+#include "jemalloc/internal/rb.h"
+#include "jemalloc/internal/thread_event_registry.h"
+
+/* Forward decl; only base_t * is used as a pointer arg below. */
+typedef struct base_s base_t;
+
+/******************************************************************************/
+/* TYPES */
+/******************************************************************************/
+
+typedef struct prof_bt_s     prof_bt_t;
+typedef struct prof_cnt_s    prof_cnt_t;
+typedef struct prof_tctx_s   prof_tctx_t;
+typedef struct prof_info_s   prof_info_t;
+typedef struct prof_gctx_s   prof_gctx_t;
+typedef struct prof_tdata_s  prof_tdata_t;
+typedef struct prof_recent_s prof_recent_t;
+
+/* Option defaults. */
+#ifdef JEMALLOC_PROF
+#	define PROF_PREFIX_DEFAULT "jeprof"
+#else
+#	define PROF_PREFIX_DEFAULT ""
+#endif
+#define LG_PROF_SAMPLE_DEFAULT 19
+#define LG_PROF_INTERVAL_DEFAULT -1
+
+/*
+ * Hard limit on stack backtrace depth.  The version of prof_backtrace() that
+ * is based on __builtin_return_address() necessarily has a hard-coded number
+ * of backtrace frame handlers, and should be kept in sync with this setting.
+ */
+#ifdef JEMALLOC_PROF_GCC
+#	define PROF_BT_MAX_LIMIT 256
+#else
+#	define PROF_BT_MAX_LIMIT UINT_MAX
+#endif
+#define PROF_BT_MAX_DEFAULT 128
+
+/* Initial hash table size. */
+#define PROF_CKH_MINITEMS 64
+
+/* Size of memory buffer to use when writing dump files. */
+#ifndef JEMALLOC_PROF
+/* Minimize memory bloat for non-prof builds. */
+#	define PROF_DUMP_BUFSIZE 1
+#elif defined(JEMALLOC_DEBUG)
+/* Use a small buffer size in debug build, mainly to facilitate testing. */
+#	define PROF_DUMP_BUFSIZE 16
+#else
+#	define PROF_DUMP_BUFSIZE 65536
+#endif
+
+/* Size of size class related tables */
+#ifdef JEMALLOC_PROF
+#	define PROF_SC_NSIZES SC_NSIZES
+#else
+/* Minimize memory bloat for non-prof builds. */
+#	define PROF_SC_NSIZES 1
+#endif
+
+/* Size of stack-allocated buffer used by prof_printf(). */
+#define PROF_PRINTF_BUFSIZE 128
+
+/*
+ * Number of mutexes shared among all gctx's.  No space is allocated for these
+ * unless profiling is enabled, so it's okay to over-provision.
+ */
+#define PROF_NCTX_LOCKS 1024
+
+/*
+ * Number of mutexes shared among all tdata's.  No space is allocated for these
+ * unless profiling is enabled, so it's okay to over-provision.
+ */
+#define PROF_NTDATA_LOCKS 256
+
+/* Minimize memory bloat for non-prof builds. */
+#ifdef JEMALLOC_PROF
+#	define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
+#else
+#	define PROF_DUMP_FILENAME_LEN 1
+#endif
+
+/* Default number of recent allocations to record. */
+#define PROF_RECENT_ALLOC_MAX_DEFAULT 0
+
+/* Thread name storage size limit. */
+#define PROF_THREAD_NAME_MAX_LEN 16
+
+/*
+ * Minimum required alignment for sampled allocations. Over-aligning sampled
+ * allocations allows us to quickly identify them on the dalloc path without
+ * resorting to metadata lookup.
+ */
+#define PROF_SAMPLE_ALIGNMENT PAGE
+#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK
+
+/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
+#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U))
+
+/******************************************************************************/
+/* STRUCTS */
+/******************************************************************************/
+
+struct prof_bt_s {
+	/* Backtrace, stored as len program counters. */
+	void   **vec;
+	unsigned len;
+};
+
+#ifdef JEMALLOC_PROF_LIBGCC
+/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
+typedef struct {
+	void    **vec;
+	unsigned *len;
+	unsigned  max;
+} prof_unwind_data_t;
+#endif
+
+struct prof_cnt_s {
+	/* Profiling counters. */
+	uint64_t curobjs;
+	uint64_t curobjs_shifted_unbiased;
+	uint64_t curbytes;
+	uint64_t curbytes_unbiased;
+	uint64_t accumobjs;
+	uint64_t accumobjs_shifted_unbiased;
+	uint64_t accumbytes;
+	uint64_t accumbytes_unbiased;
+};
+
+typedef enum {
+	prof_tctx_state_initializing,
+	prof_tctx_state_nominal,
+	prof_tctx_state_dumping,
+	prof_tctx_state_purgatory /* Dumper must finish destroying. */
+} prof_tctx_state_t;
+
+struct prof_tctx_s {
+	/* Thread data for thread that performed the allocation. */
+	prof_tdata_t *tdata;
+
+	/*
+	 * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
+	 * defunct during teardown.
+	 */
+	uint64_t thr_uid;
+	uint64_t thr_discrim;
+
+	/*
+	 * Reference count of how many times this tctx object is referenced in
+	 * recent allocation / deallocation records, protected by tdata->lock.
+	 */
+	uint64_t recent_count;
+
+	/* Profiling counters, protected by tdata->lock. */
+	prof_cnt_t cnts;
+
+	/* Associated global context. */
+	prof_gctx_t *gctx;
+
+	/*
+	 * UID that distinguishes multiple tctx's created by the same thread,
+	 * but coexisting in gctx->tctxs.  There are two ways that such
+	 * coexistence can occur:
+	 * - A dumper thread can cause a tctx to be retained in the purgatory
+	 *   state.
+	 * - Although a single "producer" thread must create all tctx's which
+	 *   share the same thr_uid, multiple "consumers" can each concurrently
+	 *   execute portions of prof_tctx_destroy().  prof_tctx_destroy() only
+	 *   gets called once each time cnts.cur{objs,bytes} drop to 0, but this
+	 *   threshold can be hit again before the first consumer finishes
+	 *   executing prof_tctx_destroy().
+	 */
+	uint64_t tctx_uid;
+
+	/* Linkage into gctx's tctxs. */
+	rb_node(prof_tctx_t) tctx_link;
+
+	/*
+	 * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
+	 * sample vs destroy race.
+	 */
+	bool prepared;
+
+	/* Current dump-related state, protected by gctx->lock. */
+	prof_tctx_state_t state;
+
+	/*
+	 * Copy of cnts snapshotted during early dump phase, protected by
+	 * dump_mtx.
+	 */
+	prof_cnt_t dump_cnts;
+};
+typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
+
+struct prof_info_s {
+	/* Time when the allocation was made. */
+	nstime_t alloc_time;
+	/* Points to the prof_tctx_t corresponding to the allocation. */
+	prof_tctx_t *alloc_tctx;
+	/* Allocation request size. */
+	size_t alloc_size;
+};
+
+struct prof_gctx_s {
+	/* Protects nlimbo, cnt_summed, and tctxs. */
+	malloc_mutex_t *lock;
+
+	/*
+	 * Number of threads that currently cause this gctx to be in a state of
+	 * limbo due to one of:
+	 *   - Initializing this gctx.
+	 *   - Initializing per thread counters associated with this gctx.
+	 *   - Preparing to destroy this gctx.
+	 *   - Dumping a heap profile that includes this gctx.
+	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
+	 * gctx.
+	 */
+	unsigned nlimbo;
+
+	/*
+	 * Tree of profile counters, one for each thread that has allocated in
+	 * this context.
+	 */
+	prof_tctx_tree_t tctxs;
+
+	/* Linkage for tree of contexts to be dumped. */
+	rb_node(prof_gctx_t) dump_link;
+
+	/* Temporary storage for summation during dump. */
+	prof_cnt_t cnt_summed;
+
+	/* Associated backtrace. */
+	prof_bt_t bt;
+
+	/* Backtrace vector, variable size, referred to by bt. */
+	void *vec[1];
+};
+typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
+
+struct prof_tdata_s {
+	malloc_mutex_t *lock;
+
+	/* Monotonically increasing unique thread identifier. */
+	uint64_t thr_uid;
+
+	/*
+	 * Monotonically increasing discriminator among tdata structures
+	 * associated with the same thr_uid.
+	 */
+	uint64_t thr_discrim;
+
+	rb_node(prof_tdata_t) tdata_link;
+
+	/*
+	 * Counter used to initialize prof_tctx_t's tctx_uid.  No locking is
+	 * necessary when incrementing this field, because only one thread ever
+	 * does so.
+	 */
+	uint64_t tctx_uid_next;
+
+	/*
+	 * Hash of (prof_bt_t *)-->(prof_tctx_t *).  Each thread tracks
+	 * backtraces for which it has non-zero allocation/deallocation counters
+	 * associated with thread-specific prof_tctx_t objects.  Other threads
+	 * may write to prof_tctx_t contents when freeing associated objects.
+	 */
+	ckh_t bt2tctx;
+
+	/* Included in heap profile dumps if has content. */
+	char thread_name[PROF_THREAD_NAME_MAX_LEN];
+
+	/* State used to avoid dumping while operating on prof internals. */
+	bool enq;
+	bool enq_idump;
+	bool enq_gdump;
+
+	/*
+	 * Set to true during an early dump phase for tdata's which are
+	 * currently being dumped.  New threads' tdata's have this initialized
+	 * to false so that they aren't accidentally included in later dump
+	 * phases.
+	 */
+	bool dumping;
+
+	/*
+	 * True if profiling is active for this tdata's thread
+	 * (thread.prof.active mallctl).
+	 */
+	bool active;
+
+	bool attached;
+	bool expired;
+
+	/* Temporary storage for summation during dump. */
+	prof_cnt_t cnt_summed;
+
+	/* Backtrace vector, used for calls to prof_backtrace(). */
+	void **vec;
+};
+typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
+
+struct prof_recent_s {
+	nstime_t alloc_time;
+	nstime_t dalloc_time;
+
+	ql_elm(prof_recent_t) link;
+	size_t       size;
+	size_t       usize;
+	atomic_p_t   alloc_edata; /* NULL means allocation has been freed. */
+	prof_tctx_t *alloc_tctx;
+	prof_tctx_t *dalloc_tctx;
+};
+
+/******************************************************************************/
+/* EXTERNS */
+/******************************************************************************/
+
+extern bool     opt_prof;
+extern bool     opt_prof_active;
+extern bool     opt_prof_thread_active_init;
+extern unsigned opt_prof_bt_max;
+extern size_t   opt_lg_prof_sample; /* Mean bytes between samples. */
+extern ssize_t opt_lg_prof_interval;    /* lg(prof_interval). */
+extern bool    opt_prof_gdump;          /* High-water memory dumping. */
+extern bool    opt_prof_final;          /* Final profile dumping. */
+extern bool    opt_prof_leak;           /* Dump leak summary at exit. */
+extern bool    opt_prof_leak_error; /* Exit with error code if memory leaked */
+extern bool    opt_prof_accum;      /* Report cumulative bytes. */
+extern bool    opt_prof_log;        /* Turn logging on at boot. */
+extern char    opt_prof_prefix[
+/* Minimize memory bloat for non-prof builds. */
+#ifdef JEMALLOC_PROF
+    PATH_MAX +
+#endif
+    1];
+extern bool opt_prof_unbias;
+
+/* Include pid namespace in profile file names. */
+extern bool opt_prof_pid_namespace;
+
+/* For recording recent allocations */
+extern ssize_t opt_prof_recent_alloc_max;
+
+/* Whether to use thread name provided by the system or by mallctl. */
+extern bool opt_prof_sys_thread_name;
+
+/* Whether to record per size class counts and request size totals. */
+extern bool opt_prof_stats;
+
+/* Accessed via prof_active_[gs]et{_unlocked,}(). */
+extern bool prof_active_state;
+
+/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
+extern bool prof_gdump_val;
+
+/* Profile dump interval, measured in bytes allocated. */
+extern uint64_t prof_interval;
+
+/*
+ * Initialized as opt_lg_prof_sample, and potentially modified during profiling
+ * resets.
+ */
+extern size_t lg_prof_sample;
+
+extern bool prof_booted;
+
+void                  prof_backtrace_hook_set(prof_backtrace_hook_t hook);
+prof_backtrace_hook_t prof_backtrace_hook_get(void);
+
+void             prof_dump_hook_set(prof_dump_hook_t hook);
+prof_dump_hook_t prof_dump_hook_get(void);
+
+void               prof_sample_hook_set(prof_sample_hook_t hook);
+prof_sample_hook_t prof_sample_hook_get(void);
+
+void                    prof_sample_free_hook_set(prof_sample_free_hook_t hook);
+prof_sample_free_hook_t prof_sample_free_hook_get(void);
+
+/* Functions only accessed in prof_inlines.h */
+prof_tdata_t *prof_tdata_init(tsd_t *tsd);
+prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
+
+void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
+void prof_malloc_sample_object(
+    tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
+void prof_free_sampled_object(
+    tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
+prof_tctx_t *prof_tctx_create(tsd_t *tsd);
+void         prof_idump(tsdn_t *tsdn);
+bool         prof_mdump(tsd_t *tsd, const char *filename);
+void         prof_gdump(tsdn_t *tsdn);
+
+void        prof_tdata_cleanup(tsd_t *tsd);
+bool        prof_active_get(tsdn_t *tsdn);
+bool        prof_active_set(tsdn_t *tsdn, bool active);
+const char *prof_thread_name_get(tsd_t *tsd);
+int         prof_thread_name_set(tsd_t *tsd, const char *thread_name);
+bool        prof_thread_active_get(tsd_t *tsd);
+bool        prof_thread_active_set(tsd_t *tsd, bool active);
+bool        prof_thread_active_init_get(tsdn_t *tsdn);
+bool        prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
+bool        prof_gdump_get(tsdn_t *tsdn);
+bool        prof_gdump_set(tsdn_t *tsdn, bool active);
+void        prof_boot0(void);
+void        prof_boot1(void);
+bool        prof_boot2(tsd_t *tsd, base_t *base);
+void        prof_prefork0(tsdn_t *tsdn);
+void        prof_prefork1(tsdn_t *tsdn);
+void        prof_postfork_parent(tsdn_t *tsdn);
+void        prof_postfork_child(tsdn_t *tsdn);
+
+uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
+
+extern te_base_cb_t prof_sample_te_handler;
+
+#endif /* JEMALLOC_INTERNAL_PROF_H */

include/jemalloc/internal/prof_data.h

@@ -29,9 +29,4 @@ void          prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
 void          prof_reset(tsd_t *tsd, size_t lg_sample);
 void          prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx);
 
-/* Used in unit tests. */
-size_t prof_tdata_count(void);
-size_t prof_bt_count(void);
-void   prof_cnt_all(prof_cnt_t *cnt_all);
-
 #endif /* JEMALLOC_INTERNAL_PROF_DATA_H */

include/jemalloc/internal/prof_externs.h

@@ -1,132 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PROF_EXTERNS_H
-#define JEMALLOC_INTERNAL_PROF_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/base.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/prof_hook.h"
-#include "jemalloc/internal/thread_event_registry.h"
-
-extern bool     opt_prof;
-extern bool     opt_prof_active;
-extern bool     opt_prof_thread_active_init;
-extern unsigned opt_prof_bt_max;
-extern size_t   opt_lg_prof_sample; /* Mean bytes between samples. */
-extern ssize_t opt_lg_prof_interval;    /* lg(prof_interval). */
-extern bool    opt_prof_gdump;          /* High-water memory dumping. */
-extern bool    opt_prof_final;          /* Final profile dumping. */
-extern bool    opt_prof_leak;           /* Dump leak summary at exit. */
-extern bool    opt_prof_leak_error; /* Exit with error code if memory leaked */
-extern bool    opt_prof_accum;      /* Report cumulative bytes. */
-extern bool    opt_prof_log;        /* Turn logging on at boot. */
-extern char    opt_prof_prefix[
-/* Minimize memory bloat for non-prof builds. */
-#ifdef JEMALLOC_PROF
-    PATH_MAX +
-#endif
-    1];
-extern bool opt_prof_unbias;
-
-/* Include pid namespace in profile file names. */
-extern bool opt_prof_pid_namespace;
-
-/* For recording recent allocations */
-extern ssize_t opt_prof_recent_alloc_max;
-
-/* Whether to use thread name provided by the system or by mallctl. */
-extern bool opt_prof_sys_thread_name;
-
-/* Whether to record per size class counts and request size totals. */
-extern bool opt_prof_stats;
-
-/* Accessed via prof_active_[gs]et{_unlocked,}(). */
-extern bool prof_active_state;
-
-/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
-extern bool prof_gdump_val;
-
-/* Profile dump interval, measured in bytes allocated. */
-extern uint64_t prof_interval;
-
-/*
- * Initialized as opt_lg_prof_sample, and potentially modified during profiling
- * resets.
- */
-extern size_t lg_prof_sample;
-
-extern bool prof_booted;
-
-void                  prof_backtrace_hook_set(prof_backtrace_hook_t hook);
-prof_backtrace_hook_t prof_backtrace_hook_get(void);
-
-void             prof_dump_hook_set(prof_dump_hook_t hook);
-prof_dump_hook_t prof_dump_hook_get(void);
-
-void               prof_sample_hook_set(prof_sample_hook_t hook);
-prof_sample_hook_t prof_sample_hook_get(void);
-
-void                    prof_sample_free_hook_set(prof_sample_free_hook_t hook);
-prof_sample_free_hook_t prof_sample_free_hook_get(void);
-
-/* Functions only accessed in prof_inlines.h */
-prof_tdata_t *prof_tdata_init(tsd_t *tsd);
-prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
-
-void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
-void prof_malloc_sample_object(
-    tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
-void prof_free_sampled_object(
-    tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
-prof_tctx_t *prof_tctx_create(tsd_t *tsd);
-void         prof_idump(tsdn_t *tsdn);
-bool         prof_mdump(tsd_t *tsd, const char *filename);
-void         prof_gdump(tsdn_t *tsdn);
-
-void        prof_tdata_cleanup(tsd_t *tsd);
-bool        prof_active_get(tsdn_t *tsdn);
-bool        prof_active_set(tsdn_t *tsdn, bool active);
-const char *prof_thread_name_get(tsd_t *tsd);
-int         prof_thread_name_set(tsd_t *tsd, const char *thread_name);
-bool        prof_thread_active_get(tsd_t *tsd);
-bool        prof_thread_active_set(tsd_t *tsd, bool active);
-bool        prof_thread_active_init_get(tsdn_t *tsdn);
-bool        prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
-bool        prof_gdump_get(tsdn_t *tsdn);
-bool        prof_gdump_set(tsdn_t *tsdn, bool active);
-void        prof_boot0(void);
-void        prof_boot1(void);
-bool        prof_boot2(tsd_t *tsd, base_t *base);
-void        prof_prefork0(tsdn_t *tsdn);
-void        prof_prefork1(tsdn_t *tsdn);
-void        prof_postfork_parent(tsdn_t *tsdn);
-void        prof_postfork_child(tsdn_t *tsdn);
-
-uint64_t prof_sample_new_event_wait(tsd_t *tsd);
-uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
-
-/*
- * The lookahead functionality facilitates events to be able to lookahead, i.e.
- * without touching the event counters, to determine whether an event would be
- * triggered.  The event counters are not advanced until the end of the
- * allocation / deallocation calls, so the lookahead can be useful if some
- * preparation work for some event must be done early in the allocation /
- * deallocation calls.
- *
- * Currently only the profiling sampling event needs the lookahead
- * functionality, so we don't yet define general purpose lookahead functions.
- */
-
-JEMALLOC_ALWAYS_INLINE bool
-te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
-	if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) {
-		return false;
-	}
-	/* The subtraction is intentionally susceptible to underflow. */
-	uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize
-	    - tsd_thread_allocated_last_event_get(tsd);
-	return accumbytes >= tsd_prof_sample_event_wait_get(tsd);
-}
-
-extern te_base_cb_t prof_sample_te_handler;
-
-#endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */

include/jemalloc/internal/prof_inlines.h

@@ -2,10 +2,9 @@
 #define JEMALLOC_INTERNAL_PROF_INLINES_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_c.h"
-#include "jemalloc/internal/prof_externs.h"
-#include "jemalloc/internal/prof_structs.h"
+#include "jemalloc/internal/prof.h"
 #include "jemalloc/internal/safety_check.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/thread_event.h"

include/jemalloc/internal/prof_log.h

@@ -9,14 +9,6 @@ extern malloc_mutex_t log_mtx;
 void prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
 bool prof_log_init(tsd_t *tsdn);
 
-/* Used in unit tests. */
-size_t prof_log_bt_count(void);
-size_t prof_log_alloc_count(void);
-size_t prof_log_thr_count(void);
-bool   prof_log_is_logging(void);
-bool   prof_log_rep_check(void);
-void   prof_log_dummy_set(bool new_value);
-
 bool prof_log_start(tsdn_t *tsdn, const char *filename);
 bool prof_log_stop(tsdn_t *tsdn);
 

include/jemalloc/internal/prof_recent.h

@@ -17,8 +17,6 @@ void edata_prof_recent_alloc_init(edata_t *edata);
 /* Used in unit tests. */
 typedef ql_head(prof_recent_t) prof_recent_list_t;
 extern prof_recent_list_t prof_recent_alloc_list;
-edata_t *prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *node);
-prof_recent_t *edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata);
 
 ssize_t prof_recent_alloc_max_ctl_read(void);
 ssize_t prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max);

include/jemalloc/internal/prof_structs.h

@@ -1,222 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PROF_STRUCTS_H
-#define JEMALLOC_INTERNAL_PROF_STRUCTS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/ckh.h"
-#include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/prng.h"
-#include "jemalloc/internal/rb.h"
-
-struct prof_bt_s {
-	/* Backtrace, stored as len program counters. */
-	void   **vec;
-	unsigned len;
-};
-
-#ifdef JEMALLOC_PROF_LIBGCC
-/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
-typedef struct {
-	void    **vec;
-	unsigned *len;
-	unsigned  max;
-} prof_unwind_data_t;
-#endif
-
-struct prof_cnt_s {
-	/* Profiling counters. */
-	uint64_t curobjs;
-	uint64_t curobjs_shifted_unbiased;
-	uint64_t curbytes;
-	uint64_t curbytes_unbiased;
-	uint64_t accumobjs;
-	uint64_t accumobjs_shifted_unbiased;
-	uint64_t accumbytes;
-	uint64_t accumbytes_unbiased;
-};
-
-typedef enum {
-	prof_tctx_state_initializing,
-	prof_tctx_state_nominal,
-	prof_tctx_state_dumping,
-	prof_tctx_state_purgatory /* Dumper must finish destroying. */
-} prof_tctx_state_t;
-
-struct prof_tctx_s {
-	/* Thread data for thread that performed the allocation. */
-	prof_tdata_t *tdata;
-
-	/*
-	 * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
-	 * defunct during teardown.
-	 */
-	uint64_t thr_uid;
-	uint64_t thr_discrim;
-
-	/*
-	 * Reference count of how many times this tctx object is referenced in
-	 * recent allocation / deallocation records, protected by tdata->lock.
-	 */
-	uint64_t recent_count;
-
-	/* Profiling counters, protected by tdata->lock. */
-	prof_cnt_t cnts;
-
-	/* Associated global context. */
-	prof_gctx_t *gctx;
-
-	/*
-	 * UID that distinguishes multiple tctx's created by the same thread,
-	 * but coexisting in gctx->tctxs.  There are two ways that such
-	 * coexistence can occur:
-	 * - A dumper thread can cause a tctx to be retained in the purgatory
-	 *   state.
-	 * - Although a single "producer" thread must create all tctx's which
-	 *   share the same thr_uid, multiple "consumers" can each concurrently
-	 *   execute portions of prof_tctx_destroy().  prof_tctx_destroy() only
-	 *   gets called once each time cnts.cur{objs,bytes} drop to 0, but this
-	 *   threshold can be hit again before the first consumer finishes
-	 *   executing prof_tctx_destroy().
-	 */
-	uint64_t tctx_uid;
-
-	/* Linkage into gctx's tctxs. */
-	rb_node(prof_tctx_t) tctx_link;
-
-	/*
-	 * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
-	 * sample vs destroy race.
-	 */
-	bool prepared;
-
-	/* Current dump-related state, protected by gctx->lock. */
-	prof_tctx_state_t state;
-
-	/*
-	 * Copy of cnts snapshotted during early dump phase, protected by
-	 * dump_mtx.
-	 */
-	prof_cnt_t dump_cnts;
-};
-typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
-
-struct prof_info_s {
-	/* Time when the allocation was made. */
-	nstime_t alloc_time;
-	/* Points to the prof_tctx_t corresponding to the allocation. */
-	prof_tctx_t *alloc_tctx;
-	/* Allocation request size. */
-	size_t alloc_size;
-};
-
-struct prof_gctx_s {
-	/* Protects nlimbo, cnt_summed, and tctxs. */
-	malloc_mutex_t *lock;
-
-	/*
-	 * Number of threads that currently cause this gctx to be in a state of
-	 * limbo due to one of:
-	 *   - Initializing this gctx.
-	 *   - Initializing per thread counters associated with this gctx.
-	 *   - Preparing to destroy this gctx.
-	 *   - Dumping a heap profile that includes this gctx.
-	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
-	 * gctx.
-	 */
-	unsigned nlimbo;
-
-	/*
-	 * Tree of profile counters, one for each thread that has allocated in
-	 * this context.
-	 */
-	prof_tctx_tree_t tctxs;
-
-	/* Linkage for tree of contexts to be dumped. */
-	rb_node(prof_gctx_t) dump_link;
-
-	/* Temporary storage for summation during dump. */
-	prof_cnt_t cnt_summed;
-
-	/* Associated backtrace. */
-	prof_bt_t bt;
-
-	/* Backtrace vector, variable size, referred to by bt. */
-	void *vec[1];
-};
-typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
-
-struct prof_tdata_s {
-	malloc_mutex_t *lock;
-
-	/* Monotonically increasing unique thread identifier. */
-	uint64_t thr_uid;
-
-	/*
-	 * Monotonically increasing discriminator among tdata structures
-	 * associated with the same thr_uid.
-	 */
-	uint64_t thr_discrim;
-
-	rb_node(prof_tdata_t) tdata_link;
-
-	/*
-	 * Counter used to initialize prof_tctx_t's tctx_uid.  No locking is
-	 * necessary when incrementing this field, because only one thread ever
-	 * does so.
-	 */
-	uint64_t tctx_uid_next;
-
-	/*
-	 * Hash of (prof_bt_t *)-->(prof_tctx_t *).  Each thread tracks
-	 * backtraces for which it has non-zero allocation/deallocation counters
-	 * associated with thread-specific prof_tctx_t objects.  Other threads
-	 * may write to prof_tctx_t contents when freeing associated objects.
-	 */
-	ckh_t bt2tctx;
-
-	/* Included in heap profile dumps if has content. */
-	char thread_name[PROF_THREAD_NAME_MAX_LEN];
-
-	/* State used to avoid dumping while operating on prof internals. */
-	bool enq;
-	bool enq_idump;
-	bool enq_gdump;
-
-	/*
-	 * Set to true during an early dump phase for tdata's which are
-	 * currently being dumped.  New threads' tdata's have this initialized
-	 * to false so that they aren't accidentally included in later dump
-	 * phases.
-	 */
-	bool dumping;
-
-	/*
-	 * True if profiling is active for this tdata's thread
-	 * (thread.prof.active mallctl).
-	 */
-	bool active;
-
-	bool attached;
-	bool expired;
-
-	/* Temporary storage for summation during dump. */
-	prof_cnt_t cnt_summed;
-
-	/* Backtrace vector, used for calls to prof_backtrace(). */
-	void **vec;
-};
-typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
-
-struct prof_recent_s {
-	nstime_t alloc_time;
-	nstime_t dalloc_time;
-
-	ql_elm(prof_recent_t) link;
-	size_t       size;
-	size_t       usize;
-	atomic_p_t   alloc_edata; /* NULL means allocation has been freed. */
-	prof_tctx_t *alloc_tctx;
-	prof_tctx_t *dalloc_tctx;
-};
-
-#endif /* JEMALLOC_INTERNAL_PROF_STRUCTS_H */

include/jemalloc/internal/prof_sys.h

@@ -4,6 +4,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prof.h"
 
 extern malloc_mutex_t prof_dump_filename_mtx;
 extern base_t        *prof_base;
@@ -20,7 +21,6 @@ void prof_fdump_impl(tsd_t *tsd);
 void prof_idump_impl(tsd_t *tsd);
 bool prof_mdump_impl(tsd_t *tsd, const char *filename);
 void prof_gdump_impl(tsd_t *tsd);
-int  prof_thread_stack_range(uintptr_t fp, uintptr_t *low, uintptr_t *high);
 
 /* Used in unit tests. */
 typedef int(prof_sys_thread_name_read_t)(char *buf, size_t limit);

include/jemalloc/internal/prof_types.h

@@ -1,94 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PROF_TYPES_H
-#define JEMALLOC_INTERNAL_PROF_TYPES_H
-
-typedef struct prof_bt_s     prof_bt_t;
-typedef struct prof_cnt_s    prof_cnt_t;
-typedef struct prof_tctx_s   prof_tctx_t;
-typedef struct prof_info_s   prof_info_t;
-typedef struct prof_gctx_s   prof_gctx_t;
-typedef struct prof_tdata_s  prof_tdata_t;
-typedef struct prof_recent_s prof_recent_t;
-
-/* Option defaults. */
-#ifdef JEMALLOC_PROF
-#	define PROF_PREFIX_DEFAULT "jeprof"
-#else
-#	define PROF_PREFIX_DEFAULT ""
-#endif
-#define LG_PROF_SAMPLE_DEFAULT 19
-#define LG_PROF_INTERVAL_DEFAULT -1
-
-/*
- * Hard limit on stack backtrace depth.  The version of prof_backtrace() that
- * is based on __builtin_return_address() necessarily has a hard-coded number
- * of backtrace frame handlers, and should be kept in sync with this setting.
- */
-#ifdef JEMALLOC_PROF_GCC
-#	define PROF_BT_MAX_LIMIT 256
-#else
-#	define PROF_BT_MAX_LIMIT UINT_MAX
-#endif
-#define PROF_BT_MAX_DEFAULT 128
-
-/* Initial hash table size. */
-#define PROF_CKH_MINITEMS 64
-
-/* Size of memory buffer to use when writing dump files. */
-#ifndef JEMALLOC_PROF
-/* Minimize memory bloat for non-prof builds. */
-#	define PROF_DUMP_BUFSIZE 1
-#elif defined(JEMALLOC_DEBUG)
-/* Use a small buffer size in debug build, mainly to facilitate testing. */
-#	define PROF_DUMP_BUFSIZE 16
-#else
-#	define PROF_DUMP_BUFSIZE 65536
-#endif
-
-/* Size of size class related tables */
-#ifdef JEMALLOC_PROF
-#	define PROF_SC_NSIZES SC_NSIZES
-#else
-/* Minimize memory bloat for non-prof builds. */
-#	define PROF_SC_NSIZES 1
-#endif
-
-/* Size of stack-allocated buffer used by prof_printf(). */
-#define PROF_PRINTF_BUFSIZE 128
-
-/*
- * Number of mutexes shared among all gctx's.  No space is allocated for these
- * unless profiling is enabled, so it's okay to over-provision.
- */
-#define PROF_NCTX_LOCKS 1024
-
-/*
- * Number of mutexes shared among all tdata's.  No space is allocated for these
- * unless profiling is enabled, so it's okay to over-provision.
- */
-#define PROF_NTDATA_LOCKS 256
-
-/* Minimize memory bloat for non-prof builds. */
-#ifdef JEMALLOC_PROF
-#	define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
-#else
-#	define PROF_DUMP_FILENAME_LEN 1
-#endif
-
-/* Default number of recent allocations to record. */
-#define PROF_RECENT_ALLOC_MAX_DEFAULT 0
-
-/* Thread name storage size limit. */
-#define PROF_THREAD_NAME_MAX_LEN 16
-
-/*
- * Minimum required alignment for sampled allocations. Over-aligning sampled
- * allocations allows us to quickly identify them on the dalloc path without
- * resorting to metadata lookup.
- */
-#define PROF_SAMPLE_ALIGNMENT PAGE
-#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK
-
-/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
-#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U))
-
-#endif /* JEMALLOC_INTERNAL_PROF_TYPES_H */

include/jemalloc/internal/safety_check.h

@@ -11,11 +11,6 @@ void safety_check_fail_sized_dealloc(
     bool current_dealloc, const void *ptr, size_t true_size, size_t input_size);
 void safety_check_fail(const char *format, ...);
 
-typedef void (*safety_check_abort_hook_t)(const char *message);
-
-/* Can set to NULL for a default. */
-void safety_check_set_abort(safety_check_abort_hook_t abort_fn);
-
 #define REDZONE_SIZE ((size_t)32)
 #define REDZONE_FILL_VALUE 0xBC
 

include/jemalloc/internal/sc.h

@@ -368,7 +368,6 @@ struct sc_data_s {
 	sc_t sc[SC_NSIZES];
 };
 
-size_t reg_size_compute(int lg_base, int lg_delta, int ndelta);
 void   sc_data_init(sc_data_t *data);
 /*
  * Updates slab sizes in [begin, end] to be pgs pages in length, if possible.

include/jemalloc/internal/sec.h

@@ -35,19 +35,20 @@ typedef struct sec_stats_s sec_stats_t;
 struct sec_stats_s {
 	/* Sum of bytes_cur across all shards. */
 	size_t bytes;
+	/* Subset of bytes that are pinned. */
+	size_t bytes_pinned;
 
 	/* Totals of bin_stats. */
 	sec_bin_stats_t total;
 };
 
-static inline void
-sec_bin_stats_init(sec_bin_stats_t *stats) {
-	stats->ndalloc_flush = 0;
-	stats->nmisses = 0;
-	stats->nhits = 0;
-	stats->ndalloc_noflush = 0;
-	stats->noverfills = 0;
-}
+typedef struct sec_pszind_stats_s sec_pszind_stats_t;
+struct sec_pszind_stats_s {
+	size_t nextents;
+	size_t bytes;
+	size_t nextents_pinned;
+	size_t bytes_pinned;
+};
 
 static inline void
 sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
@@ -61,6 +62,7 @@ sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
 static inline void
 sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 	dst->bytes += src->bytes;
+	dst->bytes_pinned += src->bytes_pinned;
 	sec_bin_stats_accum(&dst->total, &src->total);
 }
 
@@ -68,16 +70,21 @@ sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 typedef struct sec_bin_s sec_bin_t;
 struct sec_bin_s {
 	/*
-	 * Protects the data members of the bin.
+	 * Protects the freelist and synchronizes counter updates.
 	 */
 	malloc_mutex_t mtx;
 
 	/*
 	 * Number of bytes in this particular bin.
 	 */
-	size_t              bytes_cur;
+	atomic_zu_t         bytes_cur;
+	atomic_zu_t         bytes_pinned_cur;
 	edata_list_active_t freelist;
-	sec_bin_stats_t     stats;
+	atomic_zu_t         nmisses;
+	atomic_zu_t         nhits;
+	atomic_zu_t         ndalloc_flush;
+	atomic_zu_t         ndalloc_noflush;
+	atomic_zu_t         noverfills;
 };
 
 typedef struct sec_s sec_t;
@@ -88,13 +95,13 @@ struct sec_s {
 };
 
 static inline bool
-sec_is_used(sec_t *sec) {
+sec_is_used(const sec_t *sec) {
 	return sec->opts.nshards != 0;
 }
 
 static inline bool
-sec_size_supported(sec_t *sec, size_t size) {
-	return sec_is_used(sec) && size <= sec->opts.max_alloc;
+sec_size_supported(const sec_t *sec, size_t size) {
+	return size <= sec->opts.max_alloc;
 }
 
 /* Min number of extents we will allocate when expanding the SEC. */
@@ -116,12 +123,21 @@ sec_size_supported(sec_t *sec, size_t size) {
 void sec_calc_nallocs_for_size(
     sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
 
-/* If sec does not have extent available, it will return NULL. */
-edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
+/*
+ * Lazily picks (and caches in *idxp) a shard for the calling thread.  Different
+ * SEC instances pass independent per-thread uint8_t slots, initialized to
+ * (uint8_t)-1.
+ */
+uint8_t sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp);
+
+/* Callers must ensure sec_size_supported(sec, size). */
+edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard);
 void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
-        edata_list_active_t *result, size_t nallocs);
+        edata_list_active_t *result, size_t nallocs, uint8_t shard);
 
 /*
+ * Callers must ensure sec_size_supported(sec, edata_size).
+ *
  * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty
  * if there are extents that need to be flushed from cache.  Please note, that
  * if we need to flush, extent(s) returned in the list to be deallocated
@@ -129,7 +145,8 @@ void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
  * considered "hot" and preserved in the cache, while "colder" ones are
  * returned).
  */
-void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list);
+void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
+    uint8_t shard);
 
 bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts);
 
@@ -142,7 +159,10 @@ void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
  * lets them fit easily into the pa_shard stats framework (which also has this
  * split), which simplifies the stats management.
  */
-void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge_pszind(
+    tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
+    sec_pszind_stats_t *stats);
 void sec_mutex_stats_read(
     tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data);
 

include/jemalloc/internal/stats.h

@@ -43,6 +43,9 @@ extern char    opt_stats_interval_opts[stats_print_tot_num_options + 1];
 #define STATS_INTERVAL_ACCUM_LG_BATCH_SIZE 6
 #define STATS_INTERVAL_ACCUM_BATCH_MAX (4 << 20)
 
+/* Per thread batch accum size for stats_interval; read by thread event. */
+extern uint64_t stats_interval_accum_batch;
+
 /* Only accessed by thread event. */
 extern te_base_cb_t stats_interval_te_handler;
 

include/jemalloc/internal/stats_internal.h

@@ -0,0 +1,340 @@
+#ifndef JEMALLOC_INTERNAL_STATS_INTERNAL_H
+#define JEMALLOC_INTERNAL_STATS_INTERNAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena.h" /* ARENA_NAME_LEN */
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/emitter.h"
+#include "jemalloc/internal/prof_stats.h"
+
+/*
+ * Internal implementation support for src/stats.c only -- do NOT include from
+ * other translation units.
+ *
+ * This header holds noisy local scaffolding for malloc_stats_print(): ctl
+ * gather plumbing, gather structs, emitter table-column helpers, and option
+ * list helpers.  It is deliberately not a reusable stats-gather API.
+ */
+
+#define CTL_GET(n, v, t)                                                       \
+	do {                                                                   \
+		size_t sz = sizeof(t);                                         \
+		xmallctl(n, (void *)v, &sz, NULL, 0);                          \
+	} while (0)
+
+#define CTL_LEAF_PREPARE(mib, miblen, name)                                    \
+	do {                                                                   \
+		assert(miblen < CTL_MAX_DEPTH);                                \
+		size_t miblen_new = CTL_MAX_DEPTH;                             \
+		xmallctlmibnametomib(mib, miblen, name, &miblen_new);          \
+		assert(miblen_new > miblen);                                   \
+	} while (0)
+
+#define CTL_LEAF(mib, miblen, leaf, v, t)                                      \
+	do {                                                                   \
+		assert(miblen < CTL_MAX_DEPTH);                                \
+		size_t miblen_new = CTL_MAX_DEPTH;                             \
+		size_t sz = sizeof(t);                                         \
+		xmallctlbymibname(                                             \
+		    mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0);  \
+		assert(miblen_new == miblen + 1);                              \
+	} while (0)
+
+#define CTL_MIB_GET(n, i, v, t, ind)                                           \
+	do {                                                                   \
+		size_t mib[CTL_MAX_DEPTH];                                     \
+		size_t miblen = sizeof(mib) / sizeof(size_t);                  \
+		size_t sz = sizeof(t);                                         \
+		xmallctlnametomib(n, mib, &miblen);                            \
+		mib[(ind)] = (i);                                              \
+		xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0);           \
+	} while (0)
+
+#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1)
+#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2)
+
+/******************************************************************************/
+/*
+ * Per-section gather structs.  The stats_gather_*() functions that fill them
+ * are static in src/stats.c.
+ */
+
+/* One arena's memory-accounting stats (stats.arenas.<i>.*). */
+typedef struct stats_arena_mem_s {
+	size_t mapped;
+	size_t retained;
+	size_t pinned;
+	size_t base;
+	size_t internal;
+	size_t metadata_edata;
+	size_t metadata_rtree;
+	size_t metadata_thp;
+	size_t tcache_bytes;
+	size_t tcache_stashed_bytes;
+	size_t resident;
+	size_t abandoned_vm;
+	size_t extent_avail;
+	size_t page; /* arenas.page, for the active-bytes conversion */
+} stats_arena_mem_t;
+
+/* One arena extents row (stats.arenas.<i>.extents.<pszind>.*). */
+typedef struct stats_arena_extent_s {
+	size_t ndirty;
+	size_t nmuzzy;
+	size_t nretained;
+	size_t npinned;
+	size_t ntotal;
+	size_t dirty_bytes;
+	size_t muzzy_bytes;
+	size_t retained_bytes;
+	size_t pinned_bytes;
+	size_t total_bytes;
+} stats_arena_extent_t;
+
+/* One arena's "basics" (stats.arenas.<i>.{nthreads,uptime,dss}). */
+typedef struct stats_arena_basics_s {
+	unsigned    nthreads;
+	uint64_t    uptime;
+	const char *dss;
+	char        name[ARENA_NAME_LEN];
+	bool        has_name; /* false for the merged/destroyed pseudo-arenas */
+} stats_arena_basics_t;
+
+/* One arena's decay / page stats (stats.arenas.<i>.*). */
+typedef struct stats_arena_decay_s {
+	ssize_t  dirty_decay_ms;
+	ssize_t  muzzy_decay_ms;
+	size_t   pactive;
+	size_t   pdirty;
+	size_t   pmuzzy;
+	uint64_t dirty_npurge;
+	uint64_t dirty_nmadvise;
+	uint64_t dirty_purged;
+	uint64_t muzzy_npurge;
+	uint64_t muzzy_nmadvise;
+	uint64_t muzzy_purged;
+} stats_arena_decay_t;
+
+/* One size class of arena allocation counts (stats.arenas.<i>.{small,large}.*). */
+typedef struct stats_arena_alloc_s {
+	size_t   allocated;
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	uint64_t nfills;
+	uint64_t nflushes;
+} stats_arena_alloc_t;
+
+/*
+ * One arena bin row (stats.arenas.<i>.bins.<j>.* + arenas.bin.<j>.*,
+ * plus prof.stats.bins.<j>.{live,accum} when profiling stats are enabled).
+ */
+typedef struct stats_arena_bin_s {
+	uint64_t nslabs;
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	uint64_t nfills;
+	uint64_t nflushes;
+	uint64_t nreslabs;
+	size_t   reg_size;
+	size_t   slab_size;
+	size_t   curregs;
+	size_t   curslabs;
+	size_t   nonfull_slabs;
+	uint32_t nregs;
+	uint32_t nshards;
+	prof_stats_t prof_live;
+	prof_stats_t prof_accum;
+} stats_arena_bin_t;
+
+/*
+ * One arena large-extent row (stats.arenas.<i>.lextents.<j>.* +
+ * arenas.lextent.<j>.*, plus prof.stats.lextents.<j>.{live,accum} when
+ * profiling stats are enabled).
+ */
+typedef struct stats_arena_lextent_s {
+	uint64_t nmalloc;
+	uint64_t ndalloc;
+	uint64_t nrequests;
+	size_t   lextent_size;
+	size_t   curlextents;
+	prof_stats_t prof_live;
+	prof_stats_t prof_accum;
+} stats_arena_lextent_t;
+
+/* HPA shard small-extent cache (stats.arenas.<i>.hpa_sec_*). */
+typedef struct stats_arena_hpa_sec_s {
+	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+	size_t sec_overfills;
+} stats_arena_hpa_sec_t;
+
+/* PAC small-extent cache (stats.arenas.<i>.pac_sec_*). */
+typedef struct stats_arena_pac_sec_s {
+	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+} stats_arena_pac_sec_t;
+
+/* HPA shard counters (stats.arenas.<i>.hpa_shard.*). */
+typedef struct stats_arena_hpa_counters_s {
+	size_t npageslabs;
+	size_t nactive;
+	size_t ndirty;
+	size_t npageslabs_nonhuge;
+	size_t nactive_nonhuge;
+	size_t ndirty_nonhuge;
+	size_t nretained_nonhuge;
+	size_t npageslabs_huge;
+	size_t nactive_huge;
+	size_t ndirty_huge;
+	uint64_t npurge_passes;
+	uint64_t npurges;
+	uint64_t nhugifies;
+	uint64_t nhugify_failures;
+	uint64_t ndehugifies;
+} stats_arena_hpa_counters_t;
+
+/* One HPA slab class (full/empty/nonfull) row. */
+typedef struct stats_arena_hpa_slab_s {
+	size_t npageslabs_huge;
+	size_t nactive_huge;
+	size_t ndirty_huge;
+	size_t npageslabs_nonhuge;
+	size_t nactive_nonhuge;
+	size_t ndirty_nonhuge;
+	size_t nretained_nonhuge;
+} stats_arena_hpa_slab_t;
+
+/* Arena configuration / size-class scalars (arenas.*). */
+typedef struct stats_arena_config_s {
+	unsigned narenas;
+	ssize_t  dirty_decay_ms;
+	ssize_t  muzzy_decay_ms;
+	size_t   quantum;
+	size_t   page;
+	size_t   hugepage;
+	bool     have_tcache_max;
+	size_t   tcache_max;
+	unsigned nbins;
+	unsigned nhbins;
+	unsigned nlextents;
+} stats_arena_config_t;
+
+/* Per-size-class geometry (emitted in JSON only). */
+typedef struct stats_arena_bin_meta_s {
+	size_t   size;
+	uint32_t nregs;
+	size_t   slab_size;
+	uint32_t nshards;
+} stats_arena_bin_meta_t;
+
+typedef struct stats_arena_lextent_meta_s {
+	size_t size;
+} stats_arena_lextent_meta_t;
+
+/* Process-wide global stats (stats.* + stats.background_thread.*). */
+typedef struct stats_global_s {
+	size_t   allocated;
+	size_t   active;
+	size_t   metadata;
+	size_t   metadata_edata;
+	size_t   metadata_rtree;
+	size_t   metadata_thp;
+	size_t   resident;
+	size_t   mapped;
+	size_t   retained;
+	size_t   pinned;
+	size_t   zero_reallocs;
+	size_t   num_background_threads;
+	uint64_t background_thread_num_runs;
+	uint64_t background_thread_run_interval;
+} stats_global_t;
+
+/******************************************************************************/
+/*
+ * Emitter table-column helper macros: declare an emitter_col_t (and, for
+ * COL_HDR, its header column) and wire it into a row.
+ */
+
+#define COL_DECLARE(column_name) emitter_col_t col_##column_name;
+
+#define COL_INIT(row_name, column_name, left_or_right, col_width, etype)       \
+	emitter_col_init(&col_##column_name, &row_name);                       \
+	col_##column_name.justify = emitter_justify_##left_or_right;           \
+	col_##column_name.width = col_width;                                   \
+	col_##column_name.type = emitter_type_##etype;
+
+#define COL(row_name, column_name, left_or_right, col_width, etype)            \
+	COL_DECLARE(column_name);                                              \
+	COL_INIT(row_name, column_name, left_or_right, col_width, etype)
+
+#define COL_HDR_DECLARE(column_name)                                           \
+	COL_DECLARE(column_name);                                              \
+	emitter_col_t header_##column_name;
+
+#define COL_HDR_INIT(                                                          \
+    row_name, column_name, human, left_or_right, col_width, etype)             \
+	COL_INIT(row_name, column_name, left_or_right, col_width, etype)       \
+	emitter_col_init(&header_##column_name, &header_##row_name);           \
+	header_##column_name.justify = emitter_justify_##left_or_right;        \
+	header_##column_name.width = col_width;                                \
+	header_##column_name.type = emitter_type_title;                        \
+	header_##column_name.str_val = human ? human : #column_name;
+
+#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \
+	COL_HDR_DECLARE(column_name)                                           \
+	COL_HDR_INIT(                                                          \
+	    row_name, column_name, human, left_or_right, col_width, etype)
+
+/*
+ * stats_general_print() config/option list helpers.  Unhygienic by design:
+ * they assume `emitter` and the standard scratch locals are in scope at the
+ * call site -- bv/bsz (bool), cpv/cpsz (const char *), uv/usz (unsigned),
+ * i64v/i64sz, u64v/u64sz, sv/ssz (size_t), ssv/sssz (ssize_t), plus bv2/ssv2
+ * for the *_MUTABLE variants.  (CONFIG_WRITE_BOOL uses CTL_GET above; the
+ * OPT_WRITE_* variants use je_mallctl directly.)
+ */
+#define CONFIG_WRITE_BOOL(name)                                                \
+	do {                                                                   \
+		CTL_GET("config." #name, &bv, bool);                           \
+		emitter_kv(                                                    \
+		    emitter, #name, "config." #name, emitter_type_bool, &bv);  \
+	} while (0)
+
+#define OPT_WRITE(name, var, size, emitter_type)                               \
+	if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) {      \
+		emitter_kv(emitter, name, "opt." name, emitter_type, &var);    \
+	}
+
+#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname)       \
+	if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0        \
+	    && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) {      \
+		emitter_kv_note(emitter, name, "opt." name, emitter_type,      \
+		    &var1, altname, emitter_type, &var2);                      \
+	}
+
+#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool)
+#define OPT_WRITE_BOOL_MUTABLE(name, altname)                                  \
+	OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname)
+
+#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned)
+
+#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64)
+#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64)
+
+#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size)
+#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize)
+#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname)                               \
+	OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname)
+
+#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string)
+
+#endif /* JEMALLOC_INTERNAL_STATS_INTERNAL_H */

include/jemalloc/internal/sz.h

@@ -3,6 +3,7 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/bit_util.h"
+#include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/util.h"
@@ -324,11 +325,13 @@ sz_s2u_compute_using_delta(size_t size) {
 	    : x - SC_LG_NGROUP - 1;
 	size_t delta = ZU(1) << lg_delta;
 	size_t delta_mask = delta - 1;
-	if (unlikely(size > SIZE_T_MAX - delta_mask)) {
+
+	size_t usize = size + delta_mask;
+	if (unlikely(usize < size)) {
+		/* size_t overflow. */
 		return 0;
 	}
-
-	size_t usize = (size + delta_mask) & ~delta_mask;
+	usize &= ~delta_mask;
 	return usize;
 }
 
@@ -358,7 +361,12 @@ sz_s2u_compute(size_t size) {
 		 * multiple of PAGE to minimize the memory overhead, especially
 		 * when using hugepages.
 		 */
-		size_t usize = PAGE_CEILING(size);
+		size_t usize = size + PAGE_MASK;
+		if (unlikely(usize < size)) {
+			/* size_t overflow. */
+			return 0;
+		}
+		usize &= ~PAGE_MASK;
 		assert(usize - size < PAGE);
 		return usize;
 	}
@@ -415,10 +423,12 @@ sz_sa2u(size_t size, size_t alignment) {
 		 *    144 | 10100000 |  32
 		 *    192 | 11000000 |  64
 		 */
-		if (unlikely(size > SIZE_T_MAX - alignment_mask)) {
+		size_t rounded = size + alignment_mask;
+		if (unlikely(rounded < size)) {
+			/* size_t overflow. */
 			return 0;
 		}
-		usize = sz_s2u((size + alignment_mask) & ~alignment_mask);
+		usize = sz_s2u(rounded & ~alignment_mask);
 		if (usize < SC_LARGE_MINCLASS) {
 			return usize;
 		}

include/jemalloc/internal/tcache.h

@@ -0,0 +1,199 @@
+#ifndef JEMALLOC_INTERNAL_TCACHE_H
+#define JEMALLOC_INTERNAL_TCACHE_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/cache_bin.h"
+#include "jemalloc/internal/ql.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/thread_event_registry.h"
+#include "jemalloc/internal/ticker.h"
+
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s arena_t;
+typedef struct base_s  base_t;
+
+/******************************************************************************/
+/* TYPES */
+/******************************************************************************/
+
+typedef struct tcache_slow_s tcache_slow_t;
+typedef struct tcache_s      tcache_t;
+typedef struct tcaches_s     tcaches_t;
+
+/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
+#define TCACHE_ZERO_INITIALIZER                                                \
+	{ 0 }
+#define TCACHE_SLOW_ZERO_INITIALIZER                                           \
+	{                                                                      \
+		{ 0 }                                                          \
+	}
+
+/* Used in TSD static initializer only. Will be initialized to opt_tcache. */
+#define TCACHE_ENABLED_ZERO_INITIALIZER false
+
+/* Used for explicit tcache only. Means flushed but not destroyed. */
+/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
+#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1)
+
+#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
+#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
+#define TCACHE_NBINS_MAX                                                       \
+	(SC_NBINS                                                              \
+	    + SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS)    \
+	    + 1)
+#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21)       /* 2M */
+#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
+#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)
+#define TCACHE_GC_LARGE_NBINS_MAX 1
+
+/******************************************************************************/
+/* STRUCTS */
+/******************************************************************************/
+
+/*
+ * The tcache state is split into the slow and hot path data.  Each has a
+ * pointer to the other, and the data always comes in pairs.  The layout of each
+ * of them varies in practice; tcache_slow lives in the TSD for the automatic
+ * tcache, and as part of a dynamic allocation for manual allocations.  Keeping
+ * a pointer to tcache_slow lets us treat these cases uniformly, rather than
+ * splitting up the tcache [de]allocation code into those paths called with the
+ * TSD tcache and those called with a manual tcache.
+ */
+
+struct tcache_slow_s {
+	/*
+	 * The descriptor lets the arena find our cache bins without seeing the
+	 * tcache definition.  This enables arenas to aggregate stats across
+	 * tcaches without having a tcache dependency.
+	 */
+	cache_bin_array_descriptor_t cache_bin_array_descriptor;
+
+	/* The arena this tcache is associated with. */
+	arena_t *arena;
+	/* The number of bins activated in the tcache. */
+	unsigned tcache_nbins;
+	/* Last time GC has been performed.  */
+	nstime_t last_gc_time;
+	/* Next bin to GC. */
+	szind_t next_gc_bin;
+	szind_t next_gc_bin_small;
+	szind_t next_gc_bin_large;
+	/* For small bins, help determine how many items to fill at a time. */
+	cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS];
+	/* For small bins, whether has been refilled since last GC. */
+	bool bin_refilled[SC_NBINS];
+	/*
+	 * For small bins, the number of items we can pretend to flush before
+	 * actually flushing.
+	 */
+	uint8_t bin_flush_delay_items[SC_NBINS];
+	/*
+	 * The start of the allocation containing the dynamic allocation for
+	 * either the cache bins alone, or the cache bin memory as well as this
+	 * tcache_slow_t and its associated tcache_t.
+	 */
+	void *dyn_alloc;
+
+	/* The associated bins. */
+	tcache_t *tcache;
+};
+
+struct tcache_s {
+	tcache_slow_t *tcache_slow;
+	cache_bin_t    bins[TCACHE_NBINS_MAX];
+};
+
+/* Linkage for list of available (previously used) explicit tcache IDs. */
+struct tcaches_s {
+	union {
+		tcache_t  *tcache;
+		tcaches_t *next;
+	};
+};
+
+/******************************************************************************/
+/* EXTERNS */
+/******************************************************************************/
+
+extern bool     opt_tcache;
+extern size_t   opt_tcache_max;
+extern ssize_t  opt_lg_tcache_nslots_mul;
+extern unsigned opt_tcache_nslots_small_min;
+extern unsigned opt_tcache_nslots_small_max;
+extern unsigned opt_tcache_nslots_large;
+extern ssize_t  opt_lg_tcache_shift;
+extern size_t   opt_tcache_gc_incr_bytes;
+extern size_t   opt_tcache_gc_delay_bytes;
+extern unsigned opt_lg_tcache_flush_small_div;
+extern unsigned opt_lg_tcache_flush_large_div;
+
+/*
+ * Number of tcache bins.  There are SC_NBINS small-object bins, plus 0 or more
+ * large-object bins.  This is only used during threads initialization and
+ * changing it will not reflect on initialized threads as expected.  Thus,
+ * it should not be changed on the fly.  To change the number of tcache bins
+ * in use, refer to tcache_nbins of each tcache.
+ */
+extern unsigned global_do_not_change_tcache_nbins;
+
+/*
+ * Maximum cached size class.  Same as above, this is only used during threads
+ * initialization and should not be changed.  To change the maximum cached size
+ * class, refer to tcache_max of each tcache.
+ */
+extern size_t global_do_not_change_tcache_maxclass;
+
+/*
+ * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
+ * usable via the MALLOCX_TCACHE() flag.  The automatic per thread tcaches are
+ * completely disjoint from this data structure.  tcaches starts off as a sparse
+ * array, so it has no physical memory footprint until individual pages are
+ * touched.  This allows the entire array to be allocated the first time an
+ * explicit tcache is created without a disproportionate impact on memory usage.
+ */
+extern tcaches_t *tcaches;
+
+size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
+void  *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
+     cache_bin_t *cache_bin, szind_t binind, bool *tcache_success);
+
+void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache,
+    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
+void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
+    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
+void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
+    cache_bin_t *cache_bin, szind_t binind, bool is_small);
+bool tcache_bin_info_default_init(
+    const char *bin_settings_segment_cur, size_t len_left);
+bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
+bool tcache_bin_ncached_max_read(
+    tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
+void tcache_arena_reassociate(
+    tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
+tcache_t *tcache_create_explicit(tsd_t *tsd);
+bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
+void      tcache_cleanup(tsd_t *tsd);
+bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
+void      tcaches_flush(tsd_t *tsd, unsigned ind);
+void      tcaches_destroy(tsd_t *tsd, unsigned ind);
+bool      tcache_boot(tsdn_t *tsdn, base_t *base);
+void      tcache_arena_associate(
+         tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
+cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor(
+         tsdn_t *tsdn, arena_t *arena);
+void tcache_prefork(tsdn_t *tsdn);
+void tcache_postfork_parent(tsdn_t *tsdn);
+void tcache_postfork_child(tsdn_t *tsdn);
+void tcache_flush(tsd_t *tsd);
+bool tsd_tcache_enabled_data_init(tsd_t *tsd);
+void tcache_enabled_set(tsd_t *tsd, bool enabled);
+
+extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size,
+    size_t alignment);
+
+void tcache_assert_initialized(tcache_t *tcache);
+
+extern te_base_cb_t tcache_gc_te_handler;
+
+#endif /* JEMALLOC_INTERNAL_TCACHE_H */

include/jemalloc/internal/tcache_externs.h

@@ -1,91 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
-#define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/base.h"
-#include "jemalloc/internal/cache_bin.h"
-#include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_types.h"
-#include "jemalloc/internal/thread_event_registry.h"
-
-extern bool     opt_tcache;
-extern size_t   opt_tcache_max;
-extern ssize_t  opt_lg_tcache_nslots_mul;
-extern unsigned opt_tcache_nslots_small_min;
-extern unsigned opt_tcache_nslots_small_max;
-extern unsigned opt_tcache_nslots_large;
-extern ssize_t  opt_lg_tcache_shift;
-extern size_t   opt_tcache_gc_incr_bytes;
-extern size_t   opt_tcache_gc_delay_bytes;
-extern unsigned opt_lg_tcache_flush_small_div;
-extern unsigned opt_lg_tcache_flush_large_div;
-
-/*
- * Number of tcache bins.  There are SC_NBINS small-object bins, plus 0 or more
- * large-object bins.  This is only used during threads initialization and
- * changing it will not reflect on initialized threads as expected.  Thus,
- * it should not be changed on the fly.  To change the number of tcache bins
- * in use, refer to tcache_nbins of each tcache.
- */
-extern unsigned global_do_not_change_tcache_nbins;
-
-/*
- * Maximum cached size class.  Same as above, this is only used during threads
- * initialization and should not be changed.  To change the maximum cached size
- * class, refer to tcache_max of each tcache.
- */
-extern size_t global_do_not_change_tcache_maxclass;
-
-/*
- * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
- * usable via the MALLOCX_TCACHE() flag.  The automatic per thread tcaches are
- * completely disjoint from this data structure.  tcaches starts off as a sparse
- * array, so it has no physical memory footprint until individual pages are
- * touched.  This allows the entire array to be allocated the first time an
- * explicit tcache is created without a disproportionate impact on memory usage.
- */
-extern tcaches_t *tcaches;
-
-size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
-void  *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
-     cache_bin_t *cache_bin, szind_t binind, bool *tcache_success);
-
-void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache,
-    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
-void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
-    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
-void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
-    cache_bin_t *cache_bin, szind_t binind, bool is_small);
-bool tcache_bin_info_default_init(
-    const char *bin_settings_segment_cur, size_t len_left);
-bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
-bool tcache_bin_ncached_max_read(
-    tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
-void tcache_arena_reassociate(
-    tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
-tcache_t *tcache_create_explicit(tsd_t *tsd);
-bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
-void      tcache_cleanup(tsd_t *tsd);
-bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
-void      tcaches_flush(tsd_t *tsd, unsigned ind);
-void      tcaches_destroy(tsd_t *tsd, unsigned ind);
-bool      tcache_boot(tsdn_t *tsdn, base_t *base);
-void      tcache_arena_associate(
-         tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
-cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor(
-         tsdn_t *tsdn, arena_t *arena);
-void tcache_prefork(tsdn_t *tsdn);
-void tcache_postfork_parent(tsdn_t *tsdn);
-void tcache_postfork_child(tsdn_t *tsdn);
-void tcache_flush(tsd_t *tsd);
-bool tsd_tcache_enabled_data_init(tsd_t *tsd);
-void tcache_enabled_set(tsd_t *tsd, bool enabled);
-
-extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size,
-    size_t alignment);
-
-void tcache_assert_initialized(tcache_t *tcache);
-
-extern te_base_cb_t tcache_gc_te_handler;
-
-#endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

include/jemalloc/internal/tcache_inlines.h

@@ -2,15 +2,15 @@
 #define JEMALLOC_INTERNAL_TCACHE_INLINES_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
+#include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/bin.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
-#include "jemalloc/internal/large_externs.h"
+#include "jemalloc/internal/large.h"
 #include "jemalloc/internal/san.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_externs.h"
+#include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/util.h"
 
 static inline bool

include/jemalloc/internal/tcache_structs.h

@@ -1,72 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H
-#define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/cache_bin.h"
-#include "jemalloc/internal/ql.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/tcache_types.h"
-#include "jemalloc/internal/ticker.h"
-
-/*
- * The tcache state is split into the slow and hot path data.  Each has a
- * pointer to the other, and the data always comes in pairs.  The layout of each
- * of them varies in practice; tcache_slow lives in the TSD for the automatic
- * tcache, and as part of a dynamic allocation for manual allocations.  Keeping
- * a pointer to tcache_slow lets us treat these cases uniformly, rather than
- * splitting up the tcache [de]allocation code into those paths called with the
- * TSD tcache and those called with a manual tcache.
- */
-
-struct tcache_slow_s {
-	/*
-	 * The descriptor lets the arena find our cache bins without seeing the
-	 * tcache definition.  This enables arenas to aggregate stats across
-	 * tcaches without having a tcache dependency.
-	 */
-	cache_bin_array_descriptor_t cache_bin_array_descriptor;
-
-	/* The arena this tcache is associated with. */
-	arena_t *arena;
-	/* The number of bins activated in the tcache. */
-	unsigned tcache_nbins;
-	/* Last time GC has been performed.  */
-	nstime_t last_gc_time;
-	/* Next bin to GC. */
-	szind_t next_gc_bin;
-	szind_t next_gc_bin_small;
-	szind_t next_gc_bin_large;
-	/* For small bins, help determine how many items to fill at a time. */
-	cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS];
-	/* For small bins, whether has been refilled since last GC. */
-	bool bin_refilled[SC_NBINS];
-	/*
-	 * For small bins, the number of items we can pretend to flush before
-	 * actually flushing.
-	 */
-	uint8_t bin_flush_delay_items[SC_NBINS];
-	/*
-	 * The start of the allocation containing the dynamic allocation for
-	 * either the cache bins alone, or the cache bin memory as well as this
-	 * tcache_slow_t and its associated tcache_t.
-	 */
-	void *dyn_alloc;
-
-	/* The associated bins. */
-	tcache_t *tcache;
-};
-
-struct tcache_s {
-	tcache_slow_t *tcache_slow;
-	cache_bin_t    bins[TCACHE_NBINS_MAX];
-};
-
-/* Linkage for list of available (previously used) explicit tcache IDs. */
-struct tcaches_s {
-	union {
-		tcache_t  *tcache;
-		tcaches_t *next;
-	};
-};
-
-#endif /* JEMALLOC_INTERNAL_TCACHE_STRUCTS_H */

include/jemalloc/internal/tcache_types.h

@@ -1,37 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H
-#define JEMALLOC_INTERNAL_TCACHE_TYPES_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/sc.h"
-
-typedef struct tcache_slow_s tcache_slow_t;
-typedef struct tcache_s      tcache_t;
-typedef struct tcaches_s     tcaches_t;
-
-/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
-#define TCACHE_ZERO_INITIALIZER                                                \
-	{ 0 }
-#define TCACHE_SLOW_ZERO_INITIALIZER                                           \
-	{                                                                      \
-		{ 0 }                                                          \
-	}
-
-/* Used in TSD static initializer only. Will be initialized to opt_tcache. */
-#define TCACHE_ENABLED_ZERO_INITIALIZER false
-
-/* Used for explicit tcache only. Means flushed but not destroyed. */
-/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
-#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1)
-
-#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
-#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
-#define TCACHE_NBINS_MAX                                                       \
-	(SC_NBINS                                                              \
-	    + SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS)    \
-	    + 1)
-#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21)       /* 2M */
-#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
-#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)
-#define TCACHE_GC_LARGE_NBINS_MAX 1
-
-#endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */

include/jemalloc/internal/test_hooks.h

@@ -1,10 +1,12 @@
 #ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H
 #define JEMALLOC_INTERNAL_TEST_HOOKS_H
 
-#include "jemalloc/internal/jemalloc_preamble.h"
-
 extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(void);
 extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void);
+extern JEMALLOC_EXPORT void (*test_hooks_safety_check_abort)(const char *);
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+extern JEMALLOC_EXPORT void (*test_hooks_tsd_bootstrap_hook)(void);
+#endif
 
 #if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
 #	define JEMALLOC_TEST_HOOK(fn, hook)                                   \

include/jemalloc/internal/tsd.h

@@ -4,7 +4,8 @@
 /*
  * We put the platform-specific data declarations and inlines into their own
  * header files to avoid cluttering this file.  They define tsd_boot0,
- * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates, tsd_get, and tsd_set.
+ * tsd_boot1, tsd_boot, tsd_booted_get, tsd_get_allocates,
+ * tsd_teardown_done, tsd_get, and tsd_set.
  */
 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
 #	include "jemalloc/internal/jemalloc_preamble.h"

include/jemalloc/internal/tsd_binshards.h

@@ -0,0 +1,24 @@
+#ifndef JEMALLOC_INTERNAL_TSD_BINSHARDS_H
+#define JEMALLOC_INTERNAL_TSD_BINSHARDS_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/sc.h"
+
+/*
+ * Per-thread cache of bin-shard assignments.  This lives in its own header
+ * (separate from bin.h) so that tsd_internals.h can pull it in for X-macro
+ * expansion without dragging in mutex.h, which itself depends on TSD machinery
+ * and would form an include-order dependency cycle.
+ */
+
+#define TSD_BINSHARDS_ZERO_INITIALIZER                                         \
+	{                                                                      \
+		{ UINT8_MAX }                                                  \
+	}
+
+typedef struct tsd_binshards_s tsd_binshards_t;
+struct tsd_binshards_s {
+	uint8_t binshard[SC_NBINS];
+};
+
+#endif /* JEMALLOC_INTERNAL_TSD_BINSHARDS_H */

include/jemalloc/internal/tsd_generic.h

@@ -27,6 +27,7 @@ void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block);
 void  tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 
 extern pthread_key_t   tsd_tsd;
+extern pthread_key_t   tsd_thread_initialized_tsd;
 extern tsd_init_head_t tsd_init_head;
 extern tsd_wrapper_t   tsd_boot_wrapper;
 extern bool            tsd_booted;
@@ -51,6 +52,12 @@ tsd_cleanup_wrapper(void *arg) {
 			return;
 		}
 	}
+	/*
+	 * Leave tsd_thread_initialized_tsd set.  It has no destructor and
+	 * intentionally outlives the TSD wrapper, so later jemalloc calls can
+	 * distinguish this fully-torn-down thread from one that has never
+	 * initialized TSD.
+	 */
 	malloc_tsd_dalloc(wrapper);
 }
 
@@ -63,6 +70,11 @@ tsd_wrapper_set(tsd_wrapper_t *wrapper) {
 		malloc_write("<jemalloc>: Error setting TSD\n");
 		abort();
 	}
+	if (pthread_setspecific(tsd_thread_initialized_tsd,
+	    (void *)&tsd_thread_initialized_tsd) != 0) {
+		malloc_write("<jemalloc>: Error setting TSD\n");
+		abort();
+	}
 }
 
 JEMALLOC_ALWAYS_INLINE tsd_wrapper_t *
@@ -116,6 +128,15 @@ tsd_boot0(void) {
 	if (pthread_key_create(&tsd_tsd, tsd_cleanup_wrapper) != 0) {
 		return true;
 	}
+	/*
+	 * This key has no destructor.  It records that the current thread once
+	 * had jemalloc TSD, so a later NULL tsd_tsd value means teardown has
+	 * finished rather than initialization not having happened yet.
+	 */
+	if (pthread_key_create(&tsd_thread_initialized_tsd, NULL) != 0) {
+		pthread_key_delete(tsd_tsd);
+		return true;
+	}
 	tsd_booted = true;
 	tsd_wrapper_set(&tsd_boot_wrapper);
 	tsd_init_finish(&tsd_init_head, &block);
@@ -160,6 +181,12 @@ tsd_get_allocates(void) {
 	return true;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return tsd_booted && pthread_getspecific(tsd_tsd) == NULL
+	    && pthread_getspecific(tsd_thread_initialized_tsd) != NULL;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_internals.h

@@ -4,20 +4,27 @@
 #define JEMALLOC_INTERNAL_TSD_INTERNALS_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/arena_decay_constants.h"
 #include "jemalloc/internal/assert.h"
-#include "jemalloc/internal/bin_types.h"
+#include "jemalloc/internal/tsd_binshards.h"
 #include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/peak.h"
-#include "jemalloc/internal/prof_types.h"
 #include "jemalloc/internal/rtree_tsd.h"
-#include "jemalloc/internal/tcache_structs.h"
-#include "jemalloc/internal/tcache_types.h"
+#include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/thread_event_registry.h"
 #include "jemalloc/internal/tsd_types.h"
 #include "jemalloc/internal/util.h"
 #include "jemalloc/internal/witness.h"
 
+/*
+ * Forward decls.  tsd_internals.h cannot include arena.h / prof.h directly:
+ * those headers' STRUCTS-section includes trigger mutex.h -> tsd.h ->
+ * tsd_generic.h, which would re-enter this file before its body finishes.
+ * Each consumer here only uses these as pointer types.
+ */
+typedef struct arena_s      arena_t;
+typedef struct prof_tdata_s prof_tdata_t;
+
 /*
  * Thread-Specific-Data layout
  *
@@ -76,6 +83,7 @@ typedef void (*test_callback_t)(int *);
 	O(arena, arena_t *, arena_t *)                                         \
 	O(arena_decay_ticker, ticker_geom_t, ticker_geom_t)                    \
 	O(sec_shard, uint8_t, uint8_t)                                         \
+	O(pac_sec_shard, uint8_t, uint8_t)                                     \
 	O(binshards, tsd_binshards_t, tsd_binshards_t)                         \
 	O(peak, peak_t, peak_t)                                                \
 	O(tcache_slow, tcache_slow_t, tcache_slow_t)                           \
@@ -95,6 +103,7 @@ typedef void (*test_callback_t)(int *);
 	    /* arena */ NULL, /* arena_decay_ticker */                         \
 	    TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),                   \
 	    /* sec_shard */ (uint8_t) - 1,                                     \
+	    /* pac_sec_shard */ (uint8_t) - 1,                                 \
 	    /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER,                    \
 	    /* peak */ PEAK_INITIALIZER, /* tcache_slow */                     \
 	    TCACHE_SLOW_ZERO_INITIALIZER,                                      \

include/jemalloc/internal/tsd_malloc_thread_cleanup.h

@@ -50,6 +50,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_tls.h

@@ -43,6 +43,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

include/jemalloc/internal/tsd_win.h

@@ -146,6 +146,11 @@ tsd_get_allocates(void) {
 	return true;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
@@ -220,6 +225,11 @@ tsd_get_allocates(void) {
 	return false;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tsd_teardown_done(void) {
+	return false;
+}
+
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

scripts/README_GH_ACTIONS.md

@@ -27,12 +27,12 @@ The script can generate workflows for different platforms:
 
 ### Linux CI (`linux-ci.yml`)
 - **test-linux** (AMD64): `ubuntu-latest` (x86_64)
-  - ~96 configurations covering GCC, Clang, various flags
+  - ~98 configurations covering GCC, Clang, various flags
 - **test-linux-arm64** (ARM64): `ubuntu-24.04-arm` (aarch64)
   - ~14 configurations including large hugepage tests
   - **Note:** Free ARM64 runners (Public Preview) - may have longer queue times during peak hours
 
-**Total:** 110 configurations
+**Total:** 112 configurations
 
 ### macOS CI (`macos-ci.yml`)
 - **test-macos** (Intel): `macos-15-intel` (x86_64)
@@ -161,7 +161,7 @@ The Windows workflow uses:
 ### Linux Build Process
 - Ubuntu Latest for AMD64, Ubuntu 24.04 for ARM64
 - Installs 32-bit cross-compilation dependencies when needed
-- Most comprehensive test matrix (110 configurations)
+- Most comprehensive test matrix (112 configurations)
 
 ## Relationship to Travis CI
 
@@ -178,4 +178,3 @@ To regenerate all workflows after modifying `gen_gh_actions.py`:
 ```
 
 **Note**: The generated files should not be edited by hand. All changes should be made to `gen_gh_actions.py` and then regenerated.
-

scripts/gen_gh_actions.py

@@ -224,11 +224,8 @@ def generate_linux_job(arch):
     if arch != ARM64:
         exclude += [LARGE_HUGEPAGE]
 
-    linux_configure_flags = list(configure_flag_unusuals)
-    linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))
-
     linux_unusuals = (compilers_unusual + feature_unusuals
-                    + linux_configure_flags + malloc_conf_unusuals)
+                    + configure_flag_unusuals + malloc_conf_unusuals)
 
     matrix_entries = generate_job_matrix_entries(os, arch, exclude, max_unusual_opts, linux_unusuals)
 
@@ -273,9 +270,39 @@ def generate_linux_job(arch):
             'CXX': 'g++',
             'CONFIGURE_FLAGS': '--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof',
             'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': 'force_tls=0',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': 'force_tls=0 --enable-debug',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
         }
     ]
 
+    # --enable-cxx-infallible-new coverage. Plain variant runs on all arches;
+    # debug-combined variant runs only on AMD64 to bound matrix size.
+    infallible_new_entries = [
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        },
+    ]
+    if arch == AMD64:
+        infallible_new_entries.append({
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new --enable-debug',
+            'EXTRA_CFLAGS': '-Werror -Wno-array-bounds'
+        })
+
     if arch == AMD64:
         for entry in manual_entries:
             job += "          - env:\n"
@@ -285,6 +312,14 @@ def generate_linux_job(arch):
                 else:
                     job += f"              {key}: {value}\n"
 
+    for entry in infallible_new_entries:
+        job += "          - env:\n"
+        for key, value in entry.items():
+            if ' ' in str(value):
+                job += f'              {key}: "{value}"\n'
+            else:
+                job += f"              {key}: {value}\n"
+
     job += f"""
     steps:
     - uses: actions/checkout@v6
@@ -385,6 +420,24 @@ def generate_macos_job(arch):
             else:
                 job += f"              {key}: {value}\n"
 
+    # --enable-cxx-infallible-new coverage on macOS (both arches).
+    macos_extra_cflags = ' '.join(get_extra_cflags(OSX, GCC.value))
+    infallible_new_entries = [
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
+            'EXTRA_CFLAGS': macos_extra_cflags
+        },
+    ]
+    for entry in infallible_new_entries:
+        job += "          - env:\n"
+        for key, value in entry.items():
+            if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
+                job += f'              {key}: "{value}"\n'
+            else:
+                job += f"              {key}: {value}\n"
+
     job += f"""
     steps:
     - uses: actions/checkout@v6
@@ -467,6 +520,24 @@ def generate_windows_job(arch):
             else:
                 job += f"              {key}: {value}\n"
 
+    # --enable-cxx-infallible-new coverage on Windows (MinGW-GCC only).
+    windows_mingw_cflags = ' '.join(get_extra_cflags(WINDOWS, GCC.value))
+    infallible_new_entries = [
+        {
+            'CC': 'gcc',
+            'CXX': 'g++',
+            'CONFIGURE_FLAGS': '--enable-cxx-infallible-new',
+            'EXTRA_CFLAGS': windows_mingw_cflags
+        },
+    ]
+    for entry in infallible_new_entries:
+        job += "          - env:\n"
+        for key, value in entry.items():
+            if ' ' in str(value) or any(c in str(value) for c in [':', ',', '#']):
+                job += f'              {key}: "{value}"\n'
+            else:
+                job += f"              {key}: {value}\n"
+
     job += f"""
     steps:
     - uses: actions/checkout@v6

scripts/gen_header_dep_graph.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+# Generate header dependency graph + tsort cycle check.
+#
+# Outputs (under build/):
+#   header_deps_baseline.txt   - sorted, unique header->header (and .c->header)
+#                                edges derived from #include "..." directives.
+#   header_tsort_order.txt     - topological order produced by tsort.
+#   header_tsort_cycles.txt    - stderr from tsort; non-empty if there's a cycle.
+#   per_tu_deps.txt            - per-translation-unit transitive header lists
+#                                (one TU per line; basenames only; sorted).
+#
+# All headers / sources are reduced to basenames so renames in the cleanup are
+# easy to spot in the diff.
+
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$repo_root"
+mkdir -p build
+
+edges_raw=build/header_edges_raw.txt
+edges_baseline=build/header_deps_baseline.txt
+tsort_order=build/header_tsort_order.txt
+tsort_cycles=build/header_tsort_cycles.txt
+per_tu=build/per_tu_deps.txt
+
+: > "$edges_raw"
+
+extract_edges() {
+    local src="$1"
+    local bn
+    bn=$(basename "$src")
+    # Match `#include "..."` (skip `<...>` system includes). Strip path, keep
+    # only the basename of the included file, so the graph collapses
+    # `../jemalloc.h` and similar relative paths into a single node.
+    awk '
+        /^[[:space:]]*#[[:space:]]*include[[:space:]]+"/ {
+            match($0, /"[^"]+"/)
+            inc = substr($0, RSTART + 1, RLENGTH - 2)
+            n = split(inc, parts, "/")
+            print parts[n]
+        }
+    ' "$src" | while read -r dep; do
+        printf '%s %s\n' "$bn" "$dep"
+    done
+}
+
+# Headers (include/jemalloc/internal/) — these are the nodes we care about for
+# cycle detection.
+for f in include/jemalloc/internal/*.h include/jemalloc/*.h; do
+    extract_edges "$f" >> "$edges_raw"
+done
+
+# Translation units (src/*.c) — included so that .c files appear as graph
+# sources in the baseline. They can't introduce cycles (nothing #includes a
+# .c), but the edges are useful when diffing later.
+for f in src/*.c; do
+    extract_edges "$f" >> "$edges_raw"
+done
+
+sort -u "$edges_raw" > "$edges_baseline"
+
+# tsort consumes "A B" pairs and reports cycles on stderr.
+: > "$tsort_cycles"
+if ! tsort "$edges_baseline" > "$tsort_order" 2> "$tsort_cycles"; then
+    echo "tsort exited non-zero; see $tsort_cycles" >&2
+fi
+
+# Per-translation-unit transitive header lists, harvested from the .d files
+# the build already produced (CC_MM=1 in the Makefile).
+: > "$per_tu"
+for d in src/*.d; do
+    [ -f "$d" ] || continue
+    tu=$(basename "${d%.d}.c")
+    # Strip the make rule prefix ("foo.o: foo.c \"), drop line continuations,
+    # collapse to basenames, sort+uniq.
+    deps=$(
+        tr '\n' ' ' < "$d" \
+            | sed -E 's/\\//g' \
+            | tr ' ' '\n' \
+            | grep -E '\.h$' \
+            | awk -F/ '{print $NF}' \
+            | sort -u \
+            | tr '\n' ' ' \
+            | sed -E 's/[[:space:]]+$//'
+    )
+    printf '%s: %s\n' "$tu" "$deps" >> "$per_tu"
+done
+sort -o "$per_tu" "$per_tu"
+
+edge_count=$(wc -l < "$edges_baseline" | tr -d ' ')
+cycle_bytes=$(wc -c < "$tsort_cycles" | tr -d ' ')
+tu_count=$(wc -l < "$per_tu" | tr -d ' ')
+echo "edges: $edge_count"
+echo "translation units: $tu_count"
+echo "tsort cycle output bytes: $cycle_bytes"
+if [ "$cycle_bytes" -gt 0 ]; then
+    echo "---- tsort cycle report ----"
+    cat "$tsort_cycles"
+fi

scripts/gen_travis.py

@@ -264,11 +264,8 @@ def generate_linux(arch):
     if arch != ARM64:
         exclude += [LARGE_HUGEPAGE]
 
-    linux_configure_flags = list(configure_flag_unusuals)
-    linux_configure_flags.append(Option.as_configure_flag("--enable-prof --enable-prof-frameptr"))
-
     linux_unusuals = (compilers_unusual + feature_unusuals
-                    + linux_configure_flags + malloc_conf_unusuals)
+                    + configure_flag_unusuals + malloc_conf_unusuals)
 
     return generate_jobs(os, arch, exclude, max_unusual_opts, linux_unusuals)
 

src/arena.c

@@ -1,16 +1,27 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
 
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/background_thread_inlines.h"
+#include "jemalloc/internal/bin_inlines.h"
 #include "jemalloc/internal/decay.h"
 #include "jemalloc/internal/ehooks.h"
 #include "jemalloc/internal/extent_dss.h"
 #include "jemalloc/internal/extent_mmap.h"
-#include "jemalloc/internal/san.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
+#include "jemalloc/internal/large.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prof.h"
+#include "jemalloc/internal/prof_inlines.h"
 #include "jemalloc/internal/rtree.h"
 #include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/san.h"
+#include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/util.h"
+#include "jemalloc/internal/witness.h"
 
 JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
 
@@ -36,6 +47,14 @@ static pa_central_t arena_pa_central_global;
 
 div_info_t arena_binind_div_info[SC_NBINS];
 
+JET_EXTERN void
+bin_dalloc_locked_begin(
+    bin_dalloc_locked_info_t *info, szind_t binind) {
+	info->div_info = arena_binind_div_info[binind];
+	info->nregs = bin_infos[binind].nregs;
+	info->ndalloc = 0;
+}
+
 size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
 size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
 
@@ -64,11 +83,6 @@ const arena_config_t arena_config_default = {
  * definition.
  */
 
-static bool arena_decay_dirty(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-static void arena_maybe_do_deferred_work(
-    tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new);
-
 /******************************************************************************/
 
 void
@@ -77,8 +91,8 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
 	*nthreads += arena_nthreads_get(arena, false);
 	*dss = dss_prec_names[arena_dss_prec_get(arena)];
-	*dirty_decay_ms = arena_decay_ms_get(arena, extent_state_dirty);
-	*muzzy_decay_ms = arena_decay_ms_get(arena, extent_state_muzzy);
+	*dirty_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_dirty);
+	*muzzy_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_muzzy);
 	pa_shard_basic_stats_merge(&arena->pa_shard, nactive, ndirty, nmuzzy);
 }
 
@@ -275,7 +289,7 @@ arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
  * from scratch (descriptors held by other threads at fork time are gone)
  * without locking.
  */
-void
+static void
 arena_cache_bin_array_postfork_child(arena_t *arena,
     cache_bin_array_descriptor_t *desc_or_null) {
 	cassert(config_stats);
@@ -297,34 +311,6 @@ arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
 	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 }
 
-static void
-arena_background_thread_inactivity_check(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
-	if (!background_thread_enabled() || is_background_thread) {
-		return;
-	}
-	background_thread_info_t *info = arena_background_thread_info_get(
-	    arena);
-	if (background_thread_indefinite_sleep(info)) {
-		arena_maybe_do_deferred_work(
-		    tsdn, arena, &arena->pa_shard.pac.decay_dirty, 0);
-	}
-}
-
-/*
- * React to deferred work generated by a PAI function.
- */
-void
-arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) {
-	witness_assert_depth_to_rank(
-	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-
-	if (decay_immediately(&arena->pa_shard.pac.decay_dirty)) {
-		arena_decay_dirty(tsdn, arena, false, true);
-	}
-	arena_background_thread_inactivity_check(tsdn, arena, false);
-}
-
 static void
 arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) {
 	cassert(config_stats);
@@ -457,188 +443,12 @@ arena_extent_ralloc_large_expand(
 	}
 }
 
-/*
- * In situations where we're not forcing a decay (i.e. because the user
- * specifically requested it), should we purge ourselves, or wait for the
- * background thread to get to it.
- */
-static pac_purge_eagerness_t
-arena_decide_unforced_purge_eagerness(bool is_background_thread) {
-	if (is_background_thread) {
-		return PAC_PURGE_ALWAYS;
-	} else if (!is_background_thread && background_thread_enabled()) {
-		return PAC_PURGE_NEVER;
-	} else {
-		return PAC_PURGE_ON_EPOCH_ADVANCE;
-	}
-}
-
-bool
-arena_decay_ms_set(
-    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms) {
-	pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
-	    /* is_background_thread */ false);
-	return pa_decay_ms_set(
-	    tsdn, &arena->pa_shard, state, decay_ms, eagerness);
-}
-
-ssize_t
-arena_decay_ms_get(arena_t *arena, extent_state_t state) {
-	return pa_decay_ms_get(&arena->pa_shard, state);
-}
-
-static bool
-arena_decay_impl(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
-    pac_decay_stats_t *decay_stats, ecache_t *ecache, bool is_background_thread,
-    bool all) {
-	if (all) {
-		malloc_mutex_lock(tsdn, &decay->mtx);
-		pac_decay_all(tsdn, &arena->pa_shard.pac, decay, decay_stats,
-		    ecache, /* fully_decay */ all);
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-
-	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
-		/* No need to wait if another thread is in progress. */
-		return true;
-	}
-	pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
-	    is_background_thread);
-	bool epoch_advanced = pac_maybe_decay_purge(
-	    tsdn, &arena->pa_shard.pac, decay, decay_stats, ecache, eagerness);
-	size_t npages_new JEMALLOC_CLANG_ANALYZER_SILENCE_INIT(0);
-	if (epoch_advanced) {
-		/* Backlog is updated on epoch advance. */
-		npages_new = decay_epoch_npages_delta(decay);
-	}
-	malloc_mutex_unlock(tsdn, &decay->mtx);
-
-	if (have_background_thread && background_thread_enabled()
-	    && epoch_advanced && !is_background_thread) {
-		arena_maybe_do_deferred_work(tsdn, arena, decay, npages_new);
-	}
-
-	return false;
-}
-
-static bool
-arena_decay_dirty(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_dirty,
-	    &arena->pa_shard.pac.stats->decay_dirty,
-	    &arena->pa_shard.pac.ecache_dirty, is_background_thread, all);
-}
-
-static bool
-arena_decay_muzzy(
-    tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	if (pa_shard_dont_decay_muzzy(&arena->pa_shard)) {
-		return false;
-	}
-	return arena_decay_impl(tsdn, arena, &arena->pa_shard.pac.decay_muzzy,
-	    &arena->pa_shard.pac.stats->decay_muzzy,
-	    &arena->pa_shard.pac.ecache_muzzy, is_background_thread, all);
-}
-
-void
-arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
-	if (all) {
-		/*
-		 * We should take a purge of "all" to mean "save as much memory
-		 * as possible", including flushing any caches (for situations
-		 * like thread death, or manual purge calls).
-		 */
-		pa_shard_flush(tsdn, &arena->pa_shard);
-	}
-	if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
-		return;
-	}
-	arena_decay_muzzy(tsdn, arena, is_background_thread, all);
-}
-
-static bool
-arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
-    background_thread_info_t *info, nstime_t *remaining_sleep,
-    size_t npages_new) {
-	malloc_mutex_assert_owner(tsdn, &info->mtx);
-
-	if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
-		return false;
-	}
-
-	if (!decay_gradually(decay)) {
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-
-	nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
-	if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
-		malloc_mutex_unlock(tsdn, &decay->mtx);
-		return false;
-	}
-	nstime_subtract(remaining_sleep, &decay->epoch);
-	if (npages_new > 0) {
-		uint64_t npurge_new = decay_npages_purge_in(
-		    decay, remaining_sleep, npages_new);
-		info->npages_to_purge_new += npurge_new;
-	}
-	malloc_mutex_unlock(tsdn, &decay->mtx);
-	return info->npages_to_purge_new
-	    > ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD;
-}
-
-/*
- * Check if deferred work needs to be done sooner than planned.
- * For decay we might want to wake up earlier because of an influx of dirty
- * pages. Rather than waiting for previously estimated time, we proactively
- * purge those pages.
- * If background thread sleeps indefinitely, always wake up because some
- * deferred work has been generated.
- */
 static void
-arena_maybe_do_deferred_work(
-    tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new) {
-	background_thread_info_t *info = arena_background_thread_info_get(
-	    arena);
-	if (malloc_mutex_trylock(tsdn, &info->mtx)) {
-		/*
-		 * Background thread may hold the mutex for a long period of
-		 * time.  We'd like to avoid the variance on application
-		 * threads.  So keep this non-blocking, and leave the work to a
-		 * future epoch.
-		 */
-		return;
-	}
-	if (!background_thread_is_started(info)) {
-		goto label_done;
-	}
-
-	nstime_t remaining_sleep;
-	if (background_thread_indefinite_sleep(info)) {
-		background_thread_wakeup_early(info, NULL);
-	} else if (arena_should_decay_early(tsdn, arena, decay, info,
-	               &remaining_sleep, npages_new)) {
-		info->npages_to_purge_new = 0;
-		background_thread_wakeup_early(info, &remaining_sleep);
-	}
-label_done:
-	malloc_mutex_unlock(tsdn, &info->mtx);
-}
-
-/* Called from background threads. */
-void
-arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) {
-	arena_decay(tsdn, arena, true, false);
-	pa_shard_do_deferred_work(tsdn, &arena->pa_shard);
-}
-
-void
 arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) {
 	bool deferred_work_generated = false;
 	pa_dalloc(tsdn, &arena->pa_shard, slab, &deferred_work_generated);
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 }
 
@@ -694,11 +504,13 @@ arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize) {
 	assert(isalloc(tsdn, ptr) == usize);
 }
 
-static size_t
+size_t
 arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
 	cassert(config_prof);
+	assert(opt_prof);
 	assert(ptr != NULL);
-	size_t usize = isalloc(tsdn, ptr);
+	size_t usize = edata_usize_get(edata);
+	assert(isalloc(tsdn, ptr) == usize);
 	size_t bumped_usize = sz_sa2u(usize, PROF_SAMPLE_ALIGNMENT);
 	assert(bumped_usize <= SC_LARGE_MINCLASS
 	    && PAGE_CEILING(bumped_usize) == bumped_usize);
@@ -710,17 +522,6 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
 
 	assert(isalloc(tsdn, ptr) == bumped_usize);
 
-	return bumped_usize;
-}
-
-static void
-arena_dalloc_promoted_impl(
-    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path, edata_t *edata) {
-	cassert(config_prof);
-	assert(opt_prof);
-
-	size_t usize = edata_usize_get(edata);
-	size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
 	if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) {
 		/*
 		 * Currently, we only do redzoning for small sampled
@@ -728,21 +529,8 @@ arena_dalloc_promoted_impl(
 		 */
 		safety_check_verify_redzone(ptr, usize, bumped_usize);
 	}
-	szind_t bumped_ind = sz_size2index(bumped_usize);
-	if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
-	    && tcache_can_cache_large(tcache, bumped_ind)) {
-		tcache_dalloc_large(
-		    tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
-	} else {
-		large_dalloc(tsdn, edata);
-	}
-}
 
-void
-arena_dalloc_promoted(
-    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
-	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
-	arena_dalloc_promoted_impl(tsdn, ptr, tcache, slow_path, edata);
+	return bumped_usize;
 }
 
 void
@@ -784,8 +572,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
 			prof_free(tsd, ptr, usize, &alloc_ctx);
 		}
 		if (config_prof && opt_prof && alloc_ctx.szind < SC_NBINS) {
-			arena_dalloc_promoted_impl(tsd_tsdn(tsd), ptr,
-			    /* tcache */ NULL, /* slow_path */ true, edata);
+			arena_prof_demote(tsd_tsdn(tsd), edata, ptr);
+			large_dalloc(tsd_tsdn(tsd), edata);
 		} else {
 			large_dalloc(tsd_tsdn(tsd), edata);
 		}
@@ -937,7 +725,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind,
 	    /* zero */ false, guarded, &deferred_work_generated);
 
 	if (deferred_work_generated) {
-		arena_handle_deferred_work(tsdn, arena);
+		pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
 	}
 
 	if (slab == NULL) {
@@ -1154,33 +942,6 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
 	}
 }
 
-void *
-arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
-    bool zero, bool slab, tcache_t *tcache) {
-	if (slab) {
-		assert(sz_can_use_slab(usize));
-		/* Small; alignment doesn't require special slab placement. */
-
-		/* usize should be a result of sz_sa2u() */
-		assert((usize & (alignment - 1)) == 0);
-
-		/*
-		 * Small usize can't come from an alignment larger than a page.
-		 */
-		assert(alignment <= PAGE);
-
-		return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
-		    zero, slab, tcache, true);
-	} else {
-		if (likely(alignment <= CACHELINE)) {
-			return large_malloc(tsdn, arena, usize, zero);
-		} else {
-			return large_palloc(
-			    tsdn, arena, usize, alignment, zero);
-		}
-	}
-}
-
 static void
 arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) {
 	szind_t  binind = edata_szind_get(edata);
@@ -1514,10 +1275,10 @@ arena_ptr_array_flush_impl(tsd_t *tsd, szind_t binind,
 	 * '...' is morally equivalent, the code itself needs slight tweaks.
 	 */
 	if (small) {
-		return arena_ptr_array_flush_impl_small(tsdn, binind, arr,
+		arena_ptr_array_flush_impl_small(tsdn, binind, arr,
 		    item_edata, nflush, stats_arena, merge_stats);
 	} else {
-		return arena_ptr_array_flush_impl_large(tsdn, binind, arr,
+		arena_ptr_array_flush_impl_large(tsdn, binind, arr,
 		    item_edata, nflush, stats_arena, merge_stats);
 	}
 }
@@ -1607,64 +1368,6 @@ done:
 	return ret;
 }
 
-static void *
-arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache) {
-	if (alignment == 0) {
-		return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
-		    zero, slab, tcache, true);
-	}
-	usize = sz_sa2u(usize, alignment);
-	if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
-		return NULL;
-	}
-	return ipalloct_explicit_slab(
-	    tsdn, usize, alignment, zero, slab, tcache, arena);
-}
-
-void *
-arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
-	size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
-	if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
-		return NULL;
-	}
-
-	if (likely(slab)) {
-		assert(sz_can_use_slab(usize));
-		/* Try to avoid moving the allocation. */
-		UNUSED size_t newsize;
-		if (!arena_ralloc_no_move(
-		        tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
-			return ptr;
-		}
-	}
-
-	if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
-		return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
-		    tcache);
-	}
-
-	/*
-	 * size and oldsize are different enough that we need to move the
-	 * object.  In that case, fall back to allocating new space and copying.
-	 */
-	void *ret = arena_ralloc_move_helper(
-	    tsdn, arena, usize, alignment, zero, slab, tcache);
-	if (ret == NULL) {
-		return NULL;
-	}
-
-	/*
-	 * Junk/zero-filling were already done by
-	 * ipalloc()/arena_malloc().
-	 */
-	size_t copysize = (usize < oldsize) ? usize : oldsize;
-	memcpy(ret, ptr, copysize);
-	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
-	return ret;
-}
-
 ehooks_t *
 arena_get_ehooks(const arena_t *arena) {
 	return base_ehooks_get(arena->base);
@@ -1673,17 +1376,17 @@ arena_get_ehooks(const arena_t *arena) {
 extent_hooks_t *
 arena_set_extent_hooks(
     tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks) {
-	background_thread_info_t *info;
-	if (have_background_thread) {
-		info = arena_background_thread_info_get(arena);
-		malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
-	}
+	/*
+	 * Serialize disabling the HPA against the background thread via the
+	 * module-owned bracket (have_background_thread-gated internally) rather
+	 * than reaching into info->mtx directly.
+	 */
+	unsigned arena_ind = arena_ind_get(arena);
+	background_thread_serialize_lock(tsd, arena_ind);
 	/* No using the HPA now that we have the custom hooks. */
 	pa_shard_disable_hpa(tsd_tsdn(tsd), &arena->pa_shard);
 	extent_hooks_t *ret = base_extent_hooks_set(arena->base, extent_hooks);
-	if (have_background_thread) {
-		malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
-	}
+	background_thread_serialize_unlock(tsd, arena_ind);
 
 	return ret;
 }
@@ -1916,13 +1619,13 @@ arena_create_huge_arena(tsd_t *tsd, unsigned ind) {
 	 */
 	if (!background_thread_enabled()
 	    && arena_dirty_decay_ms_default_get() > 0) {
-		arena_decay_ms_set(
-		    tsd_tsdn(tsd), huge_arena, extent_state_dirty, 0);
+		pa_decay_ms_set(
+		    tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_dirty, 0);
 	}
 	if (!background_thread_enabled()
 	    && arena_muzzy_decay_ms_default_get() > 0) {
-		arena_decay_ms_set(
-		    tsd_tsdn(tsd), huge_arena, extent_state_muzzy, 0);
+		pa_decay_ms_set(
+		    tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_muzzy, 0);
 	}
 
 	return huge_arena;

src/arenas_management.c

@@ -1,11 +1,17 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
 
+#include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/arenas_management.h"
+#include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/background_thread_inlines.h"
 #include "jemalloc/internal/jemalloc_init.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
+#include "jemalloc/internal/arena_inlines.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache.h"
 
 JEMALLOC_ALIGNED(CACHELINE)
 atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
@@ -196,8 +202,8 @@ arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
 		 * Purge if the old arena has no associated threads anymore and
 		 * no background threads.
 		 */
-		arena_decay(tsd_tsdn(tsd), oldarena,
-		    /* is_background_thread */ false, /* all */ true);
+		pa_shard_flush(tsd_tsdn(tsd), &oldarena->pa_shard,
+		    /* all */ true);
 	}
 }
 

src/background_thread.c

@@ -1,7 +1,18 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
 
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arenas_management.h"
 #include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/background_thread.h"
+#include "jemalloc/internal/background_thread_inlines.h"
+#include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/deferral.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
+#include "jemalloc/internal/malloc_io.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prof.h"
+#include "jemalloc/internal/tcache.h"
+#include "jemalloc/internal/witness.h"
 
 JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
 
@@ -23,6 +34,72 @@ size_t     max_background_threads;
 /* Thread info per-index. */
 background_thread_info_t *background_thread_info;
 
+/******************************************************************************/
+/*
+ * Config-independent lifecycle/state-ownership helpers.  Defined
+ * unconditionally and gated at runtime on have_background_thread, so callers in
+ * ctl.c / arena.c never touch background_thread_lock or info->state directly;
+ * they compile to runtime no-ops when !have_background_thread.
+ */
+
+void
+background_thread_arena_reset_begin(tsd_t *tsd, unsigned arena_ind) {
+	/* Temporarily disable the background thread during arena reset. */
+	if (have_background_thread) {
+		/*
+		 * Hold background_thread_lock across the whole arena-reset
+		 * body (acquired here, released in _finish) so a concurrent
+		 * background_threads_enable() cannot start a background
+		 * thread mid-reset.  This must happen whenever the feature
+		 * is compiled in (have_background_thread), even when the
+		 * thread is not currently enabled; the state transition
+		 * below is separately gated on background_thread_enabled().
+		 */
+		malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock);
+		if (background_thread_enabled()) {
+			background_thread_info_t *info =
+			    background_thread_info_get(arena_ind);
+			assert(info->state == background_thread_started);
+			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+			info->state = background_thread_paused;
+			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+		}
+	}
+}
+
+void
+background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		if (background_thread_enabled()) {
+			background_thread_info_t *info =
+			    background_thread_info_get(arena_ind);
+			assert(info->state == background_thread_paused);
+			malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+			info->state = background_thread_started;
+			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+		}
+		malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock);
+	}
+}
+
+void
+background_thread_serialize_lock(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		background_thread_info_t *info =
+		    background_thread_info_get(arena_ind);
+		malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx);
+	}
+}
+
+void
+background_thread_serialize_unlock(tsd_t *tsd, unsigned arena_ind) {
+	if (have_background_thread) {
+		background_thread_info_t *info =
+		    background_thread_info_get(arena_ind);
+		malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
+	}
+}
+
 /******************************************************************************/
 
 #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
@@ -181,6 +258,50 @@ background_thread_cond_wait(
 	return ret;
 }
 
+static int
+background_thread_cond_init(pthread_cond_t *cond) {
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	pthread_condattr_t cond_attr;
+	int ret = pthread_condattr_init(&cond_attr);
+	if (ret != 0) {
+		return ret;
+	}
+	ret = pthread_condattr_setclock(&cond_attr, CLOCK_MONOTONIC);
+	if (ret != 0) {
+		pthread_condattr_destroy(&cond_attr);
+		return ret;
+	}
+	ret = pthread_cond_init(cond, &cond_attr);
+	pthread_condattr_destroy(&cond_attr);
+	return ret;
+#else
+	return pthread_cond_init(cond, NULL);
+#endif
+}
+
+/*
+ * Fill in the absolute deadline for pthread_cond_timedwait.  The clock read
+ * here MUST match the clock the condvar was initialized with in
+ * background_thread_cond_init, otherwise the deadline is interpreted against
+ * the wrong epoch.
+ */
+static void
+background_thread_wakeup_ts_init(struct timespec *ts, uint64_t interval) {
+	nstime_t wakeup;
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	struct timespec now;
+	clock_gettime(CLOCK_MONOTONIC, &now);
+	nstime_init2(&wakeup, now.tv_sec, now.tv_nsec);
+#else
+	struct timeval tv;
+	gettimeofday(&tv, NULL);
+	nstime_init2(&wakeup, tv.tv_sec, tv.tv_usec * 1000);
+#endif
+	nstime_iadd(&wakeup, interval);
+	ts->tv_sec = (size_t)nstime_sec(&wakeup);
+	ts->tv_nsec = (size_t)nstime_nsec(&wakeup);
+}
+
 static void
 background_thread_sleep(
     tsdn_t *tsdn, background_thread_info_t *info, uint64_t interval) {
@@ -189,11 +310,8 @@ background_thread_sleep(
 	}
 	info->npages_to_purge_new = 0;
 
-	struct timeval tv;
-	/* Specific clock required by timedwait. */
-	gettimeofday(&tv, NULL);
 	nstime_t before_sleep;
-	nstime_init2(&before_sleep, tv.tv_sec, tv.tv_usec * 1000);
+	nstime_init_update(&before_sleep);
 
 	int ret;
 	if (interval == BACKGROUND_THREAD_INDEFINITE_SLEEP) {
@@ -213,21 +331,16 @@ background_thread_sleep(
 		background_thread_wakeup_time_set(
 		    tsdn, info, nstime_ns(&next_wakeup));
 
-		nstime_t ts_wakeup;
-		nstime_copy(&ts_wakeup, &before_sleep);
-		nstime_iadd(&ts_wakeup, interval);
 		struct timespec ts;
-		ts.tv_sec = (size_t)nstime_sec(&ts_wakeup);
-		ts.tv_nsec = (size_t)nstime_nsec(&ts_wakeup);
+		background_thread_wakeup_ts_init(&ts, interval);
 
 		assert(!background_thread_indefinite_sleep(info));
 		ret = background_thread_cond_wait(info, &ts);
 		assert(ret == ETIMEDOUT || ret == 0);
 	}
 	if (config_stats) {
-		gettimeofday(&tv, NULL);
 		nstime_t after_sleep;
-		nstime_init2(&after_sleep, tv.tv_sec, tv.tv_usec * 1000);
+		nstime_init_update(&after_sleep);
 		if (nstime_compare(&after_sleep, &before_sleep) > 0) {
 			nstime_subtract(&after_sleep, &before_sleep);
 			nstime_add(&info->tot_sleep_time, &after_sleep);
@@ -252,7 +365,7 @@ background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) {
 static inline void
 background_work_sleep_once(
     tsdn_t *tsdn, background_thread_info_t *info, unsigned ind) {
-	uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX;
+	uint64_t ns_until_deferred = DEFERRED_WORK_MAX;
 	unsigned narenas = narenas_total_get();
 	bool     slept_indefinitely = background_thread_indefinite_sleep(info);
 
@@ -267,7 +380,8 @@ background_work_sleep_once(
 		 * work that caused this thread to wake up is scheduled for.
 		 */
 		if (!slept_indefinitely) {
-			arena_do_deferred_work(tsdn, arena);
+			pa_shard_do_deferred_work(tsdn, &arena->pa_shard,
+			    /* is_background_thread */ true);
 		}
 		if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) {
 			/* Min interval will be used. */
@@ -281,7 +395,7 @@ background_work_sleep_once(
 	}
 
 	uint64_t sleep_ns;
-	if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) {
+	if (ns_until_deferred == DEFERRED_WORK_MAX) {
 		sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP;
 	} else {
 		sleep_ns = (ns_until_deferred
@@ -741,7 +855,7 @@ background_thread_postfork_child(tsdn_t *tsdn) {
 		background_thread_info_t *info = &background_thread_info[i];
 		malloc_mutex_lock(tsdn, &info->mtx);
 		info->state = background_thread_stopped;
-		int ret = pthread_cond_init(&info->cond, NULL);
+		int ret = background_thread_cond_init(&info->cond);
 		assert(ret == 0);
 		background_thread_info_init(tsdn, info);
 		malloc_mutex_unlock(tsdn, &info->mtx);
@@ -859,7 +973,7 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) {
 		        malloc_mutex_address_ordered)) {
 			return true;
 		}
-		if (pthread_cond_init(&info->cond, NULL)) {
+		if (background_thread_cond_init(&info->cond)) {
 			return true;
 		}
 		malloc_mutex_lock(tsdn, &info->mtx);

src/base.c

@@ -1,11 +1,15 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
 
+#include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/base.h"
+#include "jemalloc/internal/ehooks.h"
 #include "jemalloc/internal/extent_mmap.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/sz.h"
 
+JET_EXTERN ehooks_t *base_ehooks_get_for_metadata(base_t *base);
+
 /*
  * In auto mode, arenas switch to huge pages for the base allocator on the
  * second base block.  a0 switches to thp on the 5th block (after 20 megabytes
@@ -15,6 +19,17 @@
 #define BASE_AUTO_THP_THRESHOLD 2
 #define BASE_AUTO_THP_THRESHOLD_A0 5
 
+/*
+ * Cap the base-block growth heuristic in base_block_alloc().  The
+ * growth heuristic reduces the number of disjoint VM ranges when new base
+ * blocks are rare, but high thread churn can cause many parallel misses for
+ * metadata allocations.  Without a cap, those misses can advance
+ * base->pind_last causing small requests to mmap multi-TiB blocks and exhaust
+ * the address space.  Large individual requests still use min_block_size and
+ * can exceed this cap.
+ */
+#define BASE_BLOCK_GROWTH_MAX ((size_t)128 << 20) /* 128 MiB */
+
 /******************************************************************************/
 /* Data. */
 
@@ -372,7 +387,11 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, ehooks_t *ehooks, unsigned ind,
 	pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS))
 	    ? *pind_last + 1
 	    : *pind_last;
-	size_t   next_block_size = base_block_size_ceil(sz_pind2sz(pind_next));
+	size_t next_block_size = base_block_size_ceil(sz_pind2sz(pind_next));
+	size_t max_block_size = base_block_size_ceil(BASE_BLOCK_GROWTH_MAX);
+	next_block_size = (next_block_size < max_block_size)
+	    ? next_block_size
+	    : max_block_size;
 	size_t   block_size = (min_block_size > next_block_size)
 	      ? min_block_size
 	      : next_block_size;
@@ -543,7 +562,7 @@ base_ehooks_get(base_t *base) {
 	return &base->ehooks;
 }
 
-ehooks_t *
+JET_EXTERN ehooks_t *
 base_ehooks_get_for_metadata(base_t *base) {
 	return &base->ehooks_base;
 }

src/bin.c

@@ -1,6 +1,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
 
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/assert.h"
 #include "jemalloc/internal/bin.h"
 #include "jemalloc/internal/sc.h"
@@ -68,7 +69,7 @@ bin_postfork_child(tsdn_t *tsdn, bin_t *bin) {
 	malloc_mutex_postfork_child(tsdn, &bin->lock);
 }
 
-void *
+JET_EXTERN void *
 bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info) {
 	void        *ret;
 	slab_data_t *slab_data = edata_slab_data_get(slab);
@@ -133,7 +134,7 @@ bin_slab_reg_alloc_batch(
 	edata_nfree_sub(slab, cnt);
 }
 
-void
+JET_EXTERN void
 bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) {
 	assert(edata_nfree_get(slab) > 0);
 	edata_heap_insert(&bin->slabs_nonfull, slab);
@@ -142,7 +143,7 @@ bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) {
 	edata_heap_remove(&bin->slabs_nonfull, slab);
 	if (config_stats) {
@@ -150,7 +151,7 @@ bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab) {
 	}
 }
 
-edata_t *
+JET_EXTERN edata_t *
 bin_slabs_nonfull_tryget(bin_t *bin) {
 	edata_t *slab = edata_heap_remove_first(&bin->slabs_nonfull);
 	if (slab == NULL) {
@@ -163,7 +164,7 @@ bin_slabs_nonfull_tryget(bin_t *bin) {
 	return slab;
 }
 
-void
+JET_EXTERN void
 bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab) {
 	assert(edata_nfree_get(slab) == 0);
 	/*
@@ -185,7 +186,7 @@ bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab) {
 	edata_list_active_remove(&bin->slabs_full, slab);
 }
 
-void
+JET_EXTERN void
 bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin) {
 	/* Dissociate slab from bin. */
 	if (slab == bin->slabcur) {
@@ -207,7 +208,7 @@ bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) {
 	assert(edata_nfree_get(slab) > 0);
 
@@ -233,7 +234,7 @@ bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin) {
 	}
 }
 
-void
+JET_EXTERN void
 bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin) {
 	malloc_mutex_assert_owner(tsdn, &bin->lock);