Improve unit test coverage in tcache, pac, hpa_central

Fix FreeBSD postfork child handler never being called: FreeBSD's libthr
calls _malloc_postfork in both parent and child (see freebsd-src
lib/libthr/thread/thr_fork.c), but jemalloc mapped it to the parent
handler only.  Detect the child via getpid() and route to
jemalloc_postfork_child, which resets nthreads and rebuilds the
descriptor queue.

Remove the child_survivor_bytes vs pre_survivor_bytes comparison: on
macOS where jemalloc registers as the default zone, internal allocations
during the postfork handler (pthread_mutex_init) can inflate the
surviving thread's tcache.

Add double-fork test to verify prefork pid is refreshed correctly when a
child process forks again.

.appveyor.yml

@@ -1,3 +1,5 @@
+image: Visual Studio 2022
+
 version: '{build}'
 
 environment:
@@ -37,7 +39,8 @@ environment:
 
 install:
   - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
-  - if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
+  - if "%MSVC%"=="amd64" call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+  - if "%MSVC%"=="x86"   call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars32.bat"
   - if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
   - pacman --noconfirm -Syuu
   - pacman --noconfirm -S autoconf

.github/workflows/check_formatting.yaml

@@ -5,6 +5,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Check for trailing whitespace
         run: scripts/check_trailing_whitespace.sh

.github/workflows/freebsd-ci.yml

@@ -25,7 +25,7 @@ jobs:
     name: FreeBSD (${{ matrix.arch }}, debug=${{ matrix.debug }}, prof=${{ matrix.prof }}${{ matrix.uncommon && ', uncommon' || '' }})
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 1
 

.github/workflows/linux-ci.yml

@@ -510,7 +510,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |
@@ -639,7 +639,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |

.github/workflows/macos-ci.yml

@@ -62,7 +62,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |
@@ -164,7 +164,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |

.github/workflows/static_analysis.yaml

@@ -7,7 +7,7 @@ jobs:
       # We build libunwind ourselves because sadly the version
       # provided by Ubuntu via apt-get is much too old.
       - name: Check out libunwind
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           repository: libunwind/libunwind
           path: libunwind
@@ -23,7 +23,7 @@ jobs:
           cd ..
           rm -rf libunwind
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       # We download LLVM directly from the latest stable release
       # on GitHub, because this tends to be much newer than the
       # version available via apt-get in Ubuntu.
@@ -54,7 +54,7 @@ jobs:
           scripts/run_static_analysis.sh static_analysis_results "$GITHUB_OUTPUT"
       - name: Upload static analysis results
         if: ${{ steps.run_static_analysis.outputs.HAS_STATIC_ANALYSIS_RESULTS }} == '1'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: static_analysis_results
           path: static_analysis_results

.github/workflows/windows-ci.yml

@@ -54,7 +54,7 @@ jobs:
               CONFIGURE_FLAGS: --enable-debug
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       shell: cmd

INSTALL.md

@@ -411,7 +411,7 @@ differ in their ease of use and flexibility.
 ### With MSVC solutions
 This is the easy, but less flexible approach. It doesn't let you specify
 arguments to the `configure` script.
-  
+
 1. Install Cygwin with at least the following packages:
    * autoconf
    * autogen
@@ -419,18 +419,19 @@ arguments to the `configure` script.
    * grep
    * sed
 
-2. Install Visual Studio 2015 or 2017 with Visual C++
+2. Install Visual Studio with Visual C++. We currently test on 2022 and provide
+solution files back to 2015.
 
 3. Add Cygwin\bin to the PATH environment variable
 
-4. Open "x64 Native Tools Command Prompt for VS 2017"
+4. Open "x64 Native Tools Command Prompt" for your version of Visual Studio
    (note: x86/x64 doesn't matter at this point)
 
 5. Generate header files:
    sh -c "CC=cl ./autogen.sh"
 
-6. Now the project can be opened and built in Visual Studio:
-   msvc\jemalloc_vc2017.sln
+6. Now the project can be opened and built in Visual Studio using the
+   corresponding solution files in the `msvc\` folder.
 
 ### With MSYS
 This is a more involved approach that offers the same configuration flexibility

Makefile.in

@@ -95,7 +95,10 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
 BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
 C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
 C_SRCS := $(srcroot)src/jemalloc.c \
+	$(srcroot)src/jemalloc_fork.c \
+	$(srcroot)src/jemalloc_init.c \
 	$(srcroot)src/arena.c \
+	$(srcroot)src/arenas_management.c \
 	$(srcroot)src/background_thread.c \
 	$(srcroot)src/base.c \
 	$(srcroot)src/bin.c \
@@ -121,7 +124,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/fxp.c \
 	$(srcroot)src/san.c \
 	$(srcroot)src/san_bump.c \
-	$(srcroot)src/hook.c \
 	$(srcroot)src/hpa.c \
 	$(srcroot)src/hpa_central.c \
 	$(srcroot)src/hpa_hooks.c \
@@ -203,12 +205,12 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/a0.c \
 	$(srcroot)test/unit/arena_decay.c \
 	$(srcroot)test/unit/arena_reset.c \
+	$(srcroot)test/unit/arenas_management.c \
 	$(srcroot)test/unit/atomic.c \
 	$(srcroot)test/unit/background_thread.c \
 	$(srcroot)test/unit/background_thread_enable.c \
 	$(srcroot)test/unit/background_thread_init.c \
 	$(srcroot)test/unit/base.c \
-	$(srcroot)test/unit/batch_alloc.c \
 	$(srcroot)test/unit/bin.c \
 	$(srcroot)test/unit/binshard.c \
 	$(srcroot)test/unit/bitmap.c \
@@ -226,7 +228,10 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/div.c \
 	$(srcroot)test/unit/double_free.c \
 	$(srcroot)test/unit/edata_cache.c \
+	$(srcroot)test/unit/emap.c \
 	$(srcroot)test/unit/emitter.c \
+	$(srcroot)test/unit/eset.c \
+	$(srcroot)test/unit/extent_dss.c \
 	$(srcroot)test/unit/extent_quantize.c \
 	${srcroot}test/unit/fb.c \
 	$(srcroot)test/unit/fork.c \
@@ -234,16 +239,19 @@ TESTS_UNIT := \
 	${srcroot}test/unit/san.c \
 	${srcroot}test/unit/san_bump.c \
 	$(srcroot)test/unit/hash.c \
-	$(srcroot)test/unit/hook.c \
 	$(srcroot)test/unit/hpa.c \
+	$(srcroot)test/unit/hpa_central.c \
 	$(srcroot)test/unit/hpa_sec_integration.c \
 	$(srcroot)test/unit/hpa_thp_always.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \
 	$(srcroot)test/unit/hpa_background_thread.c \
+	$(srcroot)test/unit/hpa_pageslab_packing.c \
 	$(srcroot)test/unit/hpdata.c \
+	$(srcroot)test/unit/extent_alloc_flags.c \
 	$(srcroot)test/unit/huge.c \
 	$(srcroot)test/unit/inspect.c \
+	$(srcroot)test/unit/jemalloc_init.c \
 	$(srcroot)test/unit/junk.c \
 	$(srcroot)test/unit/junk_alloc.c \
 	$(srcroot)test/unit/junk_free.c \
@@ -261,6 +269,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/ncached_max.c \
 	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
+	$(srcroot)test/unit/pac.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/peak.c \
@@ -289,7 +298,6 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/safety_check.c \
 	$(srcroot)test/unit/sc.c \
 	$(srcroot)test/unit/sec.c \
-	$(srcroot)test/unit/seq.c \
 	$(srcroot)test/unit/SFMT.c \
 	$(srcroot)test/unit/size_check.c \
 	$(srcroot)test/unit/size_classes.c \
@@ -301,6 +309,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/sz.c \
 	$(srcroot)test/unit/tcache_init.c \
 	$(srcroot)test/unit/tcache_max.c \
+	$(srcroot)test/unit/tcache_gc.c \
 	$(srcroot)test/unit/test_hooks.c \
 	$(srcroot)test/unit/thread_event.c \
 	$(srcroot)test/unit/ticker.c \
@@ -314,8 +323,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/zero_reallocs.c
 ifeq (@enable_prof@, 1)
 TESTS_UNIT += \
-	$(srcroot)test/unit/arena_reset_prof.c \
-	$(srcroot)test/unit/batch_alloc_prof.c
+	$(srcroot)test/unit/arena_reset_prof.c
 endif
 TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
 	$(srcroot)test/integration/allocated.c \
@@ -337,9 +345,11 @@ TESTS_INTEGRATION += \
 endif
 ifeq (@enable_cxx@, 1)
 CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
-TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp \
-	$(srcroot)test/integration/cpp/infallible_new_true.cpp \
+TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
+ifeq (@enable_cxx_exceptions@, 1)
+TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new_true.cpp \
 	$(srcroot)test/integration/cpp/infallible_new_false.cpp
+endif
 else
 CPP_SRCS :=
 TESTS_INTEGRATION_CPP :=
@@ -347,9 +357,7 @@ endif
 TESTS_ANALYZE := $(srcroot)test/analyze/prof_bias.c \
 	$(srcroot)test/analyze/rand.c \
 	$(srcroot)test/analyze/sizes.c
-TESTS_STRESS := $(srcroot)test/stress/batch_alloc.c \
-	$(srcroot)test/stress/fill_flush.c \
-	$(srcroot)test/stress/hookbench.c \
+TESTS_STRESS := $(srcroot)test/stress/fill_flush.c \
 	$(srcroot)test/stress/large_microbench.c \
 	$(srcroot)test/stress/mallctl.c \
 	$(srcroot)test/stress/microbench.c
@@ -609,38 +617,38 @@ build_lib: build_lib_static
 endif
 
 install_bin:
-	$(INSTALL) -d $(BINDIR)
+	$(INSTALL) -d '$(BINDIR)'
 	@for b in $(BINS); do \
-	echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \
-	$(INSTALL) -m 755 $$b $(BINDIR); \
+	echo "$(INSTALL) -m 755 $$b '$(BINDIR)'"; \
+	$(INSTALL) -m 755 $$b '$(BINDIR)'; \
 done
 
 install_include:
-	$(INSTALL) -d $(INCLUDEDIR)/jemalloc
+	$(INSTALL) -d '$(INCLUDEDIR)/jemalloc'
 	@for h in $(C_HDRS); do \
-	echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
-	$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
+	echo "$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'"; \
+	$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'; \
 done
 
 install_lib_shared: $(DSOS)
-	$(INSTALL) -d $(LIBDIR)
-	$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
+	$(INSTALL) -d '$(LIBDIR)'
+	$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) '$(LIBDIR)'
 ifneq ($(SOREV),$(SO))
-	ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
+	ln -sf $(LIBJEMALLOC).$(SOREV) '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
 endif
 
 install_lib_static: $(STATIC_LIBS)
-	$(INSTALL) -d $(LIBDIR)
+	$(INSTALL) -d '$(LIBDIR)'
 	@for l in $(STATIC_LIBS); do \
-	echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \
-	$(INSTALL) -m 755 $$l $(LIBDIR); \
+	echo "$(INSTALL) -m 644 $$l '$(LIBDIR)'"; \
+	$(INSTALL) -m 644 $$l '$(LIBDIR)'; \
 done
 
 install_lib_pc: $(PC)
-	$(INSTALL) -d $(LIBDIR)/pkgconfig
+	$(INSTALL) -d '$(LIBDIR)/pkgconfig'
 	@for l in $(PC); do \
-	echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \
-	$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
+	echo "$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'"; \
+	$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'; \
 done
 
 ifeq ($(enable_shared), 1)
@@ -652,17 +660,17 @@ endif
 install_lib: install_lib_pc
 
 install_doc_html: build_doc_html
-	$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
+	$(INSTALL) -d '$(DATADIR)/doc/jemalloc$(install_suffix)'
 	@for d in $(DOCS_HTML); do \
-	echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
-	$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
+	echo "$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'"; \
+	$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'; \
 done
 
 install_doc_man: build_doc_man
-	$(INSTALL) -d $(MANDIR)/man3
+	$(INSTALL) -d '$(MANDIR)/man3'
 	@for d in $(DOCS_MAN3); do \
-	echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \
-	$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
+	echo "$(INSTALL) -m 644 $$d '$(MANDIR)/man3'"; \
+	$(INSTALL) -m 644 $$d '$(MANDIR)/man3'; \
 done
 
 install_doc: install_doc_html install_doc_man
@@ -674,23 +682,23 @@ install: install_doc
 endif
 
 uninstall_bin:
-	$(RM) -v $(foreach b,$(notdir $(BINS)),$(BINDIR)/$(b))
+	$(RM) -v $(foreach b,$(notdir $(BINS)),'$(BINDIR)/$(b)')
 
 uninstall_include:
-	$(RM) -v $(foreach h,$(notdir $(C_HDRS)),$(INCLUDEDIR)/jemalloc/$(h))
-	rmdir -v $(INCLUDEDIR)/jemalloc
+	$(RM) -v $(foreach h,$(notdir $(C_HDRS)),'$(INCLUDEDIR)/jemalloc/$(h)')
+	rmdir -v '$(INCLUDEDIR)/jemalloc'
 
 uninstall_lib_shared:
-	$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SOREV)
+	$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SOREV)'
 ifneq ($(SOREV),$(SO))
-	$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SO)
+	$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
 endif
 
 uninstall_lib_static:
-	$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),$(LIBDIR)/$(l))
+	$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),'$(LIBDIR)/$(l)')
 
 uninstall_lib_pc:
-	$(RM) -v $(foreach p,$(notdir $(PC)),$(LIBDIR)/pkgconfig/$(p))
+	$(RM) -v $(foreach p,$(notdir $(PC)),'$(LIBDIR)/pkgconfig/$(p)')
 
 ifeq ($(enable_shared), 1)
 uninstall_lib: uninstall_lib_shared
@@ -701,11 +709,11 @@ endif
 uninstall_lib: uninstall_lib_pc
 
 uninstall_doc_html:
-	$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),$(DATADIR)/doc/jemalloc$(install_suffix)/$(d))
-	rmdir -v $(DATADIR)/doc/jemalloc$(install_suffix)
+	$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),'$(DATADIR)/doc/jemalloc$(install_suffix)/$(d)')
+	rmdir -v '$(DATADIR)/doc/jemalloc$(install_suffix)'
 
 uninstall_doc_man:
-	$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),$(MANDIR)/man3/$(d))
+	$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),'$(MANDIR)/man3/$(d)')
 
 uninstall_doc: uninstall_doc_html uninstall_doc_man
 

configure.ac

@@ -158,29 +158,27 @@ AC_SUBST([abs_objroot])
 
 dnl Munge install path variables.
 case "$prefix" in
-   *\ * ) AC_MSG_ERROR([Prefix should not contain spaces]) ;;
    "NONE" ) prefix="/usr/local" ;;
 esac
 case "$exec_prefix" in
-   *\ * ) AC_MSG_ERROR([Exec prefix should not contain spaces]) ;;
    "NONE" ) exec_prefix=$prefix ;;
 esac
 PREFIX=$prefix
 AC_SUBST([PREFIX])
-BINDIR=`eval echo $bindir`
-BINDIR=`eval echo $BINDIR`
+BINDIR=`eval echo "\"$bindir\""`
+BINDIR=`eval echo "\"$BINDIR\""`
 AC_SUBST([BINDIR])
-INCLUDEDIR=`eval echo $includedir`
-INCLUDEDIR=`eval echo $INCLUDEDIR`
+INCLUDEDIR=`eval echo "\"$includedir\""`
+INCLUDEDIR=`eval echo "\"$INCLUDEDIR\""`
 AC_SUBST([INCLUDEDIR])
-LIBDIR=`eval echo $libdir`
-LIBDIR=`eval echo $LIBDIR`
+LIBDIR=`eval echo "\"$libdir\""`
+LIBDIR=`eval echo "\"$LIBDIR\""`
 AC_SUBST([LIBDIR])
-DATADIR=`eval echo $datadir`
-DATADIR=`eval echo $DATADIR`
+DATADIR=`eval echo "\"$datadir\""`
+DATADIR=`eval echo "\"$DATADIR\""`
 AC_SUBST([DATADIR])
-MANDIR=`eval echo $mandir`
-MANDIR=`eval echo $MANDIR`
+MANDIR=`eval echo "\"$mandir\""`
+MANDIR=`eval echo "\"$MANDIR\""`
 AC_SUBST([MANDIR])
 
 dnl Support for building documentation.
@@ -376,7 +374,30 @@ fi
 if test "x$enable_cxx" = "x1"; then
   AC_DEFINE([JEMALLOC_ENABLE_CXX], [ ], [ ])
 fi
+if test "x$enable_cxx" = "x1"; then
+  dnl Now check whether the C++ compiler has exceptions enabled.
+  AC_LANG_PUSH([C++])
+  SAVED_CXXFLAGS="${CXXFLAGS}"
+  CXXFLAGS="${CXXFLAGS} ${EXTRA_CXXFLAGS}"
+  JE_COMPILABLE([C++ exception support], [
+#include <new>
+], [
+	try {
+		throw std::bad_alloc();
+	} catch (const std::bad_alloc &) {
+	}
+], [je_cv_cxx_exceptions])
+  CXXFLAGS="${SAVED_CXXFLAGS}"
+  AC_LANG_POP([C++])
+  if test "x${je_cv_cxx_exceptions}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_HAVE_CXX_EXCEPTIONS], [ ], [ ])
+    enable_cxx_exceptions="1"
+  else
+    enable_cxx_exceptions="0"
+  fi
+fi
 AC_SUBST([enable_cxx])
+AC_SUBST([enable_cxx_exceptions])
 AC_SUBST([CONFIGURE_CXXFLAGS])
 AC_SUBST([SPECIFIED_CXXFLAGS])
 AC_SUBST([EXTRA_CXXFLAGS])
@@ -2041,6 +2062,9 @@ if test "x${LG_PAGE}" != "xundefined" ; then
 else
    AC_MSG_ERROR([cannot determine value for LG_PAGE])
 fi
+if test "${LG_PAGE}" -lt 8 ; then
+   AC_MSG_ERROR([LG_PAGE must be at least 8 (PAGE >= 256)])
+fi
 
 AC_ARG_WITH([lg_hugepage],
   [AS_HELP_STRING([--with-lg-hugepage=<lg-hugepage>],

doc/jemalloc.xml.in

@@ -2088,7 +2088,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <parameter>*oldlenp</parameter> must be
         <code language="C">sizeof(<type>char *</type>)</code>.
         Arena names are also included in the output of <link
-        linkend="stats_print"><function>malloc_stats_print()</function></link>.
+        linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
         </para></listitem>
       </varlistentry>
 
@@ -2172,6 +2172,15 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         addition there may be extents created prior to the application having an
         opportunity to take over extent allocation.</para>
 
+        <para>An extent must be operated on (dalloc, destroy, commit, decommit,
+        purge, split, merge) by a hook capable of handling it, normally the hook
+        that allocated it. Replacing hooks on a live arena is tricky and thus
+        discouraged. If the hook is replaced anyway, the new hook should forward
+        operations on extents it did not allocate to the previous hook (e.g.,
+        the new dalloc dispatches to the previous dalloc for an
+        old-hook-allocated extent). The new hook should also avoid merging
+        extents allocated by different hooks.</para>
+
         <programlisting language="C"><![CDATA[
 typedef extent_hooks_s extent_hooks_t;
 struct extent_hooks_s {
@@ -2236,6 +2245,18 @@ struct extent_hooks_s {
         linkend="arena.i.dss"><mallctl>arena.&lt;i&gt;.dss</mallctl></link>
         setting irrelevant.</para>
 
+        <para>The alloc hook may bitwise-OR
+        <constant>EXTENT_ALLOC_FLAG_PINNED</constant> into the low bits of
+        the returned pointer to indicate that the backing memory is
+        non-reclaimable (e.g. HugeTLB pages) and should be reused
+        preferentially; in that case <parameter>*commit</parameter> must also
+        be set to true.  jemalloc strips the low byte before use.  The
+        pinned attribute is per-extent rather than per-hook: a single alloc
+        hook may return pinned and non-pinned extents in different calls.
+        Pinned-ness is set at allocation, inherited through splits, and
+        never changes after that.  Pinned and non-pinned extents are never
+        merged together.</para>
+
         <funcsynopsis><funcprototype>
           <funcdef>typedef bool <function>(extent_dalloc_t)</function></funcdef>
           <paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
@@ -2457,7 +2478,7 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Hugepage size.  This value is also reported in the
         output of <link
-        linkend="stats_print"><function>malloc_stats_print()</function></link>.
+        linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
         </para></listitem>
       </varlistentry>
 
@@ -2768,11 +2789,11 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Maximum number of bytes in physically resident data
         pages mapped by the allocator, comprising all pages dedicated to
-        allocator metadata, pages backing active allocations, and unused dirty
-        pages.  This is a maximum rather than precise because pages may not
-        actually be physically resident if they correspond to demand-zeroed
-        virtual memory that has not yet been touched.  This is a multiple of the
-        page size, and is larger than <link
+        allocator metadata, pages backing active allocations, unused dirty
+        pages, and pinned pages.  This is a maximum rather than precise because
+        pages may not actually be physically resident if they correspond to
+        demand-zeroed virtual memory that has not yet been touched.  This is a
+        multiple of the page size, and is larger than <link
         linkend="stats.active"><mallctl>stats.active</mallctl></link>.</para></listitem>
       </varlistentry>
 
@@ -2811,6 +2832,22 @@ struct extent_hooks_s {
         </para></listitem>
       </varlistentry>
 
+      <varlistentry id="stats.pinned">
+        <term>
+          <mallctl>stats.pinned</mallctl>
+          (<type>size_t</type>)
+          <literal>r-</literal>
+          [<option>--enable-stats</option>]
+        </term>
+        <listitem><para>Total number of bytes in unused extents backed by
+        non-reclaimable memory.  Pinned extents are tracked separately from
+        dirty, muzzy, and retained extents because they are excluded from
+        decay and purging; unlike <link
+        linkend="stats.retained"><mallctl>stats.retained</mallctl></link>,
+        pinned bytes are included in <link
+        linkend="stats.mapped"><mallctl>stats.mapped</mallctl></link>.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="stats.zero_reallocs">
         <term>
           <mallctl>stats.zero_reallocs</mallctl>
@@ -3089,6 +3126,18 @@ struct extent_hooks_s {
         details.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="stats.arenas.i.pinned">
+        <term>
+          <mallctl>stats.arenas.&lt;i&gt;.pinned</mallctl>
+          (<type>size_t</type>)
+          <literal>r-</literal>
+          [<option>--enable-stats</option>]
+        </term>
+        <listitem><para>Number of pinned bytes.  See <link
+        linkend="stats.pinned"><mallctl>stats.pinned</mallctl></link> for
+        details.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="stats.arenas.i.extent_avail">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.extent_avail</mallctl>
@@ -3146,11 +3195,11 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Maximum number of bytes in physically resident data
         pages mapped by the arena, comprising all pages dedicated to allocator
-        metadata, pages backing active allocations, and unused dirty pages.
-        This is a maximum rather than precise because pages may not actually be
-        physically resident if they correspond to demand-zeroed virtual memory
-        that has not yet been touched.  This is a multiple of the page
-        size.</para></listitem>
+        metadata, pages backing active allocations, unused dirty pages, and
+        pinned pages.  This is a maximum rather than precise because pages
+        may not actually be physically resident if they correspond to
+        demand-zeroed virtual memory that has not yet been touched.  This is
+        a multiple of the page size.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.dirty_npurge">
@@ -3493,7 +3542,7 @@ struct extent_hooks_s {
         </term>
         <listitem><para> Number of extents of the given type in this arena in
 	the bucket corresponding to page size index &lt;j&gt;. The extent type
-	is one of dirty, muzzy, or retained.</para></listitem>
+	is one of dirty, muzzy, retained, or pinned.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.extents.bytes">
@@ -3505,7 +3554,7 @@ struct extent_hooks_s {
         </term>
 	<listitem><para> Sum of the bytes managed by extents of the given type
 	in this arena in the bucket corresponding to page size index &lt;j&gt;.
-	The extent type is one of dirty, muzzy, or retained.</para></listitem>
+	The extent type is one of dirty, muzzy, retained, or pinned.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.lextents.j.nmalloc">
@@ -3625,6 +3674,19 @@ struct extent_hooks_s {
         counters</link>.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="stats.arenas.i.mutexes.extents_pinned">
+        <term>
+          <mallctl>stats.arenas.&lt;i&gt;.mutexes.extents_pinned.{counter}</mallctl>
+          (<type>counter specific type</type>) <literal>r-</literal>
+          [<option>--enable-stats</option>]
+        </term>
+        <listitem><para>Statistics on <varname>arena.&lt;i&gt;.extents_pinned
+        </varname> mutex (arena scope; pinned extents related).
+        <mallctl>{counter}</mallctl> is one of the counters in <link
+        linkend="mutex_counters">mutex profiling
+        counters</link>.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="stats.arenas.i.mutexes.decay_dirty">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.mutexes.decay_dirty.{counter}</mallctl>

include/jemalloc/internal/arena_externs.h

@@ -7,7 +7,6 @@
 #include "jemalloc/internal/div.h"
 #include "jemalloc/internal/emap.h"
 #include "jemalloc/internal/extent_dss.h"
-#include "jemalloc/internal/hook.h"
 #include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/stats.h"
 
@@ -51,11 +50,11 @@ void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
 edata_t *arena_extent_alloc_large(
     tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
 void arena_extent_dalloc_large_prep(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata);
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata);
 void arena_extent_ralloc_large_shrink(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
 void arena_extent_ralloc_large_expand(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
 bool arena_decay_ms_set(
     tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
 ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
@@ -86,14 +85,13 @@ void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
 bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
      size_t extra, bool zero, size_t *newsize);
 void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
-    hook_ralloc_args_t *hook_args);
-dss_prec_t      arena_dss_prec_get(arena_t *arena);
-ehooks_t       *arena_get_ehooks(arena_t *arena);
+    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache);
+dss_prec_t      arena_dss_prec_get(const arena_t *arena);
+ehooks_t       *arena_get_ehooks(const arena_t *arena);
 extent_hooks_t *arena_set_extent_hooks(
     tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
 bool    arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
-void    arena_name_get(arena_t *arena, char *name);
+void    arena_name_get(const arena_t *arena, char *name);
 void    arena_name_set(arena_t *arena, const char *name);
 ssize_t arena_dirty_decay_ms_default_get(void);
 bool    arena_dirty_decay_ms_default_set(ssize_t decay_ms);
@@ -101,15 +99,22 @@ ssize_t arena_muzzy_decay_ms_default_get(void);
 bool    arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
 bool    arena_retain_grow_limit_get_set(
        tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
-unsigned arena_nthreads_get(arena_t *arena, bool internal);
+unsigned arena_nthreads_get(const arena_t *arena, bool internal);
 void     arena_nthreads_inc(arena_t *arena, bool internal);
 void     arena_nthreads_dec(arena_t *arena, bool internal);
 arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
 bool     arena_init_huge(tsdn_t *tsdn, arena_t *a0);
+bool     arena_ind_is_huge(unsigned ind);
 arena_t *arena_choose_huge(tsd_t *tsd);
-size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
-    void **ptrs, size_t nfill, bool zero);
 bool   arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
+void  *arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind);
+void   arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *desc);
+void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *desc);
+void   arena_cache_bin_array_postfork_child(arena_t *arena,
+       cache_bin_array_descriptor_t *desc_or_null);
+void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);
@@ -120,6 +125,7 @@ void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
 void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
-void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
+void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *surviving_desc);
 
 #endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */

include/jemalloc/internal/arena_inlines_a.h

@@ -20,7 +20,7 @@ arena_internal_sub(arena_t *arena, size_t size) {
 }
 
 static inline size_t
-arena_internal_get(arena_t *arena) {
+arena_internal_get(const arena_t *arena) {
 	return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
 }
 

include/jemalloc/internal/arena_inlines_b.h

@@ -21,7 +21,7 @@
 #include "jemalloc/internal/ticker.h"
 
 static inline arena_t *
-arena_get_from_edata(edata_t *edata) {
+arena_get_from_edata(const edata_t *edata) {
 	return (arena_t *)atomic_load_p(
 	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
 }
@@ -203,9 +203,7 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
 			assert(sz_can_use_slab(size));
 			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
 			    size, ind, zero, slow_path);
-		} else if (likely(ind < tcache_nbins_get(tcache->tcache_slow)
-		               && !tcache_bin_disabled(ind, &tcache->bins[ind],
-		                   tcache->tcache_slow))) {
+		} else if (likely(tcache_can_cache_large(tcache, ind))) {
 			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
 			    size, ind, zero, slow_path);
 		}
@@ -319,9 +317,7 @@ arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
 	if (unlikely(is_sample_promoted)) {
 		arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
 	} else {
-		if (szind < tcache_nbins_get(tcache->tcache_slow)
-		    && !tcache_bin_disabled(
-		        szind, &tcache->bins[szind], tcache->tcache_slow)) {
+		if (tcache_can_cache_large(tcache, szind)) {
 			tcache_dalloc_large(
 			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
 		} else {

include/jemalloc/internal/arena_structs.h

@@ -48,15 +48,14 @@ struct arena_s {
 	arena_stats_t stats;
 
 	/*
-	 * Lists of tcaches and cache_bin_array_descriptors for extant threads
-	 * associated with this arena.  Stats from these are merged
-	 * incrementally, and at exit if opt_stats_print is enabled.
+	 * List of cache_bin_array_descriptors for extant threads associated
+	 * with this arena.  Stats from these are merged incrementally, and at
+	 * exit if opt_stats_print is enabled.
 	 *
-	 * Synchronization: tcache_ql_mtx.
+	 * Synchronization: cache_bin_array_descriptor_ql_mtx.
 	 */
-	ql_head(tcache_slow_t) tcache_ql;
 	ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
-	malloc_mutex_t tcache_ql_mtx;
+	malloc_mutex_t cache_bin_array_descriptor_ql_mtx;
 
 	/*
 	 * Represents a dss_prec_t, but atomically.
@@ -105,7 +104,11 @@ struct arena_s {
 	    "Do not use this field directly. "
 	    "Use `arena_get_bin` instead.")
 	JEMALLOC_ALIGNED(CACHELINE)
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+	bin_t all_bins[];
+#else
 	bin_t all_bins[0];
+#endif
 };
 
 #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */

include/jemalloc/internal/arenas_management.h

@@ -0,0 +1,48 @@
+#ifndef JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
+#define JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
+
+#include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/*
+ * Arenas that are used to service external requests.  Not all elements of the
+ * arenas array are necessarily used; arenas are created lazily as needed.
+ *
+ * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
+ * arenas.  arenas[narenas_auto..narenas_total) are only used if the application
+ * takes some action to create them and allocate from them.
+ *
+ * Points to an arena_t.
+ */
+extern atomic_p_t arenas[];
+
+/* Number of arenas used for automatic multiplexing of threads and arenas. */
+extern unsigned narenas_auto;
+
+/* Base index for manual arenas. */
+extern unsigned manual_arena_base;
+
+void     narenas_total_set(unsigned narenas);
+void     narenas_total_inc(void);
+unsigned narenas_total_get(void);
+void     narenas_auto_set(unsigned n);
+void     manual_arena_base_set(unsigned base);
+
+void    *a0malloc(size_t size);
+void     a0dalloc(void *ptr);
+void    *a0ialloc(size_t size, bool zero, bool is_internal);
+void     a0idalloc(void *ptr, bool is_internal);
+void     arena_set(unsigned ind, arena_t *arena);
+arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
+arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
+void     arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
+void     iarena_cleanup(tsd_t *tsd);
+void     arena_cleanup(tsd_t *tsd);
+
+bool     arenas_management_boot(void);
+void     arenas_management_prefork(tsdn_t *tsdn);
+void     arenas_management_postfork_parent(tsdn_t *tsdn);
+void     arenas_management_postfork_child(tsdn_t *tsdn);
+
+#endif /* JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H */

include/jemalloc/internal/bin.h

@@ -95,11 +95,21 @@ bool  bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, bool is_auto,
 void *bin_malloc_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin,
     szind_t binind);
 
+/* Slab queries. */
+void *bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin);
+
 /* Bin selection. */
 bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     unsigned *binshard_p);
 
 /* Stats. */
+static inline void
+bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) {
+	malloc_mutex_lock(tsdn, &bin->lock);
+	bin->stats.nrequests += n;
+	malloc_mutex_unlock(tsdn, &bin->lock);
+}
+
 static inline void
 bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
 	malloc_mutex_lock(tsdn, &bin->lock);

include/jemalloc/internal/bin_inlines.h

@@ -24,8 +24,8 @@ struct bin_dalloc_locked_info_s {
 
 /* Find the region index of a pointer within a slab. */
 JEMALLOC_ALWAYS_INLINE size_t
-bin_slab_regind_impl(
-    div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) {
+bin_slab_regind_impl(const div_info_t *div_info, szind_t binind,
+    const edata_t *slab, const void *ptr) {
 	size_t diff, regind;
 
 	/* Freeing a pointer outside the slab can cause assertion failure. */
@@ -45,8 +45,8 @@ bin_slab_regind_impl(
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-bin_slab_regind(bin_dalloc_locked_info_t *info, szind_t binind,
-    edata_t *slab, const void *ptr) {
+bin_slab_regind(const bin_dalloc_locked_info_t *info, szind_t binind,
+    const edata_t *slab, const void *ptr) {
 	size_t regind = bin_slab_regind_impl(
 	    &info->div_info, binind, slab, ptr);
 	return regind;

include/jemalloc/internal/bit_util.h

@@ -195,7 +195,7 @@ fls_u(unsigned x) {
 	/*								\
 	 * Algorithm from an old AMD optimization reference manual.	\
 	 * We're putting a little bit more work than you might expect	\
-	 * into the no-instrinsic case, since we only support the	\
+	 * into the no-intrinsic case, since we only support the	\
 	 * GCC intrinsics spelling of popcount (for now).  Detecting	\
 	 * whether or not the popcount builtin is actually useable in	\
 	 * MSVC is nontrivial.						\
@@ -212,7 +212,7 @@ fls_u(unsigned x) {
 	 * That costs an extra mask relative to this, though.		\
 	 */								\
 	x = x - ((x >> 1) & (0x55U * bmul));				\
-	/* Replace each 4 bits with their sideays sum.  0x3 = 0b0011. */\
+	/* Replace each 4 bits with their sideways sum.  0x3 = 0b0011. */\
 	x = (x & (bmul * 0x33U)) + ((x >> 2) & (bmul * 0x33U));		\
 	/*								\
 	 * Replace each 8 bits with their sideways sum.  Note that we	\

include/jemalloc/internal/bitmap.h

@@ -181,7 +181,7 @@ void   bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
 size_t bitmap_size(const bitmap_info_t *binfo);
 
 static inline bool
-bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
+bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {
 #ifdef BITMAP_USE_TREE
 	size_t   rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
 	bitmap_t rg = bitmap[rgoff];
@@ -200,7 +200,7 @@ bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
 }
 
 static inline bool
-bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
+bitmap_get(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
 	size_t   goff;
 	bitmap_t g;
 

include/jemalloc/internal/cache_bin.h

@@ -442,19 +442,6 @@ cache_bin_alloc(cache_bin_t *bin, bool *success) {
 	return cache_bin_alloc_impl(bin, success, true);
 }
 
-JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
-cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) {
-	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
-	if (n > num) {
-		n = (cache_bin_sz_t)num;
-	}
-	memcpy(out, bin->stack_head, n * sizeof(void *));
-	bin->stack_head += n;
-	cache_bin_low_water_adjust(bin);
-
-	return n;
-}
-
 JEMALLOC_ALWAYS_INLINE bool
 cache_bin_full(cache_bin_t *bin) {
 	return (
@@ -536,14 +523,13 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
 /* Get the number of stashed pointers. */
 JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
 cache_bin_nstashed_get_internal(cache_bin_t *bin) {
-	cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
 	cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
 	    bin);
 
 	cache_bin_sz_t n = cache_bin_diff(
 	                       bin, low_bits_low_bound, bin->low_bits_full)
 	    / sizeof(void *);
-	assert(n <= ncached_max);
+	assert(n <= cache_bin_ncached_max_get(bin));
 	if (config_debug && n != 0) {
 		/* Below are for assertions only. */
 		void **low_bound = cache_bin_low_bound_get(bin);

include/jemalloc/internal/ctl.h

@@ -63,6 +63,7 @@ typedef struct ctl_stats_s {
 	size_t resident;
 	size_t mapped;
 	size_t retained;
+	size_t pinned;
 
 	background_thread_stats_t background_thread;
 	mutex_prof_data_t mutex_prof_data[mutex_prof_num_global_mutexes];

include/jemalloc/internal/div.h

@@ -24,7 +24,7 @@ struct div_info_s {
 void div_init(div_info_t *div_info, size_t divisor);
 
 static inline size_t
-div_compute(div_info_t *div_info, size_t n) {
+div_compute(const div_info_t *div_info, size_t n) {
 	assert(n <= (uint32_t)-1);
 	/*
 	 * This generates, e.g. mov; imul; shr on x86-64. On a 32-bit machine,

include/jemalloc/internal/ecache.h

@@ -23,27 +23,27 @@ struct ecache_s {
 };
 
 static inline size_t
-ecache_npages_get(ecache_t *ecache) {
+ecache_npages_get(const ecache_t *ecache) {
 	return eset_npages_get(&ecache->eset)
 	    + eset_npages_get(&ecache->guarded_eset);
 }
 
 /* Get the number of extents in the given page size index. */
 static inline size_t
-ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
+ecache_nextents_get(const ecache_t *ecache, pszind_t ind) {
 	return eset_nextents_get(&ecache->eset, ind)
 	    + eset_nextents_get(&ecache->guarded_eset, ind);
 }
 
 /* Get the sum total bytes of the extents in the given page size index. */
 static inline size_t
-ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
+ecache_nbytes_get(const ecache_t *ecache, pszind_t ind) {
 	return eset_nbytes_get(&ecache->eset, ind)
 	    + eset_nbytes_get(&ecache->guarded_eset, ind);
 }
 
 static inline unsigned
-ecache_ind_get(ecache_t *ecache) {
+ecache_ind_get(const ecache_t *ecache) {
 	return ecache->ind;
 }
 

include/jemalloc/internal/edata.h

@@ -34,9 +34,10 @@ enum extent_state_e {
 	extent_state_dirty = 1,
 	extent_state_muzzy = 2,
 	extent_state_retained = 3,
-	extent_state_transition = 4, /* States below are intermediate. */
-	extent_state_merging = 5,
-	extent_state_max = 5 /* Sanity checking only. */
+	extent_state_pinned = 4,
+	extent_state_transition = 5, /* States below are intermediate. */
+	extent_state_merging = 6,
+	extent_state_max = 6 /* Sanity checking only. */
 };
 typedef enum extent_state_e extent_state_t;
 
@@ -47,8 +48,8 @@ enum extent_head_state_e {
 typedef enum extent_head_state_e extent_head_state_t;
 
 /*
- * Which implementation of the page allocator interface, (PAI, defined in
- * pai.h) owns the given extent?
+ * Which page allocator implementation (PAC or HPA) owns the given extent?
+ * Used by PA to route expand/shrink/dalloc to the correct implementation.
  */
 enum extent_pai_e { EXTENT_PAI_PAC = 0, EXTENT_PAI_HPA = 1 };
 typedef enum extent_pai_e extent_pai_t;
@@ -94,8 +95,8 @@ struct edata_cmp_summary_s {
 
 /* Extent (span of pages).  Use accessor functions for e_* fields. */
 typedef struct edata_s edata_t;
-ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM);
-ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM);
+ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM)
+ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM)
 struct edata_s {
 	/*
 	 * Bitfield containing several fields:
@@ -110,8 +111,10 @@ struct edata_s {
 	 * i: szind
 	 * f: nfree
 	 * s: bin_shard
+	 * h: is_head
+	 * n: pinned
 	 *
-	 * 00000000 ... 0000ssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
+	 * 00000000 ... 0nhsssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
 	 *
 	 * arena_ind: Arena from which this extent came, or all 1 bits if
 	 *            unassociated.
@@ -145,6 +148,10 @@ struct edata_s {
 	 * nfree: Number of free regions in slab.
 	 *
 	 * bin_shard: the shard of the bin from which this extent came.
+	 *
+	 * is_head: see comments in ehooks_default_merge_impl().
+	 *
+	 * pinned: true if the alloc hook signaled non-reclaimable backing.
 	 */
 	uint64_t e_bits;
 #define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT)                         \
@@ -210,6 +217,16 @@ struct edata_s {
 #define EDATA_BITS_IS_HEAD_MASK                                                \
 	MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
 
+#define EDATA_BITS_PINNED_WIDTH 1
+#define EDATA_BITS_PINNED_SHIFT                                                \
+	(EDATA_BITS_IS_HEAD_WIDTH + EDATA_BITS_IS_HEAD_SHIFT)
+#define EDATA_BITS_PINNED_MASK                                                 \
+	MASK(EDATA_BITS_PINNED_WIDTH, EDATA_BITS_PINNED_SHIFT)
+
+#if (EDATA_BITS_PINNED_SHIFT + EDATA_BITS_PINNED_WIDTH > 64)
+#error "edata_t e_bits overflow"
+#endif
+
 	/* Pointer to the extent that this structure is responsible for. */
 	void *e_addr;
 
@@ -538,6 +555,29 @@ edata_ps_set(edata_t *edata, hpdata_t *ps) {
 	edata->e_ps = ps;
 }
 
+static inline bool
+edata_pinned_get(const edata_t *edata) {
+	return (bool)((edata->e_bits & EDATA_BITS_PINNED_MASK)
+	    >> EDATA_BITS_PINNED_SHIFT);
+}
+
+static inline void
+edata_pinned_set(edata_t *edata, bool pinned) {
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_PINNED_MASK)
+	    | ((uint64_t)pinned << EDATA_BITS_PINNED_SHIFT);
+}
+
+static inline void
+edata_hook_flags_init(edata_t *edata, unsigned alloc_flags) {
+	edata_pinned_set(edata,
+	    (alloc_flags & EXTENT_ALLOC_FLAG_PINNED) != 0);
+}
+
+static inline unsigned
+edata_alloc_flags_get(const edata_t *edata) {
+	return edata_pinned_get(edata) ? EXTENT_ALLOC_FLAG_PINNED : 0;
+}
+
 static inline void
 edata_szind_set(edata_t *edata, szind_t szind) {
 	assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
@@ -645,7 +685,7 @@ edata_prof_recent_alloc_set_dont_call_directly(
 }
 
 static inline bool
-edata_is_head_get(edata_t *edata) {
+edata_is_head_get(const edata_t *edata) {
 	return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK)
 	    >> EDATA_BITS_IS_HEAD_SHIFT);
 }
@@ -686,6 +726,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
 	edata_committed_set(edata, committed);
 	edata_pai_set(edata, pai);
 	edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
+	edata_hook_flags_init(edata, 0);
 	if (config_prof) {
 		edata_prof_tctx_set(edata, NULL);
 	}
@@ -711,6 +752,7 @@ edata_binit(
 	 * wasting a state bit to encode this fact.
 	 */
 	edata_pai_set(edata, EXTENT_PAI_PAC);
+	edata_hook_flags_init(edata, 0);
 }
 
 static inline int

include/jemalloc/internal/ehooks.h

@@ -110,12 +110,12 @@ ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
 }
 
 static inline extent_hooks_t *
-ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
+ehooks_get_extent_hooks_ptr(const ehooks_t *ehooks) {
 	return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
 }
 
 static inline bool
-ehooks_are_default(ehooks_t *ehooks) {
+ehooks_are_default(const ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)
 	    == &ehooks_default_extent_hooks;
 }
@@ -126,7 +126,7 @@ ehooks_are_default(ehooks_t *ehooks) {
  * include some checks for such cases.
  */
 static inline bool
-ehooks_dalloc_will_fail(ehooks_t *ehooks) {
+ehooks_dalloc_will_fail(const ehooks_t *ehooks) {
 	if (ehooks_are_default(ehooks)) {
 		return opt_retain;
 	} else {
@@ -135,17 +135,17 @@ ehooks_dalloc_will_fail(ehooks_t *ehooks) {
 }
 
 static inline bool
-ehooks_split_will_fail(ehooks_t *ehooks) {
+ehooks_split_will_fail(const ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
 }
 
 static inline bool
-ehooks_merge_will_fail(ehooks_t *ehooks) {
+ehooks_merge_will_fail(const ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
 }
 
 static inline bool
-ehooks_guard_will_fail(ehooks_t *ehooks) {
+ehooks_guard_will_fail(const ehooks_t *ehooks) {
 	/*
 	 * Before the guard hooks are officially introduced, limit the use to
 	 * the default hooks only.
@@ -191,7 +191,7 @@ ehooks_debug_zero_check(void *addr, size_t size) {
 
 static inline void *
 ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
-    size_t alignment, bool *zero, bool *commit) {
+    size_t alignment, bool *zero, bool *commit, unsigned *alloc_flags) {
 	bool            orig_zero = *zero;
 	void           *ret;
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
@@ -204,6 +204,18 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
 		    alignment, zero, commit, ehooks_ind_get(ehooks));
 		ehooks_post_reentrancy(tsdn);
 	}
+#if LG_PAGE < 8
+#  error "Extent alloc flags require page size of at least 256"
+#endif
+	if (ret != NULL) {
+		*alloc_flags = (unsigned)((uintptr_t)ret
+		    & EXTENT_ALLOC_FLAG_MASK);
+		ret = (void *)((byte_t *)ret - *alloc_flags);
+		/* Pinned hooks must also set *commit; pinned bypasses commit/decommit. */
+		assert(!(*alloc_flags & EXTENT_ALLOC_FLAG_PINNED) || *commit);
+	} else {
+		*alloc_flags = 0;
+	}
 	assert(new_addr == NULL || ret == NULL || new_addr == ret);
 	assert(!orig_zero || *zero);
 	if (*zero && ret != NULL) {

include/jemalloc/internal/emap.h

@@ -211,6 +211,7 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) {
 	assert(edata_state_get(inner) == extent_state_active);
 	assert(edata_state_get(outer) == extent_state_merging);
 	assert(!edata_guarded_get(inner) && !edata_guarded_get(outer));
+	assert(edata_pinned_get(inner) == edata_pinned_get(outer));
 	assert(edata_base_get(inner) == edata_past_get(outer)
 	    || edata_base_get(outer) == edata_past_get(inner));
 }

include/jemalloc/internal/eset.h

@@ -60,11 +60,11 @@ struct eset_s {
 
 void eset_init(eset_t *eset, extent_state_t state);
 
-size_t eset_npages_get(eset_t *eset);
+size_t eset_npages_get(const eset_t *eset);
 /* Get the number of extents in the given page size index. */
-size_t eset_nextents_get(eset_t *eset, pszind_t ind);
+size_t eset_nextents_get(const eset_t *eset, pszind_t ind);
 /* Get the sum total bytes of the extents in the given page size index. */
-size_t eset_nbytes_get(eset_t *eset, pszind_t ind);
+size_t eset_nbytes_get(const eset_t *eset, pszind_t ind);
 
 void eset_insert(eset_t *eset, edata_t *edata);
 void eset_remove(eset_t *eset, edata_t *edata);
@@ -73,6 +73,6 @@ void eset_remove(eset_t *eset, edata_t *edata);
  * null if no such item could be found.
  */
 edata_t *eset_fit(eset_t *eset, size_t esize, size_t alignment, bool exact_only,
-    unsigned lg_max_fit);
+    unsigned lg_max_fit, bool prefer_small);
 
 #endif /* JEMALLOC_INTERNAL_ESET_H */

include/jemalloc/internal/extent.h

@@ -91,7 +91,7 @@ extent_neighbor_head_state_mergeable(
 }
 
 JEMALLOC_ALWAYS_INLINE bool
-extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
+extent_can_acquire_neighbor(const edata_t *edata, rtree_contents_t contents,
     extent_pai_t pai, extent_state_t expected_state, bool forward,
     bool expanding) {
 	edata_t *neighbor = contents.edata;
@@ -120,6 +120,10 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
 			 */
 			return false;
 		}
+		/* Do not merge pinned and non-pinned extents. */
+		if (edata_pinned_get(edata) != edata_pinned_get(neighbor)) {
+			return false;
+		}
 	} else {
 		if (neighbor_state == extent_state_active) {
 			return false;

include/jemalloc/internal/extent_dss.h

@@ -27,4 +27,9 @@ bool       extent_in_dss(void *addr);
 bool       extent_dss_mergeable(void *addr_a, void *addr_b);
 void       extent_dss_boot(void);
 
+#ifdef JEMALLOC_JET
+typedef void *(*extent_dss_sbrk_hook_t)(intptr_t);
+extern extent_dss_sbrk_hook_t extent_dss_sbrk_hook;
+#endif
+
 #endif /* JEMALLOC_INTERNAL_EXTENT_DSS_H */

include/jemalloc/internal/fb.h

@@ -25,7 +25,7 @@ fb_init(fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_empty(fb_group_t *fb, size_t nbits) {
+fb_empty(const fb_group_t *fb, size_t nbits) {
 	size_t ngroups = FB_NGROUPS(nbits);
 	for (size_t i = 0; i < ngroups; i++) {
 		if (fb[i] != 0) {
@@ -36,7 +36,7 @@ fb_empty(fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_full(fb_group_t *fb, size_t nbits) {
+fb_full(const fb_group_t *fb, size_t nbits) {
 	size_t ngroups = FB_NGROUPS(nbits);
 	size_t trailing_bits = nbits % FB_GROUP_BITS;
 	size_t limit = (trailing_bits == 0 ? ngroups : ngroups - 1);
@@ -52,7 +52,7 @@ fb_full(fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_get(fb_group_t *fb, size_t nbits, size_t bit) {
+fb_get(const fb_group_t *fb, size_t nbits, size_t bit) {
 	assert(bit < nbits);
 	size_t group_ind = bit / FB_GROUP_BITS;
 	size_t bit_ind = bit % FB_GROUP_BITS;
@@ -156,15 +156,21 @@ fb_scount_visitor(void *ctx, fb_group_t *fb, fb_group_t mask) {
 
 /* Finds the number of set bit in the of length cnt starting at start. */
 JEMALLOC_ALWAYS_INLINE size_t
-fb_scount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
+fb_scount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
 	size_t scount = 0;
-	fb_visit_impl(fb, nbits, &fb_scount_visitor, &scount, start, cnt);
+	/*
+	 * fb_visit_impl is shared with mutating visitors (e.g. fb_set_range),
+	 * so it takes a non-const fb.  fb_scount_visitor only reads, so the
+	 * cast is safe.
+	 */
+	fb_visit_impl((fb_group_t *)fb, nbits, &fb_scount_visitor, &scount,
+	    start, cnt);
 	return scount;
 }
 
 /* Finds the number of unset bit in the of length cnt starting at start. */
 JEMALLOC_ALWAYS_INLINE size_t
-fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
+fb_ucount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
 	size_t scount = fb_scount(fb, nbits, start, cnt);
 	return cnt - scount;
 }
@@ -176,8 +182,8 @@ fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
  * Returns the number of bits in the bitmap if no such bit exists.
  */
 JEMALLOC_ALWAYS_INLINE ssize_t
-fb_find_impl(
-    fb_group_t *fb, size_t nbits, size_t start, bool val, bool forward) {
+fb_find_impl(const fb_group_t *fb, size_t nbits, size_t start, bool val,
+    bool forward) {
 	assert(start < nbits);
 	size_t  ngroups = FB_NGROUPS(nbits);
 	ssize_t group_ind = start / FB_GROUP_BITS;
@@ -226,14 +232,14 @@ fb_find_impl(
  * number of bits in the bitmap if no such bit exists.
  */
 static inline size_t
-fb_ffu(fb_group_t *fb, size_t nbits, size_t min_bit) {
+fb_ffu(const fb_group_t *fb, size_t nbits, size_t min_bit) {
 	return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ false,
 	    /* forward */ true);
 }
 
 /* The same, but looks for an unset bit. */
 static inline size_t
-fb_ffs(fb_group_t *fb, size_t nbits, size_t min_bit) {
+fb_ffs(const fb_group_t *fb, size_t nbits, size_t min_bit) {
 	return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ true,
 	    /* forward */ true);
 }
@@ -243,21 +249,21 @@ fb_ffs(fb_group_t *fb, size_t nbits, size_t min_bit) {
  * no such bit exists.
  */
 static inline ssize_t
-fb_flu(fb_group_t *fb, size_t nbits, size_t max_bit) {
+fb_flu(const fb_group_t *fb, size_t nbits, size_t max_bit) {
 	return fb_find_impl(fb, nbits, max_bit, /* val */ false,
 	    /* forward */ false);
 }
 
 static inline ssize_t
-fb_fls(fb_group_t *fb, size_t nbits, size_t max_bit) {
+fb_fls(const fb_group_t *fb, size_t nbits, size_t max_bit) {
 	return fb_find_impl(fb, nbits, max_bit, /* val */ true,
 	    /* forward */ false);
 }
 
 /* Returns whether or not we found a range. */
 JEMALLOC_ALWAYS_INLINE bool
-fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len, bool val, bool forward) {
+fb_iter_range_impl(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len, bool val, bool forward) {
 	assert(start < nbits);
 	ssize_t next_range_begin = fb_find_impl(fb, nbits, start, val, forward);
 	if ((forward && next_range_begin == (ssize_t)nbits)
@@ -286,8 +292,8 @@ fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
  * touching *r_begin or *r_end).
  */
 static inline bool
-fb_srange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_srange_iter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ true, /* forward */ true);
 }
@@ -297,30 +303,30 @@ fb_srange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
  * forwards.  (The position returned is still the earliest bit in the range).
  */
 static inline bool
-fb_srange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_srange_riter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ true, /* forward */ false);
 }
 
 /* Similar to fb_srange_iter, but searches for unset bits. */
 static inline bool
-fb_urange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_urange_iter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ false, /* forward */ true);
 }
 
 /* Similar to fb_srange_riter, but searches for unset bits. */
 static inline bool
-fb_urange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_urange_riter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ false, /* forward */ false);
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
+fb_range_longest_impl(const fb_group_t *fb, size_t nbits, bool val) {
 	size_t begin = 0;
 	size_t longest_len = 0;
 	size_t len = 0;
@@ -336,12 +342,12 @@ fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
 }
 
 static inline size_t
-fb_srange_longest(fb_group_t *fb, size_t nbits) {
+fb_srange_longest(const fb_group_t *fb, size_t nbits) {
 	return fb_range_longest_impl(fb, nbits, /* val */ true);
 }
 
 static inline size_t
-fb_urange_longest(fb_group_t *fb, size_t nbits) {
+fb_urange_longest(const fb_group_t *fb, size_t nbits) {
 	return fb_range_longest_impl(fb, nbits, /* val */ false);
 }
 

include/jemalloc/internal/hook.h

@@ -1,163 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_HOOK_H
-#define JEMALLOC_INTERNAL_HOOK_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/tsd.h"
-
-/*
- * This API is *extremely* experimental, and may get ripped out, changed in API-
- * and ABI-incompatible ways, be insufficiently or incorrectly documented, etc.
- *
- * It allows hooking the stateful parts of the API to see changes as they
- * happen.
- *
- * Allocation hooks are called after the allocation is done, free hooks are
- * called before the free is done, and expand hooks are called after the
- * allocation is expanded.
- *
- * For realloc and rallocx, if the expansion happens in place, the expansion
- * hook is called.  If it is moved, then the alloc hook is called on the new
- * location, and then the free hook is called on the old location (i.e. both
- * hooks are invoked in between the alloc and the dalloc).
- *
- * If we return NULL from OOM, then usize might not be trustworthy.  Calling
- * realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0)
- * only calls the free hook.  (Calling realloc(NULL, 0) is treated as malloc(0),
- * and only calls the alloc hook).
- *
- * Reentrancy:
- *   Reentrancy is guarded against from within the hook implementation.  If you
- *   call allocator functions from within a hook, the hooks will not be invoked
- *   again.
- * Threading:
- *   The installation of a hook synchronizes with all its uses.  If you can
- *   prove the installation of a hook happens-before a jemalloc entry point,
- *   then the hook will get invoked (unless there's a racing removal).
- *
- *   Hook insertion appears to be atomic at a per-thread level (i.e. if a thread
- *   allocates and has the alloc hook invoked, then a subsequent free on the
- *   same thread will also have the free hook invoked).
- *
- *   The *removal* of a hook does *not* block until all threads are done with
- *   the hook.  Hook authors have to be resilient to this, and need some
- *   out-of-band mechanism for cleaning up any dynamically allocated memory
- *   associated with their hook.
- * Ordering:
- *   Order of hook execution is unspecified, and may be different than insertion
- *   order.
- */
-
-#define HOOK_MAX 4
-
-enum hook_alloc_e {
-	hook_alloc_malloc,
-	hook_alloc_posix_memalign,
-	hook_alloc_aligned_alloc,
-	hook_alloc_calloc,
-	hook_alloc_memalign,
-	hook_alloc_valloc,
-	hook_alloc_pvalloc,
-	hook_alloc_mallocx,
-
-	/* The reallocating functions have both alloc and dalloc variants */
-	hook_alloc_realloc,
-	hook_alloc_rallocx,
-};
-/*
- * We put the enum typedef after the enum, since this file may get included by
- * jemalloc_cpp.cpp, and C++ disallows enum forward declarations.
- */
-typedef enum hook_alloc_e hook_alloc_t;
-
-enum hook_dalloc_e {
-	hook_dalloc_free,
-	hook_dalloc_dallocx,
-	hook_dalloc_sdallocx,
-
-	/*
-	 * The dalloc halves of reallocation (not called if in-place expansion
-	 * happens).
-	 */
-	hook_dalloc_realloc,
-	hook_dalloc_rallocx,
-};
-typedef enum hook_dalloc_e hook_dalloc_t;
-
-enum hook_expand_e {
-	hook_expand_realloc,
-	hook_expand_rallocx,
-	hook_expand_xallocx,
-};
-typedef enum hook_expand_e hook_expand_t;
-
-typedef void (*hook_alloc)(void *extra, hook_alloc_t type, void *result,
-    uintptr_t result_raw, uintptr_t args_raw[3]);
-
-typedef void (*hook_dalloc)(
-    void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
-
-typedef void (*hook_expand)(void *extra, hook_expand_t type, void *address,
-    size_t old_usize, size_t new_usize, uintptr_t result_raw,
-    uintptr_t args_raw[4]);
-
-typedef struct hooks_s hooks_t;
-struct hooks_s {
-	hook_alloc  alloc_hook;
-	hook_dalloc dalloc_hook;
-	hook_expand expand_hook;
-	void       *extra;
-};
-
-/*
- * Begin implementation details; everything above this point might one day live
- * in a public API.  Everything below this point never will.
- */
-
-/*
- * The realloc pathways haven't gotten any refactoring love in a while, and it's
- * fairly difficult to pass information from the entry point to the hooks.  We
- * put the informaiton the hooks will need into a struct to encapsulate
- * everything.
- *
- * Much of these pathways are force-inlined, so that the compiler can avoid
- * materializing this struct until we hit an extern arena function.  For fairly
- * goofy reasons, *many* of the realloc paths hit an extern arena function.
- * These paths are cold enough that it doesn't matter; eventually, we should
- * rewrite the realloc code to make the expand-in-place and the
- * free-then-realloc paths more orthogonal, at which point we don't need to
- * spread the hook logic all over the place.
- */
-typedef struct hook_ralloc_args_s hook_ralloc_args_t;
-struct hook_ralloc_args_s {
-	/* I.e. as opposed to rallocx. */
-	bool is_realloc;
-	/*
-	 * The expand hook takes 4 arguments, even if only 3 are actually used;
-	 * we add an extra one in case the user decides to memcpy without
-	 * looking too closely at the hooked function.
-	 */
-	uintptr_t args[4];
-};
-
-/*
- * Returns an opaque handle to be used when removing the hook.  NULL means that
- * we couldn't install the hook.
- */
-bool hook_boot(void);
-
-void *hook_install(tsdn_t *tsdn, hooks_t *to_install);
-/* Uninstalls the hook with the handle previously returned from hook_install. */
-void hook_remove(tsdn_t *tsdn, void *opaque);
-
-/* Hooks */
-
-void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
-    uintptr_t args_raw[3]);
-
-void hook_invoke_dalloc(
-    hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
-
-void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
-    size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
-
-#endif /* JEMALLOC_INTERNAL_HOOK_H */

include/jemalloc/internal/hpa.h

@@ -10,7 +10,6 @@
 #include "jemalloc/internal/hpa_hooks.h"
 #include "jemalloc/internal/hpa_opts.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/psset.h"
 #include "jemalloc/internal/sec.h"
 
@@ -51,6 +50,60 @@ struct hpa_shard_nonderived_stats_s {
 	 * Guarded by mtx.
 	 */
 	uint64_t ndehugifies;
+
+	/*
+	 * Distribution of the min number of extents we will try to allocate
+	 * from a single hpa_alloc() call.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the max number of extents we will try to allocate
+	 * from a single hpa_alloc() call.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of extents allocated for a single
+	 * hpa_alloc() call and a single mtx lock acquisition.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of ps out of which we allocated extents
+	 * for a single hpa_alloc() call and a single mtx lock acquisition.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of pages allocated from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of extents allocated from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the total elapsed time (ns) for allocating extents
+	 * from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
 };
 
 /* Completely derived; only used by CTL. */
@@ -63,12 +116,6 @@ struct hpa_shard_stats_s {
 
 typedef struct hpa_shard_s hpa_shard_t;
 struct hpa_shard_s {
-	/*
-	 * pai must be the first member; we cast from a pointer to it to a
-	 * pointer to the hpa_shard_t.
-	 */
-	pai_t pai;
-
 	/* The central allocator we get our hugepages from. */
 	hpa_central_t *central;
 
@@ -145,7 +192,7 @@ struct hpa_shard_s {
 
 bool hpa_hugepage_size_exceeds_limit(void);
 /*
- * Whether or not the HPA can be used given the current configuration.  This is
+ * Whether or not the HPA can be used given the current configuration.  This
  * is not necessarily a guarantee that it backs its allocations by hugepages,
  * just that it can function properly given the system it's running on.
  */
@@ -154,6 +201,18 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
     emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
     const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts);
 
+edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
+    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
+    bool *deferred_work_generated);
+bool hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
+    size_t old_size, size_t new_size, bool zero,
+    bool *deferred_work_generated);
+bool hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
+    size_t old_size, size_t new_size, bool *deferred_work_generated);
+void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
+    bool *deferred_work_generated);
+uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
+
 void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
 void hpa_shard_stats_merge(
     tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst);

include/jemalloc/internal/hpdata.h

@@ -27,7 +27,7 @@
  */
 #define PSSET_ENUMERATE_MAX_NUM 32
 typedef struct hpdata_s hpdata_t;
-ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM);
+ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM)
 struct hpdata_s {
 	/*
 	 * We likewise follow the edata convention of mangling names and forcing
@@ -136,7 +136,7 @@ TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty)
 TYPED_LIST(hpdata_purge_list, hpdata_t, ql_link_purge)
 TYPED_LIST(hpdata_hugify_list, hpdata_t, ql_link_hugify)
 
-ph_proto(, hpdata_age_heap, hpdata_t);
+ph_proto(, hpdata_age_heap, hpdata_t)
 
 static inline void *
 hpdata_addr_get(const hpdata_t *hpdata) {
@@ -209,7 +209,7 @@ hpdata_allow_hugify(hpdata_t *hpdata, nstime_t now) {
 }
 
 static inline nstime_t
-hpdata_time_hugify_allowed(hpdata_t *hpdata) {
+hpdata_time_hugify_allowed(const hpdata_t *hpdata) {
 	return hpdata->h_time_hugify_allowed;
 }
 
@@ -305,7 +305,7 @@ hpdata_ndirty_get(const hpdata_t *hpdata) {
 }
 
 static inline size_t
-hpdata_nretained_get(hpdata_t *hpdata) {
+hpdata_nretained_get(const hpdata_t *hpdata) {
 	return HUGEPAGE_PAGES - hpdata->h_ntouched;
 }
 
@@ -330,7 +330,7 @@ hpdata_purged_when_empty_and_huge_set(hpdata_t *hpdata, bool v) {
 }
 
 static inline void
-hpdata_assert_empty(hpdata_t *hpdata) {
+hpdata_assert_empty(const hpdata_t *hpdata) {
 	assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES));
 	assert(hpdata->h_nactive == 0);
 }
@@ -341,7 +341,7 @@ hpdata_assert_empty(hpdata_t *hpdata) {
  * match computed ones).
  */
 static inline bool
-hpdata_consistent(hpdata_t *hpdata) {
+hpdata_consistent(const hpdata_t *hpdata) {
 	bool res = true;
 
 	const size_t active_urange_longest = fb_urange_longest(
@@ -432,6 +432,60 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
+/*
+ * For buffering extent allocations we will perform out of
+ * a single ps.
+ */
+typedef struct hpdata_alloc_offset_s hpdata_alloc_offset_t;
+struct hpdata_alloc_offset_s {
+	/*
+	 * Index on the active bitmap for the extent to allocate.
+	 * It is used to know which bits we'll need to set when we perform
+	 * the allocation. They are in the range [index, index + npages).
+	 */
+	size_t index;
+
+	/*
+	 * The length of the free bit range on the active bitmap,
+	 * starting at index, before setting the bits in the range
+	 * [index, index + npages).
+	 * It is used to determine whether one of the allocations
+	 * used up the longest free range on the active bitmap.
+	 * If it did, we might have to update the longest free range
+	 * metadata on the hpdata.
+	 */
+	size_t len_before;
+
+	/*
+	 * The length of the longest free range in the range [0, index).
+	 * When we need to update the longest free range on the hpdata,
+	 * the new value is either longest_len (the max up to index),
+	 * len_before - npages (what's left after we carve up the free
+	 * range starting at index), or the max in the range
+	 * [index + len_before, HUGEPAGE_PAGES), whichever is greater.
+	 */
+	size_t longest_len;
+};
+
+/*
+ * Given an hpdata that can serve an allocation request of size sz,
+ * find between one and max_nallocs offsets that can satisfy such
+ * an allocation request and buffer them in offsets (without actually
+ * reserving any space or updating hpdata). Return the number
+ * of offsets discovered.
+ */
+size_t hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t max_nallocs);
+/* Reserve the allocation for the given offset. */
+void *hpdata_reserve_alloc_offset(
+    hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset);
+/*
+ * Do any work that needs to be done after performing allocations
+ * from a single hpdata.
+ */
+void hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t nallocs);
+
 /*
  * The hpdata_purge_prepare_t allows grabbing the metadata required to purge
  * subranges of a hugepage while holding a lock, drop the lock during the actual

include/jemalloc/internal/jemalloc_fork.h

@@ -0,0 +1,8 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_FORK_H
+#define JEMALLOC_INTERNAL_JEMALLOC_FORK_H
+
+void jemalloc_prefork(void);
+void jemalloc_postfork_parent(void);
+void jemalloc_postfork_child(void);
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_FORK_H */

include/jemalloc/internal/jemalloc_init.h

@@ -0,0 +1,40 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_INIT_H
+#define JEMALLOC_INTERNAL_JEMALLOC_INIT_H
+
+enum malloc_init_e {
+	malloc_init_uninitialized = 3,
+	malloc_init_a0_initialized = 2,
+	malloc_init_recursible = 1,
+	malloc_init_initialized = 0 /* Common case --> jnz. */
+};
+typedef enum malloc_init_e malloc_init_t;
+
+extern malloc_init_t malloc_init_state;
+
+bool malloc_is_initializer(void);
+
+bool malloc_init_hard_a0(void);
+bool malloc_init_hard(void);
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_init_a0(void) {
+	if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
+		return malloc_init_hard_a0();
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_initialized(void) {
+	return (malloc_init_state == malloc_init_initialized);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_init(void) {
+	if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
+		return true;
+	}
+	return false;
+}
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_INIT_H */

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -465,6 +465,9 @@
 /* Is C++ support being built? */
 #undef JEMALLOC_ENABLE_CXX
 
+/* Are C++ exceptions enabled? */
+#undef JEMALLOC_HAVE_CXX_EXCEPTIONS
+
 /* Performs additional size checks when defined. */
 #undef JEMALLOC_OPT_SIZE_CHECKS
 

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -37,7 +37,6 @@ extern bool                  opt_zero;
 extern unsigned              opt_narenas;
 extern fxp_t                 opt_narenas_ratio;
 extern zero_realloc_action_t opt_zero_realloc_action;
-extern malloc_init_t         malloc_init_state;
 extern const char *const     zero_realloc_mode_names[];
 extern atomic_zu_t           zero_realloc_count;
 extern bool                  opt_cache_oblivious;
@@ -54,36 +53,9 @@ extern uintptr_t san_cache_bin_nonfast_mask;
 /* Number of CPUs. */
 extern unsigned ncpus;
 
-/* Number of arenas used for automatic multiplexing of threads and arenas. */
-extern unsigned narenas_auto;
-
-/* Base index for manual arenas. */
-extern unsigned manual_arena_base;
-
-/*
- * Arenas that are used to service external requests.  Not all elements of the
- * arenas array are necessarily used; arenas are created lazily as needed.
- */
-extern atomic_p_t arenas[];
-
-extern unsigned huge_arena_ind;
-
-void    *a0malloc(size_t size);
-void     a0dalloc(void *ptr);
 void    *bootstrap_malloc(size_t size);
 void    *bootstrap_calloc(size_t num, size_t size);
 void     bootstrap_free(void *ptr);
-void     arena_set(unsigned ind, arena_t *arena);
-unsigned narenas_total_get(void);
-arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
-arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
-void     arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
-void     iarena_cleanup(tsd_t *tsd);
-void     arena_cleanup(tsd_t *tsd);
-size_t   batch_alloc(void **ptrs, size_t num, size_t size, int flags);
-void     jemalloc_prefork(void);
-void     jemalloc_postfork_parent(void);
-void     jemalloc_postfork_child(void);
 void     sdallocx_default(void *ptr, size_t size, int flags);
 void     free_default(void *ptr);
 void    *malloc_default(size_t size);

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -4,6 +4,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/arena_externs.h"
 #include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/arenas_management.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bit_util.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"

include/jemalloc/internal/jemalloc_internal_inlines_b.h

@@ -3,9 +3,22 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/arena_inlines_a.h"
+#include "jemalloc/internal/arenas_management.h"
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_a.h"
 
+static inline void
+thread_migrate_arena(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
+	assert(oldarena != NULL);
+	assert(newarena != NULL);
+
+	arena_migrate(tsd, oldarena, newarena);
+	if (tcache_available(tsd)) {
+		tcache_arena_reassociate(tsd_tsdn(tsd),
+		    tsd_tcache_slowp_get(tsd), newarena);
+	}
+}
+
 static inline void
 percpu_arena_update(tsd_t *tsd, unsigned cpu) {
 	assert(have_percpu_arena);
@@ -18,15 +31,7 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
 		arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
 		assert(newarena != NULL);
 
-		/* Set new arena/tcache associations. */
-		arena_migrate(tsd, oldarena, newarena);
-		tcache_t *tcache = tcache_get(tsd);
-		if (tcache != NULL) {
-			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
-			assert(tcache_slow->arena != NULL);
-			tcache_arena_reassociate(
-			    tsd_tsdn(tsd), tcache_slow, tcache, newarena);
-		}
+		thread_migrate_arena(tsd, oldarena, newarena);
 	}
 }
 
@@ -50,18 +55,17 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
 		assert(ret);
 		if (tcache_available(tsd)) {
 			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
-			tcache_t      *tcache = tsd_tcachep_get(tsd);
 			if (tcache_slow->arena != NULL) {
 				/* See comments in tsd_tcache_data_init().*/
 				assert(tcache_slow->arena
 				    == arena_get(tsd_tsdn(tsd), 0, false));
 				if (tcache_slow->arena != ret) {
 					tcache_arena_reassociate(tsd_tsdn(tsd),
-					    tcache_slow, tcache, ret);
+					    tcache_slow, ret);
 				}
 			} else {
 				tcache_arena_associate(
-				    tsd_tsdn(tsd), tcache_slow, tcache, ret);
+				    tsd_tsdn(tsd), tcache_slow, ret);
 			}
 		}
 	}
@@ -97,7 +101,7 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) {
 }
 
 static inline bool
-arena_is_auto(arena_t *arena) {
+arena_is_auto(const arena_t *arena) {
 	assert(narenas_auto > 0);
 
 	return (arena_ind_get(arena) < manual_arena_base);

include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -5,7 +5,7 @@
 #include "jemalloc/internal/arena_externs.h"
 #include "jemalloc/internal/arena_inlines_b.h"
 #include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/hook.h"
+#include "jemalloc/internal/jemalloc_init.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/log.h"
 #include "jemalloc/internal/sz.h"
@@ -174,8 +174,7 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
-    hook_ralloc_args_t *hook_args) {
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	void  *p;
@@ -196,27 +195,13 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 	memcpy(p, ptr, copysize);
-	hook_invoke_alloc(
-	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, p,
-	    (uintptr_t)p, hook_args->args);
-	hook_invoke_dalloc(
-	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
-	    ptr, hook_args->args);
 	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
 	return p;
 }
 
-/*
- * is_realloc threads through the knowledge of whether or not this call comes
- * from je_realloc (as opposed to je_rallocx); this ensures that we pass the
- * correct entry point into any hooks.
- * Note that these functions are all force-inlined, so no actual bool gets
- * passed-around anywhere.
- */
 JEMALLOC_ALWAYS_INLINE void *
 iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
-    hook_ralloc_args_t *hook_args) {
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
 	assert(ptr != NULL);
 	assert(size != 0);
 	witness_assert_depth_to_rank(
@@ -229,27 +214,26 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 		 * and copy.
 		 */
 		return iralloct_realign(tsdn, ptr, oldsize, size, alignment,
-		    zero, slab, tcache, arena, hook_args);
+		    zero, slab, tcache, arena);
 	}
 
 	return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
-	    slab, tcache, hook_args);
+	    slab, tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
-    size_t usize, bool zero, tcache_t *tcache, arena_t *arena,
-    hook_ralloc_args_t *hook_args) {
+    size_t usize, bool zero, tcache_t *tcache, arena_t *arena) {
 	bool slab = sz_can_use_slab(usize);
 	return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero,
-	    slab, tcache, arena, hook_args);
+	    slab, tcache, arena);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
-    size_t usize, bool zero, hook_ralloc_args_t *hook_args) {
+    size_t usize, bool zero) {
 	return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, usize,
-	    zero, tcache_get(tsd), NULL, hook_args);
+	    zero, tcache_get(tsd), NULL);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -280,11 +264,6 @@ fastpath_success_finish(
 	}
 }
 
-JEMALLOC_ALWAYS_INLINE bool
-malloc_initialized(void) {
-	return (malloc_init_state == malloc_init_initialized);
-}
-
 /*
  * malloc() fastpath.  Included here so that we can inline it into operator new;
  * function call overhead there is non-negligible as a fraction of total CPU in

include/jemalloc/internal/jemalloc_internal_types.h

@@ -20,14 +20,6 @@ typedef enum zero_realloc_action_e zero_realloc_action_t;
 /* Signature of write callback. */
 typedef void(write_cb_t)(void *, const char *);
 
-enum malloc_init_e {
-	malloc_init_uninitialized = 3,
-	malloc_init_a0_initialized = 2,
-	malloc_init_recursible = 1,
-	malloc_init_initialized = 0 /* Common case --> jnz. */
-};
-typedef enum malloc_init_e malloc_init_t;
-
 /*
  * Flags bits:
  *

include/jemalloc/internal/large_externs.h

@@ -3,7 +3,6 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/hook.h"
 
 void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
 void *large_palloc(
@@ -11,8 +10,7 @@ void *large_palloc(
 bool  large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
      size_t usize_max, bool zero);
 void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
-    size_t alignment, bool zero, tcache_t *tcache,
-    hook_ralloc_args_t *hook_args);
+    size_t alignment, bool zero, tcache_t *tcache);
 
 void   large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata);
 void   large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);

include/jemalloc/internal/mutex_prof.h

@@ -30,6 +30,7 @@ typedef enum {
 	OP(extents_dirty)                                                      \
 	OP(extents_muzzy)                                                      \
 	OP(extents_retained)                                                   \
+	OP(extents_pinned)                                                     \
 	OP(decay_dirty)                                                        \
 	OP(decay_muzzy)                                                        \
 	OP(base)                                                               \

include/jemalloc/internal/pa.h

@@ -10,15 +10,13 @@
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/lockedint.h"
 #include "jemalloc/internal/pac.h"
-#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/sec.h"
 
 /*
  * The page allocator; responsible for acquiring pages of memory for
- * allocations.  It picks the implementation of the page allocator interface
- * (i.e. a pai_t) to handle a given page-level allocation request.  For now, the
- * only such implementation is the PAC code ("page allocator classic"), but
- * others will be coming soon.
+ * allocations.  It dispatches each page-level allocation request to either
+ * the PAC ("page allocator classic") or the HPA ("hugepage allocator")
+ * by calling their pac_*() / hpa_*() entry points directly.
  */
 
 typedef struct pa_central_s pa_central_t;
@@ -96,7 +94,7 @@ struct pa_shard_s {
 	/* Allocates from a PAC. */
 	pac_t pac;
 
-	hpa_shard_t hpa_shard;
+	hpa_shard_t hpa;
 
 	/* The source of edata_t objects. */
 	edata_cache_t edata_cache;
@@ -120,7 +118,7 @@ pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
 }
 
 static inline ehooks_t *
-pa_shard_ehooks_get(pa_shard_t *shard) {
+pa_shard_ehooks_get(const pa_shard_t *shard) {
 	return base_ehooks_get(shard->base);
 }
 
@@ -221,12 +219,12 @@ void pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
 
-size_t pa_shard_nactive(pa_shard_t *shard);
-size_t pa_shard_ndirty(pa_shard_t *shard);
-size_t pa_shard_nmuzzy(pa_shard_t *shard);
+size_t pa_shard_nactive(const pa_shard_t *shard);
+size_t pa_shard_ndirty(const pa_shard_t *shard);
+size_t pa_shard_nmuzzy(const pa_shard_t *shard);
 
 void pa_shard_basic_stats_merge(
-    pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
+    const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
 
 void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,

include/jemalloc/internal/pac.h

@@ -4,10 +4,11 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/decay.h"
 #include "jemalloc/internal/ecache.h"
+#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
 #include "jemalloc/internal/lockedint.h"
-#include "jemalloc/internal/pai.h"
+#include "jemalloc/internal/tsd_types.h"
 #include "san_bump.h"
 
 /*
@@ -51,6 +52,8 @@ struct pac_estats_s {
 	size_t muzzy_bytes;
 	size_t nretained;
 	size_t retained_bytes;
+	size_t npinned;
+	size_t pinned_bytes;
 };
 
 typedef struct pac_stats_s pac_stats_t;
@@ -61,9 +64,14 @@ struct pac_stats_s {
 	/*
 	 * Number of unused virtual memory bytes currently retained.  Retained
 	 * bytes are technically mapped (though always decommitted or purged),
-	 * but they are excluded from the mapped statistic (above).
+	 * but they are excluded from pac_mapped.
 	 */
 	size_t retained; /* Derived. */
+	/*
+	 * Number of bytes in pinned (non-reclaimable) extents currently
+	 * cached.  Unlike retained, pinned bytes count toward pac_mapped.
+	 */
+	size_t pinned;   /* Derived. */
 
 	/*
 	 * Number of bytes currently mapped, excluding retained memory (and any
@@ -80,11 +88,8 @@ struct pac_stats_s {
 
 typedef struct pac_s pac_t;
 struct pac_s {
-	/*
-	 * Must be the first member (we convert it to a PAC given only a
-	 * pointer).  The handle to the allocation interface.
-	 */
-	pai_t pai;
+	/* True once pinned memory has been seen. */
+	atomic_b_t has_pinned;
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.
@@ -94,6 +99,7 @@ struct pac_s {
 	ecache_t ecache_dirty;
 	ecache_t ecache_muzzy;
 	ecache_t ecache_retained;
+	ecache_t ecache_pinned;
 
 	base_t        *base;
 	emap_t        *emap;
@@ -155,13 +161,39 @@ bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
     ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
     malloc_mutex_t *stats_mtx);
 
+edata_t *pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment,
+    bool zero, bool guarded, bool frequent_reuse,
+    bool *deferred_work_generated);
+bool pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
+    size_t new_size, bool zero, bool *deferred_work_generated);
+bool pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
+    size_t new_size, bool *deferred_work_generated);
+void pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
+    bool *deferred_work_generated);
+uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac);
+
 static inline size_t
-pac_mapped(pac_t *pac) {
+pac_mapped(const pac_t *pac) {
 	return atomic_load_zu(&pac->stats->pac_mapped, ATOMIC_RELAXED);
 }
 
+void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+    ecache_t *ecache, edata_t *edata);
+
+static inline void
+pac_record_grown(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+    edata_t *edata) {
+	bool pinned = edata_pinned_get(edata);
+	if (pinned && config_stats) {
+		atomic_fetch_add_zu(&pac->stats->pac_mapped,
+		    edata_size_get(edata), ATOMIC_RELAXED);
+	}
+	extent_record(tsdn, pac, ehooks,
+	    pinned ? &pac->ecache_pinned : &pac->ecache_retained, edata);
+}
+
 static inline ehooks_t *
-pac_ehooks_get(pac_t *pac) {
+pac_ehooks_get(const pac_t *pac) {
 	return base_ehooks_get(pac->base);
 }
 

include/jemalloc/internal/pai.h

@@ -1,63 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PAI_H
-#define JEMALLOC_INTERNAL_PAI_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/tsd_types.h"
-
-/* An interface for page allocation. */
-
-typedef struct pai_s pai_t;
-struct pai_s {
-	/* Returns NULL on failure. */
-	edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
-	    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
-	    bool *deferred_work_generated);
-	bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-	    size_t old_size, size_t new_size, bool zero,
-	    bool *deferred_work_generated);
-	bool (*shrink)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-	    size_t old_size, size_t new_size, bool *deferred_work_generated);
-	void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-	    bool *deferred_work_generated);
-	uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self);
-};
-
-/*
- * These are just simple convenience functions to avoid having to reference the
- * same pai_t twice on every invocation.
- */
-
-static inline edata_t *
-pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
-    bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
-	return self->alloc(tsdn, self, size, alignment, zero, guarded,
-	    frequent_reuse, deferred_work_generated);
-}
-
-static inline bool
-pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size, bool zero, bool *deferred_work_generated) {
-	return self->expand(tsdn, self, edata, old_size, new_size, zero,
-	    deferred_work_generated);
-}
-
-static inline bool
-pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size, bool *deferred_work_generated) {
-	return self->shrink(
-	    tsdn, self, edata, old_size, new_size, deferred_work_generated);
-}
-
-static inline void
-pai_dalloc(
-    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
-	self->dalloc(tsdn, self, edata, deferred_work_generated);
-}
-
-static inline uint64_t
-pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
-	return self->time_until_deferred_work(tsdn, self);
-}
-
-#endif /* JEMALLOC_INTERNAL_PAI_H */

include/jemalloc/internal/prof_externs.h

@@ -114,50 +114,17 @@ uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
  *
  * Currently only the profiling sampling event needs the lookahead
  * functionality, so we don't yet define general purpose lookahead functions.
- *
- * Surplus is a terminology referring to the amount of bytes beyond what's
- * needed for triggering an event, which can be a useful quantity to have in
- * general when lookahead is being called.
- *
- * This function returns true if allocation of usize would go above the next
- * trigger for prof event, and false otherwise.
- * If function returns true surplus will contain number of bytes beyond that
- * trigger.
  */
 
 JEMALLOC_ALWAYS_INLINE bool
-te_prof_sample_event_lookahead_surplus(
-    tsd_t *tsd, size_t usize, size_t *surplus) {
-	if (surplus != NULL) {
-		/*
-		 * This is a dead store: the surplus will be overwritten before
-		 * any read.  The initialization suppresses compiler warnings.
-		 * Meanwhile, using SIZE_MAX to initialize is good for
-		 * debugging purpose, because a valid surplus value is strictly
-		 * less than usize, which is at most SIZE_MAX.
-		 */
-		*surplus = SIZE_MAX;
-	}
+te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
 	if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) {
 		return false;
 	}
 	/* The subtraction is intentionally susceptible to underflow. */
 	uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize
 	    - tsd_thread_allocated_last_event_get(tsd);
-	uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd);
-	if (accumbytes < sample_wait) {
-		return false;
-	}
-	assert(accumbytes - sample_wait < (uint64_t)usize);
-	if (surplus != NULL) {
-		*surplus = (size_t)(accumbytes - sample_wait);
-	}
-	return true;
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
-	return te_prof_sample_event_lookahead_surplus(tsd, usize, NULL);
+	return accumbytes >= tsd_prof_sample_event_wait_get(tsd);
 }
 
 extern te_base_cb_t prof_sample_te_handler;

include/jemalloc/internal/psset.h

@@ -134,17 +134,17 @@ void psset_insert(psset_t *psset, hpdata_t *ps);
 void psset_remove(psset_t *psset, hpdata_t *ps);
 
 static inline size_t
-psset_npageslabs(psset_t *psset) {
+psset_npageslabs(const psset_t *psset) {
 	return psset->stats.merged.npageslabs;
 }
 
 static inline size_t
-psset_nactive(psset_t *psset) {
+psset_nactive(const psset_t *psset) {
 	return psset->stats.merged.nactive;
 }
 
 static inline size_t
-psset_ndirty(psset_t *psset) {
+psset_ndirty(const psset_t *psset) {
 	return psset->stats.merged.ndirty;
 }
 

include/jemalloc/internal/sc.h

@@ -268,12 +268,17 @@
 #define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP))
 #define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP)
 
-/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */
-#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2))
-#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP))
-
-/* The largest size class supported. */
-#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA)
+/*
+ * The largest size class supported.  Spell this out directly to avoid
+ * expanding subtractive arithmetic at every use site.
+ */
+#if LG_SIZEOF_PTR == 3
+#	define SC_LARGE_MAXCLASS 0x7000000000000000ULL
+#elif LG_SIZEOF_PTR == 2
+#	define SC_LARGE_MAXCLASS 0x70000000ULL
+#else
+#	error "Unsupported pointer size"
+#endif
 
 /* Maximum number of regions in one slab. */
 #ifndef CONFIG_LG_SLAB_MAXREGS

include/jemalloc/internal/sec.h

@@ -4,9 +4,10 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/sec_opts.h"
+#include "jemalloc/internal/tsd_types.h"
 
 /*
  * Small extent cache.
@@ -96,6 +97,25 @@ sec_size_supported(sec_t *sec, size_t size) {
 	return sec_is_used(sec) && size <= sec->opts.max_alloc;
 }
 
+/* Min number of extents we will allocate when expanding the SEC. */
+#define SEC_MIN_NALLOCS 2
+
+/* Max number of extents we will allocate out of a single huge page. */
+#define SEC_MAX_NALLOCS 8
+
+/* Attempt to fill the SEC up to max_bytes / SEC_MAX_BYTES_DIV */
+#define SEC_MAX_BYTES_DIV 4
+
+/*
+ * Calculate the min and max number of extents we will try to allocate
+ * when expanding the SEC. We will attempt to allocate at least min
+ * extents and up to max extents depending on whether we can allocate
+ * them out of a huge page we have already allocated out of. Both
+ * min and max should the in the range [1, SEC_MAX_NALLOCS].
+ */
+void sec_calc_nallocs_for_size(
+    sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
+
 /* If sec does not have extent available, it will return NULL. */
 edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
 void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,

include/jemalloc/internal/sec_opts.h

@@ -27,16 +27,9 @@ struct sec_opts_s {
 	 * until we are 1/4 below max_bytes.
 	 */
 	size_t max_bytes;
-	/*
-	 * When we can't satisfy an allocation out of the SEC because there are
-	 * no available ones cached, allocator will allocate a batch with extra
-	 * batch_fill_extra extents of the same size.
-	 */
-	size_t batch_fill_extra;
 };
 
 #define SEC_OPTS_NSHARDS_DEFAULT 2
-#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3
 #define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024))
 #define SEC_OPTS_MAX_BYTES_DEFAULT                                             \
 	((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT)                       \
@@ -45,6 +38,6 @@ struct sec_opts_s {
 
 #define SEC_OPTS_DEFAULT                                                       \
 	{SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT,                 \
-	    SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT}
+	    SEC_OPTS_MAX_BYTES_DEFAULT}
 
 #endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

include/jemalloc/internal/seq.h

@@ -1,58 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_SEQ_H
-#define JEMALLOC_INTERNAL_SEQ_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/atomic.h"
-
-/*
- * A simple seqlock implementation.
- */
-
-/* clang-format off */
-#define seq_define(type, short_type)					\
-typedef struct {							\
-	atomic_zu_t seq;						\
-	atomic_zu_t data[						\
-	    (sizeof(type) + sizeof(size_t) - 1) / sizeof(size_t)];	\
-} seq_##short_type##_t;							\
-									\
-/*									\
- * No internal synchronization -- the caller must ensure that there's	\
- * only a single writer at a time.					\
- */									\
-static inline void							\
-seq_store_##short_type(seq_##short_type##_t *dst, type *src) {		\
-	size_t buf[sizeof(dst->data) / sizeof(size_t)];			\
-	buf[sizeof(buf) / sizeof(size_t) - 1] = 0;			\
-	memcpy(buf, src, sizeof(type));					\
-	size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED);	\
-	atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED);	\
-	atomic_fence(ATOMIC_RELEASE);					\
-	for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) {	\
-		atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED);	\
-	}								\
-	atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE);	\
-}									\
-									\
-/* Returns whether or not the read was consistent. */			\
-static inline bool							\
-seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) {	\
-	size_t buf[sizeof(src->data) / sizeof(size_t)];			\
-	size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE);	\
-	if (seq1 % 2 != 0) {						\
-		return false;						\
-	}								\
-	for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) {	\
-		buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED);	\
-	}								\
-	atomic_fence(ATOMIC_ACQUIRE);					\
-	size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED);	\
-	if (seq1 != seq2) {						\
-		return false;						\
-	}								\
-	memcpy(dst, buf, sizeof(type));					\
-	return true;							\
-}
-/* clang-format on */
-
-#endif /* JEMALLOC_INTERNAL_SEQ_H */

include/jemalloc/internal/sz.h

@@ -314,12 +314,20 @@ sz_size2index_usize_fastpath(size_t size, szind_t *ind, size_t *usize) {
 
 JEMALLOC_ALWAYS_INLINE size_t
 sz_s2u_compute_using_delta(size_t size) {
+	if (unlikely(size > SC_LARGE_MAXCLASS)) {
+		return 0;
+	}
+
 	size_t x = lg_floor((size << 1) - 1);
 	size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1)
 	    ? LG_QUANTUM
 	    : x - SC_LG_NGROUP - 1;
 	size_t delta = ZU(1) << lg_delta;
 	size_t delta_mask = delta - 1;
+	if (unlikely(size > SIZE_T_MAX - delta_mask)) {
+		return 0;
+	}
+
 	size_t usize = (size + delta_mask) & ~delta_mask;
 	return usize;
 }
@@ -385,7 +393,11 @@ JEMALLOC_ALWAYS_INLINE size_t
 sz_sa2u(size_t size, size_t alignment) {
 	size_t usize;
 
-	assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
+	if (unlikely(alignment == 0)) {
+		return 0;
+	}
+	size_t alignment_mask = alignment - 1;
+	assert((alignment_mask & alignment) == 0);
 
 	/* Try for a small size class. */
 	if (size <= SC_SMALL_MAXCLASS && alignment <= PAGE) {
@@ -403,7 +415,10 @@ sz_sa2u(size_t size, size_t alignment) {
 		 *    144 | 10100000 |  32
 		 *    192 | 11000000 |  64
 		 */
-		usize = sz_s2u(ALIGNMENT_CEILING(size, alignment));
+		if (unlikely(size > SIZE_T_MAX - alignment_mask)) {
+			return 0;
+		}
+		usize = sz_s2u((size + alignment_mask) & ~alignment_mask);
 		if (usize < SC_LARGE_MINCLASS) {
 			return usize;
 		}

include/jemalloc/internal/tcache_externs.h

@@ -62,17 +62,18 @@ bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
 bool tcache_bin_ncached_max_read(
     tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
 void tcache_arena_reassociate(
-    tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
+    tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
 tcache_t *tcache_create_explicit(tsd_t *tsd);
 bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
 void      tcache_cleanup(tsd_t *tsd);
-void      tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
 bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
 void      tcaches_flush(tsd_t *tsd, unsigned ind);
 void      tcaches_destroy(tsd_t *tsd, unsigned ind);
 bool      tcache_boot(tsdn_t *tsdn, base_t *base);
 void      tcache_arena_associate(
-         tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
+         tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
+cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor(
+         tsdn_t *tsdn, arena_t *arena);
 void tcache_prefork(tsdn_t *tsdn);
 void tcache_postfork_parent(tsdn_t *tsdn);
 void tcache_postfork_child(tsdn_t *tsdn);

include/jemalloc/internal/tcache_inlines.h

@@ -19,7 +19,7 @@ tcache_enabled_get(tsd_t *tsd) {
 }
 
 static inline unsigned
-tcache_nbins_get(tcache_slow_t *tcache_slow) {
+tcache_nbins_get(const tcache_slow_t *tcache_slow) {
 	assert(tcache_slow != NULL);
 	unsigned nbins = tcache_slow->tcache_nbins;
 	assert(nbins <= TCACHE_NBINS_MAX);
@@ -27,7 +27,7 @@ tcache_nbins_get(tcache_slow_t *tcache_slow) {
 }
 
 static inline size_t
-tcache_max_get(tcache_slow_t *tcache_slow) {
+tcache_max_get(const tcache_slow_t *tcache_slow) {
 	assert(tcache_slow != NULL);
 	size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1);
 	assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
@@ -86,6 +86,13 @@ tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_slow_t *tcache_slow) {
 	return disabled;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tcache_can_cache_large(tcache_t *tcache, szind_t ind) {
+	return ind < tcache_nbins_get(tcache->tcache_slow)
+	    && !tcache_bin_disabled(ind, &tcache->bins[ind],
+	        tcache->tcache_slow);
+}
+
 JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
     szind_t binind, bool zero, bool slow_path) {

include/jemalloc/internal/tcache_structs.h

@@ -19,9 +19,6 @@
  */
 
 struct tcache_slow_s {
-	/* Lets us track all the tcaches in an arena. */
-	ql_elm(tcache_slow_t) link;
-
 	/*
 	 * The descriptor lets the arena find our cache bins without seeing the
 	 * tcache definition.  This enables arenas to aggregate stats across

include/jemalloc/internal/tsd.h

@@ -37,13 +37,12 @@ TSD_DATA_SLOWER
 JEMALLOC_ALWAYS_INLINE t *						\
 tsd_##n##p_get(tsd_t *tsd) {						\
 	/*								\
-	 * Because the state might change asynchronously if it's	\
-	 * nominal, we need to make sure that we only read it once.	\
+	 * Read the state once, so that a transition between		\
+	 * nominal states does not confuse this assertion.		\
 	 */								\
 	uint8_t state = tsd_state_get(tsd);				\
 	assert(state == tsd_state_nominal ||				\
 	    state == tsd_state_nominal_slow ||				\
-	    state == tsd_state_nominal_recompute ||			\
 	    state == tsd_state_reincarnated ||				\
 	    state == tsd_state_minimal_initialized);			\
 	return tsd_##n##p_get_unsafe(tsd);				\
@@ -95,11 +94,6 @@ TSD_DATA_SLOWER
 
 JEMALLOC_ALWAYS_INLINE void
 tsd_assert_fast(tsd_t *tsd) {
-	/*
-	 * Note that our fastness assertion does *not* include global slowness
-	 * counters; it's not in general possible to ensure that they won't
-	 * change asynchronously from underneath us.
-	 */
 	assert(!malloc_slow && tsd_tcache_enabled_get(tsd)
 	    && tsd_reentrancy_level_get(tsd) == 0);
 }

include/jemalloc/internal/tsd_internals.h

@@ -10,7 +10,6 @@
 #include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/peak.h"
 #include "jemalloc/internal/prof_types.h"
-#include "jemalloc/internal/ql.h"
 #include "jemalloc/internal/rtree_tsd.h"
 #include "jemalloc/internal/tcache_structs.h"
 #include "jemalloc/internal/tcache_types.h"
@@ -57,8 +56,6 @@ typedef void (*test_callback_t)(int *);
 #	define MALLOC_TEST_TSD_INITIALIZER
 #endif
 
-typedef ql_elm(tsd_t) tsd_link_t;
-
 /*  O(name,			type,			nullable type) */
 #define TSD_DATA_SLOW                                                          \
 	O(tcache_enabled, bool, bool)                                          \
@@ -80,8 +77,6 @@ typedef ql_elm(tsd_t) tsd_link_t;
 	O(arena_decay_ticker, ticker_geom_t, ticker_geom_t)                    \
 	O(sec_shard, uint8_t, uint8_t)                                         \
 	O(binshards, tsd_binshards_t, tsd_binshards_t)                         \
-	O(tsd_link, tsd_link_t, tsd_link_t)                                    \
-	O(in_hook, bool, bool)                                                 \
 	O(peak, peak_t, peak_t)                                                \
 	O(tcache_slow, tcache_slow_t, tcache_slow_t)                           \
 	O(rtree_ctx, rtree_ctx_t, rtree_ctx_t)
@@ -101,9 +96,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
 	    TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),                   \
 	    /* sec_shard */ (uint8_t) - 1,                                     \
 	    /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER,                    \
-	    /* tsd_link */ {NULL}, /* in_hook */ false,                        \
-	    /* peak */ PEAK_INITIALIZER,                                       \
-	    /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER,                    \
+	    /* peak */ PEAK_INITIALIZER, /* tcache_slow */                     \
+	    TCACHE_SLOW_ZERO_INITIALIZER,                                      \
 	    /* rtree_ctx */ RTREE_CTX_INITIALIZER,
 
 /*  O(name,			type,			nullable type) */
@@ -132,7 +126,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
 #define TSD_INITIALIZER                                                        \
 	{                                                                      \
 		TSD_DATA_SLOW_INITIALIZER                                      \
-		/* state */ ATOMIC_INIT(tsd_state_uninitialized),              \
+		/* state */ tsd_state_uninitialized,                           \
 		    TSD_DATA_FAST_INITIALIZER TSD_DATA_SLOWER_INITIALIZER      \
 	}
 
@@ -148,17 +142,6 @@ void   tsd_cleanup(void *arg);
 tsd_t *tsd_fetch_slow(tsd_t *tsd, bool minimal);
 void   tsd_state_set(tsd_t *tsd, uint8_t new_state);
 void   tsd_slow_update(tsd_t *tsd);
-void   tsd_prefork(tsd_t *tsd);
-void   tsd_postfork_parent(tsd_t *tsd);
-void   tsd_postfork_child(tsd_t *tsd);
-
-/*
- * Call ..._inc when your module wants to take all threads down the slow paths,
- * and ..._dec when it no longer needs to.
- */
-void tsd_global_slow_inc(tsdn_t *tsdn);
-void tsd_global_slow_dec(tsdn_t *tsdn);
-bool tsd_global_slow(void);
 
 #define TSD_MIN_INIT_STATE_MAX_FETCHED (128)
 
@@ -167,39 +150,29 @@ enum {
 	tsd_state_nominal = 0,
 	/* Initialized but on slow path. */
 	tsd_state_nominal_slow = 1,
-	/*
-	 * Some thread has changed global state in such a way that all nominal
-	 * threads need to recompute their fast / slow status the next time they
-	 * get a chance.
-	 *
-	 * Any thread can change another thread's status *to* recompute, but
-	 * threads are the only ones who can change their status *from*
-	 * recompute.
-	 */
-	tsd_state_nominal_recompute = 2,
 	/*
 	 * The above nominal states should be lower values.  We use
 	 * tsd_nominal_max to separate nominal states from threads in the
 	 * process of being born / dying.
 	 */
-	tsd_state_nominal_max = 2,
+	tsd_state_nominal_max = 1,
 
 	/*
 	 * A thread might free() during its death as its only allocator action;
 	 * in such scenarios, we need tsd, but set up in such a way that no
 	 * cleanup is necessary.
 	 */
-	tsd_state_minimal_initialized = 3,
+	tsd_state_minimal_initialized = 2,
 	/* States during which we know we're in thread death. */
-	tsd_state_purgatory = 4,
-	tsd_state_reincarnated = 5,
+	tsd_state_purgatory = 3,
+	tsd_state_reincarnated = 4,
 	/*
 	 * What it says on the tin; tsd that hasn't been initialized.  Note
 	 * that even when the tsd struct lives in TLS, when need to keep track
 	 * of stuff like whether or not our pthread destructors have been
 	 * scheduled, so this really truly is different than the nominal state.
 	 */
-	tsd_state_uninitialized = 6
+	tsd_state_uninitialized = 5
 };
 
 /*
@@ -209,18 +182,6 @@ enum {
  */
 #define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n
 
-#ifdef JEMALLOC_U8_ATOMICS
-#	define tsd_state_t atomic_u8_t
-#	define tsd_atomic_load atomic_load_u8
-#	define tsd_atomic_store atomic_store_u8
-#	define tsd_atomic_exchange atomic_exchange_u8
-#else
-#	define tsd_state_t atomic_u32_t
-#	define tsd_atomic_load atomic_load_u32
-#	define tsd_atomic_store atomic_store_u32
-#	define tsd_atomic_exchange atomic_exchange_u32
-#endif
-
 /* The actual tsd. */
 struct tsd_s {
 	/*
@@ -232,11 +193,8 @@ struct tsd_s {
 #define O(n, t, nt) t TSD_MANGLE(n);
 
 	TSD_DATA_SLOW
-	/*
-	 * We manually limit the state to just a single byte.  Unless the 8-bit
-	 * atomics are unavailable (which is rare).
-	 */
-	tsd_state_t state;
+	/* Encodes one of tsd_state_*; mutated only by the owning thread. */
+	uint8_t state;
 	TSD_DATA_FAST
 	TSD_DATA_SLOWER
 #undef O
@@ -244,13 +202,7 @@ struct tsd_s {
 
 JEMALLOC_ALWAYS_INLINE uint8_t
 tsd_state_get(tsd_t *tsd) {
-	/*
-	 * This should be atomic.  Unfortunately, compilers right now can't tell
-	 * that this can be done as a memory comparison, and forces a load into
-	 * a register that hurts fast-path performance.
-	 */
-	/* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */
-	return *(uint8_t *)&tsd->state;
+	return tsd->state;
 }
 
 /*

include/jemalloc/internal/util.h

@@ -6,6 +6,18 @@
 
 #define UTIL_INLINE static inline
 
+JEMALLOC_ALWAYS_INLINE
+size_t
+min_zu(size_t a, size_t b) {
+	return (a < b) ? a : b;
+}
+
+JEMALLOC_ALWAYS_INLINE
+size_t
+max_zu(size_t a, size_t b) {
+	return (a > b) ? a : b;
+}
+
 /* Junk fill patterns. */
 #ifndef JEMALLOC_ALLOC_JUNK
 #	define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)

include/jemalloc/internal/witness.h

@@ -44,7 +44,7 @@ enum witness_rank_e {
 	 */
 	WITNESS_RANK_CORE,
 	WITNESS_RANK_DECAY = WITNESS_RANK_CORE,
-	WITNESS_RANK_TCACHE_QL,
+	WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
 
 	WITNESS_RANK_SEC_BIN,
 

include/jemalloc/jemalloc_typedefs.h.in

@@ -4,6 +4,36 @@ extern "C" {
 
 typedef struct extent_hooks_s extent_hooks_t;
 
+/*
+ * Extent alloc flags.  A custom extent_alloc hook may OR these into the
+ * returned pointer; jemalloc strips the low bits before use.  Safe because
+ * returned addresses are at least page-aligned (PAGE >= 256).
+ *
+ * EXTENT_ALLOC_FLAG_PINNED: backing memory is non-reclaimable.
+ * Pinned extents are excluded from decay/purging and cached separately for
+ * preferential reuse.  A hook returning this flag must also set *commit to
+ * true: pinned memory bypasses jemalloc's commit/decommit machinery.
+ *
+ * The pinned attribute is per-extent: a single hook may return pinned and
+ * non-pinned extents in different calls.  Pinned and non-pinned extents are
+ * never merged together (the merge would change the reclamation policy of
+ * one half), so pinned-ness is set at allocation and inherited through
+ * splits, but never changes after that.
+ *
+ * Example (HugeTLB alloc hook):
+ *   void *my_alloc(extent_hooks_t *h, void *new_addr, size_t size,
+ *       size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
+ *       void *addr = mmap(NULL, size, PROT_READ|PROT_WRITE,
+ *           MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0);
+ *       if (addr == MAP_FAILED) return NULL;
+ *       *zero = true;
+ *       *commit = true;
+ *       return (void *)((uintptr_t)addr | EXTENT_ALLOC_FLAG_PINNED);
+ *   }
+ */
+#define EXTENT_ALLOC_FLAG_PINNED    0x1U
+#define EXTENT_ALLOC_FLAG_MASK      0xFFU
+
 /*
  * void *
  * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -36,6 +36,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
+    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -44,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
+    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -67,6 +69,8 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -31,6 +31,9 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\conf.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -202,5 +205,14 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\arenas_management.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -36,6 +36,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
+    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -44,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
+    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -67,6 +69,8 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -31,6 +31,9 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\conf.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -202,5 +205,14 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\arenas_management.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj

@@ -36,6 +36,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
+    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -44,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
+    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -67,6 +69,8 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters

@@ -31,6 +31,9 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\conf.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -202,5 +205,14 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\arenas_management.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj

@@ -36,6 +36,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
+    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -44,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
+    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -67,6 +69,8 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters

@@ -31,6 +31,9 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\conf.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -202,5 +205,14 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\arenas_management.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

scripts/gen_gh_actions.py

@@ -287,7 +287,7 @@ def generate_linux_job(arch):
 
     job += f"""
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |
@@ -302,7 +302,7 @@ def generate_linux_job(arch):
         cat /etc/os-release || true
         echo ""
         echo "=== CPU Info ==="
-        lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\(s\):" || true
+        lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\\(s\\):" || true
 
     - name: Install dependencies (32-bit)
       if: matrix.env.CROSS_COMPILE_32BIT == 'yes'
@@ -387,7 +387,7 @@ def generate_macos_job(arch):
 
     job += f"""
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |
@@ -469,7 +469,7 @@ def generate_windows_job(arch):
 
     job += f"""
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       shell: cmd
@@ -590,7 +590,7 @@ def generate_freebsd_job(arch):
     name: FreeBSD (${{{{ matrix.arch }}}}, debug=${{{{ matrix.debug }}}}, prof=${{{{ matrix.prof }}}}${{{{ matrix.uncommon && ', uncommon' || '' }}}})
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 1
 

src/arena.c

@@ -18,7 +18,7 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
 /* Data. */
 
 /*
- * Define names for both unininitialized and initialized phases, so that
+ * Define names for both uninitialized and initialized phases, so that
  * options and mallctl processing are straightforward.
  */
 const char *const percpu_arena_mode_names[] = {
@@ -44,10 +44,10 @@ static unsigned nbins_total;
 
 /*
  * a0 is used to handle huge requests before malloc init completes. After
- * that,the huge_arena_ind is updated to point to the actual huge arena,
+ * that, the huge_arena_ind is updated to point to the actual huge arena,
  * which is the last one of the auto arenas.
  */
-unsigned  huge_arena_ind = 0;
+static unsigned huge_arena_ind = 0;
 bool      opt_huge_arena_pac_thp = false;
 pac_thp_t huge_arena_pac_thp = {.thp_madvise = false,
     .auto_thp_switched = false,
@@ -164,7 +164,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	/* Currently cached bytes and sanitizer-stashed bytes in tcache. */
 	astats->tcache_bytes = 0;
 	astats->tcache_stashed_bytes = 0;
-	malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
+	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 	cache_bin_array_descriptor_t *descriptor;
 	ql_foreach (descriptor, &arena->cache_bin_array_descriptor_ql, link) {
 		for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
@@ -183,8 +183,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	}
 	malloc_mutex_prof_read(tsdn,
 	    &astats->mutex_prof_data[arena_prof_mutex_tcache_list],
-	    &arena->tcache_ql_mtx);
-	malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
+	    &arena->cache_bin_array_descriptor_ql_mtx);
+	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 
 #define READ_ARENA_MUTEX_PROF_DATA(mtx, ind)                                   \
 	malloc_mutex_lock(tsdn, &arena->mtx);                                  \
@@ -211,6 +211,92 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	}
 }
 
+void *
+arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind) {
+	assert(szind < SC_NBINS);
+	bin_t *bin = bin_choose(tsdn, arena, szind, NULL);
+	assert(bin != NULL);
+	return bin_current_slab_addr(tsdn, bin);
+}
+
+void
+arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
+    cache_bin_array_descriptor_t *desc) {
+	cassert(config_stats);
+	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+	ql_tail_insert(&arena->cache_bin_array_descriptor_ql, desc, link);
+	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+}
+
+static void
+arena_cache_bin_stats_flush(tsdn_t *tsdn, arena_t *arena,
+    cache_bin_array_descriptor_t *desc) {
+	cassert(config_stats);
+	for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
+		cache_bin_t *cache_bin = &desc->bins[i];
+		if (cache_bin_disabled(cache_bin)) {
+			continue;
+		}
+		if (i < SC_NBINS) {
+			bin_t *bin = bin_choose(tsdn, arena, i, NULL);
+			bin_stats_nrequests_add(tsdn, bin,
+			    cache_bin->tstats.nrequests);
+		} else {
+			arena_stats_large_flush_nrequests_add(tsdn,
+			    &arena->stats, i, cache_bin->tstats.nrequests);
+		}
+		cache_bin->tstats.nrequests = 0;
+	}
+}
+
+void
+arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
+    cache_bin_array_descriptor_t *desc) {
+	cassert(config_stats);
+	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+	if (config_debug) {
+		bool                          in_ql = false;
+		cache_bin_array_descriptor_t *iter;
+		ql_foreach (iter, &arena->cache_bin_array_descriptor_ql, link) {
+			if (iter == desc) {
+				in_ql = true;
+				break;
+			}
+		}
+		assert(in_ql);
+	}
+	ql_remove(&arena->cache_bin_array_descriptor_ql, desc, link);
+	arena_cache_bin_stats_flush(tsdn, arena, desc);
+	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+}
+
+/*
+ * Postfork-child entry: child is single-threaded, so the queue is rebuilt
+ * from scratch (descriptors held by other threads at fork time are gone)
+ * without locking.
+ */
+void
+arena_cache_bin_array_postfork_child(arena_t *arena,
+    cache_bin_array_descriptor_t *desc_or_null) {
+	cassert(config_stats);
+	ql_new(&arena->cache_bin_array_descriptor_ql);
+	if (desc_or_null != NULL) {
+		ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
+		    desc_or_null, link);
+	}
+}
+
+void
+arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
+	cassert(config_stats);
+	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+	cache_bin_array_descriptor_t *desc;
+	ql_foreach (desc, &arena->cache_bin_array_descriptor_ql, link) {
+		arena_cache_bin_stats_flush(tsdn, arena, desc);
+	}
+	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+}
+
 static void
 arena_background_thread_inactivity_check(
     tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
@@ -343,7 +429,8 @@ arena_extent_alloc_large(
 }
 
 void
-arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
+arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
+    const edata_t *edata) {
 	if (config_stats) {
 		arena_large_dalloc_stats_update(
 		    tsdn, arena, edata_usize_get(edata));
@@ -352,7 +439,7 @@ arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
 
 void
 arena_extent_ralloc_large_shrink(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) {
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize) {
 	size_t usize = edata_usize_get(edata);
 
 	if (config_stats) {
@@ -362,7 +449,7 @@ arena_extent_ralloc_large_shrink(
 
 void
 arena_extent_ralloc_large_expand(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) {
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize) {
 	size_t usize = edata_usize_get(edata);
 
 	if (config_stats) {
@@ -643,9 +730,7 @@ arena_dalloc_promoted_impl(
 	}
 	szind_t bumped_ind = sz_size2index(bumped_usize);
 	if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
-	    && bumped_ind < TCACHE_NBINS_MAX
-	    && !tcache_bin_disabled(
-	        bumped_ind, &tcache->bins[bumped_ind], tcache->tcache_slow)) {
+	    && tcache_can_cache_large(tcache, bumped_ind)) {
 		tcache_dalloc_large(
 		    tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
 	} else {
@@ -796,6 +881,8 @@ arena_prepare_base_deletion(tsd_t *tsd, base_t *base_to_destroy) {
 		    tsd, &pac->ecache_muzzy.mtx, delayed_mtx, &n_delayed);
 		arena_prepare_base_deletion_sync(
 		    tsd, &pac->ecache_retained.mtx, delayed_mtx, &n_delayed);
+		arena_prepare_base_deletion_sync(
+		    tsd, &pac->ecache_pinned.mtx, delayed_mtx, &n_delayed);
 	}
 	arena_prepare_base_deletion_sync_finish(tsd, delayed_mtx, n_delayed);
 }
@@ -997,76 +1084,6 @@ label_refill:
 	return filled;
 }
 
-size_t
-arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
-    void **ptrs, size_t nfill, bool zero) {
-	assert(binind < SC_NBINS);
-	const bin_info_t *bin_info = &bin_infos[binind];
-	const size_t      nregs = bin_info->nregs;
-	assert(nregs > 0);
-	const size_t usize = bin_info->reg_size;
-
-	const bool manual_arena = !arena_is_auto(arena);
-	unsigned   binshard;
-	bin_t     *bin = bin_choose(tsdn, arena, binind, &binshard);
-
-	size_t              nslab = 0;
-	size_t              filled = 0;
-	edata_t            *slab = NULL;
-	edata_list_active_t fulls;
-	edata_list_active_init(&fulls);
-
-	while (filled < nfill
-	    && (slab = arena_slab_alloc(
-	            tsdn, arena, binind, binshard, bin_info))
-	        != NULL) {
-		assert((size_t)edata_nfree_get(slab) == nregs);
-		++nslab;
-		size_t batch = nfill - filled;
-		if (batch > nregs) {
-			batch = nregs;
-		}
-		assert(batch > 0);
-		bin_slab_reg_alloc_batch(
-		    slab, bin_info, (unsigned)batch, &ptrs[filled]);
-		assert(edata_addr_get(slab) == ptrs[filled]);
-		if (zero) {
-			memset(ptrs[filled], 0, batch * usize);
-		}
-		filled += batch;
-		if (batch == nregs) {
-			if (manual_arena) {
-				edata_list_active_append(&fulls, slab);
-			}
-			slab = NULL;
-		}
-	}
-
-	malloc_mutex_lock(tsdn, &bin->lock);
-	/*
-	 * Only the last slab can be non-empty, and the last slab is non-empty
-	 * iff slab != NULL.
-	 */
-	if (slab != NULL) {
-		bin_lower_slab(tsdn, !manual_arena, slab, bin);
-	}
-	if (manual_arena) {
-		edata_list_active_concat(&bin->slabs_full, &fulls);
-	}
-	assert(edata_list_active_empty(&fulls));
-	if (config_stats) {
-		bin->stats.nslabs += nslab;
-		bin->stats.curslabs += nslab;
-		bin->stats.nmalloc += filled;
-		bin->stats.nrequests += filled;
-		bin->stats.curregs += filled;
-	}
-	malloc_mutex_unlock(tsdn, &bin->lock);
-
-	arena_decay_tick(tsdn, arena);
-	return filled;
-}
-
 static void *
 arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {
 	assert(binind < SC_NBINS);
@@ -1607,8 +1624,7 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
 
 void *
 arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
-    hook_ralloc_args_t *hook_args) {
+    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
 	size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
 	if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
 		return NULL;
@@ -1620,18 +1636,13 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
 		UNUSED size_t newsize;
 		if (!arena_ralloc_no_move(
 		        tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
-			hook_invoke_expand(hook_args->is_realloc
-			        ? hook_expand_realloc
-			        : hook_expand_rallocx,
-			    ptr, oldsize, usize, (uintptr_t)ptr,
-			    hook_args->args);
 			return ptr;
 		}
 	}
 
 	if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
 		return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
-		    tcache, hook_args);
+		    tcache);
 	}
 
 	/*
@@ -1644,13 +1655,6 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
 		return NULL;
 	}
 
-	hook_invoke_alloc(
-	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx,
-	    ret, (uintptr_t)ret, hook_args->args);
-	hook_invoke_dalloc(
-	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
-	    ptr, hook_args->args);
-
 	/*
 	 * Junk/zero-filling were already done by
 	 * ipalloc()/arena_malloc().
@@ -1662,7 +1666,7 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
 }
 
 ehooks_t *
-arena_get_ehooks(arena_t *arena) {
+arena_get_ehooks(const arena_t *arena) {
 	return base_ehooks_get(arena->base);
 }
 
@@ -1685,7 +1689,7 @@ arena_set_extent_hooks(
 }
 
 dss_prec_t
-arena_dss_prec_get(arena_t *arena) {
+arena_dss_prec_get(const arena_t *arena) {
 	return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE);
 }
 
@@ -1699,8 +1703,9 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) {
 }
 
 void
-arena_name_get(arena_t *arena, char *name) {
-	char *end = (char *)memchr((void *)arena->name, '\0', ARENA_NAME_LEN);
+arena_name_get(const arena_t *arena, char *name) {
+	const char *end = (const char *)memchr(
+	    arena->name, '\0', ARENA_NAME_LEN);
 	assert(end != NULL);
 	size_t len = (uintptr_t)end - (uintptr_t)arena->name + 1;
 	assert(len > 0 && len <= ARENA_NAME_LEN);
@@ -1751,7 +1756,7 @@ arena_retain_grow_limit_get_set(
 }
 
 unsigned
-arena_nthreads_get(arena_t *arena, bool internal) {
+arena_nthreads_get(const arena_t *arena, bool internal) {
 	return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED);
 }
 
@@ -1799,10 +1804,11 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
 			goto label_error;
 		}
 
-		ql_new(&arena->tcache_ql);
 		ql_new(&arena->cache_bin_array_descriptor_ql);
-		if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql",
-		        WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) {
+		if (malloc_mutex_init(&arena->cache_bin_array_descriptor_ql_mtx,
+		        "cache_bin_array_descriptor_ql",
+		        WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
+		        malloc_mutex_rank_exclusive)) {
 			goto label_error;
 		}
 	}
@@ -1954,9 +1960,6 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
 		huge_arena_ind = narenas_total_get();
 		assert(huge_arena_ind != 0);
 		oversize_threshold = opt_oversize_threshold;
-		/* a0 init happened before malloc_conf_init. */
-		atomic_store_zu(&a0->pa_shard.pac.oversize_threshold,
-		    oversize_threshold, ATOMIC_RELAXED);
 		/* Initialize huge_arena_pac_thp fields. */
 		base_t *b0 = a0->base;
 		/* Make sure that b0 thp auto-switch won't happen concurrently here. */
@@ -1974,9 +1977,18 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
 		huge_enabled = true;
 	}
 
+	/* a0 init happened before malloc_conf_init. */
+	atomic_store_zu(&a0->pa_shard.pac.oversize_threshold,
+	    oversize_threshold, ATOMIC_RELAXED);
+
 	return huge_enabled;
 }
 
+bool
+arena_ind_is_huge(unsigned ind) {
+	return huge_arena_ind != 0 && ind == huge_arena_ind;
+}
+
 bool
 arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) {
 	arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
@@ -2006,7 +2018,7 @@ arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
 void
 arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
 	if (config_stats) {
-		malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx);
+		malloc_mutex_prefork(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 	}
 }
 
@@ -2059,12 +2071,13 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
 	base_postfork_parent(tsdn, arena->base);
 	pa_shard_postfork_parent(tsdn, &arena->pa_shard);
 	if (config_stats) {
-		malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
+		malloc_mutex_postfork_parent(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 	}
 }
 
 void
-arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
+arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
+    cache_bin_array_descriptor_t *surviving_desc) {
 	atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
 	atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
 	if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) {
@@ -2074,19 +2087,9 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
 		arena_nthreads_inc(arena, true);
 	}
 	if (config_stats) {
-		ql_new(&arena->tcache_ql);
-		ql_new(&arena->cache_bin_array_descriptor_ql);
-		tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn));
-		if (tcache_slow != NULL && tcache_slow->arena == arena) {
-			tcache_t *tcache = tcache_slow->tcache;
-			ql_elm_new(tcache_slow, link);
-			ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
-			cache_bin_array_descriptor_init(
-			    &tcache_slow->cache_bin_array_descriptor,
-			    tcache->bins);
-			ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
-			    &tcache_slow->cache_bin_array_descriptor, link);
-		}
+		malloc_mutex_postfork_child(tsdn,
+		    &arena->cache_bin_array_descriptor_ql_mtx);
+		arena_cache_bin_array_postfork_child(arena, surviving_desc);
 	}
 
 	for (unsigned i = 0; i < nbins_total; i++) {
@@ -2097,7 +2100,4 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
 	malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
 	base_postfork_child(tsdn, arena->base);
 	pa_shard_postfork_child(tsdn, &arena->pa_shard);
-	if (config_stats) {
-		malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
-	}
 }

src/arenas_management.c

@@ -0,0 +1,354 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/arenas_management.h"
+#include "jemalloc/internal/jemalloc_init.h"
+#include "jemalloc/internal/malloc_io.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/sz.h"
+
+JEMALLOC_ALIGNED(CACHELINE)
+atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
+/* Below two are read-only after initialization. */
+unsigned   narenas_auto;
+unsigned   manual_arena_base;
+
+static atomic_u_t narenas_total;
+
+static malloc_mutex_t arenas_lock;
+
+bool
+arenas_management_boot(void) {
+	return malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
+	    malloc_mutex_rank_exclusive);
+}
+
+void
+arenas_management_prefork(tsdn_t *tsdn) {
+	malloc_mutex_prefork(tsdn, &arenas_lock);
+}
+
+void
+arenas_management_postfork_parent(tsdn_t *tsdn) {
+	malloc_mutex_postfork_parent(tsdn, &arenas_lock);
+}
+
+void
+arenas_management_postfork_child(tsdn_t *tsdn) {
+	malloc_mutex_postfork_child(tsdn, &arenas_lock);
+}
+
+void
+narenas_total_set(unsigned narenas) {
+	atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
+}
+
+void
+narenas_total_inc(void) {
+	atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
+}
+
+unsigned
+narenas_total_get(void) {
+	return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
+}
+
+void
+narenas_auto_set(unsigned n) {
+	narenas_auto = n;
+}
+
+void
+manual_arena_base_set(unsigned base) {
+	manual_arena_base = base;
+}
+
+/*
+ * The a0*() functions are used instead of i{d,}alloc() in situations that
+ * cannot tolerate TLS variable access.
+ */
+
+void *
+a0ialloc(size_t size, bool zero, bool is_internal) {
+	if (unlikely(malloc_init_a0())) {
+		return NULL;
+	}
+
+	return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
+	    is_internal, arena_get(TSDN_NULL, 0, true), true);
+}
+
+void
+a0idalloc(void *ptr, bool is_internal) {
+	idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
+}
+
+void *
+a0malloc(size_t size) {
+	return a0ialloc(size, false, true);
+}
+
+void
+a0dalloc(void *ptr) {
+	a0idalloc(ptr, true);
+}
+
+void
+arena_set(unsigned ind, arena_t *arena) {
+	atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
+}
+
+/* Create a new arena and insert it into the arenas array at index ind. */
+static arena_t *
+arena_init_locked(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
+	arena_t *arena;
+
+	assert(ind <= narenas_total_get());
+	if (ind >= MALLOCX_ARENA_LIMIT) {
+		return NULL;
+	}
+	if (ind == narenas_total_get()) {
+		narenas_total_inc();
+	}
+
+	/*
+	 * Another thread may have already initialized arenas[ind] if it's an
+	 * auto arena.
+	 */
+	arena = arena_get(tsdn, ind, false);
+	if (arena != NULL) {
+		assert(arena_is_auto(arena));
+		return arena;
+	}
+
+	/* Actually initialize the arena. */
+	arena = arena_new(tsdn, ind, config);
+
+	return arena;
+}
+
+static void
+arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
+	if (ind == 0) {
+		return;
+	}
+
+	if (have_background_thread) {
+		if (background_thread_create(tsdn_tsd(tsdn), ind)) {
+			malloc_printf(
+			    "<jemalloc>: error in background thread "
+			    "creation for arena %u. Abort.\n",
+			    ind);
+			abort();
+		}
+	}
+}
+
+arena_t *
+arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
+	arena_t *arena;
+
+	malloc_mutex_lock(tsdn, &arenas_lock);
+	arena = arena_init_locked(tsdn, ind, config);
+	malloc_mutex_unlock(tsdn, &arenas_lock);
+
+	arena_new_create_background_thread(tsdn, ind);
+
+	return arena;
+}
+
+static void
+arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
+	arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
+	arena_nthreads_inc(arena, internal);
+
+	if (internal) {
+		tsd_iarena_set(tsd, arena);
+	} else {
+		tsd_arena_set(tsd, arena);
+		/*
+		 * While shard acts as a random seed, the cast below should
+		 * not make much difference.
+		 */
+		uint8_t shard = (uint8_t)atomic_fetch_add_u(
+		    &arena->binshard_next, 1, ATOMIC_RELAXED);
+		tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
+		for (unsigned i = 0; i < SC_NBINS; i++) {
+			assert(bin_infos[i].n_shards > 0
+			    && bin_infos[i].n_shards <= BIN_SHARDS_MAX);
+			bins->binshard[i] = shard % bin_infos[i].n_shards;
+		}
+	}
+}
+
+void
+arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
+	assert(oldarena != NULL);
+	assert(newarena != NULL);
+
+	arena_nthreads_dec(oldarena, false);
+	arena_nthreads_inc(newarena, false);
+	tsd_arena_set(tsd, newarena);
+
+	if (arena_nthreads_get(oldarena, false) == 0
+	    && !background_thread_enabled()) {
+		/*
+		 * Purge if the old arena has no associated threads anymore and
+		 * no background threads.
+		 */
+		arena_decay(tsd_tsdn(tsd), oldarena,
+		    /* is_background_thread */ false, /* all */ true);
+	}
+}
+
+static void
+arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
+	arena_t *arena;
+
+	arena = arena_get(tsd_tsdn(tsd), ind, false);
+	arena_nthreads_dec(arena, internal);
+
+	if (internal) {
+		tsd_iarena_set(tsd, NULL);
+	} else {
+		tsd_arena_set(tsd, NULL);
+	}
+}
+
+/* Slow path, called only by arena_choose(). */
+arena_t *
+arena_choose_hard(tsd_t *tsd, bool internal) {
+	arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
+
+	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
+		unsigned choose = percpu_arena_choose();
+		ret = arena_get(tsd_tsdn(tsd), choose, true);
+		assert(ret != NULL);
+		arena_bind(tsd, arena_ind_get(ret), false);
+		arena_bind(tsd, arena_ind_get(ret), true);
+
+		return ret;
+	}
+
+	if (narenas_auto > 1) {
+		unsigned i, j, choose[2], first_null;
+		bool     is_new_arena[2];
+
+		/*
+		 * Determine binding for both non-internal and internal
+		 * allocation.
+		 *
+		 *   choose[0]: For application allocation.
+		 *   choose[1]: For internal metadata allocation.
+		 */
+
+		for (j = 0; j < 2; j++) {
+			choose[j] = 0;
+			is_new_arena[j] = false;
+		}
+
+		first_null = narenas_auto;
+		malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
+		assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
+		for (i = 1; i < narenas_auto; i++) {
+			if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
+				/*
+				 * Choose the first arena that has the lowest
+				 * number of threads assigned to it.
+				 */
+				for (j = 0; j < 2; j++) {
+					if (arena_nthreads_get(
+					        arena_get(
+					            tsd_tsdn(tsd), i, false),
+					        !!j)
+					    < arena_nthreads_get(
+					        arena_get(tsd_tsdn(tsd),
+					            choose[j], false),
+					        !!j)) {
+						choose[j] = i;
+					}
+				}
+			} else if (first_null == narenas_auto) {
+				/*
+				 * Record the index of the first uninitialized
+				 * arena, in case all extant arenas are in use.
+				 *
+				 * NB: It is possible for there to be
+				 * discontinuities in terms of initialized
+				 * versus uninitialized arenas, due to the
+				 * "thread.arena" mallctl.
+				 */
+				first_null = i;
+			}
+		}
+
+		for (j = 0; j < 2; j++) {
+			if (arena_nthreads_get(
+			        arena_get(tsd_tsdn(tsd), choose[j], false), !!j)
+			        == 0
+			    || first_null == narenas_auto) {
+				/*
+				 * Use an unloaded arena, or the least loaded
+				 * arena if all arenas are already initialized.
+				 */
+				if (!!j == internal) {
+					ret = arena_get(
+					    tsd_tsdn(tsd), choose[j], false);
+				}
+			} else {
+				arena_t *arena;
+
+				/* Initialize a new arena. */
+				choose[j] = first_null;
+				arena = arena_init_locked(tsd_tsdn(tsd),
+				    choose[j], &arena_config_default);
+				if (arena == NULL) {
+					malloc_mutex_unlock(
+					    tsd_tsdn(tsd), &arenas_lock);
+					return NULL;
+				}
+				is_new_arena[j] = true;
+				if (!!j == internal) {
+					ret = arena;
+				}
+			}
+			arena_bind(tsd, choose[j], !!j);
+		}
+		malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
+
+		for (j = 0; j < 2; j++) {
+			if (is_new_arena[j]) {
+				assert(choose[j] > 0);
+				arena_new_create_background_thread(
+				    tsd_tsdn(tsd), choose[j]);
+			}
+		}
+
+	} else {
+		ret = arena_get(tsd_tsdn(tsd), 0, false);
+		arena_bind(tsd, 0, false);
+		arena_bind(tsd, 0, true);
+	}
+
+	return ret;
+}
+
+void
+iarena_cleanup(tsd_t *tsd) {
+	arena_t *iarena;
+
+	iarena = tsd_iarena_get(tsd);
+	if (iarena != NULL) {
+		arena_unbind(tsd, arena_ind_get(iarena), true);
+	}
+}
+
+void
+arena_cleanup(tsd_t *tsd) {
+	arena_t *arena;
+
+	arena = tsd_arena_get(tsd);
+	if (arena != NULL) {
+		arena_unbind(tsd, arena_ind_get(arena), false);
+	}
+}

src/base.c

@@ -52,8 +52,9 @@ base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) {
 	if (ehooks_are_default(ehooks)) {
 		addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit);
 	} else {
-		addr = ehooks_alloc(
-		    tsdn, ehooks, NULL, size, alignment, &zero, &commit);
+		UNUSED unsigned flags;
+		addr = ehooks_alloc(tsdn, ehooks, NULL, size, alignment, &zero,
+		    &commit, &flags);
 	}
 
 	return addr;
@@ -112,7 +113,7 @@ label_done:
 }
 
 static inline bool
-base_edata_is_reused(edata_t *edata) {
+base_edata_is_reused(const edata_t *edata) {
 	/*
 	 * Borrow the guarded bit to indicate if the extent is a recycled one,
 	 * i.e. the ones returned to base for reuse; currently only tcache bin
@@ -133,8 +134,8 @@ base_edata_init(
 }
 
 static size_t
-base_get_num_blocks(base_t *base, bool with_new_block) {
-	base_block_t *b = base->blocks;
+base_get_num_blocks(const base_t *base, bool with_new_block) {
+	const base_block_t *b = base->blocks;
 	assert(b != NULL);
 
 	size_t n_blocks = with_new_block ? 2 : 1;

src/bin.c

@@ -330,3 +330,16 @@ bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
 	}
 	return arena_get_bin(arena, binind, binshard);
 }
+
+void *
+bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin) {
+	malloc_mutex_lock(tsdn, &bin->lock);
+	edata_t *slab = (bin->slabcur != NULL)
+	    ? bin->slabcur
+	    : edata_heap_first(&bin->slabs_nonfull);
+	assert(slab != NULL || edata_heap_empty(&bin->slabs_nonfull));
+	void *ret = (slab != NULL) ? edata_addr_get(slab) : NULL;
+	assert(ret != NULL || slab == NULL);
+	malloc_mutex_unlock(tsdn, &bin->lock);
+	return ret;
+}

src/cache_bin.c

@@ -10,8 +10,8 @@ const uintptr_t disabled_bin = JUNK_ADDR;
 void
 cache_bin_info_init(cache_bin_info_t *info, cache_bin_sz_t ncached_max) {
 	assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
-	size_t stack_size = (size_t)ncached_max * sizeof(void *);
-	assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
+	assert((size_t)ncached_max * sizeof(void *)
+	    < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
 	info->ncached_max = (cache_bin_sz_t)ncached_max;
 }
 
@@ -94,9 +94,8 @@ cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
 	bin->low_bits_full = (cache_bin_sz_t)(uintptr_t)full_position;
 	bin->low_bits_empty = (cache_bin_sz_t)(uintptr_t)empty_position;
 	cache_bin_info_init(&bin->bin_info, info->ncached_max);
-	cache_bin_sz_t free_spots = cache_bin_diff(bin, bin->low_bits_full,
-	    (cache_bin_sz_t)(uintptr_t)bin->stack_head);
-	assert(free_spots == bin_stack_size);
+	assert(cache_bin_diff(bin, bin->low_bits_full,
+	    (cache_bin_sz_t)(uintptr_t)bin->stack_head) == bin_stack_size);
 	if (!cache_bin_disabled(bin)) {
 		assert(cache_bin_ncached_get_local(bin) == 0);
 	}

src/conf.c

@@ -225,7 +225,8 @@ conf_error(
 		/* However, tolerate experimental features. */
 		return;
 	}
-	const char  *deprecated[] = {"hpa_sec_bytes_after_flush"};
+	const char *deprecated[] = {
+	    "hpa_sec_bytes_after_flush", "hpa_sec_batch_fill_extra"};
 	const size_t deprecated_cnt = (sizeof(deprecated)
 	    / sizeof(deprecated[0]));
 	for (size_t i = 0; i < deprecated_cnt; ++i) {
@@ -952,9 +953,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
 			    "hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
 			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
-			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
-			    "hpa_sec_batch_fill_extra", 1, HUGEPAGE_PAGES,
-			    CONF_CHECK_MIN, CONF_CHECK_MAX, true);
 
 			if (CONF_MATCH("slab_sizes")) {
 				if (CONF_MATCH_VALUE("default")) {

src/ctl.c

@@ -115,7 +115,6 @@ CTL_PROTO(opt_hpa_dirty_mult)
 CTL_PROTO(opt_hpa_sec_nshards)
 CTL_PROTO(opt_hpa_sec_max_alloc)
 CTL_PROTO(opt_hpa_sec_max_bytes)
-CTL_PROTO(opt_hpa_sec_batch_fill_extra)
 CTL_PROTO(opt_huge_arena_pac_thp)
 CTL_PROTO(opt_metadata_thp)
 CTL_PROTO(opt_retain)
@@ -257,9 +256,11 @@ INDEX_PROTO(stats_arenas_i_lextents_j)
 CTL_PROTO(stats_arenas_i_extents_j_ndirty)
 CTL_PROTO(stats_arenas_i_extents_j_nmuzzy)
 CTL_PROTO(stats_arenas_i_extents_j_nretained)
+CTL_PROTO(stats_arenas_i_extents_j_npinned)
 CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes)
 CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes)
 CTL_PROTO(stats_arenas_i_extents_j_retained_bytes)
+CTL_PROTO(stats_arenas_i_extents_j_pinned_bytes)
 INDEX_PROTO(stats_arenas_i_extents_j)
 
 /* Merged set of stats for HPA shard. */
@@ -309,6 +310,14 @@ CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)
 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)
 
 INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j)
+CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_min_extents)
+CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_max_extents)
+CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_extents)
+CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_ps)
+CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_pages_per_ps)
+CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_extents_per_ps)
+CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps)
+INDEX_PROTO(stats_arenas_i_hpa_shard_alloc_j)
 
 CTL_PROTO(stats_arenas_i_nthreads)
 CTL_PROTO(stats_arenas_i_uptime)
@@ -320,6 +329,7 @@ CTL_PROTO(stats_arenas_i_pdirty)
 CTL_PROTO(stats_arenas_i_pmuzzy)
 CTL_PROTO(stats_arenas_i_mapped)
 CTL_PROTO(stats_arenas_i_retained)
+CTL_PROTO(stats_arenas_i_pinned)
 CTL_PROTO(stats_arenas_i_extent_avail)
 CTL_PROTO(stats_arenas_i_dirty_npurge)
 CTL_PROTO(stats_arenas_i_dirty_nmadvise)
@@ -355,10 +365,9 @@ CTL_PROTO(stats_metadata_thp)
 CTL_PROTO(stats_resident)
 CTL_PROTO(stats_mapped)
 CTL_PROTO(stats_retained)
+CTL_PROTO(stats_pinned)
 CTL_PROTO(stats_zero_reallocs)
 CTL_PROTO(approximate_stats_active)
-CTL_PROTO(experimental_hooks_install)
-CTL_PROTO(experimental_hooks_remove)
 CTL_PROTO(experimental_hooks_prof_backtrace)
 CTL_PROTO(experimental_hooks_prof_dump)
 CTL_PROTO(experimental_hooks_prof_sample)
@@ -371,7 +380,6 @@ CTL_PROTO(experimental_arenas_i_pactivep)
 INDEX_PROTO(experimental_arenas_i)
 CTL_PROTO(experimental_prof_recent_alloc_max)
 CTL_PROTO(experimental_prof_recent_alloc_dump)
-CTL_PROTO(experimental_batch_alloc)
 CTL_PROTO(experimental_arenas_create_ext)
 
 #define MUTEX_STATS_CTL_PROTO_GEN(n)                                           \
@@ -487,7 +495,6 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
     {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
     {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
     {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
-    {NAME("hpa_sec_batch_fill_extra"), CTL(opt_hpa_sec_batch_fill_extra)},
     {NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
     {NAME("metadata_thp"), CTL(opt_metadata_thp)},
     {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)},
@@ -697,9 +704,11 @@ static const ctl_named_node_t stats_arenas_i_extents_j_node[] = {
     {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)},
     {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)},
     {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)},
+    {NAME("npinned"), CTL(stats_arenas_i_extents_j_npinned)},
     {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)},
     {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)},
-    {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)}};
+    {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)},
+    {NAME("pinned_bytes"), CTL(stats_arenas_i_extents_j_pinned_bytes)}};
 
 static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = {
     {NAME(""), CHILD(named, stats_arenas_i_extents_j)}};
@@ -778,6 +787,23 @@ static const ctl_named_node_t
 static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] =
     {{INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}};
 
+static const ctl_named_node_t stats_arenas_i_hpa_shard_alloc_j_node[] = {
+    {NAME("min_extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_min_extents)},
+    {NAME("max_extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_max_extents)},
+    {NAME("extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_extents)},
+    {NAME("ps"), CTL(stats_arenas_i_hpa_shard_alloc_j_ps)},
+    {NAME("pages_per_ps"), CTL(stats_arenas_i_hpa_shard_alloc_j_pages_per_ps)},
+    {NAME("extents_per_ps"),
+        CTL(stats_arenas_i_hpa_shard_alloc_j_extents_per_ps)},
+    {NAME("total_elapsed_ns_per_ps"),
+        CTL(stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps)}};
+
+static const ctl_named_node_t super_stats_arenas_i_hpa_shard_alloc_j_node[] = {
+    {NAME(""), CHILD(named, stats_arenas_i_hpa_shard_alloc_j)}};
+
+static const ctl_indexed_node_t stats_arenas_i_hpa_shard_alloc_node[] = {
+    {INDEX(stats_arenas_i_hpa_shard_alloc_j)}};
+
 static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
     {NAME("npageslabs"), CTL(stats_arenas_i_hpa_shard_npageslabs)},
     {NAME("nactive"), CTL(stats_arenas_i_hpa_shard_nactive)},
@@ -791,6 +817,8 @@ static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
     {NAME("nhugify_failures"), CTL(stats_arenas_i_hpa_shard_nhugify_failures)},
     {NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)},
 
+    {NAME("alloc"), CHILD(indexed, stats_arenas_i_hpa_shard_alloc)},
+
     {NAME("full_slabs"), CHILD(named, stats_arenas_i_hpa_shard_full_slabs)},
     {NAME("empty_slabs"), CHILD(named, stats_arenas_i_hpa_shard_empty_slabs)},
     {NAME("nonfull_slabs"),
@@ -807,6 +835,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
     {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)},
     {NAME("mapped"), CTL(stats_arenas_i_mapped)},
     {NAME("retained"), CTL(stats_arenas_i_retained)},
+    {NAME("pinned"), CTL(stats_arenas_i_pinned)},
     {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)},
     {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)},
     {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)},
@@ -872,6 +901,7 @@ static const ctl_named_node_t stats_node[] = {
     {NAME("resident"), CTL(stats_resident)},
     {NAME("mapped"), CTL(stats_mapped)},
     {NAME("retained"), CTL(stats_retained)},
+    {NAME("pinned"), CTL(stats_pinned)},
     {NAME("background_thread"), CHILD(named, stats_background_thread)},
     {NAME("mutexes"), CHILD(named, stats_mutexes)},
     {NAME("arenas"), CHILD(indexed, stats_arenas)},
@@ -879,8 +909,6 @@ static const ctl_named_node_t stats_node[] = {
 };
 
 static const ctl_named_node_t experimental_hooks_node[] = {
-    {NAME("install"), CTL(experimental_hooks_install)},
-    {NAME("remove"), CTL(experimental_hooks_remove)},
     {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)},
     {NAME("prof_dump"), CTL(experimental_hooks_prof_dump)},
     {NAME("prof_sample"), CTL(experimental_hooks_prof_sample)},
@@ -911,8 +939,7 @@ static const ctl_named_node_t experimental_node[] = {
     {NAME("utilization"), CHILD(named, experimental_utilization)},
     {NAME("arenas"), CHILD(indexed, experimental_arenas)},
     {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)},
-    {NAME("prof_recent"), CHILD(named, experimental_prof_recent)},
-    {NAME("batch_alloc"), CTL(experimental_batch_alloc)}};
+    {NAME("prof_recent"), CHILD(named, experimental_prof_recent)}};
 
 static const ctl_named_node_t root_node[] = {{NAME("version"), CTL(version)},
     {NAME("epoch"), CTL(epoch)},
@@ -1111,6 +1138,8 @@ ctl_arena_stats_sdmerge(
 			sdstats->astats.mapped += astats->astats.mapped;
 			sdstats->astats.pa_shard_stats.pac_stats.retained +=
 			    astats->astats.pa_shard_stats.pac_stats.retained;
+			sdstats->astats.pa_shard_stats.pac_stats.pinned +=
+			    astats->astats.pa_shard_stats.pac_stats.pinned;
 			sdstats->astats.pa_shard_stats.edata_avail +=
 			    astats->astats.pa_shard_stats.edata_avail;
 		}
@@ -1247,12 +1276,16 @@ ctl_arena_stats_sdmerge(
 			sdstats->estats[i].nmuzzy += astats->estats[i].nmuzzy;
 			sdstats->estats[i].nretained +=
 			    astats->estats[i].nretained;
+			sdstats->estats[i].npinned +=
+			    astats->estats[i].npinned;
 			sdstats->estats[i].dirty_bytes +=
 			    astats->estats[i].dirty_bytes;
 			sdstats->estats[i].muzzy_bytes +=
 			    astats->estats[i].muzzy_bytes;
 			sdstats->estats[i].retained_bytes +=
 			    astats->estats[i].retained_bytes;
+			sdstats->estats[i].pinned_bytes +=
+			    astats->estats[i].pinned_bytes;
 		}
 
 		/* Merge HPA stats. */
@@ -1367,6 +1400,8 @@ ctl_refresh(tsdn_t *tsdn) {
 		ctl_stats->mapped = ctl_sarena->astats->astats.mapped;
 		ctl_stats->retained = ctl_sarena->astats->astats.pa_shard_stats
 		                          .pac_stats.retained;
+		ctl_stats->pinned = ctl_sarena->astats->astats.pa_shard_stats
+		                        .pac_stats.pinned;
 
 		ctl_background_thread_stats_read(tsdn);
 
@@ -2173,8 +2208,6 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
-CTL_RO_NL_GEN(
-    opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, size_t)
 CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
 CTL_RO_NL_GEN(
     opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
@@ -2304,13 +2337,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 			ret = EAGAIN;
 			goto label_return;
 		}
-		/* Set new arena/tcache associations. */
-		arena_migrate(tsd, oldarena, newarena);
-		if (tcache_available(tsd)) {
-			tcache_arena_reassociate(tsd_tsdn(tsd),
-			    tsd_tcache_slowp_get(tsd), tsd_tcachep_get(tsd),
-			    newarena);
-		}
+		thread_migrate_arena(tsd, oldarena, newarena);
 	}
 
 	ret = 0;
@@ -3721,6 +3748,7 @@ CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t)
 CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
 CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
 CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
+CTL_RO_CGEN(config_stats, stats_pinned, ctl_stats->pinned, size_t)
 
 CTL_RO_CGEN(config_stats, stats_background_thread_num_threads,
     ctl_stats->background_thread.num_threads, size_t)
@@ -3786,6 +3814,8 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
     arenas_i(mib[2])->astats->astats.mapped, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
     arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.retained, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_pinned,
+    arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.pinned, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail,
     arenas_i(mib[2])->astats->astats.pa_shard_stats.edata_avail, size_t)
 
@@ -3958,9 +3988,10 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
 		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_dirty.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_muzzy.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_retained.mtx);
+		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_pinned.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.decay_dirty.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.decay_muzzy.mtx);
-		MUTEX_PROF_RESET(arena->tcache_ql_mtx);
+		MUTEX_PROF_RESET(arena->cache_bin_array_descriptor_ql_mtx);
 		MUTEX_PROF_RESET(arena->base->mtx);
 
 		for (szind_t j = 0; j < SC_NBINS; j++) {
@@ -4029,17 +4060,21 @@ stats_arenas_i_lextents_j_index(
 }
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty,
-    arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t);
+    arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy,
-    arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t);
+    arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained,
-    arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t);
+    arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_npinned,
+    arenas_i(mib[2])->astats->estats[mib[4]].npinned, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes,
-    arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t);
+    arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes,
-    arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t);
+    arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes,
-    arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t);
+    arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_pinned_bytes,
+    arenas_i(mib[2])->astats->estats[mib[4]].pinned_bytes, size_t)
 
 static const ctl_named_node_t *
 stats_arenas_i_extents_j_index(
@@ -4051,86 +4086,116 @@ stats_arenas_i_extents_j_index(
 }
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npageslabs,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.npageslabs, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.npageslabs, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nactive,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.nactive, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.nactive, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndirty,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.ndirty, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.ndirty, size_t)
 
 /* Nonhuge slabs */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_npageslabs_nonhuge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].npageslabs, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].npageslabs, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_nactive_nonhuge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].nactive, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].nactive, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_ndirty_nonhuge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].ndirty, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].ndirty, size_t)
 
 /* Huge slabs */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_npageslabs_huge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].npageslabs, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].npageslabs, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_nactive_huge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].nactive, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].nactive, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_ndirty_huge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].ndirty, size_t);
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].ndirty, size_t)
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurge_passes,
     arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurge_passes,
-    uint64_t);
+    uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurges,
-    arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t);
+    arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies,
-    arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t);
+    arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugify_failures,
     arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugify_failures,
-    uint64_t);
+    uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies,
-    arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t);
+    arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t)
+
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_min_extents,
+    arenas_i(mib[2])
+        ->astats->hpastats.nonderived_stats.hpa_alloc_min_extents[mib[5]],
+    uint64_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_max_extents,
+    arenas_i(mib[2])
+        ->astats->hpastats.nonderived_stats.hpa_alloc_max_extents[mib[5]],
+    uint64_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_extents,
+    arenas_i(mib[2])
+        ->astats->hpastats.nonderived_stats.hpa_alloc_extents[mib[5]],
+    uint64_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_ps,
+    arenas_i(mib[2])->astats->hpastats.nonderived_stats.hpa_alloc_ps[mib[5]],
+    uint64_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_pages_per_ps,
+    arenas_i(mib[2])
+        ->astats->hpastats.nonderived_stats.hpa_alloc_pages_per_ps[mib[5]],
+    uint64_t);
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_extents_per_ps,
+    arenas_i(mib[2])
+        ->astats->hpastats.nonderived_stats.hpa_alloc_extents_per_ps[mib[5]],
+    uint64_t);
+CTL_RO_CGEN(config_stats,
+    stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps,
+    arenas_i(mib[2])
+        ->astats->hpastats.nonderived_stats
+        .hpa_alloc_total_elapsed_ns_per_ps[mib[5]],
+    uint64_t);
 
 /* Full, nonhuge */
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ndirty,
-    size_t);
+    size_t)
 
 /* Full, huge */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ndirty,
-    size_t);
+    size_t)
 
 /* Empty, nonhuge */
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].npageslabs,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].nactive,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].ndirty,
-    size_t);
+    size_t)
 
 /* Empty, huge */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].npageslabs,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].nactive,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].ndirty,
-    size_t);
+    size_t)
 
 /* Nonfull, nonhuge */
 CTL_RO_CGEN(config_stats,
@@ -4138,19 +4203,19 @@ CTL_RO_CGEN(config_stats,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0]
         .npageslabs,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0]
         .nactive,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0]
         .ndirty,
-    size_t);
+    size_t)
 
 /* Nonfull, huge */
 CTL_RO_CGEN(config_stats,
@@ -4158,17 +4223,17 @@ CTL_RO_CGEN(config_stats,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1]
         .npageslabs,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1]
         .nactive,
-    size_t);
+    size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1]
         .ndirty,
-    size_t);
+    size_t)
 
 static const ctl_named_node_t *
 stats_arenas_i_hpa_shard_nonfull_slabs_j_index(
@@ -4179,6 +4244,15 @@ stats_arenas_i_hpa_shard_nonfull_slabs_j_index(
 	return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node;
 }
 
+static const ctl_named_node_t *
+stats_arenas_i_hpa_shard_alloc_j_index(
+    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
+	if (j > SEC_MAX_NALLOCS) {
+		return NULL;
+	}
+	return super_stats_arenas_i_hpa_shard_alloc_j_node;
+}
+
 static bool
 ctl_arenas_i_verify(size_t i) {
 	size_t a = arenas_i2a_impl(i, true, true);
@@ -4205,51 +4279,6 @@ label_return:
 	return ret;
 }
 
-static int
-experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
-    void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
-	int ret;
-	if (oldp == NULL || oldlenp == NULL || newp == NULL) {
-		ret = EINVAL;
-		goto label_return;
-	}
-	/*
-	 * Note: this is a *private* struct.  This is an experimental interface;
-	 * forcing the user to know the jemalloc internals well enough to
-	 * extract the ABI hopefully ensures nobody gets too comfortable with
-	 * this API, which can change at a moment's notice.
-	 */
-	hooks_t hooks;
-	WRITE(hooks, hooks_t);
-	void *handle = hook_install(tsd_tsdn(tsd), &hooks);
-	if (handle == NULL) {
-		ret = EAGAIN;
-		goto label_return;
-	}
-	READ(handle, void *);
-
-	ret = 0;
-label_return:
-	return ret;
-}
-
-static int
-experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
-    void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
-	int ret;
-	WRITEONLY();
-	void *handle = NULL;
-	WRITE(handle, void *);
-	if (handle == NULL) {
-		ret = EINVAL;
-		goto label_return;
-	}
-	hook_remove(tsd_tsdn(tsd), handle);
-	ret = 0;
-label_return:
-	return ret;
-}
-
 /*
  * Output six memory utilization entries for an input pointer, the first one of
  * type (void *) and the remaining five of type size_t, describing the following
@@ -4589,34 +4618,6 @@ label_return:
 	return ret;
 }
 
-typedef struct batch_alloc_packet_s batch_alloc_packet_t;
-struct batch_alloc_packet_s {
-	void **ptrs;
-	size_t num;
-	size_t size;
-	int    flags;
-};
-
-static int
-experimental_batch_alloc_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
-    void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
-	int ret;
-
-	VERIFY_READ(size_t);
-
-	batch_alloc_packet_t batch_alloc_packet;
-	ASSURED_WRITE(batch_alloc_packet, batch_alloc_packet_t);
-	size_t filled = batch_alloc(batch_alloc_packet.ptrs,
-	    batch_alloc_packet.num, batch_alloc_packet.size,
-	    batch_alloc_packet.flags);
-	READ(filled, size_t);
-
-	ret = 0;
-
-label_return:
-	return ret;
-}
-
 static int
 prof_stats_bins_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
     void *oldp, size_t *oldlenp, void *newp, size_t newlen) {

src/emap.c

@@ -50,7 +50,8 @@ emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
 	assert(!edata_state_in_transition(expected_state));
 	assert(expected_state == extent_state_dirty
 	    || expected_state == extent_state_muzzy
-	    || expected_state == extent_state_retained);
+	    || expected_state == extent_state_retained
+	    || expected_state == extent_state_pinned);
 
 	void *neighbor_addr = forward ? edata_past_get(edata)
 	                              : edata_before_get(edata);

src/eset.c

@@ -32,17 +32,17 @@ eset_init(eset_t *eset, extent_state_t state) {
 }
 
 size_t
-eset_npages_get(eset_t *eset) {
+eset_npages_get(const eset_t *eset) {
 	return atomic_load_zu(&eset->npages, ATOMIC_RELAXED);
 }
 
 size_t
-eset_nextents_get(eset_t *eset, pszind_t pind) {
+eset_nextents_get(const eset_t *eset, pszind_t pind) {
 	return atomic_load_zu(&eset->bin_stats[pind].nextents, ATOMIC_RELAXED);
 }
 
 size_t
-eset_nbytes_get(eset_t *eset, pszind_t pind) {
+eset_nbytes_get(const eset_t *eset, pszind_t pind) {
 	return atomic_load_zu(&eset->bin_stats[pind].nbytes, ATOMIC_RELAXED);
 }
 
@@ -98,7 +98,10 @@ eset_insert(eset_t *eset, edata_t *edata) {
 		eset_stats_add(eset, pind, size);
 	}
 
-	edata_list_inactive_append(&eset->lru, edata);
+	/* Pinned extents skip LRU as they do not decay. */
+	if (!edata_pinned_get(edata)) {
+		edata_list_inactive_append(&eset->lru, edata);
+	}
 	size_t npages = size >> LG_PAGE;
 	/*
 	 * All modifications to npages hold the mutex (as asserted above), so we
@@ -143,7 +146,9 @@ eset_remove(eset_t *eset, edata_t *edata) {
 			    edata_heap_first(&eset->bins[pind].heap));
 		}
 	}
-	edata_list_inactive_remove(&eset->lru, edata);
+	if (!edata_pinned_get(edata)) {
+		edata_list_inactive_remove(&eset->lru, edata);
+	}
 	size_t npages = size >> LG_PAGE;
 	/*
 	 * As in eset_insert, we hold eset->mtx and so don't need atomic
@@ -279,10 +284,15 @@ eset_fit_alignment(
  * avoiding reusing and splitting large extents for smaller sizes.  In practice,
  * it's set to opt_lg_extent_max_active_fit for the dirty eset and SC_PTR_BITS
  * for others.
+ *
+ * If prefer_small is true, return as soon as the smallest fitting bin yields a
+ * candidate, instead of scanning further bins for an older/lower extent.
+ * Useful for fragmentation control for the pinned pool.
  */
 static edata_t *
 eset_first_fit(
-    eset_t *eset, size_t size, bool exact_only, unsigned lg_max_fit) {
+    eset_t *eset, size_t size, bool exact_only, unsigned lg_max_fit,
+    bool prefer_small) {
 	edata_t                     *ret = NULL;
 	edata_cmp_summary_t ret_summ JEMALLOC_CC_SILENCE_INIT({0});
 
@@ -327,6 +337,9 @@ eset_first_fit(
 	if (sz_large_size_classes_disabled() && pind != pind_prev) {
 		ret = eset_enumerate_search(eset, size, pind_prev,
 		    /* exact_only */ false, &ret_summ);
+		if (prefer_small && ret != NULL) {
+			return ret;
+		}
 	}
 
 	for (pszind_t i =
@@ -363,6 +376,9 @@ eset_first_fit(
 			           edata_cmp_summary_get(edata))
 			        == 0);
 			ret = edata;
+			if (prefer_small) {
+				return ret;
+			}
 			ret_summ = eset->bins[i].heap_min;
 		}
 		if (i == SC_NPSIZES) {
@@ -376,14 +392,15 @@ eset_first_fit(
 
 edata_t *
 eset_fit(eset_t *eset, size_t esize, size_t alignment, bool exact_only,
-    unsigned lg_max_fit) {
+    unsigned lg_max_fit, bool prefer_small) {
 	size_t max_size = esize + PAGE_CEILING(alignment) - PAGE;
 	/* Beware size_t wrap-around. */
 	if (max_size < esize) {
 		return NULL;
 	}
 
-	edata_t *edata = eset_first_fit(eset, max_size, exact_only, lg_max_fit);
+	edata_t *edata = eset_first_fit(eset, max_size, exact_only, lg_max_fit,
+	    prefer_small);
 
 	if (alignment > PAGE && edata == NULL) {
 		/*

src/extent.c

@@ -17,9 +17,8 @@ size_t opt_process_madvise_max_batch =
 #ifdef JEMALLOC_HAVE_PROCESS_MADVISE
     PROCESS_MADVISE_MAX_BATCH_DEFAULT;
 #else
-    0
+    0;
 #endif
-;
 
 static bool extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
     size_t offset, size_t length, bool growing_retained);
@@ -70,6 +69,7 @@ extent_may_force_decay(pac_t *pac) {
 static bool
 extent_try_delayed_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, edata_t *edata) {
+	malloc_mutex_assert_owner(tsdn, &ecache->mtx);
 	emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active);
 
 	bool coalesced;
@@ -212,6 +212,7 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 	switch (ecache->state) {
 	case extent_state_dirty:
 	case extent_state_muzzy:
+	case extent_state_pinned:
 		emap_update_edata_state(
 		    tsdn, pac->emap, edata, extent_state_active);
 		break;
@@ -244,7 +245,11 @@ extents_abandon_vm(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 	}
 	/*
 	 * Leak extent after making sure its pages have already been purged, so
-	 * that this is only a virtual memory leak.
+	 * that this is only a virtual memory leak, except when the extent is
+	 * pinned/unpurgeable, for which a real memory leak happens.  This is
+	 * acceptable because reaching this path requires that an extent split
+	 * fail, which is already an exceptional condition (typically an OOM
+	 * on edata_t allocation).
 	 */
 	if (ecache->state == extent_state_dirty) {
 		if (extent_purge_lazy_impl(
@@ -434,8 +439,12 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
 		 * then no longer satify a request for its original size.  To
 		 * limit this effect, when delayed coalescing is enabled, we
 		 * put a cap on how big an extent we can split for a request.
+		 *
+		 * Pinned extents are exempt: they are never purged, so the cap
+		 * doesn't apply.
 		 */
-		unsigned lg_max_fit = ecache->delay_coalesce
+		unsigned lg_max_fit = (ecache->delay_coalesce
+		    && ecache != &pac->ecache_pinned)
 		    ? (unsigned)opt_lg_extent_max_active_fit
 		    : SC_PTR_BITS;
 
@@ -448,7 +457,13 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
 		 * allocations.
 		 */
 		bool exact_only = (!maps_coalesce && !opt_retain) || guarded;
-		edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit);
+		/*
+		 * When selecting a pinned extent, avoid breaking larger extent
+		 * if a smaller one works.
+		 */
+		bool prefer_small = (ecache == &pac->ecache_pinned);
+		edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit,
+		    prefer_small);
 	}
 	if (edata == NULL) {
 		return NULL;
@@ -733,10 +748,9 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	}
 	bool zeroed = false;
 	bool committed = false;
-
-	void *ptr = ehooks_alloc(
-	    tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed, &committed);
-
+	unsigned flags = 0;
+	void *ptr = ehooks_alloc(tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed,
+	    &committed, &flags);
 	if (ptr == NULL) {
 		edata_cache_put(tsdn, pac->edata_cache, edata);
 		goto label_err;
@@ -746,6 +760,10 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	edata_init(edata, ind, ptr, alloc_size, false, SC_NSIZES,
 	    extent_sn_next(pac), extent_state_active, zeroed, committed,
 	    EXTENT_PAI_PAC, EXTENT_IS_HEAD);
+	edata_hook_flags_init(edata, flags);
+	if (flags & EXTENT_ALLOC_FLAG_PINNED) {
+		atomic_store_b(&pac->has_pinned, true, ATOMIC_RELAXED);
+	}
 
 	if (extent_register_no_gdump_add(tsdn, pac, edata)) {
 		edata_cache_put(tsdn, pac->edata_cache, edata);
@@ -767,12 +785,10 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 
 	if (result == extent_split_interior_ok) {
 		if (lead != NULL) {
-			extent_record(
-			    tsdn, pac, ehooks, &pac->ecache_retained, lead);
+			pac_record_grown(tsdn, pac, ehooks, lead);
 		}
 		if (trail != NULL) {
-			extent_record(
-			    tsdn, pac, ehooks, &pac->ecache_retained, trail);
+			pac_record_grown(tsdn, pac, ehooks, trail);
 		}
 	} else {
 		/*
@@ -784,8 +800,7 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 			if (config_prof) {
 				extent_gdump_add(tsdn, to_salvage);
 			}
-			extent_record(tsdn, pac, ehooks, &pac->ecache_retained,
-			    to_salvage);
+			pac_record_grown(tsdn, pac, ehooks, to_salvage);
 		}
 		if (to_leak != NULL) {
 			extent_deregister_no_gdump_sub(tsdn, pac, to_leak);
@@ -796,6 +811,8 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	}
 
 	if (*commit && !edata_committed_get(edata)) {
+		/* Pinned memory must be committed by the hook. */
+		assert(!edata_pinned_get(edata));
 		if (extent_commit_impl(
 		        tsdn, ehooks, edata, 0, edata_size_get(edata), true)) {
 			extent_record(
@@ -815,15 +832,18 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 
 	/*
 	 * Increment extent_grow_next if doing so wouldn't exceed the allowed
-	 * range.
+	 * range.  Skip for pinned: pinned memory is a finite resource;
+	 * oversized remnants waste it.
 	 */
 	/* All opportunities for failure are past. */
-	exp_grow_size_commit(&pac->exp_grow, exp_grow_skip);
+	if (!(flags & EXTENT_ALLOC_FLAG_PINNED)) {
+		exp_grow_size_commit(&pac->exp_grow, exp_grow_skip);
+	}
 	malloc_mutex_unlock(tsdn, &pac->grow_mtx);
 
 	if (huge_arena_pac_thp.thp_madvise) {
 		/* Avoid using HUGEPAGE when the grow size is less than HUGEPAGE. */
-		if (ind != 0 && ind == huge_arena_ind
+		if (arena_ind_is_huge(ind)
 		    && ehooks_are_default(ehooks)
 		    && likely(alloc_size >= HUGEPAGE)) {
 			extent_handle_huge_arena_thp(tsdn, &huge_arena_pac_thp,
@@ -1020,11 +1040,13 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 		edata = extent_try_coalesce(
 		    tsdn, pac, ehooks, ecache, edata, &coalesced_unused);
 	} else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) {
-		assert(ecache == &pac->ecache_dirty);
+		assert(edata_pinned_get(edata)
+		    ? (ecache == &pac->ecache_pinned)
+		    : (ecache == &pac->ecache_dirty));
 		/* Always coalesce large extents eagerly. */
 		/**
 		* Maximum size limit (max_size) for large extents waiting to be coalesced
-		* in dirty ecache.
+		* in pinned/dirty ecache.
 		*
 		* When set to a non-zero value, this parameter restricts the maximum size
 		* of large extents after coalescing. If the combined size of two extents
@@ -1056,7 +1078,9 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 			edata = extent_try_coalesce_large(tsdn, pac, ehooks,
 			    ecache, edata, max_size, &coalesced);
 		} while (coalesced);
-		if (edata_size_get(edata) >= atomic_load_zu(
+		/* Pinned extents cannot be purged; skip the oversize shortcut. */
+		if (ecache == &pac->ecache_dirty
+		    && edata_size_get(edata) >= atomic_load_zu(
 		        &pac->oversize_threshold, ATOMIC_RELAXED)
 		    && !background_thread_enabled()
 		    && extent_may_force_decay(pac)) {
@@ -1119,8 +1143,9 @@ extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void *new_addr,
 		return NULL;
 	}
 	size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
-	void  *addr = ehooks_alloc(
-            tsdn, ehooks, new_addr, size, palignment, &zero, commit);
+	unsigned flags = 0;
+	void  *addr = ehooks_alloc(tsdn, ehooks, new_addr, size, palignment,
+	    &zero, commit, &flags);
 	if (addr == NULL) {
 		edata_cache_put(tsdn, pac->edata_cache, edata);
 		return NULL;
@@ -1129,6 +1154,10 @@ extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void *new_addr,
 	    /* slab */ false, SC_NSIZES, extent_sn_next(pac),
 	    extent_state_active, zero, *commit, EXTENT_PAI_PAC,
 	    opt_retain ? EXTENT_IS_HEAD : EXTENT_NOT_HEAD);
+	edata_hook_flags_init(edata, flags);
+	if (flags & EXTENT_ALLOC_FLAG_PINNED) {
+		atomic_store_b(&pac->has_pinned, true, ATOMIC_RELAXED);
+	}
 	/*
 	 * Retained memory is not counted towards gdump.  Only if an extent is
 	 * allocated as a separate mapping, i.e. growing_retained is false, then
@@ -1328,6 +1357,7 @@ extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata,
 	    /* slab */ false, SC_NSIZES, edata_sn_get(edata),
 	    edata_state_get(edata), edata_zeroed_get(edata),
 	    edata_committed_get(edata), EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
+	edata_hook_flags_init(trail, edata_alloc_flags_get(edata));
 	emap_prepare_t prepare;
 	bool           err = emap_split_prepare(
             tsdn, pac->emap, &prepare, edata, size_a, trail, size_b);
@@ -1412,6 +1442,8 @@ extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a,
 	                                        : edata_sn_get(b));
 	edata_zeroed_set(a, edata_zeroed_get(a) && edata_zeroed_get(b));
 
+	assert(edata_pinned_get(a) == edata_pinned_get(b));
+
 	emap_merge_commit(tsdn, pac->emap, &prepare, a, b);
 
 	edata_cache_put(tsdn, pac->edata_cache, b);

src/extent_dss.c

@@ -33,11 +33,20 @@ static atomic_b_t dss_exhausted;
 /* Atomic current upper limit on DSS addresses. */
 static atomic_p_t dss_max;
 
+#ifdef JEMALLOC_JET
+extent_dss_sbrk_hook_t extent_dss_sbrk_hook = NULL;
+#endif
+
 /******************************************************************************/
 
 static void *
 extent_dss_sbrk(intptr_t increment) {
 #ifdef JEMALLOC_DSS
+#ifdef JEMALLOC_JET
+	if (extent_dss_sbrk_hook != NULL) {
+		return extent_dss_sbrk_hook(increment);
+	}
+#endif
 	return sbrk(increment);
 #else
 	not_implemented();

src/hook.c

@@ -1,192 +0,0 @@
-#include "jemalloc/internal/jemalloc_preamble.h"
-
-#include "jemalloc/internal/hook.h"
-
-#include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/seq.h"
-
-typedef struct hooks_internal_s hooks_internal_t;
-struct hooks_internal_s {
-	hooks_t hooks;
-	bool    in_use;
-};
-
-seq_define(hooks_internal_t, hooks)
-
-    static atomic_u_t nhooks = ATOMIC_INIT(0);
-static seq_hooks_t    hooks[HOOK_MAX];
-static malloc_mutex_t hooks_mu;
-
-bool
-hook_boot(void) {
-	return malloc_mutex_init(
-	    &hooks_mu, "hooks", WITNESS_RANK_HOOK, malloc_mutex_rank_exclusive);
-}
-
-static void *
-hook_install_locked(hooks_t *to_install) {
-	hooks_internal_t hooks_internal;
-	for (int i = 0; i < HOOK_MAX; i++) {
-		bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]);
-		/* We hold mu; no concurrent access. */
-		assert(success);
-		if (!hooks_internal.in_use) {
-			hooks_internal.hooks = *to_install;
-			hooks_internal.in_use = true;
-			seq_store_hooks(&hooks[i], &hooks_internal);
-			atomic_store_u(&nhooks,
-			    atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1,
-			    ATOMIC_RELAXED);
-			return &hooks[i];
-		}
-	}
-	return NULL;
-}
-
-void *
-hook_install(tsdn_t *tsdn, hooks_t *to_install) {
-	malloc_mutex_lock(tsdn, &hooks_mu);
-	void *ret = hook_install_locked(to_install);
-	if (ret != NULL) {
-		tsd_global_slow_inc(tsdn);
-	}
-	malloc_mutex_unlock(tsdn, &hooks_mu);
-	return ret;
-}
-
-static void
-hook_remove_locked(seq_hooks_t *to_remove) {
-	hooks_internal_t hooks_internal;
-	bool success = seq_try_load_hooks(&hooks_internal, to_remove);
-	/* We hold mu; no concurrent access. */
-	assert(success);
-	/* Should only remove hooks that were added. */
-	assert(hooks_internal.in_use);
-	hooks_internal.in_use = false;
-	seq_store_hooks(to_remove, &hooks_internal);
-	atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1,
-	    ATOMIC_RELAXED);
-}
-
-void
-hook_remove(tsdn_t *tsdn, void *opaque) {
-	if (config_debug) {
-		char *hooks_begin = (char *)&hooks[0];
-		char *hooks_end = (char *)&hooks[HOOK_MAX];
-		char *hook = (char *)opaque;
-		assert(hooks_begin <= hook && hook < hooks_end
-		    && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0);
-	}
-	malloc_mutex_lock(tsdn, &hooks_mu);
-	hook_remove_locked((seq_hooks_t *)opaque);
-	tsd_global_slow_dec(tsdn);
-	malloc_mutex_unlock(tsdn, &hooks_mu);
-}
-
-#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr)                                \
-	for (int for_each_hook_counter = 0; for_each_hook_counter < HOOK_MAX;  \
-	     for_each_hook_counter++) {                                        \
-		bool for_each_hook_success = seq_try_load_hooks(               \
-		    (hooks_internal_ptr), &hooks[for_each_hook_counter]);      \
-		if (!for_each_hook_success) {                                  \
-			continue;                                              \
-		}                                                              \
-		if (!(hooks_internal_ptr)->in_use) {                           \
-			continue;                                              \
-		}
-#define FOR_EACH_HOOK_END }
-
-static bool *
-hook_reentrantp(void) {
-	/*
-	 * We prevent user reentrancy within hooks.  This is basically just a
-	 * thread-local bool that triggers an early-exit.
-	 *
-	 * We don't fold in_hook into reentrancy.  There are two reasons for
-	 * this:
-	 * - Right now, we turn on reentrancy during things like extent hook
-	 *   execution.  Allocating during extent hooks is not officially
-	 *   supported, but we don't want to break it for the time being.  These
-	 *   sorts of allocations should probably still be hooked, though.
-	 * - If a hook allocates, we may want it to be relatively fast (after
-	 *   all, it executes on every allocator operation).  Turning on
-	 *   reentrancy is a fairly heavyweight mode (disabling tcache,
-	 *   redirecting to arena 0, etc.).  It's possible we may one day want
-	 *   to turn on reentrant mode here, if it proves too difficult to keep
-	 *   this working.  But that's fairly easy for us to see; OTOH, people
-	 *   not using hooks because they're too slow is easy for us to miss.
-	 *
-	 * The tricky part is
-	 * that this code might get invoked even if we don't have access to tsd.
-	 * This function mimics getting a pointer to thread-local data, except
-	 * that it might secretly return a pointer to some global data if we
-	 * know that the caller will take the early-exit path.
-	 * If we return a bool that indicates that we are reentrant, then the
-	 * caller will go down the early exit path, leaving the global
-	 * untouched.
-	 */
-	static bool in_hook_global = true;
-	tsdn_t     *tsdn = tsdn_fetch();
-	bool       *in_hook = tsdn_in_hookp_get(tsdn);
-	if (in_hook != NULL) {
-		return in_hook;
-	}
-	return &in_hook_global;
-}
-
-#define HOOK_PROLOGUE                                                          \
-	if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) {             \
-		return;                                                        \
-	}                                                                      \
-	bool *in_hook = hook_reentrantp();                                     \
-	if (*in_hook) {                                                        \
-		return;                                                        \
-	}                                                                      \
-	*in_hook = true;
-
-#define HOOK_EPILOGUE *in_hook = false;
-
-void
-hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
-    uintptr_t args_raw[3]) {
-	HOOK_PROLOGUE
-
-	hooks_internal_t hook;
-	FOR_EACH_HOOK_BEGIN(&hook)
-	hook_alloc h = hook.hooks.alloc_hook;
-	if (h != NULL) {
-		h(hook.hooks.extra, type, result, result_raw, args_raw);
-	}
-	FOR_EACH_HOOK_END
-
-	HOOK_EPILOGUE
-}
-
-void
-hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) {
-	HOOK_PROLOGUE
-	hooks_internal_t hook;
-	FOR_EACH_HOOK_BEGIN(&hook)
-	hook_dalloc h = hook.hooks.dalloc_hook;
-	if (h != NULL) {
-		h(hook.hooks.extra, type, address, args_raw);
-	}
-	FOR_EACH_HOOK_END
-	HOOK_EPILOGUE
-}
-
-void
-hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
-    size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) {
-	HOOK_PROLOGUE
-	hooks_internal_t hook;
-	FOR_EACH_HOOK_BEGIN(&hook)
-	hook_expand h = hook.hooks.expand_hook;
-	if (h != NULL) {
-		h(hook.hooks.extra, type, address, old_usize, new_usize,
-		    result_raw, args_raw);
-	}
-	FOR_EACH_HOOK_END
-	HOOK_EPILOGUE
-}

src/hpa.c

@@ -8,18 +8,7 @@
 #include "jemalloc/internal/witness.h"
 #include "jemalloc/internal/jemalloc_probe.h"
 
-static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
-    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
-    bool *deferred_work_generated);
-static bool     hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-        size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
-static bool     hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-        size_t old_size, size_t new_size, bool *deferred_work_generated);
-static void     hpa_dalloc(
-        tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
-static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
-
-static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
+static void hpa_dalloc_batch(tsdn_t *tsdn, hpa_shard_t *shard,
     edata_list_active_t *list, bool *deferred_work_generated);
 
 const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
@@ -46,7 +35,7 @@ hpa_supported(void) {
 		return false;
 	}
 	/*
-	 * We fundamentally rely on a address-space-hungry growth strategy for
+	 * We fundamentally rely on an address-space-hungry growth strategy for
 	 * hugepages.
 	 */
 	if (LG_SIZEOF_PTR != 3) {
@@ -109,17 +98,19 @@ hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
 	shard->stats.nhugifies = 0;
 	shard->stats.nhugify_failures = 0;
 	shard->stats.ndehugifies = 0;
-
-	/*
-	 * Fill these in last, so that if an hpa_shard gets used despite
-	 * initialization failing, we'll at least crash instead of just
-	 * operating on corrupted data.
-	 */
-	shard->pai.alloc = &hpa_alloc;
-	shard->pai.expand = &hpa_expand;
-	shard->pai.shrink = &hpa_shrink;
-	shard->pai.dalloc = &hpa_dalloc;
-	shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work;
+	memset(shard->stats.hpa_alloc_min_extents, 0,
+	    sizeof(shard->stats.hpa_alloc_min_extents));
+	memset(shard->stats.hpa_alloc_max_extents, 0,
+	    sizeof(shard->stats.hpa_alloc_max_extents));
+	memset(shard->stats.hpa_alloc_extents, 0,
+	    sizeof(shard->stats.hpa_alloc_extents));
+	memset(shard->stats.hpa_alloc_ps, 0, sizeof(shard->stats.hpa_alloc_ps));
+	memset(shard->stats.hpa_alloc_pages_per_ps, 0,
+	    sizeof(shard->stats.hpa_alloc_pages_per_ps));
+	memset(shard->stats.hpa_alloc_extents_per_ps, 0,
+	    sizeof(shard->stats.hpa_alloc_extents_per_ps));
+	memset(shard->stats.hpa_alloc_total_elapsed_ns_per_ps, 0,
+	    sizeof(shard->stats.hpa_alloc_total_elapsed_ns_per_ps));
 
 	err = sec_init(tsdn, &shard->sec, base, sec_opts);
 	if (err) {
@@ -145,6 +136,18 @@ hpa_shard_nonderived_stats_accum(
 	dst->nhugifies += src->nhugifies;
 	dst->nhugify_failures += src->nhugify_failures;
 	dst->ndehugifies += src->ndehugifies;
+	for (size_t i = 0; i <= SEC_MAX_NALLOCS; i++) {
+		dst->hpa_alloc_min_extents[i] += src->hpa_alloc_min_extents[i];
+		dst->hpa_alloc_max_extents[i] += src->hpa_alloc_max_extents[i];
+		dst->hpa_alloc_extents[i] += src->hpa_alloc_extents[i];
+		dst->hpa_alloc_ps[i] += src->hpa_alloc_ps[i];
+		dst->hpa_alloc_pages_per_ps[i] +=
+		    src->hpa_alloc_pages_per_ps[i];
+		dst->hpa_alloc_extents_per_ps[i] +=
+		    src->hpa_alloc_extents_per_ps[i];
+		dst->hpa_alloc_total_elapsed_ns_per_ps[i] +=
+		    src->hpa_alloc_total_elapsed_ns_per_ps[i];
+	}
 }
 
 void
@@ -651,37 +654,18 @@ hpa_shard_maybe_do_deferred_work(
 }
 
 static edata_t *
-hpa_try_alloc_one_no_grow(
-    tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) {
+hpa_try_alloc_one_offset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
+    hpdata_t *ps, hpdata_alloc_offset_t *alloc_offset, bool *oom) {
+	assert(*oom == false);
 	malloc_mutex_assert_owner(tsdn, &shard->mtx);
 
-	bool     err;
 	edata_t *edata = edata_cache_fast_get(tsdn, &shard->ecf);
 	if (edata == NULL) {
 		*oom = true;
 		return NULL;
 	}
 
-	hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
-	if (ps == NULL) {
-		edata_cache_fast_put(tsdn, &shard->ecf, edata);
-		return NULL;
-	}
-
-	psset_update_begin(&shard->psset, ps);
-
-	if (hpdata_empty(ps)) {
-		/*
-		 * If the pageslab used to be empty, treat it as though it's
-		 * brand new for fragmentation-avoidance purposes; what we're
-		 * trying to approximate is the age of the allocations *in* that
-		 * pageslab, and the allocations in the new pageslab are by
-		 * definition the youngest in this hpa shard.
-		 */
-		hpdata_age_set(ps, shard->age_counter++);
-	}
-
-	void *addr = hpdata_reserve_alloc(ps, size);
+	void *addr = hpdata_reserve_alloc_offset(ps, size, alloc_offset);
 	JE_USDT(hpa_alloc, 5, shard->ind, addr, size, hpdata_nactive_get(ps),
 	    hpdata_age_get(ps));
 	edata_init(edata, shard->ind, addr, size, /* slab */ false, SC_NSIZES,
@@ -693,12 +677,12 @@ hpa_try_alloc_one_no_grow(
 	/*
 	 * This could theoretically be moved outside of the critical section,
 	 * but that introduces the potential for a race.  Without the lock, the
-	 * (initially nonempty, since this is the reuse pathway) pageslab we
+     * (initially nonempty, since this is the reuse pathway) pageslab we
 	 * allocated out of could become otherwise empty while the lock is
 	 * dropped.  This would force us to deal with a pageslab eviction down
 	 * the error pathway, which is a pain.
 	 */
-	err = emap_register_boundary(
+	const bool err = emap_register_boundary(
 	    tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false);
 	if (err) {
 		hpdata_unreserve(
@@ -715,32 +699,118 @@ hpa_try_alloc_one_no_grow(
 		 * principle that we didn't *really* affect shard state (we
 		 * tweaked the stats, but our tweaks weren't really accurate).
 		 */
-		psset_update_end(&shard->psset, ps);
 		edata_cache_fast_put(tsdn, &shard->ecf, edata);
 		*oom = true;
 		return NULL;
 	}
 
-	hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
-	psset_update_end(&shard->psset, ps);
 	return edata;
 }
 
 static size_t
-hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
-    size_t size, bool *oom, size_t nallocs, edata_list_active_t *results,
+hpa_try_alloc_from_one_ps(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
+    size_t max_nallocs, bool *oom, edata_list_active_t *results,
     bool *deferred_work_generated) {
+	assert(size <= HUGEPAGE);
+	assert(size <= shard->opts.slab_max_alloc || size == sz_s2u(size));
+	assert(*oom == false);
 	malloc_mutex_assert_owner(tsdn, &shard->mtx);
+
+	nstime_t start;
+	nstime_init_update(&start);
+
+	hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
+	if (ps == NULL) {
+		return 0;
+	}
+
+	assert(max_nallocs <= SEC_MAX_NALLOCS);
+	hpdata_alloc_offset_t alloc_offsets[SEC_MAX_NALLOCS];
+	const size_t          nallocs = hpdata_find_alloc_offsets(
+            ps, size, alloc_offsets, max_nallocs);
+
+	psset_update_begin(&shard->psset, ps);
+
+	if (hpdata_empty(ps)) {
+		/*
+         * If the pageslab used to be empty, treat it as though it's
+		 * brand new for fragmentation-avoidance purposes; what we're
+		 * trying to approximate is the age of the allocations *in* that
+		 * pageslab, and the allocations in the new pageslab are by
+		 * definition the youngest in this hpa shard.
+		 */
+		hpdata_age_set(ps, shard->age_counter++);
+	}
+
 	size_t nsuccess = 0;
-	for (; nsuccess < nallocs; nsuccess++) {
-		edata_t *edata = hpa_try_alloc_one_no_grow(
-		    tsdn, shard, size, oom);
+	for (; nsuccess < nallocs; nsuccess += 1) {
+		edata_t *edata = hpa_try_alloc_one_offset(
+		    tsdn, shard, size, ps, (alloc_offsets + nsuccess), oom);
 		if (edata == NULL) {
 			break;
 		}
+
 		edata_list_active_append(results, edata);
 	}
 
+	hpdata_post_reserve_alloc_offsets(ps, size, alloc_offsets, nsuccess);
+	hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
+	psset_update_end(&shard->psset, ps);
+
+	const uint64_t elapsed_ns = nstime_ns_since(&start);
+	assert(nsuccess <= SEC_MAX_NALLOCS);
+	shard->stats.hpa_alloc_pages_per_ps[nsuccess] += nsuccess
+	    * (size >> LG_PAGE);
+	shard->stats.hpa_alloc_extents_per_ps[nsuccess] += 1;
+	shard->stats.hpa_alloc_total_elapsed_ns_per_ps[nsuccess] += elapsed_ns;
+
+	return nsuccess;
+}
+
+static size_t
+hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
+    size_t size, size_t min_nallocs, size_t max_nallocs,
+    bool update_min_max_stats, bool *oom, edata_list_active_t *results,
+    bool *deferred_work_generated) {
+	assert(*oom == false);
+	malloc_mutex_assert_owner(tsdn, &shard->mtx);
+
+	/*
+	 * As we require the shard mtx lock to update the stats,
+	 * we do the update the first time this function is called from
+	 * hpa_alloc_batch_psset().
+	 */
+	if (update_min_max_stats) {
+		assert(min_nallocs <= SEC_MAX_NALLOCS);
+		shard->stats.hpa_alloc_min_extents[min_nallocs] += 1;
+		assert(max_nallocs <= SEC_MAX_NALLOCS);
+		shard->stats.hpa_alloc_max_extents[max_nallocs] += 1;
+	}
+
+	size_t nsuccess = 0;
+	size_t ps_count = 0;
+	while (true) {
+		assert(1 <= min_nallocs);
+		assert(nsuccess < min_nallocs);
+		assert(min_nallocs <= max_nallocs);
+		const size_t nallocs = hpa_try_alloc_from_one_ps(tsdn, shard,
+		    size, max_nallocs - nsuccess, oom, results,
+		    deferred_work_generated);
+		if (nallocs == 0 || *oom) {
+			break;
+		}
+		nsuccess += nallocs;
+		ps_count += 1;
+		if (min_nallocs <= nsuccess) {
+			break;
+		}
+	}
+
+	assert(nsuccess <= SEC_MAX_NALLOCS);
+	shard->stats.hpa_alloc_extents[nsuccess] += 1;
+	assert(ps_count <= SEC_MAX_NALLOCS);
+	shard->stats.hpa_alloc_ps[ps_count] += 1;
+
 	hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
 	*deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard);
 	return nsuccess;
@@ -748,27 +818,26 @@ hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
 
 static size_t
 hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
-    bool *oom, size_t nallocs, edata_list_active_t *results,
-    bool *deferred_work_generated) {
+    size_t min_nallocs, size_t max_nallocs, bool update_min_max_stats,
+    bool *oom, edata_list_active_t *results, bool *deferred_work_generated) {
 	malloc_mutex_lock(tsdn, &shard->mtx);
-	size_t nsuccess = hpa_try_alloc_batch_no_grow_locked(
-	    tsdn, shard, size, oom, nallocs, results, deferred_work_generated);
+	const size_t nsuccess = hpa_try_alloc_batch_no_grow_locked(tsdn, shard,
+	    size, min_nallocs, max_nallocs, update_min_max_stats, oom, results,
+	    deferred_work_generated);
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 	return nsuccess;
 }
 
 static size_t
 hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
-    size_t nallocs, edata_list_active_t *results,
+    size_t min_nallocs, size_t max_nallocs, edata_list_active_t *results,
     bool *deferred_work_generated) {
-	assert(size <= HUGEPAGE);
-	assert(size <= shard->opts.slab_max_alloc || size == sz_s2u(size));
 	bool oom = false;
 
-	size_t nsuccess = hpa_try_alloc_batch_no_grow(
-	    tsdn, shard, size, &oom, nallocs, results, deferred_work_generated);
-
-	if (nsuccess == nallocs || oom) {
+	size_t nsuccess = hpa_try_alloc_batch_no_grow(tsdn, shard, size,
+	    min_nallocs, max_nallocs, /* update_min_max_stats */ true, &oom,
+	    results, deferred_work_generated);
+	if (min_nallocs <= nsuccess || oom) {
 		return nsuccess;
 	}
 
@@ -777,13 +846,18 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
 	 * try to grow.
 	 */
 	malloc_mutex_lock(tsdn, &shard->grow_mtx);
+
 	/*
 	 * Check for grow races; maybe some earlier thread expanded the psset
 	 * in between when we dropped the main mutex and grabbed the grow mutex.
 	 */
-	nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
-	    nallocs - nsuccess, results, deferred_work_generated);
-	if (nsuccess == nallocs || oom) {
+	assert(nsuccess < min_nallocs);
+	assert(min_nallocs <= max_nallocs);
+	nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size,
+	    min_nallocs - nsuccess, max_nallocs - nsuccess,
+	    /* update_min_max_stats */ false, &oom, results,
+	    deferred_work_generated);
+	if (min_nallocs <= nsuccess || oom) {
 		malloc_mutex_unlock(tsdn, &shard->grow_mtx);
 		return nsuccess;
 	}
@@ -807,28 +881,19 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
 	 */
 	malloc_mutex_lock(tsdn, &shard->mtx);
 	psset_insert(&shard->psset, ps);
-	nsuccess += hpa_try_alloc_batch_no_grow_locked(tsdn, shard, size, &oom,
-	    nallocs - nsuccess, results, deferred_work_generated);
+	assert(nsuccess < min_nallocs);
+	assert(min_nallocs <= max_nallocs);
+	nsuccess += hpa_try_alloc_batch_no_grow_locked(tsdn, shard, size,
+	    min_nallocs - nsuccess, max_nallocs - nsuccess,
+	    /* update_min_max_stats */ false, &oom, results,
+	    deferred_work_generated);
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 
-	/*
-	 * Drop grow_mtx before doing deferred work; other threads blocked on it
-	 * should be allowed to proceed while we're working.
-	 */
 	malloc_mutex_unlock(tsdn, &shard->grow_mtx);
 
 	return nsuccess;
 }
 
-static hpa_shard_t *
-hpa_from_pai(pai_t *self) {
-	assert(self->alloc == &hpa_alloc);
-	assert(self->expand == &hpa_expand);
-	assert(self->shrink == &hpa_shrink);
-	assert(self->dalloc == &hpa_dalloc);
-	return (hpa_shard_t *)self;
-}
-
 static void
 hpa_assert_results(
     tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *results) {
@@ -854,9 +919,10 @@ hpa_assert_results(
 	}
 }
 
-static edata_t *
-hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
-    bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
+edata_t *
+hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
+    bool zero, bool guarded, bool frequent_reuse,
+    bool *deferred_work_generated) {
 	assert((size & PAGE_MASK) == 0);
 	assert(!guarded);
 	witness_assert_depth_to_rank(
@@ -866,7 +932,6 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
 	if (alignment > PAGE || zero) {
 		return NULL;
 	}
-	hpa_shard_t *shard = hpa_from_pai(self);
 
 	/*
 	 * frequent_use here indicates this request comes from the arena bins,
@@ -886,13 +951,13 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
 	if (edata != NULL) {
 		return edata;
 	}
-	size_t              nallocs = sec_size_supported(&shard->sec, size)
-	                 ? shard->sec.opts.batch_fill_extra + 1
-	                 : 1;
 	edata_list_active_t results;
 	edata_list_active_init(&results);
-	size_t nsuccess = hpa_alloc_batch_psset(
-	    tsdn, shard, size, nallocs, &results, deferred_work_generated);
+	size_t min_nallocs, max_nallocs;
+	sec_calc_nallocs_for_size(
+	    &shard->sec, size, &min_nallocs, &max_nallocs);
+	size_t nsuccess = hpa_alloc_batch_psset(tsdn, shard, size, min_nallocs,
+	    max_nallocs, &results, deferred_work_generated);
 	hpa_assert_results(tsdn, shard, &results);
 	edata = edata_list_active_first(&results);
 
@@ -907,7 +972,7 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
 		/* Unlikely rollback in case of overfill */
 		if (!edata_list_active_empty(&results)) {
 			hpa_dalloc_batch(
-			    tsdn, self, &results, deferred_work_generated);
+			    tsdn, shard, &results, deferred_work_generated);
 		}
 	}
 	witness_assert_depth_to_rank(
@@ -915,15 +980,15 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
 	return edata;
 }
 
-static bool
-hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+bool
+hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
     size_t new_size, bool zero, bool *deferred_work_generated) {
 	/* Expand not yet supported. */
 	return true;
 }
 
-static bool
-hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+bool
+hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
     size_t new_size, bool *deferred_work_generated) {
 	/* Shrink not yet supported. */
 	return true;
@@ -982,10 +1047,8 @@ hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
 }
 
 static void
-hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
+hpa_dalloc_batch(tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *list,
     bool *deferred_work_generated) {
-	hpa_shard_t *shard = hpa_from_pai(self);
-
 	edata_t *edata;
 	ql_foreach (edata, &list->head, ql_link_active) {
 		hpa_dalloc_prepare_unlocked(tsdn, shard, edata);
@@ -1003,16 +1066,15 @@ hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 }
 
-static void
-hpa_dalloc(
-    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
+void
+hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
+    bool *deferred_work_generated) {
 	assert(!edata_guarded_get(edata));
 
 	edata_list_active_t dalloc_list;
 	edata_list_active_init(&dalloc_list);
 	edata_list_active_append(&dalloc_list, edata);
 
-	hpa_shard_t *shard = hpa_from_pai(self);
 	sec_dalloc(tsdn, &shard->sec, &dalloc_list);
 	if (edata_list_active_empty(&dalloc_list)) {
 		/* sec consumed the pointer */
@@ -1020,17 +1082,16 @@ hpa_dalloc(
 		return;
 	}
 	/* We may have more than one pointer to flush now */
-	hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
+	hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated);
 }
 
 /*
  * Calculate time until either purging or hugification ought to happen.
  * Called by background threads.
  */
-static uint64_t
-hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
-	hpa_shard_t *shard = hpa_from_pai(self);
-	uint64_t     time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
+uint64_t
+hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
+	uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
 
 	malloc_mutex_lock(tsdn, &shard->mtx);
 
@@ -1090,8 +1151,7 @@ hpa_sec_flush_impl(tsdn_t *tsdn, hpa_shard_t *shard) {
 
 	sec_flush(tsdn, &shard->sec, &to_flush);
 	bool deferred_work_generated;
-	hpa_dalloc_batch(
-	    tsdn, (pai_t *)shard, &to_flush, &deferred_work_generated);
+	hpa_dalloc_batch(tsdn, shard, &to_flush, &deferred_work_generated);
 }
 
 void

src/hpdata.c

@@ -170,6 +170,171 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
 	hpdata_assert_consistent(hpdata);
 }
 
+size_t
+hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t max_nallocs) {
+	hpdata_assert_consistent(hpdata);
+	assert((sz & PAGE_MASK) == 0);
+	assert(1 <= max_nallocs);
+	const size_t npages = sz >> LG_PAGE;
+	/* We should be able to find at least one allocation */
+	assert(npages <= hpdata_longest_free_range_get(hpdata));
+
+	size_t nallocs = 0;
+	size_t start = 0;
+	size_t longest_len = 0;
+	while (true) {
+		size_t begin = 0;
+		size_t len = 0;
+
+		const bool found = fb_urange_iter(
+		    hpdata->active_pages, HUGEPAGE_PAGES, start, &begin, &len);
+		if (!found) {
+			/* we should have found at least one */
+			assert(0 < nallocs);
+			break;
+		}
+
+		/* carve up the free range, if it's large enough */
+		while (npages <= len) {
+			offsets->len_before = len;
+			offsets->index = begin;
+			offsets->longest_len = longest_len;
+			offsets += 1;
+
+			nallocs += 1;
+			if (nallocs == max_nallocs) {
+				/* cause start to be == HUGEPAGE_PAGES to break out of the outer loop */
+				begin = HUGEPAGE_PAGES;
+				len = 0;
+				break;
+			}
+
+			begin += npages;
+			len -= npages;
+		}
+
+		start = begin + len;
+		assert(start <= HUGEPAGE_PAGES);
+		if (start == HUGEPAGE_PAGES) {
+			break;
+		}
+		longest_len = max_zu(longest_len, len);
+	}
+
+	/* post-conditions */
+	assert(1 <= nallocs);
+	assert(nallocs <= max_nallocs);
+
+	return nallocs;
+}
+
+void *
+hpdata_reserve_alloc_offset(
+    hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset) {
+	/*
+	 * This is a metadata change; the hpdata should therefore either not be
+	 * in the psset, or should have explicitly marked itself as being
+	 * mid-update.
+	 */
+	assert(!hpdata->h_in_psset || hpdata->h_updating);
+	assert(hpdata->h_alloc_allowed);
+	assert((sz & PAGE_MASK) == 0);
+	const size_t npages = sz >> LG_PAGE;
+	const size_t index = offset->index;
+
+	fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, index, npages);
+	hpdata->h_nactive += npages;
+
+	/*
+	 * We might be about to dirty some memory for the first time; update our
+	 * count if so.
+	 */
+	size_t new_dirty = fb_ucount(
+	    hpdata->touched_pages, HUGEPAGE_PAGES, index, npages);
+	fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, index, npages);
+	hpdata->h_ntouched += new_dirty;
+
+	return (void *)((byte_t *)hpdata_addr_get(hpdata) + (index << LG_PAGE));
+}
+
+void
+hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t nallocs) {
+	assert((sz & PAGE_MASK) == 0);
+	const size_t npages = sz >> LG_PAGE;
+
+	if (nallocs == 0) {
+		return;
+	}
+
+	size_t max_len = offsets[0].len_before;
+	for (size_t i = 1; i < nallocs; i += 1) {
+		max_len = max_zu(max_len, offsets[i].len_before);
+	}
+
+	const size_t prev_longest = hpdata_longest_free_range_get(hpdata);
+	assert(max_len <= prev_longest);
+	if (max_len < prev_longest) {
+		/* no need to update the hpdata longest range */
+		return;
+	}
+
+	/*
+	 * If we allocated out of a range that was the longest in the hpdata, it
+	 * might be the only one of that size and we'll have to adjust the
+	 * metadata.
+	 */
+	const size_t len_before = offsets[nallocs - 1].len_before;
+	size_t       start = offsets[nallocs - 1].index + len_before;
+	size_t       longest_len = max_zu(
+            offsets[nallocs - 1].longest_len, len_before - npages);
+
+	const size_t rest = HUGEPAGE_PAGES - start;
+	/*
+	 * Only look at the rest if we think we'll find a range longer than what
+	 * we already have. This also implicitly checks for rest == 0, so we don't
+	 * have to check before the first call to fb_urange_iter().
+	 */
+	if (longest_len < rest) {
+		while (true) {
+			size_t begin = 0;
+			size_t len = 0;
+
+			const bool found = fb_urange_iter(hpdata->active_pages,
+			    HUGEPAGE_PAGES, start, &begin, &len);
+			if (!found) {
+				break;
+			}
+
+			if (longest_len < len) {
+				if (len == prev_longest) {
+					/* it's already set to the right value */
+					assert(hpdata_longest_free_range_get(
+					           hpdata)
+					    == len);
+					assert(fb_urange_longest(
+					           hpdata->active_pages,
+					           HUGEPAGE_PAGES)
+					    == len);
+					return;
+				}
+				longest_len = len;
+			}
+
+			start = begin + len;
+			assert(start <= HUGEPAGE_PAGES);
+			if (start == HUGEPAGE_PAGES) {
+				break;
+			}
+		}
+	}
+
+	assert(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES)
+	    == longest_len);
+	hpdata_longest_free_range_set(hpdata, longest_len);
+}
+
 size_t
 hpdata_purge_begin(
     hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t *nranges) {

src/inspect.c

@@ -53,8 +53,7 @@ inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
 	assert(*nfree <= *nregs);
 	assert(*nfree * edata_usize_get(edata) <= *size);
 
-	arena_t *arena = (arena_t *)atomic_load_p(
-	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
+	arena_t *arena = arena_get_from_edata(edata);
 	assert(arena != NULL);
 	const unsigned binshard = edata_binshard_get(edata);
 	bin_t         *bin = arena_get_bin(arena, szind, binshard);

src/jemalloc_cpp.cpp

@@ -1,4 +1,4 @@
-#include <mutex>
+#include <exception>
 #include <new>
 // NOLINTBEGIN(misc-use-anonymous-namespace)
 
@@ -78,29 +78,30 @@ handleOOM(std::size_t size, bool nothrow) {
 	void *ptr = nullptr;
 
 	while (ptr == nullptr) {
-		std::new_handler handler;
-		// GCC-4.8 and clang 4.0 do not have std::get_new_handler.
-		{
-			static std::mutex           mtx;
-			std::lock_guard<std::mutex> lock(mtx);
-
-			handler = std::set_new_handler(nullptr);
-			std::set_new_handler(handler);
-		}
+		std::new_handler handler = std::get_new_handler();
 		if (handler == nullptr)
 			break;
 
+#ifdef JEMALLOC_HAVE_CXX_EXCEPTIONS
 		try {
 			handler();
 		} catch (const std::bad_alloc &) {
 			break;
 		}
+#else
+		handler();
+#endif
 
 		ptr = je_malloc(size);
 	}
 
-	if (ptr == nullptr && !nothrow)
-		std::__throw_bad_alloc();
+	if (ptr == nullptr && !nothrow) {
+#ifdef JEMALLOC_HAVE_CXX_EXCEPTIONS
+		throw std::bad_alloc();
+#else
+		std::terminate();
+#endif
+	}
 	return ptr;
 }
 

src/jemalloc_fork.c

@@ -0,0 +1,182 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/arenas_management.h"
+#include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/jemalloc_fork.h"
+#include "jemalloc/internal/jemalloc_init.h"
+
+/******************************************************************************/
+/*
+ * The following functions are used by threading libraries for protection of
+ * malloc during fork().
+ */
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+/*
+ * When JEMALLOC_MUTEX_INIT_CB is defined, pthread_atfork registration is
+ * skipped and the platform calls _malloc_prefork/_malloc_postfork directly.
+ * FreeBSD's libthr calls _malloc_postfork in both parent and child.  Detect
+ * the child by pid change so we route to jemalloc_postfork_child, which resets
+ * per-arena state the parent handler does not touch (nthreads, descriptor
+ * queues).  The check is harmless on any platform that only calls
+ * _malloc_postfork in the parent.
+ */
+static pid_t jemalloc_prefork_pid;
+#endif
+
+#ifndef JEMALLOC_MUTEX_INIT_CB
+void
+jemalloc_prefork(void)
+#else
+JEMALLOC_EXPORT void
+_malloc_prefork(void)
+#endif
+{
+	tsd_t   *tsd;
+	unsigned i, j, narenas;
+	arena_t *arena;
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+	if (!malloc_initialized()) {
+		return;
+	}
+#endif
+	assert(malloc_initialized());
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+	jemalloc_prefork_pid = getpid();
+#endif
+
+	tsd = tsd_fetch();
+
+	narenas = narenas_total_get();
+
+	witness_prefork(tsd_witness_tsdp_get(tsd));
+	/* Acquire all mutexes in a safe order. */
+	ctl_prefork(tsd_tsdn(tsd));
+	tcache_prefork(tsd_tsdn(tsd));
+	arenas_management_prefork(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_prefork0(tsd_tsdn(tsd));
+	}
+	prof_prefork0(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_prefork1(tsd_tsdn(tsd));
+	}
+	/* Break arena prefork into stages to preserve lock order. */
+	for (i = 0; i < 9; i++) {
+		for (j = 0; j < narenas; j++) {
+			if ((arena = arena_get(tsd_tsdn(tsd), j, false))
+			    != NULL) {
+				switch (i) {
+				case 0:
+					arena_prefork0(tsd_tsdn(tsd), arena);
+					break;
+				case 1:
+					arena_prefork1(tsd_tsdn(tsd), arena);
+					break;
+				case 2:
+					arena_prefork2(tsd_tsdn(tsd), arena);
+					break;
+				case 3:
+					arena_prefork3(tsd_tsdn(tsd), arena);
+					break;
+				case 4:
+					arena_prefork4(tsd_tsdn(tsd), arena);
+					break;
+				case 5:
+					arena_prefork5(tsd_tsdn(tsd), arena);
+					break;
+				case 6:
+					arena_prefork6(tsd_tsdn(tsd), arena);
+					break;
+				case 7:
+					arena_prefork7(tsd_tsdn(tsd), arena);
+					break;
+				case 8:
+					arena_prefork8(tsd_tsdn(tsd), arena);
+					break;
+				default:
+					not_reached();
+				}
+			}
+		}
+	}
+	prof_prefork1(tsd_tsdn(tsd));
+	stats_prefork(tsd_tsdn(tsd));
+}
+
+#ifndef JEMALLOC_MUTEX_INIT_CB
+void
+jemalloc_postfork_parent(void)
+#else
+JEMALLOC_EXPORT void
+_malloc_postfork(void)
+#endif
+{
+	tsd_t   *tsd;
+	unsigned i, narenas;
+
+#ifdef JEMALLOC_MUTEX_INIT_CB
+	if (!malloc_initialized()) {
+		return;
+	}
+	if (getpid() != jemalloc_prefork_pid) {
+		jemalloc_postfork_child();
+		return;
+	}
+#endif
+	assert(malloc_initialized());
+
+	tsd = tsd_fetch();
+
+	witness_postfork_parent(tsd_witness_tsdp_get(tsd));
+	/* Release all mutexes, now that fork() has completed. */
+	stats_postfork_parent(tsd_tsdn(tsd));
+	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
+		arena_t *arena;
+
+		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
+			arena_postfork_parent(tsd_tsdn(tsd), arena);
+		}
+	}
+	prof_postfork_parent(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_postfork_parent(tsd_tsdn(tsd));
+	}
+	arenas_management_postfork_parent(tsd_tsdn(tsd));
+	tcache_postfork_parent(tsd_tsdn(tsd));
+	ctl_postfork_parent(tsd_tsdn(tsd));
+}
+
+void
+jemalloc_postfork_child(void) {
+	tsd_t   *tsd;
+	unsigned i, narenas;
+
+	assert(malloc_initialized());
+
+	tsd = tsd_fetch();
+
+	witness_postfork_child(tsd_witness_tsdp_get(tsd));
+	/* Release all mutexes, now that fork() has completed. */
+	stats_postfork_child(tsd_tsdn(tsd));
+	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
+		arena_t *arena;
+
+		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
+			cache_bin_array_descriptor_t *desc =
+			    tcache_postfork_arena_descriptor(
+			        tsd_tsdn(tsd), arena);
+			arena_postfork_child(tsd_tsdn(tsd), arena, desc);
+		}
+	}
+	prof_postfork_child(tsd_tsdn(tsd));
+	if (have_background_thread) {
+		background_thread_postfork_child(tsd_tsdn(tsd));
+	}
+	arenas_management_postfork_child(tsd_tsdn(tsd));
+	tcache_postfork_child(tsd_tsdn(tsd));
+	ctl_postfork_child(tsd_tsdn(tsd));
+}

src/jemalloc_init.c

@@ -0,0 +1,700 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/arenas_management.h"
+#include "jemalloc/internal/conf.h"
+#include "jemalloc/internal/ctl.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/extent_dss.h"
+#include "jemalloc/internal/extent_mmap.h"
+#include "jemalloc/internal/jemalloc_fork.h"
+#include "jemalloc/internal/jemalloc_init.h"
+#include "jemalloc/internal/malloc_io.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/san.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/spin.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/thread_event.h"
+
+#ifdef JEMALLOC_THREADED_INIT
+/* Used to let the initializing thread recursively allocate. */
+#	define NO_INITIALIZER ((unsigned long)0)
+#	define INITIALIZER pthread_self()
+static pthread_t malloc_initializer = NO_INITIALIZER;
+#else
+#	define NO_INITIALIZER false
+#	define INITIALIZER true
+static bool malloc_initializer = NO_INITIALIZER;
+#endif
+
+bool
+malloc_is_initializer(void) {
+#ifdef JEMALLOC_THREADED_INIT
+	return pthread_equal(malloc_initializer, pthread_self());
+#else
+	return malloc_initializer;
+#endif
+}
+
+#ifdef JEMALLOC_THREADED_INIT
+static bool
+malloc_initializer_is_set(void) {
+	return malloc_initializer != NO_INITIALIZER;
+}
+#endif
+
+static void
+malloc_initializer_set(void) {
+	malloc_initializer = INITIALIZER;
+}
+
+/* Used to avoid initialization races. */
+#ifdef _WIN32
+#	if _WIN32_WINNT >= 0x0600
+static malloc_mutex_t init_lock = SRWLOCK_INIT;
+#	else
+static malloc_mutex_t init_lock;
+static bool           init_lock_initialized = false;
+
+JEMALLOC_ATTR(constructor)
+static void WINAPI
+_init_init_lock(void) {
+	/*
+	 * If another constructor in the same binary is using mallctl to e.g.
+	 * set up extent hooks, it may end up running before this one, and
+	 * malloc_init_hard will crash trying to lock the uninitialized lock. So
+	 * we force an initialization of the lock in malloc_init_hard as well.
+	 * We don't try to care about atomicity of the accessed to the
+	 * init_lock_initialized boolean, since it really only matters early in
+	 * the process creation, before any separate thread normally starts
+	 * doing anything.
+	 */
+	if (!init_lock_initialized) {
+		malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT,
+		    malloc_mutex_rank_exclusive);
+	}
+	init_lock_initialized = true;
+}
+
+#		ifdef _MSC_VER
+#			pragma section(".CRT$XCU", read)
+JEMALLOC_SECTION(".CRT$XCU")
+JEMALLOC_ATTR(used)
+static const void(WINAPI *init_init_lock)(void) = _init_init_lock;
+#		endif
+#	endif
+#else
+static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
+#endif
+
+malloc_init_t malloc_init_state = malloc_init_uninitialized;
+
+/* When malloc_slow is true, set the corresponding bits for sanity check. */
+enum {
+	flag_opt_junk_alloc = (1U),
+	flag_opt_junk_free = (1U << 1),
+	flag_opt_zero = (1U << 2),
+	flag_opt_utrace = (1U << 3),
+	flag_opt_xmalloc = (1U << 4)
+};
+static uint8_t malloc_slow_flags;
+
+static void
+malloc_slow_flag_init(void) {
+	/*
+	 * Combine the runtime options into malloc_slow for fast path.  Called
+	 * after bootstrap is complete.
+	 */
+	malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
+	    | (opt_junk_free ? flag_opt_junk_free : 0)
+	    | (opt_zero ? flag_opt_zero : 0)
+	    | (opt_utrace ? flag_opt_utrace : 0)
+	    | (opt_xmalloc ? flag_opt_xmalloc : 0);
+
+	malloc_slow = (malloc_slow_flags != 0);
+}
+
+static void stats_print_atexit(void);
+static unsigned malloc_ncpus(void);
+static bool malloc_cpu_count_is_deterministic(void);
+
+static bool
+malloc_init_hard_needed(void) {
+	if (malloc_initialized()
+	    || (malloc_is_initializer()
+	        && malloc_init_state == malloc_init_recursible)) {
+		/*
+		 * Another thread initialized the allocator before this one
+		 * acquired init_lock, or this thread is the initializing
+		 * thread, and it is recursively allocating.
+		 */
+		return false;
+	}
+#ifdef JEMALLOC_THREADED_INIT
+	if (malloc_initializer_is_set() && !malloc_is_initializer()) {
+		/* Busy-wait until the initializing thread completes. */
+		spin_t spinner = SPIN_INITIALIZER;
+		do {
+			malloc_mutex_unlock(TSDN_NULL, &init_lock);
+			spin_adaptive(&spinner);
+			malloc_mutex_lock(TSDN_NULL, &init_lock);
+		} while (!malloc_initialized());
+		return false;
+	}
+#endif
+	return true;
+}
+
+static bool
+malloc_init_hard_a0_locked(void) {
+	malloc_initializer_set();
+
+	JEMALLOC_DIAGNOSTIC_PUSH
+	JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
+	sc_data_t sc_data = {0};
+	JEMALLOC_DIAGNOSTIC_POP
+
+	/*
+	 * Ordering here is somewhat tricky; we need sc_boot() first, since that
+	 * determines what the size classes will be, and then
+	 * malloc_conf_init(), since any slab size tweaking will need to be done
+	 * before sz_boot and bin_info_boot, which assume that the values they
+	 * read out of sc_data_global are final.
+	 */
+	sc_boot(&sc_data);
+	unsigned bin_shard_sizes[SC_NBINS];
+	bin_shard_sizes_boot(bin_shard_sizes);
+	/*
+	 * prof_boot0 only initializes opt_prof_prefix.  We need to do it before
+	 * we parse malloc_conf options, in case malloc_conf parsing overwrites
+	 * it.
+	 */
+	if (config_prof) {
+		prof_boot0();
+	}
+	char readlink_buf[PATH_MAX + 1];
+	readlink_buf[0] = '\0';
+	malloc_conf_init(&sc_data, bin_shard_sizes, readlink_buf);
+	san_init(opt_lg_san_uaf_align);
+	sz_boot(&sc_data, opt_cache_oblivious);
+	bin_info_boot(&sc_data, bin_shard_sizes);
+
+	if (opt_stats_print) {
+		/* Print statistics at exit. */
+		if (atexit(stats_print_atexit) != 0) {
+			malloc_write("<jemalloc>: Error in atexit()\n");
+			if (opt_abort) {
+				abort();
+			}
+		}
+	}
+
+	if (stats_boot()) {
+		return true;
+	}
+	if (pages_boot()) {
+		return true;
+	}
+	if (base_boot(TSDN_NULL)) {
+		return true;
+	}
+	/* emap_global is static, hence zeroed. */
+	if (emap_init(&arena_emap_global, b0get(), /* zeroed */ true)) {
+		return true;
+	}
+	if (extent_boot()) {
+		return true;
+	}
+	if (ctl_boot()) {
+		return true;
+	}
+	if (config_prof) {
+		prof_boot1();
+	}
+	if (opt_hpa && !hpa_supported()) {
+		malloc_printf(
+		    "<jemalloc>: HPA not supported in the current "
+		    "configuration; %s.",
+		    opt_abort_conf ? "aborting" : "disabling");
+		if (opt_abort_conf) {
+			malloc_abort_invalid_conf();
+		} else {
+			opt_hpa = false;
+		}
+	}
+	if (arena_boot(&sc_data, b0get(), opt_hpa)) {
+		return true;
+	}
+	if (tcache_boot(TSDN_NULL, b0get())) {
+		return true;
+	}
+	if (arenas_management_boot()) {
+		return true;
+	}
+	experimental_thread_events_boot();
+	/*
+	 * Create enough scaffolding to allow recursive allocation in
+	 * malloc_ncpus().
+	 */
+	narenas_auto_set(1);
+	manual_arena_base_set(narenas_auto + 1);
+	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
+	/*
+	 * Initialize one arena here.  The rest are lazily created in
+	 * arena_choose_hard().
+	 */
+	if (arena_init(TSDN_NULL, 0, &arena_config_default) == NULL) {
+		return true;
+	}
+
+	if (opt_hpa && !hpa_supported()) {
+		malloc_printf(
+		    "<jemalloc>: HPA not supported in the current "
+		    "configuration; %s.",
+		    opt_abort_conf ? "aborting" : "disabling");
+		if (opt_abort_conf) {
+			malloc_abort_invalid_conf();
+		} else {
+			opt_hpa = false;
+		}
+	}
+
+	malloc_init_state = malloc_init_a0_initialized;
+
+	size_t buf_len = strlen(readlink_buf);
+	if (buf_len > 0) {
+		void *readlink_allocated = a0malloc(buf_len + 1);
+		if (readlink_allocated != NULL) {
+			memcpy(readlink_allocated, readlink_buf, buf_len + 1);
+			opt_malloc_conf_symlink = readlink_allocated;
+		}
+	}
+
+	return false;
+}
+
+bool
+malloc_init_hard_a0(void) {
+	bool ret;
+
+	malloc_mutex_lock(TSDN_NULL, &init_lock);
+	ret = malloc_init_hard_a0_locked();
+	malloc_mutex_unlock(TSDN_NULL, &init_lock);
+	return ret;
+}
+
+static void
+stats_print_atexit(void) {
+	if (config_stats) {
+		tsdn_t  *tsdn;
+		unsigned narenas, i;
+
+		tsdn = tsdn_fetch();
+
+		/*
+		 * Merge stats from extant threads.  This is racy, since
+		 * individual threads do not lock when recording tcache stats
+		 * events.  As a consequence, the final stats may be slightly
+		 * out of date by the time they are reported, if other threads
+		 * continue to allocate.
+		 */
+		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
+			arena_t *arena = arena_get(tsdn, i, false);
+			if (arena != NULL) {
+				arena_cache_bins_stats_merge(tsdn, arena);
+			}
+		}
+	}
+	je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
+}
+
+static unsigned
+malloc_ncpus(void) {
+	long result;
+
+#ifdef _WIN32
+	SYSTEM_INFO si;
+	GetSystemInfo(&si);
+	result = si.dwNumberOfProcessors;
+#elif defined(CPU_COUNT)
+	/*
+	 * glibc >= 2.6 has the CPU_COUNT macro.
+	 *
+	 * glibc's sysconf() uses isspace().  glibc allocates for the first time
+	 * *before* setting up the isspace tables.  Therefore we need a
+	 * different method to get the number of CPUs.
+	 *
+	 * The getaffinity approach is also preferred when only a subset of CPUs
+	 * is available, to avoid using more arenas than necessary.
+	 */
+	{
+#	if defined(__FreeBSD__) || defined(__DragonFly__)
+		cpuset_t set;
+#	else
+		cpu_set_t set;
+#	endif
+#	if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
+		sched_getaffinity(0, sizeof(set), &set);
+#	else
+		pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
+#	endif
+		result = CPU_COUNT(&set);
+	}
+#else
+	result = sysconf(_SC_NPROCESSORS_ONLN);
+#endif
+	return ((result == -1) ? 1 : (unsigned)result);
+}
+
+/*
+ * Ensure that number of CPUs is determistinc, i.e. it is the same based on:
+ * - sched_getaffinity()
+ * - _SC_NPROCESSORS_ONLN
+ * - _SC_NPROCESSORS_CONF
+ * Since otherwise tricky things is possible with percpu arenas in use.
+ */
+static bool
+malloc_cpu_count_is_deterministic(void) {
+#ifdef _WIN32
+	return true;
+#else
+	long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
+	long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
+	if (cpu_onln != cpu_conf) {
+		return false;
+	}
+#	if defined(CPU_COUNT)
+#		if defined(__FreeBSD__) || defined(__DragonFly__)
+	cpuset_t set;
+#		else
+	cpu_set_t set;
+#		endif /* __FreeBSD__ */
+#		if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
+	sched_getaffinity(0, sizeof(set), &set);
+#		else  /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
+	pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
+#		endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
+	long cpu_affinity = CPU_COUNT(&set);
+	if (cpu_affinity != cpu_conf) {
+		return false;
+	}
+#	endif         /* CPU_COUNT */
+	return true;
+#endif
+}
+
+/* Initialize data structures which may trigger recursive allocation. */
+static bool
+malloc_init_hard_recursible(void) {
+	malloc_init_state = malloc_init_recursible;
+
+	ncpus = malloc_ncpus();
+	if (opt_percpu_arena != percpu_arena_disabled) {
+		bool cpu_count_is_deterministic =
+		    malloc_cpu_count_is_deterministic();
+		if (!cpu_count_is_deterministic) {
+			/*
+			 * If # of CPU is not deterministic, and narenas not
+			 * specified, disables per cpu arena since it may not
+			 * detect CPU IDs properly.
+			 */
+			if (opt_narenas == 0) {
+				opt_percpu_arena = percpu_arena_disabled;
+				malloc_write(
+				    "<jemalloc>: Number of CPUs "
+				    "detected is not deterministic. Per-CPU "
+				    "arena disabled.\n");
+				if (opt_abort_conf) {
+					malloc_abort_invalid_conf();
+				}
+				if (opt_abort) {
+					abort();
+				}
+			}
+		}
+	}
+
+#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
+    && !defined(JEMALLOC_ZONE) && !defined(_WIN32)                             \
+    && !defined(__native_client__))
+	/* LinuxThreads' pthread_atfork() allocates. */
+	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
+	        jemalloc_postfork_child)
+	    != 0) {
+		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
+		if (opt_abort) {
+			abort();
+		}
+		return true;
+	}
+#endif
+
+	if (background_thread_boot0()) {
+		return true;
+	}
+
+	return false;
+}
+
+static unsigned
+malloc_narenas_default(void) {
+	assert(ncpus > 0);
+	/*
+	 * For SMP systems, create more than one arena per CPU by
+	 * default.
+	 */
+	if (ncpus > 1) {
+		fxp_t    fxp_ncpus = FXP_INIT_INT(ncpus);
+		fxp_t    goal = fxp_mul(fxp_ncpus, opt_narenas_ratio);
+		uint32_t int_goal = fxp_round_nearest(goal);
+		if (int_goal == 0) {
+			return 1;
+		}
+		return int_goal;
+	} else {
+		return 1;
+	}
+}
+
+static percpu_arena_mode_t
+percpu_arena_as_initialized(percpu_arena_mode_t mode) {
+	assert(!malloc_initialized());
+	assert(mode <= percpu_arena_disabled);
+
+	if (mode != percpu_arena_disabled) {
+		mode += percpu_arena_mode_enabled_base;
+	}
+
+	return mode;
+}
+
+static bool
+malloc_init_narenas(tsdn_t *tsdn) {
+	assert(ncpus > 0);
+
+	if (opt_percpu_arena != percpu_arena_disabled) {
+		if (!have_percpu_arena || malloc_getcpu() < 0) {
+			opt_percpu_arena = percpu_arena_disabled;
+			malloc_printf(
+			    "<jemalloc>: perCPU arena getcpu() not "
+			    "available. Setting narenas to %u.\n",
+			    opt_narenas ? opt_narenas
+			                : malloc_narenas_default());
+			if (opt_abort) {
+				abort();
+			}
+		} else {
+			if (ncpus >= MALLOCX_ARENA_LIMIT) {
+				malloc_printf(
+				    "<jemalloc>: narenas w/ percpu"
+				    "arena beyond limit (%d)\n",
+				    ncpus);
+				if (opt_abort) {
+					abort();
+				}
+				return true;
+			}
+			/* NB: opt_percpu_arena isn't fully initialized yet. */
+			if (percpu_arena_as_initialized(opt_percpu_arena)
+			        == per_phycpu_arena
+			    && ncpus % 2 != 0) {
+				malloc_printf(
+				    "<jemalloc>: invalid "
+				    "configuration -- per physical CPU arena "
+				    "with odd number (%u) of CPUs (no hyper "
+				    "threading?).\n",
+				    ncpus);
+				if (opt_abort)
+					abort();
+			}
+			unsigned n = percpu_arena_ind_limit(
+			    percpu_arena_as_initialized(opt_percpu_arena));
+			if (opt_narenas < n) {
+				/*
+				 * If narenas is specified with percpu_arena
+				 * enabled, actual narenas is set as the greater
+				 * of the two. percpu_arena_choose will be free
+				 * to use any of the arenas based on CPU
+				 * id. This is conservative (at a small cost)
+				 * but ensures correctness.
+				 *
+				 * If for some reason the ncpus determined at
+				 * boot is not the actual number (e.g. because
+				 * of affinity setting from numactl), reserving
+				 * narenas this way provides a workaround for
+				 * percpu_arena.
+				 */
+				opt_narenas = n;
+			}
+		}
+	}
+	if (opt_narenas == 0) {
+		opt_narenas = malloc_narenas_default();
+	}
+	assert(opt_narenas > 0);
+
+	narenas_auto_set(opt_narenas);
+	/*
+	 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
+	 */
+	if (narenas_auto >= MALLOCX_ARENA_LIMIT) {
+		narenas_auto_set(MALLOCX_ARENA_LIMIT - 1);
+		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
+		    narenas_auto);
+	}
+	narenas_total_set(narenas_auto);
+	if (arena_init_huge(tsdn, arena_get(tsdn, 0, false))) {
+		narenas_total_inc();
+	}
+	manual_arena_base_set(narenas_total_get());
+
+	return false;
+}
+
+static void
+malloc_init_percpu(void) {
+	opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena);
+}
+
+static bool
+malloc_init_hard_finish(void) {
+	if (malloc_mutex_boot()) {
+		return true;
+	}
+
+	malloc_init_state = malloc_init_initialized;
+	malloc_slow_flag_init();
+
+	return false;
+}
+
+static void
+malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) {
+	malloc_mutex_assert_owner(tsdn, &init_lock);
+	malloc_mutex_unlock(tsdn, &init_lock);
+	if (reentrancy_set) {
+		assert(!tsdn_null(tsdn));
+		tsd_t *tsd = tsdn_tsd(tsdn);
+		assert(tsd_reentrancy_level_get(tsd) > 0);
+		post_reentrancy(tsd);
+	}
+}
+
+bool
+malloc_init_hard(void) {
+	tsd_t *tsd;
+
+	assert(TCACHE_MAXCLASS_LIMIT <= USIZE_GROW_SLOW_THRESHOLD);
+	assert(SC_LOOKUP_MAXCLASS <= USIZE_GROW_SLOW_THRESHOLD);
+	/*
+	 * This asserts an extreme case where TINY_MAXCLASS is larger
+	 * than LARGE_MINCLASS.  It could only happen if some constants
+	 * are configured miserably wrong.
+	 */
+	assert(SC_NTINY == 0 || SC_LG_TINY_MAXCLASS <= SC_LG_LARGE_MINCLASS);
+
+#if defined(_WIN32) && _WIN32_WINNT < 0x0600
+	_init_init_lock();
+#endif
+	malloc_mutex_lock(TSDN_NULL, &init_lock);
+
+#define UNLOCK_RETURN(tsdn, ret, reentrancy)                                   \
+	malloc_init_hard_cleanup(tsdn, reentrancy);                            \
+	return ret;
+
+	if (!malloc_init_hard_needed()) {
+		UNLOCK_RETURN(TSDN_NULL, false, false)
+	}
+
+	if (malloc_init_state != malloc_init_a0_initialized
+	    && malloc_init_hard_a0_locked()) {
+		UNLOCK_RETURN(TSDN_NULL, true, false)
+	}
+
+	malloc_mutex_unlock(TSDN_NULL, &init_lock);
+	/* Recursive allocation relies on functional tsd. */
+	tsd = malloc_tsd_boot0();
+	if (tsd == NULL) {
+		return true;
+	}
+	if (malloc_init_hard_recursible()) {
+		return true;
+	}
+
+	malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
+	/* Set reentrancy level to 1 during init. */
+	pre_reentrancy(tsd, NULL);
+	/* Initialize narenas before prof_boot2 (for allocation). */
+	if (malloc_init_narenas(tsd_tsdn(tsd))
+	    || background_thread_boot1(tsd_tsdn(tsd), b0get())) {
+		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
+	}
+	if (opt_hpa) {
+		/*
+		 * We didn't initialize arena 0 hpa_shard in arena_new, because
+		 * background_thread_enabled wasn't initialized yet, but we
+		 * need it to set correct value for deferral_allowed.
+		 */
+		arena_t         *a0 = arena_get(tsd_tsdn(tsd), 0, false);
+		hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
+		hpa_shard_opts.deferral_allowed = background_thread_enabled();
+		if (pa_shard_enable_hpa(tsd_tsdn(tsd), &a0->pa_shard,
+		        &hpa_shard_opts, &opt_hpa_sec_opts)) {
+			UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
+		}
+	}
+	if (config_prof && prof_boot2(tsd, b0get())) {
+		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
+	}
+
+	malloc_init_percpu();
+
+	if (malloc_init_hard_finish()) {
+		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
+	}
+	post_reentrancy(tsd);
+	malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
+
+	witness_assert_lockless(
+	    witness_tsd_tsdn(tsd_witness_tsdp_get_unsafe(tsd)));
+	malloc_tsd_boot1();
+	/* Update TSD after tsd_boot1. */
+	tsd = tsd_fetch();
+	if (opt_background_thread) {
+		assert(have_background_thread);
+		/*
+		 * Need to finish init & unlock first before creating background
+		 * threads (pthread_create depends on malloc).  ctl_init (which
+		 * sets isthreaded) needs to be called without holding any lock.
+		 */
+		background_thread_ctl_init(tsd_tsdn(tsd));
+		if (background_thread_create(tsd, 0)) {
+			return true;
+		}
+	}
+#undef UNLOCK_RETURN
+	return false;
+}
+
+/*
+ * If an application creates a thread before doing any allocation in the main
+ * thread, then calls fork(2) in the main thread followed by memory allocation
+ * in the child process, a race can occur that results in deadlock within the
+ * child: the main thread may have forked while the created thread had
+ * partially initialized the allocator.  Ordinarily jemalloc prevents
+ * fork/malloc races via the following functions it registers during
+ * initialization using pthread_atfork(), but of course that does no good if
+ * the allocator isn't fully initialized at fork time.  The following library
+ * constructor is a partial solution to this problem.  It may still be possible
+ * to trigger the deadlock described above, but doing so would involve forking
+ * via a library constructor that runs before jemalloc's runs.
+ */
+#ifndef JEMALLOC_JET
+JEMALLOC_ATTR(constructor)
+static void
+jemalloc_constructor(void) {
+	malloc_init();
+}
+#endif

src/large.c

@@ -184,8 +184,7 @@ large_ralloc_move_helper(
 
 void *
 large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
-    size_t alignment, bool zero, tcache_t *tcache,
-    hook_ralloc_args_t *hook_args) {
+    size_t alignment, bool zero, tcache_t *tcache) {
 	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
 
 	size_t oldusize = edata_usize_get(edata);
@@ -196,9 +195,6 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
 
 	/* Try to avoid moving the allocation. */
 	if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) {
-		hook_invoke_expand(hook_args->is_realloc ? hook_expand_realloc
-		                                         : hook_expand_rallocx,
-		    ptr, oldusize, usize, (uintptr_t)ptr, hook_args->args);
 		return edata_addr_get(edata);
 	}
 
@@ -213,13 +209,6 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
 		return NULL;
 	}
 
-	hook_invoke_alloc(
-	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx,
-	    ret, (uintptr_t)ret, hook_args->args);
-	hook_invoke_dalloc(
-	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
-	    ptr, hook_args->args);
-
 	size_t copysize = (usize < oldusize) ? usize : oldusize;
 	memcpy(ret, edata_addr_get(edata), copysize);
 	isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true);

src/malloc_io.c

@@ -105,8 +105,7 @@ buferror(int err, char *buf, size_t buflen) {
     && defined(_GNU_SOURCE)
 	char *b = strerror_r(err, buf, buflen);
 	if (b != buf) {
-		strncpy(buf, b, buflen);
-		buf[buflen - 1] = '\0';
+		malloc_snprintf(buf, buflen, "%s", b);
 	}
 	return 0;
 #else

src/pa.c

@@ -67,7 +67,7 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
 bool
 pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
     const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
-	if (hpa_shard_init(tsdn, &shard->hpa_shard, &shard->central->hpa,
+	if (hpa_shard_init(tsdn, &shard->hpa, &shard->central->hpa,
 	        shard->emap, shard->base, &shard->edata_cache, shard->ind,
 	        hpa_opts, hpa_sec_opts)) {
 		return true;
@@ -82,7 +82,7 @@ void
 pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
 	atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
 	if (shard->ever_used_hpa) {
-		hpa_shard_disable(tsdn, &shard->hpa_shard);
+		hpa_shard_disable(tsdn, &shard->hpa);
 	}
 }
 
@@ -95,7 +95,7 @@ pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (shard->ever_used_hpa) {
-		hpa_shard_flush(tsdn, &shard->hpa_shard);
+		hpa_shard_flush(tsdn, &shard->hpa);
 	}
 }
 
@@ -108,16 +108,10 @@ void
 pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
 	pac_destroy(tsdn, &shard->pac);
 	if (shard->ever_used_hpa) {
-		hpa_shard_destroy(tsdn, &shard->hpa_shard);
+		hpa_shard_destroy(tsdn, &shard->hpa);
 	}
 }
 
-static pai_t *
-pa_get_pai(pa_shard_t *shard, edata_t *edata) {
-	return (edata_pai_get(edata) == EXTENT_PAI_PAC ? &shard->pac.pai
-	                                               : &shard->hpa_shard.pai);
-}
-
 edata_t *
 pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
     bool slab, szind_t szind, bool zero, bool guarded,
@@ -128,7 +122,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
 
 	edata_t *edata = NULL;
 	if (!guarded && pa_shard_uses_hpa(shard)) {
-		edata = pai_alloc(tsdn, &shard->hpa_shard.pai, size, alignment,
+		edata = hpa_alloc(tsdn, &shard->hpa, size, alignment,
 		    zero, /* guarded */ false, slab, deferred_work_generated);
 	}
 	/*
@@ -136,7 +130,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
 	 * allocation request.
 	 */
 	if (edata == NULL) {
-		edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero,
+		edata = pac_alloc(tsdn, &shard->pac, size, alignment, zero,
 		    guarded, slab, deferred_work_generated);
 	}
 	if (edata != NULL) {
@@ -164,10 +158,15 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 	}
 	size_t expand_amount = new_size - old_size;
 
-	pai_t *pai = pa_get_pai(shard, edata);
-
-	bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero,
-	    deferred_work_generated);
+	/*
+	 * HPA expand always fails (it's a stub); skip the call entirely for
+	 * HPA-owned extents.
+	 */
+	if (edata_pai_get(edata) == EXTENT_PAI_HPA) {
+		return true;
+	}
+	bool error = pac_expand(tsdn, &shard->pac, edata, old_size, new_size,
+	    zero, deferred_work_generated);
 	if (error) {
 		return true;
 	}
@@ -189,9 +188,15 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 	}
 	size_t shrink_amount = old_size - new_size;
 
-	pai_t *pai = pa_get_pai(shard, edata);
-	bool   error = pai_shrink(
-            tsdn, pai, edata, old_size, new_size, deferred_work_generated);
+	/*
+	 * HPA shrink always fails (it's a stub); skip the call entirely for
+	 * HPA-owned extents.
+	 */
+	if (edata_pai_get(edata) == EXTENT_PAI_HPA) {
+		return true;
+	}
+	bool error = pac_shrink(tsdn, &shard->pac, edata, old_size, new_size,
+	    deferred_work_generated);
 	if (error) {
 		return true;
 	}
@@ -216,8 +221,11 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
 	edata_addr_set(edata, edata_base_get(edata));
 	edata_szind_set(edata, SC_NSIZES);
 	pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE);
-	pai_t *pai = pa_get_pai(shard, edata);
-	pai_dalloc(tsdn, pai, edata, deferred_work_generated);
+	if (edata_pai_get(edata) == EXTENT_PAI_HPA) {
+		hpa_dalloc(tsdn, &shard->hpa, edata, deferred_work_generated);
+	} else {
+		pac_dalloc(tsdn, &shard->pac, edata, deferred_work_generated);
+	}
 }
 
 bool
@@ -236,14 +244,14 @@ pa_shard_set_deferral_allowed(
     tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed) {
 	if (pa_shard_uses_hpa(shard)) {
 		hpa_shard_set_deferral_allowed(
-		    tsdn, &shard->hpa_shard, deferral_allowed);
+		    tsdn, &shard->hpa, deferral_allowed);
 	}
 }
 
 void
 pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (pa_shard_uses_hpa(shard)) {
-		hpa_shard_do_deferred_work(tsdn, &shard->hpa_shard);
+		hpa_shard_do_deferred_work(tsdn, &shard->hpa);
 	}
 }
 
@@ -254,14 +262,14 @@ pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
  */
 uint64_t
 pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
-	uint64_t time = pai_time_until_deferred_work(tsdn, &shard->pac.pai);
+	uint64_t time = pac_time_until_deferred_work(tsdn, &shard->pac);
 	if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
 		return time;
 	}
 
 	if (pa_shard_uses_hpa(shard)) {
-		uint64_t hpa = pai_time_until_deferred_work(
-		    tsdn, &shard->hpa_shard.pai);
+		uint64_t hpa = hpa_time_until_deferred_work(
+		    tsdn, &shard->hpa);
 		if (hpa < time) {
 			time = hpa;
 		}

src/pa_extra.c

@@ -17,7 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (shard->ever_used_hpa) {
-		hpa_shard_prefork2(tsdn, &shard->hpa_shard);
+		hpa_shard_prefork2(tsdn, &shard->hpa);
 	}
 }
 
@@ -25,7 +25,7 @@ void
 pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) {
 	malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx);
 	if (shard->ever_used_hpa) {
-		hpa_shard_prefork3(tsdn, &shard->hpa_shard);
+		hpa_shard_prefork3(tsdn, &shard->hpa);
 	}
 }
 
@@ -34,8 +34,9 @@ pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_prefork(tsdn, &shard->pac.ecache_dirty);
 	ecache_prefork(tsdn, &shard->pac.ecache_muzzy);
 	ecache_prefork(tsdn, &shard->pac.ecache_retained);
+	ecache_prefork(tsdn, &shard->pac.ecache_pinned);
 	if (shard->ever_used_hpa) {
-		hpa_shard_prefork4(tsdn, &shard->hpa_shard);
+		hpa_shard_prefork4(tsdn, &shard->hpa);
 	}
 }
 
@@ -50,11 +51,12 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_dirty);
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_muzzy);
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
+	ecache_postfork_parent(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
-		hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
+		hpa_shard_postfork_parent(tsdn, &shard->hpa);
 	}
 }
 
@@ -64,36 +66,37 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_child(tsdn, &shard->pac.ecache_dirty);
 	ecache_postfork_child(tsdn, &shard->pac.ecache_muzzy);
 	ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
+	ecache_postfork_child(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
-		hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
+		hpa_shard_postfork_child(tsdn, &shard->hpa);
 	}
 }
 
 size_t
-pa_shard_nactive(pa_shard_t *shard) {
+pa_shard_nactive(const pa_shard_t *shard) {
 	return atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
 }
 
 size_t
-pa_shard_ndirty(pa_shard_t *shard) {
+pa_shard_ndirty(const pa_shard_t *shard) {
 	size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty);
 	if (shard->ever_used_hpa) {
-		ndirty += psset_ndirty(&shard->hpa_shard.psset);
+		ndirty += psset_ndirty(&shard->hpa.psset);
 	}
 	return ndirty;
 }
 
 size_t
-pa_shard_nmuzzy(pa_shard_t *shard) {
+pa_shard_nmuzzy(const pa_shard_t *shard) {
 	return ecache_npages_get(&shard->pac.ecache_muzzy);
 }
 
 void
 pa_shard_basic_stats_merge(
-    pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
+    const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
 	*nactive += pa_shard_nactive(shard);
 	*ndirty += pa_shard_ndirty(shard);
 	*nmuzzy += pa_shard_nmuzzy(shard);
@@ -107,12 +110,15 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 
 	pa_shard_stats_out->pac_stats.retained +=
 	    ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
+	pa_shard_stats_out->pac_stats.pinned +=
+	    ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE;
 	pa_shard_stats_out->edata_avail += atomic_load_zu(
 	    &shard->edata_cache.count, ATOMIC_RELAXED);
 
 	size_t resident_pgs = 0;
 	resident_pgs += pa_shard_nactive(shard);
 	resident_pgs += pa_shard_ndirty(shard);
+	resident_pgs += ecache_npages_get(&shard->pac.ecache_pinned);
 	*resident += (resident_pgs << LG_PAGE);
 
 	/* Dirty decay stats */
@@ -147,26 +153,31 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 	    atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED));
 
 	for (pszind_t i = 0; i < SC_NPSIZES; i++) {
-		size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
-		    retained_bytes;
+		size_t dirty, muzzy, retained, pinned, dirty_bytes,
+		    muzzy_bytes, retained_bytes, pinned_bytes;
 		dirty = ecache_nextents_get(&shard->pac.ecache_dirty, i);
 		muzzy = ecache_nextents_get(&shard->pac.ecache_muzzy, i);
 		retained = ecache_nextents_get(&shard->pac.ecache_retained, i);
+		pinned = ecache_nextents_get(&shard->pac.ecache_pinned, i);
 		dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i);
 		muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i);
 		retained_bytes = ecache_nbytes_get(
 		    &shard->pac.ecache_retained, i);
+		pinned_bytes = ecache_nbytes_get(
+		    &shard->pac.ecache_pinned, i);
 
 		estats_out[i].ndirty = dirty;
 		estats_out[i].nmuzzy = muzzy;
 		estats_out[i].nretained = retained;
+		estats_out[i].npinned = pinned;
 		estats_out[i].dirty_bytes = dirty_bytes;
 		estats_out[i].muzzy_bytes = muzzy_bytes;
 		estats_out[i].retained_bytes = retained_bytes;
+		estats_out[i].pinned_bytes = pinned_bytes;
 	}
 
 	if (shard->ever_used_hpa) {
-		hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
+		hpa_shard_stats_merge(tsdn, &shard->hpa, hpa_stats_out);
 	}
 }
 
@@ -189,6 +200,8 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 	    &shard->pac.ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy);
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 	    &shard->pac.ecache_retained.mtx, arena_prof_mutex_extents_retained);
+	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
+	    &shard->pac.ecache_pinned.mtx, arena_prof_mutex_extents_pinned);
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 	    &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty);
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
@@ -196,11 +209,11 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 
 	if (shard->ever_used_hpa) {
 		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
-		    &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard);
+		    &shard->hpa.mtx, arena_prof_mutex_hpa_shard);
 		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
-		    &shard->hpa_shard.grow_mtx,
+		    &shard->hpa.grow_mtx,
 		    arena_prof_mutex_hpa_shard_grow);
-		sec_mutex_stats_read(tsdn, &shard->hpa_shard.sec,
+		sec_mutex_stats_read(tsdn, &shard->hpa.sec,
 		    &mutex_prof_data[arena_prof_mutex_hpa_sec]);
 	}
 }

src/pac.c

@@ -4,17 +4,6 @@
 #include "jemalloc/internal/pac.h"
 #include "jemalloc/internal/san.h"
 
-static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
-    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
-    bool *deferred_work_generated);
-static bool     pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-        size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
-static bool     pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-        size_t old_size, size_t new_size, bool *deferred_work_generated);
-static void     pac_dalloc_impl(
-        tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
-static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
-
 static inline void
 pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay,
     pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) {
@@ -31,6 +20,7 @@ pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay,
 		return;
 	case extent_state_active:
 	case extent_state_retained:
+	case extent_state_pinned:
 	case extent_state_transition:
 	case extent_state_merging:
 	default:
@@ -72,6 +62,12 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	        /* delay_coalesce */ false)) {
 		return true;
 	}
+	/* Pinned extents: no decay, delayed coalesce. */
+	if (ecache_init(tsdn, &pac->ecache_pinned, extent_state_pinned, ind,
+	        /* delay_coalesce */ true)) {
+		return true;
+	}
+	atomic_store_b(&pac->has_pinned, false, ATOMIC_RELAXED);
 	exp_grow_init(&pac->exp_grow);
 	if (malloc_mutex_init(&pac->grow_mtx, "extent_grow",
 	        WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
@@ -96,12 +92,6 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	pac->stats_mtx = stats_mtx;
 	atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
 
-	pac->pai.alloc = &pac_alloc_impl;
-	pac->pai.expand = &pac_expand_impl;
-	pac->pai.shrink = &pac_shrink_impl;
-	pac->pai.dalloc = &pac_dalloc_impl;
-	pac->pai.time_until_deferred_work = &pac_time_until_deferred_work;
-
 	return false;
 }
 
@@ -110,6 +100,14 @@ pac_may_have_muzzy(pac_t *pac) {
 	return pac_decay_ms_get(pac, extent_state_muzzy) != 0;
 }
 
+static inline void
+pac_ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+    edata_t *edata) {
+	ecache_dalloc(tsdn, pac, ehooks,
+	    edata_pinned_get(edata) ? &pac->ecache_pinned : &pac->ecache_dirty,
+	    edata);
+}
+
 static size_t
 pac_alloc_retained_batched_size(size_t size) {
 	if (size > SC_LARGE_MAXCLASS) {
@@ -133,8 +131,22 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	assert(!guarded || alignment <= PAGE);
 	size_t newly_mapped_size = 0;
 
-	edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
-	    NULL, size, alignment, zero, guarded);
+	edata_t *edata = NULL;
+
+	/*
+	 * Guarded allocations need surrounding guard pages, which the pinned
+	 * pool does not maintain; skip ecache_pinned in that case.
+	 */
+	if (!guarded && atomic_load_b(&pac->has_pinned, ATOMIC_RELAXED)
+	    && ecache_npages_get(&pac->ecache_pinned) > 0) {
+		edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_pinned,
+		    NULL, size, alignment, zero, guarded);
+	}
+
+	if (edata == NULL) {
+		edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
+		    NULL, size, alignment, zero, guarded);
+	}
 
 	if (edata == NULL && pac_may_have_muzzy(pac)) {
 		edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
@@ -180,12 +192,10 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 			    edata, size, batched_size - size,
 			    /* holding_core_locks */ false);
 			if (trail == NULL) {
-				ecache_dalloc(tsdn, pac, ehooks,
-				    &pac->ecache_retained, edata);
+				pac_record_grown(tsdn, pac, ehooks, edata);
 				edata = NULL;
 			} else {
-				ecache_dalloc(tsdn, pac, ehooks,
-				    &pac->ecache_dirty, trail);
+				pac_ecache_dalloc(tsdn, pac, ehooks, trail);
 			}
 		}
 
@@ -238,11 +248,10 @@ pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	return edata;
 }
 
-static edata_t *
-pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
+edata_t *
+pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment,
     bool zero, bool guarded, bool frequent_reuse,
     bool *deferred_work_generated) {
-	pac_t    *pac = (pac_t *)self;
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	edata_t *edata = NULL;
@@ -265,10 +274,11 @@ pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
 	return edata;
 }
 
-static bool
-pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+bool
+pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
     size_t new_size, bool zero, bool *deferred_work_generated) {
-	pac_t    *pac = (pac_t *)self;
+	assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
+
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	size_t mapped_add = 0;
@@ -277,23 +287,53 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
 	if (ehooks_merge_will_fail(ehooks)) {
 		return true;
 	}
-	edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
-	    edata, expand_amount, PAGE, zero, /* guarded*/ false);
-	if (trail == NULL) {
-		trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
-		    edata, expand_amount, PAGE, zero, /* guarded*/ false);
+	edata_t *trail = NULL;
+	if (edata_pinned_get(edata)) {
+		trail = ecache_alloc(tsdn, pac, ehooks,
+		    &pac->ecache_pinned, edata, expand_amount,
+		    PAGE, zero, /* guarded */ false);
+		if (trail == NULL) {
+			/*
+			 * Only ecache_pinned can hold a mergeable neighbor;
+			 * dirty, muzzy, and retained extents are non-pinned.
+			 * Pinned memory is already committed, and hooks are
+			 * unlikely to reserve adjacent pinned space for growth,
+			 * so don't consult the hook to grow in place.
+			 */
+			return true;
+		}
+		assert(edata_pinned_get(trail));
+	} else {
+		trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
+		    edata, expand_amount, PAGE, zero, /* guarded */ false);
+		if (trail == NULL) {
+			trail = ecache_alloc(tsdn, pac, ehooks,
+			    &pac->ecache_muzzy, edata, expand_amount,
+			    PAGE, zero, /* guarded */ false);
+		}
+		if (trail == NULL) {
+			trail = ecache_alloc_grow(tsdn, pac, ehooks,
+			    &pac->ecache_retained, edata, expand_amount,
+			    PAGE, zero, /* guarded */ false);
+			mapped_add = expand_amount;
+		}
+		if (trail == NULL) {
+			return true;
+		}
 	}
-	if (trail == NULL) {
-		trail = ecache_alloc_grow(tsdn, pac, ehooks,
-		    &pac->ecache_retained, edata, expand_amount, PAGE, zero,
-		    /* guarded */ false);
-		mapped_add = expand_amount;
-	}
-	if (trail == NULL) {
-		return true;
-	}
-	if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
-		extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
+	/* extent_merge_wrapper requires matching pinnedness. */
+	if ((edata_pinned_get(edata) != edata_pinned_get(trail))
+	    || extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
+		if (edata_pinned_get(trail)) {
+			if (config_stats) {
+				atomic_fetch_add_zu(&pac->stats->pac_mapped,
+				    mapped_add, ATOMIC_RELAXED);
+			}
+			ecache_dalloc(tsdn, pac, ehooks,
+			    &pac->ecache_pinned, trail);
+		} else {
+			extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
+		}
 		return true;
 	}
 	if (config_stats && mapped_add > 0) {
@@ -303,10 +343,11 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
 	return false;
 }
 
-static bool
-pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+bool
+pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
     size_t new_size, bool *deferred_work_generated) {
-	pac_t    *pac = (pac_t *)self;
+	assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
+
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	size_t shrink_amount = old_size - new_size;
@@ -320,15 +361,19 @@ pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
 	if (trail == NULL) {
 		return true;
 	}
-	ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail);
-	*deferred_work_generated = true;
+	bool pinned = edata_pinned_get(trail);
+	pac_ecache_dalloc(tsdn, pac, ehooks, trail);
+	if (!pinned) {
+		*deferred_work_generated = true;
+	}
 	return false;
 }
 
-static void
-pac_dalloc_impl(
-    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
-	pac_t    *pac = (pac_t *)self;
+void
+pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
+    bool *deferred_work_generated) {
+	assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
+
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	if (edata_guarded_get(edata)) {
@@ -352,9 +397,11 @@ pac_dalloc_impl(
 		}
 	}
 
-	ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
-	/* Purging of deallocated pages is deferred */
-	*deferred_work_generated = true;
+	bool pinned = edata_pinned_get(edata);
+	pac_ecache_dalloc(tsdn, pac, ehooks, edata);
+	if (!pinned) {
+		*deferred_work_generated = true;
+	}
 }
 
 static inline uint64_t
@@ -370,10 +417,9 @@ pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) {
 	return result;
 }
 
-static uint64_t
-pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
+uint64_t
+pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac) {
 	uint64_t time;
-	pac_t   *pac = (pac_t *)self;
 
 	time = pac_ns_until_purge(
 	    tsdn, &pac->decay_dirty, ecache_npages_get(&pac->ecache_dirty));
@@ -543,6 +589,7 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
 			break;
 		case extent_state_active:
 		case extent_state_retained:
+		case extent_state_pinned:
 		case extent_state_transition:
 		case extent_state_merging:
 		default:
@@ -721,7 +768,48 @@ pac_destroy(tsdn_t *tsdn, pac_t *pac) {
 	 * dss-based extents for later reuse.
 	 */
 	ehooks_t *ehooks = pac_ehooks_get(pac);
-	edata_t  *edata;
+	edata_t *edata;
+	if (atomic_load_b(&pac->has_pinned, ATOMIC_RELAXED)) {
+		/*
+		 * Reroute pinned extents through ecache_retained: clearing the
+		 * pinned bit lets retained's eager coalesce merge fragments
+		 * back to their original OS-allocation bases, so the destroy
+		 * hook can release whole reservations (required on platforms
+		 * like Windows where VirtualFree only accepts the original
+		 * VirtualAlloc base).  Subtract from pac_mapped along the way
+		 * because retained is excluded from stats.mapped.
+		 */
+		edata_list_inactive_t pinned_list;
+		edata_list_inactive_init(&pinned_list);
+		malloc_mutex_lock(tsdn, &pac->ecache_pinned.mtx);
+		assert(eset_npages_get(&pac->ecache_pinned.guarded_eset) == 0);
+		size_t pinned_bytes =
+		    eset_npages_get(&pac->ecache_pinned.eset) << LG_PAGE;
+		while (eset_npages_get(&pac->ecache_pinned.eset) > 0) {
+			edata = eset_fit(&pac->ecache_pinned.eset,
+			    PAGE, PAGE, /* exact_only */ false, SC_PTR_BITS,
+			    /* prefer_small */ false);
+			assert(edata != NULL);
+			assert(edata_pinned_get(edata));
+			eset_remove(&pac->ecache_pinned.eset, edata);
+			emap_update_edata_state(tsdn, pac->emap, edata,
+			    extent_state_active);
+			edata_pinned_set(edata, false);
+			edata_list_inactive_append(&pinned_list, edata);
+		}
+		malloc_mutex_unlock(tsdn, &pac->ecache_pinned.mtx);
+		if (config_stats && pinned_bytes > 0) {
+			atomic_fetch_sub_zu(&pac->stats->pac_mapped,
+			    pinned_bytes, ATOMIC_RELAXED);
+		}
+		while ((edata = edata_list_inactive_first(&pinned_list))
+		    != NULL) {
+			edata_list_inactive_remove(&pinned_list, edata);
+			extent_record(tsdn, pac, ehooks,
+			    &pac->ecache_retained, edata);
+		}
+	}
+	assert(ecache_npages_get(&pac->ecache_pinned) == 0);
 	while (
 	    (edata = ecache_evict(tsdn, pac, ehooks, &pac->ecache_retained, 0))
 	    != NULL) {

src/pages.c

@@ -662,10 +662,6 @@ pages_purge_process_madvise_impl(
 		return true;
 	}
 
-	/*
-	 * TODO: remove this save/restore of errno after supporting errno
-	 * preservation for free() call properly.
-	 */
 	int    saved_errno = get_errno();
 	size_t purged_bytes = (size_t)syscall(JE_SYS_PROCESS_MADVISE_NR,
 	    PIDFD_SELF, (struct iovec *)vec, vec_len, MADV_DONTNEED, 0);

src/psset.c

@@ -349,7 +349,7 @@ psset_enumerate_search(psset_t *psset, pszind_t pind, size_t size) {
 
 	while ((ps = hpdata_age_heap_enumerate_next(
 	            &psset->pageslabs[pind], &helper))) {
-		if (hpdata_longest_free_range_get(ps) >= size) {
+		if ((hpdata_longest_free_range_get(ps) << LG_PAGE) >= size) {
 			return ps;
 		}
 	}

src/sec.c

@@ -90,6 +90,35 @@ sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) {
 	return &sec->bins[ind];
 }
 
+void
+sec_calc_nallocs_for_size(
+    sec_t *sec, size_t size, size_t *min_nallocs_ret, size_t *max_nallocs_ret) {
+	size_t min_nallocs = 1;
+	size_t max_nallocs = 1;
+
+	if (sec_size_supported(sec, size)) {
+		/*
+		 * This attempts to fill up to 1/SEC_MAX_BYTES_DIV of the SEC.
+		 * If we go much over that, we might cause purging.
+		 * This is mainly an issue when max_bytes is small (256K)
+		 * and size is large. For larger max_bytes, we will
+		 * almost always end up with SEC_MAX_NALLOCS.
+		 */
+		size_t nallocs = sec->opts.max_bytes / size / SEC_MAX_BYTES_DIV;
+		nallocs = max_zu(nallocs, SEC_MIN_NALLOCS);
+		min_nallocs = SEC_MIN_NALLOCS;
+		max_nallocs = min_zu(nallocs, SEC_MAX_NALLOCS);
+	}
+
+	/* post-conditions */
+	assert(1 <= min_nallocs);
+	assert(min_nallocs <= max_nallocs);
+	assert(max_nallocs <= SEC_MAX_NALLOCS);
+
+	*min_nallocs_ret = min_nallocs;
+	*max_nallocs_ret = max_nallocs;
+}
+
 static edata_t *
 sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
 	malloc_mutex_assert_owner(tsdn, &bin->mtx);

src/stats.c

@@ -712,6 +712,8 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 	COL_HDR(row, muzzy, NULL, right, 13, size)
 	COL_HDR(row, nretained, NULL, right, 13, size)
 	COL_HDR(row, retained, NULL, right, 13, size)
+	COL_HDR(row, npinned, NULL, right, 13, size)
+	COL_HDR(row, pinned, NULL, right, 13, size)
 	COL_HDR(row, ntotal, NULL, right, 13, size)
 	COL_HDR(row, total, NULL, right, 13, size)
 
@@ -728,22 +730,27 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 
 	in_gap = false;
 	for (j = 0; j < SC_NPSIZES; j++) {
-		size_t ndirty, nmuzzy, nretained, total, dirty_bytes,
-		    muzzy_bytes, retained_bytes, total_bytes;
+		size_t ndirty, nmuzzy, nretained, npinned, total,
+		    dirty_bytes, muzzy_bytes, retained_bytes, pinned_bytes,
+		    total_bytes;
 		stats_arenas_mib[4] = j;
 
 		CTL_LEAF(stats_arenas_mib, 5, "ndirty", &ndirty, size_t);
 		CTL_LEAF(stats_arenas_mib, 5, "nmuzzy", &nmuzzy, size_t);
 		CTL_LEAF(stats_arenas_mib, 5, "nretained", &nretained, size_t);
+		CTL_LEAF(stats_arenas_mib, 5, "npinned", &npinned, size_t);
 		CTL_LEAF(
 		    stats_arenas_mib, 5, "dirty_bytes", &dirty_bytes, size_t);
 		CTL_LEAF(
 		    stats_arenas_mib, 5, "muzzy_bytes", &muzzy_bytes, size_t);
 		CTL_LEAF(stats_arenas_mib, 5, "retained_bytes", &retained_bytes,
 		    size_t);
+		CTL_LEAF(stats_arenas_mib, 5, "pinned_bytes", &pinned_bytes,
+		    size_t);
 
-		total = ndirty + nmuzzy + nretained;
-		total_bytes = dirty_bytes + muzzy_bytes + retained_bytes;
+		total = ndirty + nmuzzy + nretained + npinned;
+		total_bytes = dirty_bytes + muzzy_bytes + retained_bytes
+		    + pinned_bytes;
 
 		in_gap_prev = in_gap;
 		in_gap = (total == 0);
@@ -758,6 +765,8 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 		emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy);
 		emitter_json_kv(
 		    emitter, "nretained", emitter_type_size, &nretained);
+		emitter_json_kv(
+		    emitter, "npinned", emitter_type_size, &npinned);
 
 		emitter_json_kv(
 		    emitter, "dirty_bytes", emitter_type_size, &dirty_bytes);
@@ -765,6 +774,8 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 		    emitter, "muzzy_bytes", emitter_type_size, &muzzy_bytes);
 		emitter_json_kv(emitter, "retained_bytes", emitter_type_size,
 		    &retained_bytes);
+		emitter_json_kv(emitter, "pinned_bytes", emitter_type_size,
+		    &pinned_bytes);
 		emitter_json_object_end(emitter);
 
 		col_size.size_val = sz_pind2sz(j);
@@ -775,6 +786,8 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 		col_muzzy.size_val = muzzy_bytes;
 		col_nretained.size_val = nretained;
 		col_retained.size_val = retained_bytes;
+		col_npinned.size_val = npinned;
+		col_pinned.size_val = pinned_bytes;
 		col_ntotal.size_val = total;
 		col_total.size_val = total_bytes;
 
@@ -889,9 +902,7 @@ stats_arena_hpa_shard_counters_print(
 	    " / sec)\n"
 	    "  Hugify failures: %" FMTu64 " (%" FMTu64
 	    " / sec)\n"
-	    "  Dehugifies: %" FMTu64 " (%" FMTu64
-	    " / sec)\n"
-	    "\n",
+	    "  Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n",
 	    npageslabs, npageslabs_huge, npageslabs_nonhuge, nactive,
 	    nactive_huge, nactive_nonhuge, ndirty, ndirty_huge, ndirty_nonhuge,
 	    nretained_nonhuge, npurge_passes,
@@ -931,6 +942,82 @@ stats_arena_hpa_shard_counters_print(
 	emitter_json_kv(
 	    emitter, "ndirty_huge", emitter_type_size, &ndirty_huge);
 	emitter_json_object_end(emitter); /* End "slabs" */
+
+	/* alloc_batch stats */
+	uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
+	uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
+	uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
+	uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
+	uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
+	uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
+	uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
+
+	size_t alloc_mib[CTL_MAX_DEPTH];
+	CTL_LEAF_PREPARE(alloc_mib, 0, "stats.arenas");
+	alloc_mib[2] = i;
+	CTL_LEAF_PREPARE(alloc_mib, 3, "hpa_shard.alloc");
+
+	for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
+		alloc_mib[5] = j;
+		CTL_LEAF(alloc_mib, 6, "min_extents", &hpa_alloc_min_extents[j],
+		    uint64_t);
+		CTL_LEAF(alloc_mib, 6, "max_extents", &hpa_alloc_max_extents[j],
+		    uint64_t);
+		CTL_LEAF(
+		    alloc_mib, 6, "extents", &hpa_alloc_extents[j], uint64_t);
+		CTL_LEAF(alloc_mib, 6, "ps", &hpa_alloc_ps[j], uint64_t);
+		CTL_LEAF(alloc_mib, 6, "pages_per_ps",
+		    &hpa_alloc_pages_per_ps[j], uint64_t);
+		CTL_LEAF(alloc_mib, 6, "extents_per_ps",
+		    &hpa_alloc_extents_per_ps[j], uint64_t);
+		CTL_LEAF(alloc_mib, 6, "total_elapsed_ns_per_ps",
+		    &hpa_alloc_total_elapsed_ns_per_ps[j], uint64_t);
+	}
+
+	emitter_table_printf(emitter, "  extent allocation distribution:\n");
+	emitter_table_printf(emitter,
+	    "  %4s %20s %20s %20s %20s %20s %20s %24s %24s\n", "",
+	    "min_extents", "max_extents",
+	    "extents", "ps", "pages_per_ps", "extents_per_ps",
+	    "total_elapsed_ns_per_ps", "elapsed_ns_per_ps");
+	for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
+		const uint64_t extents_per_ps = hpa_alloc_extents_per_ps[j];
+		const uint64_t total_elapsed_ns_per_ps =
+		    hpa_alloc_total_elapsed_ns_per_ps[j];
+		const uint64_t elapsed_ns_per_ps = (extents_per_ps != 0)
+		    ? (total_elapsed_ns_per_ps / extents_per_ps)
+		    : 0;
+		emitter_table_printf(emitter,
+		    "  %4zu %20" FMTu64 " %20" FMTu64 " %20" FMTu64
+		    " %20" FMTu64 " %20" FMTu64 " %20" FMTu64 " %24" FMTu64
+		    " %24" FMTu64 "\n",
+		    j, hpa_alloc_min_extents[j], hpa_alloc_max_extents[j],
+		    hpa_alloc_extents[j],
+		    hpa_alloc_ps[j], hpa_alloc_pages_per_ps[j], extents_per_ps,
+		    total_elapsed_ns_per_ps, elapsed_ns_per_ps);
+	}
+	emitter_table_printf(emitter, "\n");
+
+	emitter_json_array_kv_begin(emitter, "extent_allocation_distribution");
+	for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
+		emitter_json_object_begin(emitter);
+		emitter_json_kv(emitter, "min_extents", emitter_type_uint64,
+		    &hpa_alloc_min_extents[j]);
+		emitter_json_kv(emitter, "max_extents", emitter_type_uint64,
+		    &hpa_alloc_max_extents[j]);
+		emitter_json_kv(emitter, "extents", emitter_type_uint64,
+		    &hpa_alloc_extents[j]);
+		emitter_json_kv(
+		    emitter, "ps", emitter_type_uint64, &hpa_alloc_ps[j]);
+		emitter_json_kv(emitter, "pages_per_ps", emitter_type_uint64,
+		    &hpa_alloc_pages_per_ps[j]);
+		emitter_json_kv(emitter, "extents_per_ps", emitter_type_uint64,
+		    &hpa_alloc_extents_per_ps[j]);
+		emitter_json_kv(emitter, "total_elapsed_ns_per_ps",
+		    emitter_type_uint64, &hpa_alloc_total_elapsed_ns_per_ps[j]);
+		emitter_json_object_end(emitter);
+	}
+	emitter_json_array_end(emitter); /* End "alloc_batch" */
 }
 
 static void
@@ -1166,7 +1253,7 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
 	unsigned    nthreads;
 	const char *dss;
 	ssize_t     dirty_decay_ms, muzzy_decay_ms;
-	size_t      page, pactive, pdirty, pmuzzy, mapped, retained;
+	size_t      page, pactive, pdirty, pmuzzy, mapped, retained, pinned;
 	size_t      base, internal, resident, metadata_edata, metadata_rtree,
 	    metadata_thp, extent_avail;
 	uint64_t dirty_npurge, dirty_nmadvise, dirty_purged;
@@ -1467,6 +1554,7 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
 
 	GET_AND_EMIT_MEM_STAT(mapped)
 	GET_AND_EMIT_MEM_STAT(retained)
+	GET_AND_EMIT_MEM_STAT(pinned)
 	GET_AND_EMIT_MEM_STAT(base)
 	GET_AND_EMIT_MEM_STAT(internal)
 	GET_AND_EMIT_MEM_STAT(metadata_edata)
@@ -1673,7 +1761,6 @@ stats_general_print(emitter_t *emitter) {
 	OPT_WRITE_SIZE_T("hpa_sec_nshards")
 	OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
 	OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
-	OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
 	OPT_WRITE_BOOL("huge_arena_pac_thp")
 	OPT_WRITE_CHAR_P("metadata_thp")
 	OPT_WRITE_INT64("mutex_max_spin")
@@ -1872,7 +1959,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 	 * the transition to the emitter code.
 	 */
 	size_t allocated, active, metadata, metadata_edata, metadata_rtree,
-	    metadata_thp, resident, mapped, retained;
+	    metadata_thp, resident, mapped, retained, pinned;
 	size_t   num_background_threads;
 	size_t   zero_reallocs;
 	uint64_t background_thread_num_runs, background_thread_run_interval;
@@ -1886,6 +1973,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 	CTL_GET("stats.resident", &resident, size_t);
 	CTL_GET("stats.mapped", &mapped, size_t);
 	CTL_GET("stats.retained", &retained, size_t);
+	CTL_GET("stats.pinned", &pinned, size_t);
 
 	CTL_GET("stats.zero_reallocs", &zero_reallocs, size_t);
 
@@ -1916,15 +2004,16 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 	emitter_json_kv(emitter, "resident", emitter_type_size, &resident);
 	emitter_json_kv(emitter, "mapped", emitter_type_size, &mapped);
 	emitter_json_kv(emitter, "retained", emitter_type_size, &retained);
+	emitter_json_kv(emitter, "pinned", emitter_type_size, &pinned);
 	emitter_json_kv(
 	    emitter, "zero_reallocs", emitter_type_size, &zero_reallocs);
 
 	emitter_table_printf(emitter,
 	    "Allocated: %zu, active: %zu, "
 	    "metadata: %zu (n_thp %zu, edata %zu, rtree %zu), resident: %zu, "
-	    "mapped: %zu, retained: %zu\n",
+	    "mapped: %zu, retained: %zu, pinned: %zu\n",
 	    allocated, active, metadata, metadata_thp, metadata_edata,
-	    metadata_rtree, resident, mapped, retained);
+	    metadata_rtree, resident, mapped, retained, pinned);
 
 	/* Strange behaviors */
 	emitter_table_printf(emitter,

src/tcache.c

@@ -203,26 +203,6 @@ tcache_gc_item_delay_compute(szind_t szind) {
 	return (uint8_t)item_delay;
 }
 
-static inline void *
-tcache_gc_small_heuristic_addr_get(
-    tsd_t *tsd, tcache_slow_t *tcache_slow, szind_t szind) {
-	assert(szind < SC_NBINS);
-	tsdn_t *tsdn = tsd_tsdn(tsd);
-	bin_t  *bin = bin_choose(tsdn, tcache_slow->arena, szind, NULL);
-	assert(bin != NULL);
-
-	malloc_mutex_lock(tsdn, &bin->lock);
-	edata_t *slab = (bin->slabcur == NULL)
-	    ? edata_heap_first(&bin->slabs_nonfull)
-	    : bin->slabcur;
-	assert(slab != NULL || edata_heap_empty(&bin->slabs_nonfull));
-	void *ret = (slab != NULL) ? edata_addr_get(slab) : NULL;
-	assert(ret != NULL || slab == NULL);
-	malloc_mutex_unlock(tsdn, &bin->lock);
-
-	return ret;
-}
-
 static inline bool
 tcache_gc_is_addr_remote(void *addr, uintptr_t min, uintptr_t max) {
 	assert(addr != NULL);
@@ -359,6 +339,55 @@ tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
 	}
 }
 
+#ifdef JEMALLOC_JET
+/*
+ * The GC helpers above are static inline so they cannot be linked from a
+ * separate translation unit.  In JET builds we expose thin wrappers with a
+ * `_test` suffix so test/unit/tcache_gc.c can exercise them directly.  These
+ * symbols are absent from the production library.
+ */
+#define JET_WRAP_RET(ret, fn, params, args)	\
+	ret fn##_test params {			\
+		return fn args;			\
+	}
+#define JET_WRAP_VOID(fn, params, args)		\
+	void fn##_test params {			\
+		fn args;			\
+	}
+
+JET_WRAP_RET(cache_bin_sz_t, tcache_gc_small_nremote_get,
+    (cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min,
+     uintptr_t *addr_max, szind_t szind, size_t nflush),
+    (cache_bin, addr, addr_min, addr_max, szind, nflush))
+
+JET_WRAP_VOID(tcache_gc_small_bin_shuffle,
+    (cache_bin_t *cache_bin, cache_bin_sz_t nremote,
+     uintptr_t addr_min, uintptr_t addr_max),
+    (cache_bin, nremote, addr_min, addr_max))
+
+JET_WRAP_RET(uint8_t, tcache_nfill_small_lg_div_get,
+    (tcache_slow_t *tcache_slow, szind_t szind),
+    (tcache_slow, szind))
+
+JET_WRAP_VOID(tcache_nfill_small_burst_prepare,
+    (tcache_slow_t *tcache_slow, szind_t szind),
+    (tcache_slow, szind))
+
+JET_WRAP_VOID(tcache_nfill_small_burst_reset,
+    (tcache_slow_t *tcache_slow, szind_t szind),
+    (tcache_slow, szind))
+
+JET_WRAP_VOID(tcache_nfill_small_gc_update,
+    (tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit),
+    (tcache_slow, szind, limit))
+
+JET_WRAP_RET(uint8_t, tcache_gc_item_delay_compute,
+    (szind_t szind), (szind))
+
+#undef JET_WRAP_RET
+#undef JET_WRAP_VOID
+#endif
+
 static bool
 tcache_gc_small(
     tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) {
@@ -417,9 +446,9 @@ tcache_gc_small(
 		goto label_flush;
 	}
 
-	/* Query arena binshard to get heuristic locality info. */
-	void *addr = tcache_gc_small_heuristic_addr_get(
-	    tsd, tcache_slow, szind);
+	/* Query arena for a locality anchor (small-bin slab address). */
+	void *addr = arena_locality_hint(tsd_tsdn(tsd), tcache_slow->arena,
+	    szind);
 	if (addr == NULL) {
 		goto label_flush;
 	}
@@ -742,58 +771,52 @@ tcache_bin_ncached_max_read(
 
 void
 tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
-    tcache_t *tcache, arena_t *arena) {
+    arena_t *arena) {
 	assert(tcache_slow->arena == NULL);
+	assert(tcache_slow->tcache != NULL);
 	tcache_slow->arena = arena;
 
 	if (config_stats) {
-		/* Link into list of extant tcaches. */
-		malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
-
-		ql_elm_new(tcache_slow, link);
-		ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
 		cache_bin_array_descriptor_init(
-		    &tcache_slow->cache_bin_array_descriptor, tcache->bins);
-		ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
-		    &tcache_slow->cache_bin_array_descriptor, link);
-
-		malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
+		    &tcache_slow->cache_bin_array_descriptor,
+		    tcache_slow->tcache->bins);
+		arena_cache_bin_array_register(tsdn, arena,
+		    &tcache_slow->cache_bin_array_descriptor);
 	}
 }
 
 static void
-tcache_arena_dissociate(
-    tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache) {
+tcache_arena_dissociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow) {
 	arena_t *arena = tcache_slow->arena;
 	assert(arena != NULL);
 	if (config_stats) {
-		/* Unlink from list of extant tcaches. */
-		malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
-		if (config_debug) {
-			bool           in_ql = false;
-			tcache_slow_t *iter;
-			ql_foreach (iter, &arena->tcache_ql, link) {
-				if (iter == tcache_slow) {
-					in_ql = true;
-					break;
-				}
-			}
-			assert(in_ql);
-		}
-		ql_remove(&arena->tcache_ql, tcache_slow, link);
-		ql_remove(&arena->cache_bin_array_descriptor_ql,
-		    &tcache_slow->cache_bin_array_descriptor, link);
-		tcache_stats_merge(tsdn, tcache_slow->tcache, arena);
-		malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
+		arena_cache_bin_array_unregister(tsdn, arena,
+		    &tcache_slow->cache_bin_array_descriptor);
 	}
 	tcache_slow->arena = NULL;
 }
 
 void
 tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
-    tcache_t *tcache, arena_t *arena) {
-	tcache_arena_dissociate(tsdn, tcache_slow, tcache);
-	tcache_arena_associate(tsdn, tcache_slow, tcache, arena);
+    arena_t *arena) {
+	tcache_arena_dissociate(tsdn, tcache_slow);
+	tcache_arena_associate(tsdn, tcache_slow, arena);
+}
+
+cache_bin_array_descriptor_t *
+tcache_postfork_arena_descriptor(tsdn_t *tsdn, arena_t *arena) {
+	if (!config_stats) {
+		return NULL;
+	}
+	tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn));
+	if (tcache_slow == NULL || tcache_slow->arena != arena) {
+		return NULL;
+	}
+	assert(tcache_slow->tcache != NULL);
+	cache_bin_array_descriptor_init(
+	    &tcache_slow->cache_bin_array_descriptor,
+	    tcache_slow->tcache->bins);
+	return &tcache_slow->cache_bin_array_descriptor;
 }
 
 static void
@@ -810,7 +833,6 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, void *mem,
 	tcache->tcache_slow = tcache_slow;
 	tcache_slow->tcache = tcache;
 
-	memset(&tcache_slow->link, 0, sizeof(ql_elm(tcache_t)));
 	nstime_init_zero(&tcache_slow->last_gc_time);
 	tcache_slow->next_gc_bin = 0;
 	tcache_slow->next_gc_bin_small = 0;
@@ -984,8 +1006,7 @@ tsd_tcache_data_init_impl(
 	if (!malloc_initialized()) {
 		/* If in initialization, assign to a0. */
 		arena = arena_get(tsd_tsdn(tsd), 0, false);
-		tcache_arena_associate(
-		    tsd_tsdn(tsd), tcache_slow, tcache, arena);
+		tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, arena);
 	} else {
 		if (arena == NULL) {
 			arena = arena_choose(tsd, NULL);
@@ -993,7 +1014,7 @@ tsd_tcache_data_init_impl(
 		/* This may happen if thread.tcache.enabled is used. */
 		if (tcache_slow->arena == NULL) {
 			tcache_arena_associate(
-			    tsd_tsdn(tsd), tcache_slow, tcache, arena);
+			    tsd_tsdn(tsd), tcache_slow, arena);
 		}
 	}
 	assert(arena == tcache_slow->arena);
@@ -1053,7 +1074,7 @@ tcache_create_explicit(tsd_t *tsd) {
 	    tsd, tcache_slow, tcache, mem, tcache_get_default_ncached_max());
 
 	tcache_arena_associate(
-	    tsd_tsdn(tsd), tcache_slow, tcache, arena_ichoose(tsd, NULL));
+	    tsd_tsdn(tsd), tcache_slow, arena_ichoose(tsd, NULL));
 
 	return tcache;
 }
@@ -1225,7 +1246,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
 	tcache_slow_t *tcache_slow = tcache->tcache_slow;
 	tcache_flush_cache(tsd, tcache);
 	arena_t *arena = tcache_slow->arena;
-	tcache_arena_dissociate(tsd_tsdn(tsd), tcache_slow, tcache);
+	tcache_arena_dissociate(tsd_tsdn(tsd), tcache_slow);
 
 	if (tsd_tcache) {
 		cache_bin_t *cache_bin = &tcache->bins[0];
@@ -1275,29 +1296,6 @@ tcache_cleanup(tsd_t *tsd) {
 	memset(tcache->bins, 0, sizeof(cache_bin_t) * TCACHE_NBINS_MAX);
 }
 
-void
-tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
-	cassert(config_stats);
-
-	/* Merge and reset tcache stats. */
-	for (unsigned i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) {
-		cache_bin_t *cache_bin = &tcache->bins[i];
-		if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
-			continue;
-		}
-		if (i < SC_NBINS) {
-			bin_t *bin = bin_choose(tsdn, arena, i, NULL);
-			malloc_mutex_lock(tsdn, &bin->lock);
-			bin->stats.nrequests += cache_bin->tstats.nrequests;
-			malloc_mutex_unlock(tsdn, &bin->lock);
-		} else {
-			arena_stats_large_flush_nrequests_add(tsdn,
-			    &arena->stats, i, cache_bin->tstats.nrequests);
-		}
-		cache_bin->tstats.nrequests = 0;
-	}
-}
-
 static bool
 tcaches_create_prep(tsd_t *tsd, base_t *base) {
 	bool err;