Merge branch 'dev'

.appveyor.yml

@@ -1,5 +1,3 @@
-image: Visual Studio 2022
-
 version: '{build}'
 
 environment:
@@ -39,8 +37,7 @@ environment:
 
 install:
   - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
-  - if "%MSVC%"=="amd64" call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
-  - if "%MSVC%"=="x86"   call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars32.bat"
+  - if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
   - if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
   - pacman --noconfirm -Syuu
   - pacman --noconfirm -S autoconf

.github/workflows/check_formatting.yaml

@@ -5,6 +5,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
       - name: Check for trailing whitespace
         run: scripts/check_trailing_whitespace.sh

.github/workflows/freebsd-ci.yml

@@ -25,7 +25,7 @@ jobs:
     name: FreeBSD (${{ matrix.arch }}, debug=${{ matrix.debug }}, prof=${{ matrix.prof }}${{ matrix.uncommon && ', uncommon' || '' }})
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 1
 

.github/workflows/linux-ci.yml

@@ -510,7 +510,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       run: |
@@ -639,7 +639,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       run: |

.github/workflows/macos-ci.yml

@@ -62,7 +62,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       run: |
@@ -164,7 +164,7 @@ jobs:
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       run: |

.github/workflows/static_analysis.yaml

@@ -7,7 +7,7 @@ jobs:
       # We build libunwind ourselves because sadly the version
       # provided by Ubuntu via apt-get is much too old.
       - name: Check out libunwind
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           repository: libunwind/libunwind
           path: libunwind
@@ -23,7 +23,7 @@ jobs:
           cd ..
           rm -rf libunwind
       - name: Check out repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
       # We download LLVM directly from the latest stable release
       # on GitHub, because this tends to be much newer than the
       # version available via apt-get in Ubuntu.
@@ -54,7 +54,7 @@ jobs:
           scripts/run_static_analysis.sh static_analysis_results "$GITHUB_OUTPUT"
       - name: Upload static analysis results
         if: ${{ steps.run_static_analysis.outputs.HAS_STATIC_ANALYSIS_RESULTS }} == '1'
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v4
         with:
           name: static_analysis_results
           path: static_analysis_results

.github/workflows/windows-ci.yml

@@ -54,7 +54,7 @@ jobs:
               CONFIGURE_FLAGS: --enable-debug
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       shell: cmd

INSTALL.md

@@ -411,7 +411,7 @@ differ in their ease of use and flexibility.
 ### With MSVC solutions
 This is the easy, but less flexible approach. It doesn't let you specify
 arguments to the `configure` script.
-
+  
 1. Install Cygwin with at least the following packages:
    * autoconf
    * autogen
@@ -419,19 +419,18 @@ arguments to the `configure` script.
    * grep
    * sed
 
-2. Install Visual Studio with Visual C++. We currently test on 2022 and provide
-solution files back to 2015.
+2. Install Visual Studio 2015 or 2017 with Visual C++
 
 3. Add Cygwin\bin to the PATH environment variable
 
-4. Open "x64 Native Tools Command Prompt" for your version of Visual Studio
+4. Open "x64 Native Tools Command Prompt for VS 2017"
    (note: x86/x64 doesn't matter at this point)
 
 5. Generate header files:
    sh -c "CC=cl ./autogen.sh"
 
-6. Now the project can be opened and built in Visual Studio using the
-   corresponding solution files in the `msvc\` folder.
+6. Now the project can be opened and built in Visual Studio:
+   msvc\jemalloc_vc2017.sln
 
 ### With MSYS
 This is a more involved approach that offers the same configuration flexibility

Makefile.in

@@ -95,10 +95,7 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
 BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
 C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
 C_SRCS := $(srcroot)src/jemalloc.c \
-	$(srcroot)src/jemalloc_fork.c \
-	$(srcroot)src/jemalloc_init.c \
 	$(srcroot)src/arena.c \
-	$(srcroot)src/arenas_management.c \
 	$(srcroot)src/background_thread.c \
 	$(srcroot)src/base.c \
 	$(srcroot)src/bin.c \
@@ -124,6 +121,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/fxp.c \
 	$(srcroot)src/san.c \
 	$(srcroot)src/san_bump.c \
+	$(srcroot)src/hook.c \
 	$(srcroot)src/hpa.c \
 	$(srcroot)src/hpa_central.c \
 	$(srcroot)src/hpa_hooks.c \
@@ -205,12 +203,12 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/a0.c \
 	$(srcroot)test/unit/arena_decay.c \
 	$(srcroot)test/unit/arena_reset.c \
-	$(srcroot)test/unit/arenas_management.c \
 	$(srcroot)test/unit/atomic.c \
 	$(srcroot)test/unit/background_thread.c \
 	$(srcroot)test/unit/background_thread_enable.c \
 	$(srcroot)test/unit/background_thread_init.c \
 	$(srcroot)test/unit/base.c \
+	$(srcroot)test/unit/batch_alloc.c \
 	$(srcroot)test/unit/bin.c \
 	$(srcroot)test/unit/binshard.c \
 	$(srcroot)test/unit/bitmap.c \
@@ -228,10 +226,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/div.c \
 	$(srcroot)test/unit/double_free.c \
 	$(srcroot)test/unit/edata_cache.c \
-	$(srcroot)test/unit/emap.c \
 	$(srcroot)test/unit/emitter.c \
-	$(srcroot)test/unit/eset.c \
-	$(srcroot)test/unit/extent_dss.c \
 	$(srcroot)test/unit/extent_quantize.c \
 	${srcroot}test/unit/fb.c \
 	$(srcroot)test/unit/fork.c \
@@ -239,19 +234,16 @@ TESTS_UNIT := \
 	${srcroot}test/unit/san.c \
 	${srcroot}test/unit/san_bump.c \
 	$(srcroot)test/unit/hash.c \
+	$(srcroot)test/unit/hook.c \
 	$(srcroot)test/unit/hpa.c \
-	$(srcroot)test/unit/hpa_central.c \
 	$(srcroot)test/unit/hpa_sec_integration.c \
 	$(srcroot)test/unit/hpa_thp_always.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \
 	$(srcroot)test/unit/hpa_background_thread.c \
-	$(srcroot)test/unit/hpa_pageslab_packing.c \
 	$(srcroot)test/unit/hpdata.c \
-	$(srcroot)test/unit/extent_alloc_flags.c \
 	$(srcroot)test/unit/huge.c \
 	$(srcroot)test/unit/inspect.c \
-	$(srcroot)test/unit/jemalloc_init.c \
 	$(srcroot)test/unit/junk.c \
 	$(srcroot)test/unit/junk_alloc.c \
 	$(srcroot)test/unit/junk_free.c \
@@ -269,7 +261,6 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/ncached_max.c \
 	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
-	$(srcroot)test/unit/pac.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/peak.c \
@@ -298,6 +289,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/safety_check.c \
 	$(srcroot)test/unit/sc.c \
 	$(srcroot)test/unit/sec.c \
+	$(srcroot)test/unit/seq.c \
 	$(srcroot)test/unit/SFMT.c \
 	$(srcroot)test/unit/size_check.c \
 	$(srcroot)test/unit/size_classes.c \
@@ -309,7 +301,6 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/sz.c \
 	$(srcroot)test/unit/tcache_init.c \
 	$(srcroot)test/unit/tcache_max.c \
-	$(srcroot)test/unit/tcache_gc.c \
 	$(srcroot)test/unit/test_hooks.c \
 	$(srcroot)test/unit/thread_event.c \
 	$(srcroot)test/unit/ticker.c \
@@ -323,7 +314,8 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/zero_reallocs.c
 ifeq (@enable_prof@, 1)
 TESTS_UNIT += \
-	$(srcroot)test/unit/arena_reset_prof.c
+	$(srcroot)test/unit/arena_reset_prof.c \
+	$(srcroot)test/unit/batch_alloc_prof.c
 endif
 TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
 	$(srcroot)test/integration/allocated.c \
@@ -345,11 +337,9 @@ TESTS_INTEGRATION += \
 endif
 ifeq (@enable_cxx@, 1)
 CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
-TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
-ifeq (@enable_cxx_exceptions@, 1)
-TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new_true.cpp \
+TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp \
+	$(srcroot)test/integration/cpp/infallible_new_true.cpp \
 	$(srcroot)test/integration/cpp/infallible_new_false.cpp
-endif
 else
 CPP_SRCS :=
 TESTS_INTEGRATION_CPP :=
@@ -357,7 +347,9 @@ endif
 TESTS_ANALYZE := $(srcroot)test/analyze/prof_bias.c \
 	$(srcroot)test/analyze/rand.c \
 	$(srcroot)test/analyze/sizes.c
-TESTS_STRESS := $(srcroot)test/stress/fill_flush.c \
+TESTS_STRESS := $(srcroot)test/stress/batch_alloc.c \
+	$(srcroot)test/stress/fill_flush.c \
+	$(srcroot)test/stress/hookbench.c \
 	$(srcroot)test/stress/large_microbench.c \
 	$(srcroot)test/stress/mallctl.c \
 	$(srcroot)test/stress/microbench.c
@@ -617,38 +609,38 @@ build_lib: build_lib_static
 endif
 
 install_bin:
-	$(INSTALL) -d '$(BINDIR)'
+	$(INSTALL) -d $(BINDIR)
 	@for b in $(BINS); do \
-	echo "$(INSTALL) -m 755 $$b '$(BINDIR)'"; \
-	$(INSTALL) -m 755 $$b '$(BINDIR)'; \
+	echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \
+	$(INSTALL) -m 755 $$b $(BINDIR); \
 done
 
 install_include:
-	$(INSTALL) -d '$(INCLUDEDIR)/jemalloc'
+	$(INSTALL) -d $(INCLUDEDIR)/jemalloc
 	@for h in $(C_HDRS); do \
-	echo "$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'"; \
-	$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'; \
+	echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
+	$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
 done
 
 install_lib_shared: $(DSOS)
-	$(INSTALL) -d '$(LIBDIR)'
-	$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) '$(LIBDIR)'
+	$(INSTALL) -d $(LIBDIR)
+	$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
 ifneq ($(SOREV),$(SO))
-	ln -sf $(LIBJEMALLOC).$(SOREV) '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
+	ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
 endif
 
 install_lib_static: $(STATIC_LIBS)
-	$(INSTALL) -d '$(LIBDIR)'
+	$(INSTALL) -d $(LIBDIR)
 	@for l in $(STATIC_LIBS); do \
-	echo "$(INSTALL) -m 644 $$l '$(LIBDIR)'"; \
-	$(INSTALL) -m 644 $$l '$(LIBDIR)'; \
+	echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \
+	$(INSTALL) -m 755 $$l $(LIBDIR); \
 done
 
 install_lib_pc: $(PC)
-	$(INSTALL) -d '$(LIBDIR)/pkgconfig'
+	$(INSTALL) -d $(LIBDIR)/pkgconfig
 	@for l in $(PC); do \
-	echo "$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'"; \
-	$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'; \
+	echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \
+	$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
 done
 
 ifeq ($(enable_shared), 1)
@@ -660,17 +652,17 @@ endif
 install_lib: install_lib_pc
 
 install_doc_html: build_doc_html
-	$(INSTALL) -d '$(DATADIR)/doc/jemalloc$(install_suffix)'
+	$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
 	@for d in $(DOCS_HTML); do \
-	echo "$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'"; \
-	$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'; \
+	echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
+	$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
 done
 
 install_doc_man: build_doc_man
-	$(INSTALL) -d '$(MANDIR)/man3'
+	$(INSTALL) -d $(MANDIR)/man3
 	@for d in $(DOCS_MAN3); do \
-	echo "$(INSTALL) -m 644 $$d '$(MANDIR)/man3'"; \
-	$(INSTALL) -m 644 $$d '$(MANDIR)/man3'; \
+	echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \
+	$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
 done
 
 install_doc: install_doc_html install_doc_man
@@ -682,23 +674,23 @@ install: install_doc
 endif
 
 uninstall_bin:
-	$(RM) -v $(foreach b,$(notdir $(BINS)),'$(BINDIR)/$(b)')
+	$(RM) -v $(foreach b,$(notdir $(BINS)),$(BINDIR)/$(b))
 
 uninstall_include:
-	$(RM) -v $(foreach h,$(notdir $(C_HDRS)),'$(INCLUDEDIR)/jemalloc/$(h)')
-	rmdir -v '$(INCLUDEDIR)/jemalloc'
+	$(RM) -v $(foreach h,$(notdir $(C_HDRS)),$(INCLUDEDIR)/jemalloc/$(h))
+	rmdir -v $(INCLUDEDIR)/jemalloc
 
 uninstall_lib_shared:
-	$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SOREV)'
+	$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SOREV)
 ifneq ($(SOREV),$(SO))
-	$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
+	$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SO)
 endif
 
 uninstall_lib_static:
-	$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),'$(LIBDIR)/$(l)')
+	$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),$(LIBDIR)/$(l))
 
 uninstall_lib_pc:
-	$(RM) -v $(foreach p,$(notdir $(PC)),'$(LIBDIR)/pkgconfig/$(p)')
+	$(RM) -v $(foreach p,$(notdir $(PC)),$(LIBDIR)/pkgconfig/$(p))
 
 ifeq ($(enable_shared), 1)
 uninstall_lib: uninstall_lib_shared
@@ -709,11 +701,11 @@ endif
 uninstall_lib: uninstall_lib_pc
 
 uninstall_doc_html:
-	$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),'$(DATADIR)/doc/jemalloc$(install_suffix)/$(d)')
-	rmdir -v '$(DATADIR)/doc/jemalloc$(install_suffix)'
+	$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),$(DATADIR)/doc/jemalloc$(install_suffix)/$(d))
+	rmdir -v $(DATADIR)/doc/jemalloc$(install_suffix)
 
 uninstall_doc_man:
-	$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),'$(MANDIR)/man3/$(d)')
+	$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),$(MANDIR)/man3/$(d))
 
 uninstall_doc: uninstall_doc_html uninstall_doc_man
 

configure.ac

@@ -158,27 +158,29 @@ AC_SUBST([abs_objroot])
 
 dnl Munge install path variables.
 case "$prefix" in
+   *\ * ) AC_MSG_ERROR([Prefix should not contain spaces]) ;;
    "NONE" ) prefix="/usr/local" ;;
 esac
 case "$exec_prefix" in
+   *\ * ) AC_MSG_ERROR([Exec prefix should not contain spaces]) ;;
    "NONE" ) exec_prefix=$prefix ;;
 esac
 PREFIX=$prefix
 AC_SUBST([PREFIX])
-BINDIR=`eval echo "\"$bindir\""`
-BINDIR=`eval echo "\"$BINDIR\""`
+BINDIR=`eval echo $bindir`
+BINDIR=`eval echo $BINDIR`
 AC_SUBST([BINDIR])
-INCLUDEDIR=`eval echo "\"$includedir\""`
-INCLUDEDIR=`eval echo "\"$INCLUDEDIR\""`
+INCLUDEDIR=`eval echo $includedir`
+INCLUDEDIR=`eval echo $INCLUDEDIR`
 AC_SUBST([INCLUDEDIR])
-LIBDIR=`eval echo "\"$libdir\""`
-LIBDIR=`eval echo "\"$LIBDIR\""`
+LIBDIR=`eval echo $libdir`
+LIBDIR=`eval echo $LIBDIR`
 AC_SUBST([LIBDIR])
-DATADIR=`eval echo "\"$datadir\""`
-DATADIR=`eval echo "\"$DATADIR\""`
+DATADIR=`eval echo $datadir`
+DATADIR=`eval echo $DATADIR`
 AC_SUBST([DATADIR])
-MANDIR=`eval echo "\"$mandir\""`
-MANDIR=`eval echo "\"$MANDIR\""`
+MANDIR=`eval echo $mandir`
+MANDIR=`eval echo $MANDIR`
 AC_SUBST([MANDIR])
 
 dnl Support for building documentation.
@@ -374,30 +376,7 @@ fi
 if test "x$enable_cxx" = "x1"; then
   AC_DEFINE([JEMALLOC_ENABLE_CXX], [ ], [ ])
 fi
-if test "x$enable_cxx" = "x1"; then
-  dnl Now check whether the C++ compiler has exceptions enabled.
-  AC_LANG_PUSH([C++])
-  SAVED_CXXFLAGS="${CXXFLAGS}"
-  CXXFLAGS="${CXXFLAGS} ${EXTRA_CXXFLAGS}"
-  JE_COMPILABLE([C++ exception support], [
-#include <new>
-], [
-	try {
-		throw std::bad_alloc();
-	} catch (const std::bad_alloc &) {
-	}
-], [je_cv_cxx_exceptions])
-  CXXFLAGS="${SAVED_CXXFLAGS}"
-  AC_LANG_POP([C++])
-  if test "x${je_cv_cxx_exceptions}" = "xyes" ; then
-    AC_DEFINE([JEMALLOC_HAVE_CXX_EXCEPTIONS], [ ], [ ])
-    enable_cxx_exceptions="1"
-  else
-    enable_cxx_exceptions="0"
-  fi
-fi
 AC_SUBST([enable_cxx])
-AC_SUBST([enable_cxx_exceptions])
 AC_SUBST([CONFIGURE_CXXFLAGS])
 AC_SUBST([SPECIFIED_CXXFLAGS])
 AC_SUBST([EXTRA_CXXFLAGS])
@@ -2062,9 +2041,6 @@ if test "x${LG_PAGE}" != "xundefined" ; then
 else
    AC_MSG_ERROR([cannot determine value for LG_PAGE])
 fi
-if test "${LG_PAGE}" -lt 8 ; then
-   AC_MSG_ERROR([LG_PAGE must be at least 8 (PAGE >= 256)])
-fi
 
 AC_ARG_WITH([lg_hugepage],
   [AS_HELP_STRING([--with-lg-hugepage=<lg-hugepage>],

doc/jemalloc.xml.in

@@ -2088,7 +2088,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <parameter>*oldlenp</parameter> must be
         <code language="C">sizeof(<type>char *</type>)</code>.
         Arena names are also included in the output of <link
-        linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
+        linkend="stats_print"><function>malloc_stats_print()</function></link>.
         </para></listitem>
       </varlistentry>
 
@@ -2172,15 +2172,6 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         addition there may be extents created prior to the application having an
         opportunity to take over extent allocation.</para>
 
-        <para>An extent must be operated on (dalloc, destroy, commit, decommit,
-        purge, split, merge) by a hook capable of handling it, normally the hook
-        that allocated it. Replacing hooks on a live arena is tricky and thus
-        discouraged. If the hook is replaced anyway, the new hook should forward
-        operations on extents it did not allocate to the previous hook (e.g.,
-        the new dalloc dispatches to the previous dalloc for an
-        old-hook-allocated extent). The new hook should also avoid merging
-        extents allocated by different hooks.</para>
-
         <programlisting language="C"><![CDATA[
 typedef extent_hooks_s extent_hooks_t;
 struct extent_hooks_s {
@@ -2245,18 +2236,6 @@ struct extent_hooks_s {
         linkend="arena.i.dss"><mallctl>arena.&lt;i&gt;.dss</mallctl></link>
         setting irrelevant.</para>
 
-        <para>The alloc hook may bitwise-OR
-        <constant>EXTENT_ALLOC_FLAG_PINNED</constant> into the low bits of
-        the returned pointer to indicate that the backing memory is
-        non-reclaimable (e.g. HugeTLB pages) and should be reused
-        preferentially; in that case <parameter>*commit</parameter> must also
-        be set to true.  jemalloc strips the low byte before use.  The
-        pinned attribute is per-extent rather than per-hook: a single alloc
-        hook may return pinned and non-pinned extents in different calls.
-        Pinned-ness is set at allocation, inherited through splits, and
-        never changes after that.  Pinned and non-pinned extents are never
-        merged together.</para>
-
         <funcsynopsis><funcprototype>
           <funcdef>typedef bool <function>(extent_dalloc_t)</function></funcdef>
           <paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
@@ -2478,7 +2457,7 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Hugepage size.  This value is also reported in the
         output of <link
-        linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
+        linkend="stats_print"><function>malloc_stats_print()</function></link>.
         </para></listitem>
       </varlistentry>
 
@@ -2789,11 +2768,11 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Maximum number of bytes in physically resident data
         pages mapped by the allocator, comprising all pages dedicated to
-        allocator metadata, pages backing active allocations, unused dirty
-        pages, and pinned pages.  This is a maximum rather than precise because
-        pages may not actually be physically resident if they correspond to
-        demand-zeroed virtual memory that has not yet been touched.  This is a
-        multiple of the page size, and is larger than <link
+        allocator metadata, pages backing active allocations, and unused dirty
+        pages.  This is a maximum rather than precise because pages may not
+        actually be physically resident if they correspond to demand-zeroed
+        virtual memory that has not yet been touched.  This is a multiple of the
+        page size, and is larger than <link
         linkend="stats.active"><mallctl>stats.active</mallctl></link>.</para></listitem>
       </varlistentry>
 
@@ -2832,22 +2811,6 @@ struct extent_hooks_s {
         </para></listitem>
       </varlistentry>
 
-      <varlistentry id="stats.pinned">
-        <term>
-          <mallctl>stats.pinned</mallctl>
-          (<type>size_t</type>)
-          <literal>r-</literal>
-          [<option>--enable-stats</option>]
-        </term>
-        <listitem><para>Total number of bytes in unused extents backed by
-        non-reclaimable memory.  Pinned extents are tracked separately from
-        dirty, muzzy, and retained extents because they are excluded from
-        decay and purging; unlike <link
-        linkend="stats.retained"><mallctl>stats.retained</mallctl></link>,
-        pinned bytes are included in <link
-        linkend="stats.mapped"><mallctl>stats.mapped</mallctl></link>.</para></listitem>
-      </varlistentry>
-
       <varlistentry id="stats.zero_reallocs">
         <term>
           <mallctl>stats.zero_reallocs</mallctl>
@@ -3126,18 +3089,6 @@ struct extent_hooks_s {
         details.</para></listitem>
       </varlistentry>
 
-      <varlistentry id="stats.arenas.i.pinned">
-        <term>
-          <mallctl>stats.arenas.&lt;i&gt;.pinned</mallctl>
-          (<type>size_t</type>)
-          <literal>r-</literal>
-          [<option>--enable-stats</option>]
-        </term>
-        <listitem><para>Number of pinned bytes.  See <link
-        linkend="stats.pinned"><mallctl>stats.pinned</mallctl></link> for
-        details.</para></listitem>
-      </varlistentry>
-
       <varlistentry id="stats.arenas.i.extent_avail">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.extent_avail</mallctl>
@@ -3195,11 +3146,11 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Maximum number of bytes in physically resident data
         pages mapped by the arena, comprising all pages dedicated to allocator
-        metadata, pages backing active allocations, unused dirty pages, and
-        pinned pages.  This is a maximum rather than precise because pages
-        may not actually be physically resident if they correspond to
-        demand-zeroed virtual memory that has not yet been touched.  This is
-        a multiple of the page size.</para></listitem>
+        metadata, pages backing active allocations, and unused dirty pages.
+        This is a maximum rather than precise because pages may not actually be
+        physically resident if they correspond to demand-zeroed virtual memory
+        that has not yet been touched.  This is a multiple of the page
+        size.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.dirty_npurge">
@@ -3542,7 +3493,7 @@ struct extent_hooks_s {
         </term>
         <listitem><para> Number of extents of the given type in this arena in
 	the bucket corresponding to page size index &lt;j&gt;. The extent type
-	is one of dirty, muzzy, retained, or pinned.</para></listitem>
+	is one of dirty, muzzy, or retained.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.extents.bytes">
@@ -3554,7 +3505,7 @@ struct extent_hooks_s {
         </term>
 	<listitem><para> Sum of the bytes managed by extents of the given type
 	in this arena in the bucket corresponding to page size index &lt;j&gt;.
-	The extent type is one of dirty, muzzy, retained, or pinned.</para></listitem>
+	The extent type is one of dirty, muzzy, or retained.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.lextents.j.nmalloc">
@@ -3674,19 +3625,6 @@ struct extent_hooks_s {
         counters</link>.</para></listitem>
       </varlistentry>
 
-      <varlistentry id="stats.arenas.i.mutexes.extents_pinned">
-        <term>
-          <mallctl>stats.arenas.&lt;i&gt;.mutexes.extents_pinned.{counter}</mallctl>
-          (<type>counter specific type</type>) <literal>r-</literal>
-          [<option>--enable-stats</option>]
-        </term>
-        <listitem><para>Statistics on <varname>arena.&lt;i&gt;.extents_pinned
-        </varname> mutex (arena scope; pinned extents related).
-        <mallctl>{counter}</mallctl> is one of the counters in <link
-        linkend="mutex_counters">mutex profiling
-        counters</link>.</para></listitem>
-      </varlistentry>
-
       <varlistentry id="stats.arenas.i.mutexes.decay_dirty">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.mutexes.decay_dirty.{counter}</mallctl>

include/jemalloc/internal/arena_externs.h

@@ -7,6 +7,7 @@
 #include "jemalloc/internal/div.h"
 #include "jemalloc/internal/emap.h"
 #include "jemalloc/internal/extent_dss.h"
+#include "jemalloc/internal/hook.h"
 #include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/stats.h"
 
@@ -50,11 +51,11 @@ void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
 edata_t *arena_extent_alloc_large(
     tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
 void arena_extent_dalloc_large_prep(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata);
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata);
 void arena_extent_ralloc_large_shrink(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
 void arena_extent_ralloc_large_expand(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
 bool arena_decay_ms_set(
     tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
 ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
@@ -85,13 +86,14 @@ void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
 bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
      size_t extra, bool zero, size_t *newsize);
 void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache);
-dss_prec_t      arena_dss_prec_get(const arena_t *arena);
-ehooks_t       *arena_get_ehooks(const arena_t *arena);
+    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
+    hook_ralloc_args_t *hook_args);
+dss_prec_t      arena_dss_prec_get(arena_t *arena);
+ehooks_t       *arena_get_ehooks(arena_t *arena);
 extent_hooks_t *arena_set_extent_hooks(
     tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
 bool    arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
-void    arena_name_get(const arena_t *arena, char *name);
+void    arena_name_get(arena_t *arena, char *name);
 void    arena_name_set(arena_t *arena, const char *name);
 ssize_t arena_dirty_decay_ms_default_get(void);
 bool    arena_dirty_decay_ms_default_set(ssize_t decay_ms);
@@ -99,22 +101,15 @@ ssize_t arena_muzzy_decay_ms_default_get(void);
 bool    arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
 bool    arena_retain_grow_limit_get_set(
        tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
-unsigned arena_nthreads_get(const arena_t *arena, bool internal);
+unsigned arena_nthreads_get(arena_t *arena, bool internal);
 void     arena_nthreads_inc(arena_t *arena, bool internal);
 void     arena_nthreads_dec(arena_t *arena, bool internal);
 arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
 bool     arena_init_huge(tsdn_t *tsdn, arena_t *a0);
-bool     arena_ind_is_huge(unsigned ind);
 arena_t *arena_choose_huge(tsd_t *tsd);
+size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
+    void **ptrs, size_t nfill, bool zero);
 bool   arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
-void  *arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind);
-void   arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
-       cache_bin_array_descriptor_t *desc);
-void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
-       cache_bin_array_descriptor_t *desc);
-void   arena_cache_bin_array_postfork_child(arena_t *arena,
-       cache_bin_array_descriptor_t *desc_or_null);
-void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);
@@ -125,7 +120,6 @@ void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
 void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
-void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
-       cache_bin_array_descriptor_t *surviving_desc);
+void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
 
 #endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */

include/jemalloc/internal/arena_inlines_a.h

@@ -20,7 +20,7 @@ arena_internal_sub(arena_t *arena, size_t size) {
 }
 
 static inline size_t
-arena_internal_get(const arena_t *arena) {
+arena_internal_get(arena_t *arena) {
 	return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
 }
 

include/jemalloc/internal/arena_inlines_b.h

@@ -21,7 +21,7 @@
 #include "jemalloc/internal/ticker.h"
 
 static inline arena_t *
-arena_get_from_edata(const edata_t *edata) {
+arena_get_from_edata(edata_t *edata) {
 	return (arena_t *)atomic_load_p(
 	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
 }
@@ -203,7 +203,9 @@ arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
 			assert(sz_can_use_slab(size));
 			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
 			    size, ind, zero, slow_path);
-		} else if (likely(tcache_can_cache_large(tcache, ind))) {
+		} else if (likely(ind < tcache_nbins_get(tcache->tcache_slow)
+		               && !tcache_bin_disabled(ind, &tcache->bins[ind],
+		                   tcache->tcache_slow))) {
 			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
 			    size, ind, zero, slow_path);
 		}
@@ -317,7 +319,9 @@ arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
 	if (unlikely(is_sample_promoted)) {
 		arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
 	} else {
-		if (tcache_can_cache_large(tcache, szind)) {
+		if (szind < tcache_nbins_get(tcache->tcache_slow)
+		    && !tcache_bin_disabled(
+		        szind, &tcache->bins[szind], tcache->tcache_slow)) {
 			tcache_dalloc_large(
 			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
 		} else {

include/jemalloc/internal/arena_structs.h

@@ -48,14 +48,15 @@ struct arena_s {
 	arena_stats_t stats;
 
 	/*
-	 * List of cache_bin_array_descriptors for extant threads associated
-	 * with this arena.  Stats from these are merged incrementally, and at
-	 * exit if opt_stats_print is enabled.
+	 * Lists of tcaches and cache_bin_array_descriptors for extant threads
+	 * associated with this arena.  Stats from these are merged
+	 * incrementally, and at exit if opt_stats_print is enabled.
 	 *
-	 * Synchronization: cache_bin_array_descriptor_ql_mtx.
+	 * Synchronization: tcache_ql_mtx.
 	 */
+	ql_head(tcache_slow_t) tcache_ql;
 	ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
-	malloc_mutex_t cache_bin_array_descriptor_ql_mtx;
+	malloc_mutex_t tcache_ql_mtx;
 
 	/*
 	 * Represents a dss_prec_t, but atomically.
@@ -104,11 +105,7 @@ struct arena_s {
 	    "Do not use this field directly. "
 	    "Use `arena_get_bin` instead.")
 	JEMALLOC_ALIGNED(CACHELINE)
-#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
-	bin_t all_bins[];
-#else
 	bin_t all_bins[0];
-#endif
 };
 
 #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */

include/jemalloc/internal/arenas_management.h

@@ -1,48 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
-#define JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
-
-#include "jemalloc/internal/arena_types.h"
-#include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/tsd_types.h"
-
-/*
- * Arenas that are used to service external requests.  Not all elements of the
- * arenas array are necessarily used; arenas are created lazily as needed.
- *
- * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
- * arenas.  arenas[narenas_auto..narenas_total) are only used if the application
- * takes some action to create them and allocate from them.
- *
- * Points to an arena_t.
- */
-extern atomic_p_t arenas[];
-
-/* Number of arenas used for automatic multiplexing of threads and arenas. */
-extern unsigned narenas_auto;
-
-/* Base index for manual arenas. */
-extern unsigned manual_arena_base;
-
-void     narenas_total_set(unsigned narenas);
-void     narenas_total_inc(void);
-unsigned narenas_total_get(void);
-void     narenas_auto_set(unsigned n);
-void     manual_arena_base_set(unsigned base);
-
-void    *a0malloc(size_t size);
-void     a0dalloc(void *ptr);
-void    *a0ialloc(size_t size, bool zero, bool is_internal);
-void     a0idalloc(void *ptr, bool is_internal);
-void     arena_set(unsigned ind, arena_t *arena);
-arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
-arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
-void     arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
-void     iarena_cleanup(tsd_t *tsd);
-void     arena_cleanup(tsd_t *tsd);
-
-bool     arenas_management_boot(void);
-void     arenas_management_prefork(tsdn_t *tsdn);
-void     arenas_management_postfork_parent(tsdn_t *tsdn);
-void     arenas_management_postfork_child(tsdn_t *tsdn);
-
-#endif /* JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H */

include/jemalloc/internal/bin.h

@@ -95,21 +95,11 @@ bool  bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, bool is_auto,
 void *bin_malloc_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin,
     szind_t binind);
 
-/* Slab queries. */
-void *bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin);
-
 /* Bin selection. */
 bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     unsigned *binshard_p);
 
 /* Stats. */
-static inline void
-bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) {
-	malloc_mutex_lock(tsdn, &bin->lock);
-	bin->stats.nrequests += n;
-	malloc_mutex_unlock(tsdn, &bin->lock);
-}
-
 static inline void
 bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
 	malloc_mutex_lock(tsdn, &bin->lock);

include/jemalloc/internal/bin_inlines.h

@@ -24,8 +24,8 @@ struct bin_dalloc_locked_info_s {
 
 /* Find the region index of a pointer within a slab. */
 JEMALLOC_ALWAYS_INLINE size_t
-bin_slab_regind_impl(const div_info_t *div_info, szind_t binind,
-    const edata_t *slab, const void *ptr) {
+bin_slab_regind_impl(
+    div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) {
 	size_t diff, regind;
 
 	/* Freeing a pointer outside the slab can cause assertion failure. */
@@ -45,8 +45,8 @@ bin_slab_regind_impl(const div_info_t *div_info, szind_t binind,
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-bin_slab_regind(const bin_dalloc_locked_info_t *info, szind_t binind,
-    const edata_t *slab, const void *ptr) {
+bin_slab_regind(bin_dalloc_locked_info_t *info, szind_t binind,
+    edata_t *slab, const void *ptr) {
 	size_t regind = bin_slab_regind_impl(
 	    &info->div_info, binind, slab, ptr);
 	return regind;

include/jemalloc/internal/bit_util.h

@@ -195,7 +195,7 @@ fls_u(unsigned x) {
 	/*								\
 	 * Algorithm from an old AMD optimization reference manual.	\
 	 * We're putting a little bit more work than you might expect	\
-	 * into the no-intrinsic case, since we only support the	\
+	 * into the no-instrinsic case, since we only support the	\
 	 * GCC intrinsics spelling of popcount (for now).  Detecting	\
 	 * whether or not the popcount builtin is actually useable in	\
 	 * MSVC is nontrivial.						\
@@ -212,7 +212,7 @@ fls_u(unsigned x) {
 	 * That costs an extra mask relative to this, though.		\
 	 */								\
 	x = x - ((x >> 1) & (0x55U * bmul));				\
-	/* Replace each 4 bits with their sideways sum.  0x3 = 0b0011. */\
+	/* Replace each 4 bits with their sideays sum.  0x3 = 0b0011. */\
 	x = (x & (bmul * 0x33U)) + ((x >> 2) & (bmul * 0x33U));		\
 	/*								\
 	 * Replace each 8 bits with their sideways sum.  Note that we	\

include/jemalloc/internal/bitmap.h

@@ -181,7 +181,7 @@ void   bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
 size_t bitmap_size(const bitmap_info_t *binfo);
 
 static inline bool
-bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {
+bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
 #ifdef BITMAP_USE_TREE
 	size_t   rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
 	bitmap_t rg = bitmap[rgoff];
@@ -200,7 +200,7 @@ bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {
 }
 
 static inline bool
-bitmap_get(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
+bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
 	size_t   goff;
 	bitmap_t g;
 

include/jemalloc/internal/cache_bin.h

@@ -442,6 +442,19 @@ cache_bin_alloc(cache_bin_t *bin, bool *success) {
 	return cache_bin_alloc_impl(bin, success, true);
 }
 
+JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
+cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) {
+	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
+	if (n > num) {
+		n = (cache_bin_sz_t)num;
+	}
+	memcpy(out, bin->stack_head, n * sizeof(void *));
+	bin->stack_head += n;
+	cache_bin_low_water_adjust(bin);
+
+	return n;
+}
+
 JEMALLOC_ALWAYS_INLINE bool
 cache_bin_full(cache_bin_t *bin) {
 	return (
@@ -523,13 +536,14 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
 /* Get the number of stashed pointers. */
 JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
 cache_bin_nstashed_get_internal(cache_bin_t *bin) {
+	cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
 	cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
 	    bin);
 
 	cache_bin_sz_t n = cache_bin_diff(
 	                       bin, low_bits_low_bound, bin->low_bits_full)
 	    / sizeof(void *);
-	assert(n <= cache_bin_ncached_max_get(bin));
+	assert(n <= ncached_max);
 	if (config_debug && n != 0) {
 		/* Below are for assertions only. */
 		void **low_bound = cache_bin_low_bound_get(bin);

include/jemalloc/internal/ctl.h

@@ -63,7 +63,6 @@ typedef struct ctl_stats_s {
 	size_t resident;
 	size_t mapped;
 	size_t retained;
-	size_t pinned;
 
 	background_thread_stats_t background_thread;
 	mutex_prof_data_t mutex_prof_data[mutex_prof_num_global_mutexes];

include/jemalloc/internal/div.h

@@ -24,7 +24,7 @@ struct div_info_s {
 void div_init(div_info_t *div_info, size_t divisor);
 
 static inline size_t
-div_compute(const div_info_t *div_info, size_t n) {
+div_compute(div_info_t *div_info, size_t n) {
 	assert(n <= (uint32_t)-1);
 	/*
 	 * This generates, e.g. mov; imul; shr on x86-64. On a 32-bit machine,

include/jemalloc/internal/ecache.h

@@ -23,27 +23,27 @@ struct ecache_s {
 };
 
 static inline size_t
-ecache_npages_get(const ecache_t *ecache) {
+ecache_npages_get(ecache_t *ecache) {
 	return eset_npages_get(&ecache->eset)
 	    + eset_npages_get(&ecache->guarded_eset);
 }
 
 /* Get the number of extents in the given page size index. */
 static inline size_t
-ecache_nextents_get(const ecache_t *ecache, pszind_t ind) {
+ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
 	return eset_nextents_get(&ecache->eset, ind)
 	    + eset_nextents_get(&ecache->guarded_eset, ind);
 }
 
 /* Get the sum total bytes of the extents in the given page size index. */
 static inline size_t
-ecache_nbytes_get(const ecache_t *ecache, pszind_t ind) {
+ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
 	return eset_nbytes_get(&ecache->eset, ind)
 	    + eset_nbytes_get(&ecache->guarded_eset, ind);
 }
 
 static inline unsigned
-ecache_ind_get(const ecache_t *ecache) {
+ecache_ind_get(ecache_t *ecache) {
 	return ecache->ind;
 }
 

include/jemalloc/internal/edata.h

@@ -34,10 +34,9 @@ enum extent_state_e {
 	extent_state_dirty = 1,
 	extent_state_muzzy = 2,
 	extent_state_retained = 3,
-	extent_state_pinned = 4,
-	extent_state_transition = 5, /* States below are intermediate. */
-	extent_state_merging = 6,
-	extent_state_max = 6 /* Sanity checking only. */
+	extent_state_transition = 4, /* States below are intermediate. */
+	extent_state_merging = 5,
+	extent_state_max = 5 /* Sanity checking only. */
 };
 typedef enum extent_state_e extent_state_t;
 
@@ -48,8 +47,8 @@ enum extent_head_state_e {
 typedef enum extent_head_state_e extent_head_state_t;
 
 /*
- * Which page allocator implementation (PAC or HPA) owns the given extent?
- * Used by PA to route expand/shrink/dalloc to the correct implementation.
+ * Which implementation of the page allocator interface, (PAI, defined in
+ * pai.h) owns the given extent?
  */
 enum extent_pai_e { EXTENT_PAI_PAC = 0, EXTENT_PAI_HPA = 1 };
 typedef enum extent_pai_e extent_pai_t;
@@ -95,8 +94,8 @@ struct edata_cmp_summary_s {
 
 /* Extent (span of pages).  Use accessor functions for e_* fields. */
 typedef struct edata_s edata_t;
-ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM)
-ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM)
+ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM);
+ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM);
 struct edata_s {
 	/*
 	 * Bitfield containing several fields:
@@ -111,10 +110,8 @@ struct edata_s {
 	 * i: szind
 	 * f: nfree
 	 * s: bin_shard
-	 * h: is_head
-	 * n: pinned
 	 *
-	 * 00000000 ... 0nhsssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
+	 * 00000000 ... 0000ssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
 	 *
 	 * arena_ind: Arena from which this extent came, or all 1 bits if
 	 *            unassociated.
@@ -148,10 +145,6 @@ struct edata_s {
 	 * nfree: Number of free regions in slab.
 	 *
 	 * bin_shard: the shard of the bin from which this extent came.
-	 *
-	 * is_head: see comments in ehooks_default_merge_impl().
-	 *
-	 * pinned: true if the alloc hook signaled non-reclaimable backing.
 	 */
 	uint64_t e_bits;
 #define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT)                         \
@@ -217,16 +210,6 @@ struct edata_s {
 #define EDATA_BITS_IS_HEAD_MASK                                                \
 	MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
 
-#define EDATA_BITS_PINNED_WIDTH 1
-#define EDATA_BITS_PINNED_SHIFT                                                \
-	(EDATA_BITS_IS_HEAD_WIDTH + EDATA_BITS_IS_HEAD_SHIFT)
-#define EDATA_BITS_PINNED_MASK                                                 \
-	MASK(EDATA_BITS_PINNED_WIDTH, EDATA_BITS_PINNED_SHIFT)
-
-#if (EDATA_BITS_PINNED_SHIFT + EDATA_BITS_PINNED_WIDTH > 64)
-#error "edata_t e_bits overflow"
-#endif
-
 	/* Pointer to the extent that this structure is responsible for. */
 	void *e_addr;
 
@@ -555,29 +538,6 @@ edata_ps_set(edata_t *edata, hpdata_t *ps) {
 	edata->e_ps = ps;
 }
 
-static inline bool
-edata_pinned_get(const edata_t *edata) {
-	return (bool)((edata->e_bits & EDATA_BITS_PINNED_MASK)
-	    >> EDATA_BITS_PINNED_SHIFT);
-}
-
-static inline void
-edata_pinned_set(edata_t *edata, bool pinned) {
-	edata->e_bits = (edata->e_bits & ~EDATA_BITS_PINNED_MASK)
-	    | ((uint64_t)pinned << EDATA_BITS_PINNED_SHIFT);
-}
-
-static inline void
-edata_hook_flags_init(edata_t *edata, unsigned alloc_flags) {
-	edata_pinned_set(edata,
-	    (alloc_flags & EXTENT_ALLOC_FLAG_PINNED) != 0);
-}
-
-static inline unsigned
-edata_alloc_flags_get(const edata_t *edata) {
-	return edata_pinned_get(edata) ? EXTENT_ALLOC_FLAG_PINNED : 0;
-}
-
 static inline void
 edata_szind_set(edata_t *edata, szind_t szind) {
 	assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
@@ -685,7 +645,7 @@ edata_prof_recent_alloc_set_dont_call_directly(
 }
 
 static inline bool
-edata_is_head_get(const edata_t *edata) {
+edata_is_head_get(edata_t *edata) {
 	return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK)
 	    >> EDATA_BITS_IS_HEAD_SHIFT);
 }
@@ -726,7 +686,6 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
 	edata_committed_set(edata, committed);
 	edata_pai_set(edata, pai);
 	edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
-	edata_hook_flags_init(edata, 0);
 	if (config_prof) {
 		edata_prof_tctx_set(edata, NULL);
 	}
@@ -752,7 +711,6 @@ edata_binit(
 	 * wasting a state bit to encode this fact.
 	 */
 	edata_pai_set(edata, EXTENT_PAI_PAC);
-	edata_hook_flags_init(edata, 0);
 }
 
 static inline int

include/jemalloc/internal/ehooks.h

@@ -110,12 +110,12 @@ ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
 }
 
 static inline extent_hooks_t *
-ehooks_get_extent_hooks_ptr(const ehooks_t *ehooks) {
+ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
 	return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
 }
 
 static inline bool
-ehooks_are_default(const ehooks_t *ehooks) {
+ehooks_are_default(ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)
 	    == &ehooks_default_extent_hooks;
 }
@@ -126,7 +126,7 @@ ehooks_are_default(const ehooks_t *ehooks) {
  * include some checks for such cases.
  */
 static inline bool
-ehooks_dalloc_will_fail(const ehooks_t *ehooks) {
+ehooks_dalloc_will_fail(ehooks_t *ehooks) {
 	if (ehooks_are_default(ehooks)) {
 		return opt_retain;
 	} else {
@@ -135,17 +135,17 @@ ehooks_dalloc_will_fail(const ehooks_t *ehooks) {
 }
 
 static inline bool
-ehooks_split_will_fail(const ehooks_t *ehooks) {
+ehooks_split_will_fail(ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
 }
 
 static inline bool
-ehooks_merge_will_fail(const ehooks_t *ehooks) {
+ehooks_merge_will_fail(ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
 }
 
 static inline bool
-ehooks_guard_will_fail(const ehooks_t *ehooks) {
+ehooks_guard_will_fail(ehooks_t *ehooks) {
 	/*
 	 * Before the guard hooks are officially introduced, limit the use to
 	 * the default hooks only.
@@ -191,7 +191,7 @@ ehooks_debug_zero_check(void *addr, size_t size) {
 
 static inline void *
 ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
-    size_t alignment, bool *zero, bool *commit, unsigned *alloc_flags) {
+    size_t alignment, bool *zero, bool *commit) {
 	bool            orig_zero = *zero;
 	void           *ret;
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
@@ -204,18 +204,6 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
 		    alignment, zero, commit, ehooks_ind_get(ehooks));
 		ehooks_post_reentrancy(tsdn);
 	}
-#if LG_PAGE < 8
-#  error "Extent alloc flags require page size of at least 256"
-#endif
-	if (ret != NULL) {
-		*alloc_flags = (unsigned)((uintptr_t)ret
-		    & EXTENT_ALLOC_FLAG_MASK);
-		ret = (void *)((byte_t *)ret - *alloc_flags);
-		/* Pinned hooks must also set *commit; pinned bypasses commit/decommit. */
-		assert(!(*alloc_flags & EXTENT_ALLOC_FLAG_PINNED) || *commit);
-	} else {
-		*alloc_flags = 0;
-	}
 	assert(new_addr == NULL || ret == NULL || new_addr == ret);
 	assert(!orig_zero || *zero);
 	if (*zero && ret != NULL) {

include/jemalloc/internal/emap.h

@@ -211,7 +211,6 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) {
 	assert(edata_state_get(inner) == extent_state_active);
 	assert(edata_state_get(outer) == extent_state_merging);
 	assert(!edata_guarded_get(inner) && !edata_guarded_get(outer));
-	assert(edata_pinned_get(inner) == edata_pinned_get(outer));
 	assert(edata_base_get(inner) == edata_past_get(outer)
 	    || edata_base_get(outer) == edata_past_get(inner));
 }

include/jemalloc/internal/eset.h

@@ -60,11 +60,11 @@ struct eset_s {
 
 void eset_init(eset_t *eset, extent_state_t state);
 
-size_t eset_npages_get(const eset_t *eset);
+size_t eset_npages_get(eset_t *eset);
 /* Get the number of extents in the given page size index. */
-size_t eset_nextents_get(const eset_t *eset, pszind_t ind);
+size_t eset_nextents_get(eset_t *eset, pszind_t ind);
 /* Get the sum total bytes of the extents in the given page size index. */
-size_t eset_nbytes_get(const eset_t *eset, pszind_t ind);
+size_t eset_nbytes_get(eset_t *eset, pszind_t ind);
 
 void eset_insert(eset_t *eset, edata_t *edata);
 void eset_remove(eset_t *eset, edata_t *edata);
@@ -73,6 +73,6 @@ void eset_remove(eset_t *eset, edata_t *edata);
  * null if no such item could be found.
  */
 edata_t *eset_fit(eset_t *eset, size_t esize, size_t alignment, bool exact_only,
-    unsigned lg_max_fit, bool prefer_small);
+    unsigned lg_max_fit);
 
 #endif /* JEMALLOC_INTERNAL_ESET_H */

include/jemalloc/internal/extent.h

@@ -91,7 +91,7 @@ extent_neighbor_head_state_mergeable(
 }
 
 JEMALLOC_ALWAYS_INLINE bool
-extent_can_acquire_neighbor(const edata_t *edata, rtree_contents_t contents,
+extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
     extent_pai_t pai, extent_state_t expected_state, bool forward,
     bool expanding) {
 	edata_t *neighbor = contents.edata;
@@ -120,10 +120,6 @@ extent_can_acquire_neighbor(const edata_t *edata, rtree_contents_t contents,
 			 */
 			return false;
 		}
-		/* Do not merge pinned and non-pinned extents. */
-		if (edata_pinned_get(edata) != edata_pinned_get(neighbor)) {
-			return false;
-		}
 	} else {
 		if (neighbor_state == extent_state_active) {
 			return false;

include/jemalloc/internal/extent_dss.h

@@ -27,9 +27,4 @@ bool       extent_in_dss(void *addr);
 bool       extent_dss_mergeable(void *addr_a, void *addr_b);
 void       extent_dss_boot(void);
 
-#ifdef JEMALLOC_JET
-typedef void *(*extent_dss_sbrk_hook_t)(intptr_t);
-extern extent_dss_sbrk_hook_t extent_dss_sbrk_hook;
-#endif
-
 #endif /* JEMALLOC_INTERNAL_EXTENT_DSS_H */

include/jemalloc/internal/fb.h

@@ -25,7 +25,7 @@ fb_init(fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_empty(const fb_group_t *fb, size_t nbits) {
+fb_empty(fb_group_t *fb, size_t nbits) {
 	size_t ngroups = FB_NGROUPS(nbits);
 	for (size_t i = 0; i < ngroups; i++) {
 		if (fb[i] != 0) {
@@ -36,7 +36,7 @@ fb_empty(const fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_full(const fb_group_t *fb, size_t nbits) {
+fb_full(fb_group_t *fb, size_t nbits) {
 	size_t ngroups = FB_NGROUPS(nbits);
 	size_t trailing_bits = nbits % FB_GROUP_BITS;
 	size_t limit = (trailing_bits == 0 ? ngroups : ngroups - 1);
@@ -52,7 +52,7 @@ fb_full(const fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_get(const fb_group_t *fb, size_t nbits, size_t bit) {
+fb_get(fb_group_t *fb, size_t nbits, size_t bit) {
 	assert(bit < nbits);
 	size_t group_ind = bit / FB_GROUP_BITS;
 	size_t bit_ind = bit % FB_GROUP_BITS;
@@ -156,21 +156,15 @@ fb_scount_visitor(void *ctx, fb_group_t *fb, fb_group_t mask) {
 
 /* Finds the number of set bit in the of length cnt starting at start. */
 JEMALLOC_ALWAYS_INLINE size_t
-fb_scount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
+fb_scount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
 	size_t scount = 0;
-	/*
-	 * fb_visit_impl is shared with mutating visitors (e.g. fb_set_range),
-	 * so it takes a non-const fb.  fb_scount_visitor only reads, so the
-	 * cast is safe.
-	 */
-	fb_visit_impl((fb_group_t *)fb, nbits, &fb_scount_visitor, &scount,
-	    start, cnt);
+	fb_visit_impl(fb, nbits, &fb_scount_visitor, &scount, start, cnt);
 	return scount;
 }
 
 /* Finds the number of unset bit in the of length cnt starting at start. */
 JEMALLOC_ALWAYS_INLINE size_t
-fb_ucount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
+fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
 	size_t scount = fb_scount(fb, nbits, start, cnt);
 	return cnt - scount;
 }
@@ -182,8 +176,8 @@ fb_ucount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
  * Returns the number of bits in the bitmap if no such bit exists.
  */
 JEMALLOC_ALWAYS_INLINE ssize_t
-fb_find_impl(const fb_group_t *fb, size_t nbits, size_t start, bool val,
-    bool forward) {
+fb_find_impl(
+    fb_group_t *fb, size_t nbits, size_t start, bool val, bool forward) {
 	assert(start < nbits);
 	size_t  ngroups = FB_NGROUPS(nbits);
 	ssize_t group_ind = start / FB_GROUP_BITS;
@@ -232,14 +226,14 @@ fb_find_impl(const fb_group_t *fb, size_t nbits, size_t start, bool val,
  * number of bits in the bitmap if no such bit exists.
  */
 static inline size_t
-fb_ffu(const fb_group_t *fb, size_t nbits, size_t min_bit) {
+fb_ffu(fb_group_t *fb, size_t nbits, size_t min_bit) {
 	return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ false,
 	    /* forward */ true);
 }
 
 /* The same, but looks for an unset bit. */
 static inline size_t
-fb_ffs(const fb_group_t *fb, size_t nbits, size_t min_bit) {
+fb_ffs(fb_group_t *fb, size_t nbits, size_t min_bit) {
 	return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ true,
 	    /* forward */ true);
 }
@@ -249,21 +243,21 @@ fb_ffs(const fb_group_t *fb, size_t nbits, size_t min_bit) {
  * no such bit exists.
  */
 static inline ssize_t
-fb_flu(const fb_group_t *fb, size_t nbits, size_t max_bit) {
+fb_flu(fb_group_t *fb, size_t nbits, size_t max_bit) {
 	return fb_find_impl(fb, nbits, max_bit, /* val */ false,
 	    /* forward */ false);
 }
 
 static inline ssize_t
-fb_fls(const fb_group_t *fb, size_t nbits, size_t max_bit) {
+fb_fls(fb_group_t *fb, size_t nbits, size_t max_bit) {
 	return fb_find_impl(fb, nbits, max_bit, /* val */ true,
 	    /* forward */ false);
 }
 
 /* Returns whether or not we found a range. */
 JEMALLOC_ALWAYS_INLINE bool
-fb_iter_range_impl(const fb_group_t *fb, size_t nbits, size_t start,
-    size_t *r_begin, size_t *r_len, bool val, bool forward) {
+fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
+    size_t *r_len, bool val, bool forward) {
 	assert(start < nbits);
 	ssize_t next_range_begin = fb_find_impl(fb, nbits, start, val, forward);
 	if ((forward && next_range_begin == (ssize_t)nbits)
@@ -292,8 +286,8 @@ fb_iter_range_impl(const fb_group_t *fb, size_t nbits, size_t start,
  * touching *r_begin or *r_end).
  */
 static inline bool
-fb_srange_iter(const fb_group_t *fb, size_t nbits, size_t start,
-    size_t *r_begin, size_t *r_len) {
+fb_srange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
+    size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ true, /* forward */ true);
 }
@@ -303,30 +297,30 @@ fb_srange_iter(const fb_group_t *fb, size_t nbits, size_t start,
  * forwards.  (The position returned is still the earliest bit in the range).
  */
 static inline bool
-fb_srange_riter(const fb_group_t *fb, size_t nbits, size_t start,
-    size_t *r_begin, size_t *r_len) {
+fb_srange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
+    size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ true, /* forward */ false);
 }
 
 /* Similar to fb_srange_iter, but searches for unset bits. */
 static inline bool
-fb_urange_iter(const fb_group_t *fb, size_t nbits, size_t start,
-    size_t *r_begin, size_t *r_len) {
+fb_urange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
+    size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ false, /* forward */ true);
 }
 
 /* Similar to fb_srange_riter, but searches for unset bits. */
 static inline bool
-fb_urange_riter(const fb_group_t *fb, size_t nbits, size_t start,
-    size_t *r_begin, size_t *r_len) {
+fb_urange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
+    size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ false, /* forward */ false);
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-fb_range_longest_impl(const fb_group_t *fb, size_t nbits, bool val) {
+fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
 	size_t begin = 0;
 	size_t longest_len = 0;
 	size_t len = 0;
@@ -342,12 +336,12 @@ fb_range_longest_impl(const fb_group_t *fb, size_t nbits, bool val) {
 }
 
 static inline size_t
-fb_srange_longest(const fb_group_t *fb, size_t nbits) {
+fb_srange_longest(fb_group_t *fb, size_t nbits) {
 	return fb_range_longest_impl(fb, nbits, /* val */ true);
 }
 
 static inline size_t
-fb_urange_longest(const fb_group_t *fb, size_t nbits) {
+fb_urange_longest(fb_group_t *fb, size_t nbits) {
 	return fb_range_longest_impl(fb, nbits, /* val */ false);
 }
 

include/jemalloc/internal/hook.h

@@ -0,0 +1,163 @@
+#ifndef JEMALLOC_INTERNAL_HOOK_H
+#define JEMALLOC_INTERNAL_HOOK_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/tsd.h"
+
+/*
+ * This API is *extremely* experimental, and may get ripped out, changed in API-
+ * and ABI-incompatible ways, be insufficiently or incorrectly documented, etc.
+ *
+ * It allows hooking the stateful parts of the API to see changes as they
+ * happen.
+ *
+ * Allocation hooks are called after the allocation is done, free hooks are
+ * called before the free is done, and expand hooks are called after the
+ * allocation is expanded.
+ *
+ * For realloc and rallocx, if the expansion happens in place, the expansion
+ * hook is called.  If it is moved, then the alloc hook is called on the new
+ * location, and then the free hook is called on the old location (i.e. both
+ * hooks are invoked in between the alloc and the dalloc).
+ *
+ * If we return NULL from OOM, then usize might not be trustworthy.  Calling
+ * realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0)
+ * only calls the free hook.  (Calling realloc(NULL, 0) is treated as malloc(0),
+ * and only calls the alloc hook).
+ *
+ * Reentrancy:
+ *   Reentrancy is guarded against from within the hook implementation.  If you
+ *   call allocator functions from within a hook, the hooks will not be invoked
+ *   again.
+ * Threading:
+ *   The installation of a hook synchronizes with all its uses.  If you can
+ *   prove the installation of a hook happens-before a jemalloc entry point,
+ *   then the hook will get invoked (unless there's a racing removal).
+ *
+ *   Hook insertion appears to be atomic at a per-thread level (i.e. if a thread
+ *   allocates and has the alloc hook invoked, then a subsequent free on the
+ *   same thread will also have the free hook invoked).
+ *
+ *   The *removal* of a hook does *not* block until all threads are done with
+ *   the hook.  Hook authors have to be resilient to this, and need some
+ *   out-of-band mechanism for cleaning up any dynamically allocated memory
+ *   associated with their hook.
+ * Ordering:
+ *   Order of hook execution is unspecified, and may be different than insertion
+ *   order.
+ */
+
+#define HOOK_MAX 4
+
+enum hook_alloc_e {
+	hook_alloc_malloc,
+	hook_alloc_posix_memalign,
+	hook_alloc_aligned_alloc,
+	hook_alloc_calloc,
+	hook_alloc_memalign,
+	hook_alloc_valloc,
+	hook_alloc_pvalloc,
+	hook_alloc_mallocx,
+
+	/* The reallocating functions have both alloc and dalloc variants */
+	hook_alloc_realloc,
+	hook_alloc_rallocx,
+};
+/*
+ * We put the enum typedef after the enum, since this file may get included by
+ * jemalloc_cpp.cpp, and C++ disallows enum forward declarations.
+ */
+typedef enum hook_alloc_e hook_alloc_t;
+
+enum hook_dalloc_e {
+	hook_dalloc_free,
+	hook_dalloc_dallocx,
+	hook_dalloc_sdallocx,
+
+	/*
+	 * The dalloc halves of reallocation (not called if in-place expansion
+	 * happens).
+	 */
+	hook_dalloc_realloc,
+	hook_dalloc_rallocx,
+};
+typedef enum hook_dalloc_e hook_dalloc_t;
+
+enum hook_expand_e {
+	hook_expand_realloc,
+	hook_expand_rallocx,
+	hook_expand_xallocx,
+};
+typedef enum hook_expand_e hook_expand_t;
+
+typedef void (*hook_alloc)(void *extra, hook_alloc_t type, void *result,
+    uintptr_t result_raw, uintptr_t args_raw[3]);
+
+typedef void (*hook_dalloc)(
+    void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
+
+typedef void (*hook_expand)(void *extra, hook_expand_t type, void *address,
+    size_t old_usize, size_t new_usize, uintptr_t result_raw,
+    uintptr_t args_raw[4]);
+
+typedef struct hooks_s hooks_t;
+struct hooks_s {
+	hook_alloc  alloc_hook;
+	hook_dalloc dalloc_hook;
+	hook_expand expand_hook;
+	void       *extra;
+};
+
+/*
+ * Begin implementation details; everything above this point might one day live
+ * in a public API.  Everything below this point never will.
+ */
+
+/*
+ * The realloc pathways haven't gotten any refactoring love in a while, and it's
+ * fairly difficult to pass information from the entry point to the hooks.  We
+ * put the informaiton the hooks will need into a struct to encapsulate
+ * everything.
+ *
+ * Much of these pathways are force-inlined, so that the compiler can avoid
+ * materializing this struct until we hit an extern arena function.  For fairly
+ * goofy reasons, *many* of the realloc paths hit an extern arena function.
+ * These paths are cold enough that it doesn't matter; eventually, we should
+ * rewrite the realloc code to make the expand-in-place and the
+ * free-then-realloc paths more orthogonal, at which point we don't need to
+ * spread the hook logic all over the place.
+ */
+typedef struct hook_ralloc_args_s hook_ralloc_args_t;
+struct hook_ralloc_args_s {
+	/* I.e. as opposed to rallocx. */
+	bool is_realloc;
+	/*
+	 * The expand hook takes 4 arguments, even if only 3 are actually used;
+	 * we add an extra one in case the user decides to memcpy without
+	 * looking too closely at the hooked function.
+	 */
+	uintptr_t args[4];
+};
+
+/*
+ * Returns an opaque handle to be used when removing the hook.  NULL means that
+ * we couldn't install the hook.
+ */
+bool hook_boot(void);
+
+void *hook_install(tsdn_t *tsdn, hooks_t *to_install);
+/* Uninstalls the hook with the handle previously returned from hook_install. */
+void hook_remove(tsdn_t *tsdn, void *opaque);
+
+/* Hooks */
+
+void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
+    uintptr_t args_raw[3]);
+
+void hook_invoke_dalloc(
+    hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
+
+void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
+    size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
+
+#endif /* JEMALLOC_INTERNAL_HOOK_H */

include/jemalloc/internal/hpa.h

@@ -10,6 +10,7 @@
 #include "jemalloc/internal/hpa_hooks.h"
 #include "jemalloc/internal/hpa_opts.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/psset.h"
 #include "jemalloc/internal/sec.h"
 
@@ -50,60 +51,6 @@ struct hpa_shard_nonderived_stats_s {
 	 * Guarded by mtx.
 	 */
 	uint64_t ndehugifies;
-
-	/*
-	 * Distribution of the min number of extents we will try to allocate
-	 * from a single hpa_alloc() call.
-	 *
-	 * Guarded by mtx.
-	 */
-	uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
-
-	/*
-	 * Distribution of the max number of extents we will try to allocate
-	 * from a single hpa_alloc() call.
-	 *
-	 * Guarded by mtx.
-	 */
-	uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
-
-	/*
-	 * Distribution of the number of extents allocated for a single
-	 * hpa_alloc() call and a single mtx lock acquisition.
-	 *
-	 * Guarded by mtx.
-	 */
-	uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
-
-	/*
-	 * Distribution of the number of ps out of which we allocated extents
-	 * for a single hpa_alloc() call and a single mtx lock acquisition.
-	 *
-	 * Guarded by mtx.
-	 */
-	uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
-
-	/*
-	 * Distribution of the number of pages allocated from a single ps.
-	 *
-	 * Guarded by mtx.
-	 */
-	uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
-
-	/*
-	 * Distribution of the number of extents allocated from a single ps.
-	 *
-	 * Guarded by mtx.
-	 */
-	uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
-
-	/*
-	 * Distribution of the total elapsed time (ns) for allocating extents
-	 * from a single ps.
-	 *
-	 * Guarded by mtx.
-	 */
-	uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
 };
 
 /* Completely derived; only used by CTL. */
@@ -116,6 +63,12 @@ struct hpa_shard_stats_s {
 
 typedef struct hpa_shard_s hpa_shard_t;
 struct hpa_shard_s {
+	/*
+	 * pai must be the first member; we cast from a pointer to it to a
+	 * pointer to the hpa_shard_t.
+	 */
+	pai_t pai;
+
 	/* The central allocator we get our hugepages from. */
 	hpa_central_t *central;
 
@@ -192,7 +145,7 @@ struct hpa_shard_s {
 
 bool hpa_hugepage_size_exceeds_limit(void);
 /*
- * Whether or not the HPA can be used given the current configuration.  This
+ * Whether or not the HPA can be used given the current configuration.  This is
  * is not necessarily a guarantee that it backs its allocations by hugepages,
  * just that it can function properly given the system it's running on.
  */
@@ -201,18 +154,6 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
     emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
     const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts);
 
-edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
-    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
-    bool *deferred_work_generated);
-bool hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool zero,
-    bool *deferred_work_generated);
-bool hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    size_t old_size, size_t new_size, bool *deferred_work_generated);
-void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    bool *deferred_work_generated);
-uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
-
 void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
 void hpa_shard_stats_merge(
     tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst);

include/jemalloc/internal/hpdata.h

@@ -27,7 +27,7 @@
  */
 #define PSSET_ENUMERATE_MAX_NUM 32
 typedef struct hpdata_s hpdata_t;
-ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM)
+ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM);
 struct hpdata_s {
 	/*
 	 * We likewise follow the edata convention of mangling names and forcing
@@ -136,7 +136,7 @@ TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty)
 TYPED_LIST(hpdata_purge_list, hpdata_t, ql_link_purge)
 TYPED_LIST(hpdata_hugify_list, hpdata_t, ql_link_hugify)
 
-ph_proto(, hpdata_age_heap, hpdata_t)
+ph_proto(, hpdata_age_heap, hpdata_t);
 
 static inline void *
 hpdata_addr_get(const hpdata_t *hpdata) {
@@ -209,7 +209,7 @@ hpdata_allow_hugify(hpdata_t *hpdata, nstime_t now) {
 }
 
 static inline nstime_t
-hpdata_time_hugify_allowed(const hpdata_t *hpdata) {
+hpdata_time_hugify_allowed(hpdata_t *hpdata) {
 	return hpdata->h_time_hugify_allowed;
 }
 
@@ -305,7 +305,7 @@ hpdata_ndirty_get(const hpdata_t *hpdata) {
 }
 
 static inline size_t
-hpdata_nretained_get(const hpdata_t *hpdata) {
+hpdata_nretained_get(hpdata_t *hpdata) {
 	return HUGEPAGE_PAGES - hpdata->h_ntouched;
 }
 
@@ -330,7 +330,7 @@ hpdata_purged_when_empty_and_huge_set(hpdata_t *hpdata, bool v) {
 }
 
 static inline void
-hpdata_assert_empty(const hpdata_t *hpdata) {
+hpdata_assert_empty(hpdata_t *hpdata) {
 	assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES));
 	assert(hpdata->h_nactive == 0);
 }
@@ -341,7 +341,7 @@ hpdata_assert_empty(const hpdata_t *hpdata) {
  * match computed ones).
  */
 static inline bool
-hpdata_consistent(const hpdata_t *hpdata) {
+hpdata_consistent(hpdata_t *hpdata) {
 	bool res = true;
 
 	const size_t active_urange_longest = fb_urange_longest(
@@ -432,60 +432,6 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
-/*
- * For buffering extent allocations we will perform out of
- * a single ps.
- */
-typedef struct hpdata_alloc_offset_s hpdata_alloc_offset_t;
-struct hpdata_alloc_offset_s {
-	/*
-	 * Index on the active bitmap for the extent to allocate.
-	 * It is used to know which bits we'll need to set when we perform
-	 * the allocation. They are in the range [index, index + npages).
-	 */
-	size_t index;
-
-	/*
-	 * The length of the free bit range on the active bitmap,
-	 * starting at index, before setting the bits in the range
-	 * [index, index + npages).
-	 * It is used to determine whether one of the allocations
-	 * used up the longest free range on the active bitmap.
-	 * If it did, we might have to update the longest free range
-	 * metadata on the hpdata.
-	 */
-	size_t len_before;
-
-	/*
-	 * The length of the longest free range in the range [0, index).
-	 * When we need to update the longest free range on the hpdata,
-	 * the new value is either longest_len (the max up to index),
-	 * len_before - npages (what's left after we carve up the free
-	 * range starting at index), or the max in the range
-	 * [index + len_before, HUGEPAGE_PAGES), whichever is greater.
-	 */
-	size_t longest_len;
-};
-
-/*
- * Given an hpdata that can serve an allocation request of size sz,
- * find between one and max_nallocs offsets that can satisfy such
- * an allocation request and buffer them in offsets (without actually
- * reserving any space or updating hpdata). Return the number
- * of offsets discovered.
- */
-size_t hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
-    hpdata_alloc_offset_t *offsets, size_t max_nallocs);
-/* Reserve the allocation for the given offset. */
-void *hpdata_reserve_alloc_offset(
-    hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset);
-/*
- * Do any work that needs to be done after performing allocations
- * from a single hpdata.
- */
-void hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
-    hpdata_alloc_offset_t *offsets, size_t nallocs);
-
 /*
  * The hpdata_purge_prepare_t allows grabbing the metadata required to purge
  * subranges of a hugepage while holding a lock, drop the lock during the actual

include/jemalloc/internal/jemalloc_fork.h

@@ -1,8 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_JEMALLOC_FORK_H
-#define JEMALLOC_INTERNAL_JEMALLOC_FORK_H
-
-void jemalloc_prefork(void);
-void jemalloc_postfork_parent(void);
-void jemalloc_postfork_child(void);
-
-#endif /* JEMALLOC_INTERNAL_JEMALLOC_FORK_H */

include/jemalloc/internal/jemalloc_init.h

@@ -1,40 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_JEMALLOC_INIT_H
-#define JEMALLOC_INTERNAL_JEMALLOC_INIT_H
-
-enum malloc_init_e {
-	malloc_init_uninitialized = 3,
-	malloc_init_a0_initialized = 2,
-	malloc_init_recursible = 1,
-	malloc_init_initialized = 0 /* Common case --> jnz. */
-};
-typedef enum malloc_init_e malloc_init_t;
-
-extern malloc_init_t malloc_init_state;
-
-bool malloc_is_initializer(void);
-
-bool malloc_init_hard_a0(void);
-bool malloc_init_hard(void);
-
-JEMALLOC_ALWAYS_INLINE bool
-malloc_init_a0(void) {
-	if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
-		return malloc_init_hard_a0();
-	}
-	return false;
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-malloc_initialized(void) {
-	return (malloc_init_state == malloc_init_initialized);
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-malloc_init(void) {
-	if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
-		return true;
-	}
-	return false;
-}
-
-#endif /* JEMALLOC_INTERNAL_JEMALLOC_INIT_H */

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -465,9 +465,6 @@
 /* Is C++ support being built? */
 #undef JEMALLOC_ENABLE_CXX
 
-/* Are C++ exceptions enabled? */
-#undef JEMALLOC_HAVE_CXX_EXCEPTIONS
-
 /* Performs additional size checks when defined. */
 #undef JEMALLOC_OPT_SIZE_CHECKS
 

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -37,6 +37,7 @@ extern bool                  opt_zero;
 extern unsigned              opt_narenas;
 extern fxp_t                 opt_narenas_ratio;
 extern zero_realloc_action_t opt_zero_realloc_action;
+extern malloc_init_t         malloc_init_state;
 extern const char *const     zero_realloc_mode_names[];
 extern atomic_zu_t           zero_realloc_count;
 extern bool                  opt_cache_oblivious;
@@ -53,9 +54,36 @@ extern uintptr_t san_cache_bin_nonfast_mask;
 /* Number of CPUs. */
 extern unsigned ncpus;
 
+/* Number of arenas used for automatic multiplexing of threads and arenas. */
+extern unsigned narenas_auto;
+
+/* Base index for manual arenas. */
+extern unsigned manual_arena_base;
+
+/*
+ * Arenas that are used to service external requests.  Not all elements of the
+ * arenas array are necessarily used; arenas are created lazily as needed.
+ */
+extern atomic_p_t arenas[];
+
+extern unsigned huge_arena_ind;
+
+void    *a0malloc(size_t size);
+void     a0dalloc(void *ptr);
 void    *bootstrap_malloc(size_t size);
 void    *bootstrap_calloc(size_t num, size_t size);
 void     bootstrap_free(void *ptr);
+void     arena_set(unsigned ind, arena_t *arena);
+unsigned narenas_total_get(void);
+arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
+arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
+void     arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
+void     iarena_cleanup(tsd_t *tsd);
+void     arena_cleanup(tsd_t *tsd);
+size_t   batch_alloc(void **ptrs, size_t num, size_t size, int flags);
+void     jemalloc_prefork(void);
+void     jemalloc_postfork_parent(void);
+void     jemalloc_postfork_child(void);
 void     sdallocx_default(void *ptr, size_t size, int flags);
 void     free_default(void *ptr);
 void    *malloc_default(size_t size);

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -4,7 +4,6 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/arena_externs.h"
 #include "jemalloc/internal/arena_types.h"
-#include "jemalloc/internal/arenas_management.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bit_util.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"

include/jemalloc/internal/jemalloc_internal_inlines_b.h

@@ -3,22 +3,9 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/arena_inlines_a.h"
-#include "jemalloc/internal/arenas_management.h"
 #include "jemalloc/internal/extent.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_a.h"
 
-static inline void
-thread_migrate_arena(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
-	assert(oldarena != NULL);
-	assert(newarena != NULL);
-
-	arena_migrate(tsd, oldarena, newarena);
-	if (tcache_available(tsd)) {
-		tcache_arena_reassociate(tsd_tsdn(tsd),
-		    tsd_tcache_slowp_get(tsd), newarena);
-	}
-}
-
 static inline void
 percpu_arena_update(tsd_t *tsd, unsigned cpu) {
 	assert(have_percpu_arena);
@@ -31,7 +18,15 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
 		arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
 		assert(newarena != NULL);
 
-		thread_migrate_arena(tsd, oldarena, newarena);
+		/* Set new arena/tcache associations. */
+		arena_migrate(tsd, oldarena, newarena);
+		tcache_t *tcache = tcache_get(tsd);
+		if (tcache != NULL) {
+			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
+			assert(tcache_slow->arena != NULL);
+			tcache_arena_reassociate(
+			    tsd_tsdn(tsd), tcache_slow, tcache, newarena);
+		}
 	}
 }
 
@@ -55,17 +50,18 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
 		assert(ret);
 		if (tcache_available(tsd)) {
 			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
+			tcache_t      *tcache = tsd_tcachep_get(tsd);
 			if (tcache_slow->arena != NULL) {
 				/* See comments in tsd_tcache_data_init().*/
 				assert(tcache_slow->arena
 				    == arena_get(tsd_tsdn(tsd), 0, false));
 				if (tcache_slow->arena != ret) {
 					tcache_arena_reassociate(tsd_tsdn(tsd),
-					    tcache_slow, ret);
+					    tcache_slow, tcache, ret);
 				}
 			} else {
 				tcache_arena_associate(
-				    tsd_tsdn(tsd), tcache_slow, ret);
+				    tsd_tsdn(tsd), tcache_slow, tcache, ret);
 			}
 		}
 	}
@@ -101,7 +97,7 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) {
 }
 
 static inline bool
-arena_is_auto(const arena_t *arena) {
+arena_is_auto(arena_t *arena) {
 	assert(narenas_auto > 0);
 
 	return (arena_ind_get(arena) < manual_arena_base);

include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -5,7 +5,7 @@
 #include "jemalloc/internal/arena_externs.h"
 #include "jemalloc/internal/arena_inlines_b.h"
 #include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/jemalloc_init.h"
+#include "jemalloc/internal/hook.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/log.h"
 #include "jemalloc/internal/sz.h"
@@ -174,7 +174,8 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
+    hook_ralloc_args_t *hook_args) {
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	void  *p;
@@ -195,13 +196,27 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 	memcpy(p, ptr, copysize);
+	hook_invoke_alloc(
+	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, p,
+	    (uintptr_t)p, hook_args->args);
+	hook_invoke_dalloc(
+	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
+	    ptr, hook_args->args);
 	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
 	return p;
 }
 
+/*
+ * is_realloc threads through the knowledge of whether or not this call comes
+ * from je_realloc (as opposed to je_rallocx); this ensures that we pass the
+ * correct entry point into any hooks.
+ * Note that these functions are all force-inlined, so no actual bool gets
+ * passed-around anywhere.
+ */
 JEMALLOC_ALWAYS_INLINE void *
 iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
+    hook_ralloc_args_t *hook_args) {
 	assert(ptr != NULL);
 	assert(size != 0);
 	witness_assert_depth_to_rank(
@@ -214,26 +229,27 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 		 * and copy.
 		 */
 		return iralloct_realign(tsdn, ptr, oldsize, size, alignment,
-		    zero, slab, tcache, arena);
+		    zero, slab, tcache, arena, hook_args);
 	}
 
 	return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
-	    slab, tcache);
+	    slab, tcache, hook_args);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
-    size_t usize, bool zero, tcache_t *tcache, arena_t *arena) {
+    size_t usize, bool zero, tcache_t *tcache, arena_t *arena,
+    hook_ralloc_args_t *hook_args) {
 	bool slab = sz_can_use_slab(usize);
 	return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero,
-	    slab, tcache, arena);
+	    slab, tcache, arena, hook_args);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
-    size_t usize, bool zero) {
+    size_t usize, bool zero, hook_ralloc_args_t *hook_args) {
 	return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, usize,
-	    zero, tcache_get(tsd), NULL);
+	    zero, tcache_get(tsd), NULL, hook_args);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -264,6 +280,11 @@ fastpath_success_finish(
 	}
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+malloc_initialized(void) {
+	return (malloc_init_state == malloc_init_initialized);
+}
+
 /*
  * malloc() fastpath.  Included here so that we can inline it into operator new;
  * function call overhead there is non-negligible as a fraction of total CPU in

include/jemalloc/internal/jemalloc_internal_types.h

@@ -20,6 +20,14 @@ typedef enum zero_realloc_action_e zero_realloc_action_t;
 /* Signature of write callback. */
 typedef void(write_cb_t)(void *, const char *);
 
+enum malloc_init_e {
+	malloc_init_uninitialized = 3,
+	malloc_init_a0_initialized = 2,
+	malloc_init_recursible = 1,
+	malloc_init_initialized = 0 /* Common case --> jnz. */
+};
+typedef enum malloc_init_e malloc_init_t;
+
 /*
  * Flags bits:
  *

include/jemalloc/internal/large_externs.h

@@ -3,6 +3,7 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/edata.h"
+#include "jemalloc/internal/hook.h"
 
 void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
 void *large_palloc(
@@ -10,7 +11,8 @@ void *large_palloc(
 bool  large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
      size_t usize_max, bool zero);
 void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
-    size_t alignment, bool zero, tcache_t *tcache);
+    size_t alignment, bool zero, tcache_t *tcache,
+    hook_ralloc_args_t *hook_args);
 
 void   large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata);
 void   large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);

include/jemalloc/internal/mutex_prof.h

@@ -30,7 +30,6 @@ typedef enum {
 	OP(extents_dirty)                                                      \
 	OP(extents_muzzy)                                                      \
 	OP(extents_retained)                                                   \
-	OP(extents_pinned)                                                     \
 	OP(decay_dirty)                                                        \
 	OP(decay_muzzy)                                                        \
 	OP(base)                                                               \

include/jemalloc/internal/pa.h

@@ -10,13 +10,15 @@
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/lockedint.h"
 #include "jemalloc/internal/pac.h"
+#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/sec.h"
 
 /*
  * The page allocator; responsible for acquiring pages of memory for
- * allocations.  It dispatches each page-level allocation request to either
- * the PAC ("page allocator classic") or the HPA ("hugepage allocator")
- * by calling their pac_*() / hpa_*() entry points directly.
+ * allocations.  It picks the implementation of the page allocator interface
+ * (i.e. a pai_t) to handle a given page-level allocation request.  For now, the
+ * only such implementation is the PAC code ("page allocator classic"), but
+ * others will be coming soon.
  */
 
 typedef struct pa_central_s pa_central_t;
@@ -94,7 +96,7 @@ struct pa_shard_s {
 	/* Allocates from a PAC. */
 	pac_t pac;
 
-	hpa_shard_t hpa;
+	hpa_shard_t hpa_shard;
 
 	/* The source of edata_t objects. */
 	edata_cache_t edata_cache;
@@ -118,7 +120,7 @@ pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
 }
 
 static inline ehooks_t *
-pa_shard_ehooks_get(const pa_shard_t *shard) {
+pa_shard_ehooks_get(pa_shard_t *shard) {
 	return base_ehooks_get(shard->base);
 }
 
@@ -219,12 +221,12 @@ void pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
 
-size_t pa_shard_nactive(const pa_shard_t *shard);
-size_t pa_shard_ndirty(const pa_shard_t *shard);
-size_t pa_shard_nmuzzy(const pa_shard_t *shard);
+size_t pa_shard_nactive(pa_shard_t *shard);
+size_t pa_shard_ndirty(pa_shard_t *shard);
+size_t pa_shard_nmuzzy(pa_shard_t *shard);
 
 void pa_shard_basic_stats_merge(
-    const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
+    pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
 
 void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,

include/jemalloc/internal/pac.h

@@ -4,11 +4,10 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/decay.h"
 #include "jemalloc/internal/ecache.h"
-#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
 #include "jemalloc/internal/lockedint.h"
-#include "jemalloc/internal/tsd_types.h"
+#include "jemalloc/internal/pai.h"
 #include "san_bump.h"
 
 /*
@@ -52,8 +51,6 @@ struct pac_estats_s {
 	size_t muzzy_bytes;
 	size_t nretained;
 	size_t retained_bytes;
-	size_t npinned;
-	size_t pinned_bytes;
 };
 
 typedef struct pac_stats_s pac_stats_t;
@@ -64,14 +61,9 @@ struct pac_stats_s {
 	/*
 	 * Number of unused virtual memory bytes currently retained.  Retained
 	 * bytes are technically mapped (though always decommitted or purged),
-	 * but they are excluded from pac_mapped.
+	 * but they are excluded from the mapped statistic (above).
 	 */
 	size_t retained; /* Derived. */
-	/*
-	 * Number of bytes in pinned (non-reclaimable) extents currently
-	 * cached.  Unlike retained, pinned bytes count toward pac_mapped.
-	 */
-	size_t pinned;   /* Derived. */
 
 	/*
 	 * Number of bytes currently mapped, excluding retained memory (and any
@@ -88,8 +80,11 @@ struct pac_stats_s {
 
 typedef struct pac_s pac_t;
 struct pac_s {
-	/* True once pinned memory has been seen. */
-	atomic_b_t has_pinned;
+	/*
+	 * Must be the first member (we convert it to a PAC given only a
+	 * pointer).  The handle to the allocation interface.
+	 */
+	pai_t pai;
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.
@@ -99,7 +94,6 @@ struct pac_s {
 	ecache_t ecache_dirty;
 	ecache_t ecache_muzzy;
 	ecache_t ecache_retained;
-	ecache_t ecache_pinned;
 
 	base_t        *base;
 	emap_t        *emap;
@@ -161,39 +155,13 @@ bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
     ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
     malloc_mutex_t *stats_mtx);
 
-edata_t *pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment,
-    bool zero, bool guarded, bool frequent_reuse,
-    bool *deferred_work_generated);
-bool pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
-    size_t new_size, bool zero, bool *deferred_work_generated);
-bool pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
-    size_t new_size, bool *deferred_work_generated);
-void pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
-    bool *deferred_work_generated);
-uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac);
-
 static inline size_t
-pac_mapped(const pac_t *pac) {
+pac_mapped(pac_t *pac) {
 	return atomic_load_zu(&pac->stats->pac_mapped, ATOMIC_RELAXED);
 }
 
-void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
-    ecache_t *ecache, edata_t *edata);
-
-static inline void
-pac_record_grown(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
-    edata_t *edata) {
-	bool pinned = edata_pinned_get(edata);
-	if (pinned && config_stats) {
-		atomic_fetch_add_zu(&pac->stats->pac_mapped,
-		    edata_size_get(edata), ATOMIC_RELAXED);
-	}
-	extent_record(tsdn, pac, ehooks,
-	    pinned ? &pac->ecache_pinned : &pac->ecache_retained, edata);
-}
-
 static inline ehooks_t *
-pac_ehooks_get(const pac_t *pac) {
+pac_ehooks_get(pac_t *pac) {
 	return base_ehooks_get(pac->base);
 }
 

include/jemalloc/internal/pai.h

@@ -0,0 +1,63 @@
+#ifndef JEMALLOC_INTERNAL_PAI_H
+#define JEMALLOC_INTERNAL_PAI_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/edata.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/* An interface for page allocation. */
+
+typedef struct pai_s pai_t;
+struct pai_s {
+	/* Returns NULL on failure. */
+	edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
+	    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
+	    bool *deferred_work_generated);
+	bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+	    size_t old_size, size_t new_size, bool zero,
+	    bool *deferred_work_generated);
+	bool (*shrink)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+	    size_t old_size, size_t new_size, bool *deferred_work_generated);
+	void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+	    bool *deferred_work_generated);
+	uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self);
+};
+
+/*
+ * These are just simple convenience functions to avoid having to reference the
+ * same pai_t twice on every invocation.
+ */
+
+static inline edata_t *
+pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
+    bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
+	return self->alloc(tsdn, self, size, alignment, zero, guarded,
+	    frequent_reuse, deferred_work_generated);
+}
+
+static inline bool
+pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+    size_t new_size, bool zero, bool *deferred_work_generated) {
+	return self->expand(tsdn, self, edata, old_size, new_size, zero,
+	    deferred_work_generated);
+}
+
+static inline bool
+pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
+    size_t new_size, bool *deferred_work_generated) {
+	return self->shrink(
+	    tsdn, self, edata, old_size, new_size, deferred_work_generated);
+}
+
+static inline void
+pai_dalloc(
+    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
+	self->dalloc(tsdn, self, edata, deferred_work_generated);
+}
+
+static inline uint64_t
+pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
+	return self->time_until_deferred_work(tsdn, self);
+}
+
+#endif /* JEMALLOC_INTERNAL_PAI_H */

include/jemalloc/internal/prof_externs.h

@@ -114,17 +114,50 @@ uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
  *
  * Currently only the profiling sampling event needs the lookahead
  * functionality, so we don't yet define general purpose lookahead functions.
+ *
+ * Surplus is a terminology referring to the amount of bytes beyond what's
+ * needed for triggering an event, which can be a useful quantity to have in
+ * general when lookahead is being called.
+ *
+ * This function returns true if allocation of usize would go above the next
+ * trigger for prof event, and false otherwise.
+ * If function returns true surplus will contain number of bytes beyond that
+ * trigger.
  */
 
 JEMALLOC_ALWAYS_INLINE bool
-te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
+te_prof_sample_event_lookahead_surplus(
+    tsd_t *tsd, size_t usize, size_t *surplus) {
+	if (surplus != NULL) {
+		/*
+		 * This is a dead store: the surplus will be overwritten before
+		 * any read.  The initialization suppresses compiler warnings.
+		 * Meanwhile, using SIZE_MAX to initialize is good for
+		 * debugging purpose, because a valid surplus value is strictly
+		 * less than usize, which is at most SIZE_MAX.
+		 */
+		*surplus = SIZE_MAX;
+	}
 	if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) {
 		return false;
 	}
 	/* The subtraction is intentionally susceptible to underflow. */
 	uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize
 	    - tsd_thread_allocated_last_event_get(tsd);
-	return accumbytes >= tsd_prof_sample_event_wait_get(tsd);
+	uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd);
+	if (accumbytes < sample_wait) {
+		return false;
+	}
+	assert(accumbytes - sample_wait < (uint64_t)usize);
+	if (surplus != NULL) {
+		*surplus = (size_t)(accumbytes - sample_wait);
+	}
+	return true;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
+	return te_prof_sample_event_lookahead_surplus(tsd, usize, NULL);
 }
 
 extern te_base_cb_t prof_sample_te_handler;

include/jemalloc/internal/psset.h

@@ -134,17 +134,17 @@ void psset_insert(psset_t *psset, hpdata_t *ps);
 void psset_remove(psset_t *psset, hpdata_t *ps);
 
 static inline size_t
-psset_npageslabs(const psset_t *psset) {
+psset_npageslabs(psset_t *psset) {
 	return psset->stats.merged.npageslabs;
 }
 
 static inline size_t
-psset_nactive(const psset_t *psset) {
+psset_nactive(psset_t *psset) {
 	return psset->stats.merged.nactive;
 }
 
 static inline size_t
-psset_ndirty(const psset_t *psset) {
+psset_ndirty(psset_t *psset) {
 	return psset->stats.merged.ndirty;
 }
 

include/jemalloc/internal/sc.h

@@ -268,17 +268,12 @@
 #define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP))
 #define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP)
 
-/*
- * The largest size class supported.  Spell this out directly to avoid
- * expanding subtractive arithmetic at every use site.
- */
-#if LG_SIZEOF_PTR == 3
-#	define SC_LARGE_MAXCLASS 0x7000000000000000ULL
-#elif LG_SIZEOF_PTR == 2
-#	define SC_LARGE_MAXCLASS 0x70000000ULL
-#else
-#	error "Unsupported pointer size"
-#endif
+/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */
+#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2))
+#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP))
+
+/* The largest size class supported. */
+#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA)
 
 /* Maximum number of regions in one slab. */
 #ifndef CONFIG_LG_SLAB_MAXREGS

include/jemalloc/internal/sec.h

@@ -4,10 +4,9 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/sec_opts.h"
-#include "jemalloc/internal/tsd_types.h"
 
 /*
  * Small extent cache.
@@ -97,25 +96,6 @@ sec_size_supported(sec_t *sec, size_t size) {
 	return sec_is_used(sec) && size <= sec->opts.max_alloc;
 }
 
-/* Min number of extents we will allocate when expanding the SEC. */
-#define SEC_MIN_NALLOCS 2
-
-/* Max number of extents we will allocate out of a single huge page. */
-#define SEC_MAX_NALLOCS 8
-
-/* Attempt to fill the SEC up to max_bytes / SEC_MAX_BYTES_DIV */
-#define SEC_MAX_BYTES_DIV 4
-
-/*
- * Calculate the min and max number of extents we will try to allocate
- * when expanding the SEC. We will attempt to allocate at least min
- * extents and up to max extents depending on whether we can allocate
- * them out of a huge page we have already allocated out of. Both
- * min and max should the in the range [1, SEC_MAX_NALLOCS].
- */
-void sec_calc_nallocs_for_size(
-    sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
-
 /* If sec does not have extent available, it will return NULL. */
 edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
 void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,

include/jemalloc/internal/sec_opts.h

@@ -27,9 +27,16 @@ struct sec_opts_s {
 	 * until we are 1/4 below max_bytes.
 	 */
 	size_t max_bytes;
+	/*
+	 * When we can't satisfy an allocation out of the SEC because there are
+	 * no available ones cached, allocator will allocate a batch with extra
+	 * batch_fill_extra extents of the same size.
+	 */
+	size_t batch_fill_extra;
 };
 
 #define SEC_OPTS_NSHARDS_DEFAULT 2
+#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3
 #define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024))
 #define SEC_OPTS_MAX_BYTES_DEFAULT                                             \
 	((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT)                       \
@@ -38,6 +45,6 @@ struct sec_opts_s {
 
 #define SEC_OPTS_DEFAULT                                                       \
 	{SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT,                 \
-	    SEC_OPTS_MAX_BYTES_DEFAULT}
+	    SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT}
 
 #endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

include/jemalloc/internal/seq.h

@@ -0,0 +1,58 @@
+#ifndef JEMALLOC_INTERNAL_SEQ_H
+#define JEMALLOC_INTERNAL_SEQ_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/atomic.h"
+
+/*
+ * A simple seqlock implementation.
+ */
+
+/* clang-format off */
+#define seq_define(type, short_type)					\
+typedef struct {							\
+	atomic_zu_t seq;						\
+	atomic_zu_t data[						\
+	    (sizeof(type) + sizeof(size_t) - 1) / sizeof(size_t)];	\
+} seq_##short_type##_t;							\
+									\
+/*									\
+ * No internal synchronization -- the caller must ensure that there's	\
+ * only a single writer at a time.					\
+ */									\
+static inline void							\
+seq_store_##short_type(seq_##short_type##_t *dst, type *src) {		\
+	size_t buf[sizeof(dst->data) / sizeof(size_t)];			\
+	buf[sizeof(buf) / sizeof(size_t) - 1] = 0;			\
+	memcpy(buf, src, sizeof(type));					\
+	size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED);	\
+	atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED);	\
+	atomic_fence(ATOMIC_RELEASE);					\
+	for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) {	\
+		atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED);	\
+	}								\
+	atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE);	\
+}									\
+									\
+/* Returns whether or not the read was consistent. */			\
+static inline bool							\
+seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) {	\
+	size_t buf[sizeof(src->data) / sizeof(size_t)];			\
+	size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE);	\
+	if (seq1 % 2 != 0) {						\
+		return false;						\
+	}								\
+	for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) {	\
+		buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED);	\
+	}								\
+	atomic_fence(ATOMIC_ACQUIRE);					\
+	size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED);	\
+	if (seq1 != seq2) {						\
+		return false;						\
+	}								\
+	memcpy(dst, buf, sizeof(type));					\
+	return true;							\
+}
+/* clang-format on */
+
+#endif /* JEMALLOC_INTERNAL_SEQ_H */

include/jemalloc/internal/sz.h

@@ -314,20 +314,12 @@ sz_size2index_usize_fastpath(size_t size, szind_t *ind, size_t *usize) {
 
 JEMALLOC_ALWAYS_INLINE size_t
 sz_s2u_compute_using_delta(size_t size) {
-	if (unlikely(size > SC_LARGE_MAXCLASS)) {
-		return 0;
-	}
-
 	size_t x = lg_floor((size << 1) - 1);
 	size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1)
 	    ? LG_QUANTUM
 	    : x - SC_LG_NGROUP - 1;
 	size_t delta = ZU(1) << lg_delta;
 	size_t delta_mask = delta - 1;
-	if (unlikely(size > SIZE_T_MAX - delta_mask)) {
-		return 0;
-	}
-
 	size_t usize = (size + delta_mask) & ~delta_mask;
 	return usize;
 }
@@ -393,11 +385,7 @@ JEMALLOC_ALWAYS_INLINE size_t
 sz_sa2u(size_t size, size_t alignment) {
 	size_t usize;
 
-	if (unlikely(alignment == 0)) {
-		return 0;
-	}
-	size_t alignment_mask = alignment - 1;
-	assert((alignment_mask & alignment) == 0);
+	assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
 
 	/* Try for a small size class. */
 	if (size <= SC_SMALL_MAXCLASS && alignment <= PAGE) {
@@ -415,10 +403,7 @@ sz_sa2u(size_t size, size_t alignment) {
 		 *    144 | 10100000 |  32
 		 *    192 | 11000000 |  64
 		 */
-		if (unlikely(size > SIZE_T_MAX - alignment_mask)) {
-			return 0;
-		}
-		usize = sz_s2u((size + alignment_mask) & ~alignment_mask);
+		usize = sz_s2u(ALIGNMENT_CEILING(size, alignment));
 		if (usize < SC_LARGE_MINCLASS) {
 			return usize;
 		}

include/jemalloc/internal/tcache_externs.h

@@ -62,18 +62,17 @@ bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
 bool tcache_bin_ncached_max_read(
     tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
 void tcache_arena_reassociate(
-    tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
+    tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
 tcache_t *tcache_create_explicit(tsd_t *tsd);
 bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
 void      tcache_cleanup(tsd_t *tsd);
+void      tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
 bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
 void      tcaches_flush(tsd_t *tsd, unsigned ind);
 void      tcaches_destroy(tsd_t *tsd, unsigned ind);
 bool      tcache_boot(tsdn_t *tsdn, base_t *base);
 void      tcache_arena_associate(
-         tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
-cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor(
-         tsdn_t *tsdn, arena_t *arena);
+         tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
 void tcache_prefork(tsdn_t *tsdn);
 void tcache_postfork_parent(tsdn_t *tsdn);
 void tcache_postfork_child(tsdn_t *tsdn);

include/jemalloc/internal/tcache_inlines.h

@@ -19,7 +19,7 @@ tcache_enabled_get(tsd_t *tsd) {
 }
 
 static inline unsigned
-tcache_nbins_get(const tcache_slow_t *tcache_slow) {
+tcache_nbins_get(tcache_slow_t *tcache_slow) {
 	assert(tcache_slow != NULL);
 	unsigned nbins = tcache_slow->tcache_nbins;
 	assert(nbins <= TCACHE_NBINS_MAX);
@@ -27,7 +27,7 @@ tcache_nbins_get(const tcache_slow_t *tcache_slow) {
 }
 
 static inline size_t
-tcache_max_get(const tcache_slow_t *tcache_slow) {
+tcache_max_get(tcache_slow_t *tcache_slow) {
 	assert(tcache_slow != NULL);
 	size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1);
 	assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
@@ -86,13 +86,6 @@ tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_slow_t *tcache_slow) {
 	return disabled;
 }
 
-JEMALLOC_ALWAYS_INLINE bool
-tcache_can_cache_large(tcache_t *tcache, szind_t ind) {
-	return ind < tcache_nbins_get(tcache->tcache_slow)
-	    && !tcache_bin_disabled(ind, &tcache->bins[ind],
-	        tcache->tcache_slow);
-}
-
 JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
     szind_t binind, bool zero, bool slow_path) {

include/jemalloc/internal/tcache_structs.h

@@ -19,6 +19,9 @@
  */
 
 struct tcache_slow_s {
+	/* Lets us track all the tcaches in an arena. */
+	ql_elm(tcache_slow_t) link;
+
 	/*
 	 * The descriptor lets the arena find our cache bins without seeing the
 	 * tcache definition.  This enables arenas to aggregate stats across

include/jemalloc/internal/tsd.h

@@ -37,12 +37,13 @@ TSD_DATA_SLOWER
 JEMALLOC_ALWAYS_INLINE t *						\
 tsd_##n##p_get(tsd_t *tsd) {						\
 	/*								\
-	 * Read the state once, so that a transition between		\
-	 * nominal states does not confuse this assertion.		\
+	 * Because the state might change asynchronously if it's	\
+	 * nominal, we need to make sure that we only read it once.	\
 	 */								\
 	uint8_t state = tsd_state_get(tsd);				\
 	assert(state == tsd_state_nominal ||				\
 	    state == tsd_state_nominal_slow ||				\
+	    state == tsd_state_nominal_recompute ||			\
 	    state == tsd_state_reincarnated ||				\
 	    state == tsd_state_minimal_initialized);			\
 	return tsd_##n##p_get_unsafe(tsd);				\
@@ -94,6 +95,11 @@ TSD_DATA_SLOWER
 
 JEMALLOC_ALWAYS_INLINE void
 tsd_assert_fast(tsd_t *tsd) {
+	/*
+	 * Note that our fastness assertion does *not* include global slowness
+	 * counters; it's not in general possible to ensure that they won't
+	 * change asynchronously from underneath us.
+	 */
 	assert(!malloc_slow && tsd_tcache_enabled_get(tsd)
 	    && tsd_reentrancy_level_get(tsd) == 0);
 }

include/jemalloc/internal/tsd_internals.h

@@ -10,6 +10,7 @@
 #include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/peak.h"
 #include "jemalloc/internal/prof_types.h"
+#include "jemalloc/internal/ql.h"
 #include "jemalloc/internal/rtree_tsd.h"
 #include "jemalloc/internal/tcache_structs.h"
 #include "jemalloc/internal/tcache_types.h"
@@ -56,6 +57,8 @@ typedef void (*test_callback_t)(int *);
 #	define MALLOC_TEST_TSD_INITIALIZER
 #endif
 
+typedef ql_elm(tsd_t) tsd_link_t;
+
 /*  O(name,			type,			nullable type) */
 #define TSD_DATA_SLOW                                                          \
 	O(tcache_enabled, bool, bool)                                          \
@@ -77,6 +80,8 @@ typedef void (*test_callback_t)(int *);
 	O(arena_decay_ticker, ticker_geom_t, ticker_geom_t)                    \
 	O(sec_shard, uint8_t, uint8_t)                                         \
 	O(binshards, tsd_binshards_t, tsd_binshards_t)                         \
+	O(tsd_link, tsd_link_t, tsd_link_t)                                    \
+	O(in_hook, bool, bool)                                                 \
 	O(peak, peak_t, peak_t)                                                \
 	O(tcache_slow, tcache_slow_t, tcache_slow_t)                           \
 	O(rtree_ctx, rtree_ctx_t, rtree_ctx_t)
@@ -96,8 +101,9 @@ typedef void (*test_callback_t)(int *);
 	    TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),                   \
 	    /* sec_shard */ (uint8_t) - 1,                                     \
 	    /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER,                    \
-	    /* peak */ PEAK_INITIALIZER, /* tcache_slow */                     \
-	    TCACHE_SLOW_ZERO_INITIALIZER,                                      \
+	    /* tsd_link */ {NULL}, /* in_hook */ false,                        \
+	    /* peak */ PEAK_INITIALIZER,                                       \
+	    /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER,                    \
 	    /* rtree_ctx */ RTREE_CTX_INITIALIZER,
 
 /*  O(name,			type,			nullable type) */
@@ -126,7 +132,7 @@ typedef void (*test_callback_t)(int *);
 #define TSD_INITIALIZER                                                        \
 	{                                                                      \
 		TSD_DATA_SLOW_INITIALIZER                                      \
-		/* state */ tsd_state_uninitialized,                           \
+		/* state */ ATOMIC_INIT(tsd_state_uninitialized),              \
 		    TSD_DATA_FAST_INITIALIZER TSD_DATA_SLOWER_INITIALIZER      \
 	}
 
@@ -142,6 +148,17 @@ void   tsd_cleanup(void *arg);
 tsd_t *tsd_fetch_slow(tsd_t *tsd, bool minimal);
 void   tsd_state_set(tsd_t *tsd, uint8_t new_state);
 void   tsd_slow_update(tsd_t *tsd);
+void   tsd_prefork(tsd_t *tsd);
+void   tsd_postfork_parent(tsd_t *tsd);
+void   tsd_postfork_child(tsd_t *tsd);
+
+/*
+ * Call ..._inc when your module wants to take all threads down the slow paths,
+ * and ..._dec when it no longer needs to.
+ */
+void tsd_global_slow_inc(tsdn_t *tsdn);
+void tsd_global_slow_dec(tsdn_t *tsdn);
+bool tsd_global_slow(void);
 
 #define TSD_MIN_INIT_STATE_MAX_FETCHED (128)
 
@@ -150,29 +167,39 @@ enum {
 	tsd_state_nominal = 0,
 	/* Initialized but on slow path. */
 	tsd_state_nominal_slow = 1,
+	/*
+	 * Some thread has changed global state in such a way that all nominal
+	 * threads need to recompute their fast / slow status the next time they
+	 * get a chance.
+	 *
+	 * Any thread can change another thread's status *to* recompute, but
+	 * threads are the only ones who can change their status *from*
+	 * recompute.
+	 */
+	tsd_state_nominal_recompute = 2,
 	/*
 	 * The above nominal states should be lower values.  We use
 	 * tsd_nominal_max to separate nominal states from threads in the
 	 * process of being born / dying.
 	 */
-	tsd_state_nominal_max = 1,
+	tsd_state_nominal_max = 2,
 
 	/*
 	 * A thread might free() during its death as its only allocator action;
 	 * in such scenarios, we need tsd, but set up in such a way that no
 	 * cleanup is necessary.
 	 */
-	tsd_state_minimal_initialized = 2,
+	tsd_state_minimal_initialized = 3,
 	/* States during which we know we're in thread death. */
-	tsd_state_purgatory = 3,
-	tsd_state_reincarnated = 4,
+	tsd_state_purgatory = 4,
+	tsd_state_reincarnated = 5,
 	/*
 	 * What it says on the tin; tsd that hasn't been initialized.  Note
 	 * that even when the tsd struct lives in TLS, when need to keep track
 	 * of stuff like whether or not our pthread destructors have been
 	 * scheduled, so this really truly is different than the nominal state.
 	 */
-	tsd_state_uninitialized = 5
+	tsd_state_uninitialized = 6
 };
 
 /*
@@ -182,6 +209,18 @@ enum {
  */
 #define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n
 
+#ifdef JEMALLOC_U8_ATOMICS
+#	define tsd_state_t atomic_u8_t
+#	define tsd_atomic_load atomic_load_u8
+#	define tsd_atomic_store atomic_store_u8
+#	define tsd_atomic_exchange atomic_exchange_u8
+#else
+#	define tsd_state_t atomic_u32_t
+#	define tsd_atomic_load atomic_load_u32
+#	define tsd_atomic_store atomic_store_u32
+#	define tsd_atomic_exchange atomic_exchange_u32
+#endif
+
 /* The actual tsd. */
 struct tsd_s {
 	/*
@@ -193,8 +232,11 @@ struct tsd_s {
 #define O(n, t, nt) t TSD_MANGLE(n);
 
 	TSD_DATA_SLOW
-	/* Encodes one of tsd_state_*; mutated only by the owning thread. */
-	uint8_t state;
+	/*
+	 * We manually limit the state to just a single byte.  Unless the 8-bit
+	 * atomics are unavailable (which is rare).
+	 */
+	tsd_state_t state;
 	TSD_DATA_FAST
 	TSD_DATA_SLOWER
 #undef O
@@ -202,7 +244,13 @@ struct tsd_s {
 
 JEMALLOC_ALWAYS_INLINE uint8_t
 tsd_state_get(tsd_t *tsd) {
-	return tsd->state;
+	/*
+	 * This should be atomic.  Unfortunately, compilers right now can't tell
+	 * that this can be done as a memory comparison, and forces a load into
+	 * a register that hurts fast-path performance.
+	 */
+	/* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */
+	return *(uint8_t *)&tsd->state;
 }
 
 /*

include/jemalloc/internal/util.h

@@ -6,18 +6,6 @@
 
 #define UTIL_INLINE static inline
 
-JEMALLOC_ALWAYS_INLINE
-size_t
-min_zu(size_t a, size_t b) {
-	return (a < b) ? a : b;
-}
-
-JEMALLOC_ALWAYS_INLINE
-size_t
-max_zu(size_t a, size_t b) {
-	return (a > b) ? a : b;
-}
-
 /* Junk fill patterns. */
 #ifndef JEMALLOC_ALLOC_JUNK
 #	define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)

include/jemalloc/internal/witness.h

@@ -44,7 +44,7 @@ enum witness_rank_e {
 	 */
 	WITNESS_RANK_CORE,
 	WITNESS_RANK_DECAY = WITNESS_RANK_CORE,
-	WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
+	WITNESS_RANK_TCACHE_QL,
 
 	WITNESS_RANK_SEC_BIN,
 

include/jemalloc/jemalloc_typedefs.h.in

@@ -4,36 +4,6 @@ extern "C" {
 
 typedef struct extent_hooks_s extent_hooks_t;
 
-/*
- * Extent alloc flags.  A custom extent_alloc hook may OR these into the
- * returned pointer; jemalloc strips the low bits before use.  Safe because
- * returned addresses are at least page-aligned (PAGE >= 256).
- *
- * EXTENT_ALLOC_FLAG_PINNED: backing memory is non-reclaimable.
- * Pinned extents are excluded from decay/purging and cached separately for
- * preferential reuse.  A hook returning this flag must also set *commit to
- * true: pinned memory bypasses jemalloc's commit/decommit machinery.
- *
- * The pinned attribute is per-extent: a single hook may return pinned and
- * non-pinned extents in different calls.  Pinned and non-pinned extents are
- * never merged together (the merge would change the reclamation policy of
- * one half), so pinned-ness is set at allocation and inherited through
- * splits, but never changes after that.
- *
- * Example (HugeTLB alloc hook):
- *   void *my_alloc(extent_hooks_t *h, void *new_addr, size_t size,
- *       size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
- *       void *addr = mmap(NULL, size, PROT_READ|PROT_WRITE,
- *           MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0);
- *       if (addr == MAP_FAILED) return NULL;
- *       *zero = true;
- *       *commit = true;
- *       return (void *)((uintptr_t)addr | EXTENT_ALLOC_FLAG_PINNED);
- *   }
- */
-#define EXTENT_ALLOC_FLAG_PINNED    0x1U
-#define EXTENT_ALLOC_FLAG_MASK      0xFFU
-
 /*
  * void *
  * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -36,7 +36,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
-    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
-    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,8 +67,6 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -31,9 +31,6 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\conf.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -205,14 +202,5 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\arenas_management.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
 </Project>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -36,7 +36,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
-    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
-    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,8 +67,6 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -31,9 +31,6 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\conf.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -205,14 +202,5 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\arenas_management.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
 </Project>

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj

@@ -36,7 +36,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
-    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
-    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,8 +67,6 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters

@@ -31,9 +31,6 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\conf.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -205,14 +202,5 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\arenas_management.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
 </Project>

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj

@@ -36,7 +36,6 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
-    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -45,7 +44,6 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
-    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -69,8 +67,6 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters

@@ -31,9 +31,6 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\conf.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -205,14 +202,5 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\arenas_management.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
 </Project>

scripts/gen_gh_actions.py

@@ -287,7 +287,7 @@ def generate_linux_job(arch):
 
     job += f"""
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       run: |
@@ -302,7 +302,7 @@ def generate_linux_job(arch):
         cat /etc/os-release || true
         echo ""
         echo "=== CPU Info ==="
-        lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\\(s\\):" || true
+        lscpu | grep -E "Architecture|CPU op-mode|Byte Order|CPU\(s\):" || true
 
     - name: Install dependencies (32-bit)
       if: matrix.env.CROSS_COMPILE_32BIT == 'yes'
@@ -387,7 +387,7 @@ def generate_macos_job(arch):
 
     job += f"""
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       run: |
@@ -469,7 +469,7 @@ def generate_windows_job(arch):
 
     job += f"""
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - name: Show OS version
       shell: cmd
@@ -590,7 +590,7 @@ def generate_freebsd_job(arch):
     name: FreeBSD (${{{{ matrix.arch }}}}, debug=${{{{ matrix.debug }}}}, prof=${{{{ matrix.prof }}}}${{{{ matrix.uncommon && ', uncommon' || '' }}}})
 
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 1
 

src/arena.c

@@ -18,7 +18,7 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS
 /* Data. */
 
 /*
- * Define names for both uninitialized and initialized phases, so that
+ * Define names for both unininitialized and initialized phases, so that
  * options and mallctl processing are straightforward.
  */
 const char *const percpu_arena_mode_names[] = {
@@ -44,10 +44,10 @@ static unsigned nbins_total;
 
 /*
  * a0 is used to handle huge requests before malloc init completes. After
- * that, the huge_arena_ind is updated to point to the actual huge arena,
+ * that,the huge_arena_ind is updated to point to the actual huge arena,
  * which is the last one of the auto arenas.
  */
-static unsigned huge_arena_ind = 0;
+unsigned  huge_arena_ind = 0;
 bool      opt_huge_arena_pac_thp = false;
 pac_thp_t huge_arena_pac_thp = {.thp_madvise = false,
     .auto_thp_switched = false,
@@ -164,7 +164,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	/* Currently cached bytes and sanitizer-stashed bytes in tcache. */
 	astats->tcache_bytes = 0;
 	astats->tcache_stashed_bytes = 0;
-	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+	malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
 	cache_bin_array_descriptor_t *descriptor;
 	ql_foreach (descriptor, &arena->cache_bin_array_descriptor_ql, link) {
 		for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
@@ -183,8 +183,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	}
 	malloc_mutex_prof_read(tsdn,
 	    &astats->mutex_prof_data[arena_prof_mutex_tcache_list],
-	    &arena->cache_bin_array_descriptor_ql_mtx);
-	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+	    &arena->tcache_ql_mtx);
+	malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
 
 #define READ_ARENA_MUTEX_PROF_DATA(mtx, ind)                                   \
 	malloc_mutex_lock(tsdn, &arena->mtx);                                  \
@@ -211,92 +211,6 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	}
 }
 
-void *
-arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind) {
-	assert(szind < SC_NBINS);
-	bin_t *bin = bin_choose(tsdn, arena, szind, NULL);
-	assert(bin != NULL);
-	return bin_current_slab_addr(tsdn, bin);
-}
-
-void
-arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
-    cache_bin_array_descriptor_t *desc) {
-	cassert(config_stats);
-	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
-	ql_tail_insert(&arena->cache_bin_array_descriptor_ql, desc, link);
-	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
-}
-
-static void
-arena_cache_bin_stats_flush(tsdn_t *tsdn, arena_t *arena,
-    cache_bin_array_descriptor_t *desc) {
-	cassert(config_stats);
-	for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
-		cache_bin_t *cache_bin = &desc->bins[i];
-		if (cache_bin_disabled(cache_bin)) {
-			continue;
-		}
-		if (i < SC_NBINS) {
-			bin_t *bin = bin_choose(tsdn, arena, i, NULL);
-			bin_stats_nrequests_add(tsdn, bin,
-			    cache_bin->tstats.nrequests);
-		} else {
-			arena_stats_large_flush_nrequests_add(tsdn,
-			    &arena->stats, i, cache_bin->tstats.nrequests);
-		}
-		cache_bin->tstats.nrequests = 0;
-	}
-}
-
-void
-arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
-    cache_bin_array_descriptor_t *desc) {
-	cassert(config_stats);
-	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
-	if (config_debug) {
-		bool                          in_ql = false;
-		cache_bin_array_descriptor_t *iter;
-		ql_foreach (iter, &arena->cache_bin_array_descriptor_ql, link) {
-			if (iter == desc) {
-				in_ql = true;
-				break;
-			}
-		}
-		assert(in_ql);
-	}
-	ql_remove(&arena->cache_bin_array_descriptor_ql, desc, link);
-	arena_cache_bin_stats_flush(tsdn, arena, desc);
-	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
-}
-
-/*
- * Postfork-child entry: child is single-threaded, so the queue is rebuilt
- * from scratch (descriptors held by other threads at fork time are gone)
- * without locking.
- */
-void
-arena_cache_bin_array_postfork_child(arena_t *arena,
-    cache_bin_array_descriptor_t *desc_or_null) {
-	cassert(config_stats);
-	ql_new(&arena->cache_bin_array_descriptor_ql);
-	if (desc_or_null != NULL) {
-		ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
-		    desc_or_null, link);
-	}
-}
-
-void
-arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
-	cassert(config_stats);
-	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
-	cache_bin_array_descriptor_t *desc;
-	ql_foreach (desc, &arena->cache_bin_array_descriptor_ql, link) {
-		arena_cache_bin_stats_flush(tsdn, arena, desc);
-	}
-	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
-}
-
 static void
 arena_background_thread_inactivity_check(
     tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
@@ -429,8 +343,7 @@ arena_extent_alloc_large(
 }
 
 void
-arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
-    const edata_t *edata) {
+arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena, edata_t *edata) {
 	if (config_stats) {
 		arena_large_dalloc_stats_update(
 		    tsdn, arena, edata_usize_get(edata));
@@ -439,7 +352,7 @@ arena_extent_dalloc_large_prep(tsdn_t *tsdn, arena_t *arena,
 
 void
 arena_extent_ralloc_large_shrink(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize) {
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) {
 	size_t usize = edata_usize_get(edata);
 
 	if (config_stats) {
@@ -449,7 +362,7 @@ arena_extent_ralloc_large_shrink(
 
 void
 arena_extent_ralloc_large_expand(
-    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize) {
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize) {
 	size_t usize = edata_usize_get(edata);
 
 	if (config_stats) {
@@ -730,7 +643,9 @@ arena_dalloc_promoted_impl(
 	}
 	szind_t bumped_ind = sz_size2index(bumped_usize);
 	if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
-	    && tcache_can_cache_large(tcache, bumped_ind)) {
+	    && bumped_ind < TCACHE_NBINS_MAX
+	    && !tcache_bin_disabled(
+	        bumped_ind, &tcache->bins[bumped_ind], tcache->tcache_slow)) {
 		tcache_dalloc_large(
 		    tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
 	} else {
@@ -881,8 +796,6 @@ arena_prepare_base_deletion(tsd_t *tsd, base_t *base_to_destroy) {
 		    tsd, &pac->ecache_muzzy.mtx, delayed_mtx, &n_delayed);
 		arena_prepare_base_deletion_sync(
 		    tsd, &pac->ecache_retained.mtx, delayed_mtx, &n_delayed);
-		arena_prepare_base_deletion_sync(
-		    tsd, &pac->ecache_pinned.mtx, delayed_mtx, &n_delayed);
 	}
 	arena_prepare_base_deletion_sync_finish(tsd, delayed_mtx, n_delayed);
 }
@@ -1084,6 +997,76 @@ label_refill:
 	return filled;
 }
 
+size_t
+arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
+    void **ptrs, size_t nfill, bool zero) {
+	assert(binind < SC_NBINS);
+	const bin_info_t *bin_info = &bin_infos[binind];
+	const size_t      nregs = bin_info->nregs;
+	assert(nregs > 0);
+	const size_t usize = bin_info->reg_size;
+
+	const bool manual_arena = !arena_is_auto(arena);
+	unsigned   binshard;
+	bin_t     *bin = bin_choose(tsdn, arena, binind, &binshard);
+
+	size_t              nslab = 0;
+	size_t              filled = 0;
+	edata_t            *slab = NULL;
+	edata_list_active_t fulls;
+	edata_list_active_init(&fulls);
+
+	while (filled < nfill
+	    && (slab = arena_slab_alloc(
+	            tsdn, arena, binind, binshard, bin_info))
+	        != NULL) {
+		assert((size_t)edata_nfree_get(slab) == nregs);
+		++nslab;
+		size_t batch = nfill - filled;
+		if (batch > nregs) {
+			batch = nregs;
+		}
+		assert(batch > 0);
+		bin_slab_reg_alloc_batch(
+		    slab, bin_info, (unsigned)batch, &ptrs[filled]);
+		assert(edata_addr_get(slab) == ptrs[filled]);
+		if (zero) {
+			memset(ptrs[filled], 0, batch * usize);
+		}
+		filled += batch;
+		if (batch == nregs) {
+			if (manual_arena) {
+				edata_list_active_append(&fulls, slab);
+			}
+			slab = NULL;
+		}
+	}
+
+	malloc_mutex_lock(tsdn, &bin->lock);
+	/*
+	 * Only the last slab can be non-empty, and the last slab is non-empty
+	 * iff slab != NULL.
+	 */
+	if (slab != NULL) {
+		bin_lower_slab(tsdn, !manual_arena, slab, bin);
+	}
+	if (manual_arena) {
+		edata_list_active_concat(&bin->slabs_full, &fulls);
+	}
+	assert(edata_list_active_empty(&fulls));
+	if (config_stats) {
+		bin->stats.nslabs += nslab;
+		bin->stats.curslabs += nslab;
+		bin->stats.nmalloc += filled;
+		bin->stats.nrequests += filled;
+		bin->stats.curregs += filled;
+	}
+	malloc_mutex_unlock(tsdn, &bin->lock);
+
+	arena_decay_tick(tsdn, arena);
+	return filled;
+}
+
 static void *
 arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) {
 	assert(binind < SC_NBINS);
@@ -1624,7 +1607,8 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
 
 void *
 arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
+    hook_ralloc_args_t *hook_args) {
 	size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
 	if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
 		return NULL;
@@ -1636,13 +1620,18 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
 		UNUSED size_t newsize;
 		if (!arena_ralloc_no_move(
 		        tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
+			hook_invoke_expand(hook_args->is_realloc
+			        ? hook_expand_realloc
+			        : hook_expand_rallocx,
+			    ptr, oldsize, usize, (uintptr_t)ptr,
+			    hook_args->args);
 			return ptr;
 		}
 	}
 
 	if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
 		return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
-		    tcache);
+		    tcache, hook_args);
 	}
 
 	/*
@@ -1655,6 +1644,13 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
 		return NULL;
 	}
 
+	hook_invoke_alloc(
+	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx,
+	    ret, (uintptr_t)ret, hook_args->args);
+	hook_invoke_dalloc(
+	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
+	    ptr, hook_args->args);
+
 	/*
 	 * Junk/zero-filling were already done by
 	 * ipalloc()/arena_malloc().
@@ -1666,7 +1662,7 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
 }
 
 ehooks_t *
-arena_get_ehooks(const arena_t *arena) {
+arena_get_ehooks(arena_t *arena) {
 	return base_ehooks_get(arena->base);
 }
 
@@ -1689,7 +1685,7 @@ arena_set_extent_hooks(
 }
 
 dss_prec_t
-arena_dss_prec_get(const arena_t *arena) {
+arena_dss_prec_get(arena_t *arena) {
 	return (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_ACQUIRE);
 }
 
@@ -1703,9 +1699,8 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) {
 }
 
 void
-arena_name_get(const arena_t *arena, char *name) {
-	const char *end = (const char *)memchr(
-	    arena->name, '\0', ARENA_NAME_LEN);
+arena_name_get(arena_t *arena, char *name) {
+	char *end = (char *)memchr((void *)arena->name, '\0', ARENA_NAME_LEN);
 	assert(end != NULL);
 	size_t len = (uintptr_t)end - (uintptr_t)arena->name + 1;
 	assert(len > 0 && len <= ARENA_NAME_LEN);
@@ -1756,7 +1751,7 @@ arena_retain_grow_limit_get_set(
 }
 
 unsigned
-arena_nthreads_get(const arena_t *arena, bool internal) {
+arena_nthreads_get(arena_t *arena, bool internal) {
 	return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED);
 }
 
@@ -1804,11 +1799,10 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
 			goto label_error;
 		}
 
+		ql_new(&arena->tcache_ql);
 		ql_new(&arena->cache_bin_array_descriptor_ql);
-		if (malloc_mutex_init(&arena->cache_bin_array_descriptor_ql_mtx,
-		        "cache_bin_array_descriptor_ql",
-		        WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
-		        malloc_mutex_rank_exclusive)) {
+		if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql",
+		        WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) {
 			goto label_error;
 		}
 	}
@@ -1960,6 +1954,9 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
 		huge_arena_ind = narenas_total_get();
 		assert(huge_arena_ind != 0);
 		oversize_threshold = opt_oversize_threshold;
+		/* a0 init happened before malloc_conf_init. */
+		atomic_store_zu(&a0->pa_shard.pac.oversize_threshold,
+		    oversize_threshold, ATOMIC_RELAXED);
 		/* Initialize huge_arena_pac_thp fields. */
 		base_t *b0 = a0->base;
 		/* Make sure that b0 thp auto-switch won't happen concurrently here. */
@@ -1977,18 +1974,9 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
 		huge_enabled = true;
 	}
 
-	/* a0 init happened before malloc_conf_init. */
-	atomic_store_zu(&a0->pa_shard.pac.oversize_threshold,
-	    oversize_threshold, ATOMIC_RELAXED);
-
 	return huge_enabled;
 }
 
-bool
-arena_ind_is_huge(unsigned ind) {
-	return huge_arena_ind != 0 && ind == huge_arena_ind;
-}
-
 bool
 arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) {
 	arena_dirty_decay_ms_default_set(opt_dirty_decay_ms);
@@ -2018,7 +2006,7 @@ arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
 void
 arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
 	if (config_stats) {
-		malloc_mutex_prefork(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+		malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx);
 	}
 }
 
@@ -2071,13 +2059,12 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
 	base_postfork_parent(tsdn, arena->base);
 	pa_shard_postfork_parent(tsdn, &arena->pa_shard);
 	if (config_stats) {
-		malloc_mutex_postfork_parent(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+		malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
 	}
 }
 
 void
-arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
-    cache_bin_array_descriptor_t *surviving_desc) {
+arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
 	atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
 	atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
 	if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) {
@@ -2087,9 +2074,19 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
 		arena_nthreads_inc(arena, true);
 	}
 	if (config_stats) {
-		malloc_mutex_postfork_child(tsdn,
-		    &arena->cache_bin_array_descriptor_ql_mtx);
-		arena_cache_bin_array_postfork_child(arena, surviving_desc);
+		ql_new(&arena->tcache_ql);
+		ql_new(&arena->cache_bin_array_descriptor_ql);
+		tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn));
+		if (tcache_slow != NULL && tcache_slow->arena == arena) {
+			tcache_t *tcache = tcache_slow->tcache;
+			ql_elm_new(tcache_slow, link);
+			ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
+			cache_bin_array_descriptor_init(
+			    &tcache_slow->cache_bin_array_descriptor,
+			    tcache->bins);
+			ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
+			    &tcache_slow->cache_bin_array_descriptor, link);
+		}
 	}
 
 	for (unsigned i = 0; i < nbins_total; i++) {
@@ -2100,4 +2097,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
 	malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
 	base_postfork_child(tsdn, arena->base);
 	pa_shard_postfork_child(tsdn, &arena->pa_shard);
+	if (config_stats) {
+		malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
+	}
 }

src/arenas_management.c

@@ -1,354 +0,0 @@
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
-
-#include "jemalloc/internal/arenas_management.h"
-#include "jemalloc/internal/jemalloc_init.h"
-#include "jemalloc/internal/malloc_io.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/sz.h"
-
-JEMALLOC_ALIGNED(CACHELINE)
-atomic_p_t arenas[MALLOCX_ARENA_LIMIT];
-/* Below two are read-only after initialization. */
-unsigned   narenas_auto;
-unsigned   manual_arena_base;
-
-static atomic_u_t narenas_total;
-
-static malloc_mutex_t arenas_lock;
-
-bool
-arenas_management_boot(void) {
-	return malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS,
-	    malloc_mutex_rank_exclusive);
-}
-
-void
-arenas_management_prefork(tsdn_t *tsdn) {
-	malloc_mutex_prefork(tsdn, &arenas_lock);
-}
-
-void
-arenas_management_postfork_parent(tsdn_t *tsdn) {
-	malloc_mutex_postfork_parent(tsdn, &arenas_lock);
-}
-
-void
-arenas_management_postfork_child(tsdn_t *tsdn) {
-	malloc_mutex_postfork_child(tsdn, &arenas_lock);
-}
-
-void
-narenas_total_set(unsigned narenas) {
-	atomic_store_u(&narenas_total, narenas, ATOMIC_RELEASE);
-}
-
-void
-narenas_total_inc(void) {
-	atomic_fetch_add_u(&narenas_total, 1, ATOMIC_RELEASE);
-}
-
-unsigned
-narenas_total_get(void) {
-	return atomic_load_u(&narenas_total, ATOMIC_ACQUIRE);
-}
-
-void
-narenas_auto_set(unsigned n) {
-	narenas_auto = n;
-}
-
-void
-manual_arena_base_set(unsigned base) {
-	manual_arena_base = base;
-}
-
-/*
- * The a0*() functions are used instead of i{d,}alloc() in situations that
- * cannot tolerate TLS variable access.
- */
-
-void *
-a0ialloc(size_t size, bool zero, bool is_internal) {
-	if (unlikely(malloc_init_a0())) {
-		return NULL;
-	}
-
-	return iallocztm(TSDN_NULL, size, sz_size2index(size), zero, NULL,
-	    is_internal, arena_get(TSDN_NULL, 0, true), true);
-}
-
-void
-a0idalloc(void *ptr, bool is_internal) {
-	idalloctm(TSDN_NULL, ptr, NULL, NULL, is_internal, true);
-}
-
-void *
-a0malloc(size_t size) {
-	return a0ialloc(size, false, true);
-}
-
-void
-a0dalloc(void *ptr) {
-	a0idalloc(ptr, true);
-}
-
-void
-arena_set(unsigned ind, arena_t *arena) {
-	atomic_store_p(&arenas[ind], arena, ATOMIC_RELEASE);
-}
-
-/* Create a new arena and insert it into the arenas array at index ind. */
-static arena_t *
-arena_init_locked(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
-	arena_t *arena;
-
-	assert(ind <= narenas_total_get());
-	if (ind >= MALLOCX_ARENA_LIMIT) {
-		return NULL;
-	}
-	if (ind == narenas_total_get()) {
-		narenas_total_inc();
-	}
-
-	/*
-	 * Another thread may have already initialized arenas[ind] if it's an
-	 * auto arena.
-	 */
-	arena = arena_get(tsdn, ind, false);
-	if (arena != NULL) {
-		assert(arena_is_auto(arena));
-		return arena;
-	}
-
-	/* Actually initialize the arena. */
-	arena = arena_new(tsdn, ind, config);
-
-	return arena;
-}
-
-static void
-arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) {
-	if (ind == 0) {
-		return;
-	}
-
-	if (have_background_thread) {
-		if (background_thread_create(tsdn_tsd(tsdn), ind)) {
-			malloc_printf(
-			    "<jemalloc>: error in background thread "
-			    "creation for arena %u. Abort.\n",
-			    ind);
-			abort();
-		}
-	}
-}
-
-arena_t *
-arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
-	arena_t *arena;
-
-	malloc_mutex_lock(tsdn, &arenas_lock);
-	arena = arena_init_locked(tsdn, ind, config);
-	malloc_mutex_unlock(tsdn, &arenas_lock);
-
-	arena_new_create_background_thread(tsdn, ind);
-
-	return arena;
-}
-
-static void
-arena_bind(tsd_t *tsd, unsigned ind, bool internal) {
-	arena_t *arena = arena_get(tsd_tsdn(tsd), ind, false);
-	arena_nthreads_inc(arena, internal);
-
-	if (internal) {
-		tsd_iarena_set(tsd, arena);
-	} else {
-		tsd_arena_set(tsd, arena);
-		/*
-		 * While shard acts as a random seed, the cast below should
-		 * not make much difference.
-		 */
-		uint8_t shard = (uint8_t)atomic_fetch_add_u(
-		    &arena->binshard_next, 1, ATOMIC_RELAXED);
-		tsd_binshards_t *bins = tsd_binshardsp_get(tsd);
-		for (unsigned i = 0; i < SC_NBINS; i++) {
-			assert(bin_infos[i].n_shards > 0
-			    && bin_infos[i].n_shards <= BIN_SHARDS_MAX);
-			bins->binshard[i] = shard % bin_infos[i].n_shards;
-		}
-	}
-}
-
-void
-arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
-	assert(oldarena != NULL);
-	assert(newarena != NULL);
-
-	arena_nthreads_dec(oldarena, false);
-	arena_nthreads_inc(newarena, false);
-	tsd_arena_set(tsd, newarena);
-
-	if (arena_nthreads_get(oldarena, false) == 0
-	    && !background_thread_enabled()) {
-		/*
-		 * Purge if the old arena has no associated threads anymore and
-		 * no background threads.
-		 */
-		arena_decay(tsd_tsdn(tsd), oldarena,
-		    /* is_background_thread */ false, /* all */ true);
-	}
-}
-
-static void
-arena_unbind(tsd_t *tsd, unsigned ind, bool internal) {
-	arena_t *arena;
-
-	arena = arena_get(tsd_tsdn(tsd), ind, false);
-	arena_nthreads_dec(arena, internal);
-
-	if (internal) {
-		tsd_iarena_set(tsd, NULL);
-	} else {
-		tsd_arena_set(tsd, NULL);
-	}
-}
-
-/* Slow path, called only by arena_choose(). */
-arena_t *
-arena_choose_hard(tsd_t *tsd, bool internal) {
-	arena_t *ret JEMALLOC_CC_SILENCE_INIT(NULL);
-
-	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)) {
-		unsigned choose = percpu_arena_choose();
-		ret = arena_get(tsd_tsdn(tsd), choose, true);
-		assert(ret != NULL);
-		arena_bind(tsd, arena_ind_get(ret), false);
-		arena_bind(tsd, arena_ind_get(ret), true);
-
-		return ret;
-	}
-
-	if (narenas_auto > 1) {
-		unsigned i, j, choose[2], first_null;
-		bool     is_new_arena[2];
-
-		/*
-		 * Determine binding for both non-internal and internal
-		 * allocation.
-		 *
-		 *   choose[0]: For application allocation.
-		 *   choose[1]: For internal metadata allocation.
-		 */
-
-		for (j = 0; j < 2; j++) {
-			choose[j] = 0;
-			is_new_arena[j] = false;
-		}
-
-		first_null = narenas_auto;
-		malloc_mutex_lock(tsd_tsdn(tsd), &arenas_lock);
-		assert(arena_get(tsd_tsdn(tsd), 0, false) != NULL);
-		for (i = 1; i < narenas_auto; i++) {
-			if (arena_get(tsd_tsdn(tsd), i, false) != NULL) {
-				/*
-				 * Choose the first arena that has the lowest
-				 * number of threads assigned to it.
-				 */
-				for (j = 0; j < 2; j++) {
-					if (arena_nthreads_get(
-					        arena_get(
-					            tsd_tsdn(tsd), i, false),
-					        !!j)
-					    < arena_nthreads_get(
-					        arena_get(tsd_tsdn(tsd),
-					            choose[j], false),
-					        !!j)) {
-						choose[j] = i;
-					}
-				}
-			} else if (first_null == narenas_auto) {
-				/*
-				 * Record the index of the first uninitialized
-				 * arena, in case all extant arenas are in use.
-				 *
-				 * NB: It is possible for there to be
-				 * discontinuities in terms of initialized
-				 * versus uninitialized arenas, due to the
-				 * "thread.arena" mallctl.
-				 */
-				first_null = i;
-			}
-		}
-
-		for (j = 0; j < 2; j++) {
-			if (arena_nthreads_get(
-			        arena_get(tsd_tsdn(tsd), choose[j], false), !!j)
-			        == 0
-			    || first_null == narenas_auto) {
-				/*
-				 * Use an unloaded arena, or the least loaded
-				 * arena if all arenas are already initialized.
-				 */
-				if (!!j == internal) {
-					ret = arena_get(
-					    tsd_tsdn(tsd), choose[j], false);
-				}
-			} else {
-				arena_t *arena;
-
-				/* Initialize a new arena. */
-				choose[j] = first_null;
-				arena = arena_init_locked(tsd_tsdn(tsd),
-				    choose[j], &arena_config_default);
-				if (arena == NULL) {
-					malloc_mutex_unlock(
-					    tsd_tsdn(tsd), &arenas_lock);
-					return NULL;
-				}
-				is_new_arena[j] = true;
-				if (!!j == internal) {
-					ret = arena;
-				}
-			}
-			arena_bind(tsd, choose[j], !!j);
-		}
-		malloc_mutex_unlock(tsd_tsdn(tsd), &arenas_lock);
-
-		for (j = 0; j < 2; j++) {
-			if (is_new_arena[j]) {
-				assert(choose[j] > 0);
-				arena_new_create_background_thread(
-				    tsd_tsdn(tsd), choose[j]);
-			}
-		}
-
-	} else {
-		ret = arena_get(tsd_tsdn(tsd), 0, false);
-		arena_bind(tsd, 0, false);
-		arena_bind(tsd, 0, true);
-	}
-
-	return ret;
-}
-
-void
-iarena_cleanup(tsd_t *tsd) {
-	arena_t *iarena;
-
-	iarena = tsd_iarena_get(tsd);
-	if (iarena != NULL) {
-		arena_unbind(tsd, arena_ind_get(iarena), true);
-	}
-}
-
-void
-arena_cleanup(tsd_t *tsd) {
-	arena_t *arena;
-
-	arena = tsd_arena_get(tsd);
-	if (arena != NULL) {
-		arena_unbind(tsd, arena_ind_get(arena), false);
-	}
-}

src/base.c

@@ -52,9 +52,8 @@ base_map(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, size_t size) {
 	if (ehooks_are_default(ehooks)) {
 		addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit);
 	} else {
-		UNUSED unsigned flags;
-		addr = ehooks_alloc(tsdn, ehooks, NULL, size, alignment, &zero,
-		    &commit, &flags);
+		addr = ehooks_alloc(
+		    tsdn, ehooks, NULL, size, alignment, &zero, &commit);
 	}
 
 	return addr;
@@ -113,7 +112,7 @@ label_done:
 }
 
 static inline bool
-base_edata_is_reused(const edata_t *edata) {
+base_edata_is_reused(edata_t *edata) {
 	/*
 	 * Borrow the guarded bit to indicate if the extent is a recycled one,
 	 * i.e. the ones returned to base for reuse; currently only tcache bin
@@ -134,8 +133,8 @@ base_edata_init(
 }
 
 static size_t
-base_get_num_blocks(const base_t *base, bool with_new_block) {
-	const base_block_t *b = base->blocks;
+base_get_num_blocks(base_t *base, bool with_new_block) {
+	base_block_t *b = base->blocks;
 	assert(b != NULL);
 
 	size_t n_blocks = with_new_block ? 2 : 1;

src/bin.c

@@ -330,16 +330,3 @@ bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
 	}
 	return arena_get_bin(arena, binind, binshard);
 }
-
-void *
-bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin) {
-	malloc_mutex_lock(tsdn, &bin->lock);
-	edata_t *slab = (bin->slabcur != NULL)
-	    ? bin->slabcur
-	    : edata_heap_first(&bin->slabs_nonfull);
-	assert(slab != NULL || edata_heap_empty(&bin->slabs_nonfull));
-	void *ret = (slab != NULL) ? edata_addr_get(slab) : NULL;
-	assert(ret != NULL || slab == NULL);
-	malloc_mutex_unlock(tsdn, &bin->lock);
-	return ret;
-}

src/cache_bin.c

@@ -10,8 +10,8 @@ const uintptr_t disabled_bin = JUNK_ADDR;
 void
 cache_bin_info_init(cache_bin_info_t *info, cache_bin_sz_t ncached_max) {
 	assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
-	assert((size_t)ncached_max * sizeof(void *)
-	    < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
+	size_t stack_size = (size_t)ncached_max * sizeof(void *);
+	assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
 	info->ncached_max = (cache_bin_sz_t)ncached_max;
 }
 
@@ -94,8 +94,9 @@ cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
 	bin->low_bits_full = (cache_bin_sz_t)(uintptr_t)full_position;
 	bin->low_bits_empty = (cache_bin_sz_t)(uintptr_t)empty_position;
 	cache_bin_info_init(&bin->bin_info, info->ncached_max);
-	assert(cache_bin_diff(bin, bin->low_bits_full,
-	    (cache_bin_sz_t)(uintptr_t)bin->stack_head) == bin_stack_size);
+	cache_bin_sz_t free_spots = cache_bin_diff(bin, bin->low_bits_full,
+	    (cache_bin_sz_t)(uintptr_t)bin->stack_head);
+	assert(free_spots == bin_stack_size);
 	if (!cache_bin_disabled(bin)) {
 		assert(cache_bin_ncached_get_local(bin) == 0);
 	}

src/conf.c

@@ -225,8 +225,7 @@ conf_error(
 		/* However, tolerate experimental features. */
 		return;
 	}
-	const char *deprecated[] = {
-	    "hpa_sec_bytes_after_flush", "hpa_sec_batch_fill_extra"};
+	const char  *deprecated[] = {"hpa_sec_bytes_after_flush"};
 	const size_t deprecated_cnt = (sizeof(deprecated)
 	    / sizeof(deprecated[0]));
 	for (size_t i = 0; i < deprecated_cnt; ++i) {
@@ -953,6 +952,9 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
 			    "hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
 			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
+			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
+			    "hpa_sec_batch_fill_extra", 1, HUGEPAGE_PAGES,
+			    CONF_CHECK_MIN, CONF_CHECK_MAX, true);
 
 			if (CONF_MATCH("slab_sizes")) {
 				if (CONF_MATCH_VALUE("default")) {

src/ctl.c

@@ -115,6 +115,7 @@ CTL_PROTO(opt_hpa_dirty_mult)
 CTL_PROTO(opt_hpa_sec_nshards)
 CTL_PROTO(opt_hpa_sec_max_alloc)
 CTL_PROTO(opt_hpa_sec_max_bytes)
+CTL_PROTO(opt_hpa_sec_batch_fill_extra)
 CTL_PROTO(opt_huge_arena_pac_thp)
 CTL_PROTO(opt_metadata_thp)
 CTL_PROTO(opt_retain)
@@ -256,11 +257,9 @@ INDEX_PROTO(stats_arenas_i_lextents_j)
 CTL_PROTO(stats_arenas_i_extents_j_ndirty)
 CTL_PROTO(stats_arenas_i_extents_j_nmuzzy)
 CTL_PROTO(stats_arenas_i_extents_j_nretained)
-CTL_PROTO(stats_arenas_i_extents_j_npinned)
 CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes)
 CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes)
 CTL_PROTO(stats_arenas_i_extents_j_retained_bytes)
-CTL_PROTO(stats_arenas_i_extents_j_pinned_bytes)
 INDEX_PROTO(stats_arenas_i_extents_j)
 
 /* Merged set of stats for HPA shard. */
@@ -310,14 +309,6 @@ CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)
 CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)
 
 INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j)
-CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_min_extents)
-CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_max_extents)
-CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_extents)
-CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_ps)
-CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_pages_per_ps)
-CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_extents_per_ps)
-CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps)
-INDEX_PROTO(stats_arenas_i_hpa_shard_alloc_j)
 
 CTL_PROTO(stats_arenas_i_nthreads)
 CTL_PROTO(stats_arenas_i_uptime)
@@ -329,7 +320,6 @@ CTL_PROTO(stats_arenas_i_pdirty)
 CTL_PROTO(stats_arenas_i_pmuzzy)
 CTL_PROTO(stats_arenas_i_mapped)
 CTL_PROTO(stats_arenas_i_retained)
-CTL_PROTO(stats_arenas_i_pinned)
 CTL_PROTO(stats_arenas_i_extent_avail)
 CTL_PROTO(stats_arenas_i_dirty_npurge)
 CTL_PROTO(stats_arenas_i_dirty_nmadvise)
@@ -365,9 +355,10 @@ CTL_PROTO(stats_metadata_thp)
 CTL_PROTO(stats_resident)
 CTL_PROTO(stats_mapped)
 CTL_PROTO(stats_retained)
-CTL_PROTO(stats_pinned)
 CTL_PROTO(stats_zero_reallocs)
 CTL_PROTO(approximate_stats_active)
+CTL_PROTO(experimental_hooks_install)
+CTL_PROTO(experimental_hooks_remove)
 CTL_PROTO(experimental_hooks_prof_backtrace)
 CTL_PROTO(experimental_hooks_prof_dump)
 CTL_PROTO(experimental_hooks_prof_sample)
@@ -380,6 +371,7 @@ CTL_PROTO(experimental_arenas_i_pactivep)
 INDEX_PROTO(experimental_arenas_i)
 CTL_PROTO(experimental_prof_recent_alloc_max)
 CTL_PROTO(experimental_prof_recent_alloc_dump)
+CTL_PROTO(experimental_batch_alloc)
 CTL_PROTO(experimental_arenas_create_ext)
 
 #define MUTEX_STATS_CTL_PROTO_GEN(n)                                           \
@@ -495,6 +487,7 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
     {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
     {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
     {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
+    {NAME("hpa_sec_batch_fill_extra"), CTL(opt_hpa_sec_batch_fill_extra)},
     {NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
     {NAME("metadata_thp"), CTL(opt_metadata_thp)},
     {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)},
@@ -704,11 +697,9 @@ static const ctl_named_node_t stats_arenas_i_extents_j_node[] = {
     {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)},
     {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)},
     {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)},
-    {NAME("npinned"), CTL(stats_arenas_i_extents_j_npinned)},
     {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)},
     {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)},
-    {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)},
-    {NAME("pinned_bytes"), CTL(stats_arenas_i_extents_j_pinned_bytes)}};
+    {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)}};
 
 static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = {
     {NAME(""), CHILD(named, stats_arenas_i_extents_j)}};
@@ -787,23 +778,6 @@ static const ctl_named_node_t
 static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] =
     {{INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}};
 
-static const ctl_named_node_t stats_arenas_i_hpa_shard_alloc_j_node[] = {
-    {NAME("min_extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_min_extents)},
-    {NAME("max_extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_max_extents)},
-    {NAME("extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_extents)},
-    {NAME("ps"), CTL(stats_arenas_i_hpa_shard_alloc_j_ps)},
-    {NAME("pages_per_ps"), CTL(stats_arenas_i_hpa_shard_alloc_j_pages_per_ps)},
-    {NAME("extents_per_ps"),
-        CTL(stats_arenas_i_hpa_shard_alloc_j_extents_per_ps)},
-    {NAME("total_elapsed_ns_per_ps"),
-        CTL(stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps)}};
-
-static const ctl_named_node_t super_stats_arenas_i_hpa_shard_alloc_j_node[] = {
-    {NAME(""), CHILD(named, stats_arenas_i_hpa_shard_alloc_j)}};
-
-static const ctl_indexed_node_t stats_arenas_i_hpa_shard_alloc_node[] = {
-    {INDEX(stats_arenas_i_hpa_shard_alloc_j)}};
-
 static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
     {NAME("npageslabs"), CTL(stats_arenas_i_hpa_shard_npageslabs)},
     {NAME("nactive"), CTL(stats_arenas_i_hpa_shard_nactive)},
@@ -817,8 +791,6 @@ static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
     {NAME("nhugify_failures"), CTL(stats_arenas_i_hpa_shard_nhugify_failures)},
     {NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)},
 
-    {NAME("alloc"), CHILD(indexed, stats_arenas_i_hpa_shard_alloc)},
-
     {NAME("full_slabs"), CHILD(named, stats_arenas_i_hpa_shard_full_slabs)},
     {NAME("empty_slabs"), CHILD(named, stats_arenas_i_hpa_shard_empty_slabs)},
     {NAME("nonfull_slabs"),
@@ -835,7 +807,6 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
     {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)},
     {NAME("mapped"), CTL(stats_arenas_i_mapped)},
     {NAME("retained"), CTL(stats_arenas_i_retained)},
-    {NAME("pinned"), CTL(stats_arenas_i_pinned)},
     {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)},
     {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)},
     {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)},
@@ -901,7 +872,6 @@ static const ctl_named_node_t stats_node[] = {
     {NAME("resident"), CTL(stats_resident)},
     {NAME("mapped"), CTL(stats_mapped)},
     {NAME("retained"), CTL(stats_retained)},
-    {NAME("pinned"), CTL(stats_pinned)},
     {NAME("background_thread"), CHILD(named, stats_background_thread)},
     {NAME("mutexes"), CHILD(named, stats_mutexes)},
     {NAME("arenas"), CHILD(indexed, stats_arenas)},
@@ -909,6 +879,8 @@ static const ctl_named_node_t stats_node[] = {
 };
 
 static const ctl_named_node_t experimental_hooks_node[] = {
+    {NAME("install"), CTL(experimental_hooks_install)},
+    {NAME("remove"), CTL(experimental_hooks_remove)},
     {NAME("prof_backtrace"), CTL(experimental_hooks_prof_backtrace)},
     {NAME("prof_dump"), CTL(experimental_hooks_prof_dump)},
     {NAME("prof_sample"), CTL(experimental_hooks_prof_sample)},
@@ -939,7 +911,8 @@ static const ctl_named_node_t experimental_node[] = {
     {NAME("utilization"), CHILD(named, experimental_utilization)},
     {NAME("arenas"), CHILD(indexed, experimental_arenas)},
     {NAME("arenas_create_ext"), CTL(experimental_arenas_create_ext)},
-    {NAME("prof_recent"), CHILD(named, experimental_prof_recent)}};
+    {NAME("prof_recent"), CHILD(named, experimental_prof_recent)},
+    {NAME("batch_alloc"), CTL(experimental_batch_alloc)}};
 
 static const ctl_named_node_t root_node[] = {{NAME("version"), CTL(version)},
     {NAME("epoch"), CTL(epoch)},
@@ -1138,8 +1111,6 @@ ctl_arena_stats_sdmerge(
 			sdstats->astats.mapped += astats->astats.mapped;
 			sdstats->astats.pa_shard_stats.pac_stats.retained +=
 			    astats->astats.pa_shard_stats.pac_stats.retained;
-			sdstats->astats.pa_shard_stats.pac_stats.pinned +=
-			    astats->astats.pa_shard_stats.pac_stats.pinned;
 			sdstats->astats.pa_shard_stats.edata_avail +=
 			    astats->astats.pa_shard_stats.edata_avail;
 		}
@@ -1276,16 +1247,12 @@ ctl_arena_stats_sdmerge(
 			sdstats->estats[i].nmuzzy += astats->estats[i].nmuzzy;
 			sdstats->estats[i].nretained +=
 			    astats->estats[i].nretained;
-			sdstats->estats[i].npinned +=
-			    astats->estats[i].npinned;
 			sdstats->estats[i].dirty_bytes +=
 			    astats->estats[i].dirty_bytes;
 			sdstats->estats[i].muzzy_bytes +=
 			    astats->estats[i].muzzy_bytes;
 			sdstats->estats[i].retained_bytes +=
 			    astats->estats[i].retained_bytes;
-			sdstats->estats[i].pinned_bytes +=
-			    astats->estats[i].pinned_bytes;
 		}
 
 		/* Merge HPA stats. */
@@ -1400,8 +1367,6 @@ ctl_refresh(tsdn_t *tsdn) {
 		ctl_stats->mapped = ctl_sarena->astats->astats.mapped;
 		ctl_stats->retained = ctl_sarena->astats->astats.pa_shard_stats
 		                          .pac_stats.retained;
-		ctl_stats->pinned = ctl_sarena->astats->astats.pa_shard_stats
-		                        .pac_stats.pinned;
 
 		ctl_background_thread_stats_read(tsdn);
 
@@ -2208,6 +2173,8 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
+CTL_RO_NL_GEN(
+    opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, size_t)
 CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
 CTL_RO_NL_GEN(
     opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
@@ -2337,7 +2304,13 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 			ret = EAGAIN;
 			goto label_return;
 		}
-		thread_migrate_arena(tsd, oldarena, newarena);
+		/* Set new arena/tcache associations. */
+		arena_migrate(tsd, oldarena, newarena);
+		if (tcache_available(tsd)) {
+			tcache_arena_reassociate(tsd_tsdn(tsd),
+			    tsd_tcache_slowp_get(tsd), tsd_tcachep_get(tsd),
+			    newarena);
+		}
 	}
 
 	ret = 0;
@@ -3748,7 +3721,6 @@ CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t)
 CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t)
 CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t)
 CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t)
-CTL_RO_CGEN(config_stats, stats_pinned, ctl_stats->pinned, size_t)
 
 CTL_RO_CGEN(config_stats, stats_background_thread_num_threads,
     ctl_stats->background_thread.num_threads, size_t)
@@ -3814,8 +3786,6 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
     arenas_i(mib[2])->astats->astats.mapped, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_retained,
     arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.retained, size_t)
-CTL_RO_CGEN(config_stats, stats_arenas_i_pinned,
-    arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats.pinned, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail,
     arenas_i(mib[2])->astats->astats.pa_shard_stats.edata_avail, size_t)
 
@@ -3988,10 +3958,9 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
 		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_dirty.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_muzzy.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_retained.mtx);
-		MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_pinned.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.decay_dirty.mtx);
 		MUTEX_PROF_RESET(arena->pa_shard.pac.decay_muzzy.mtx);
-		MUTEX_PROF_RESET(arena->cache_bin_array_descriptor_ql_mtx);
+		MUTEX_PROF_RESET(arena->tcache_ql_mtx);
 		MUTEX_PROF_RESET(arena->base->mtx);
 
 		for (szind_t j = 0; j < SC_NBINS; j++) {
@@ -4060,21 +4029,17 @@ stats_arenas_i_lextents_j_index(
 }
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty,
-    arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t)
+    arenas_i(mib[2])->astats->estats[mib[4]].ndirty, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy,
-    arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t)
+    arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained,
-    arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t)
-CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_npinned,
-    arenas_i(mib[2])->astats->estats[mib[4]].npinned, size_t)
+    arenas_i(mib[2])->astats->estats[mib[4]].nretained, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes,
-    arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t)
+    arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes,
-    arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t)
+    arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes,
-    arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t)
-CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_pinned_bytes,
-    arenas_i(mib[2])->astats->estats[mib[4]].pinned_bytes, size_t)
+    arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, size_t);
 
 static const ctl_named_node_t *
 stats_arenas_i_extents_j_index(
@@ -4086,116 +4051,86 @@ stats_arenas_i_extents_j_index(
 }
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npageslabs,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.npageslabs, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.npageslabs, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nactive,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.nactive, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.nactive, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndirty,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.ndirty, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.merged.ndirty, size_t);
 
 /* Nonhuge slabs */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_npageslabs_nonhuge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].npageslabs, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].npageslabs, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_nactive_nonhuge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].nactive, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].nactive, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_ndirty_nonhuge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].ndirty, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[0].ndirty, size_t);
 
 /* Huge slabs */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_npageslabs_huge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].npageslabs, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].npageslabs, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_nactive_huge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].nactive, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].nactive, size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_slabs_ndirty_huge,
-    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].ndirty, size_t)
+    arenas_i(mib[2])->astats->hpastats.psset_stats.slabs[1].ndirty, size_t);
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurge_passes,
     arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurge_passes,
-    uint64_t)
+    uint64_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_npurges,
-    arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t)
+    arenas_i(mib[2])->astats->hpastats.nonderived_stats.npurges, uint64_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugifies,
-    arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t)
+    arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugifies, uint64_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugify_failures,
     arenas_i(mib[2])->astats->hpastats.nonderived_stats.nhugify_failures,
-    uint64_t)
+    uint64_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies,
-    arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t)
-
-CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_min_extents,
-    arenas_i(mib[2])
-        ->astats->hpastats.nonderived_stats.hpa_alloc_min_extents[mib[5]],
-    uint64_t);
-CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_max_extents,
-    arenas_i(mib[2])
-        ->astats->hpastats.nonderived_stats.hpa_alloc_max_extents[mib[5]],
-    uint64_t);
-CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_extents,
-    arenas_i(mib[2])
-        ->astats->hpastats.nonderived_stats.hpa_alloc_extents[mib[5]],
-    uint64_t);
-CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_ps,
-    arenas_i(mib[2])->astats->hpastats.nonderived_stats.hpa_alloc_ps[mib[5]],
-    uint64_t);
-CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_pages_per_ps,
-    arenas_i(mib[2])
-        ->astats->hpastats.nonderived_stats.hpa_alloc_pages_per_ps[mib[5]],
-    uint64_t);
-CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_extents_per_ps,
-    arenas_i(mib[2])
-        ->astats->hpastats.nonderived_stats.hpa_alloc_extents_per_ps[mib[5]],
-    uint64_t);
-CTL_RO_CGEN(config_stats,
-    stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps,
-    arenas_i(mib[2])
-        ->astats->hpastats.nonderived_stats
-        .hpa_alloc_total_elapsed_ns_per_ps[mib[5]],
-    uint64_t);
+    arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t);
 
 /* Full, nonhuge */
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].npageslabs,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].nactive,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[0].ndirty,
-    size_t)
+    size_t);
 
 /* Full, huge */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].npageslabs,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].nactive,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ndirty_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.full_slabs[1].ndirty,
-    size_t)
+    size_t);
 
 /* Empty, nonhuge */
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].npageslabs,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].nactive,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[0].ndirty,
-    size_t)
+    size_t);
 
 /* Empty, huge */
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].npageslabs,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_nactive_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].nactive,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge,
     arenas_i(mib[2])->astats->hpastats.psset_stats.empty_slabs[1].ndirty,
-    size_t)
+    size_t);
 
 /* Nonfull, nonhuge */
 CTL_RO_CGEN(config_stats,
@@ -4203,19 +4138,19 @@ CTL_RO_CGEN(config_stats,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0]
         .npageslabs,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0]
         .nactive,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats,
     stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][0]
         .ndirty,
-    size_t)
+    size_t);
 
 /* Nonfull, huge */
 CTL_RO_CGEN(config_stats,
@@ -4223,17 +4158,17 @@ CTL_RO_CGEN(config_stats,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1]
         .npageslabs,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1]
         .nactive,
-    size_t)
+    size_t);
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge,
     arenas_i(mib[2])
         ->astats->hpastats.psset_stats.nonfull_slabs[mib[5]][1]
         .ndirty,
-    size_t)
+    size_t);
 
 static const ctl_named_node_t *
 stats_arenas_i_hpa_shard_nonfull_slabs_j_index(
@@ -4244,15 +4179,6 @@ stats_arenas_i_hpa_shard_nonfull_slabs_j_index(
 	return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node;
 }
 
-static const ctl_named_node_t *
-stats_arenas_i_hpa_shard_alloc_j_index(
-    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
-	if (j > SEC_MAX_NALLOCS) {
-		return NULL;
-	}
-	return super_stats_arenas_i_hpa_shard_alloc_j_node;
-}
-
 static bool
 ctl_arenas_i_verify(size_t i) {
 	size_t a = arenas_i2a_impl(i, true, true);
@@ -4279,6 +4205,51 @@ label_return:
 	return ret;
 }
 
+static int
+experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+    void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+	int ret;
+	if (oldp == NULL || oldlenp == NULL || newp == NULL) {
+		ret = EINVAL;
+		goto label_return;
+	}
+	/*
+	 * Note: this is a *private* struct.  This is an experimental interface;
+	 * forcing the user to know the jemalloc internals well enough to
+	 * extract the ABI hopefully ensures nobody gets too comfortable with
+	 * this API, which can change at a moment's notice.
+	 */
+	hooks_t hooks;
+	WRITE(hooks, hooks_t);
+	void *handle = hook_install(tsd_tsdn(tsd), &hooks);
+	if (handle == NULL) {
+		ret = EAGAIN;
+		goto label_return;
+	}
+	READ(handle, void *);
+
+	ret = 0;
+label_return:
+	return ret;
+}
+
+static int
+experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+    void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+	int ret;
+	WRITEONLY();
+	void *handle = NULL;
+	WRITE(handle, void *);
+	if (handle == NULL) {
+		ret = EINVAL;
+		goto label_return;
+	}
+	hook_remove(tsd_tsdn(tsd), handle);
+	ret = 0;
+label_return:
+	return ret;
+}
+
 /*
  * Output six memory utilization entries for an input pointer, the first one of
  * type (void *) and the remaining five of type size_t, describing the following
@@ -4618,6 +4589,34 @@ label_return:
 	return ret;
 }
 
+typedef struct batch_alloc_packet_s batch_alloc_packet_t;
+struct batch_alloc_packet_s {
+	void **ptrs;
+	size_t num;
+	size_t size;
+	int    flags;
+};
+
+static int
+experimental_batch_alloc_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+    void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+	int ret;
+
+	VERIFY_READ(size_t);
+
+	batch_alloc_packet_t batch_alloc_packet;
+	ASSURED_WRITE(batch_alloc_packet, batch_alloc_packet_t);
+	size_t filled = batch_alloc(batch_alloc_packet.ptrs,
+	    batch_alloc_packet.num, batch_alloc_packet.size,
+	    batch_alloc_packet.flags);
+	READ(filled, size_t);
+
+	ret = 0;
+
+label_return:
+	return ret;
+}
+
 static int
 prof_stats_bins_i_live_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
     void *oldp, size_t *oldlenp, void *newp, size_t newlen) {

src/emap.c

@@ -50,8 +50,7 @@ emap_try_acquire_edata_neighbor_impl(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
 	assert(!edata_state_in_transition(expected_state));
 	assert(expected_state == extent_state_dirty
 	    || expected_state == extent_state_muzzy
-	    || expected_state == extent_state_retained
-	    || expected_state == extent_state_pinned);
+	    || expected_state == extent_state_retained);
 
 	void *neighbor_addr = forward ? edata_past_get(edata)
 	                              : edata_before_get(edata);

src/eset.c

@@ -32,17 +32,17 @@ eset_init(eset_t *eset, extent_state_t state) {
 }
 
 size_t
-eset_npages_get(const eset_t *eset) {
+eset_npages_get(eset_t *eset) {
 	return atomic_load_zu(&eset->npages, ATOMIC_RELAXED);
 }
 
 size_t
-eset_nextents_get(const eset_t *eset, pszind_t pind) {
+eset_nextents_get(eset_t *eset, pszind_t pind) {
 	return atomic_load_zu(&eset->bin_stats[pind].nextents, ATOMIC_RELAXED);
 }
 
 size_t
-eset_nbytes_get(const eset_t *eset, pszind_t pind) {
+eset_nbytes_get(eset_t *eset, pszind_t pind) {
 	return atomic_load_zu(&eset->bin_stats[pind].nbytes, ATOMIC_RELAXED);
 }
 
@@ -98,10 +98,7 @@ eset_insert(eset_t *eset, edata_t *edata) {
 		eset_stats_add(eset, pind, size);
 	}
 
-	/* Pinned extents skip LRU as they do not decay. */
-	if (!edata_pinned_get(edata)) {
-		edata_list_inactive_append(&eset->lru, edata);
-	}
+	edata_list_inactive_append(&eset->lru, edata);
 	size_t npages = size >> LG_PAGE;
 	/*
 	 * All modifications to npages hold the mutex (as asserted above), so we
@@ -146,9 +143,7 @@ eset_remove(eset_t *eset, edata_t *edata) {
 			    edata_heap_first(&eset->bins[pind].heap));
 		}
 	}
-	if (!edata_pinned_get(edata)) {
-		edata_list_inactive_remove(&eset->lru, edata);
-	}
+	edata_list_inactive_remove(&eset->lru, edata);
 	size_t npages = size >> LG_PAGE;
 	/*
 	 * As in eset_insert, we hold eset->mtx and so don't need atomic
@@ -284,15 +279,10 @@ eset_fit_alignment(
  * avoiding reusing and splitting large extents for smaller sizes.  In practice,
  * it's set to opt_lg_extent_max_active_fit for the dirty eset and SC_PTR_BITS
  * for others.
- *
- * If prefer_small is true, return as soon as the smallest fitting bin yields a
- * candidate, instead of scanning further bins for an older/lower extent.
- * Useful for fragmentation control for the pinned pool.
  */
 static edata_t *
 eset_first_fit(
-    eset_t *eset, size_t size, bool exact_only, unsigned lg_max_fit,
-    bool prefer_small) {
+    eset_t *eset, size_t size, bool exact_only, unsigned lg_max_fit) {
 	edata_t                     *ret = NULL;
 	edata_cmp_summary_t ret_summ JEMALLOC_CC_SILENCE_INIT({0});
 
@@ -337,9 +327,6 @@ eset_first_fit(
 	if (sz_large_size_classes_disabled() && pind != pind_prev) {
 		ret = eset_enumerate_search(eset, size, pind_prev,
 		    /* exact_only */ false, &ret_summ);
-		if (prefer_small && ret != NULL) {
-			return ret;
-		}
 	}
 
 	for (pszind_t i =
@@ -376,9 +363,6 @@ eset_first_fit(
 			           edata_cmp_summary_get(edata))
 			        == 0);
 			ret = edata;
-			if (prefer_small) {
-				return ret;
-			}
 			ret_summ = eset->bins[i].heap_min;
 		}
 		if (i == SC_NPSIZES) {
@@ -392,15 +376,14 @@ eset_first_fit(
 
 edata_t *
 eset_fit(eset_t *eset, size_t esize, size_t alignment, bool exact_only,
-    unsigned lg_max_fit, bool prefer_small) {
+    unsigned lg_max_fit) {
 	size_t max_size = esize + PAGE_CEILING(alignment) - PAGE;
 	/* Beware size_t wrap-around. */
 	if (max_size < esize) {
 		return NULL;
 	}
 
-	edata_t *edata = eset_first_fit(eset, max_size, exact_only, lg_max_fit,
-	    prefer_small);
+	edata_t *edata = eset_first_fit(eset, max_size, exact_only, lg_max_fit);
 
 	if (alignment > PAGE && edata == NULL) {
 		/*

src/extent.c

@@ -17,8 +17,9 @@ size_t opt_process_madvise_max_batch =
 #ifdef JEMALLOC_HAVE_PROCESS_MADVISE
     PROCESS_MADVISE_MAX_BATCH_DEFAULT;
 #else
-    0;
+    0
 #endif
+;
 
 static bool extent_commit_impl(tsdn_t *tsdn, ehooks_t *ehooks, edata_t *edata,
     size_t offset, size_t length, bool growing_retained);
@@ -69,7 +70,6 @@ extent_may_force_decay(pac_t *pac) {
 static bool
 extent_try_delayed_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
     ecache_t *ecache, edata_t *edata) {
-	malloc_mutex_assert_owner(tsdn, &ecache->mtx);
 	emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active);
 
 	bool coalesced;
@@ -212,7 +212,6 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 	switch (ecache->state) {
 	case extent_state_dirty:
 	case extent_state_muzzy:
-	case extent_state_pinned:
 		emap_update_edata_state(
 		    tsdn, pac->emap, edata, extent_state_active);
 		break;
@@ -245,11 +244,7 @@ extents_abandon_vm(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 	}
 	/*
 	 * Leak extent after making sure its pages have already been purged, so
-	 * that this is only a virtual memory leak, except when the extent is
-	 * pinned/unpurgeable, for which a real memory leak happens.  This is
-	 * acceptable because reaching this path requires that an extent split
-	 * fail, which is already an exceptional condition (typically an OOM
-	 * on edata_t allocation).
+	 * that this is only a virtual memory leak.
 	 */
 	if (ecache->state == extent_state_dirty) {
 		if (extent_purge_lazy_impl(
@@ -439,12 +434,8 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
 		 * then no longer satify a request for its original size.  To
 		 * limit this effect, when delayed coalescing is enabled, we
 		 * put a cap on how big an extent we can split for a request.
-		 *
-		 * Pinned extents are exempt: they are never purged, so the cap
-		 * doesn't apply.
 		 */
-		unsigned lg_max_fit = (ecache->delay_coalesce
-		    && ecache != &pac->ecache_pinned)
+		unsigned lg_max_fit = ecache->delay_coalesce
 		    ? (unsigned)opt_lg_extent_max_active_fit
 		    : SC_PTR_BITS;
 
@@ -457,13 +448,7 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
 		 * allocations.
 		 */
 		bool exact_only = (!maps_coalesce && !opt_retain) || guarded;
-		/*
-		 * When selecting a pinned extent, avoid breaking larger extent
-		 * if a smaller one works.
-		 */
-		bool prefer_small = (ecache == &pac->ecache_pinned);
-		edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit,
-		    prefer_small);
+		edata = eset_fit(eset, size, alignment, exact_only, lg_max_fit);
 	}
 	if (edata == NULL) {
 		return NULL;
@@ -748,9 +733,10 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	}
 	bool zeroed = false;
 	bool committed = false;
-	unsigned flags = 0;
-	void *ptr = ehooks_alloc(tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed,
-	    &committed, &flags);
+
+	void *ptr = ehooks_alloc(
+	    tsdn, ehooks, NULL, alloc_size, PAGE, &zeroed, &committed);
+
 	if (ptr == NULL) {
 		edata_cache_put(tsdn, pac->edata_cache, edata);
 		goto label_err;
@@ -760,10 +746,6 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	edata_init(edata, ind, ptr, alloc_size, false, SC_NSIZES,
 	    extent_sn_next(pac), extent_state_active, zeroed, committed,
 	    EXTENT_PAI_PAC, EXTENT_IS_HEAD);
-	edata_hook_flags_init(edata, flags);
-	if (flags & EXTENT_ALLOC_FLAG_PINNED) {
-		atomic_store_b(&pac->has_pinned, true, ATOMIC_RELAXED);
-	}
 
 	if (extent_register_no_gdump_add(tsdn, pac, edata)) {
 		edata_cache_put(tsdn, pac->edata_cache, edata);
@@ -785,10 +767,12 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 
 	if (result == extent_split_interior_ok) {
 		if (lead != NULL) {
-			pac_record_grown(tsdn, pac, ehooks, lead);
+			extent_record(
+			    tsdn, pac, ehooks, &pac->ecache_retained, lead);
 		}
 		if (trail != NULL) {
-			pac_record_grown(tsdn, pac, ehooks, trail);
+			extent_record(
+			    tsdn, pac, ehooks, &pac->ecache_retained, trail);
 		}
 	} else {
 		/*
@@ -800,7 +784,8 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 			if (config_prof) {
 				extent_gdump_add(tsdn, to_salvage);
 			}
-			pac_record_grown(tsdn, pac, ehooks, to_salvage);
+			extent_record(tsdn, pac, ehooks, &pac->ecache_retained,
+			    to_salvage);
 		}
 		if (to_leak != NULL) {
 			extent_deregister_no_gdump_sub(tsdn, pac, to_leak);
@@ -811,8 +796,6 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	}
 
 	if (*commit && !edata_committed_get(edata)) {
-		/* Pinned memory must be committed by the hook. */
-		assert(!edata_pinned_get(edata));
 		if (extent_commit_impl(
 		        tsdn, ehooks, edata, 0, edata_size_get(edata), true)) {
 			extent_record(
@@ -832,18 +815,15 @@ extent_grow_retained(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 
 	/*
 	 * Increment extent_grow_next if doing so wouldn't exceed the allowed
-	 * range.  Skip for pinned: pinned memory is a finite resource;
-	 * oversized remnants waste it.
+	 * range.
 	 */
 	/* All opportunities for failure are past. */
-	if (!(flags & EXTENT_ALLOC_FLAG_PINNED)) {
-		exp_grow_size_commit(&pac->exp_grow, exp_grow_skip);
-	}
+	exp_grow_size_commit(&pac->exp_grow, exp_grow_skip);
 	malloc_mutex_unlock(tsdn, &pac->grow_mtx);
 
 	if (huge_arena_pac_thp.thp_madvise) {
 		/* Avoid using HUGEPAGE when the grow size is less than HUGEPAGE. */
-		if (arena_ind_is_huge(ind)
+		if (ind != 0 && ind == huge_arena_ind
 		    && ehooks_are_default(ehooks)
 		    && likely(alloc_size >= HUGEPAGE)) {
 			extent_handle_huge_arena_thp(tsdn, &huge_arena_pac_thp,
@@ -1040,13 +1020,11 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 		edata = extent_try_coalesce(
 		    tsdn, pac, ehooks, ecache, edata, &coalesced_unused);
 	} else if (edata_size_get(edata) >= SC_LARGE_MINCLASS) {
-		assert(edata_pinned_get(edata)
-		    ? (ecache == &pac->ecache_pinned)
-		    : (ecache == &pac->ecache_dirty));
+		assert(ecache == &pac->ecache_dirty);
 		/* Always coalesce large extents eagerly. */
 		/**
 		* Maximum size limit (max_size) for large extents waiting to be coalesced
-		* in pinned/dirty ecache.
+		* in dirty ecache.
 		*
 		* When set to a non-zero value, this parameter restricts the maximum size
 		* of large extents after coalescing. If the combined size of two extents
@@ -1078,9 +1056,7 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
 			edata = extent_try_coalesce_large(tsdn, pac, ehooks,
 			    ecache, edata, max_size, &coalesced);
 		} while (coalesced);
-		/* Pinned extents cannot be purged; skip the oversize shortcut. */
-		if (ecache == &pac->ecache_dirty
-		    && edata_size_get(edata) >= atomic_load_zu(
+		if (edata_size_get(edata) >= atomic_load_zu(
 		        &pac->oversize_threshold, ATOMIC_RELAXED)
 		    && !background_thread_enabled()
 		    && extent_may_force_decay(pac)) {
@@ -1143,9 +1119,8 @@ extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void *new_addr,
 		return NULL;
 	}
 	size_t palignment = ALIGNMENT_CEILING(alignment, PAGE);
-	unsigned flags = 0;
-	void  *addr = ehooks_alloc(tsdn, ehooks, new_addr, size, palignment,
-	    &zero, commit, &flags);
+	void  *addr = ehooks_alloc(
+            tsdn, ehooks, new_addr, size, palignment, &zero, commit);
 	if (addr == NULL) {
 		edata_cache_put(tsdn, pac->edata_cache, edata);
 		return NULL;
@@ -1154,10 +1129,6 @@ extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, void *new_addr,
 	    /* slab */ false, SC_NSIZES, extent_sn_next(pac),
 	    extent_state_active, zero, *commit, EXTENT_PAI_PAC,
 	    opt_retain ? EXTENT_IS_HEAD : EXTENT_NOT_HEAD);
-	edata_hook_flags_init(edata, flags);
-	if (flags & EXTENT_ALLOC_FLAG_PINNED) {
-		atomic_store_b(&pac->has_pinned, true, ATOMIC_RELAXED);
-	}
 	/*
 	 * Retained memory is not counted towards gdump.  Only if an extent is
 	 * allocated as a separate mapping, i.e. growing_retained is false, then
@@ -1357,7 +1328,6 @@ extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *edata,
 	    /* slab */ false, SC_NSIZES, edata_sn_get(edata),
 	    edata_state_get(edata), edata_zeroed_get(edata),
 	    edata_committed_get(edata), EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
-	edata_hook_flags_init(trail, edata_alloc_flags_get(edata));
 	emap_prepare_t prepare;
 	bool           err = emap_split_prepare(
             tsdn, pac->emap, &prepare, edata, size_a, trail, size_b);
@@ -1442,8 +1412,6 @@ extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a,
 	                                        : edata_sn_get(b));
 	edata_zeroed_set(a, edata_zeroed_get(a) && edata_zeroed_get(b));
 
-	assert(edata_pinned_get(a) == edata_pinned_get(b));
-
 	emap_merge_commit(tsdn, pac->emap, &prepare, a, b);
 
 	edata_cache_put(tsdn, pac->edata_cache, b);

src/extent_dss.c

@@ -33,20 +33,11 @@ static atomic_b_t dss_exhausted;
 /* Atomic current upper limit on DSS addresses. */
 static atomic_p_t dss_max;
 
-#ifdef JEMALLOC_JET
-extent_dss_sbrk_hook_t extent_dss_sbrk_hook = NULL;
-#endif
-
 /******************************************************************************/
 
 static void *
 extent_dss_sbrk(intptr_t increment) {
 #ifdef JEMALLOC_DSS
-#ifdef JEMALLOC_JET
-	if (extent_dss_sbrk_hook != NULL) {
-		return extent_dss_sbrk_hook(increment);
-	}
-#endif
 	return sbrk(increment);
 #else
 	not_implemented();

src/hook.c

@@ -0,0 +1,192 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+#include "jemalloc/internal/hook.h"
+
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/seq.h"
+
+typedef struct hooks_internal_s hooks_internal_t;
+struct hooks_internal_s {
+	hooks_t hooks;
+	bool    in_use;
+};
+
+seq_define(hooks_internal_t, hooks)
+
+    static atomic_u_t nhooks = ATOMIC_INIT(0);
+static seq_hooks_t    hooks[HOOK_MAX];
+static malloc_mutex_t hooks_mu;
+
+bool
+hook_boot(void) {
+	return malloc_mutex_init(
+	    &hooks_mu, "hooks", WITNESS_RANK_HOOK, malloc_mutex_rank_exclusive);
+}
+
+static void *
+hook_install_locked(hooks_t *to_install) {
+	hooks_internal_t hooks_internal;
+	for (int i = 0; i < HOOK_MAX; i++) {
+		bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]);
+		/* We hold mu; no concurrent access. */
+		assert(success);
+		if (!hooks_internal.in_use) {
+			hooks_internal.hooks = *to_install;
+			hooks_internal.in_use = true;
+			seq_store_hooks(&hooks[i], &hooks_internal);
+			atomic_store_u(&nhooks,
+			    atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1,
+			    ATOMIC_RELAXED);
+			return &hooks[i];
+		}
+	}
+	return NULL;
+}
+
+void *
+hook_install(tsdn_t *tsdn, hooks_t *to_install) {
+	malloc_mutex_lock(tsdn, &hooks_mu);
+	void *ret = hook_install_locked(to_install);
+	if (ret != NULL) {
+		tsd_global_slow_inc(tsdn);
+	}
+	malloc_mutex_unlock(tsdn, &hooks_mu);
+	return ret;
+}
+
+static void
+hook_remove_locked(seq_hooks_t *to_remove) {
+	hooks_internal_t hooks_internal;
+	bool success = seq_try_load_hooks(&hooks_internal, to_remove);
+	/* We hold mu; no concurrent access. */
+	assert(success);
+	/* Should only remove hooks that were added. */
+	assert(hooks_internal.in_use);
+	hooks_internal.in_use = false;
+	seq_store_hooks(to_remove, &hooks_internal);
+	atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1,
+	    ATOMIC_RELAXED);
+}
+
+void
+hook_remove(tsdn_t *tsdn, void *opaque) {
+	if (config_debug) {
+		char *hooks_begin = (char *)&hooks[0];
+		char *hooks_end = (char *)&hooks[HOOK_MAX];
+		char *hook = (char *)opaque;
+		assert(hooks_begin <= hook && hook < hooks_end
+		    && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0);
+	}
+	malloc_mutex_lock(tsdn, &hooks_mu);
+	hook_remove_locked((seq_hooks_t *)opaque);
+	tsd_global_slow_dec(tsdn);
+	malloc_mutex_unlock(tsdn, &hooks_mu);
+}
+
+#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr)                                \
+	for (int for_each_hook_counter = 0; for_each_hook_counter < HOOK_MAX;  \
+	     for_each_hook_counter++) {                                        \
+		bool for_each_hook_success = seq_try_load_hooks(               \
+		    (hooks_internal_ptr), &hooks[for_each_hook_counter]);      \
+		if (!for_each_hook_success) {                                  \
+			continue;                                              \
+		}                                                              \
+		if (!(hooks_internal_ptr)->in_use) {                           \
+			continue;                                              \
+		}
+#define FOR_EACH_HOOK_END }
+
+static bool *
+hook_reentrantp(void) {
+	/*
+	 * We prevent user reentrancy within hooks.  This is basically just a
+	 * thread-local bool that triggers an early-exit.
+	 *
+	 * We don't fold in_hook into reentrancy.  There are two reasons for
+	 * this:
+	 * - Right now, we turn on reentrancy during things like extent hook
+	 *   execution.  Allocating during extent hooks is not officially
+	 *   supported, but we don't want to break it for the time being.  These
+	 *   sorts of allocations should probably still be hooked, though.
+	 * - If a hook allocates, we may want it to be relatively fast (after
+	 *   all, it executes on every allocator operation).  Turning on
+	 *   reentrancy is a fairly heavyweight mode (disabling tcache,
+	 *   redirecting to arena 0, etc.).  It's possible we may one day want
+	 *   to turn on reentrant mode here, if it proves too difficult to keep
+	 *   this working.  But that's fairly easy for us to see; OTOH, people
+	 *   not using hooks because they're too slow is easy for us to miss.
+	 *
+	 * The tricky part is
+	 * that this code might get invoked even if we don't have access to tsd.
+	 * This function mimics getting a pointer to thread-local data, except
+	 * that it might secretly return a pointer to some global data if we
+	 * know that the caller will take the early-exit path.
+	 * If we return a bool that indicates that we are reentrant, then the
+	 * caller will go down the early exit path, leaving the global
+	 * untouched.
+	 */
+	static bool in_hook_global = true;
+	tsdn_t     *tsdn = tsdn_fetch();
+	bool       *in_hook = tsdn_in_hookp_get(tsdn);
+	if (in_hook != NULL) {
+		return in_hook;
+	}
+	return &in_hook_global;
+}
+
+#define HOOK_PROLOGUE                                                          \
+	if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) {             \
+		return;                                                        \
+	}                                                                      \
+	bool *in_hook = hook_reentrantp();                                     \
+	if (*in_hook) {                                                        \
+		return;                                                        \
+	}                                                                      \
+	*in_hook = true;
+
+#define HOOK_EPILOGUE *in_hook = false;
+
+void
+hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
+    uintptr_t args_raw[3]) {
+	HOOK_PROLOGUE
+
+	hooks_internal_t hook;
+	FOR_EACH_HOOK_BEGIN(&hook)
+	hook_alloc h = hook.hooks.alloc_hook;
+	if (h != NULL) {
+		h(hook.hooks.extra, type, result, result_raw, args_raw);
+	}
+	FOR_EACH_HOOK_END
+
+	HOOK_EPILOGUE
+}
+
+void
+hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) {
+	HOOK_PROLOGUE
+	hooks_internal_t hook;
+	FOR_EACH_HOOK_BEGIN(&hook)
+	hook_dalloc h = hook.hooks.dalloc_hook;
+	if (h != NULL) {
+		h(hook.hooks.extra, type, address, args_raw);
+	}
+	FOR_EACH_HOOK_END
+	HOOK_EPILOGUE
+}
+
+void
+hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
+    size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) {
+	HOOK_PROLOGUE
+	hooks_internal_t hook;
+	FOR_EACH_HOOK_BEGIN(&hook)
+	hook_expand h = hook.hooks.expand_hook;
+	if (h != NULL) {
+		h(hook.hooks.extra, type, address, old_usize, new_usize,
+		    result_raw, args_raw);
+	}
+	FOR_EACH_HOOK_END
+	HOOK_EPILOGUE
+}

src/hpa.c

@@ -8,7 +8,18 @@
 #include "jemalloc/internal/witness.h"
 #include "jemalloc/internal/jemalloc_probe.h"
 
-static void hpa_dalloc_batch(tsdn_t *tsdn, hpa_shard_t *shard,
+static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
+    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
+    bool *deferred_work_generated);
+static bool     hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+        size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
+static bool     hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+        size_t old_size, size_t new_size, bool *deferred_work_generated);
+static void     hpa_dalloc(
+        tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
+static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
+
+static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
     edata_list_active_t *list, bool *deferred_work_generated);
 
 const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
@@ -35,7 +46,7 @@ hpa_supported(void) {
 		return false;
 	}
 	/*
-	 * We fundamentally rely on an address-space-hungry growth strategy for
+	 * We fundamentally rely on a address-space-hungry growth strategy for
 	 * hugepages.
 	 */
 	if (LG_SIZEOF_PTR != 3) {
@@ -98,19 +109,17 @@ hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
 	shard->stats.nhugifies = 0;
 	shard->stats.nhugify_failures = 0;
 	shard->stats.ndehugifies = 0;
-	memset(shard->stats.hpa_alloc_min_extents, 0,
-	    sizeof(shard->stats.hpa_alloc_min_extents));
-	memset(shard->stats.hpa_alloc_max_extents, 0,
-	    sizeof(shard->stats.hpa_alloc_max_extents));
-	memset(shard->stats.hpa_alloc_extents, 0,
-	    sizeof(shard->stats.hpa_alloc_extents));
-	memset(shard->stats.hpa_alloc_ps, 0, sizeof(shard->stats.hpa_alloc_ps));
-	memset(shard->stats.hpa_alloc_pages_per_ps, 0,
-	    sizeof(shard->stats.hpa_alloc_pages_per_ps));
-	memset(shard->stats.hpa_alloc_extents_per_ps, 0,
-	    sizeof(shard->stats.hpa_alloc_extents_per_ps));
-	memset(shard->stats.hpa_alloc_total_elapsed_ns_per_ps, 0,
-	    sizeof(shard->stats.hpa_alloc_total_elapsed_ns_per_ps));
+
+	/*
+	 * Fill these in last, so that if an hpa_shard gets used despite
+	 * initialization failing, we'll at least crash instead of just
+	 * operating on corrupted data.
+	 */
+	shard->pai.alloc = &hpa_alloc;
+	shard->pai.expand = &hpa_expand;
+	shard->pai.shrink = &hpa_shrink;
+	shard->pai.dalloc = &hpa_dalloc;
+	shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work;
 
 	err = sec_init(tsdn, &shard->sec, base, sec_opts);
 	if (err) {
@@ -136,18 +145,6 @@ hpa_shard_nonderived_stats_accum(
 	dst->nhugifies += src->nhugifies;
 	dst->nhugify_failures += src->nhugify_failures;
 	dst->ndehugifies += src->ndehugifies;
-	for (size_t i = 0; i <= SEC_MAX_NALLOCS; i++) {
-		dst->hpa_alloc_min_extents[i] += src->hpa_alloc_min_extents[i];
-		dst->hpa_alloc_max_extents[i] += src->hpa_alloc_max_extents[i];
-		dst->hpa_alloc_extents[i] += src->hpa_alloc_extents[i];
-		dst->hpa_alloc_ps[i] += src->hpa_alloc_ps[i];
-		dst->hpa_alloc_pages_per_ps[i] +=
-		    src->hpa_alloc_pages_per_ps[i];
-		dst->hpa_alloc_extents_per_ps[i] +=
-		    src->hpa_alloc_extents_per_ps[i];
-		dst->hpa_alloc_total_elapsed_ns_per_ps[i] +=
-		    src->hpa_alloc_total_elapsed_ns_per_ps[i];
-	}
 }
 
 void
@@ -654,18 +651,37 @@ hpa_shard_maybe_do_deferred_work(
 }
 
 static edata_t *
-hpa_try_alloc_one_offset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
-    hpdata_t *ps, hpdata_alloc_offset_t *alloc_offset, bool *oom) {
-	assert(*oom == false);
+hpa_try_alloc_one_no_grow(
+    tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) {
 	malloc_mutex_assert_owner(tsdn, &shard->mtx);
 
+	bool     err;
 	edata_t *edata = edata_cache_fast_get(tsdn, &shard->ecf);
 	if (edata == NULL) {
 		*oom = true;
 		return NULL;
 	}
 
-	void *addr = hpdata_reserve_alloc_offset(ps, size, alloc_offset);
+	hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
+	if (ps == NULL) {
+		edata_cache_fast_put(tsdn, &shard->ecf, edata);
+		return NULL;
+	}
+
+	psset_update_begin(&shard->psset, ps);
+
+	if (hpdata_empty(ps)) {
+		/*
+		 * If the pageslab used to be empty, treat it as though it's
+		 * brand new for fragmentation-avoidance purposes; what we're
+		 * trying to approximate is the age of the allocations *in* that
+		 * pageslab, and the allocations in the new pageslab are by
+		 * definition the youngest in this hpa shard.
+		 */
+		hpdata_age_set(ps, shard->age_counter++);
+	}
+
+	void *addr = hpdata_reserve_alloc(ps, size);
 	JE_USDT(hpa_alloc, 5, shard->ind, addr, size, hpdata_nactive_get(ps),
 	    hpdata_age_get(ps));
 	edata_init(edata, shard->ind, addr, size, /* slab */ false, SC_NSIZES,
@@ -677,12 +693,12 @@ hpa_try_alloc_one_offset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
 	/*
 	 * This could theoretically be moved outside of the critical section,
 	 * but that introduces the potential for a race.  Without the lock, the
-     * (initially nonempty, since this is the reuse pathway) pageslab we
+	 * (initially nonempty, since this is the reuse pathway) pageslab we
 	 * allocated out of could become otherwise empty while the lock is
 	 * dropped.  This would force us to deal with a pageslab eviction down
 	 * the error pathway, which is a pain.
 	 */
-	const bool err = emap_register_boundary(
+	err = emap_register_boundary(
 	    tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false);
 	if (err) {
 		hpdata_unreserve(
@@ -699,118 +715,32 @@ hpa_try_alloc_one_offset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
 		 * principle that we didn't *really* affect shard state (we
 		 * tweaked the stats, but our tweaks weren't really accurate).
 		 */
+		psset_update_end(&shard->psset, ps);
 		edata_cache_fast_put(tsdn, &shard->ecf, edata);
 		*oom = true;
 		return NULL;
 	}
 
+	hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
+	psset_update_end(&shard->psset, ps);
 	return edata;
 }
 
 static size_t
-hpa_try_alloc_from_one_ps(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
-    size_t max_nallocs, bool *oom, edata_list_active_t *results,
+hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
+    size_t size, bool *oom, size_t nallocs, edata_list_active_t *results,
     bool *deferred_work_generated) {
-	assert(size <= HUGEPAGE);
-	assert(size <= shard->opts.slab_max_alloc || size == sz_s2u(size));
-	assert(*oom == false);
 	malloc_mutex_assert_owner(tsdn, &shard->mtx);
-
-	nstime_t start;
-	nstime_init_update(&start);
-
-	hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
-	if (ps == NULL) {
-		return 0;
-	}
-
-	assert(max_nallocs <= SEC_MAX_NALLOCS);
-	hpdata_alloc_offset_t alloc_offsets[SEC_MAX_NALLOCS];
-	const size_t          nallocs = hpdata_find_alloc_offsets(
-            ps, size, alloc_offsets, max_nallocs);
-
-	psset_update_begin(&shard->psset, ps);
-
-	if (hpdata_empty(ps)) {
-		/*
-         * If the pageslab used to be empty, treat it as though it's
-		 * brand new for fragmentation-avoidance purposes; what we're
-		 * trying to approximate is the age of the allocations *in* that
-		 * pageslab, and the allocations in the new pageslab are by
-		 * definition the youngest in this hpa shard.
-		 */
-		hpdata_age_set(ps, shard->age_counter++);
-	}
-
 	size_t nsuccess = 0;
-	for (; nsuccess < nallocs; nsuccess += 1) {
-		edata_t *edata = hpa_try_alloc_one_offset(
-		    tsdn, shard, size, ps, (alloc_offsets + nsuccess), oom);
+	for (; nsuccess < nallocs; nsuccess++) {
+		edata_t *edata = hpa_try_alloc_one_no_grow(
+		    tsdn, shard, size, oom);
 		if (edata == NULL) {
 			break;
 		}
-
 		edata_list_active_append(results, edata);
 	}
 
-	hpdata_post_reserve_alloc_offsets(ps, size, alloc_offsets, nsuccess);
-	hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
-	psset_update_end(&shard->psset, ps);
-
-	const uint64_t elapsed_ns = nstime_ns_since(&start);
-	assert(nsuccess <= SEC_MAX_NALLOCS);
-	shard->stats.hpa_alloc_pages_per_ps[nsuccess] += nsuccess
-	    * (size >> LG_PAGE);
-	shard->stats.hpa_alloc_extents_per_ps[nsuccess] += 1;
-	shard->stats.hpa_alloc_total_elapsed_ns_per_ps[nsuccess] += elapsed_ns;
-
-	return nsuccess;
-}
-
-static size_t
-hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
-    size_t size, size_t min_nallocs, size_t max_nallocs,
-    bool update_min_max_stats, bool *oom, edata_list_active_t *results,
-    bool *deferred_work_generated) {
-	assert(*oom == false);
-	malloc_mutex_assert_owner(tsdn, &shard->mtx);
-
-	/*
-	 * As we require the shard mtx lock to update the stats,
-	 * we do the update the first time this function is called from
-	 * hpa_alloc_batch_psset().
-	 */
-	if (update_min_max_stats) {
-		assert(min_nallocs <= SEC_MAX_NALLOCS);
-		shard->stats.hpa_alloc_min_extents[min_nallocs] += 1;
-		assert(max_nallocs <= SEC_MAX_NALLOCS);
-		shard->stats.hpa_alloc_max_extents[max_nallocs] += 1;
-	}
-
-	size_t nsuccess = 0;
-	size_t ps_count = 0;
-	while (true) {
-		assert(1 <= min_nallocs);
-		assert(nsuccess < min_nallocs);
-		assert(min_nallocs <= max_nallocs);
-		const size_t nallocs = hpa_try_alloc_from_one_ps(tsdn, shard,
-		    size, max_nallocs - nsuccess, oom, results,
-		    deferred_work_generated);
-		if (nallocs == 0 || *oom) {
-			break;
-		}
-		nsuccess += nallocs;
-		ps_count += 1;
-		if (min_nallocs <= nsuccess) {
-			break;
-		}
-	}
-
-	assert(nsuccess <= SEC_MAX_NALLOCS);
-	shard->stats.hpa_alloc_extents[nsuccess] += 1;
-	assert(ps_count <= SEC_MAX_NALLOCS);
-	shard->stats.hpa_alloc_ps[ps_count] += 1;
-
 	hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
 	*deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard);
 	return nsuccess;
@@ -818,26 +748,27 @@ hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
 
 static size_t
 hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
-    size_t min_nallocs, size_t max_nallocs, bool update_min_max_stats,
-    bool *oom, edata_list_active_t *results, bool *deferred_work_generated) {
+    bool *oom, size_t nallocs, edata_list_active_t *results,
+    bool *deferred_work_generated) {
 	malloc_mutex_lock(tsdn, &shard->mtx);
-	const size_t nsuccess = hpa_try_alloc_batch_no_grow_locked(tsdn, shard,
-	    size, min_nallocs, max_nallocs, update_min_max_stats, oom, results,
-	    deferred_work_generated);
+	size_t nsuccess = hpa_try_alloc_batch_no_grow_locked(
+	    tsdn, shard, size, oom, nallocs, results, deferred_work_generated);
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 	return nsuccess;
 }
 
 static size_t
 hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
-    size_t min_nallocs, size_t max_nallocs, edata_list_active_t *results,
+    size_t nallocs, edata_list_active_t *results,
     bool *deferred_work_generated) {
+	assert(size <= HUGEPAGE);
+	assert(size <= shard->opts.slab_max_alloc || size == sz_s2u(size));
 	bool oom = false;
 
-	size_t nsuccess = hpa_try_alloc_batch_no_grow(tsdn, shard, size,
-	    min_nallocs, max_nallocs, /* update_min_max_stats */ true, &oom,
-	    results, deferred_work_generated);
-	if (min_nallocs <= nsuccess || oom) {
+	size_t nsuccess = hpa_try_alloc_batch_no_grow(
+	    tsdn, shard, size, &oom, nallocs, results, deferred_work_generated);
+
+	if (nsuccess == nallocs || oom) {
 		return nsuccess;
 	}
 
@@ -846,18 +777,13 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
 	 * try to grow.
 	 */
 	malloc_mutex_lock(tsdn, &shard->grow_mtx);
-
 	/*
 	 * Check for grow races; maybe some earlier thread expanded the psset
 	 * in between when we dropped the main mutex and grabbed the grow mutex.
 	 */
-	assert(nsuccess < min_nallocs);
-	assert(min_nallocs <= max_nallocs);
-	nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size,
-	    min_nallocs - nsuccess, max_nallocs - nsuccess,
-	    /* update_min_max_stats */ false, &oom, results,
-	    deferred_work_generated);
-	if (min_nallocs <= nsuccess || oom) {
+	nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
+	    nallocs - nsuccess, results, deferred_work_generated);
+	if (nsuccess == nallocs || oom) {
 		malloc_mutex_unlock(tsdn, &shard->grow_mtx);
 		return nsuccess;
 	}
@@ -881,19 +807,28 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
 	 */
 	malloc_mutex_lock(tsdn, &shard->mtx);
 	psset_insert(&shard->psset, ps);
-	assert(nsuccess < min_nallocs);
-	assert(min_nallocs <= max_nallocs);
-	nsuccess += hpa_try_alloc_batch_no_grow_locked(tsdn, shard, size,
-	    min_nallocs - nsuccess, max_nallocs - nsuccess,
-	    /* update_min_max_stats */ false, &oom, results,
-	    deferred_work_generated);
+	nsuccess += hpa_try_alloc_batch_no_grow_locked(tsdn, shard, size, &oom,
+	    nallocs - nsuccess, results, deferred_work_generated);
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 
+	/*
+	 * Drop grow_mtx before doing deferred work; other threads blocked on it
+	 * should be allowed to proceed while we're working.
+	 */
 	malloc_mutex_unlock(tsdn, &shard->grow_mtx);
 
 	return nsuccess;
 }
 
+static hpa_shard_t *
+hpa_from_pai(pai_t *self) {
+	assert(self->alloc == &hpa_alloc);
+	assert(self->expand == &hpa_expand);
+	assert(self->shrink == &hpa_shrink);
+	assert(self->dalloc == &hpa_dalloc);
+	return (hpa_shard_t *)self;
+}
+
 static void
 hpa_assert_results(
     tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *results) {
@@ -919,10 +854,9 @@ hpa_assert_results(
 	}
 }
 
-edata_t *
-hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
-    bool zero, bool guarded, bool frequent_reuse,
-    bool *deferred_work_generated) {
+static edata_t *
+hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
+    bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
 	assert((size & PAGE_MASK) == 0);
 	assert(!guarded);
 	witness_assert_depth_to_rank(
@@ -932,6 +866,7 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	if (alignment > PAGE || zero) {
 		return NULL;
 	}
+	hpa_shard_t *shard = hpa_from_pai(self);
 
 	/*
 	 * frequent_use here indicates this request comes from the arena bins,
@@ -951,13 +886,13 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	if (edata != NULL) {
 		return edata;
 	}
+	size_t              nallocs = sec_size_supported(&shard->sec, size)
+	                 ? shard->sec.opts.batch_fill_extra + 1
+	                 : 1;
 	edata_list_active_t results;
 	edata_list_active_init(&results);
-	size_t min_nallocs, max_nallocs;
-	sec_calc_nallocs_for_size(
-	    &shard->sec, size, &min_nallocs, &max_nallocs);
-	size_t nsuccess = hpa_alloc_batch_psset(tsdn, shard, size, min_nallocs,
-	    max_nallocs, &results, deferred_work_generated);
+	size_t nsuccess = hpa_alloc_batch_psset(
+	    tsdn, shard, size, nallocs, &results, deferred_work_generated);
 	hpa_assert_results(tsdn, shard, &results);
 	edata = edata_list_active_first(&results);
 
@@ -972,7 +907,7 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 		/* Unlikely rollback in case of overfill */
 		if (!edata_list_active_empty(&results)) {
 			hpa_dalloc_batch(
-			    tsdn, shard, &results, deferred_work_generated);
+			    tsdn, self, &results, deferred_work_generated);
 		}
 	}
 	witness_assert_depth_to_rank(
@@ -980,15 +915,15 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	return edata;
 }
 
-bool
-hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
+static bool
+hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
     size_t new_size, bool zero, bool *deferred_work_generated) {
 	/* Expand not yet supported. */
 	return true;
 }
 
-bool
-hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size,
+static bool
+hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
     size_t new_size, bool *deferred_work_generated) {
 	/* Shrink not yet supported. */
 	return true;
@@ -1047,8 +982,10 @@ hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) {
 }
 
 static void
-hpa_dalloc_batch(tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *list,
+hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
     bool *deferred_work_generated) {
+	hpa_shard_t *shard = hpa_from_pai(self);
+
 	edata_t *edata;
 	ql_foreach (edata, &list->head, ql_link_active) {
 		hpa_dalloc_prepare_unlocked(tsdn, shard, edata);
@@ -1066,15 +1003,16 @@ hpa_dalloc_batch(tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *list,
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 }
 
-void
-hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
-    bool *deferred_work_generated) {
+static void
+hpa_dalloc(
+    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
 	assert(!edata_guarded_get(edata));
 
 	edata_list_active_t dalloc_list;
 	edata_list_active_init(&dalloc_list);
 	edata_list_active_append(&dalloc_list, edata);
 
+	hpa_shard_t *shard = hpa_from_pai(self);
 	sec_dalloc(tsdn, &shard->sec, &dalloc_list);
 	if (edata_list_active_empty(&dalloc_list)) {
 		/* sec consumed the pointer */
@@ -1082,16 +1020,17 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
 		return;
 	}
 	/* We may have more than one pointer to flush now */
-	hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated);
+	hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
 }
 
 /*
  * Calculate time until either purging or hugification ought to happen.
  * Called by background threads.
  */
-uint64_t
-hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
-	uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
+static uint64_t
+hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
+	hpa_shard_t *shard = hpa_from_pai(self);
+	uint64_t     time_ns = BACKGROUND_THREAD_DEFERRED_MAX;
 
 	malloc_mutex_lock(tsdn, &shard->mtx);
 
@@ -1151,7 +1090,8 @@ hpa_sec_flush_impl(tsdn_t *tsdn, hpa_shard_t *shard) {
 
 	sec_flush(tsdn, &shard->sec, &to_flush);
 	bool deferred_work_generated;
-	hpa_dalloc_batch(tsdn, shard, &to_flush, &deferred_work_generated);
+	hpa_dalloc_batch(
+	    tsdn, (pai_t *)shard, &to_flush, &deferred_work_generated);
 }
 
 void

src/hpdata.c

@@ -170,171 +170,6 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
 	hpdata_assert_consistent(hpdata);
 }
 
-size_t
-hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
-    hpdata_alloc_offset_t *offsets, size_t max_nallocs) {
-	hpdata_assert_consistent(hpdata);
-	assert((sz & PAGE_MASK) == 0);
-	assert(1 <= max_nallocs);
-	const size_t npages = sz >> LG_PAGE;
-	/* We should be able to find at least one allocation */
-	assert(npages <= hpdata_longest_free_range_get(hpdata));
-
-	size_t nallocs = 0;
-	size_t start = 0;
-	size_t longest_len = 0;
-	while (true) {
-		size_t begin = 0;
-		size_t len = 0;
-
-		const bool found = fb_urange_iter(
-		    hpdata->active_pages, HUGEPAGE_PAGES, start, &begin, &len);
-		if (!found) {
-			/* we should have found at least one */
-			assert(0 < nallocs);
-			break;
-		}
-
-		/* carve up the free range, if it's large enough */
-		while (npages <= len) {
-			offsets->len_before = len;
-			offsets->index = begin;
-			offsets->longest_len = longest_len;
-			offsets += 1;
-
-			nallocs += 1;
-			if (nallocs == max_nallocs) {
-				/* cause start to be == HUGEPAGE_PAGES to break out of the outer loop */
-				begin = HUGEPAGE_PAGES;
-				len = 0;
-				break;
-			}
-
-			begin += npages;
-			len -= npages;
-		}
-
-		start = begin + len;
-		assert(start <= HUGEPAGE_PAGES);
-		if (start == HUGEPAGE_PAGES) {
-			break;
-		}
-		longest_len = max_zu(longest_len, len);
-	}
-
-	/* post-conditions */
-	assert(1 <= nallocs);
-	assert(nallocs <= max_nallocs);
-
-	return nallocs;
-}
-
-void *
-hpdata_reserve_alloc_offset(
-    hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset) {
-	/*
-	 * This is a metadata change; the hpdata should therefore either not be
-	 * in the psset, or should have explicitly marked itself as being
-	 * mid-update.
-	 */
-	assert(!hpdata->h_in_psset || hpdata->h_updating);
-	assert(hpdata->h_alloc_allowed);
-	assert((sz & PAGE_MASK) == 0);
-	const size_t npages = sz >> LG_PAGE;
-	const size_t index = offset->index;
-
-	fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, index, npages);
-	hpdata->h_nactive += npages;
-
-	/*
-	 * We might be about to dirty some memory for the first time; update our
-	 * count if so.
-	 */
-	size_t new_dirty = fb_ucount(
-	    hpdata->touched_pages, HUGEPAGE_PAGES, index, npages);
-	fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, index, npages);
-	hpdata->h_ntouched += new_dirty;
-
-	return (void *)((byte_t *)hpdata_addr_get(hpdata) + (index << LG_PAGE));
-}
-
-void
-hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
-    hpdata_alloc_offset_t *offsets, size_t nallocs) {
-	assert((sz & PAGE_MASK) == 0);
-	const size_t npages = sz >> LG_PAGE;
-
-	if (nallocs == 0) {
-		return;
-	}
-
-	size_t max_len = offsets[0].len_before;
-	for (size_t i = 1; i < nallocs; i += 1) {
-		max_len = max_zu(max_len, offsets[i].len_before);
-	}
-
-	const size_t prev_longest = hpdata_longest_free_range_get(hpdata);
-	assert(max_len <= prev_longest);
-	if (max_len < prev_longest) {
-		/* no need to update the hpdata longest range */
-		return;
-	}
-
-	/*
-	 * If we allocated out of a range that was the longest in the hpdata, it
-	 * might be the only one of that size and we'll have to adjust the
-	 * metadata.
-	 */
-	const size_t len_before = offsets[nallocs - 1].len_before;
-	size_t       start = offsets[nallocs - 1].index + len_before;
-	size_t       longest_len = max_zu(
-            offsets[nallocs - 1].longest_len, len_before - npages);
-
-	const size_t rest = HUGEPAGE_PAGES - start;
-	/*
-	 * Only look at the rest if we think we'll find a range longer than what
-	 * we already have. This also implicitly checks for rest == 0, so we don't
-	 * have to check before the first call to fb_urange_iter().
-	 */
-	if (longest_len < rest) {
-		while (true) {
-			size_t begin = 0;
-			size_t len = 0;
-
-			const bool found = fb_urange_iter(hpdata->active_pages,
-			    HUGEPAGE_PAGES, start, &begin, &len);
-			if (!found) {
-				break;
-			}
-
-			if (longest_len < len) {
-				if (len == prev_longest) {
-					/* it's already set to the right value */
-					assert(hpdata_longest_free_range_get(
-					           hpdata)
-					    == len);
-					assert(fb_urange_longest(
-					           hpdata->active_pages,
-					           HUGEPAGE_PAGES)
-					    == len);
-					return;
-				}
-				longest_len = len;
-			}
-
-			start = begin + len;
-			assert(start <= HUGEPAGE_PAGES);
-			if (start == HUGEPAGE_PAGES) {
-				break;
-			}
-		}
-	}
-
-	assert(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES)
-	    == longest_len);
-	hpdata_longest_free_range_set(hpdata, longest_len);
-}
-
 size_t
 hpdata_purge_begin(
     hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t *nranges) {

src/inspect.c

@@ -53,7 +53,8 @@ inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
 	assert(*nfree <= *nregs);
 	assert(*nfree * edata_usize_get(edata) <= *size);
 
-	arena_t *arena = arena_get_from_edata(edata);
+	arena_t *arena = (arena_t *)atomic_load_p(
+	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
 	assert(arena != NULL);
 	const unsigned binshard = edata_binshard_get(edata);
 	bin_t         *bin = arena_get_bin(arena, szind, binshard);

src/jemalloc_cpp.cpp

@@ -1,4 +1,4 @@
-#include <exception>
+#include <mutex>
 #include <new>
 // NOLINTBEGIN(misc-use-anonymous-namespace)
 
@@ -78,30 +78,29 @@ handleOOM(std::size_t size, bool nothrow) {
 	void *ptr = nullptr;
 
 	while (ptr == nullptr) {
-		std::new_handler handler = std::get_new_handler();
+		std::new_handler handler;
+		// GCC-4.8 and clang 4.0 do not have std::get_new_handler.
+		{
+			static std::mutex           mtx;
+			std::lock_guard<std::mutex> lock(mtx);
+
+			handler = std::set_new_handler(nullptr);
+			std::set_new_handler(handler);
+		}
 		if (handler == nullptr)
 			break;
 
-#ifdef JEMALLOC_HAVE_CXX_EXCEPTIONS
 		try {
 			handler();
 		} catch (const std::bad_alloc &) {
 			break;
 		}
-#else
-		handler();
-#endif
 
 		ptr = je_malloc(size);
 	}
 
-	if (ptr == nullptr && !nothrow) {
-#ifdef JEMALLOC_HAVE_CXX_EXCEPTIONS
-		throw std::bad_alloc();
-#else
-		std::terminate();
-#endif
-	}
+	if (ptr == nullptr && !nothrow)
+		std::__throw_bad_alloc();
 	return ptr;
 }
 

src/jemalloc_fork.c

@@ -1,182 +0,0 @@
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
-
-#include "jemalloc/internal/arenas_management.h"
-#include "jemalloc/internal/ctl.h"
-#include "jemalloc/internal/jemalloc_fork.h"
-#include "jemalloc/internal/jemalloc_init.h"
-
-/******************************************************************************/
-/*
- * The following functions are used by threading libraries for protection of
- * malloc during fork().
- */
-
-#ifdef JEMALLOC_MUTEX_INIT_CB
-/*
- * When JEMALLOC_MUTEX_INIT_CB is defined, pthread_atfork registration is
- * skipped and the platform calls _malloc_prefork/_malloc_postfork directly.
- * FreeBSD's libthr calls _malloc_postfork in both parent and child.  Detect
- * the child by pid change so we route to jemalloc_postfork_child, which resets
- * per-arena state the parent handler does not touch (nthreads, descriptor
- * queues).  The check is harmless on any platform that only calls
- * _malloc_postfork in the parent.
- */
-static pid_t jemalloc_prefork_pid;
-#endif
-
-#ifndef JEMALLOC_MUTEX_INIT_CB
-void
-jemalloc_prefork(void)
-#else
-JEMALLOC_EXPORT void
-_malloc_prefork(void)
-#endif
-{
-	tsd_t   *tsd;
-	unsigned i, j, narenas;
-	arena_t *arena;
-
-#ifdef JEMALLOC_MUTEX_INIT_CB
-	if (!malloc_initialized()) {
-		return;
-	}
-#endif
-	assert(malloc_initialized());
-
-#ifdef JEMALLOC_MUTEX_INIT_CB
-	jemalloc_prefork_pid = getpid();
-#endif
-
-	tsd = tsd_fetch();
-
-	narenas = narenas_total_get();
-
-	witness_prefork(tsd_witness_tsdp_get(tsd));
-	/* Acquire all mutexes in a safe order. */
-	ctl_prefork(tsd_tsdn(tsd));
-	tcache_prefork(tsd_tsdn(tsd));
-	arenas_management_prefork(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_prefork0(tsd_tsdn(tsd));
-	}
-	prof_prefork0(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_prefork1(tsd_tsdn(tsd));
-	}
-	/* Break arena prefork into stages to preserve lock order. */
-	for (i = 0; i < 9; i++) {
-		for (j = 0; j < narenas; j++) {
-			if ((arena = arena_get(tsd_tsdn(tsd), j, false))
-			    != NULL) {
-				switch (i) {
-				case 0:
-					arena_prefork0(tsd_tsdn(tsd), arena);
-					break;
-				case 1:
-					arena_prefork1(tsd_tsdn(tsd), arena);
-					break;
-				case 2:
-					arena_prefork2(tsd_tsdn(tsd), arena);
-					break;
-				case 3:
-					arena_prefork3(tsd_tsdn(tsd), arena);
-					break;
-				case 4:
-					arena_prefork4(tsd_tsdn(tsd), arena);
-					break;
-				case 5:
-					arena_prefork5(tsd_tsdn(tsd), arena);
-					break;
-				case 6:
-					arena_prefork6(tsd_tsdn(tsd), arena);
-					break;
-				case 7:
-					arena_prefork7(tsd_tsdn(tsd), arena);
-					break;
-				case 8:
-					arena_prefork8(tsd_tsdn(tsd), arena);
-					break;
-				default:
-					not_reached();
-				}
-			}
-		}
-	}
-	prof_prefork1(tsd_tsdn(tsd));
-	stats_prefork(tsd_tsdn(tsd));
-}
-
-#ifndef JEMALLOC_MUTEX_INIT_CB
-void
-jemalloc_postfork_parent(void)
-#else
-JEMALLOC_EXPORT void
-_malloc_postfork(void)
-#endif
-{
-	tsd_t   *tsd;
-	unsigned i, narenas;
-
-#ifdef JEMALLOC_MUTEX_INIT_CB
-	if (!malloc_initialized()) {
-		return;
-	}
-	if (getpid() != jemalloc_prefork_pid) {
-		jemalloc_postfork_child();
-		return;
-	}
-#endif
-	assert(malloc_initialized());
-
-	tsd = tsd_fetch();
-
-	witness_postfork_parent(tsd_witness_tsdp_get(tsd));
-	/* Release all mutexes, now that fork() has completed. */
-	stats_postfork_parent(tsd_tsdn(tsd));
-	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
-		arena_t *arena;
-
-		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
-			arena_postfork_parent(tsd_tsdn(tsd), arena);
-		}
-	}
-	prof_postfork_parent(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_postfork_parent(tsd_tsdn(tsd));
-	}
-	arenas_management_postfork_parent(tsd_tsdn(tsd));
-	tcache_postfork_parent(tsd_tsdn(tsd));
-	ctl_postfork_parent(tsd_tsdn(tsd));
-}
-
-void
-jemalloc_postfork_child(void) {
-	tsd_t   *tsd;
-	unsigned i, narenas;
-
-	assert(malloc_initialized());
-
-	tsd = tsd_fetch();
-
-	witness_postfork_child(tsd_witness_tsdp_get(tsd));
-	/* Release all mutexes, now that fork() has completed. */
-	stats_postfork_child(tsd_tsdn(tsd));
-	for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
-		arena_t *arena;
-
-		if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) {
-			cache_bin_array_descriptor_t *desc =
-			    tcache_postfork_arena_descriptor(
-			        tsd_tsdn(tsd), arena);
-			arena_postfork_child(tsd_tsdn(tsd), arena, desc);
-		}
-	}
-	prof_postfork_child(tsd_tsdn(tsd));
-	if (have_background_thread) {
-		background_thread_postfork_child(tsd_tsdn(tsd));
-	}
-	arenas_management_postfork_child(tsd_tsdn(tsd));
-	tcache_postfork_child(tsd_tsdn(tsd));
-	ctl_postfork_child(tsd_tsdn(tsd));
-}

src/jemalloc_init.c

@@ -1,700 +0,0 @@
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
-
-#include "jemalloc/internal/arenas_management.h"
-#include "jemalloc/internal/conf.h"
-#include "jemalloc/internal/ctl.h"
-#include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/extent_dss.h"
-#include "jemalloc/internal/extent_mmap.h"
-#include "jemalloc/internal/jemalloc_fork.h"
-#include "jemalloc/internal/jemalloc_init.h"
-#include "jemalloc/internal/malloc_io.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/safety_check.h"
-#include "jemalloc/internal/san.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/spin.h"
-#include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/thread_event.h"
-
-#ifdef JEMALLOC_THREADED_INIT
-/* Used to let the initializing thread recursively allocate. */
-#	define NO_INITIALIZER ((unsigned long)0)
-#	define INITIALIZER pthread_self()
-static pthread_t malloc_initializer = NO_INITIALIZER;
-#else
-#	define NO_INITIALIZER false
-#	define INITIALIZER true
-static bool malloc_initializer = NO_INITIALIZER;
-#endif
-
-bool
-malloc_is_initializer(void) {
-#ifdef JEMALLOC_THREADED_INIT
-	return pthread_equal(malloc_initializer, pthread_self());
-#else
-	return malloc_initializer;
-#endif
-}
-
-#ifdef JEMALLOC_THREADED_INIT
-static bool
-malloc_initializer_is_set(void) {
-	return malloc_initializer != NO_INITIALIZER;
-}
-#endif
-
-static void
-malloc_initializer_set(void) {
-	malloc_initializer = INITIALIZER;
-}
-
-/* Used to avoid initialization races. */
-#ifdef _WIN32
-#	if _WIN32_WINNT >= 0x0600
-static malloc_mutex_t init_lock = SRWLOCK_INIT;
-#	else
-static malloc_mutex_t init_lock;
-static bool           init_lock_initialized = false;
-
-JEMALLOC_ATTR(constructor)
-static void WINAPI
-_init_init_lock(void) {
-	/*
-	 * If another constructor in the same binary is using mallctl to e.g.
-	 * set up extent hooks, it may end up running before this one, and
-	 * malloc_init_hard will crash trying to lock the uninitialized lock. So
-	 * we force an initialization of the lock in malloc_init_hard as well.
-	 * We don't try to care about atomicity of the accessed to the
-	 * init_lock_initialized boolean, since it really only matters early in
-	 * the process creation, before any separate thread normally starts
-	 * doing anything.
-	 */
-	if (!init_lock_initialized) {
-		malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT,
-		    malloc_mutex_rank_exclusive);
-	}
-	init_lock_initialized = true;
-}
-
-#		ifdef _MSC_VER
-#			pragma section(".CRT$XCU", read)
-JEMALLOC_SECTION(".CRT$XCU")
-JEMALLOC_ATTR(used)
-static const void(WINAPI *init_init_lock)(void) = _init_init_lock;
-#		endif
-#	endif
-#else
-static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
-#endif
-
-malloc_init_t malloc_init_state = malloc_init_uninitialized;
-
-/* When malloc_slow is true, set the corresponding bits for sanity check. */
-enum {
-	flag_opt_junk_alloc = (1U),
-	flag_opt_junk_free = (1U << 1),
-	flag_opt_zero = (1U << 2),
-	flag_opt_utrace = (1U << 3),
-	flag_opt_xmalloc = (1U << 4)
-};
-static uint8_t malloc_slow_flags;
-
-static void
-malloc_slow_flag_init(void) {
-	/*
-	 * Combine the runtime options into malloc_slow for fast path.  Called
-	 * after bootstrap is complete.
-	 */
-	malloc_slow_flags |= (opt_junk_alloc ? flag_opt_junk_alloc : 0)
-	    | (opt_junk_free ? flag_opt_junk_free : 0)
-	    | (opt_zero ? flag_opt_zero : 0)
-	    | (opt_utrace ? flag_opt_utrace : 0)
-	    | (opt_xmalloc ? flag_opt_xmalloc : 0);
-
-	malloc_slow = (malloc_slow_flags != 0);
-}
-
-static void stats_print_atexit(void);
-static unsigned malloc_ncpus(void);
-static bool malloc_cpu_count_is_deterministic(void);
-
-static bool
-malloc_init_hard_needed(void) {
-	if (malloc_initialized()
-	    || (malloc_is_initializer()
-	        && malloc_init_state == malloc_init_recursible)) {
-		/*
-		 * Another thread initialized the allocator before this one
-		 * acquired init_lock, or this thread is the initializing
-		 * thread, and it is recursively allocating.
-		 */
-		return false;
-	}
-#ifdef JEMALLOC_THREADED_INIT
-	if (malloc_initializer_is_set() && !malloc_is_initializer()) {
-		/* Busy-wait until the initializing thread completes. */
-		spin_t spinner = SPIN_INITIALIZER;
-		do {
-			malloc_mutex_unlock(TSDN_NULL, &init_lock);
-			spin_adaptive(&spinner);
-			malloc_mutex_lock(TSDN_NULL, &init_lock);
-		} while (!malloc_initialized());
-		return false;
-	}
-#endif
-	return true;
-}
-
-static bool
-malloc_init_hard_a0_locked(void) {
-	malloc_initializer_set();
-
-	JEMALLOC_DIAGNOSTIC_PUSH
-	JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS
-	sc_data_t sc_data = {0};
-	JEMALLOC_DIAGNOSTIC_POP
-
-	/*
-	 * Ordering here is somewhat tricky; we need sc_boot() first, since that
-	 * determines what the size classes will be, and then
-	 * malloc_conf_init(), since any slab size tweaking will need to be done
-	 * before sz_boot and bin_info_boot, which assume that the values they
-	 * read out of sc_data_global are final.
-	 */
-	sc_boot(&sc_data);
-	unsigned bin_shard_sizes[SC_NBINS];
-	bin_shard_sizes_boot(bin_shard_sizes);
-	/*
-	 * prof_boot0 only initializes opt_prof_prefix.  We need to do it before
-	 * we parse malloc_conf options, in case malloc_conf parsing overwrites
-	 * it.
-	 */
-	if (config_prof) {
-		prof_boot0();
-	}
-	char readlink_buf[PATH_MAX + 1];
-	readlink_buf[0] = '\0';
-	malloc_conf_init(&sc_data, bin_shard_sizes, readlink_buf);
-	san_init(opt_lg_san_uaf_align);
-	sz_boot(&sc_data, opt_cache_oblivious);
-	bin_info_boot(&sc_data, bin_shard_sizes);
-
-	if (opt_stats_print) {
-		/* Print statistics at exit. */
-		if (atexit(stats_print_atexit) != 0) {
-			malloc_write("<jemalloc>: Error in atexit()\n");
-			if (opt_abort) {
-				abort();
-			}
-		}
-	}
-
-	if (stats_boot()) {
-		return true;
-	}
-	if (pages_boot()) {
-		return true;
-	}
-	if (base_boot(TSDN_NULL)) {
-		return true;
-	}
-	/* emap_global is static, hence zeroed. */
-	if (emap_init(&arena_emap_global, b0get(), /* zeroed */ true)) {
-		return true;
-	}
-	if (extent_boot()) {
-		return true;
-	}
-	if (ctl_boot()) {
-		return true;
-	}
-	if (config_prof) {
-		prof_boot1();
-	}
-	if (opt_hpa && !hpa_supported()) {
-		malloc_printf(
-		    "<jemalloc>: HPA not supported in the current "
-		    "configuration; %s.",
-		    opt_abort_conf ? "aborting" : "disabling");
-		if (opt_abort_conf) {
-			malloc_abort_invalid_conf();
-		} else {
-			opt_hpa = false;
-		}
-	}
-	if (arena_boot(&sc_data, b0get(), opt_hpa)) {
-		return true;
-	}
-	if (tcache_boot(TSDN_NULL, b0get())) {
-		return true;
-	}
-	if (arenas_management_boot()) {
-		return true;
-	}
-	experimental_thread_events_boot();
-	/*
-	 * Create enough scaffolding to allow recursive allocation in
-	 * malloc_ncpus().
-	 */
-	narenas_auto_set(1);
-	manual_arena_base_set(narenas_auto + 1);
-	memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
-	/*
-	 * Initialize one arena here.  The rest are lazily created in
-	 * arena_choose_hard().
-	 */
-	if (arena_init(TSDN_NULL, 0, &arena_config_default) == NULL) {
-		return true;
-	}
-
-	if (opt_hpa && !hpa_supported()) {
-		malloc_printf(
-		    "<jemalloc>: HPA not supported in the current "
-		    "configuration; %s.",
-		    opt_abort_conf ? "aborting" : "disabling");
-		if (opt_abort_conf) {
-			malloc_abort_invalid_conf();
-		} else {
-			opt_hpa = false;
-		}
-	}
-
-	malloc_init_state = malloc_init_a0_initialized;
-
-	size_t buf_len = strlen(readlink_buf);
-	if (buf_len > 0) {
-		void *readlink_allocated = a0malloc(buf_len + 1);
-		if (readlink_allocated != NULL) {
-			memcpy(readlink_allocated, readlink_buf, buf_len + 1);
-			opt_malloc_conf_symlink = readlink_allocated;
-		}
-	}
-
-	return false;
-}
-
-bool
-malloc_init_hard_a0(void) {
-	bool ret;
-
-	malloc_mutex_lock(TSDN_NULL, &init_lock);
-	ret = malloc_init_hard_a0_locked();
-	malloc_mutex_unlock(TSDN_NULL, &init_lock);
-	return ret;
-}
-
-static void
-stats_print_atexit(void) {
-	if (config_stats) {
-		tsdn_t  *tsdn;
-		unsigned narenas, i;
-
-		tsdn = tsdn_fetch();
-
-		/*
-		 * Merge stats from extant threads.  This is racy, since
-		 * individual threads do not lock when recording tcache stats
-		 * events.  As a consequence, the final stats may be slightly
-		 * out of date by the time they are reported, if other threads
-		 * continue to allocate.
-		 */
-		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
-			arena_t *arena = arena_get(tsdn, i, false);
-			if (arena != NULL) {
-				arena_cache_bins_stats_merge(tsdn, arena);
-			}
-		}
-	}
-	je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
-}
-
-static unsigned
-malloc_ncpus(void) {
-	long result;
-
-#ifdef _WIN32
-	SYSTEM_INFO si;
-	GetSystemInfo(&si);
-	result = si.dwNumberOfProcessors;
-#elif defined(CPU_COUNT)
-	/*
-	 * glibc >= 2.6 has the CPU_COUNT macro.
-	 *
-	 * glibc's sysconf() uses isspace().  glibc allocates for the first time
-	 * *before* setting up the isspace tables.  Therefore we need a
-	 * different method to get the number of CPUs.
-	 *
-	 * The getaffinity approach is also preferred when only a subset of CPUs
-	 * is available, to avoid using more arenas than necessary.
-	 */
-	{
-#	if defined(__FreeBSD__) || defined(__DragonFly__)
-		cpuset_t set;
-#	else
-		cpu_set_t set;
-#	endif
-#	if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
-		sched_getaffinity(0, sizeof(set), &set);
-#	else
-		pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
-#	endif
-		result = CPU_COUNT(&set);
-	}
-#else
-	result = sysconf(_SC_NPROCESSORS_ONLN);
-#endif
-	return ((result == -1) ? 1 : (unsigned)result);
-}
-
-/*
- * Ensure that number of CPUs is determistinc, i.e. it is the same based on:
- * - sched_getaffinity()
- * - _SC_NPROCESSORS_ONLN
- * - _SC_NPROCESSORS_CONF
- * Since otherwise tricky things is possible with percpu arenas in use.
- */
-static bool
-malloc_cpu_count_is_deterministic(void) {
-#ifdef _WIN32
-	return true;
-#else
-	long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
-	long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
-	if (cpu_onln != cpu_conf) {
-		return false;
-	}
-#	if defined(CPU_COUNT)
-#		if defined(__FreeBSD__) || defined(__DragonFly__)
-	cpuset_t set;
-#		else
-	cpu_set_t set;
-#		endif /* __FreeBSD__ */
-#		if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
-	sched_getaffinity(0, sizeof(set), &set);
-#		else  /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
-	pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
-#		endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
-	long cpu_affinity = CPU_COUNT(&set);
-	if (cpu_affinity != cpu_conf) {
-		return false;
-	}
-#	endif         /* CPU_COUNT */
-	return true;
-#endif
-}
-
-/* Initialize data structures which may trigger recursive allocation. */
-static bool
-malloc_init_hard_recursible(void) {
-	malloc_init_state = malloc_init_recursible;
-
-	ncpus = malloc_ncpus();
-	if (opt_percpu_arena != percpu_arena_disabled) {
-		bool cpu_count_is_deterministic =
-		    malloc_cpu_count_is_deterministic();
-		if (!cpu_count_is_deterministic) {
-			/*
-			 * If # of CPU is not deterministic, and narenas not
-			 * specified, disables per cpu arena since it may not
-			 * detect CPU IDs properly.
-			 */
-			if (opt_narenas == 0) {
-				opt_percpu_arena = percpu_arena_disabled;
-				malloc_write(
-				    "<jemalloc>: Number of CPUs "
-				    "detected is not deterministic. Per-CPU "
-				    "arena disabled.\n");
-				if (opt_abort_conf) {
-					malloc_abort_invalid_conf();
-				}
-				if (opt_abort) {
-					abort();
-				}
-			}
-		}
-	}
-
-#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
-    && !defined(JEMALLOC_ZONE) && !defined(_WIN32)                             \
-    && !defined(__native_client__))
-	/* LinuxThreads' pthread_atfork() allocates. */
-	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
-	        jemalloc_postfork_child)
-	    != 0) {
-		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
-		if (opt_abort) {
-			abort();
-		}
-		return true;
-	}
-#endif
-
-	if (background_thread_boot0()) {
-		return true;
-	}
-
-	return false;
-}
-
-static unsigned
-malloc_narenas_default(void) {
-	assert(ncpus > 0);
-	/*
-	 * For SMP systems, create more than one arena per CPU by
-	 * default.
-	 */
-	if (ncpus > 1) {
-		fxp_t    fxp_ncpus = FXP_INIT_INT(ncpus);
-		fxp_t    goal = fxp_mul(fxp_ncpus, opt_narenas_ratio);
-		uint32_t int_goal = fxp_round_nearest(goal);
-		if (int_goal == 0) {
-			return 1;
-		}
-		return int_goal;
-	} else {
-		return 1;
-	}
-}
-
-static percpu_arena_mode_t
-percpu_arena_as_initialized(percpu_arena_mode_t mode) {
-	assert(!malloc_initialized());
-	assert(mode <= percpu_arena_disabled);
-
-	if (mode != percpu_arena_disabled) {
-		mode += percpu_arena_mode_enabled_base;
-	}
-
-	return mode;
-}
-
-static bool
-malloc_init_narenas(tsdn_t *tsdn) {
-	assert(ncpus > 0);
-
-	if (opt_percpu_arena != percpu_arena_disabled) {
-		if (!have_percpu_arena || malloc_getcpu() < 0) {
-			opt_percpu_arena = percpu_arena_disabled;
-			malloc_printf(
-			    "<jemalloc>: perCPU arena getcpu() not "
-			    "available. Setting narenas to %u.\n",
-			    opt_narenas ? opt_narenas
-			                : malloc_narenas_default());
-			if (opt_abort) {
-				abort();
-			}
-		} else {
-			if (ncpus >= MALLOCX_ARENA_LIMIT) {
-				malloc_printf(
-				    "<jemalloc>: narenas w/ percpu"
-				    "arena beyond limit (%d)\n",
-				    ncpus);
-				if (opt_abort) {
-					abort();
-				}
-				return true;
-			}
-			/* NB: opt_percpu_arena isn't fully initialized yet. */
-			if (percpu_arena_as_initialized(opt_percpu_arena)
-			        == per_phycpu_arena
-			    && ncpus % 2 != 0) {
-				malloc_printf(
-				    "<jemalloc>: invalid "
-				    "configuration -- per physical CPU arena "
-				    "with odd number (%u) of CPUs (no hyper "
-				    "threading?).\n",
-				    ncpus);
-				if (opt_abort)
-					abort();
-			}
-			unsigned n = percpu_arena_ind_limit(
-			    percpu_arena_as_initialized(opt_percpu_arena));
-			if (opt_narenas < n) {
-				/*
-				 * If narenas is specified with percpu_arena
-				 * enabled, actual narenas is set as the greater
-				 * of the two. percpu_arena_choose will be free
-				 * to use any of the arenas based on CPU
-				 * id. This is conservative (at a small cost)
-				 * but ensures correctness.
-				 *
-				 * If for some reason the ncpus determined at
-				 * boot is not the actual number (e.g. because
-				 * of affinity setting from numactl), reserving
-				 * narenas this way provides a workaround for
-				 * percpu_arena.
-				 */
-				opt_narenas = n;
-			}
-		}
-	}
-	if (opt_narenas == 0) {
-		opt_narenas = malloc_narenas_default();
-	}
-	assert(opt_narenas > 0);
-
-	narenas_auto_set(opt_narenas);
-	/*
-	 * Limit the number of arenas to the indexing range of MALLOCX_ARENA().
-	 */
-	if (narenas_auto >= MALLOCX_ARENA_LIMIT) {
-		narenas_auto_set(MALLOCX_ARENA_LIMIT - 1);
-		malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
-		    narenas_auto);
-	}
-	narenas_total_set(narenas_auto);
-	if (arena_init_huge(tsdn, arena_get(tsdn, 0, false))) {
-		narenas_total_inc();
-	}
-	manual_arena_base_set(narenas_total_get());
-
-	return false;
-}
-
-static void
-malloc_init_percpu(void) {
-	opt_percpu_arena = percpu_arena_as_initialized(opt_percpu_arena);
-}
-
-static bool
-malloc_init_hard_finish(void) {
-	if (malloc_mutex_boot()) {
-		return true;
-	}
-
-	malloc_init_state = malloc_init_initialized;
-	malloc_slow_flag_init();
-
-	return false;
-}
-
-static void
-malloc_init_hard_cleanup(tsdn_t *tsdn, bool reentrancy_set) {
-	malloc_mutex_assert_owner(tsdn, &init_lock);
-	malloc_mutex_unlock(tsdn, &init_lock);
-	if (reentrancy_set) {
-		assert(!tsdn_null(tsdn));
-		tsd_t *tsd = tsdn_tsd(tsdn);
-		assert(tsd_reentrancy_level_get(tsd) > 0);
-		post_reentrancy(tsd);
-	}
-}
-
-bool
-malloc_init_hard(void) {
-	tsd_t *tsd;
-
-	assert(TCACHE_MAXCLASS_LIMIT <= USIZE_GROW_SLOW_THRESHOLD);
-	assert(SC_LOOKUP_MAXCLASS <= USIZE_GROW_SLOW_THRESHOLD);
-	/*
-	 * This asserts an extreme case where TINY_MAXCLASS is larger
-	 * than LARGE_MINCLASS.  It could only happen if some constants
-	 * are configured miserably wrong.
-	 */
-	assert(SC_NTINY == 0 || SC_LG_TINY_MAXCLASS <= SC_LG_LARGE_MINCLASS);
-
-#if defined(_WIN32) && _WIN32_WINNT < 0x0600
-	_init_init_lock();
-#endif
-	malloc_mutex_lock(TSDN_NULL, &init_lock);
-
-#define UNLOCK_RETURN(tsdn, ret, reentrancy)                                   \
-	malloc_init_hard_cleanup(tsdn, reentrancy);                            \
-	return ret;
-
-	if (!malloc_init_hard_needed()) {
-		UNLOCK_RETURN(TSDN_NULL, false, false)
-	}
-
-	if (malloc_init_state != malloc_init_a0_initialized
-	    && malloc_init_hard_a0_locked()) {
-		UNLOCK_RETURN(TSDN_NULL, true, false)
-	}
-
-	malloc_mutex_unlock(TSDN_NULL, &init_lock);
-	/* Recursive allocation relies on functional tsd. */
-	tsd = malloc_tsd_boot0();
-	if (tsd == NULL) {
-		return true;
-	}
-	if (malloc_init_hard_recursible()) {
-		return true;
-	}
-
-	malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
-	/* Set reentrancy level to 1 during init. */
-	pre_reentrancy(tsd, NULL);
-	/* Initialize narenas before prof_boot2 (for allocation). */
-	if (malloc_init_narenas(tsd_tsdn(tsd))
-	    || background_thread_boot1(tsd_tsdn(tsd), b0get())) {
-		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
-	}
-	if (opt_hpa) {
-		/*
-		 * We didn't initialize arena 0 hpa_shard in arena_new, because
-		 * background_thread_enabled wasn't initialized yet, but we
-		 * need it to set correct value for deferral_allowed.
-		 */
-		arena_t         *a0 = arena_get(tsd_tsdn(tsd), 0, false);
-		hpa_shard_opts_t hpa_shard_opts = opt_hpa_opts;
-		hpa_shard_opts.deferral_allowed = background_thread_enabled();
-		if (pa_shard_enable_hpa(tsd_tsdn(tsd), &a0->pa_shard,
-		        &hpa_shard_opts, &opt_hpa_sec_opts)) {
-			UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
-		}
-	}
-	if (config_prof && prof_boot2(tsd, b0get())) {
-		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
-	}
-
-	malloc_init_percpu();
-
-	if (malloc_init_hard_finish()) {
-		UNLOCK_RETURN(tsd_tsdn(tsd), true, true)
-	}
-	post_reentrancy(tsd);
-	malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
-
-	witness_assert_lockless(
-	    witness_tsd_tsdn(tsd_witness_tsdp_get_unsafe(tsd)));
-	malloc_tsd_boot1();
-	/* Update TSD after tsd_boot1. */
-	tsd = tsd_fetch();
-	if (opt_background_thread) {
-		assert(have_background_thread);
-		/*
-		 * Need to finish init & unlock first before creating background
-		 * threads (pthread_create depends on malloc).  ctl_init (which
-		 * sets isthreaded) needs to be called without holding any lock.
-		 */
-		background_thread_ctl_init(tsd_tsdn(tsd));
-		if (background_thread_create(tsd, 0)) {
-			return true;
-		}
-	}
-#undef UNLOCK_RETURN
-	return false;
-}
-
-/*
- * If an application creates a thread before doing any allocation in the main
- * thread, then calls fork(2) in the main thread followed by memory allocation
- * in the child process, a race can occur that results in deadlock within the
- * child: the main thread may have forked while the created thread had
- * partially initialized the allocator.  Ordinarily jemalloc prevents
- * fork/malloc races via the following functions it registers during
- * initialization using pthread_atfork(), but of course that does no good if
- * the allocator isn't fully initialized at fork time.  The following library
- * constructor is a partial solution to this problem.  It may still be possible
- * to trigger the deadlock described above, but doing so would involve forking
- * via a library constructor that runs before jemalloc's runs.
- */
-#ifndef JEMALLOC_JET
-JEMALLOC_ATTR(constructor)
-static void
-jemalloc_constructor(void) {
-	malloc_init();
-}
-#endif

src/large.c

@@ -184,7 +184,8 @@ large_ralloc_move_helper(
 
 void *
 large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
-    size_t alignment, bool zero, tcache_t *tcache) {
+    size_t alignment, bool zero, tcache_t *tcache,
+    hook_ralloc_args_t *hook_args) {
 	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
 
 	size_t oldusize = edata_usize_get(edata);
@@ -195,6 +196,9 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
 
 	/* Try to avoid moving the allocation. */
 	if (!large_ralloc_no_move(tsdn, edata, usize, usize, zero)) {
+		hook_invoke_expand(hook_args->is_realloc ? hook_expand_realloc
+		                                         : hook_expand_rallocx,
+		    ptr, oldusize, usize, (uintptr_t)ptr, hook_args->args);
 		return edata_addr_get(edata);
 	}
 
@@ -209,6 +213,13 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
 		return NULL;
 	}
 
+	hook_invoke_alloc(
+	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx,
+	    ret, (uintptr_t)ret, hook_args->args);
+	hook_invoke_dalloc(
+	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
+	    ptr, hook_args->args);
+
 	size_t copysize = (usize < oldusize) ? usize : oldusize;
 	memcpy(ret, edata_addr_get(edata), copysize);
 	isdalloct(tsdn, edata_addr_get(edata), oldusize, tcache, NULL, true);

src/malloc_io.c

@@ -105,7 +105,8 @@ buferror(int err, char *buf, size_t buflen) {
     && defined(_GNU_SOURCE)
 	char *b = strerror_r(err, buf, buflen);
 	if (b != buf) {
-		malloc_snprintf(buf, buflen, "%s", b);
+		strncpy(buf, b, buflen);
+		buf[buflen - 1] = '\0';
 	}
 	return 0;
 #else

src/pa.c

@@ -67,7 +67,7 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
 bool
 pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
     const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
-	if (hpa_shard_init(tsdn, &shard->hpa, &shard->central->hpa,
+	if (hpa_shard_init(tsdn, &shard->hpa_shard, &shard->central->hpa,
 	        shard->emap, shard->base, &shard->edata_cache, shard->ind,
 	        hpa_opts, hpa_sec_opts)) {
 		return true;
@@ -82,7 +82,7 @@ void
 pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
 	atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
 	if (shard->ever_used_hpa) {
-		hpa_shard_disable(tsdn, &shard->hpa);
+		hpa_shard_disable(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -95,7 +95,7 @@ pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (shard->ever_used_hpa) {
-		hpa_shard_flush(tsdn, &shard->hpa);
+		hpa_shard_flush(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -108,10 +108,16 @@ void
 pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
 	pac_destroy(tsdn, &shard->pac);
 	if (shard->ever_used_hpa) {
-		hpa_shard_destroy(tsdn, &shard->hpa);
+		hpa_shard_destroy(tsdn, &shard->hpa_shard);
 	}
 }
 
+static pai_t *
+pa_get_pai(pa_shard_t *shard, edata_t *edata) {
+	return (edata_pai_get(edata) == EXTENT_PAI_PAC ? &shard->pac.pai
+	                                               : &shard->hpa_shard.pai);
+}
+
 edata_t *
 pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
     bool slab, szind_t szind, bool zero, bool guarded,
@@ -122,7 +128,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
 
 	edata_t *edata = NULL;
 	if (!guarded && pa_shard_uses_hpa(shard)) {
-		edata = hpa_alloc(tsdn, &shard->hpa, size, alignment,
+		edata = pai_alloc(tsdn, &shard->hpa_shard.pai, size, alignment,
 		    zero, /* guarded */ false, slab, deferred_work_generated);
 	}
 	/*
@@ -130,7 +136,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
 	 * allocation request.
 	 */
 	if (edata == NULL) {
-		edata = pac_alloc(tsdn, &shard->pac, size, alignment, zero,
+		edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero,
 		    guarded, slab, deferred_work_generated);
 	}
 	if (edata != NULL) {
@@ -158,15 +164,10 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 	}
 	size_t expand_amount = new_size - old_size;
 
-	/*
-	 * HPA expand always fails (it's a stub); skip the call entirely for
-	 * HPA-owned extents.
-	 */
-	if (edata_pai_get(edata) == EXTENT_PAI_HPA) {
-		return true;
-	}
-	bool error = pac_expand(tsdn, &shard->pac, edata, old_size, new_size,
-	    zero, deferred_work_generated);
+	pai_t *pai = pa_get_pai(shard, edata);
+
+	bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero,
+	    deferred_work_generated);
 	if (error) {
 		return true;
 	}
@@ -188,15 +189,9 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size,
 	}
 	size_t shrink_amount = old_size - new_size;
 
-	/*
-	 * HPA shrink always fails (it's a stub); skip the call entirely for
-	 * HPA-owned extents.
-	 */
-	if (edata_pai_get(edata) == EXTENT_PAI_HPA) {
-		return true;
-	}
-	bool error = pac_shrink(tsdn, &shard->pac, edata, old_size, new_size,
-	    deferred_work_generated);
+	pai_t *pai = pa_get_pai(shard, edata);
+	bool   error = pai_shrink(
+            tsdn, pai, edata, old_size, new_size, deferred_work_generated);
 	if (error) {
 		return true;
 	}
@@ -221,11 +216,8 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata,
 	edata_addr_set(edata, edata_base_get(edata));
 	edata_szind_set(edata, SC_NSIZES);
 	pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE);
-	if (edata_pai_get(edata) == EXTENT_PAI_HPA) {
-		hpa_dalloc(tsdn, &shard->hpa, edata, deferred_work_generated);
-	} else {
-		pac_dalloc(tsdn, &shard->pac, edata, deferred_work_generated);
-	}
+	pai_t *pai = pa_get_pai(shard, edata);
+	pai_dalloc(tsdn, pai, edata, deferred_work_generated);
 }
 
 bool
@@ -244,14 +236,14 @@ pa_shard_set_deferral_allowed(
     tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed) {
 	if (pa_shard_uses_hpa(shard)) {
 		hpa_shard_set_deferral_allowed(
-		    tsdn, &shard->hpa, deferral_allowed);
+		    tsdn, &shard->hpa_shard, deferral_allowed);
 	}
 }
 
 void
 pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (pa_shard_uses_hpa(shard)) {
-		hpa_shard_do_deferred_work(tsdn, &shard->hpa);
+		hpa_shard_do_deferred_work(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -262,14 +254,14 @@ pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
  */
 uint64_t
 pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) {
-	uint64_t time = pac_time_until_deferred_work(tsdn, &shard->pac);
+	uint64_t time = pai_time_until_deferred_work(tsdn, &shard->pac.pai);
 	if (time == BACKGROUND_THREAD_DEFERRED_MIN) {
 		return time;
 	}
 
 	if (pa_shard_uses_hpa(shard)) {
-		uint64_t hpa = hpa_time_until_deferred_work(
-		    tsdn, &shard->hpa);
+		uint64_t hpa = pai_time_until_deferred_work(
+		    tsdn, &shard->hpa_shard.pai);
 		if (hpa < time) {
 			time = hpa;
 		}

src/pa_extra.c

@@ -17,7 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (shard->ever_used_hpa) {
-		hpa_shard_prefork2(tsdn, &shard->hpa);
+		hpa_shard_prefork2(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -25,7 +25,7 @@ void
 pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) {
 	malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx);
 	if (shard->ever_used_hpa) {
-		hpa_shard_prefork3(tsdn, &shard->hpa);
+		hpa_shard_prefork3(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -34,9 +34,8 @@ pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_prefork(tsdn, &shard->pac.ecache_dirty);
 	ecache_prefork(tsdn, &shard->pac.ecache_muzzy);
 	ecache_prefork(tsdn, &shard->pac.ecache_retained);
-	ecache_prefork(tsdn, &shard->pac.ecache_pinned);
 	if (shard->ever_used_hpa) {
-		hpa_shard_prefork4(tsdn, &shard->hpa);
+		hpa_shard_prefork4(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -51,12 +50,11 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_dirty);
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_muzzy);
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
-	ecache_postfork_parent(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
-		hpa_shard_postfork_parent(tsdn, &shard->hpa);
+		hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -66,37 +64,36 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_child(tsdn, &shard->pac.ecache_dirty);
 	ecache_postfork_child(tsdn, &shard->pac.ecache_muzzy);
 	ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
-	ecache_postfork_child(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
-		hpa_shard_postfork_child(tsdn, &shard->hpa);
+		hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
 	}
 }
 
 size_t
-pa_shard_nactive(const pa_shard_t *shard) {
+pa_shard_nactive(pa_shard_t *shard) {
 	return atomic_load_zu(&shard->nactive, ATOMIC_RELAXED);
 }
 
 size_t
-pa_shard_ndirty(const pa_shard_t *shard) {
+pa_shard_ndirty(pa_shard_t *shard) {
 	size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty);
 	if (shard->ever_used_hpa) {
-		ndirty += psset_ndirty(&shard->hpa.psset);
+		ndirty += psset_ndirty(&shard->hpa_shard.psset);
 	}
 	return ndirty;
 }
 
 size_t
-pa_shard_nmuzzy(const pa_shard_t *shard) {
+pa_shard_nmuzzy(pa_shard_t *shard) {
 	return ecache_npages_get(&shard->pac.ecache_muzzy);
 }
 
 void
 pa_shard_basic_stats_merge(
-    const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
+    pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy) {
 	*nactive += pa_shard_nactive(shard);
 	*ndirty += pa_shard_ndirty(shard);
 	*nmuzzy += pa_shard_nmuzzy(shard);
@@ -110,15 +107,12 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 
 	pa_shard_stats_out->pac_stats.retained +=
 	    ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
-	pa_shard_stats_out->pac_stats.pinned +=
-	    ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE;
 	pa_shard_stats_out->edata_avail += atomic_load_zu(
 	    &shard->edata_cache.count, ATOMIC_RELAXED);
 
 	size_t resident_pgs = 0;
 	resident_pgs += pa_shard_nactive(shard);
 	resident_pgs += pa_shard_ndirty(shard);
-	resident_pgs += ecache_npages_get(&shard->pac.ecache_pinned);
 	*resident += (resident_pgs << LG_PAGE);
 
 	/* Dirty decay stats */
@@ -153,31 +147,26 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 	    atomic_load_zu(&shard->pac.stats->abandoned_vm, ATOMIC_RELAXED));
 
 	for (pszind_t i = 0; i < SC_NPSIZES; i++) {
-		size_t dirty, muzzy, retained, pinned, dirty_bytes,
-		    muzzy_bytes, retained_bytes, pinned_bytes;
+		size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes,
+		    retained_bytes;
 		dirty = ecache_nextents_get(&shard->pac.ecache_dirty, i);
 		muzzy = ecache_nextents_get(&shard->pac.ecache_muzzy, i);
 		retained = ecache_nextents_get(&shard->pac.ecache_retained, i);
-		pinned = ecache_nextents_get(&shard->pac.ecache_pinned, i);
 		dirty_bytes = ecache_nbytes_get(&shard->pac.ecache_dirty, i);
 		muzzy_bytes = ecache_nbytes_get(&shard->pac.ecache_muzzy, i);
 		retained_bytes = ecache_nbytes_get(
 		    &shard->pac.ecache_retained, i);
-		pinned_bytes = ecache_nbytes_get(
-		    &shard->pac.ecache_pinned, i);
 
 		estats_out[i].ndirty = dirty;
 		estats_out[i].nmuzzy = muzzy;
 		estats_out[i].nretained = retained;
-		estats_out[i].npinned = pinned;
 		estats_out[i].dirty_bytes = dirty_bytes;
 		estats_out[i].muzzy_bytes = muzzy_bytes;
 		estats_out[i].retained_bytes = retained_bytes;
-		estats_out[i].pinned_bytes = pinned_bytes;
 	}
 
 	if (shard->ever_used_hpa) {
-		hpa_shard_stats_merge(tsdn, &shard->hpa, hpa_stats_out);
+		hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
 	}
 }
 
@@ -200,8 +189,6 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 	    &shard->pac.ecache_muzzy.mtx, arena_prof_mutex_extents_muzzy);
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 	    &shard->pac.ecache_retained.mtx, arena_prof_mutex_extents_retained);
-	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
-	    &shard->pac.ecache_pinned.mtx, arena_prof_mutex_extents_pinned);
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 	    &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty);
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
@@ -209,11 +196,11 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 
 	if (shard->ever_used_hpa) {
 		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
-		    &shard->hpa.mtx, arena_prof_mutex_hpa_shard);
+		    &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard);
 		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
-		    &shard->hpa.grow_mtx,
+		    &shard->hpa_shard.grow_mtx,
 		    arena_prof_mutex_hpa_shard_grow);
-		sec_mutex_stats_read(tsdn, &shard->hpa.sec,
+		sec_mutex_stats_read(tsdn, &shard->hpa_shard.sec,
 		    &mutex_prof_data[arena_prof_mutex_hpa_sec]);
 	}
 }

src/pac.c

@@ -4,6 +4,17 @@
 #include "jemalloc/internal/pac.h"
 #include "jemalloc/internal/san.h"
 
+static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size,
+    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
+    bool *deferred_work_generated);
+static bool     pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+        size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
+static bool     pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata,
+        size_t old_size, size_t new_size, bool *deferred_work_generated);
+static void     pac_dalloc_impl(
+        tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
+static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
+
 static inline void
 pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay,
     pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) {
@@ -20,7 +31,6 @@ pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay,
 		return;
 	case extent_state_active:
 	case extent_state_retained:
-	case extent_state_pinned:
 	case extent_state_transition:
 	case extent_state_merging:
 	default:
@@ -62,12 +72,6 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	        /* delay_coalesce */ false)) {
 		return true;
 	}
-	/* Pinned extents: no decay, delayed coalesce. */
-	if (ecache_init(tsdn, &pac->ecache_pinned, extent_state_pinned, ind,
-	        /* delay_coalesce */ true)) {
-		return true;
-	}
-	atomic_store_b(&pac->has_pinned, false, ATOMIC_RELAXED);
 	exp_grow_init(&pac->exp_grow);
 	if (malloc_mutex_init(&pac->grow_mtx, "extent_grow",
 	        WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
@@ -92,6 +96,12 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	pac->stats_mtx = stats_mtx;
 	atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
 
+	pac->pai.alloc = &pac_alloc_impl;
+	pac->pai.expand = &pac_expand_impl;
+	pac->pai.shrink = &pac_shrink_impl;
+	pac->pai.dalloc = &pac_dalloc_impl;
+	pac->pai.time_until_deferred_work = &pac_time_until_deferred_work;
+
 	return false;
 }
 
@@ -100,14 +110,6 @@ pac_may_have_muzzy(pac_t *pac) {
 	return pac_decay_ms_get(pac, extent_state_muzzy) != 0;
 }
 
-static inline void
-pac_ecache_dalloc(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
-    edata_t *edata) {
-	ecache_dalloc(tsdn, pac, ehooks,
-	    edata_pinned_get(edata) ? &pac->ecache_pinned : &pac->ecache_dirty,
-	    edata);
-}
-
 static size_t
 pac_alloc_retained_batched_size(size_t size) {
 	if (size > SC_LARGE_MAXCLASS) {
@@ -131,22 +133,8 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	assert(!guarded || alignment <= PAGE);
 	size_t newly_mapped_size = 0;
 
-	edata_t *edata = NULL;
-
-	/*
-	 * Guarded allocations need surrounding guard pages, which the pinned
-	 * pool does not maintain; skip ecache_pinned in that case.
-	 */
-	if (!guarded && atomic_load_b(&pac->has_pinned, ATOMIC_RELAXED)
-	    && ecache_npages_get(&pac->ecache_pinned) > 0) {
-		edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_pinned,
-		    NULL, size, alignment, zero, guarded);
-	}
-
-	if (edata == NULL) {
-		edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
-		    NULL, size, alignment, zero, guarded);
-	}
+	edata_t *edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
+	    NULL, size, alignment, zero, guarded);
 
 	if (edata == NULL && pac_may_have_muzzy(pac)) {
 		edata = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
@@ -192,10 +180,12 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 			    edata, size, batched_size - size,
 			    /* holding_core_locks */ false);
 			if (trail == NULL) {
-				pac_record_grown(tsdn, pac, ehooks, edata);
+				ecache_dalloc(tsdn, pac, ehooks,
+				    &pac->ecache_retained, edata);
 				edata = NULL;
 			} else {
-				pac_ecache_dalloc(tsdn, pac, ehooks, trail);
+				ecache_dalloc(tsdn, pac, ehooks,
+				    &pac->ecache_dirty, trail);
 			}
 		}
 
@@ -248,10 +238,11 @@ pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 	return edata;
 }
 
-edata_t *
-pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment,
+static edata_t *
+pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment,
     bool zero, bool guarded, bool frequent_reuse,
     bool *deferred_work_generated) {
+	pac_t    *pac = (pac_t *)self;
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	edata_t *edata = NULL;
@@ -274,11 +265,10 @@ pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment,
 	return edata;
 }
 
-bool
-pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
+static bool
+pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
     size_t new_size, bool zero, bool *deferred_work_generated) {
-	assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
-
+	pac_t    *pac = (pac_t *)self;
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	size_t mapped_add = 0;
@@ -287,53 +277,23 @@ pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
 	if (ehooks_merge_will_fail(ehooks)) {
 		return true;
 	}
-	edata_t *trail = NULL;
-	if (edata_pinned_get(edata)) {
-		trail = ecache_alloc(tsdn, pac, ehooks,
-		    &pac->ecache_pinned, edata, expand_amount,
-		    PAGE, zero, /* guarded */ false);
-		if (trail == NULL) {
-			/*
-			 * Only ecache_pinned can hold a mergeable neighbor;
-			 * dirty, muzzy, and retained extents are non-pinned.
-			 * Pinned memory is already committed, and hooks are
-			 * unlikely to reserve adjacent pinned space for growth,
-			 * so don't consult the hook to grow in place.
-			 */
-			return true;
-		}
-		assert(edata_pinned_get(trail));
-	} else {
-		trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
-		    edata, expand_amount, PAGE, zero, /* guarded */ false);
-		if (trail == NULL) {
-			trail = ecache_alloc(tsdn, pac, ehooks,
-			    &pac->ecache_muzzy, edata, expand_amount,
-			    PAGE, zero, /* guarded */ false);
-		}
-		if (trail == NULL) {
-			trail = ecache_alloc_grow(tsdn, pac, ehooks,
-			    &pac->ecache_retained, edata, expand_amount,
-			    PAGE, zero, /* guarded */ false);
-			mapped_add = expand_amount;
-		}
-		if (trail == NULL) {
-			return true;
-		}
+	edata_t *trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_dirty,
+	    edata, expand_amount, PAGE, zero, /* guarded*/ false);
+	if (trail == NULL) {
+		trail = ecache_alloc(tsdn, pac, ehooks, &pac->ecache_muzzy,
+		    edata, expand_amount, PAGE, zero, /* guarded*/ false);
 	}
-	/* extent_merge_wrapper requires matching pinnedness. */
-	if ((edata_pinned_get(edata) != edata_pinned_get(trail))
-	    || extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
-		if (edata_pinned_get(trail)) {
-			if (config_stats) {
-				atomic_fetch_add_zu(&pac->stats->pac_mapped,
-				    mapped_add, ATOMIC_RELAXED);
-			}
-			ecache_dalloc(tsdn, pac, ehooks,
-			    &pac->ecache_pinned, trail);
-		} else {
-			extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
-		}
+	if (trail == NULL) {
+		trail = ecache_alloc_grow(tsdn, pac, ehooks,
+		    &pac->ecache_retained, edata, expand_amount, PAGE, zero,
+		    /* guarded */ false);
+		mapped_add = expand_amount;
+	}
+	if (trail == NULL) {
+		return true;
+	}
+	if (extent_merge_wrapper(tsdn, pac, ehooks, edata, trail)) {
+		extent_dalloc_wrapper(tsdn, pac, ehooks, trail);
 		return true;
 	}
 	if (config_stats && mapped_add > 0) {
@@ -343,11 +303,10 @@ pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
 	return false;
 }
 
-bool
-pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
+static bool
+pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
     size_t new_size, bool *deferred_work_generated) {
-	assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
-
+	pac_t    *pac = (pac_t *)self;
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	size_t shrink_amount = old_size - new_size;
@@ -361,19 +320,15 @@ pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
 	if (trail == NULL) {
 		return true;
 	}
-	bool pinned = edata_pinned_get(trail);
-	pac_ecache_dalloc(tsdn, pac, ehooks, trail);
-	if (!pinned) {
-		*deferred_work_generated = true;
-	}
+	ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, trail);
+	*deferred_work_generated = true;
 	return false;
 }
 
-void
-pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
-    bool *deferred_work_generated) {
-	assert(edata_pai_get(edata) == EXTENT_PAI_PAC);
-
+static void
+pac_dalloc_impl(
+    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
+	pac_t    *pac = (pac_t *)self;
 	ehooks_t *ehooks = pac_ehooks_get(pac);
 
 	if (edata_guarded_get(edata)) {
@@ -397,11 +352,9 @@ pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
 		}
 	}
 
-	bool pinned = edata_pinned_get(edata);
-	pac_ecache_dalloc(tsdn, pac, ehooks, edata);
-	if (!pinned) {
-		*deferred_work_generated = true;
-	}
+	ecache_dalloc(tsdn, pac, ehooks, &pac->ecache_dirty, edata);
+	/* Purging of deallocated pages is deferred */
+	*deferred_work_generated = true;
 }
 
 static inline uint64_t
@@ -417,9 +370,10 @@ pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) {
 	return result;
 }
 
-uint64_t
-pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac) {
+static uint64_t
+pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
 	uint64_t time;
+	pac_t   *pac = (pac_t *)self;
 
 	time = pac_ns_until_purge(
 	    tsdn, &pac->decay_dirty, ecache_npages_get(&pac->ecache_dirty));
@@ -589,7 +543,6 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
 			break;
 		case extent_state_active:
 		case extent_state_retained:
-		case extent_state_pinned:
 		case extent_state_transition:
 		case extent_state_merging:
 		default:
@@ -768,48 +721,7 @@ pac_destroy(tsdn_t *tsdn, pac_t *pac) {
 	 * dss-based extents for later reuse.
 	 */
 	ehooks_t *ehooks = pac_ehooks_get(pac);
-	edata_t *edata;
-	if (atomic_load_b(&pac->has_pinned, ATOMIC_RELAXED)) {
-		/*
-		 * Reroute pinned extents through ecache_retained: clearing the
-		 * pinned bit lets retained's eager coalesce merge fragments
-		 * back to their original OS-allocation bases, so the destroy
-		 * hook can release whole reservations (required on platforms
-		 * like Windows where VirtualFree only accepts the original
-		 * VirtualAlloc base).  Subtract from pac_mapped along the way
-		 * because retained is excluded from stats.mapped.
-		 */
-		edata_list_inactive_t pinned_list;
-		edata_list_inactive_init(&pinned_list);
-		malloc_mutex_lock(tsdn, &pac->ecache_pinned.mtx);
-		assert(eset_npages_get(&pac->ecache_pinned.guarded_eset) == 0);
-		size_t pinned_bytes =
-		    eset_npages_get(&pac->ecache_pinned.eset) << LG_PAGE;
-		while (eset_npages_get(&pac->ecache_pinned.eset) > 0) {
-			edata = eset_fit(&pac->ecache_pinned.eset,
-			    PAGE, PAGE, /* exact_only */ false, SC_PTR_BITS,
-			    /* prefer_small */ false);
-			assert(edata != NULL);
-			assert(edata_pinned_get(edata));
-			eset_remove(&pac->ecache_pinned.eset, edata);
-			emap_update_edata_state(tsdn, pac->emap, edata,
-			    extent_state_active);
-			edata_pinned_set(edata, false);
-			edata_list_inactive_append(&pinned_list, edata);
-		}
-		malloc_mutex_unlock(tsdn, &pac->ecache_pinned.mtx);
-		if (config_stats && pinned_bytes > 0) {
-			atomic_fetch_sub_zu(&pac->stats->pac_mapped,
-			    pinned_bytes, ATOMIC_RELAXED);
-		}
-		while ((edata = edata_list_inactive_first(&pinned_list))
-		    != NULL) {
-			edata_list_inactive_remove(&pinned_list, edata);
-			extent_record(tsdn, pac, ehooks,
-			    &pac->ecache_retained, edata);
-		}
-	}
-	assert(ecache_npages_get(&pac->ecache_pinned) == 0);
+	edata_t  *edata;
 	while (
 	    (edata = ecache_evict(tsdn, pac, ehooks, &pac->ecache_retained, 0))
 	    != NULL) {

src/pages.c

@@ -662,6 +662,10 @@ pages_purge_process_madvise_impl(
 		return true;
 	}
 
+	/*
+	 * TODO: remove this save/restore of errno after supporting errno
+	 * preservation for free() call properly.
+	 */
 	int    saved_errno = get_errno();
 	size_t purged_bytes = (size_t)syscall(JE_SYS_PROCESS_MADVISE_NR,
 	    PIDFD_SELF, (struct iovec *)vec, vec_len, MADV_DONTNEED, 0);

src/psset.c

@@ -349,7 +349,7 @@ psset_enumerate_search(psset_t *psset, pszind_t pind, size_t size) {
 
 	while ((ps = hpdata_age_heap_enumerate_next(
 	            &psset->pageslabs[pind], &helper))) {
-		if ((hpdata_longest_free_range_get(ps) << LG_PAGE) >= size) {
+		if (hpdata_longest_free_range_get(ps) >= size) {
 			return ps;
 		}
 	}

src/sec.c

@@ -90,35 +90,6 @@ sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) {
 	return &sec->bins[ind];
 }
 
-void
-sec_calc_nallocs_for_size(
-    sec_t *sec, size_t size, size_t *min_nallocs_ret, size_t *max_nallocs_ret) {
-	size_t min_nallocs = 1;
-	size_t max_nallocs = 1;
-
-	if (sec_size_supported(sec, size)) {
-		/*
-		 * This attempts to fill up to 1/SEC_MAX_BYTES_DIV of the SEC.
-		 * If we go much over that, we might cause purging.
-		 * This is mainly an issue when max_bytes is small (256K)
-		 * and size is large. For larger max_bytes, we will
-		 * almost always end up with SEC_MAX_NALLOCS.
-		 */
-		size_t nallocs = sec->opts.max_bytes / size / SEC_MAX_BYTES_DIV;
-		nallocs = max_zu(nallocs, SEC_MIN_NALLOCS);
-		min_nallocs = SEC_MIN_NALLOCS;
-		max_nallocs = min_zu(nallocs, SEC_MAX_NALLOCS);
-	}
-
-	/* post-conditions */
-	assert(1 <= min_nallocs);
-	assert(min_nallocs <= max_nallocs);
-	assert(max_nallocs <= SEC_MAX_NALLOCS);
-
-	*min_nallocs_ret = min_nallocs;
-	*max_nallocs_ret = max_nallocs;
-}
-
 static edata_t *
 sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
 	malloc_mutex_assert_owner(tsdn, &bin->mtx);

src/stats.c

@@ -712,8 +712,6 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 	COL_HDR(row, muzzy, NULL, right, 13, size)
 	COL_HDR(row, nretained, NULL, right, 13, size)
 	COL_HDR(row, retained, NULL, right, 13, size)
-	COL_HDR(row, npinned, NULL, right, 13, size)
-	COL_HDR(row, pinned, NULL, right, 13, size)
 	COL_HDR(row, ntotal, NULL, right, 13, size)
 	COL_HDR(row, total, NULL, right, 13, size)
 
@@ -730,27 +728,22 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 
 	in_gap = false;
 	for (j = 0; j < SC_NPSIZES; j++) {
-		size_t ndirty, nmuzzy, nretained, npinned, total,
-		    dirty_bytes, muzzy_bytes, retained_bytes, pinned_bytes,
-		    total_bytes;
+		size_t ndirty, nmuzzy, nretained, total, dirty_bytes,
+		    muzzy_bytes, retained_bytes, total_bytes;
 		stats_arenas_mib[4] = j;
 
 		CTL_LEAF(stats_arenas_mib, 5, "ndirty", &ndirty, size_t);
 		CTL_LEAF(stats_arenas_mib, 5, "nmuzzy", &nmuzzy, size_t);
 		CTL_LEAF(stats_arenas_mib, 5, "nretained", &nretained, size_t);
-		CTL_LEAF(stats_arenas_mib, 5, "npinned", &npinned, size_t);
 		CTL_LEAF(
 		    stats_arenas_mib, 5, "dirty_bytes", &dirty_bytes, size_t);
 		CTL_LEAF(
 		    stats_arenas_mib, 5, "muzzy_bytes", &muzzy_bytes, size_t);
 		CTL_LEAF(stats_arenas_mib, 5, "retained_bytes", &retained_bytes,
 		    size_t);
-		CTL_LEAF(stats_arenas_mib, 5, "pinned_bytes", &pinned_bytes,
-		    size_t);
 
-		total = ndirty + nmuzzy + nretained + npinned;
-		total_bytes = dirty_bytes + muzzy_bytes + retained_bytes
-		    + pinned_bytes;
+		total = ndirty + nmuzzy + nretained;
+		total_bytes = dirty_bytes + muzzy_bytes + retained_bytes;
 
 		in_gap_prev = in_gap;
 		in_gap = (total == 0);
@@ -765,8 +758,6 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 		emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy);
 		emitter_json_kv(
 		    emitter, "nretained", emitter_type_size, &nretained);
-		emitter_json_kv(
-		    emitter, "npinned", emitter_type_size, &npinned);
 
 		emitter_json_kv(
 		    emitter, "dirty_bytes", emitter_type_size, &dirty_bytes);
@@ -774,8 +765,6 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 		    emitter, "muzzy_bytes", emitter_type_size, &muzzy_bytes);
 		emitter_json_kv(emitter, "retained_bytes", emitter_type_size,
 		    &retained_bytes);
-		emitter_json_kv(emitter, "pinned_bytes", emitter_type_size,
-		    &pinned_bytes);
 		emitter_json_object_end(emitter);
 
 		col_size.size_val = sz_pind2sz(j);
@@ -786,8 +775,6 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 		col_muzzy.size_val = muzzy_bytes;
 		col_nretained.size_val = nretained;
 		col_retained.size_val = retained_bytes;
-		col_npinned.size_val = npinned;
-		col_pinned.size_val = pinned_bytes;
 		col_ntotal.size_val = total;
 		col_total.size_val = total_bytes;
 
@@ -902,7 +889,9 @@ stats_arena_hpa_shard_counters_print(
 	    " / sec)\n"
 	    "  Hugify failures: %" FMTu64 " (%" FMTu64
 	    " / sec)\n"
-	    "  Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n",
+	    "  Dehugifies: %" FMTu64 " (%" FMTu64
+	    " / sec)\n"
+	    "\n",
 	    npageslabs, npageslabs_huge, npageslabs_nonhuge, nactive,
 	    nactive_huge, nactive_nonhuge, ndirty, ndirty_huge, ndirty_nonhuge,
 	    nretained_nonhuge, npurge_passes,
@@ -942,82 +931,6 @@ stats_arena_hpa_shard_counters_print(
 	emitter_json_kv(
 	    emitter, "ndirty_huge", emitter_type_size, &ndirty_huge);
 	emitter_json_object_end(emitter); /* End "slabs" */
-
-	/* alloc_batch stats */
-	uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
-	uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
-	uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
-	uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
-	uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
-	uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
-	uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
-
-	size_t alloc_mib[CTL_MAX_DEPTH];
-	CTL_LEAF_PREPARE(alloc_mib, 0, "stats.arenas");
-	alloc_mib[2] = i;
-	CTL_LEAF_PREPARE(alloc_mib, 3, "hpa_shard.alloc");
-
-	for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
-		alloc_mib[5] = j;
-		CTL_LEAF(alloc_mib, 6, "min_extents", &hpa_alloc_min_extents[j],
-		    uint64_t);
-		CTL_LEAF(alloc_mib, 6, "max_extents", &hpa_alloc_max_extents[j],
-		    uint64_t);
-		CTL_LEAF(
-		    alloc_mib, 6, "extents", &hpa_alloc_extents[j], uint64_t);
-		CTL_LEAF(alloc_mib, 6, "ps", &hpa_alloc_ps[j], uint64_t);
-		CTL_LEAF(alloc_mib, 6, "pages_per_ps",
-		    &hpa_alloc_pages_per_ps[j], uint64_t);
-		CTL_LEAF(alloc_mib, 6, "extents_per_ps",
-		    &hpa_alloc_extents_per_ps[j], uint64_t);
-		CTL_LEAF(alloc_mib, 6, "total_elapsed_ns_per_ps",
-		    &hpa_alloc_total_elapsed_ns_per_ps[j], uint64_t);
-	}
-
-	emitter_table_printf(emitter, "  extent allocation distribution:\n");
-	emitter_table_printf(emitter,
-	    "  %4s %20s %20s %20s %20s %20s %20s %24s %24s\n", "",
-	    "min_extents", "max_extents",
-	    "extents", "ps", "pages_per_ps", "extents_per_ps",
-	    "total_elapsed_ns_per_ps", "elapsed_ns_per_ps");
-	for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
-		const uint64_t extents_per_ps = hpa_alloc_extents_per_ps[j];
-		const uint64_t total_elapsed_ns_per_ps =
-		    hpa_alloc_total_elapsed_ns_per_ps[j];
-		const uint64_t elapsed_ns_per_ps = (extents_per_ps != 0)
-		    ? (total_elapsed_ns_per_ps / extents_per_ps)
-		    : 0;
-		emitter_table_printf(emitter,
-		    "  %4zu %20" FMTu64 " %20" FMTu64 " %20" FMTu64
-		    " %20" FMTu64 " %20" FMTu64 " %20" FMTu64 " %24" FMTu64
-		    " %24" FMTu64 "\n",
-		    j, hpa_alloc_min_extents[j], hpa_alloc_max_extents[j],
-		    hpa_alloc_extents[j],
-		    hpa_alloc_ps[j], hpa_alloc_pages_per_ps[j], extents_per_ps,
-		    total_elapsed_ns_per_ps, elapsed_ns_per_ps);
-	}
-	emitter_table_printf(emitter, "\n");
-
-	emitter_json_array_kv_begin(emitter, "extent_allocation_distribution");
-	for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
-		emitter_json_object_begin(emitter);
-		emitter_json_kv(emitter, "min_extents", emitter_type_uint64,
-		    &hpa_alloc_min_extents[j]);
-		emitter_json_kv(emitter, "max_extents", emitter_type_uint64,
-		    &hpa_alloc_max_extents[j]);
-		emitter_json_kv(emitter, "extents", emitter_type_uint64,
-		    &hpa_alloc_extents[j]);
-		emitter_json_kv(
-		    emitter, "ps", emitter_type_uint64, &hpa_alloc_ps[j]);
-		emitter_json_kv(emitter, "pages_per_ps", emitter_type_uint64,
-		    &hpa_alloc_pages_per_ps[j]);
-		emitter_json_kv(emitter, "extents_per_ps", emitter_type_uint64,
-		    &hpa_alloc_extents_per_ps[j]);
-		emitter_json_kv(emitter, "total_elapsed_ns_per_ps",
-		    emitter_type_uint64, &hpa_alloc_total_elapsed_ns_per_ps[j]);
-		emitter_json_object_end(emitter);
-	}
-	emitter_json_array_end(emitter); /* End "alloc_batch" */
 }
 
 static void
@@ -1253,7 +1166,7 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
 	unsigned    nthreads;
 	const char *dss;
 	ssize_t     dirty_decay_ms, muzzy_decay_ms;
-	size_t      page, pactive, pdirty, pmuzzy, mapped, retained, pinned;
+	size_t      page, pactive, pdirty, pmuzzy, mapped, retained;
 	size_t      base, internal, resident, metadata_edata, metadata_rtree,
 	    metadata_thp, extent_avail;
 	uint64_t dirty_npurge, dirty_nmadvise, dirty_purged;
@@ -1554,7 +1467,6 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
 
 	GET_AND_EMIT_MEM_STAT(mapped)
 	GET_AND_EMIT_MEM_STAT(retained)
-	GET_AND_EMIT_MEM_STAT(pinned)
 	GET_AND_EMIT_MEM_STAT(base)
 	GET_AND_EMIT_MEM_STAT(internal)
 	GET_AND_EMIT_MEM_STAT(metadata_edata)
@@ -1761,6 +1673,7 @@ stats_general_print(emitter_t *emitter) {
 	OPT_WRITE_SIZE_T("hpa_sec_nshards")
 	OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
 	OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
+	OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
 	OPT_WRITE_BOOL("huge_arena_pac_thp")
 	OPT_WRITE_CHAR_P("metadata_thp")
 	OPT_WRITE_INT64("mutex_max_spin")
@@ -1959,7 +1872,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 	 * the transition to the emitter code.
 	 */
 	size_t allocated, active, metadata, metadata_edata, metadata_rtree,
-	    metadata_thp, resident, mapped, retained, pinned;
+	    metadata_thp, resident, mapped, retained;
 	size_t   num_background_threads;
 	size_t   zero_reallocs;
 	uint64_t background_thread_num_runs, background_thread_run_interval;
@@ -1973,7 +1886,6 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 	CTL_GET("stats.resident", &resident, size_t);
 	CTL_GET("stats.mapped", &mapped, size_t);
 	CTL_GET("stats.retained", &retained, size_t);
-	CTL_GET("stats.pinned", &pinned, size_t);
 
 	CTL_GET("stats.zero_reallocs", &zero_reallocs, size_t);
 
@@ -2004,16 +1916,15 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed,
 	emitter_json_kv(emitter, "resident", emitter_type_size, &resident);
 	emitter_json_kv(emitter, "mapped", emitter_type_size, &mapped);
 	emitter_json_kv(emitter, "retained", emitter_type_size, &retained);
-	emitter_json_kv(emitter, "pinned", emitter_type_size, &pinned);
 	emitter_json_kv(
 	    emitter, "zero_reallocs", emitter_type_size, &zero_reallocs);
 
 	emitter_table_printf(emitter,
 	    "Allocated: %zu, active: %zu, "
 	    "metadata: %zu (n_thp %zu, edata %zu, rtree %zu), resident: %zu, "
-	    "mapped: %zu, retained: %zu, pinned: %zu\n",
+	    "mapped: %zu, retained: %zu\n",
 	    allocated, active, metadata, metadata_thp, metadata_edata,
-	    metadata_rtree, resident, mapped, retained, pinned);
+	    metadata_rtree, resident, mapped, retained);
 
 	/* Strange behaviors */
 	emitter_table_printf(emitter,

src/tcache.c

@@ -203,6 +203,26 @@ tcache_gc_item_delay_compute(szind_t szind) {
 	return (uint8_t)item_delay;
 }
 
+static inline void *
+tcache_gc_small_heuristic_addr_get(
+    tsd_t *tsd, tcache_slow_t *tcache_slow, szind_t szind) {
+	assert(szind < SC_NBINS);
+	tsdn_t *tsdn = tsd_tsdn(tsd);
+	bin_t  *bin = bin_choose(tsdn, tcache_slow->arena, szind, NULL);
+	assert(bin != NULL);
+
+	malloc_mutex_lock(tsdn, &bin->lock);
+	edata_t *slab = (bin->slabcur == NULL)
+	    ? edata_heap_first(&bin->slabs_nonfull)
+	    : bin->slabcur;
+	assert(slab != NULL || edata_heap_empty(&bin->slabs_nonfull));
+	void *ret = (slab != NULL) ? edata_addr_get(slab) : NULL;
+	assert(ret != NULL || slab == NULL);
+	malloc_mutex_unlock(tsdn, &bin->lock);
+
+	return ret;
+}
+
 static inline bool
 tcache_gc_is_addr_remote(void *addr, uintptr_t min, uintptr_t max) {
 	assert(addr != NULL);
@@ -339,55 +359,6 @@ tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
 	}
 }
 
-#ifdef JEMALLOC_JET
-/*
- * The GC helpers above are static inline so they cannot be linked from a
- * separate translation unit.  In JET builds we expose thin wrappers with a
- * `_test` suffix so test/unit/tcache_gc.c can exercise them directly.  These
- * symbols are absent from the production library.
- */
-#define JET_WRAP_RET(ret, fn, params, args)	\
-	ret fn##_test params {			\
-		return fn args;			\
-	}
-#define JET_WRAP_VOID(fn, params, args)		\
-	void fn##_test params {			\
-		fn args;			\
-	}
-
-JET_WRAP_RET(cache_bin_sz_t, tcache_gc_small_nremote_get,
-    (cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min,
-     uintptr_t *addr_max, szind_t szind, size_t nflush),
-    (cache_bin, addr, addr_min, addr_max, szind, nflush))
-
-JET_WRAP_VOID(tcache_gc_small_bin_shuffle,
-    (cache_bin_t *cache_bin, cache_bin_sz_t nremote,
-     uintptr_t addr_min, uintptr_t addr_max),
-    (cache_bin, nremote, addr_min, addr_max))
-
-JET_WRAP_RET(uint8_t, tcache_nfill_small_lg_div_get,
-    (tcache_slow_t *tcache_slow, szind_t szind),
-    (tcache_slow, szind))
-
-JET_WRAP_VOID(tcache_nfill_small_burst_prepare,
-    (tcache_slow_t *tcache_slow, szind_t szind),
-    (tcache_slow, szind))
-
-JET_WRAP_VOID(tcache_nfill_small_burst_reset,
-    (tcache_slow_t *tcache_slow, szind_t szind),
-    (tcache_slow, szind))
-
-JET_WRAP_VOID(tcache_nfill_small_gc_update,
-    (tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit),
-    (tcache_slow, szind, limit))
-
-JET_WRAP_RET(uint8_t, tcache_gc_item_delay_compute,
-    (szind_t szind), (szind))
-
-#undef JET_WRAP_RET
-#undef JET_WRAP_VOID
-#endif
-
 static bool
 tcache_gc_small(
     tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) {
@@ -446,9 +417,9 @@ tcache_gc_small(
 		goto label_flush;
 	}
 
-	/* Query arena for a locality anchor (small-bin slab address). */
-	void *addr = arena_locality_hint(tsd_tsdn(tsd), tcache_slow->arena,
-	    szind);
+	/* Query arena binshard to get heuristic locality info. */
+	void *addr = tcache_gc_small_heuristic_addr_get(
+	    tsd, tcache_slow, szind);
 	if (addr == NULL) {
 		goto label_flush;
 	}
@@ -771,52 +742,58 @@ tcache_bin_ncached_max_read(
 
 void
 tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
-    arena_t *arena) {
+    tcache_t *tcache, arena_t *arena) {
 	assert(tcache_slow->arena == NULL);
-	assert(tcache_slow->tcache != NULL);
 	tcache_slow->arena = arena;
 
 	if (config_stats) {
+		/* Link into list of extant tcaches. */
+		malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
+
+		ql_elm_new(tcache_slow, link);
+		ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
 		cache_bin_array_descriptor_init(
-		    &tcache_slow->cache_bin_array_descriptor,
-		    tcache_slow->tcache->bins);
-		arena_cache_bin_array_register(tsdn, arena,
-		    &tcache_slow->cache_bin_array_descriptor);
+		    &tcache_slow->cache_bin_array_descriptor, tcache->bins);
+		ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
+		    &tcache_slow->cache_bin_array_descriptor, link);
+
+		malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
 	}
 }
 
 static void
-tcache_arena_dissociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow) {
+tcache_arena_dissociate(
+    tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache) {
 	arena_t *arena = tcache_slow->arena;
 	assert(arena != NULL);
 	if (config_stats) {
-		arena_cache_bin_array_unregister(tsdn, arena,
-		    &tcache_slow->cache_bin_array_descriptor);
+		/* Unlink from list of extant tcaches. */
+		malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
+		if (config_debug) {
+			bool           in_ql = false;
+			tcache_slow_t *iter;
+			ql_foreach (iter, &arena->tcache_ql, link) {
+				if (iter == tcache_slow) {
+					in_ql = true;
+					break;
+				}
+			}
+			assert(in_ql);
+		}
+		ql_remove(&arena->tcache_ql, tcache_slow, link);
+		ql_remove(&arena->cache_bin_array_descriptor_ql,
+		    &tcache_slow->cache_bin_array_descriptor, link);
+		tcache_stats_merge(tsdn, tcache_slow->tcache, arena);
+		malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
 	}
 	tcache_slow->arena = NULL;
 }
 
 void
 tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
-    arena_t *arena) {
-	tcache_arena_dissociate(tsdn, tcache_slow);
-	tcache_arena_associate(tsdn, tcache_slow, arena);
-}
-
-cache_bin_array_descriptor_t *
-tcache_postfork_arena_descriptor(tsdn_t *tsdn, arena_t *arena) {
-	if (!config_stats) {
-		return NULL;
-	}
-	tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn));
-	if (tcache_slow == NULL || tcache_slow->arena != arena) {
-		return NULL;
-	}
-	assert(tcache_slow->tcache != NULL);
-	cache_bin_array_descriptor_init(
-	    &tcache_slow->cache_bin_array_descriptor,
-	    tcache_slow->tcache->bins);
-	return &tcache_slow->cache_bin_array_descriptor;
+    tcache_t *tcache, arena_t *arena) {
+	tcache_arena_dissociate(tsdn, tcache_slow, tcache);
+	tcache_arena_associate(tsdn, tcache_slow, tcache, arena);
 }
 
 static void
@@ -833,6 +810,7 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, void *mem,
 	tcache->tcache_slow = tcache_slow;
 	tcache_slow->tcache = tcache;
 
+	memset(&tcache_slow->link, 0, sizeof(ql_elm(tcache_t)));
 	nstime_init_zero(&tcache_slow->last_gc_time);
 	tcache_slow->next_gc_bin = 0;
 	tcache_slow->next_gc_bin_small = 0;
@@ -1006,7 +984,8 @@ tsd_tcache_data_init_impl(
 	if (!malloc_initialized()) {
 		/* If in initialization, assign to a0. */
 		arena = arena_get(tsd_tsdn(tsd), 0, false);
-		tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, arena);
+		tcache_arena_associate(
+		    tsd_tsdn(tsd), tcache_slow, tcache, arena);
 	} else {
 		if (arena == NULL) {
 			arena = arena_choose(tsd, NULL);
@@ -1014,7 +993,7 @@ tsd_tcache_data_init_impl(
 		/* This may happen if thread.tcache.enabled is used. */
 		if (tcache_slow->arena == NULL) {
 			tcache_arena_associate(
-			    tsd_tsdn(tsd), tcache_slow, arena);
+			    tsd_tsdn(tsd), tcache_slow, tcache, arena);
 		}
 	}
 	assert(arena == tcache_slow->arena);
@@ -1074,7 +1053,7 @@ tcache_create_explicit(tsd_t *tsd) {
 	    tsd, tcache_slow, tcache, mem, tcache_get_default_ncached_max());
 
 	tcache_arena_associate(
-	    tsd_tsdn(tsd), tcache_slow, arena_ichoose(tsd, NULL));
+	    tsd_tsdn(tsd), tcache_slow, tcache, arena_ichoose(tsd, NULL));
 
 	return tcache;
 }
@@ -1246,7 +1225,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
 	tcache_slow_t *tcache_slow = tcache->tcache_slow;
 	tcache_flush_cache(tsd, tcache);
 	arena_t *arena = tcache_slow->arena;
-	tcache_arena_dissociate(tsd_tsdn(tsd), tcache_slow);
+	tcache_arena_dissociate(tsd_tsdn(tsd), tcache_slow, tcache);
 
 	if (tsd_tcache) {
 		cache_bin_t *cache_bin = &tcache->bins[0];
@@ -1296,6 +1275,29 @@ tcache_cleanup(tsd_t *tsd) {
 	memset(tcache->bins, 0, sizeof(cache_bin_t) * TCACHE_NBINS_MAX);
 }
 
+void
+tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
+	cassert(config_stats);
+
+	/* Merge and reset tcache stats. */
+	for (unsigned i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) {
+		cache_bin_t *cache_bin = &tcache->bins[i];
+		if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
+			continue;
+		}
+		if (i < SC_NBINS) {
+			bin_t *bin = bin_choose(tsdn, arena, i, NULL);
+			malloc_mutex_lock(tsdn, &bin->lock);
+			bin->stats.nrequests += cache_bin->tstats.nrequests;
+			malloc_mutex_unlock(tsdn, &bin->lock);
+		} else {
+			arena_stats_large_flush_nrequests_add(tsdn,
+			    &arena->stats, i, cache_bin->tstats.nrequests);
+		}
+		cache_bin->tstats.nrequests = 0;
+	}
+}
+
 static bool
 tcaches_create_prep(tsd_t *tsd, base_t *base) {
 	bool err;