Limit TSD bootstrap hook to tests

free_sized() and free_aligned_sized() forward straight to sdallocx(), which
expects a non-NULL pointer and asserts on it in debug builds. C23 says both
should accept NULL and do nothing, like free(NULL) does, so a NULL argument
either trips that assert or feeds NULL into the dealloc path in release builds.

It is not hard to hit. glibc 2.41 ships free_sized()/free_aligned_sized(), and
a C++ sized delete of a null pointer compiles down to a free_sized() call. Once
jemalloc is preloaded its versions take over, and that NULL call takes down the
process. I ran into it with GTK4/GLib apps under LD_PRELOAD.

Check for NULL first, the way free() already does, and add an integration test
covering the NULL case for both functions.

While here, give free_aligned_sized() its own core.free_aligned_sized.entry
and .exit logging and call je_sdallocx_impl() directly rather than the
je_sdallocx() wrapper, so it mirrors free_sized() and no longer logs under
sdallocx. The C++ sized-delete paths (sizedDeleteImpl, alignedSizedDeleteImpl)
get the same treatment: log entry/exit unconditionally and guard the call with
likely(ptr != nullptr).

.appveyor.yml

@@ -1,3 +1,5 @@
+image: Visual Studio 2022
+
 version: '{build}'
 
 environment:
@@ -37,7 +39,8 @@ environment:
 
 install:
   - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
-  - if defined MSVC call "c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %MSVC%
+  - if "%MSVC%"=="amd64" call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+  - if "%MSVC%"=="x86"   call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars32.bat"
   - if defined MSVC pacman --noconfirm -Rsc mingw-w64-%CPU%-gcc gcc
   - pacman --noconfirm -Syuu
   - pacman --noconfirm -S autoconf

.github/workflows/check_formatting.yaml

@@ -5,6 +5,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       - name: Check for trailing whitespace
         run: scripts/check_trailing_whitespace.sh

.github/workflows/freebsd-ci.yml

@@ -25,7 +25,7 @@ jobs:
     name: FreeBSD (${{ matrix.arch }}, debug=${{ matrix.debug }}, prof=${{ matrix.prof }}${{ matrix.uncommon && ', uncommon' || '' }})
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 1
 

.github/workflows/linux-ci.yml

@@ -508,9 +508,19 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: "--enable-cxx-infallible-new --enable-debug"
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |
@@ -637,9 +647,14 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=background_thread:true"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |

.github/workflows/macos-ci.yml

@@ -60,9 +60,14 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |
@@ -162,9 +167,14 @@ jobs:
               CXX: g++
               CONFIGURE_FLAGS: "--with-malloc-conf=percpu_arena:percpu"
               EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: "-Werror -Wno-array-bounds -Wno-unknown-warning-option -Wno-ignored-attributes -Wno-deprecated-declarations"
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       run: |

.github/workflows/static_analysis.yaml

@@ -7,7 +7,7 @@ jobs:
       # We build libunwind ourselves because sadly the version
       # provided by Ubuntu via apt-get is much too old.
       - name: Check out libunwind
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           repository: libunwind/libunwind
           path: libunwind
@@ -23,7 +23,7 @@ jobs:
           cd ..
           rm -rf libunwind
       - name: Check out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       # We download LLVM directly from the latest stable release
       # on GitHub, because this tends to be much newer than the
       # version available via apt-get in Ubuntu.
@@ -54,7 +54,7 @@ jobs:
           scripts/run_static_analysis.sh static_analysis_results "$GITHUB_OUTPUT"
       - name: Upload static analysis results
         if: ${{ steps.run_static_analysis.outputs.HAS_STATIC_ANALYSIS_RESULTS }} == '1'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: static_analysis_results
           path: static_analysis_results

.github/workflows/windows-ci.yml

@@ -52,9 +52,14 @@ jobs:
               CXX: cl.exe
               CROSS_COMPILE_32BIT: yes
               CONFIGURE_FLAGS: --enable-debug
+          - env:
+              CC: gcc
+              CXX: g++
+              CONFIGURE_FLAGS: --enable-cxx-infallible-new
+              EXTRA_CFLAGS: -fcommon
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Show OS version
       shell: cmd

INSTALL.md

@@ -217,6 +217,15 @@ any of the following arguments (not a definitive list) to 'configure':
     Disable C++ integration.  This will cause new and delete operator
     implementations to be omitted.
 
+* `--enable-cxx-infallible-new`
+
+    Make the throwing `operator new` abort on allocation failure (logging the
+    requested size) instead of throwing `std::bad_alloc`; the `std::nothrow`
+    overloads still return null.  Disabled by default, and has no effect when
+    C++ integration is disabled (`--disable-cxx`).  When enabled, under LTO
+    this lets the compiler treat `operator new` as non-throwing and elide
+    exception-handling cleanup in callers.
+
 * `--with-xslroot=<path>`
 
     Specify where to find DocBook XSL stylesheets when building the
@@ -411,7 +420,7 @@ differ in their ease of use and flexibility.
 ### With MSVC solutions
 This is the easy, but less flexible approach. It doesn't let you specify
 arguments to the `configure` script.
-  
+
 1. Install Cygwin with at least the following packages:
    * autoconf
    * autogen
@@ -419,18 +428,19 @@ arguments to the `configure` script.
    * grep
    * sed
 
-2. Install Visual Studio 2015 or 2017 with Visual C++
+2. Install Visual Studio with Visual C++. We currently test on 2022 and provide
+solution files back to 2015.
 
 3. Add Cygwin\bin to the PATH environment variable
 
-4. Open "x64 Native Tools Command Prompt for VS 2017"
+4. Open "x64 Native Tools Command Prompt" for your version of Visual Studio
    (note: x86/x64 doesn't matter at this point)
 
 5. Generate header files:
    sh -c "CC=cl ./autogen.sh"
 
-6. Now the project can be opened and built in Visual Studio:
-   msvc\jemalloc_vc2017.sln
+6. Now the project can be opened and built in Visual Studio using the
+   corresponding solution files in the `msvc\` folder.
 
 ### With MSYS
 This is a more involved approach that offers the same configuration flexibility

Makefile.in

@@ -95,7 +95,10 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix)
 BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof
 C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h
 C_SRCS := $(srcroot)src/jemalloc.c \
+	$(srcroot)src/jemalloc_fork.c \
+	$(srcroot)src/jemalloc_init.c \
 	$(srcroot)src/arena.c \
+	$(srcroot)src/arenas_management.c \
 	$(srcroot)src/background_thread.c \
 	$(srcroot)src/base.c \
 	$(srcroot)src/bin.c \
@@ -121,7 +124,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/fxp.c \
 	$(srcroot)src/san.c \
 	$(srcroot)src/san_bump.c \
-	$(srcroot)src/hook.c \
 	$(srcroot)src/hpa.c \
 	$(srcroot)src/hpa_central.c \
 	$(srcroot)src/hpa_hooks.c \
@@ -130,6 +132,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/inspect.c \
 	$(srcroot)src/large.c \
 	$(srcroot)src/log.c \
+	$(srcroot)src/malloc_dispatch.c \
 	$(srcroot)src/malloc_io.c \
 	$(srcroot)src/conf.c \
 	$(srcroot)src/mutex.c \
@@ -203,12 +206,12 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/a0.c \
 	$(srcroot)test/unit/arena_decay.c \
 	$(srcroot)test/unit/arena_reset.c \
+	$(srcroot)test/unit/arenas_management.c \
 	$(srcroot)test/unit/atomic.c \
 	$(srcroot)test/unit/background_thread.c \
 	$(srcroot)test/unit/background_thread_enable.c \
 	$(srcroot)test/unit/background_thread_init.c \
 	$(srcroot)test/unit/base.c \
-	$(srcroot)test/unit/batch_alloc.c \
 	$(srcroot)test/unit/bin.c \
 	$(srcroot)test/unit/binshard.c \
 	$(srcroot)test/unit/bitmap.c \
@@ -226,7 +229,10 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/div.c \
 	$(srcroot)test/unit/double_free.c \
 	$(srcroot)test/unit/edata_cache.c \
+	$(srcroot)test/unit/emap.c \
 	$(srcroot)test/unit/emitter.c \
+	$(srcroot)test/unit/eset.c \
+	$(srcroot)test/unit/extent_dss.c \
 	$(srcroot)test/unit/extent_quantize.c \
 	${srcroot}test/unit/fb.c \
 	$(srcroot)test/unit/fork.c \
@@ -234,16 +240,19 @@ TESTS_UNIT := \
 	${srcroot}test/unit/san.c \
 	${srcroot}test/unit/san_bump.c \
 	$(srcroot)test/unit/hash.c \
-	$(srcroot)test/unit/hook.c \
 	$(srcroot)test/unit/hpa.c \
+	$(srcroot)test/unit/hpa_central.c \
 	$(srcroot)test/unit/hpa_sec_integration.c \
 	$(srcroot)test/unit/hpa_thp_always.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \
 	$(srcroot)test/unit/hpa_background_thread.c \
+	$(srcroot)test/unit/hpa_pageslab_packing.c \
 	$(srcroot)test/unit/hpdata.c \
+	$(srcroot)test/unit/extent_alloc_flags.c \
 	$(srcroot)test/unit/huge.c \
 	$(srcroot)test/unit/inspect.c \
+	$(srcroot)test/unit/jemalloc_init.c \
 	$(srcroot)test/unit/junk.c \
 	$(srcroot)test/unit/junk_alloc.c \
 	$(srcroot)test/unit/junk_free.c \
@@ -252,6 +261,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/log.c \
 	$(srcroot)test/unit/mallctl.c \
 	$(srcroot)test/unit/malloc_conf_2.c \
+	$(srcroot)test/unit/malloc_dispatch.c \
 	$(srcroot)test/unit/malloc_io.c \
 	$(srcroot)test/unit/math.c \
 	$(srcroot)test/unit/mpsc_queue.c \
@@ -261,6 +271,8 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/ncached_max.c \
 	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
+	$(srcroot)test/unit/pac.c \
+	$(srcroot)test/unit/pac_sec_integration.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/peak.c \
@@ -289,7 +301,6 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/safety_check.c \
 	$(srcroot)test/unit/sc.c \
 	$(srcroot)test/unit/sec.c \
-	$(srcroot)test/unit/seq.c \
 	$(srcroot)test/unit/SFMT.c \
 	$(srcroot)test/unit/size_check.c \
 	$(srcroot)test/unit/size_classes.c \
@@ -301,6 +312,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/sz.c \
 	$(srcroot)test/unit/tcache_init.c \
 	$(srcroot)test/unit/tcache_max.c \
+	$(srcroot)test/unit/tcache_gc.c \
 	$(srcroot)test/unit/test_hooks.c \
 	$(srcroot)test/unit/thread_event.c \
 	$(srcroot)test/unit/ticker.c \
@@ -314,12 +326,12 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/zero_reallocs.c
 ifeq (@enable_prof@, 1)
 TESTS_UNIT += \
-	$(srcroot)test/unit/arena_reset_prof.c \
-	$(srcroot)test/unit/batch_alloc_prof.c
+	$(srcroot)test/unit/arena_reset_prof.c
 endif
 TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \
 	$(srcroot)test/integration/allocated.c \
 	$(srcroot)test/integration/extent.c \
+	$(srcroot)test/integration/free_sized.c \
 	$(srcroot)test/integration/malloc.c \
 	$(srcroot)test/integration/mallocx.c \
 	$(srcroot)test/integration/MALLOCX_ARENA.c \
@@ -337,9 +349,14 @@ TESTS_INTEGRATION += \
 endif
 ifeq (@enable_cxx@, 1)
 CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
-TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp \
-	$(srcroot)test/integration/cpp/infallible_new_true.cpp \
-	$(srcroot)test/integration/cpp/infallible_new_false.cpp
+TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
+ifeq (@enable_cxx_exceptions@, 1)
+ifeq (@enable_cxx_infallible_new@, 1)
+TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/infallible_new.cpp
+else
+TESTS_INTEGRATION_CPP += $(srcroot)test/integration/cpp/failing_new.cpp
+endif
+endif
 else
 CPP_SRCS :=
 TESTS_INTEGRATION_CPP :=
@@ -347,9 +364,7 @@ endif
 TESTS_ANALYZE := $(srcroot)test/analyze/prof_bias.c \
 	$(srcroot)test/analyze/rand.c \
 	$(srcroot)test/analyze/sizes.c
-TESTS_STRESS := $(srcroot)test/stress/batch_alloc.c \
-	$(srcroot)test/stress/fill_flush.c \
-	$(srcroot)test/stress/hookbench.c \
+TESTS_STRESS := $(srcroot)test/stress/fill_flush.c \
 	$(srcroot)test/stress/large_microbench.c \
 	$(srcroot)test/stress/mallctl.c \
 	$(srcroot)test/stress/microbench.c
@@ -609,38 +624,38 @@ build_lib: build_lib_static
 endif
 
 install_bin:
-	$(INSTALL) -d $(BINDIR)
+	$(INSTALL) -d '$(BINDIR)'
 	@for b in $(BINS); do \
-	echo "$(INSTALL) -m 755 $$b $(BINDIR)"; \
-	$(INSTALL) -m 755 $$b $(BINDIR); \
+	echo "$(INSTALL) -m 755 $$b '$(BINDIR)'"; \
+	$(INSTALL) -m 755 $$b '$(BINDIR)'; \
 done
 
 install_include:
-	$(INSTALL) -d $(INCLUDEDIR)/jemalloc
+	$(INSTALL) -d '$(INCLUDEDIR)/jemalloc'
 	@for h in $(C_HDRS); do \
-	echo "$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
-	$(INSTALL) -m 644 $$h $(INCLUDEDIR)/jemalloc; \
+	echo "$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'"; \
+	$(INSTALL) -m 644 $$h '$(INCLUDEDIR)/jemalloc'; \
 done
 
 install_lib_shared: $(DSOS)
-	$(INSTALL) -d $(LIBDIR)
-	$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR)
+	$(INSTALL) -d '$(LIBDIR)'
+	$(INSTALL) -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) '$(LIBDIR)'
 ifneq ($(SOREV),$(SO))
-	ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO)
+	ln -sf $(LIBJEMALLOC).$(SOREV) '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
 endif
 
 install_lib_static: $(STATIC_LIBS)
-	$(INSTALL) -d $(LIBDIR)
+	$(INSTALL) -d '$(LIBDIR)'
 	@for l in $(STATIC_LIBS); do \
-	echo "$(INSTALL) -m 755 $$l $(LIBDIR)"; \
-	$(INSTALL) -m 755 $$l $(LIBDIR); \
+	echo "$(INSTALL) -m 644 $$l '$(LIBDIR)'"; \
+	$(INSTALL) -m 644 $$l '$(LIBDIR)'; \
 done
 
 install_lib_pc: $(PC)
-	$(INSTALL) -d $(LIBDIR)/pkgconfig
+	$(INSTALL) -d '$(LIBDIR)/pkgconfig'
 	@for l in $(PC); do \
-	echo "$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig"; \
-	$(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \
+	echo "$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'"; \
+	$(INSTALL) -m 644 $$l '$(LIBDIR)/pkgconfig'; \
 done
 
 ifeq ($(enable_shared), 1)
@@ -652,17 +667,17 @@ endif
 install_lib: install_lib_pc
 
 install_doc_html: build_doc_html
-	$(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix)
+	$(INSTALL) -d '$(DATADIR)/doc/jemalloc$(install_suffix)'
 	@for d in $(DOCS_HTML); do \
-	echo "$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \
-	$(INSTALL) -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \
+	echo "$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'"; \
+	$(INSTALL) -m 644 $$d '$(DATADIR)/doc/jemalloc$(install_suffix)'; \
 done
 
 install_doc_man: build_doc_man
-	$(INSTALL) -d $(MANDIR)/man3
+	$(INSTALL) -d '$(MANDIR)/man3'
 	@for d in $(DOCS_MAN3); do \
-	echo "$(INSTALL) -m 644 $$d $(MANDIR)/man3"; \
-	$(INSTALL) -m 644 $$d $(MANDIR)/man3; \
+	echo "$(INSTALL) -m 644 $$d '$(MANDIR)/man3'"; \
+	$(INSTALL) -m 644 $$d '$(MANDIR)/man3'; \
 done
 
 install_doc: install_doc_html install_doc_man
@@ -674,23 +689,23 @@ install: install_doc
 endif
 
 uninstall_bin:
-	$(RM) -v $(foreach b,$(notdir $(BINS)),$(BINDIR)/$(b))
+	$(RM) -v $(foreach b,$(notdir $(BINS)),'$(BINDIR)/$(b)')
 
 uninstall_include:
-	$(RM) -v $(foreach h,$(notdir $(C_HDRS)),$(INCLUDEDIR)/jemalloc/$(h))
-	rmdir -v $(INCLUDEDIR)/jemalloc
+	$(RM) -v $(foreach h,$(notdir $(C_HDRS)),'$(INCLUDEDIR)/jemalloc/$(h)')
+	rmdir -v '$(INCLUDEDIR)/jemalloc'
 
 uninstall_lib_shared:
-	$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SOREV)
+	$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SOREV)'
 ifneq ($(SOREV),$(SO))
-	$(RM) -v $(LIBDIR)/$(LIBJEMALLOC).$(SO)
+	$(RM) -v '$(LIBDIR)/$(LIBJEMALLOC).$(SO)'
 endif
 
 uninstall_lib_static:
-	$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),$(LIBDIR)/$(l))
+	$(RM) -v $(foreach l,$(notdir $(STATIC_LIBS)),'$(LIBDIR)/$(l)')
 
 uninstall_lib_pc:
-	$(RM) -v $(foreach p,$(notdir $(PC)),$(LIBDIR)/pkgconfig/$(p))
+	$(RM) -v $(foreach p,$(notdir $(PC)),'$(LIBDIR)/pkgconfig/$(p)')
 
 ifeq ($(enable_shared), 1)
 uninstall_lib: uninstall_lib_shared
@@ -701,11 +716,11 @@ endif
 uninstall_lib: uninstall_lib_pc
 
 uninstall_doc_html:
-	$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),$(DATADIR)/doc/jemalloc$(install_suffix)/$(d))
-	rmdir -v $(DATADIR)/doc/jemalloc$(install_suffix)
+	$(RM) -v $(foreach d,$(notdir $(DOCS_HTML)),'$(DATADIR)/doc/jemalloc$(install_suffix)/$(d)')
+	rmdir -v '$(DATADIR)/doc/jemalloc$(install_suffix)'
 
 uninstall_doc_man:
-	$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),$(MANDIR)/man3/$(d))
+	$(RM) -v $(foreach d,$(notdir $(DOCS_MAN3)),'$(MANDIR)/man3/$(d)')
 
 uninstall_doc: uninstall_doc_html uninstall_doc_man
 

configure.ac

@@ -158,29 +158,27 @@ AC_SUBST([abs_objroot])
 
 dnl Munge install path variables.
 case "$prefix" in
-   *\ * ) AC_MSG_ERROR([Prefix should not contain spaces]) ;;
    "NONE" ) prefix="/usr/local" ;;
 esac
 case "$exec_prefix" in
-   *\ * ) AC_MSG_ERROR([Exec prefix should not contain spaces]) ;;
    "NONE" ) exec_prefix=$prefix ;;
 esac
 PREFIX=$prefix
 AC_SUBST([PREFIX])
-BINDIR=`eval echo $bindir`
-BINDIR=`eval echo $BINDIR`
+BINDIR=`eval echo "\"$bindir\""`
+BINDIR=`eval echo "\"$BINDIR\""`
 AC_SUBST([BINDIR])
-INCLUDEDIR=`eval echo $includedir`
-INCLUDEDIR=`eval echo $INCLUDEDIR`
+INCLUDEDIR=`eval echo "\"$includedir\""`
+INCLUDEDIR=`eval echo "\"$INCLUDEDIR\""`
 AC_SUBST([INCLUDEDIR])
-LIBDIR=`eval echo $libdir`
-LIBDIR=`eval echo $LIBDIR`
+LIBDIR=`eval echo "\"$libdir\""`
+LIBDIR=`eval echo "\"$LIBDIR\""`
 AC_SUBST([LIBDIR])
-DATADIR=`eval echo $datadir`
-DATADIR=`eval echo $DATADIR`
+DATADIR=`eval echo "\"$datadir\""`
+DATADIR=`eval echo "\"$DATADIR\""`
 AC_SUBST([DATADIR])
-MANDIR=`eval echo $mandir`
-MANDIR=`eval echo $MANDIR`
+MANDIR=`eval echo "\"$mandir\""`
+MANDIR=`eval echo "\"$MANDIR\""`
 AC_SUBST([MANDIR])
 
 dnl Support for building documentation.
@@ -376,7 +374,53 @@ fi
 if test "x$enable_cxx" = "x1"; then
   AC_DEFINE([JEMALLOC_ENABLE_CXX], [ ], [ ])
 fi
+if test "x$enable_cxx" = "x1"; then
+  dnl Now check whether the C++ compiler has exceptions enabled.
+  AC_LANG_PUSH([C++])
+  SAVED_CXXFLAGS="${CXXFLAGS}"
+  CXXFLAGS="${CXXFLAGS} ${EXTRA_CXXFLAGS}"
+  JE_COMPILABLE([C++ exception support], [
+#include <new>
+], [
+	try {
+		throw std::bad_alloc();
+	} catch (const std::bad_alloc &) {
+	}
+], [je_cv_cxx_exceptions])
+  CXXFLAGS="${SAVED_CXXFLAGS}"
+  AC_LANG_POP([C++])
+  if test "x${je_cv_cxx_exceptions}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_HAVE_CXX_EXCEPTIONS], [ ], [ ])
+    enable_cxx_exceptions="1"
+  else
+    enable_cxx_exceptions="0"
+  fi
+fi
+
+dnl Do not enable infallible-new by default. When enabled, the throwing
+dnl operator new aborts on OOM (with size logged) instead of throwing
+dnl std::bad_alloc; nothrow new still returns null. With LTO this lets the
+dnl compiler prove operator new doesn't throw and elide exception cleanup.
+AC_ARG_ENABLE([cxx-infallible-new],
+  [AS_HELP_STRING([--enable-cxx-infallible-new],
+    [Abort on OOM in throwing operator new instead of throwing std::bad_alloc])],
+[if test "x$enable_cxx_infallible_new" = "xno" ; then
+  enable_cxx_infallible_new="0"
+else
+  enable_cxx_infallible_new="1"
+fi
+],
+[enable_cxx_infallible_new="0"]
+)
+if test "x$enable_cxx" != "x1" ; then
+  enable_cxx_infallible_new="0"
+fi
+if test "x$enable_cxx_infallible_new" = "x1" ; then
+  AC_DEFINE([JEMALLOC_INFALLIBLE_NEW], [ ], [ ])
+fi
 AC_SUBST([enable_cxx])
+AC_SUBST([enable_cxx_exceptions])
+AC_SUBST([enable_cxx_infallible_new])
 AC_SUBST([CONFIGURE_CXXFLAGS])
 AC_SUBST([SPECIFIED_CXXFLAGS])
 AC_SUBST([EXTRA_CXXFLAGS])
@@ -556,9 +600,10 @@ typedef unsigned __int32 uint32_t;
       else
         AC_MSG_ERROR([cannot determine number of significant virtual address bits])
       fi
-      AC_CACHE_CHECK([rdtscp support],
-		     [je_cv_rdtscp],
-		     AC_RUN_IFELSE([AC_LANG_PROGRAM(
+    fi
+    AC_CACHE_CHECK([rdtscp support],
+		   [je_cv_rdtscp],
+		   AC_RUN_IFELSE([AC_LANG_PROGRAM(
 [[
 #include <stdint.h>
 ]],
@@ -567,12 +612,11 @@ typedef unsigned __int32 uint32_t;
       asm volatile("rdtscp" : "=d"(dx) ::);
       return 0;
 ]])],
-      [je_cv_rdtscp=yes],
-      [je_cv_rdtscp=no],
-      [je_cv_rdtscp=no]))
-      if test "x${je_cv_rdtscp}" = "xyes"; then
-        AC_DEFINE([JEMALLOC_HAVE_RDTSCP], [ ], [ ])
-      fi
+    [je_cv_rdtscp=yes],
+    [je_cv_rdtscp=no],
+    [je_cv_rdtscp=no]))
+    if test "x${je_cv_rdtscp}" = "xyes"; then
+      AC_DEFINE([JEMALLOC_HAVE_RDTSCP], [ ], [ ])
     fi
     ;;
   *)
@@ -2041,6 +2085,9 @@ if test "x${LG_PAGE}" != "xundefined" ; then
 else
    AC_MSG_ERROR([cannot determine value for LG_PAGE])
 fi
+if test "${LG_PAGE}" -lt 8 ; then
+   AC_MSG_ERROR([LG_PAGE must be at least 8 (PAGE >= 256)])
+fi
 
 AC_ARG_WITH([lg_hugepage],
   [AS_HELP_STRING([--with-lg-hugepage=<lg-hugepage>],

doc/jemalloc.xml.in

@@ -846,6 +846,16 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay",
         build configuration.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="config.infallible_new">
+        <term>
+          <mallctl>config.infallible_new</mallctl>
+          (<type>bool</type>)
+          <literal>r-</literal>
+        </term>
+        <listitem><para><option>--enable-cxx-infallible-new</option> was
+        specified during build configuration.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="config.lazy_lock">
         <term>
           <mallctl>config.lazy_lock</mallctl>
@@ -2088,7 +2098,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <parameter>*oldlenp</parameter> must be
         <code language="C">sizeof(<type>char *</type>)</code>.
         Arena names are also included in the output of <link
-        linkend="stats_print"><function>malloc_stats_print()</function></link>.
+        linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
         </para></listitem>
       </varlistentry>
 
@@ -2172,6 +2182,15 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         addition there may be extents created prior to the application having an
         opportunity to take over extent allocation.</para>
 
+        <para>An extent must be operated on (dalloc, destroy, commit, decommit,
+        purge, split, merge) by a hook capable of handling it, normally the hook
+        that allocated it. Replacing hooks on a live arena is tricky and thus
+        discouraged. If the hook is replaced anyway, the new hook should forward
+        operations on extents it did not allocate to the previous hook (e.g.,
+        the new dalloc dispatches to the previous dalloc for an
+        old-hook-allocated extent). The new hook should also avoid merging
+        extents allocated by different hooks.</para>
+
         <programlisting language="C"><![CDATA[
 typedef extent_hooks_s extent_hooks_t;
 struct extent_hooks_s {
@@ -2236,6 +2255,18 @@ struct extent_hooks_s {
         linkend="arena.i.dss"><mallctl>arena.&lt;i&gt;.dss</mallctl></link>
         setting irrelevant.</para>
 
+        <para>The alloc hook may bitwise-OR
+        <constant>EXTENT_ALLOC_FLAG_PINNED</constant> into the low bits of
+        the returned pointer to indicate that the backing memory is
+        non-reclaimable (e.g. HugeTLB pages) and should be reused
+        preferentially; in that case <parameter>*commit</parameter> must also
+        be set to true.  jemalloc strips the low byte before use.  The
+        pinned attribute is per-extent rather than per-hook: a single alloc
+        hook may return pinned and non-pinned extents in different calls.
+        Pinned-ness is set at allocation, inherited through splits, and
+        never changes after that.  Pinned and non-pinned extents are never
+        merged together.</para>
+
         <funcsynopsis><funcprototype>
           <funcdef>typedef bool <function>(extent_dalloc_t)</function></funcdef>
           <paramdef>extent_hooks_t *<parameter>extent_hooks</parameter></paramdef>
@@ -2457,7 +2488,7 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Hugepage size.  This value is also reported in the
         output of <link
-        linkend="stats_print"><function>malloc_stats_print()</function></link>.
+        linkend="malloc_stats_print_opts"><function>malloc_stats_print()</function></link>.
         </para></listitem>
       </varlistentry>
 
@@ -2768,11 +2799,11 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Maximum number of bytes in physically resident data
         pages mapped by the allocator, comprising all pages dedicated to
-        allocator metadata, pages backing active allocations, and unused dirty
-        pages.  This is a maximum rather than precise because pages may not
-        actually be physically resident if they correspond to demand-zeroed
-        virtual memory that has not yet been touched.  This is a multiple of the
-        page size, and is larger than <link
+        allocator metadata, pages backing active allocations, unused dirty
+        pages, and pinned pages.  This is a maximum rather than precise because
+        pages may not actually be physically resident if they correspond to
+        demand-zeroed virtual memory that has not yet been touched.  This is a
+        multiple of the page size, and is larger than <link
         linkend="stats.active"><mallctl>stats.active</mallctl></link>.</para></listitem>
       </varlistentry>
 
@@ -2811,6 +2842,22 @@ struct extent_hooks_s {
         </para></listitem>
       </varlistentry>
 
+      <varlistentry id="stats.pinned">
+        <term>
+          <mallctl>stats.pinned</mallctl>
+          (<type>size_t</type>)
+          <literal>r-</literal>
+          [<option>--enable-stats</option>]
+        </term>
+        <listitem><para>Total number of bytes in unused extents backed by
+        non-reclaimable memory.  Pinned extents are tracked separately from
+        dirty, muzzy, and retained extents because they are excluded from
+        decay and purging; unlike <link
+        linkend="stats.retained"><mallctl>stats.retained</mallctl></link>,
+        pinned bytes are included in <link
+        linkend="stats.mapped"><mallctl>stats.mapped</mallctl></link>.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="stats.zero_reallocs">
         <term>
           <mallctl>stats.zero_reallocs</mallctl>
@@ -3089,6 +3136,18 @@ struct extent_hooks_s {
         details.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="stats.arenas.i.pinned">
+        <term>
+          <mallctl>stats.arenas.&lt;i&gt;.pinned</mallctl>
+          (<type>size_t</type>)
+          <literal>r-</literal>
+          [<option>--enable-stats</option>]
+        </term>
+        <listitem><para>Number of pinned bytes.  See <link
+        linkend="stats.pinned"><mallctl>stats.pinned</mallctl></link> for
+        details.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="stats.arenas.i.extent_avail">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.extent_avail</mallctl>
@@ -3146,11 +3205,11 @@ struct extent_hooks_s {
         </term>
         <listitem><para>Maximum number of bytes in physically resident data
         pages mapped by the arena, comprising all pages dedicated to allocator
-        metadata, pages backing active allocations, and unused dirty pages.
-        This is a maximum rather than precise because pages may not actually be
-        physically resident if they correspond to demand-zeroed virtual memory
-        that has not yet been touched.  This is a multiple of the page
-        size.</para></listitem>
+        metadata, pages backing active allocations, unused dirty pages, and
+        pinned pages.  This is a maximum rather than precise because pages
+        may not actually be physically resident if they correspond to
+        demand-zeroed virtual memory that has not yet been touched.  This is
+        a multiple of the page size.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.dirty_npurge">
@@ -3493,7 +3552,7 @@ struct extent_hooks_s {
         </term>
         <listitem><para> Number of extents of the given type in this arena in
 	the bucket corresponding to page size index &lt;j&gt;. The extent type
-	is one of dirty, muzzy, or retained.</para></listitem>
+	is one of dirty, muzzy, retained, or pinned.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.extents.bytes">
@@ -3505,7 +3564,7 @@ struct extent_hooks_s {
         </term>
 	<listitem><para> Sum of the bytes managed by extents of the given type
 	in this arena in the bucket corresponding to page size index &lt;j&gt;.
-	The extent type is one of dirty, muzzy, or retained.</para></listitem>
+	The extent type is one of dirty, muzzy, retained, or pinned.</para></listitem>
       </varlistentry>
 
       <varlistentry id="stats.arenas.i.lextents.j.nmalloc">
@@ -3625,6 +3684,19 @@ struct extent_hooks_s {
         counters</link>.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="stats.arenas.i.mutexes.extents_pinned">
+        <term>
+          <mallctl>stats.arenas.&lt;i&gt;.mutexes.extents_pinned.{counter}</mallctl>
+          (<type>counter specific type</type>) <literal>r-</literal>
+          [<option>--enable-stats</option>]
+        </term>
+        <listitem><para>Statistics on <varname>arena.&lt;i&gt;.extents_pinned
+        </varname> mutex (arena scope; pinned extents related).
+        <mallctl>{counter}</mallctl> is one of the counters in <link
+        linkend="mutex_counters">mutex profiling
+        counters</link>.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="stats.arenas.i.mutexes.decay_dirty">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.mutexes.decay_dirty.{counter}</mallctl>

include/jemalloc/internal/arena.h

@@ -0,0 +1,290 @@
+#ifndef JEMALLOC_INTERNAL_ARENA_H
+#define JEMALLOC_INTERNAL_ARENA_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_decay_constants.h"
+#include "jemalloc/internal/arena_stats.h"
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/bin.h"
+#include "jemalloc/internal/bitmap.h"
+#include "jemalloc/internal/counter.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/ecache.h"
+#include "jemalloc/internal/edata_cache.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/extent_dss.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/nstime.h"
+#include "jemalloc/internal/pa.h"
+#include "jemalloc/internal/pages.h"
+#include "jemalloc/internal/ql.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/stats.h"
+#include "jemalloc/internal/ticker.h"
+
+/******************************************************************************/
+/* TYPES */
+/******************************************************************************/
+
+/* Default decay times in milliseconds. */
+#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
+#define MUZZY_DECAY_MS_DEFAULT (0)
+/* Maximum length of the arena name. */
+#define ARENA_NAME_LEN 32
+
+typedef enum {
+	percpu_arena_mode_names_base = 0, /* Used for options processing. */
+
+	/*
+	 * *_uninit are used only during bootstrapping, and must correspond
+	 * to initialized variant plus percpu_arena_mode_enabled_base.
+	 */
+	percpu_arena_uninit = 0,
+	per_phycpu_arena_uninit = 1,
+
+	/* All non-disabled modes must come after percpu_arena_disabled. */
+	percpu_arena_disabled = 2,
+
+	percpu_arena_mode_names_limit = 3, /* Used for options processing. */
+	percpu_arena_mode_enabled_base = 3,
+
+	percpu_arena = 3,
+	per_phycpu_arena = 4 /* Hyper threads share arena. */
+} percpu_arena_mode_t;
+
+#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
+#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
+
+/*
+ * When allocation_size >= oversize_threshold, use the dedicated huge arena
+ * (unless have explicitly spicified arena index).  0 disables the feature.
+ */
+#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
+
+struct arena_config_s {
+	/* extent hooks to be used for the arena */
+	extent_hooks_t *extent_hooks;
+
+	/*
+	 * Use extent hooks for metadata (base) allocations when true.
+	 */
+	bool metadata_use_hooks;
+};
+
+typedef struct arena_config_s arena_config_t;
+
+extern const arena_config_t arena_config_default;
+
+/******************************************************************************/
+/* STRUCTS */
+/******************************************************************************/
+
+struct arena_s {
+	/*
+	 * Number of threads currently assigned to this arena.  Each thread has
+	 * two distinct assignments, one for application-serving allocation, and
+	 * the other for internal metadata allocation.  Internal metadata must
+	 * not be allocated from arenas explicitly created via the arenas.create
+	 * mallctl, because the arena.<i>.reset mallctl indiscriminately
+	 * discards all allocations for the affected arena.
+	 *
+	 *   0: Application allocation.
+	 *   1: Internal metadata allocation.
+	 *
+	 * Synchronization: atomic.
+	 */
+	atomic_u_t nthreads[2];
+
+	/* Next bin shard for binding new threads. Synchronization: atomic. */
+	atomic_u_t binshard_next;
+
+	/*
+	 * When percpu_arena is enabled, to amortize the cost of reading /
+	 * updating the current CPU id, track the most recent thread accessing
+	 * this arena, and only read CPU if there is a mismatch.
+	 */
+	tsdn_t *last_thd;
+
+	/* Synchronization: internal. */
+	arena_stats_t stats;
+
+	/*
+	 * List of cache_bin_array_descriptors for extant threads associated
+	 * with this arena.  Stats from these are merged incrementally, and at
+	 * exit if opt_stats_print is enabled.
+	 *
+	 * Synchronization: cache_bin_array_descriptor_ql_mtx.
+	 */
+	ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
+	malloc_mutex_t cache_bin_array_descriptor_ql_mtx;
+
+	/*
+	 * Represents a dss_prec_t, but atomically.
+	 *
+	 * Synchronization: atomic.
+	 */
+	atomic_u_t dss_prec;
+
+	/*
+	 * Extant large allocations.
+	 *
+	 * Synchronization: large_mtx.
+	 */
+	edata_list_active_t large;
+	/* Synchronizes all large allocation/update/deallocation. */
+	malloc_mutex_t large_mtx;
+
+	/* The page-level allocator shard this arena uses. */
+	pa_shard_t pa_shard;
+
+	/*
+	 * A cached copy of base->ind.  This can get accessed on hot paths;
+	 * looking it up in base requires an extra pointer hop / cache miss.
+	 */
+	unsigned ind;
+
+	/*
+	 * Base allocator, from which arena metadata are allocated.
+	 *
+	 * Synchronization: internal.
+	 */
+	base_t *base;
+	/* Used to determine uptime.  Read-only after initialization. */
+	nstime_t create_time;
+
+	/* The name of the arena. */
+	char name[ARENA_NAME_LEN];
+
+	/*
+	 * The arena is allocated alongside its bins; really this is a
+	 * dynamically sized array determined by the binshard settings.
+	 * Enforcing cacheline-alignment to minimize the number of cachelines
+	 * touched on the hot paths.
+	 */
+	JEMALLOC_WARN_ON_USAGE(
+	    "Do not use this field directly. "
+	    "Use `arena_get_bin` instead.")
+	JEMALLOC_ALIGNED(CACHELINE)
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
+	bin_t all_bins[];
+#else
+	bin_t all_bins[0];
+#endif
+};
+
+/******************************************************************************/
+/* EXTERNS */
+/******************************************************************************/
+
+/*
+ * When the amount of pages to be purged exceeds this amount, deferred purge
+ * should happen.
+ */
+#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
+
+extern ssize_t opt_dirty_decay_ms;
+extern ssize_t opt_muzzy_decay_ms;
+
+extern percpu_arena_mode_t opt_percpu_arena;
+extern const char *const   percpu_arena_mode_names[];
+
+extern div_info_t arena_binind_div_info[SC_NBINS];
+
+extern emap_t arena_emap_global;
+
+extern size_t opt_oversize_threshold;
+extern size_t oversize_threshold;
+
+extern bool      opt_huge_arena_pac_thp;
+extern pac_thp_t huge_arena_pac_thp;
+
+/*
+ * arena_bin_offsets[binind] is the offset of the first bin shard for size class
+ * binind.
+ */
+extern uint32_t arena_bin_offsets[SC_NBINS];
+
+void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
+    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
+    size_t *nactive, size_t *ndirty, size_t *nmuzzy);
+void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
+    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
+    size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
+    bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
+    hpa_shard_stats_t *hpastats);
+void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
+edata_t *arena_extent_alloc_large(
+    tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
+void arena_extent_dalloc_large_prep(
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata);
+void arena_extent_ralloc_large_shrink(
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
+void arena_extent_ralloc_large_expand(
+    tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize);
+bool arena_decay_ms_set(
+    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
+ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
+void    arena_decay(
+       tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
+uint64_t       arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
+void           arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
+void           arena_reset(tsd_t *tsd, arena_t *arena);
+void           arena_destroy(tsd_t *tsd, arena_t *arena);
+cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
+    szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min,
+    const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats);
+
+void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
+    bool zero, bool slab);
+void  arena_prof_promote(
+     tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
+size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
+
+void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
+void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
+     cache_bin_ptr_array_t *arr, unsigned nflush, bool small,
+     arena_t *stats_arena, cache_bin_stats_t merge_stats);
+bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
+     size_t extra, bool zero, size_t *newsize);
+dss_prec_t      arena_dss_prec_get(const arena_t *arena);
+ehooks_t       *arena_get_ehooks(const arena_t *arena);
+extent_hooks_t *arena_set_extent_hooks(
+    tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
+bool    arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
+void    arena_name_get(const arena_t *arena, char *name);
+void    arena_name_set(arena_t *arena, const char *name);
+ssize_t arena_dirty_decay_ms_default_get(void);
+bool    arena_dirty_decay_ms_default_set(ssize_t decay_ms);
+ssize_t arena_muzzy_decay_ms_default_get(void);
+bool    arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
+bool    arena_retain_grow_limit_get_set(
+       tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
+unsigned arena_nthreads_get(const arena_t *arena, bool internal);
+void     arena_nthreads_inc(arena_t *arena, bool internal);
+void     arena_nthreads_dec(arena_t *arena, bool internal);
+arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
+bool     arena_init_huge(tsdn_t *tsdn, arena_t *a0);
+bool     arena_ind_is_huge(unsigned ind);
+arena_t *arena_choose_huge(tsd_t *tsd);
+bool   arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
+void  *arena_locality_hint(tsdn_t *tsdn, arena_t *arena, szind_t szind);
+void   arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *desc);
+void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *desc);
+void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork3(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork4(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork5(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
+void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
+void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
+void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena,
+       cache_bin_array_descriptor_t *surviving_desc);
+
+#endif /* JEMALLOC_INTERNAL_ARENA_H */

include/jemalloc/internal/arena_decay_constants.h

@@ -0,0 +1,13 @@
+#ifndef JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H
+#define JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H
+
+/*
+ * Minimal header so both arena.h and tsd_internals.h can share decay-related
+ * constants without dragging the full arena types into the tsd parse chain
+ * (which is loaded long before arena.h via ckh.h -> tsd.h).
+ */
+
+/* Number of event ticks between time checks. */
+#define ARENA_DECAY_NTICKS_PER_UPDATE 1000
+
+#endif /* JEMALLOC_INTERNAL_ARENA_DECAY_CONSTANTS_H */

include/jemalloc/internal/arena_externs.h

@@ -1,125 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_EXTERNS_H
-#define JEMALLOC_INTERNAL_ARENA_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_stats.h"
-#include "jemalloc/internal/bin.h"
-#include "jemalloc/internal/div.h"
-#include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/extent_dss.h"
-#include "jemalloc/internal/hook.h"
-#include "jemalloc/internal/pages.h"
-#include "jemalloc/internal/stats.h"
-
-/*
- * When the amount of pages to be purged exceeds this amount, deferred purge
- * should happen.
- */
-#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024)
-
-extern ssize_t opt_dirty_decay_ms;
-extern ssize_t opt_muzzy_decay_ms;
-
-extern percpu_arena_mode_t opt_percpu_arena;
-extern const char *const   percpu_arena_mode_names[];
-
-extern div_info_t arena_binind_div_info[SC_NBINS];
-
-extern emap_t arena_emap_global;
-
-extern size_t opt_oversize_threshold;
-extern size_t oversize_threshold;
-
-extern bool      opt_huge_arena_pac_thp;
-extern pac_thp_t huge_arena_pac_thp;
-
-/*
- * arena_bin_offsets[binind] is the offset of the first bin shard for size class
- * binind.
- */
-extern uint32_t arena_bin_offsets[SC_NBINS];
-
-void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
-    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
-    size_t *nactive, size_t *ndirty, size_t *nmuzzy);
-void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
-    const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
-    size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
-    bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
-    hpa_shard_stats_t *hpastats);
-void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
-edata_t *arena_extent_alloc_large(
-    tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
-void arena_extent_dalloc_large_prep(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata);
-void arena_extent_ralloc_large_shrink(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
-void arena_extent_ralloc_large_expand(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t oldusize);
-bool arena_decay_ms_set(
-    tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms);
-ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state);
-void    arena_decay(
-       tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all);
-uint64_t       arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena);
-void           arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena);
-void           arena_reset(tsd_t *tsd, arena_t *arena);
-void           arena_destroy(tsd_t *tsd, arena_t *arena);
-cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
-    szind_t binind, cache_bin_ptr_array_t *arr, const cache_bin_sz_t nfill_min,
-    const cache_bin_sz_t nfill_max, cache_bin_stats_t merge_stats);
-
-void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
-    bool zero, bool slab);
-void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
-    bool zero, bool slab, tcache_t *tcache);
-void  arena_prof_promote(
-     tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
-void arena_dalloc_promoted(
-    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
-void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
-
-void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
-void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
-     cache_bin_ptr_array_t *arr, unsigned nflush, bool small,
-     arena_t *stats_arena, cache_bin_stats_t merge_stats);
-bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-     size_t extra, bool zero, size_t *newsize);
-void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache,
-    hook_ralloc_args_t *hook_args);
-dss_prec_t      arena_dss_prec_get(arena_t *arena);
-ehooks_t       *arena_get_ehooks(arena_t *arena);
-extent_hooks_t *arena_set_extent_hooks(
-    tsd_t *tsd, arena_t *arena, extent_hooks_t *extent_hooks);
-bool    arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
-void    arena_name_get(arena_t *arena, char *name);
-void    arena_name_set(arena_t *arena, const char *name);
-ssize_t arena_dirty_decay_ms_default_get(void);
-bool    arena_dirty_decay_ms_default_set(ssize_t decay_ms);
-ssize_t arena_muzzy_decay_ms_default_get(void);
-bool    arena_muzzy_decay_ms_default_set(ssize_t decay_ms);
-bool    arena_retain_grow_limit_get_set(
-       tsd_t *tsd, arena_t *arena, size_t *old_limit, size_t *new_limit);
-unsigned arena_nthreads_get(arena_t *arena, bool internal);
-void     arena_nthreads_inc(arena_t *arena, bool internal);
-void     arena_nthreads_dec(arena_t *arena, bool internal);
-arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
-bool     arena_init_huge(tsdn_t *tsdn, arena_t *a0);
-arena_t *arena_choose_huge(tsd_t *tsd);
-size_t arena_fill_small_fresh(tsdn_t *tsdn, arena_t *arena, szind_t binind,
-    void **ptrs, size_t nfill, bool zero);
-bool   arena_boot(sc_data_t *sc_data, base_t *base, bool hpa);
-void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork3(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork4(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork5(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
-void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
-void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
-void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
-
-#endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */

include/jemalloc/internal/arena_inlines.h

@@ -0,0 +1,400 @@
+#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_H
+#define JEMALLOC_INTERNAL_ARENA_INLINES_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arenas_management.h"
+#include "jemalloc/internal/bin_inlines.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/extent.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/large.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prof.h"
+#include "jemalloc/internal/rtree.h"
+#include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache.h"
+#include "jemalloc/internal/ticker.h"
+
+/* Cheap field accessors. */
+
+static inline unsigned
+arena_ind_get(const arena_t *arena) {
+	return arena->ind;
+}
+
+static inline void
+arena_internal_add(arena_t *arena, size_t size) {
+	atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
+}
+
+static inline void
+arena_internal_sub(arena_t *arena, size_t size) {
+	atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
+}
+
+static inline size_t
+arena_internal_get(const arena_t *arena) {
+	return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
+}
+
+static inline bool
+arena_is_auto(const arena_t *arena) {
+	assert(narenas_auto > 0);
+
+	return (arena_ind_get(arena) < manual_arena_base);
+}
+
+static inline arena_t *
+arena_get_from_edata(const edata_t *edata) {
+	return (arena_t *)atomic_load_p(
+	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
+}
+
+/* Arena selection and migration. */
+
+static inline void
+thread_migrate_arena(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) {
+	assert(oldarena != NULL);
+	assert(newarena != NULL);
+
+	arena_migrate(tsd, oldarena, newarena);
+	if (tcache_available(tsd)) {
+		tcache_arena_reassociate(tsd_tsdn(tsd),
+		    tsd_tcache_slowp_get(tsd), newarena);
+	}
+}
+
+static inline void
+percpu_arena_update(tsd_t *tsd, unsigned cpu) {
+	assert(have_percpu_arena);
+	arena_t *oldarena = tsd_arena_get(tsd);
+	assert(oldarena != NULL);
+	unsigned oldind = arena_ind_get(oldarena);
+
+	if (oldind != cpu) {
+		unsigned newind = cpu;
+		arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
+		assert(newarena != NULL);
+
+		thread_migrate_arena(tsd, oldarena, newarena);
+	}
+}
+
+/* Choose an arena based on a per-thread value. */
+static inline arena_t *
+arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
+	arena_t *ret;
+
+	if (arena != NULL) {
+		return arena;
+	}
+
+	/* During reentrancy, arena 0 is the safest bet. */
+	if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
+		return arena_get(tsd_tsdn(tsd), 0, true);
+	}
+
+	ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
+	if (unlikely(ret == NULL)) {
+		ret = arena_choose_hard(tsd, internal);
+		assert(ret);
+		if (tcache_available(tsd)) {
+			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
+			if (tcache_slow->arena != NULL) {
+				/* See comments in tsd_tcache_data_init().*/
+				assert(tcache_slow->arena
+				    == arena_get(tsd_tsdn(tsd), 0, false));
+				if (tcache_slow->arena != ret) {
+					tcache_arena_reassociate(tsd_tsdn(tsd),
+					    tcache_slow, ret);
+				}
+			} else {
+				tcache_arena_associate(
+				    tsd_tsdn(tsd), tcache_slow, ret);
+			}
+		}
+	}
+
+	/*
+	 * Note that for percpu arena, if the current arena is outside of the
+	 * auto percpu arena range, (i.e. thread is assigned to a manually
+	 * managed arena), then percpu arena is skipped.
+	 */
+	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
+	    && !internal
+	    && (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
+	    && (ret->last_thd != tsd_tsdn(tsd))) {
+		unsigned ind = percpu_arena_choose();
+		if (arena_ind_get(ret) != ind) {
+			percpu_arena_update(tsd, ind);
+			ret = tsd_arena_get(tsd);
+		}
+		ret->last_thd = tsd_tsdn(tsd);
+	}
+
+	return ret;
+}
+
+static inline arena_t *
+arena_choose(tsd_t *tsd, arena_t *arena) {
+	return arena_choose_impl(tsd, arena, false);
+}
+
+static inline arena_t *
+arena_ichoose(tsd_t *tsd, arena_t *arena) {
+	return arena_choose_impl(tsd, arena, true);
+}
+
+JEMALLOC_ALWAYS_INLINE arena_t *
+arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
+	if (arena != NULL) {
+		return arena;
+	}
+
+	/*
+	 * For huge allocations, use the dedicated huge arena if both are true:
+	 * 1) is using auto arena selection (i.e. arena == NULL), and 2) the
+	 * thread is not assigned to a manual arena.
+	 */
+	arena_t *tsd_arena = tsd_arena_get(tsd);
+	if (tsd_arena == NULL) {
+		tsd_arena = arena_choose(tsd, NULL);
+	}
+
+	size_t threshold = atomic_load_zu(
+	    &tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
+	if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) {
+		return arena_choose_huge(tsd);
+	}
+
+	return tsd_arena;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
+	if (!config_opt_safety_checks) {
+		return false;
+	}
+
+	/*
+	 * Eagerly detect double free and sized dealloc bugs for large sizes.
+	 * The cost is low enough (as edata will be accessed anyway) to be
+	 * enabled all the time.
+	 */
+	if (unlikely(edata == NULL
+	        || edata_state_get(edata) != extent_state_active)) {
+		safety_check_fail(
+		    "Invalid deallocation detected: "
+		    "pages being freed (%p) not currently active, "
+		    "possibly caused by double free bugs.",
+		    ptr);
+		return true;
+	}
+	if (unlikely(input_size != edata_usize_get(edata)
+	        || input_size > SC_LARGE_MAXCLASS)) {
+		safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
+		    /* true_size */ edata_usize_get(edata), input_size);
+		return true;
+	}
+
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
+    prof_info_t *prof_info, bool reset_recent) {
+	cassert(config_prof);
+	assert(ptr != NULL);
+	assert(prof_info != NULL);
+
+	edata_t *edata = NULL;
+	bool     is_slab;
+
+	/* Static check. */
+	if (alloc_ctx == NULL) {
+		edata = emap_edata_lookup(
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
+		is_slab = edata_slab_get(edata);
+	} else if (unlikely(!(is_slab = alloc_ctx->slab))) {
+		edata = emap_edata_lookup(
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
+	}
+
+	if (unlikely(!is_slab)) {
+		/* edata must have been initialized at this point. */
+		assert(edata != NULL);
+		size_t usize = (alloc_ctx == NULL)
+		    ? edata_usize_get(edata)
+		    : emap_alloc_ctx_usize_get(alloc_ctx);
+		if (reset_recent
+		    && large_dalloc_safety_checks(edata, ptr, usize)) {
+			prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
+			return;
+		}
+		large_prof_info_get(tsd, edata, prof_info, reset_recent);
+	} else {
+		prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
+		/*
+		 * No need to set other fields in prof_info; they will never be
+		 * accessed if alloc_tctx == PROF_TCTX_SENTINEL.
+		 */
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_tctx_reset(
+    tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
+	cassert(config_prof);
+	assert(ptr != NULL);
+
+	/* Static check. */
+	if (alloc_ctx == NULL) {
+		edata_t *edata = emap_edata_lookup(
+		    tsd_tsdn(tsd), &arena_emap_global, ptr);
+		if (unlikely(!edata_slab_get(edata))) {
+			large_prof_tctx_reset(edata);
+		}
+	} else {
+		if (unlikely(!alloc_ctx->slab)) {
+			edata_t *edata = emap_edata_lookup(
+			    tsd_tsdn(tsd), &arena_emap_global, ptr);
+			large_prof_tctx_reset(edata);
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
+	cassert(config_prof);
+	assert(ptr != NULL);
+
+	edata_t *edata = emap_edata_lookup(
+	    tsd_tsdn(tsd), &arena_emap_global, ptr);
+	assert(!edata_slab_get(edata));
+
+	large_prof_tctx_reset(edata);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_prof_info_set(
+    tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
+	cassert(config_prof);
+
+	assert(!edata_slab_get(edata));
+	large_prof_info_set(edata, tctx, size);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
+	if (unlikely(tsdn_null(tsdn))) {
+		return;
+	}
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	/*
+	 * We use the ticker_geom_t to avoid having per-arena state in the tsd.
+	 * Instead of having a countdown-until-decay timer running for every
+	 * arena in every thread, we flip a coin once per tick, whose
+	 * probability of coming up heads is 1/nticks; this is effectively the
+	 * operation of the ticker_geom_t.  Each arena has the same chance of a
+	 * coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can
+	 * use a single ticker for all of them.
+	 */
+	ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
+	uint64_t      *prng_state = tsd_prng_statep_get(tsd);
+	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
+	        tsd_reentrancy_level_get(tsd) > 0))) {
+		arena_decay(tsdn, arena, false, false);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
+	arena_decay_ticks(tsdn, arena, 1);
+}
+
+JEMALLOC_ALWAYS_INLINE arena_t *
+arena_aalloc(tsdn_t *tsdn, const void *ptr) {
+	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	unsigned arena_ind = edata_arena_ind_get(edata);
+	return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_RELAXED);
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
+arena_salloc(tsdn_t *tsdn, const void *ptr) {
+	assert(ptr != NULL);
+	emap_alloc_ctx_t alloc_ctx;
+	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
+	assert(alloc_ctx.szind != SC_NSIZES);
+
+	return emap_alloc_ctx_usize_get(&alloc_ctx);
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
+arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
+	/*
+	 * Return 0 if ptr is not within an extent managed by jemalloc.  This
+	 * function has two extra costs relative to isalloc():
+	 * - The rtree calls cannot claim to be dependent lookups, which induces
+	 *   rtree lookup load dependencies.
+	 * - The lookup may fail, so there is an extra branch to check for
+	 *   failure.
+	 */
+
+	emap_full_alloc_ctx_t full_alloc_ctx;
+	bool                  missing = emap_full_alloc_ctx_try_lookup(
+            tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
+	if (missing) {
+		return 0;
+	}
+
+	if (full_alloc_ctx.edata == NULL) {
+		return 0;
+	}
+	assert(edata_state_get(full_alloc_ctx.edata) == extent_state_active);
+	/* Only slab members should be looked up via interior pointers. */
+	assert(edata_addr_get(full_alloc_ctx.edata) == ptr
+	    || edata_slab_get(full_alloc_ctx.edata));
+
+	assert(full_alloc_ctx.szind != SC_NSIZES);
+
+	return edata_usize_get(full_alloc_ctx.edata);
+}
+
+static inline void
+arena_cache_oblivious_randomize(
+    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
+	assert(edata_base_get(edata) == edata_addr_get(edata));
+
+	if (alignment < PAGE) {
+		unsigned lg_range = LG_PAGE
+		    - lg_floor(CACHELINE_CEILING(alignment));
+		size_t r;
+		if (!tsdn_null(tsdn)) {
+			tsd_t *tsd = tsdn_tsd(tsdn);
+			r = (size_t)prng_lg_range_u64(
+			    tsd_prng_statep_get(tsd), lg_range);
+		} else {
+			uint64_t stack_value = (uint64_t)(uintptr_t)&r;
+			r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
+		}
+		uintptr_t random_offset = ((uintptr_t)r)
+		    << (LG_PAGE - lg_range);
+		edata->e_addr = (void *)((byte_t *)edata->e_addr
+		    + random_offset);
+		assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
+		    == edata->e_addr);
+	}
+}
+
+static inline bin_t *
+arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
+	bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]);
+	return shard0 + binshard;
+}
+
+#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_H */

include/jemalloc/internal/arena_inlines_a.h

@@ -1,27 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H
-#define JEMALLOC_INTERNAL_ARENA_INLINES_A_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_structs.h"
-
-static inline unsigned
-arena_ind_get(const arena_t *arena) {
-	return arena->ind;
-}
-
-static inline void
-arena_internal_add(arena_t *arena, size_t size) {
-	atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
-}
-
-static inline void
-arena_internal_sub(arena_t *arena, size_t size) {
-	atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
-}
-
-static inline size_t
-arena_internal_get(arena_t *arena) {
-	return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
-}
-
-#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */

include/jemalloc/internal/arena_inlines_b.h

@@ -1,538 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_B_H
-#define JEMALLOC_INTERNAL_ARENA_INLINES_B_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/arena_structs.h"
-#include "jemalloc/internal/bin_inlines.h"
-#include "jemalloc/internal/div.h"
-#include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
-#include "jemalloc/internal/jemalloc_internal_types.h"
-#include "jemalloc/internal/large_externs.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/prof_externs.h"
-#include "jemalloc/internal/prof_structs.h"
-#include "jemalloc/internal/rtree.h"
-#include "jemalloc/internal/safety_check.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_inlines.h"
-#include "jemalloc/internal/ticker.h"
-
-static inline arena_t *
-arena_get_from_edata(edata_t *edata) {
-	return (arena_t *)atomic_load_p(
-	    &arenas[edata_arena_ind_get(edata)], ATOMIC_RELAXED);
-}
-
-JEMALLOC_ALWAYS_INLINE arena_t *
-arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) {
-	if (arena != NULL) {
-		return arena;
-	}
-
-	/*
-	 * For huge allocations, use the dedicated huge arena if both are true:
-	 * 1) is using auto arena selection (i.e. arena == NULL), and 2) the
-	 * thread is not assigned to a manual arena.
-	 */
-	arena_t *tsd_arena = tsd_arena_get(tsd);
-	if (tsd_arena == NULL) {
-		tsd_arena = arena_choose(tsd, NULL);
-	}
-
-	size_t threshold = atomic_load_zu(
-	    &tsd_arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
-	if (unlikely(size >= threshold) && arena_is_auto(tsd_arena)) {
-		return arena_choose_huge(tsd);
-	}
-
-	return tsd_arena;
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-large_dalloc_safety_checks(edata_t *edata, const void *ptr, size_t input_size) {
-	if (!config_opt_safety_checks) {
-		return false;
-	}
-
-	/*
-	 * Eagerly detect double free and sized dealloc bugs for large sizes.
-	 * The cost is low enough (as edata will be accessed anyway) to be
-	 * enabled all the time.
-	 */
-	if (unlikely(edata == NULL
-	        || edata_state_get(edata) != extent_state_active)) {
-		safety_check_fail(
-		    "Invalid deallocation detected: "
-		    "pages being freed (%p) not currently active, "
-		    "possibly caused by double free bugs.",
-		    ptr);
-		return true;
-	}
-	if (unlikely(input_size != edata_usize_get(edata)
-	        || input_size > SC_LARGE_MAXCLASS)) {
-		safety_check_fail_sized_dealloc(/* current_dealloc */ true, ptr,
-		    /* true_size */ edata_usize_get(edata), input_size);
-		return true;
-	}
-
-	return false;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_info_get(tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx,
-    prof_info_t *prof_info, bool reset_recent) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-	assert(prof_info != NULL);
-
-	edata_t *edata = NULL;
-	bool     is_slab;
-
-	/* Static check. */
-	if (alloc_ctx == NULL) {
-		edata = emap_edata_lookup(
-		    tsd_tsdn(tsd), &arena_emap_global, ptr);
-		is_slab = edata_slab_get(edata);
-	} else if (unlikely(!(is_slab = alloc_ctx->slab))) {
-		edata = emap_edata_lookup(
-		    tsd_tsdn(tsd), &arena_emap_global, ptr);
-	}
-
-	if (unlikely(!is_slab)) {
-		/* edata must have been initialized at this point. */
-		assert(edata != NULL);
-		size_t usize = (alloc_ctx == NULL)
-		    ? edata_usize_get(edata)
-		    : emap_alloc_ctx_usize_get(alloc_ctx);
-		if (reset_recent
-		    && large_dalloc_safety_checks(edata, ptr, usize)) {
-			prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
-			return;
-		}
-		large_prof_info_get(tsd, edata, prof_info, reset_recent);
-	} else {
-		prof_info->alloc_tctx = PROF_TCTX_SENTINEL;
-		/*
-		 * No need to set other fields in prof_info; they will never be
-		 * accessed if alloc_tctx == PROF_TCTX_SENTINEL.
-		 */
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_tctx_reset(
-    tsd_t *tsd, const void *ptr, emap_alloc_ctx_t *alloc_ctx) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-
-	/* Static check. */
-	if (alloc_ctx == NULL) {
-		edata_t *edata = emap_edata_lookup(
-		    tsd_tsdn(tsd), &arena_emap_global, ptr);
-		if (unlikely(!edata_slab_get(edata))) {
-			large_prof_tctx_reset(edata);
-		}
-	} else {
-		if (unlikely(!alloc_ctx->slab)) {
-			edata_t *edata = emap_edata_lookup(
-			    tsd_tsdn(tsd), &arena_emap_global, ptr);
-			large_prof_tctx_reset(edata);
-		}
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_tctx_reset_sampled(tsd_t *tsd, const void *ptr) {
-	cassert(config_prof);
-	assert(ptr != NULL);
-
-	edata_t *edata = emap_edata_lookup(
-	    tsd_tsdn(tsd), &arena_emap_global, ptr);
-	assert(!edata_slab_get(edata));
-
-	large_prof_tctx_reset(edata);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_prof_info_set(
-    tsd_t *tsd, edata_t *edata, prof_tctx_t *tctx, size_t size) {
-	cassert(config_prof);
-
-	assert(!edata_slab_get(edata));
-	large_prof_info_set(edata, tctx, size);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) {
-	if (unlikely(tsdn_null(tsdn))) {
-		return;
-	}
-	tsd_t *tsd = tsdn_tsd(tsdn);
-	/*
-	 * We use the ticker_geom_t to avoid having per-arena state in the tsd.
-	 * Instead of having a countdown-until-decay timer running for every
-	 * arena in every thread, we flip a coin once per tick, whose
-	 * probability of coming up heads is 1/nticks; this is effectively the
-	 * operation of the ticker_geom_t.  Each arena has the same chance of a
-	 * coinflip coming up heads (1/ARENA_DECAY_NTICKS_PER_UPDATE), so we can
-	 * use a single ticker for all of them.
-	 */
-	ticker_geom_t *decay_ticker = tsd_arena_decay_tickerp_get(tsd);
-	uint64_t      *prng_state = tsd_prng_statep_get(tsd);
-	if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks,
-	        tsd_reentrancy_level_get(tsd) > 0))) {
-		arena_decay(tsdn, arena, false, false);
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
-	arena_decay_ticks(tsdn, arena, 1);
-}
-
-JEMALLOC_ALWAYS_INLINE void *
-arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
-    bool slab, tcache_t *tcache, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-
-	if (likely(tcache != NULL)) {
-		if (likely(slab)) {
-			assert(sz_can_use_slab(size));
-			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
-			    size, ind, zero, slow_path);
-		} else if (likely(ind < tcache_nbins_get(tcache->tcache_slow)
-		               && !tcache_bin_disabled(ind, &tcache->bins[ind],
-		                   tcache->tcache_slow))) {
-			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
-			    size, ind, zero, slow_path);
-		}
-		/* (size > tcache_max) case falls through. */
-	}
-
-	return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
-}
-
-JEMALLOC_ALWAYS_INLINE arena_t *
-arena_aalloc(tsdn_t *tsdn, const void *ptr) {
-	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
-	unsigned arena_ind = edata_arena_ind_get(edata);
-	return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_RELAXED);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_salloc(tsdn_t *tsdn, const void *ptr) {
-	assert(ptr != NULL);
-	emap_alloc_ctx_t alloc_ctx;
-	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
-	assert(alloc_ctx.szind != SC_NSIZES);
-
-	return emap_alloc_ctx_usize_get(&alloc_ctx);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
-	/*
-	 * Return 0 if ptr is not within an extent managed by jemalloc.  This
-	 * function has two extra costs relative to isalloc():
-	 * - The rtree calls cannot claim to be dependent lookups, which induces
-	 *   rtree lookup load dependencies.
-	 * - The lookup may fail, so there is an extra branch to check for
-	 *   failure.
-	 */
-
-	emap_full_alloc_ctx_t full_alloc_ctx;
-	bool                  missing = emap_full_alloc_ctx_try_lookup(
-            tsdn, &arena_emap_global, ptr, &full_alloc_ctx);
-	if (missing) {
-		return 0;
-	}
-
-	if (full_alloc_ctx.edata == NULL) {
-		return 0;
-	}
-	assert(edata_state_get(full_alloc_ctx.edata) == extent_state_active);
-	/* Only slab members should be looked up via interior pointers. */
-	assert(edata_addr_get(full_alloc_ctx.edata) == ptr
-	    || edata_slab_get(full_alloc_ctx.edata));
-
-	assert(full_alloc_ctx.szind != SC_NSIZES);
-
-	return edata_usize_get(full_alloc_ctx.edata);
-}
-
-static inline void
-arena_dalloc_large_no_tcache(
-    tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
-	/*
-	 * szind is still needed in this function mainly becuase
-	 * szind < SC_NBINS determines not only if this is a small alloc,
-	 * but also if szind is valid (an inactive extent would have
-	 * szind == SC_NSIZES).
-	 */
-	if (config_prof && unlikely(szind < SC_NBINS)) {
-		arena_dalloc_promoted(tsdn, ptr, NULL, true);
-	} else {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		if (large_dalloc_safety_checks(edata, ptr, usize)) {
-			/* See the comment in isfree. */
-			return;
-		}
-		large_dalloc(tsdn, edata);
-	}
-}
-
-static inline void
-arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
-	assert(ptr != NULL);
-
-	emap_alloc_ctx_t alloc_ctx;
-	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.szind < SC_NSIZES);
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		arena_dalloc_small(tsdn, ptr);
-	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx));
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
-    size_t usize, bool slow_path) {
-	assert(!tsdn_null(tsdn) && tcache != NULL);
-	bool is_sample_promoted = config_prof && szind < SC_NBINS;
-	if (unlikely(is_sample_promoted)) {
-		arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
-	} else {
-		if (szind < tcache_nbins_get(tcache->tcache_slow)
-		    && !tcache_bin_disabled(
-		        szind, &tcache->bins[szind], tcache->tcache_slow)) {
-			tcache_dalloc_large(
-			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
-		} else {
-			edata_t *edata = emap_edata_lookup(
-			    tsdn, &arena_emap_global, ptr);
-			if (large_dalloc_safety_checks(edata, ptr, usize)) {
-				/* See the comment in isfree. */
-				return;
-			}
-			large_dalloc(tsdn, edata);
-		}
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
-	if (!config_debug) {
-		return false;
-	}
-	edata_t   *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
-	szind_t    binind = edata_szind_get(edata);
-	div_info_t div_info = arena_binind_div_info[binind];
-	/*
-	 * Calls the internal function bin_slab_regind_impl because the
-	 * safety check does not require a lock.
-	 */
-	size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
-	slab_data_t      *slab_data = edata_slab_data_get(edata);
-	const bin_info_t *bin_info = &bin_infos[binind];
-	assert(edata_nfree_get(edata) < bin_info->nregs);
-	if (unlikely(!bitmap_get(
-	        slab_data->bitmap, &bin_info->bitmap_info, regind))) {
-		safety_check_fail(
-		    "Invalid deallocation detected: the pointer being freed (%p) not "
-		    "currently active, possibly caused by double free bugs.\n",
-		    ptr);
-		return true;
-	}
-	return false;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
-    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-	assert(ptr != NULL);
-
-	if (unlikely(tcache == NULL)) {
-		arena_dalloc_no_tcache(tsdn, ptr);
-		return;
-	}
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (caller_alloc_ctx != NULL) {
-		alloc_ctx = *caller_alloc_ctx;
-	} else {
-		util_assume(tsdn != NULL);
-		emap_alloc_ctx_lookup(
-		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-	}
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.szind < SC_NSIZES);
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-			return;
-		}
-		tcache_dalloc_small(
-		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
-	} else {
-		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
-	}
-}
-
-static inline void
-arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
-	assert(ptr != NULL);
-	assert(size <= SC_LARGE_MAXCLASS);
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (!config_prof || !opt_prof) {
-		/*
-		 * There is no risk of being confused by a promoted sampled
-		 * object, so base szind and slab on the given size.
-		 */
-		szind_t szind = sz_size2index(size);
-		emap_alloc_ctx_init(
-		    &alloc_ctx, szind, (szind < SC_NBINS), size);
-	}
-
-	if ((config_prof && opt_prof) || config_debug) {
-		emap_alloc_ctx_lookup(
-		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
-		assert(alloc_ctx.szind == sz_size2index(size));
-		assert((config_prof && opt_prof)
-		    || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
-
-		if (config_debug) {
-			edata_t *edata = emap_edata_lookup(
-			    tsdn, &arena_emap_global, ptr);
-			assert(alloc_ctx.szind == edata_szind_get(edata));
-			assert(alloc_ctx.slab == edata_slab_get(edata));
-		}
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		arena_dalloc_small(tsdn, ptr);
-	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx));
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
-    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-	assert(ptr != NULL);
-	assert(size <= SC_LARGE_MAXCLASS);
-
-	if (unlikely(tcache == NULL)) {
-		arena_sdalloc_no_tcache(tsdn, ptr, size);
-		return;
-	}
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (config_prof && opt_prof) {
-		if (caller_alloc_ctx == NULL) {
-			/* Uncommon case and should be a static check. */
-			emap_alloc_ctx_lookup(
-			    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-			assert(alloc_ctx.szind == sz_size2index(size));
-			assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
-		} else {
-			alloc_ctx = *caller_alloc_ctx;
-		}
-	} else {
-		/*
-		 * There is no risk of being confused by a promoted sampled
-		 * object, so base szind and slab on the given size.
-		 */
-		alloc_ctx.szind = sz_size2index(size);
-		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
-	}
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		emap_alloc_ctx_init(
-		    &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-			return;
-		}
-		tcache_dalloc_small(
-		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
-	} else {
-		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    sz_s2u(size), slow_path);
-	}
-}
-
-static inline void
-arena_cache_oblivious_randomize(
-    tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
-	assert(edata_base_get(edata) == edata_addr_get(edata));
-
-	if (alignment < PAGE) {
-		unsigned lg_range = LG_PAGE
-		    - lg_floor(CACHELINE_CEILING(alignment));
-		size_t r;
-		if (!tsdn_null(tsdn)) {
-			tsd_t *tsd = tsdn_tsd(tsdn);
-			r = (size_t)prng_lg_range_u64(
-			    tsd_prng_statep_get(tsd), lg_range);
-		} else {
-			uint64_t stack_value = (uint64_t)(uintptr_t)&r;
-			r = (size_t)prng_lg_range_u64(&stack_value, lg_range);
-		}
-		uintptr_t random_offset = ((uintptr_t)r)
-		    << (LG_PAGE - lg_range);
-		edata->e_addr = (void *)((byte_t *)edata->e_addr
-		    + random_offset);
-		assert(ALIGNMENT_ADDR2BASE(edata->e_addr, alignment)
-		    == edata->e_addr);
-	}
-}
-
-static inline bin_t *
-arena_get_bin(arena_t *arena, szind_t binind, unsigned binshard) {
-	bin_t *shard0 = (bin_t *)((byte_t *)arena + arena_bin_offsets[binind]);
-	return shard0 + binshard;
-}
-
-#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */

include/jemalloc/internal/arena_structs.h

@@ -1,111 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_H
-#define JEMALLOC_INTERNAL_ARENA_STRUCTS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_stats.h"
-#include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/bin.h"
-#include "jemalloc/internal/bitmap.h"
-#include "jemalloc/internal/counter.h"
-#include "jemalloc/internal/ecache.h"
-#include "jemalloc/internal/edata_cache.h"
-#include "jemalloc/internal/extent_dss.h"
-#include "jemalloc/internal/jemalloc_internal_types.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/nstime.h"
-#include "jemalloc/internal/pa.h"
-#include "jemalloc/internal/ql.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/ticker.h"
-
-struct arena_s {
-	/*
-	 * Number of threads currently assigned to this arena.  Each thread has
-	 * two distinct assignments, one for application-serving allocation, and
-	 * the other for internal metadata allocation.  Internal metadata must
-	 * not be allocated from arenas explicitly created via the arenas.create
-	 * mallctl, because the arena.<i>.reset mallctl indiscriminately
-	 * discards all allocations for the affected arena.
-	 *
-	 *   0: Application allocation.
-	 *   1: Internal metadata allocation.
-	 *
-	 * Synchronization: atomic.
-	 */
-	atomic_u_t nthreads[2];
-
-	/* Next bin shard for binding new threads. Synchronization: atomic. */
-	atomic_u_t binshard_next;
-
-	/*
-	 * When percpu_arena is enabled, to amortize the cost of reading /
-	 * updating the current CPU id, track the most recent thread accessing
-	 * this arena, and only read CPU if there is a mismatch.
-	 */
-	tsdn_t *last_thd;
-
-	/* Synchronization: internal. */
-	arena_stats_t stats;
-
-	/*
-	 * Lists of tcaches and cache_bin_array_descriptors for extant threads
-	 * associated with this arena.  Stats from these are merged
-	 * incrementally, and at exit if opt_stats_print is enabled.
-	 *
-	 * Synchronization: tcache_ql_mtx.
-	 */
-	ql_head(tcache_slow_t) tcache_ql;
-	ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql;
-	malloc_mutex_t tcache_ql_mtx;
-
-	/*
-	 * Represents a dss_prec_t, but atomically.
-	 *
-	 * Synchronization: atomic.
-	 */
-	atomic_u_t dss_prec;
-
-	/*
-	 * Extant large allocations.
-	 *
-	 * Synchronization: large_mtx.
-	 */
-	edata_list_active_t large;
-	/* Synchronizes all large allocation/update/deallocation. */
-	malloc_mutex_t large_mtx;
-
-	/* The page-level allocator shard this arena uses. */
-	pa_shard_t pa_shard;
-
-	/*
-	 * A cached copy of base->ind.  This can get accessed on hot paths;
-	 * looking it up in base requires an extra pointer hop / cache miss.
-	 */
-	unsigned ind;
-
-	/*
-	 * Base allocator, from which arena metadata are allocated.
-	 *
-	 * Synchronization: internal.
-	 */
-	base_t *base;
-	/* Used to determine uptime.  Read-only after initialization. */
-	nstime_t create_time;
-
-	/* The name of the arena. */
-	char name[ARENA_NAME_LEN];
-
-	/*
-	 * The arena is allocated alongside its bins; really this is a
-	 * dynamically sized array determined by the binshard settings.
-	 * Enforcing cacheline-alignment to minimize the number of cachelines
-	 * touched on the hot paths.
-	 */
-	JEMALLOC_WARN_ON_USAGE(
-	    "Do not use this field directly. "
-	    "Use `arena_get_bin` instead.")
-	JEMALLOC_ALIGNED(CACHELINE)
-	bin_t all_bins[0];
-};
-
-#endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_H */

include/jemalloc/internal/arena_types.h

@@ -1,60 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H
-#define JEMALLOC_INTERNAL_ARENA_TYPES_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/sc.h"
-
-/* Default decay times in milliseconds. */
-#define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000)
-#define MUZZY_DECAY_MS_DEFAULT (0)
-/* Number of event ticks between time checks. */
-#define ARENA_DECAY_NTICKS_PER_UPDATE 1000
-/* Maximum length of the arena name. */
-#define ARENA_NAME_LEN 32
-
-typedef struct arena_s arena_t;
-
-typedef enum {
-	percpu_arena_mode_names_base = 0, /* Used for options processing. */
-
-	/*
-	 * *_uninit are used only during bootstrapping, and must correspond
-	 * to initialized variant plus percpu_arena_mode_enabled_base.
-	 */
-	percpu_arena_uninit = 0,
-	per_phycpu_arena_uninit = 1,
-
-	/* All non-disabled modes must come after percpu_arena_disabled. */
-	percpu_arena_disabled = 2,
-
-	percpu_arena_mode_names_limit = 3, /* Used for options processing. */
-	percpu_arena_mode_enabled_base = 3,
-
-	percpu_arena = 3,
-	per_phycpu_arena = 4 /* Hyper threads share arena. */
-} percpu_arena_mode_t;
-
-#define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base)
-#define PERCPU_ARENA_DEFAULT percpu_arena_disabled
-
-/*
- * When allocation_size >= oversize_threshold, use the dedicated huge arena
- * (unless have explicitly spicified arena index).  0 disables the feature.
- */
-#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20)
-
-struct arena_config_s {
-	/* extent hooks to be used for the arena */
-	extent_hooks_t *extent_hooks;
-
-	/*
-	 * Use extent hooks for metadata (base) allocations when true.
-	 */
-	bool metadata_use_hooks;
-};
-
-typedef struct arena_config_s arena_config_t;
-
-extern const arena_config_t arena_config_default;
-
-#endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */

include/jemalloc/internal/arenas_management.h

@@ -0,0 +1,48 @@
+#ifndef JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
+#define JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H
+
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/*
+ * Arenas that are used to service external requests.  Not all elements of the
+ * arenas array are necessarily used; arenas are created lazily as needed.
+ *
+ * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
+ * arenas.  arenas[narenas_auto..narenas_total) are only used if the application
+ * takes some action to create them and allocate from them.
+ *
+ * Points to an arena_t.
+ */
+extern atomic_p_t arenas[];
+
+/* Number of arenas used for automatic multiplexing of threads and arenas. */
+extern unsigned narenas_auto;
+
+/* Base index for manual arenas. */
+extern unsigned manual_arena_base;
+
+void     narenas_total_set(unsigned narenas);
+void     narenas_total_inc(void);
+unsigned narenas_total_get(void);
+void     narenas_auto_set(unsigned n);
+void     manual_arena_base_set(unsigned base);
+
+void    *a0malloc(size_t size);
+void     a0dalloc(void *ptr);
+void    *a0ialloc(size_t size, bool zero, bool is_internal);
+void     a0idalloc(void *ptr, bool is_internal);
+void     arena_set(unsigned ind, arena_t *arena);
+arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
+arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
+void     arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
+void     iarena_cleanup(tsd_t *tsd);
+void     arena_cleanup(tsd_t *tsd);
+
+bool     arenas_management_boot(void);
+void     arenas_management_prefork(tsdn_t *tsdn);
+void     arenas_management_postfork_parent(tsdn_t *tsdn);
+void     arenas_management_postfork_child(tsdn_t *tsdn);
+
+#endif /* JEMALLOC_INTERNAL_ARENAS_MANAGEMENT_H */

include/jemalloc/internal/background_thread.h

@@ -1,11 +1,10 @@
-#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
-#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H
+#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_H
+#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/base.h"
 #include "jemalloc/internal/mutex.h"
 
-/* This file really combines "structs" and "types", but only transitionally. */
-
 #if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
 #	define JEMALLOC_PTHREAD_CREATE_WRAPPER
 #endif
@@ -66,4 +65,33 @@ struct background_thread_stats_s {
 };
 typedef struct background_thread_stats_s background_thread_stats_t;
 
-#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H */
+extern bool                      opt_background_thread;
+extern size_t                    opt_max_background_threads;
+extern malloc_mutex_t            background_thread_lock;
+extern atomic_b_t                background_thread_enabled_state;
+extern size_t                    n_background_threads;
+extern size_t                    max_background_threads;
+extern background_thread_info_t *background_thread_info;
+
+bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
+bool background_threads_enable(tsd_t *tsd);
+bool background_threads_disable(tsd_t *tsd);
+bool background_thread_is_started(background_thread_info_t *info);
+void background_thread_wakeup_early(
+    background_thread_info_t *info, nstime_t *remaining_sleep);
+void background_thread_prefork0(tsdn_t *tsdn);
+void background_thread_prefork1(tsdn_t *tsdn);
+void background_thread_postfork_parent(tsdn_t *tsdn);
+void background_thread_postfork_child(tsdn_t *tsdn);
+bool background_thread_stats_read(
+    tsdn_t *tsdn, background_thread_stats_t *stats);
+void background_thread_ctl_init(tsdn_t *tsdn);
+
+#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
+extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
+    void *(*)(void *), void *__restrict);
+#endif
+bool background_thread_boot0(void);
+bool background_thread_boot1(tsdn_t *tsdn, base_t *base);
+
+#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_H */

include/jemalloc/internal/background_thread_externs.h

@@ -1,38 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
-#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/background_thread_structs.h"
-#include "jemalloc/internal/base.h"
-#include "jemalloc/internal/mutex.h"
-
-extern bool                      opt_background_thread;
-extern size_t                    opt_max_background_threads;
-extern malloc_mutex_t            background_thread_lock;
-extern atomic_b_t                background_thread_enabled_state;
-extern size_t                    n_background_threads;
-extern size_t                    max_background_threads;
-extern background_thread_info_t *background_thread_info;
-
-bool background_thread_create(tsd_t *tsd, unsigned arena_ind);
-bool background_threads_enable(tsd_t *tsd);
-bool background_threads_disable(tsd_t *tsd);
-bool background_thread_is_started(background_thread_info_t *info);
-void background_thread_wakeup_early(
-    background_thread_info_t *info, nstime_t *remaining_sleep);
-void background_thread_prefork0(tsdn_t *tsdn);
-void background_thread_prefork1(tsdn_t *tsdn);
-void background_thread_postfork_parent(tsdn_t *tsdn);
-void background_thread_postfork_child(tsdn_t *tsdn);
-bool background_thread_stats_read(
-    tsdn_t *tsdn, background_thread_stats_t *stats);
-void background_thread_ctl_init(tsdn_t *tsdn);
-
-#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER
-extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *,
-    void *(*)(void *), void *__restrict);
-#endif
-bool background_thread_boot0(void);
-bool background_thread_boot1(tsdn_t *tsdn, base_t *base);
-
-#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H */

include/jemalloc/internal/background_thread_inlines.h

@@ -2,9 +2,9 @@
 #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_INLINES_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_inlines_a.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/background_thread_externs.h"
+#include "jemalloc/internal/background_thread.h"
 
 JEMALLOC_ALWAYS_INLINE bool
 background_thread_enabled(void) {

include/jemalloc/internal/base.h

@@ -106,7 +106,6 @@ base_t *base_new(tsdn_t *tsdn, unsigned ind, const extent_hooks_t *extent_hooks,
     bool metadata_use_hooks);
 void    base_delete(tsdn_t *tsdn, base_t *base);
 ehooks_t       *base_ehooks_get(base_t *base);
-ehooks_t       *base_ehooks_get_for_metadata(base_t *base);
 extent_hooks_t *base_extent_hooks_set(
     base_t *base, extent_hooks_t *extent_hooks);
 void    *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment);

include/jemalloc/internal/bin.h

@@ -4,11 +4,17 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/bin_info.h"
 #include "jemalloc/internal/bin_stats.h"
-#include "jemalloc/internal/bin_types.h"
+#include "jemalloc/internal/div.h"
 #include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/sc.h"
 
+/* Forward decl; only used as a pointer type in bin_choose. */
+typedef struct arena_s arena_t;
+
+#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH)
+#define N_BIN_SHARDS_DEFAULT 1
+
 /*
  * A bin contains a set of extents that are currently being used for slab
  * allocations.
@@ -63,23 +69,13 @@ void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin);
 void bin_postfork_child(tsdn_t *tsdn, bin_t *bin);
 
 /* Slab region allocation. */
-void *bin_slab_reg_alloc(edata_t *slab, const bin_info_t *bin_info);
 void  bin_slab_reg_alloc_batch(
      edata_t *slab, const bin_info_t *bin_info, unsigned cnt, void **ptrs);
 
 /* Slab list management. */
-void     bin_slabs_nonfull_insert(bin_t *bin, edata_t *slab);
-void     bin_slabs_nonfull_remove(bin_t *bin, edata_t *slab);
-edata_t *bin_slabs_nonfull_tryget(bin_t *bin);
-void     bin_slabs_full_insert(bool is_auto, bin_t *bin, edata_t *slab);
 void     bin_slabs_full_remove(bool is_auto, bin_t *bin, edata_t *slab);
 
-/* Slab association / demotion. */
-void bin_dissociate_slab(bool is_auto, edata_t *slab, bin_t *bin);
-void bin_lower_slab(tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
-
 /* Deallocation helpers (called under bin lock). */
-void bin_dalloc_slab_prepare(tsdn_t *tsdn, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_empty(
     tsdn_t *tsdn, bool is_auto, edata_t *slab, bin_t *bin);
 void bin_dalloc_locked_handle_newly_nonempty(
@@ -95,27 +91,18 @@ bool  bin_refill_slabcur_no_fresh_slab(tsdn_t *tsdn, bool is_auto,
 void *bin_malloc_no_fresh_slab(tsdn_t *tsdn, bool is_auto, bin_t *bin,
     szind_t binind);
 
+/* Slab queries. */
+void *bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin);
+
 /* Bin selection. */
 bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind,
     unsigned *binshard_p);
 
-/* Stats. */
-static inline void
-bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
-	malloc_mutex_lock(tsdn, &bin->lock);
-	malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
-	bin_stats_t *stats = &dst_bin_stats->stats_data;
-	stats->nmalloc += bin->stats.nmalloc;
-	stats->ndalloc += bin->stats.ndalloc;
-	stats->nrequests += bin->stats.nrequests;
-	stats->curregs += bin->stats.curregs;
-	stats->nfills += bin->stats.nfills;
-	stats->nflushes += bin->stats.nflushes;
-	stats->nslabs += bin->stats.nslabs;
-	stats->reslabs += bin->stats.reslabs;
-	stats->curslabs += bin->stats.curslabs;
-	stats->nonfull_slabs += bin->stats.nonfull_slabs;
-	malloc_mutex_unlock(tsdn, &bin->lock);
-}
+typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
+struct bin_dalloc_locked_info_s {
+	div_info_t div_info;
+	uint32_t   nregs;
+	uint64_t   ndalloc;
+};
 
 #endif /* JEMALLOC_INTERNAL_BIN_H */

include/jemalloc/internal/bin_inlines.h

@@ -7,25 +7,38 @@
 #include "jemalloc/internal/bitmap.h"
 #include "jemalloc/internal/div.h"
 #include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/mutex.h"
 
-/*
- * The dalloc bin info contains just the information that the common paths need
- * during tcache flushes.  By force-inlining these paths, and using local copies
- * of data (so that the compiler knows it's constant), we avoid a whole bunch of
- * redundant loads and stores by leaving this information in registers.
- */
-typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t;
-struct bin_dalloc_locked_info_s {
-	div_info_t div_info;
-	uint32_t   nregs;
-	uint64_t   ndalloc;
-};
+/* Stats. */
+static inline void
+bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) {
+	malloc_mutex_lock(tsdn, &bin->lock);
+	bin->stats.nrequests += n;
+	malloc_mutex_unlock(tsdn, &bin->lock);
+}
+
+static inline void
+bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) {
+	malloc_mutex_lock(tsdn, &bin->lock);
+	malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock);
+	bin_stats_t *stats = &dst_bin_stats->stats_data;
+	stats->nmalloc += bin->stats.nmalloc;
+	stats->ndalloc += bin->stats.ndalloc;
+	stats->nrequests += bin->stats.nrequests;
+	stats->curregs += bin->stats.curregs;
+	stats->nfills += bin->stats.nfills;
+	stats->nflushes += bin->stats.nflushes;
+	stats->nslabs += bin->stats.nslabs;
+	stats->reslabs += bin->stats.reslabs;
+	stats->curslabs += bin->stats.curslabs;
+	stats->nonfull_slabs += bin->stats.nonfull_slabs;
+	malloc_mutex_unlock(tsdn, &bin->lock);
+}
 
 /* Find the region index of a pointer within a slab. */
 JEMALLOC_ALWAYS_INLINE size_t
-bin_slab_regind_impl(
-    div_info_t *div_info, szind_t binind, edata_t *slab, const void *ptr) {
+bin_slab_regind_impl(const div_info_t *div_info, szind_t binind,
+    const edata_t *slab, const void *ptr) {
 	size_t diff, regind;
 
 	/* Freeing a pointer outside the slab can cause assertion failure. */
@@ -45,21 +58,13 @@ bin_slab_regind_impl(
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-bin_slab_regind(bin_dalloc_locked_info_t *info, szind_t binind,
-    edata_t *slab, const void *ptr) {
+bin_slab_regind(const bin_dalloc_locked_info_t *info, szind_t binind,
+    const edata_t *slab, const void *ptr) {
 	size_t regind = bin_slab_regind_impl(
 	    &info->div_info, binind, slab, ptr);
 	return regind;
 }
 
-JEMALLOC_ALWAYS_INLINE void
-bin_dalloc_locked_begin(
-    bin_dalloc_locked_info_t *info, szind_t binind) {
-	info->div_info = arena_binind_div_info[binind];
-	info->nregs = bin_infos[binind].nregs;
-	info->ndalloc = 0;
-}
-
 /*
  * Does the deallocation work associated with freeing a single pointer (a
  * "step") in between a bin_dalloc_locked begin and end call.

include/jemalloc/internal/bin_types.h

@@ -1,21 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H
-#define JEMALLOC_INTERNAL_BIN_TYPES_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/sc.h"
-
-#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH)
-#define N_BIN_SHARDS_DEFAULT 1
-
-/* Used in TSD static initializer only. Real init in arena_bind(). */
-#define TSD_BINSHARDS_ZERO_INITIALIZER                                         \
-	{                                                                      \
-		{ UINT8_MAX }                                                  \
-	}
-
-typedef struct tsd_binshards_s tsd_binshards_t;
-struct tsd_binshards_s {
-	uint8_t binshard[SC_NBINS];
-};
-
-#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */

include/jemalloc/internal/bit_util.h

@@ -195,7 +195,7 @@ fls_u(unsigned x) {
 	/*								\
 	 * Algorithm from an old AMD optimization reference manual.	\
 	 * We're putting a little bit more work than you might expect	\
-	 * into the no-instrinsic case, since we only support the	\
+	 * into the no-intrinsic case, since we only support the	\
 	 * GCC intrinsics spelling of popcount (for now).  Detecting	\
 	 * whether or not the popcount builtin is actually useable in	\
 	 * MSVC is nontrivial.						\
@@ -212,7 +212,7 @@ fls_u(unsigned x) {
 	 * That costs an extra mask relative to this, though.		\
 	 */								\
 	x = x - ((x >> 1) & (0x55U * bmul));				\
-	/* Replace each 4 bits with their sideays sum.  0x3 = 0b0011. */\
+	/* Replace each 4 bits with their sideways sum.  0x3 = 0b0011. */\
 	x = (x & (bmul * 0x33U)) + ((x >> 2) & (bmul * 0x33U));		\
 	/*								\
 	 * Replace each 8 bits with their sideways sum.  Note that we	\

include/jemalloc/internal/bitmap.h

@@ -176,12 +176,10 @@ typedef struct bitmap_info_s {
 #endif /* BITMAP_USE_TREE */
 } bitmap_info_t;
 
-void   bitmap_info_init(bitmap_info_t *binfo, size_t nbits);
 void   bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo, bool fill);
-size_t bitmap_size(const bitmap_info_t *binfo);
 
 static inline bool
-bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
+bitmap_full(const bitmap_t *bitmap, const bitmap_info_t *binfo) {
 #ifdef BITMAP_USE_TREE
 	size_t   rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
 	bitmap_t rg = bitmap[rgoff];
@@ -200,7 +198,7 @@ bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
 }
 
 static inline bool
-bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
+bitmap_get(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
 	size_t   goff;
 	bitmap_t g;
 

include/jemalloc/internal/buf_writer.h

@@ -25,7 +25,6 @@ typedef struct {
 
 bool       buf_writer_init(tsdn_t *tsdn, buf_writer_t *buf_writer,
           write_cb_t *write_cb, void *cbopaque, char *buf, size_t buf_len);
-void       buf_writer_flush(buf_writer_t *buf_writer);
 write_cb_t buf_writer_cb;
 void       buf_writer_terminate(tsdn_t *tsdn, buf_writer_t *buf_writer);
 

include/jemalloc/internal/cache_bin.h

@@ -442,19 +442,6 @@ cache_bin_alloc(cache_bin_t *bin, bool *success) {
 	return cache_bin_alloc_impl(bin, success, true);
 }
 
-JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
-cache_bin_alloc_batch(cache_bin_t *bin, size_t num, void **out) {
-	cache_bin_sz_t n = cache_bin_ncached_get_internal(bin);
-	if (n > num) {
-		n = (cache_bin_sz_t)num;
-	}
-	memcpy(out, bin->stack_head, n * sizeof(void *));
-	bin->stack_head += n;
-	cache_bin_low_water_adjust(bin);
-
-	return n;
-}
-
 JEMALLOC_ALWAYS_INLINE bool
 cache_bin_full(cache_bin_t *bin) {
 	return (
@@ -536,14 +523,13 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
 /* Get the number of stashed pointers. */
 JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
 cache_bin_nstashed_get_internal(cache_bin_t *bin) {
-	cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
 	cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
 	    bin);
 
 	cache_bin_sz_t n = cache_bin_diff(
 	                       bin, low_bits_low_bound, bin->low_bits_full)
 	    / sizeof(void *);
-	assert(n <= ncached_max);
+	assert(n <= cache_bin_ncached_max_get(bin));
 	if (config_debug && n != 0) {
 		/* Below are for assertions only. */
 		void **low_bound = cache_bin_low_bound_get(bin);

include/jemalloc/internal/ckh.h

@@ -93,10 +93,4 @@ bool ckh_remove(
     tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data);
 bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
 
-/* Some useful hash and comparison functions for strings and pointers. */
-void ckh_string_hash(const void *key, size_t r_hash[2]);
-bool ckh_string_keycomp(const void *k1, const void *k2);
-void ckh_pointer_hash(const void *key, size_t r_hash[2]);
-bool ckh_pointer_keycomp(const void *k1, const void *k2);
-
 #endif /* JEMALLOC_INTERNAL_CKH_H */

include/jemalloc/internal/conf.h

@@ -7,9 +7,9 @@ void malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
     char readlink_buf[PATH_MAX + 1]);
 void malloc_abort_invalid_conf(void);
 
-#ifdef JEMALLOC_JET
 extern bool had_conf_error;
 
+#ifdef JEMALLOC_JET
 bool conf_next(char const **opts_p, char const **k_p, size_t *klen_p,
     char const **v_p, size_t *vlen_p);
 void conf_error(

include/jemalloc/internal/ctl.h

@@ -3,7 +3,7 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/arena_stats.h"
-#include "jemalloc/internal/background_thread_structs.h"
+#include "jemalloc/internal/background_thread.h"
 #include "jemalloc/internal/bin_stats.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/malloc_io.h"
@@ -63,6 +63,7 @@ typedef struct ctl_stats_s {
 	size_t resident;
 	size_t mapped;
 	size_t retained;
+	size_t pinned;
 
 	background_thread_stats_t background_thread;
 	mutex_prof_data_t mutex_prof_data[mutex_prof_num_global_mutexes];

include/jemalloc/internal/div.h

@@ -24,7 +24,7 @@ struct div_info_s {
 void div_init(div_info_t *div_info, size_t divisor);
 
 static inline size_t
-div_compute(div_info_t *div_info, size_t n) {
+div_compute(const div_info_t *div_info, size_t n) {
 	assert(n <= (uint32_t)-1);
 	/*
 	 * This generates, e.g. mov; imul; shr on x86-64. On a 32-bit machine,

include/jemalloc/internal/ecache.h

@@ -23,27 +23,27 @@ struct ecache_s {
 };
 
 static inline size_t
-ecache_npages_get(ecache_t *ecache) {
+ecache_npages_get(const ecache_t *ecache) {
 	return eset_npages_get(&ecache->eset)
 	    + eset_npages_get(&ecache->guarded_eset);
 }
 
 /* Get the number of extents in the given page size index. */
 static inline size_t
-ecache_nextents_get(ecache_t *ecache, pszind_t ind) {
+ecache_nextents_get(const ecache_t *ecache, pszind_t ind) {
 	return eset_nextents_get(&ecache->eset, ind)
 	    + eset_nextents_get(&ecache->guarded_eset, ind);
 }
 
 /* Get the sum total bytes of the extents in the given page size index. */
 static inline size_t
-ecache_nbytes_get(ecache_t *ecache, pszind_t ind) {
+ecache_nbytes_get(const ecache_t *ecache, pszind_t ind) {
 	return eset_nbytes_get(&ecache->eset, ind)
 	    + eset_nbytes_get(&ecache->guarded_eset, ind);
 }
 
 static inline unsigned
-ecache_ind_get(ecache_t *ecache) {
+ecache_ind_get(const ecache_t *ecache) {
 	return ecache->ind;
 }
 

include/jemalloc/internal/edata.h

@@ -8,13 +8,16 @@
 #include "jemalloc/internal/hpdata.h"
 #include "jemalloc/internal/nstime.h"
 #include "jemalloc/internal/ph.h"
-#include "jemalloc/internal/prof_types.h"
 #include "jemalloc/internal/ql.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/slab_data.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/typed_list.h"
 
+/* Opaque to edata; only stored as pointers in e_prof_info_t. */
+typedef struct prof_tctx_s   prof_tctx_t;
+typedef struct prof_recent_s prof_recent_t;
+
 /*
  * sizeof(edata_t) is 128 bytes on 64-bit architectures.  Ensure the alignment
  * to free up the low bits in the rtree leaf.
@@ -34,9 +37,10 @@ enum extent_state_e {
 	extent_state_dirty = 1,
 	extent_state_muzzy = 2,
 	extent_state_retained = 3,
-	extent_state_transition = 4, /* States below are intermediate. */
-	extent_state_merging = 5,
-	extent_state_max = 5 /* Sanity checking only. */
+	extent_state_pinned = 4,
+	extent_state_transition = 5, /* States below are intermediate. */
+	extent_state_merging = 6,
+	extent_state_max = 6 /* Sanity checking only. */
 };
 typedef enum extent_state_e extent_state_t;
 
@@ -47,8 +51,8 @@ enum extent_head_state_e {
 typedef enum extent_head_state_e extent_head_state_t;
 
 /*
- * Which implementation of the page allocator interface, (PAI, defined in
- * pai.h) owns the given extent?
+ * Which page allocator implementation (PAC or HPA) owns the given extent?
+ * Used by PA to route expand/shrink/dalloc to the correct implementation.
  */
 enum extent_pai_e { EXTENT_PAI_PAC = 0, EXTENT_PAI_HPA = 1 };
 typedef enum extent_pai_e extent_pai_t;
@@ -94,8 +98,8 @@ struct edata_cmp_summary_s {
 
 /* Extent (span of pages).  Use accessor functions for e_* fields. */
 typedef struct edata_s edata_t;
-ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM);
-ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM);
+ph_structs(edata_avail, edata_t, ESET_ENUMERATE_MAX_NUM)
+ph_structs(edata_heap, edata_t, ESET_ENUMERATE_MAX_NUM)
 struct edata_s {
 	/*
 	 * Bitfield containing several fields:
@@ -110,8 +114,10 @@ struct edata_s {
 	 * i: szind
 	 * f: nfree
 	 * s: bin_shard
+	 * h: is_head
+	 * n: pinned
 	 *
-	 * 00000000 ... 0000ssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
+	 * 00000000 ... 0nhsssss ssffffff ffffiiii iiiitttg zpcbaaaa aaaaaaaa
 	 *
 	 * arena_ind: Arena from which this extent came, or all 1 bits if
 	 *            unassociated.
@@ -145,6 +151,10 @@ struct edata_s {
 	 * nfree: Number of free regions in slab.
 	 *
 	 * bin_shard: the shard of the bin from which this extent came.
+	 *
+	 * is_head: see comments in ehooks_default_merge_impl().
+	 *
+	 * pinned: true if the alloc hook signaled non-reclaimable backing.
 	 */
 	uint64_t e_bits;
 #define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT)                         \
@@ -210,6 +220,16 @@ struct edata_s {
 #define EDATA_BITS_IS_HEAD_MASK                                                \
 	MASK(EDATA_BITS_IS_HEAD_WIDTH, EDATA_BITS_IS_HEAD_SHIFT)
 
+#define EDATA_BITS_PINNED_WIDTH 1
+#define EDATA_BITS_PINNED_SHIFT                                                \
+	(EDATA_BITS_IS_HEAD_WIDTH + EDATA_BITS_IS_HEAD_SHIFT)
+#define EDATA_BITS_PINNED_MASK                                                 \
+	MASK(EDATA_BITS_PINNED_WIDTH, EDATA_BITS_PINNED_SHIFT)
+
+#if (EDATA_BITS_PINNED_SHIFT + EDATA_BITS_PINNED_WIDTH > 64)
+#error "edata_t e_bits overflow"
+#endif
+
 	/* Pointer to the extent that this structure is responsible for. */
 	void *e_addr;
 
@@ -538,6 +558,29 @@ edata_ps_set(edata_t *edata, hpdata_t *ps) {
 	edata->e_ps = ps;
 }
 
+static inline bool
+edata_pinned_get(const edata_t *edata) {
+	return (bool)((edata->e_bits & EDATA_BITS_PINNED_MASK)
+	    >> EDATA_BITS_PINNED_SHIFT);
+}
+
+static inline void
+edata_pinned_set(edata_t *edata, bool pinned) {
+	edata->e_bits = (edata->e_bits & ~EDATA_BITS_PINNED_MASK)
+	    | ((uint64_t)pinned << EDATA_BITS_PINNED_SHIFT);
+}
+
+static inline void
+edata_hook_flags_init(edata_t *edata, unsigned alloc_flags) {
+	edata_pinned_set(edata,
+	    (alloc_flags & EXTENT_ALLOC_FLAG_PINNED) != 0);
+}
+
+static inline unsigned
+edata_alloc_flags_get(const edata_t *edata) {
+	return edata_pinned_get(edata) ? EXTENT_ALLOC_FLAG_PINNED : 0;
+}
+
 static inline void
 edata_szind_set(edata_t *edata, szind_t szind) {
 	assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */
@@ -645,7 +688,7 @@ edata_prof_recent_alloc_set_dont_call_directly(
 }
 
 static inline bool
-edata_is_head_get(edata_t *edata) {
+edata_is_head_get(const edata_t *edata) {
 	return (bool)((edata->e_bits & EDATA_BITS_IS_HEAD_MASK)
 	    >> EDATA_BITS_IS_HEAD_SHIFT);
 }
@@ -686,6 +729,7 @@ edata_init(edata_t *edata, unsigned arena_ind, void *addr, size_t size,
 	edata_committed_set(edata, committed);
 	edata_pai_set(edata, pai);
 	edata_is_head_set(edata, is_head == EXTENT_IS_HEAD);
+	edata_hook_flags_init(edata, 0);
 	if (config_prof) {
 		edata_prof_tctx_set(edata, NULL);
 	}
@@ -711,6 +755,7 @@ edata_binit(
 	 * wasting a state bit to encode this fact.
 	 */
 	edata_pai_set(edata, EXTENT_PAI_PAC);
+	edata_hook_flags_init(edata, 0);
 }
 
 static inline int

include/jemalloc/internal/ehooks.h

@@ -57,13 +57,6 @@ bool ehooks_default_purge_lazy_impl(void *addr, size_t offset, size_t length);
 bool ehooks_default_purge_forced_impl(void *addr, size_t offset, size_t length);
 #endif
 bool ehooks_default_split_impl(void);
-/*
- * Merge is the only default extent hook we declare -- see the comment in
- * ehooks_merge.
- */
-bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a,
-    size_t size_a, void *addr_b, size_t size_b, bool committed,
-    unsigned arena_ind);
 bool ehooks_default_merge_impl(tsdn_t *tsdn, void *addr_a, void *addr_b);
 void ehooks_default_zero_impl(void *addr, size_t size);
 void ehooks_default_guard_impl(void *guard1, void *guard2);
@@ -110,12 +103,12 @@ ehooks_set_extent_hooks_ptr(ehooks_t *ehooks, extent_hooks_t *extent_hooks) {
 }
 
 static inline extent_hooks_t *
-ehooks_get_extent_hooks_ptr(ehooks_t *ehooks) {
+ehooks_get_extent_hooks_ptr(const ehooks_t *ehooks) {
 	return (extent_hooks_t *)atomic_load_p(&ehooks->ptr, ATOMIC_ACQUIRE);
 }
 
 static inline bool
-ehooks_are_default(ehooks_t *ehooks) {
+ehooks_are_default(const ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)
 	    == &ehooks_default_extent_hooks;
 }
@@ -126,7 +119,7 @@ ehooks_are_default(ehooks_t *ehooks) {
  * include some checks for such cases.
  */
 static inline bool
-ehooks_dalloc_will_fail(ehooks_t *ehooks) {
+ehooks_dalloc_will_fail(const ehooks_t *ehooks) {
 	if (ehooks_are_default(ehooks)) {
 		return opt_retain;
 	} else {
@@ -135,17 +128,17 @@ ehooks_dalloc_will_fail(ehooks_t *ehooks) {
 }
 
 static inline bool
-ehooks_split_will_fail(ehooks_t *ehooks) {
+ehooks_split_will_fail(const ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)->split == NULL;
 }
 
 static inline bool
-ehooks_merge_will_fail(ehooks_t *ehooks) {
+ehooks_merge_will_fail(const ehooks_t *ehooks) {
 	return ehooks_get_extent_hooks_ptr(ehooks)->merge == NULL;
 }
 
 static inline bool
-ehooks_guard_will_fail(ehooks_t *ehooks) {
+ehooks_guard_will_fail(const ehooks_t *ehooks) {
 	/*
 	 * Before the guard hooks are officially introduced, limit the use to
 	 * the default hooks only.
@@ -191,7 +184,7 @@ ehooks_debug_zero_check(void *addr, size_t size) {
 
 static inline void *
 ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
-    size_t alignment, bool *zero, bool *commit) {
+    size_t alignment, bool *zero, bool *commit, unsigned *alloc_flags) {
 	bool            orig_zero = *zero;
 	void           *ret;
 	extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks);
@@ -204,6 +197,18 @@ ehooks_alloc(tsdn_t *tsdn, ehooks_t *ehooks, void *new_addr, size_t size,
 		    alignment, zero, commit, ehooks_ind_get(ehooks));
 		ehooks_post_reentrancy(tsdn);
 	}
+#if LG_PAGE < 8
+#  error "Extent alloc flags require page size of at least 256"
+#endif
+	if (ret != NULL) {
+		*alloc_flags = (unsigned)((uintptr_t)ret
+		    & EXTENT_ALLOC_FLAG_MASK);
+		ret = (void *)((byte_t *)ret - *alloc_flags);
+		/* Pinned hooks must also set *commit; pinned bypasses commit/decommit. */
+		assert(!(*alloc_flags & EXTENT_ALLOC_FLAG_PINNED) || *commit);
+	} else {
+		*alloc_flags = 0;
+	}
 	assert(new_addr == NULL || ret == NULL || new_addr == ret);
 	assert(!orig_zero || *zero);
 	if (*zero && ret != NULL) {

include/jemalloc/internal/emap.h

@@ -211,6 +211,7 @@ extent_assert_can_coalesce(const edata_t *inner, const edata_t *outer) {
 	assert(edata_state_get(inner) == extent_state_active);
 	assert(edata_state_get(outer) == extent_state_merging);
 	assert(!edata_guarded_get(inner) && !edata_guarded_get(outer));
+	assert(edata_pinned_get(inner) == edata_pinned_get(outer));
 	assert(edata_base_get(inner) == edata_past_get(outer)
 	    || edata_base_get(outer) == edata_past_get(inner));
 }

include/jemalloc/internal/eset.h

@@ -60,11 +60,11 @@ struct eset_s {
 
 void eset_init(eset_t *eset, extent_state_t state);
 
-size_t eset_npages_get(eset_t *eset);
+size_t eset_npages_get(const eset_t *eset);
 /* Get the number of extents in the given page size index. */
-size_t eset_nextents_get(eset_t *eset, pszind_t ind);
+size_t eset_nextents_get(const eset_t *eset, pszind_t ind);
 /* Get the sum total bytes of the extents in the given page size index. */
-size_t eset_nbytes_get(eset_t *eset, pszind_t ind);
+size_t eset_nbytes_get(const eset_t *eset, pszind_t ind);
 
 void eset_insert(eset_t *eset, edata_t *edata);
 void eset_remove(eset_t *eset, edata_t *edata);
@@ -73,6 +73,6 @@ void eset_remove(eset_t *eset, edata_t *edata);
  * null if no such item could be found.
  */
 edata_t *eset_fit(eset_t *eset, size_t esize, size_t alignment, bool exact_only,
-    unsigned lg_max_fit);
+    unsigned lg_max_fit, bool prefer_small);
 
 #endif /* JEMALLOC_INTERNAL_ESET_H */

include/jemalloc/internal/extent.h

@@ -91,7 +91,7 @@ extent_neighbor_head_state_mergeable(
 }
 
 JEMALLOC_ALWAYS_INLINE bool
-extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
+extent_can_acquire_neighbor(const edata_t *edata, rtree_contents_t contents,
     extent_pai_t pai, extent_state_t expected_state, bool forward,
     bool expanding) {
 	edata_t *neighbor = contents.edata;
@@ -120,6 +120,10 @@ extent_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
 			 */
 			return false;
 		}
+		/* Do not merge pinned and non-pinned extents. */
+		if (edata_pinned_get(edata) != edata_pinned_get(neighbor)) {
+			return false;
+		}
 	} else {
 		if (neighbor_state == extent_state_active) {
 			return false;

include/jemalloc/internal/extent_dss.h

@@ -2,9 +2,11 @@
 #define JEMALLOC_INTERNAL_EXTENT_DSS_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_types.h"
 #include "jemalloc/internal/tsd_types.h"
 
+/* Forward decl; arena.h includes us, so we can't include arena.h back. */
+typedef struct arena_s arena_t;
+
 typedef enum {
 	dss_prec_disabled = 0,
 	dss_prec_primary = 1,
@@ -27,4 +29,9 @@ bool       extent_in_dss(void *addr);
 bool       extent_dss_mergeable(void *addr_a, void *addr_b);
 void       extent_dss_boot(void);
 
+#ifdef JEMALLOC_JET
+typedef void *(*extent_dss_sbrk_hook_t)(intptr_t);
+extern extent_dss_sbrk_hook_t extent_dss_sbrk_hook;
+#endif
+
 #endif /* JEMALLOC_INTERNAL_EXTENT_DSS_H */

include/jemalloc/internal/fb.h

@@ -25,7 +25,7 @@ fb_init(fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_empty(fb_group_t *fb, size_t nbits) {
+fb_empty(const fb_group_t *fb, size_t nbits) {
 	size_t ngroups = FB_NGROUPS(nbits);
 	for (size_t i = 0; i < ngroups; i++) {
 		if (fb[i] != 0) {
@@ -36,7 +36,7 @@ fb_empty(fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_full(fb_group_t *fb, size_t nbits) {
+fb_full(const fb_group_t *fb, size_t nbits) {
 	size_t ngroups = FB_NGROUPS(nbits);
 	size_t trailing_bits = nbits % FB_GROUP_BITS;
 	size_t limit = (trailing_bits == 0 ? ngroups : ngroups - 1);
@@ -52,7 +52,7 @@ fb_full(fb_group_t *fb, size_t nbits) {
 }
 
 static inline bool
-fb_get(fb_group_t *fb, size_t nbits, size_t bit) {
+fb_get(const fb_group_t *fb, size_t nbits, size_t bit) {
 	assert(bit < nbits);
 	size_t group_ind = bit / FB_GROUP_BITS;
 	size_t bit_ind = bit % FB_GROUP_BITS;
@@ -156,15 +156,21 @@ fb_scount_visitor(void *ctx, fb_group_t *fb, fb_group_t mask) {
 
 /* Finds the number of set bit in the of length cnt starting at start. */
 JEMALLOC_ALWAYS_INLINE size_t
-fb_scount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
+fb_scount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
 	size_t scount = 0;
-	fb_visit_impl(fb, nbits, &fb_scount_visitor, &scount, start, cnt);
+	/*
+	 * fb_visit_impl is shared with mutating visitors (e.g. fb_set_range),
+	 * so it takes a non-const fb.  fb_scount_visitor only reads, so the
+	 * cast is safe.
+	 */
+	fb_visit_impl((fb_group_t *)fb, nbits, &fb_scount_visitor, &scount,
+	    start, cnt);
 	return scount;
 }
 
 /* Finds the number of unset bit in the of length cnt starting at start. */
 JEMALLOC_ALWAYS_INLINE size_t
-fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
+fb_ucount(const fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
 	size_t scount = fb_scount(fb, nbits, start, cnt);
 	return cnt - scount;
 }
@@ -176,8 +182,8 @@ fb_ucount(fb_group_t *fb, size_t nbits, size_t start, size_t cnt) {
  * Returns the number of bits in the bitmap if no such bit exists.
  */
 JEMALLOC_ALWAYS_INLINE ssize_t
-fb_find_impl(
-    fb_group_t *fb, size_t nbits, size_t start, bool val, bool forward) {
+fb_find_impl(const fb_group_t *fb, size_t nbits, size_t start, bool val,
+    bool forward) {
 	assert(start < nbits);
 	size_t  ngroups = FB_NGROUPS(nbits);
 	ssize_t group_ind = start / FB_GROUP_BITS;
@@ -226,14 +232,14 @@ fb_find_impl(
  * number of bits in the bitmap if no such bit exists.
  */
 static inline size_t
-fb_ffu(fb_group_t *fb, size_t nbits, size_t min_bit) {
+fb_ffu(const fb_group_t *fb, size_t nbits, size_t min_bit) {
 	return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ false,
 	    /* forward */ true);
 }
 
 /* The same, but looks for an unset bit. */
 static inline size_t
-fb_ffs(fb_group_t *fb, size_t nbits, size_t min_bit) {
+fb_ffs(const fb_group_t *fb, size_t nbits, size_t min_bit) {
 	return (size_t)fb_find_impl(fb, nbits, min_bit, /* val */ true,
 	    /* forward */ true);
 }
@@ -243,21 +249,21 @@ fb_ffs(fb_group_t *fb, size_t nbits, size_t min_bit) {
  * no such bit exists.
  */
 static inline ssize_t
-fb_flu(fb_group_t *fb, size_t nbits, size_t max_bit) {
+fb_flu(const fb_group_t *fb, size_t nbits, size_t max_bit) {
 	return fb_find_impl(fb, nbits, max_bit, /* val */ false,
 	    /* forward */ false);
 }
 
 static inline ssize_t
-fb_fls(fb_group_t *fb, size_t nbits, size_t max_bit) {
+fb_fls(const fb_group_t *fb, size_t nbits, size_t max_bit) {
 	return fb_find_impl(fb, nbits, max_bit, /* val */ true,
 	    /* forward */ false);
 }
 
 /* Returns whether or not we found a range. */
 JEMALLOC_ALWAYS_INLINE bool
-fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len, bool val, bool forward) {
+fb_iter_range_impl(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len, bool val, bool forward) {
 	assert(start < nbits);
 	ssize_t next_range_begin = fb_find_impl(fb, nbits, start, val, forward);
 	if ((forward && next_range_begin == (ssize_t)nbits)
@@ -286,8 +292,8 @@ fb_iter_range_impl(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
  * touching *r_begin or *r_end).
  */
 static inline bool
-fb_srange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_srange_iter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ true, /* forward */ true);
 }
@@ -297,30 +303,30 @@ fb_srange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
  * forwards.  (The position returned is still the earliest bit in the range).
  */
 static inline bool
-fb_srange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_srange_riter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ true, /* forward */ false);
 }
 
 /* Similar to fb_srange_iter, but searches for unset bits. */
 static inline bool
-fb_urange_iter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_urange_iter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ false, /* forward */ true);
 }
 
 /* Similar to fb_srange_riter, but searches for unset bits. */
 static inline bool
-fb_urange_riter(fb_group_t *fb, size_t nbits, size_t start, size_t *r_begin,
-    size_t *r_len) {
+fb_urange_riter(const fb_group_t *fb, size_t nbits, size_t start,
+    size_t *r_begin, size_t *r_len) {
 	return fb_iter_range_impl(fb, nbits, start, r_begin, r_len,
 	    /* val */ false, /* forward */ false);
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
-fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
+fb_range_longest_impl(const fb_group_t *fb, size_t nbits, bool val) {
 	size_t begin = 0;
 	size_t longest_len = 0;
 	size_t len = 0;
@@ -336,12 +342,12 @@ fb_range_longest_impl(fb_group_t *fb, size_t nbits, bool val) {
 }
 
 static inline size_t
-fb_srange_longest(fb_group_t *fb, size_t nbits) {
+fb_srange_longest(const fb_group_t *fb, size_t nbits) {
 	return fb_range_longest_impl(fb, nbits, /* val */ true);
 }
 
 static inline size_t
-fb_urange_longest(fb_group_t *fb, size_t nbits) {
+fb_urange_longest(const fb_group_t *fb, size_t nbits) {
 	return fb_range_longest_impl(fb, nbits, /* val */ false);
 }
 

include/jemalloc/internal/hook.h

@@ -1,163 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_HOOK_H
-#define JEMALLOC_INTERNAL_HOOK_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/tsd.h"
-
-/*
- * This API is *extremely* experimental, and may get ripped out, changed in API-
- * and ABI-incompatible ways, be insufficiently or incorrectly documented, etc.
- *
- * It allows hooking the stateful parts of the API to see changes as they
- * happen.
- *
- * Allocation hooks are called after the allocation is done, free hooks are
- * called before the free is done, and expand hooks are called after the
- * allocation is expanded.
- *
- * For realloc and rallocx, if the expansion happens in place, the expansion
- * hook is called.  If it is moved, then the alloc hook is called on the new
- * location, and then the free hook is called on the old location (i.e. both
- * hooks are invoked in between the alloc and the dalloc).
- *
- * If we return NULL from OOM, then usize might not be trustworthy.  Calling
- * realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0)
- * only calls the free hook.  (Calling realloc(NULL, 0) is treated as malloc(0),
- * and only calls the alloc hook).
- *
- * Reentrancy:
- *   Reentrancy is guarded against from within the hook implementation.  If you
- *   call allocator functions from within a hook, the hooks will not be invoked
- *   again.
- * Threading:
- *   The installation of a hook synchronizes with all its uses.  If you can
- *   prove the installation of a hook happens-before a jemalloc entry point,
- *   then the hook will get invoked (unless there's a racing removal).
- *
- *   Hook insertion appears to be atomic at a per-thread level (i.e. if a thread
- *   allocates and has the alloc hook invoked, then a subsequent free on the
- *   same thread will also have the free hook invoked).
- *
- *   The *removal* of a hook does *not* block until all threads are done with
- *   the hook.  Hook authors have to be resilient to this, and need some
- *   out-of-band mechanism for cleaning up any dynamically allocated memory
- *   associated with their hook.
- * Ordering:
- *   Order of hook execution is unspecified, and may be different than insertion
- *   order.
- */
-
-#define HOOK_MAX 4
-
-enum hook_alloc_e {
-	hook_alloc_malloc,
-	hook_alloc_posix_memalign,
-	hook_alloc_aligned_alloc,
-	hook_alloc_calloc,
-	hook_alloc_memalign,
-	hook_alloc_valloc,
-	hook_alloc_pvalloc,
-	hook_alloc_mallocx,
-
-	/* The reallocating functions have both alloc and dalloc variants */
-	hook_alloc_realloc,
-	hook_alloc_rallocx,
-};
-/*
- * We put the enum typedef after the enum, since this file may get included by
- * jemalloc_cpp.cpp, and C++ disallows enum forward declarations.
- */
-typedef enum hook_alloc_e hook_alloc_t;
-
-enum hook_dalloc_e {
-	hook_dalloc_free,
-	hook_dalloc_dallocx,
-	hook_dalloc_sdallocx,
-
-	/*
-	 * The dalloc halves of reallocation (not called if in-place expansion
-	 * happens).
-	 */
-	hook_dalloc_realloc,
-	hook_dalloc_rallocx,
-};
-typedef enum hook_dalloc_e hook_dalloc_t;
-
-enum hook_expand_e {
-	hook_expand_realloc,
-	hook_expand_rallocx,
-	hook_expand_xallocx,
-};
-typedef enum hook_expand_e hook_expand_t;
-
-typedef void (*hook_alloc)(void *extra, hook_alloc_t type, void *result,
-    uintptr_t result_raw, uintptr_t args_raw[3]);
-
-typedef void (*hook_dalloc)(
-    void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
-
-typedef void (*hook_expand)(void *extra, hook_expand_t type, void *address,
-    size_t old_usize, size_t new_usize, uintptr_t result_raw,
-    uintptr_t args_raw[4]);
-
-typedef struct hooks_s hooks_t;
-struct hooks_s {
-	hook_alloc  alloc_hook;
-	hook_dalloc dalloc_hook;
-	hook_expand expand_hook;
-	void       *extra;
-};
-
-/*
- * Begin implementation details; everything above this point might one day live
- * in a public API.  Everything below this point never will.
- */
-
-/*
- * The realloc pathways haven't gotten any refactoring love in a while, and it's
- * fairly difficult to pass information from the entry point to the hooks.  We
- * put the informaiton the hooks will need into a struct to encapsulate
- * everything.
- *
- * Much of these pathways are force-inlined, so that the compiler can avoid
- * materializing this struct until we hit an extern arena function.  For fairly
- * goofy reasons, *many* of the realloc paths hit an extern arena function.
- * These paths are cold enough that it doesn't matter; eventually, we should
- * rewrite the realloc code to make the expand-in-place and the
- * free-then-realloc paths more orthogonal, at which point we don't need to
- * spread the hook logic all over the place.
- */
-typedef struct hook_ralloc_args_s hook_ralloc_args_t;
-struct hook_ralloc_args_s {
-	/* I.e. as opposed to rallocx. */
-	bool is_realloc;
-	/*
-	 * The expand hook takes 4 arguments, even if only 3 are actually used;
-	 * we add an extra one in case the user decides to memcpy without
-	 * looking too closely at the hooked function.
-	 */
-	uintptr_t args[4];
-};
-
-/*
- * Returns an opaque handle to be used when removing the hook.  NULL means that
- * we couldn't install the hook.
- */
-bool hook_boot(void);
-
-void *hook_install(tsdn_t *tsdn, hooks_t *to_install);
-/* Uninstalls the hook with the handle previously returned from hook_install. */
-void hook_remove(tsdn_t *tsdn, void *opaque);
-
-/* Hooks */
-
-void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw,
-    uintptr_t args_raw[3]);
-
-void hook_invoke_dalloc(
-    hook_dalloc_t type, void *address, uintptr_t args_raw[3]);
-
-void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize,
-    size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]);
-
-#endif /* JEMALLOC_INTERNAL_HOOK_H */

include/jemalloc/internal/hpa.h

@@ -10,7 +10,6 @@
 #include "jemalloc/internal/hpa_hooks.h"
 #include "jemalloc/internal/hpa_opts.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/psset.h"
 #include "jemalloc/internal/sec.h"
 
@@ -51,6 +50,60 @@ struct hpa_shard_nonderived_stats_s {
 	 * Guarded by mtx.
 	 */
 	uint64_t ndehugifies;
+
+	/*
+	 * Distribution of the min number of extents we will try to allocate
+	 * from a single hpa_alloc() call.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the max number of extents we will try to allocate
+	 * from a single hpa_alloc() call.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of extents allocated for a single
+	 * hpa_alloc() call and a single mtx lock acquisition.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of ps out of which we allocated extents
+	 * for a single hpa_alloc() call and a single mtx lock acquisition.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of pages allocated from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of extents allocated from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the total elapsed time (ns) for allocating extents
+	 * from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
 };
 
 /* Completely derived; only used by CTL. */
@@ -63,12 +116,6 @@ struct hpa_shard_stats_s {
 
 typedef struct hpa_shard_s hpa_shard_t;
 struct hpa_shard_s {
-	/*
-	 * pai must be the first member; we cast from a pointer to it to a
-	 * pointer to the hpa_shard_t.
-	 */
-	pai_t pai;
-
 	/* The central allocator we get our hugepages from. */
 	hpa_central_t *central;
 
@@ -143,9 +190,8 @@ struct hpa_shard_s {
 	nstime_t last_time_work_attempted;
 };
 
-bool hpa_hugepage_size_exceeds_limit(void);
 /*
- * Whether or not the HPA can be used given the current configuration.  This is
+ * Whether or not the HPA can be used given the current configuration.  This
  * is not necessarily a guarantee that it backs its allocations by hugepages,
  * just that it can function properly given the system it's running on.
  */
@@ -154,6 +200,13 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
     emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
     const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts);
 
+edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
+    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
+    bool *deferred_work_generated);
+void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
+    bool *deferred_work_generated);
+uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
+
 void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
 void hpa_shard_stats_merge(
     tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst);

include/jemalloc/internal/hpa_opts.h

@@ -108,11 +108,6 @@ struct hpa_shard_opts_s {
 	 */
 	uint64_t min_purge_interval_ms;
 
-	/*
-	 * Maximum number of hugepages to purge on each purging attempt.
-	 */
-	ssize_t experimental_max_purge_nhp;
-
 	/*
 	 * Minimum number of inactive bytes needed for a non-empty page to be
 	 * considered purgable.
@@ -176,8 +171,6 @@ struct hpa_shard_opts_s {
 	false,								\
 	/* min_purge_interval_ms */					\
 	5 * 1000,							\
-	/* experimental_max_purge_nhp */				\
-	-1,      							\
 	/* size_t purge_threshold */					\
 	PAGE,								\
 	/* min_purge_delay_ms */             				\

include/jemalloc/internal/hpdata.h

@@ -27,7 +27,7 @@
  */
 #define PSSET_ENUMERATE_MAX_NUM 32
 typedef struct hpdata_s hpdata_t;
-ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM);
+ph_structs(hpdata_age_heap, hpdata_t, PSSET_ENUMERATE_MAX_NUM)
 struct hpdata_s {
 	/*
 	 * We likewise follow the edata convention of mangling names and forcing
@@ -136,7 +136,7 @@ TYPED_LIST(hpdata_empty_list, hpdata_t, ql_link_empty)
 TYPED_LIST(hpdata_purge_list, hpdata_t, ql_link_purge)
 TYPED_LIST(hpdata_hugify_list, hpdata_t, ql_link_hugify)
 
-ph_proto(, hpdata_age_heap, hpdata_t);
+ph_proto(, hpdata_age_heap, hpdata_t)
 
 static inline void *
 hpdata_addr_get(const hpdata_t *hpdata) {
@@ -209,7 +209,7 @@ hpdata_allow_hugify(hpdata_t *hpdata, nstime_t now) {
 }
 
 static inline nstime_t
-hpdata_time_hugify_allowed(hpdata_t *hpdata) {
+hpdata_time_hugify_allowed(const hpdata_t *hpdata) {
 	return hpdata->h_time_hugify_allowed;
 }
 
@@ -305,7 +305,7 @@ hpdata_ndirty_get(const hpdata_t *hpdata) {
 }
 
 static inline size_t
-hpdata_nretained_get(hpdata_t *hpdata) {
+hpdata_nretained_get(const hpdata_t *hpdata) {
 	return HUGEPAGE_PAGES - hpdata->h_ntouched;
 }
 
@@ -330,7 +330,7 @@ hpdata_purged_when_empty_and_huge_set(hpdata_t *hpdata, bool v) {
 }
 
 static inline void
-hpdata_assert_empty(hpdata_t *hpdata) {
+hpdata_assert_empty(const hpdata_t *hpdata) {
 	assert(fb_empty(hpdata->active_pages, HUGEPAGE_PAGES));
 	assert(hpdata->h_nactive == 0);
 }
@@ -341,7 +341,7 @@ hpdata_assert_empty(hpdata_t *hpdata) {
  * match computed ones).
  */
 static inline bool
-hpdata_consistent(hpdata_t *hpdata) {
+hpdata_consistent(const hpdata_t *hpdata) {
 	bool res = true;
 
 	const size_t active_urange_longest = fb_urange_longest(
@@ -425,13 +425,62 @@ hpdata_full(const hpdata_t *hpdata) {
 
 void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 
-/*
- * Given an hpdata which can serve an allocation request, pick and reserve an
- * offset within that allocation.
- */
-void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
+/*
+ * For buffering extent allocations we will perform out of
+ * a single ps.
+ */
+typedef struct hpdata_alloc_offset_s hpdata_alloc_offset_t;
+struct hpdata_alloc_offset_s {
+	/*
+	 * Index on the active bitmap for the extent to allocate.
+	 * It is used to know which bits we'll need to set when we perform
+	 * the allocation. They are in the range [index, index + npages).
+	 */
+	size_t index;
+
+	/*
+	 * The length of the free bit range on the active bitmap,
+	 * starting at index, before setting the bits in the range
+	 * [index, index + npages).
+	 * It is used to determine whether one of the allocations
+	 * used up the longest free range on the active bitmap.
+	 * If it did, we might have to update the longest free range
+	 * metadata on the hpdata.
+	 */
+	size_t len_before;
+
+	/*
+	 * The length of the longest free range in the range [0, index).
+	 * When we need to update the longest free range on the hpdata,
+	 * the new value is either longest_len (the max up to index),
+	 * len_before - npages (what's left after we carve up the free
+	 * range starting at index), or the max in the range
+	 * [index + len_before, HUGEPAGE_PAGES), whichever is greater.
+	 */
+	size_t longest_len;
+};
+
+/*
+ * Given an hpdata that can serve an allocation request of size sz,
+ * find between one and max_nallocs offsets that can satisfy such
+ * an allocation request and buffer them in offsets (without actually
+ * reserving any space or updating hpdata). Return the number
+ * of offsets discovered.
+ */
+size_t hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t max_nallocs);
+/* Reserve the allocation for the given offset. */
+void *hpdata_reserve_alloc_offset(
+    hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset);
+/*
+ * Do any work that needs to be done after performing allocations
+ * from a single hpdata.
+ */
+void hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t nallocs);
+
 /*
  * The hpdata_purge_prepare_t allows grabbing the metadata required to purge
  * subranges of a hugepage while holding a lock, drop the lock during the actual

include/jemalloc/internal/inspect.h

@@ -11,8 +11,7 @@
  */
 
 /*
- * The following two structs are for experimental purposes. See
- * experimental_utilization_query_ctl and
+ * The following struct is for experimental purposes. See
  * experimental_utilization_batch_query_ctl in src/ctl.c.
  */
 typedef struct inspect_extent_util_stats_s inspect_extent_util_stats_t;
@@ -22,22 +21,7 @@ struct inspect_extent_util_stats_s {
 	size_t size;
 };
 
-typedef struct inspect_extent_util_stats_verbose_s
-    inspect_extent_util_stats_verbose_t;
-
-struct inspect_extent_util_stats_verbose_s {
-	void  *slabcur_addr;
-	size_t nfree;
-	size_t nregs;
-	size_t size;
-	size_t bin_nfree;
-	size_t bin_nregs;
-};
-
 void inspect_extent_util_stats_get(
     tsdn_t *tsdn, const void *ptr, size_t *nfree, size_t *nregs, size_t *size);
-void inspect_extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr,
-    size_t *nfree, size_t *nregs, size_t *size, size_t *bin_nfree,
-    size_t *bin_nregs, void **slabcur_addr);
 
 #endif /* JEMALLOC_INTERNAL_INSPECT_H */

include/jemalloc/internal/jemalloc_fork.h

@@ -0,0 +1,8 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_FORK_H
+#define JEMALLOC_INTERNAL_JEMALLOC_FORK_H
+
+void jemalloc_prefork(void);
+void jemalloc_postfork_parent(void);
+void jemalloc_postfork_child(void);
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_FORK_H */

include/jemalloc/internal/jemalloc_init.h

@@ -0,0 +1,40 @@
+#ifndef JEMALLOC_INTERNAL_JEMALLOC_INIT_H
+#define JEMALLOC_INTERNAL_JEMALLOC_INIT_H
+
+enum malloc_init_e {
+	malloc_init_uninitialized = 3,
+	malloc_init_a0_initialized = 2,
+	malloc_init_recursible = 1,
+	malloc_init_initialized = 0 /* Common case --> jnz. */
+};
+typedef enum malloc_init_e malloc_init_t;
+
+extern malloc_init_t malloc_init_state;
+
+bool malloc_is_initializer(void);
+
+bool malloc_init_hard_a0(void);
+bool malloc_init_hard(void);
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_init_a0(void) {
+	if (unlikely(malloc_init_state == malloc_init_uninitialized)) {
+		return malloc_init_hard_a0();
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_initialized(void) {
+	return (malloc_init_state == malloc_init_initialized);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_init(void) {
+	if (unlikely(!malloc_initialized()) && malloc_init_hard()) {
+		return true;
+	}
+	return false;
+}
+
+#endif /* JEMALLOC_INTERNAL_JEMALLOC_INIT_H */

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -465,6 +465,16 @@
 /* Is C++ support being built? */
 #undef JEMALLOC_ENABLE_CXX
 
+/* Are C++ exceptions enabled? */
+#undef JEMALLOC_HAVE_CXX_EXCEPTIONS
+
+/*
+ * If defined, throwing operator new aborts on OOM (with size logged) instead
+ * of throwing std::bad_alloc. Nothrow new still returns null. With LTO this
+ * lets the compiler prove operator new is no-throw and elide exception cleanup.
+ */
+#undef JEMALLOC_INFALLIBLE_NEW
+
 /* Performs additional size checks when defined. */
 #undef JEMALLOC_OPT_SIZE_CHECKS
 

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -1,7 +1,6 @@
 #ifndef JEMALLOC_INTERNAL_EXTERNS_H
 #define JEMALLOC_INTERNAL_EXTERNS_H
 
-#include "jemalloc/internal/arena_types.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/fxp.h"
 #include "jemalloc/internal/hpa_opts.h"
@@ -22,6 +21,7 @@ extern bool             opt_confirm_conf;
 extern bool             opt_hpa;
 extern hpa_shard_opts_t opt_hpa_opts;
 extern sec_opts_t       opt_hpa_sec_opts;
+extern sec_opts_t       opt_pac_sec_opts;
 
 extern const char *opt_junk;
 extern bool        opt_junk_alloc;
@@ -31,13 +31,11 @@ extern void (*JET_MUTABLE junk_alloc_callback)(void *ptr, size_t size);
 extern void (*JET_MUTABLE invalid_conf_abort)(void);
 extern bool                  opt_utrace;
 extern bool                  opt_xmalloc;
-extern bool                  opt_experimental_infallible_new;
 extern bool                  opt_experimental_tcache_gc;
 extern bool                  opt_zero;
 extern unsigned              opt_narenas;
 extern fxp_t                 opt_narenas_ratio;
 extern zero_realloc_action_t opt_zero_realloc_action;
-extern malloc_init_t         malloc_init_state;
 extern const char *const     zero_realloc_mode_names[];
 extern atomic_zu_t           zero_realloc_count;
 extern bool                  opt_cache_oblivious;
@@ -47,6 +45,7 @@ extern bool                  opt_disable_large_size_classes;
 
 extern const char *opt_malloc_conf_symlink;
 extern const char *opt_malloc_conf_env_var;
+extern const char *je_malloc_conf_2_conf_harder;
 
 /* Escape free-fastpath when ptr & mask == 0 (for sanitization purpose). */
 extern uintptr_t san_cache_bin_nonfast_mask;
@@ -54,36 +53,9 @@ extern uintptr_t san_cache_bin_nonfast_mask;
 /* Number of CPUs. */
 extern unsigned ncpus;
 
-/* Number of arenas used for automatic multiplexing of threads and arenas. */
-extern unsigned narenas_auto;
-
-/* Base index for manual arenas. */
-extern unsigned manual_arena_base;
-
-/*
- * Arenas that are used to service external requests.  Not all elements of the
- * arenas array are necessarily used; arenas are created lazily as needed.
- */
-extern atomic_p_t arenas[];
-
-extern unsigned huge_arena_ind;
-
-void    *a0malloc(size_t size);
-void     a0dalloc(void *ptr);
 void    *bootstrap_malloc(size_t size);
 void    *bootstrap_calloc(size_t num, size_t size);
 void     bootstrap_free(void *ptr);
-void     arena_set(unsigned ind, arena_t *arena);
-unsigned narenas_total_get(void);
-arena_t *arena_init(tsdn_t *tsdn, unsigned ind, const arena_config_t *config);
-arena_t *arena_choose_hard(tsd_t *tsd, bool internal);
-void     arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena);
-void     iarena_cleanup(tsd_t *tsd);
-void     arena_cleanup(tsd_t *tsd);
-size_t   batch_alloc(void **ptrs, size_t num, size_t size, int flags);
-void     jemalloc_prefork(void);
-void     jemalloc_postfork_parent(void);
-void     jemalloc_postfork_child(void);
 void     sdallocx_default(void *ptr, size_t size, int flags);
 void     free_default(void *ptr);
 void    *malloc_default(size_t size);

include/jemalloc/internal/jemalloc_internal_includes.h

@@ -1,84 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_INCLUDES_H
-#define JEMALLOC_INTERNAL_INCLUDES_H
-
-/*
- * jemalloc can conceptually be broken into components (arena, tcache, etc.),
- * but there are circular dependencies that cannot be broken without
- * substantial performance degradation.
- *
- * Historically, we dealt with this by each header into four sections (types,
- * structs, externs, and inlines), and included each header file multiple times
- * in this file, picking out the portion we want on each pass using the
- * following #defines:
- *   JEMALLOC_H_TYPES   : Preprocessor-defined constants and pseudo-opaque data
- *                        types.
- *   JEMALLOC_H_STRUCTS : Data structures.
- *   JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
- *   JEMALLOC_H_INLINES : Inline functions.
- *
- * We're moving toward a world in which the dependencies are explicit; each file
- * will #include the headers it depends on (rather than relying on them being
- * implicitly available via this file including every header file in the
- * project).
- *
- * We're now in an intermediate state: we've broken up the header files to avoid
- * having to include each one multiple times, but have not yet moved the
- * dependency information into the header files (i.e. we still rely on the
- * ordering in this file to ensure all a header's dependencies are available in
- * its translation unit).  Each component is now broken up into multiple header
- * files, corresponding to the sections above (e.g. instead of "foo.h", we now
- * have "foo_types.h", "foo_structs.h", "foo_externs.h", "foo_inlines.h").
- *
- * Those files which have been converted to explicitly include their
- * inter-component dependencies are now in the initial HERMETIC HEADERS
- * section.  All headers may still rely on jemalloc_preamble.h (which, by fiat,
- * must be included first in every translation unit) for system headers and
- * global jemalloc definitions, however.
- */
-
-/******************************************************************************/
-/* TYPES */
-/******************************************************************************/
-
-#include "jemalloc/internal/arena_types.h"
-#include "jemalloc/internal/tcache_types.h"
-#include "jemalloc/internal/prof_types.h"
-
-/******************************************************************************/
-/* STRUCTS */
-/******************************************************************************/
-
-#include "jemalloc/internal/prof_structs.h"
-#include "jemalloc/internal/arena_structs.h"
-#include "jemalloc/internal/tcache_structs.h"
-#include "jemalloc/internal/background_thread_structs.h"
-
-/******************************************************************************/
-/* EXTERNS */
-/******************************************************************************/
-
-#include "jemalloc/internal/jemalloc_internal_externs.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/large_externs.h"
-#include "jemalloc/internal/tcache_externs.h"
-#include "jemalloc/internal/prof_externs.h"
-#include "jemalloc/internal/background_thread_externs.h"
-
-/******************************************************************************/
-/* INLINES */
-/******************************************************************************/
-
-#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
-/*
- * Include portions of arena code interleaved with tcache code in order to
- * resolve circular dependencies.
- */
-#include "jemalloc/internal/arena_inlines_a.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
-#include "jemalloc/internal/tcache_inlines.h"
-#include "jemalloc/internal/arena_inlines_b.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
-#include "jemalloc/internal/prof_inlines.h"
-#include "jemalloc/internal/background_thread_inlines.h"
-
-#endif /* JEMALLOC_INTERNAL_INCLUDES_H */

include/jemalloc/internal/jemalloc_internal_inlines_a.h

@@ -2,13 +2,13 @@
 #define JEMALLOC_INTERNAL_INLINES_A_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arenas_management.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/bit_util.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/tcache_externs.h"
+#include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/ticker.h"
 
 JEMALLOC_ALWAYS_INLINE malloc_cpuid_t

include/jemalloc/internal/jemalloc_internal_inlines_b.h

@@ -1,106 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_INLINES_B_H
-#define JEMALLOC_INTERNAL_INLINES_B_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_inlines_a.h"
-#include "jemalloc/internal/extent.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
-
-static inline void
-percpu_arena_update(tsd_t *tsd, unsigned cpu) {
-	assert(have_percpu_arena);
-	arena_t *oldarena = tsd_arena_get(tsd);
-	assert(oldarena != NULL);
-	unsigned oldind = arena_ind_get(oldarena);
-
-	if (oldind != cpu) {
-		unsigned newind = cpu;
-		arena_t *newarena = arena_get(tsd_tsdn(tsd), newind, true);
-		assert(newarena != NULL);
-
-		/* Set new arena/tcache associations. */
-		arena_migrate(tsd, oldarena, newarena);
-		tcache_t *tcache = tcache_get(tsd);
-		if (tcache != NULL) {
-			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
-			assert(tcache_slow->arena != NULL);
-			tcache_arena_reassociate(
-			    tsd_tsdn(tsd), tcache_slow, tcache, newarena);
-		}
-	}
-}
-
-/* Choose an arena based on a per-thread value. */
-static inline arena_t *
-arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
-	arena_t *ret;
-
-	if (arena != NULL) {
-		return arena;
-	}
-
-	/* During reentrancy, arena 0 is the safest bet. */
-	if (unlikely(tsd_reentrancy_level_get(tsd) > 0)) {
-		return arena_get(tsd_tsdn(tsd), 0, true);
-	}
-
-	ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd);
-	if (unlikely(ret == NULL)) {
-		ret = arena_choose_hard(tsd, internal);
-		assert(ret);
-		if (tcache_available(tsd)) {
-			tcache_slow_t *tcache_slow = tsd_tcache_slowp_get(tsd);
-			tcache_t      *tcache = tsd_tcachep_get(tsd);
-			if (tcache_slow->arena != NULL) {
-				/* See comments in tsd_tcache_data_init().*/
-				assert(tcache_slow->arena
-				    == arena_get(tsd_tsdn(tsd), 0, false));
-				if (tcache_slow->arena != ret) {
-					tcache_arena_reassociate(tsd_tsdn(tsd),
-					    tcache_slow, tcache, ret);
-				}
-			} else {
-				tcache_arena_associate(
-				    tsd_tsdn(tsd), tcache_slow, tcache, ret);
-			}
-		}
-	}
-
-	/*
-	 * Note that for percpu arena, if the current arena is outside of the
-	 * auto percpu arena range, (i.e. thread is assigned to a manually
-	 * managed arena), then percpu arena is skipped.
-	 */
-	if (have_percpu_arena && PERCPU_ARENA_ENABLED(opt_percpu_arena)
-	    && !internal
-	    && (arena_ind_get(ret) < percpu_arena_ind_limit(opt_percpu_arena))
-	    && (ret->last_thd != tsd_tsdn(tsd))) {
-		unsigned ind = percpu_arena_choose();
-		if (arena_ind_get(ret) != ind) {
-			percpu_arena_update(tsd, ind);
-			ret = tsd_arena_get(tsd);
-		}
-		ret->last_thd = tsd_tsdn(tsd);
-	}
-
-	return ret;
-}
-
-static inline arena_t *
-arena_choose(tsd_t *tsd, arena_t *arena) {
-	return arena_choose_impl(tsd, arena, false);
-}
-
-static inline arena_t *
-arena_ichoose(tsd_t *tsd, arena_t *arena) {
-	return arena_choose_impl(tsd, arena, true);
-}
-
-static inline bool
-arena_is_auto(arena_t *arena) {
-	assert(narenas_auto > 0);
-
-	return (arena_ind_get(arena) < manual_arena_base);
-}
-
-#endif /* JEMALLOC_INTERNAL_INLINES_B_H */

include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -2,12 +2,13 @@
 #define JEMALLOC_INTERNAL_INLINES_C_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
-#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/emap.h"
-#include "jemalloc/internal/hook.h"
+#include "jemalloc/internal/jemalloc_init.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/log.h"
+#include "jemalloc/internal/malloc_dispatch_inlines.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/thread_event.h"
 #include "jemalloc/internal/witness.h"
@@ -67,7 +68,7 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
 		    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	}
 
-	ret = arena_malloc(
+	ret = malloc_dispatch_malloc(
 	    tsdn, arena, size, ind, zero, slab, tcache, slow_path);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -102,7 +103,8 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 
-	ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
+	ret = malloc_dispatch_palloc(
+	    tsdn, arena, usize, alignment, zero, slab, tcache);
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -156,7 +158,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
 	    && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
 		assert(tcache == NULL);
 	}
-	arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
+	malloc_dispatch_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -169,13 +171,12 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-	arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
+	malloc_dispatch_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
-    hook_ralloc_args_t *hook_args) {
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	void  *p;
@@ -196,27 +197,13 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 	 */
 	copysize = (size < oldsize) ? size : oldsize;
 	memcpy(p, ptr, copysize);
-	hook_invoke_alloc(
-	    hook_args->is_realloc ? hook_alloc_realloc : hook_alloc_rallocx, p,
-	    (uintptr_t)p, hook_args->args);
-	hook_invoke_dalloc(
-	    hook_args->is_realloc ? hook_dalloc_realloc : hook_dalloc_rallocx,
-	    ptr, hook_args->args);
 	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
 	return p;
 }
 
-/*
- * is_realloc threads through the knowledge of whether or not this call comes
- * from je_realloc (as opposed to je_rallocx); this ensures that we pass the
- * correct entry point into any hooks.
- * Note that these functions are all force-inlined, so no actual bool gets
- * passed-around anywhere.
- */
 JEMALLOC_ALWAYS_INLINE void *
 iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena,
-    hook_ralloc_args_t *hook_args) {
+    size_t alignment, bool zero, bool slab, tcache_t *tcache, arena_t *arena) {
 	assert(ptr != NULL);
 	assert(size != 0);
 	witness_assert_depth_to_rank(
@@ -229,27 +216,26 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 		 * and copy.
 		 */
 		return iralloct_realign(tsdn, ptr, oldsize, size, alignment,
-		    zero, slab, tcache, arena, hook_args);
+		    zero, slab, tcache, arena);
 	}
 
-	return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
-	    slab, tcache, hook_args);
+	return malloc_dispatch_ralloc(
+	    tsdn, arena, ptr, oldsize, size, alignment, zero, slab, tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
-    size_t usize, bool zero, tcache_t *tcache, arena_t *arena,
-    hook_ralloc_args_t *hook_args) {
+    size_t usize, bool zero, tcache_t *tcache, arena_t *arena) {
 	bool slab = sz_can_use_slab(usize);
 	return iralloct_explicit_slab(tsdn, ptr, oldsize, size, alignment, zero,
-	    slab, tcache, arena, hook_args);
+	    slab, tcache, arena);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
 iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
-    size_t usize, bool zero, hook_ralloc_args_t *hook_args) {
+    size_t usize, bool zero) {
 	return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, usize,
-	    zero, tcache_get(tsd), NULL, hook_args);
+	    zero, tcache_get(tsd), NULL);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -280,11 +266,6 @@ fastpath_success_finish(
 	}
 }
 
-JEMALLOC_ALWAYS_INLINE bool
-malloc_initialized(void) {
-	return (malloc_init_state == malloc_init_initialized);
-}
-
 /*
  * malloc() fastpath.  Included here so that we can inline it into operator new;
  * function call overhead there is non-negligible as a fraction of total CPU in

include/jemalloc/internal/jemalloc_internal_macros.h

@@ -39,9 +39,11 @@
 #ifdef JEMALLOC_JET
 #	define JET_MUTABLE
 #	define JET_EXTERN extern
+#	define JET_EXTERN_INLINE extern
 #else
 #	define JET_MUTABLE const
 #	define JET_EXTERN static
+#	define JET_EXTERN_INLINE static inline
 #endif
 
 #define JEMALLOC_VA_ARGS_HEAD(head, ...) head

include/jemalloc/internal/jemalloc_internal_types.h

@@ -20,14 +20,6 @@ typedef enum zero_realloc_action_e zero_realloc_action_t;
 /* Signature of write callback. */
 typedef void(write_cb_t)(void *, const char *);
 
-enum malloc_init_e {
-	malloc_init_uninitialized = 3,
-	malloc_init_a0_initialized = 2,
-	malloc_init_recursible = 1,
-	malloc_init_initialized = 0 /* Common case --> jnz. */
-};
-typedef enum malloc_init_e malloc_init_t;
-
 /*
  * Flags bits:
  *

include/jemalloc/internal/jemalloc_preamble.h.in

@@ -238,6 +238,18 @@ static const bool config_enable_cxx =
 #endif
 ;
 
+/*
+ * Whether the throwing operator new aborts on OOM instead of throwing
+ * std::bad_alloc (--enable-cxx-infallible-new).
+ */
+static const bool config_infallible_new =
+#ifdef JEMALLOC_INFALLIBLE_NEW
+    true
+#else
+    false
+#endif
+;
+
 #if defined(_WIN32) || defined(__APPLE__) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
 /* Currently percpu_arena depends on sched_getcpu. */
 #define JEMALLOC_PERCPU_ARENA

include/jemalloc/internal/large.h

@@ -1,9 +1,12 @@
-#ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H
-#define JEMALLOC_INTERNAL_LARGE_EXTERNS_H
+#ifndef JEMALLOC_INTERNAL_LARGE_H
+#define JEMALLOC_INTERNAL_LARGE_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/hook.h"
+
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s     arena_t;
+typedef struct prof_info_s prof_info_t;
 
 void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
 void *large_palloc(
@@ -11,8 +14,7 @@ void *large_palloc(
 bool  large_ralloc_no_move(tsdn_t *tsdn, edata_t *edata, size_t usize_min,
      size_t usize_max, bool zero);
 void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize,
-    size_t alignment, bool zero, tcache_t *tcache,
-    hook_ralloc_args_t *hook_args);
+    size_t alignment, bool zero, tcache_t *tcache);
 
 void   large_dalloc_prep_locked(tsdn_t *tsdn, edata_t *edata);
 void   large_dalloc_finish(tsdn_t *tsdn, edata_t *edata);
@@ -22,4 +24,4 @@ void   large_prof_info_get(
 void large_prof_tctx_reset(edata_t *edata);
 void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size);
 
-#endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */
+#endif /* JEMALLOC_INTERNAL_LARGE_H */

include/jemalloc/internal/malloc_dispatch.h

@@ -0,0 +1,19 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s  arena_t;
+typedef struct tcache_s tcache_t;
+
+void *malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache);
+void  malloc_dispatch_dalloc_promoted(
+     tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
+void *malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr,
+    size_t oldsize, size_t size, size_t alignment, bool zero, bool slab,
+    tcache_t *tcache);
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_H */

include/jemalloc/internal/malloc_dispatch_inlines.h

@@ -0,0 +1,276 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena.h"
+#include "jemalloc/internal/arena_inlines.h"
+#include "jemalloc/internal/bin_inlines.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/large.h"
+#include "jemalloc/internal/malloc_dispatch.h"
+#include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache_inlines.h"
+
+JEMALLOC_ALWAYS_INLINE void *
+malloc_dispatch_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
+    bool zero, bool slab, tcache_t *tcache, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+
+	if (likely(tcache != NULL)) {
+		if (likely(slab)) {
+			assert(sz_can_use_slab(size));
+			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
+		} else if (likely(tcache_can_cache_large(tcache, ind))) {
+			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
+		}
+		/* (size > tcache_max) case falls through. */
+	}
+
+	return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
+}
+
+static inline void
+malloc_dispatch_dalloc_large_no_tcache(
+    tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
+	/*
+	 * szind both classifies small vs large and validates the extent --
+	 * inactive extents have szind == SC_NSIZES.
+	 */
+	if (config_prof && unlikely(szind < SC_NBINS)) {
+		malloc_dispatch_dalloc_promoted(tsdn, ptr, NULL, true);
+	} else {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		if (large_dalloc_safety_checks(edata, ptr, usize)) {
+			/* See the comment in isfree. */
+			return;
+		}
+		large_dalloc(tsdn, edata);
+	}
+}
+
+static inline void
+malloc_dispatch_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
+	assert(ptr != NULL);
+
+	emap_alloc_ctx_t alloc_ctx;
+	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.szind < SC_NSIZES);
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		malloc_dispatch_dalloc_large_no_tcache(
+		    tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    szind_t szind, size_t usize, bool slow_path) {
+	assert(!tsdn_null(tsdn) && tcache != NULL);
+	bool is_sample_promoted = config_prof && szind < SC_NBINS;
+	if (unlikely(is_sample_promoted)) {
+		malloc_dispatch_dalloc_promoted(tsdn, ptr, tcache, slow_path);
+	} else {
+		if (tcache_can_cache_large(tcache, szind)) {
+			tcache_dalloc_large(
+			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
+		} else {
+			edata_t *edata = emap_edata_lookup(
+			    tsdn, &arena_emap_global, ptr);
+			if (large_dalloc_safety_checks(edata, ptr, usize)) {
+				/* See the comment in isfree. */
+				return;
+			}
+			large_dalloc(tsdn, edata);
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_dispatch_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
+	if (!config_debug) {
+		return false;
+	}
+	edata_t   *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	szind_t    binind = edata_szind_get(edata);
+	div_info_t div_info = arena_binind_div_info[binind];
+	/*
+	 * Calls the internal function bin_slab_regind_impl because the
+	 * safety check does not require a lock.
+	 */
+	size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
+	slab_data_t      *slab_data = edata_slab_data_get(edata);
+	const bin_info_t *bin_info = &bin_infos[binind];
+	assert(edata_nfree_get(edata) < bin_info->nregs);
+	if (unlikely(!bitmap_get(
+	        slab_data->bitmap, &bin_info->bitmap_info, regind))) {
+		safety_check_fail(
+		    "Invalid deallocation detected: the pointer being freed (%p) not "
+		    "currently active, possibly caused by double free bugs.\n",
+		    ptr);
+		return true;
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+
+	if (unlikely(tcache == NULL)) {
+		malloc_dispatch_dalloc_no_tcache(tsdn, ptr);
+		return;
+	}
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (caller_alloc_ctx != NULL) {
+		alloc_ctx = *caller_alloc_ctx;
+	} else {
+		util_assume(tsdn != NULL);
+		emap_alloc_ctx_lookup(
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+	}
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.szind < SC_NSIZES);
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+	} else {
+		malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
+	}
+}
+
+static inline void
+malloc_dispatch_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
+	assert(ptr != NULL);
+	assert(size <= SC_LARGE_MAXCLASS);
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (!config_prof || !opt_prof) {
+		/*
+		 * There is no risk of being confused by a promoted sampled
+		 * object, so base szind and slab on the given size.
+		 */
+		szind_t szind = sz_size2index(size);
+		emap_alloc_ctx_init(
+		    &alloc_ctx, szind, (szind < SC_NBINS), size);
+	}
+
+	if ((config_prof && opt_prof) || config_debug) {
+		emap_alloc_ctx_lookup(
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+		assert(alloc_ctx.szind == sz_size2index(size));
+		assert((config_prof && opt_prof)
+		    || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
+
+		if (config_debug) {
+			edata_t *edata = emap_edata_lookup(
+			    tsdn, &arena_emap_global, ptr);
+			assert(alloc_ctx.szind == edata_szind_get(edata));
+			assert(alloc_ctx.slab == edata_slab_get(edata));
+		}
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		malloc_dispatch_dalloc_large_no_tcache(
+		    tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+	assert(size <= SC_LARGE_MAXCLASS);
+
+	if (unlikely(tcache == NULL)) {
+		malloc_dispatch_sdalloc_no_tcache(tsdn, ptr, size);
+		return;
+	}
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (config_prof && opt_prof) {
+		if (caller_alloc_ctx == NULL) {
+			/* Uncommon case and should be a static check. */
+			emap_alloc_ctx_lookup(
+			    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+			assert(alloc_ctx.szind == sz_size2index(size));
+			assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
+		} else {
+			alloc_ctx = *caller_alloc_ctx;
+		}
+	} else {
+		/*
+		 * There is no risk of being confused by a promoted sampled
+		 * object, so base szind and slab on the given size.
+		 */
+		alloc_ctx.szind = sz_size2index(size);
+		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
+	}
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		emap_alloc_ctx_init(
+		    &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+	} else {
+		malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+		    sz_s2u(size), slow_path);
+	}
+}
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H */

include/jemalloc/internal/mutex_prof.h

@@ -30,13 +30,15 @@ typedef enum {
 	OP(extents_dirty)                                                      \
 	OP(extents_muzzy)                                                      \
 	OP(extents_retained)                                                   \
+	OP(extents_pinned)                                                     \
 	OP(decay_dirty)                                                        \
 	OP(decay_muzzy)                                                        \
 	OP(base)                                                               \
 	OP(tcache_list)                                                        \
 	OP(hpa_shard)                                                          \
 	OP(hpa_shard_grow)                                                     \
-	OP(hpa_sec)
+	OP(hpa_sec)                                                            \
+	OP(pac_sec)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,

include/jemalloc/internal/nstime.h

@@ -28,7 +28,6 @@ static const nstime_t nstime_zero = NSTIME_ZERO_INITIALIZER;
 void     nstime_init(nstime_t *time, uint64_t ns);
 void     nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
 uint64_t nstime_ns(const nstime_t *time);
-uint64_t nstime_ms(const nstime_t *time);
 uint64_t nstime_sec(const nstime_t *time);
 uint64_t nstime_nsec(const nstime_t *time);
 void     nstime_copy(nstime_t *time, const nstime_t *source);
@@ -36,11 +35,9 @@ int      nstime_compare(const nstime_t *a, const nstime_t *b);
 void     nstime_add(nstime_t *time, const nstime_t *addend);
 void     nstime_iadd(nstime_t *time, uint64_t addend);
 void     nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
-void     nstime_isubtract(nstime_t *time, uint64_t subtrahend);
 void     nstime_imultiply(nstime_t *time, uint64_t multiplier);
 void     nstime_idivide(nstime_t *time, uint64_t divisor);
 uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
-uint64_t nstime_ns_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ms_between(const nstime_t *earlier, const nstime_t *later);
 uint64_t nstime_ns_since(const nstime_t *past);
 uint64_t nstime_ms_since(const nstime_t *past);

include/jemalloc/internal/pa.h

@@ -10,15 +10,13 @@
 #include "jemalloc/internal/hpa.h"
 #include "jemalloc/internal/lockedint.h"
 #include "jemalloc/internal/pac.h"
-#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/sec.h"
 
 /*
  * The page allocator; responsible for acquiring pages of memory for
- * allocations.  It picks the implementation of the page allocator interface
- * (i.e. a pai_t) to handle a given page-level allocation request.  For now, the
- * only such implementation is the PAC code ("page allocator classic"), but
- * others will be coming soon.
+ * allocations.  It dispatches each page-level allocation request to either
+ * the PAC ("page allocator classic") or the HPA ("hugepage allocator")
+ * by calling their pac_*() / hpa_*() entry points directly.
  */
 
 typedef struct pa_central_s pa_central_t;
@@ -93,10 +91,7 @@ struct pa_shard_s {
 	 */
 	bool ever_used_hpa;
 
-	/* Allocates from a PAC. */
-	pac_t pac;
-
-	hpa_shard_t hpa_shard;
+	hpa_shard_t hpa;
 
 	/* The source of edata_t objects. */
 	edata_cache_t edata_cache;
@@ -111,6 +106,9 @@ struct pa_shard_s {
 
 	/* The base from which we get the ehooks and allocate metadat. */
 	base_t *base;
+
+	/* Allocates from a PAC. */
+	pac_t pac;
 };
 
 static inline bool
@@ -120,7 +118,7 @@ pa_shard_dont_decay_muzzy(pa_shard_t *shard) {
 }
 
 static inline ehooks_t *
-pa_shard_ehooks_get(pa_shard_t *shard) {
+pa_shard_ehooks_get(const pa_shard_t *shard) {
 	return base_ehooks_get(shard->base);
 }
 
@@ -221,12 +219,10 @@ void pa_shard_prefork5(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard);
 void pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard);
 
-size_t pa_shard_nactive(pa_shard_t *shard);
-size_t pa_shard_ndirty(pa_shard_t *shard);
-size_t pa_shard_nmuzzy(pa_shard_t *shard);
+size_t pa_shard_nactive(const pa_shard_t *shard);
 
 void pa_shard_basic_stats_merge(
-    pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
+    const pa_shard_t *shard, size_t *nactive, size_t *ndirty, size_t *nmuzzy);
 
 void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,

include/jemalloc/internal/pac.h

@@ -4,10 +4,12 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/decay.h"
 #include "jemalloc/internal/ecache.h"
+#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
 #include "jemalloc/internal/lockedint.h"
-#include "jemalloc/internal/pai.h"
+#include "jemalloc/internal/sec.h"
+#include "jemalloc/internal/tsd_types.h"
 #include "san_bump.h"
 
 /*
@@ -51,6 +53,8 @@ struct pac_estats_s {
 	size_t muzzy_bytes;
 	size_t nretained;
 	size_t retained_bytes;
+	size_t npinned;
+	size_t pinned_bytes;
 };
 
 typedef struct pac_stats_s pac_stats_t;
@@ -61,9 +65,14 @@ struct pac_stats_s {
 	/*
 	 * Number of unused virtual memory bytes currently retained.  Retained
 	 * bytes are technically mapped (though always decommitted or purged),
-	 * but they are excluded from the mapped statistic (above).
+	 * but they are excluded from pac_mapped.
 	 */
 	size_t retained; /* Derived. */
+	/*
+	 * Number of bytes in pinned (non-reclaimable) extents currently
+	 * cached.  Unlike retained, pinned bytes count toward pac_mapped.
+	 */
+	size_t pinned;   /* Derived. */
 
 	/*
 	 * Number of bytes currently mapped, excluding retained memory (and any
@@ -76,15 +85,24 @@ struct pac_stats_s {
 
 	/* VM space had to be leaked (undocumented).  Normally 0. */
 	atomic_zu_t abandoned_vm;
+
+	/* PAC SEC stats.  Derived. */
+	sec_stats_t pac_sec_stats;
 };
 
 typedef struct pac_s pac_t;
 struct pac_s {
+	/* Small extent cache in front of PAC ecaches to reduce contention. */
+	sec_t sec;
 	/*
-	 * Must be the first member (we convert it to a PAC given only a
-	 * pointer).  The handle to the allocation interface.
+	 * Runtime gate for PAC SEC.  0 disables (when SEC is not configured or
+	 * dirty_decay_ms == 0); otherwise mirrors sec.opts.max_alloc.
 	 */
-	pai_t pai;
+	atomic_zu_t sec_max_alloc;
+
+	/* True once pinned memory has been seen. */
+	atomic_b_t has_pinned;
+
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.
@@ -94,6 +112,7 @@ struct pac_s {
 	ecache_t ecache_dirty;
 	ecache_t ecache_muzzy;
 	ecache_t ecache_retained;
+	ecache_t ecache_pinned;
 
 	base_t        *base;
 	emap_t        *emap;
@@ -155,13 +174,39 @@ bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
     ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
     malloc_mutex_t *stats_mtx);
 
+edata_t *pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment,
+    bool zero, bool guarded, bool frequent_reuse,
+    bool *deferred_work_generated);
+bool pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
+    size_t new_size, bool zero, bool *deferred_work_generated);
+bool pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size,
+    size_t new_size, bool *deferred_work_generated);
+void pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
+    bool *deferred_work_generated);
+uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac);
+
 static inline size_t
-pac_mapped(pac_t *pac) {
+pac_mapped(const pac_t *pac) {
 	return atomic_load_zu(&pac->stats->pac_mapped, ATOMIC_RELAXED);
 }
 
+void extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+    ecache_t *ecache, edata_t *edata);
+
+static inline void
+pac_record_grown(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
+    edata_t *edata) {
+	bool pinned = edata_pinned_get(edata);
+	if (pinned && config_stats) {
+		atomic_fetch_add_zu(&pac->stats->pac_mapped,
+		    edata_size_get(edata), ATOMIC_RELAXED);
+	}
+	extent_record(tsdn, pac, ehooks,
+	    pinned ? &pac->ecache_pinned : &pac->ecache_retained, edata);
+}
+
 static inline ehooks_t *
-pac_ehooks_get(pac_t *pac) {
+pac_ehooks_get(const pac_t *pac) {
 	return base_ehooks_get(pac->base);
 }
 
@@ -202,7 +247,8 @@ bool    pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
        ssize_t decay_ms, pac_purge_eagerness_t eagerness);
 ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
-void pac_reset(tsdn_t *tsdn, pac_t *pac);
 void pac_destroy(tsdn_t *tsdn, pac_t *pac);
 
+void pac_sec_flush(tsdn_t *tsdn, pac_t *pac);
+
 #endif /* JEMALLOC_INTERNAL_PAC_H */

include/jemalloc/internal/pai.h

@@ -1,63 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PAI_H
-#define JEMALLOC_INTERNAL_PAI_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/tsd_types.h"
-
-/* An interface for page allocation. */
-
-typedef struct pai_s pai_t;
-struct pai_s {
-	/* Returns NULL on failure. */
-	edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
-	    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
-	    bool *deferred_work_generated);
-	bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-	    size_t old_size, size_t new_size, bool zero,
-	    bool *deferred_work_generated);
-	bool (*shrink)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-	    size_t old_size, size_t new_size, bool *deferred_work_generated);
-	void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-	    bool *deferred_work_generated);
-	uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self);
-};
-
-/*
- * These are just simple convenience functions to avoid having to reference the
- * same pai_t twice on every invocation.
- */
-
-static inline edata_t *
-pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
-    bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
-	return self->alloc(tsdn, self, size, alignment, zero, guarded,
-	    frequent_reuse, deferred_work_generated);
-}
-
-static inline bool
-pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size, bool zero, bool *deferred_work_generated) {
-	return self->expand(tsdn, self, edata, old_size, new_size, zero,
-	    deferred_work_generated);
-}
-
-static inline bool
-pai_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size, bool *deferred_work_generated) {
-	return self->shrink(
-	    tsdn, self, edata, old_size, new_size, deferred_work_generated);
-}
-
-static inline void
-pai_dalloc(
-    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
-	self->dalloc(tsdn, self, edata, deferred_work_generated);
-}
-
-static inline uint64_t
-pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
-	return self->time_until_deferred_work(tsdn, self);
-}
-
-#endif /* JEMALLOC_INTERNAL_PAI_H */

include/jemalloc/internal/peak_event.h

@@ -2,6 +2,7 @@
 #define JEMALLOC_INTERNAL_PEAK_EVENT_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/thread_event_registry.h"
 #include "jemalloc/internal/tsd_types.h"
 
 /*

include/jemalloc/internal/prof.h

@@ -0,0 +1,426 @@
+#ifndef JEMALLOC_INTERNAL_PROF_H
+#define JEMALLOC_INTERNAL_PROF_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/ckh.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prng.h"
+#include "jemalloc/internal/prof_hook.h"
+#include "jemalloc/internal/rb.h"
+#include "jemalloc/internal/thread_event_registry.h"
+
+/* Forward decl; only base_t * is used as a pointer arg below. */
+typedef struct base_s base_t;
+
+/******************************************************************************/
+/* TYPES */
+/******************************************************************************/
+
+typedef struct prof_bt_s     prof_bt_t;
+typedef struct prof_cnt_s    prof_cnt_t;
+typedef struct prof_tctx_s   prof_tctx_t;
+typedef struct prof_info_s   prof_info_t;
+typedef struct prof_gctx_s   prof_gctx_t;
+typedef struct prof_tdata_s  prof_tdata_t;
+typedef struct prof_recent_s prof_recent_t;
+
+/* Option defaults. */
+#ifdef JEMALLOC_PROF
+#	define PROF_PREFIX_DEFAULT "jeprof"
+#else
+#	define PROF_PREFIX_DEFAULT ""
+#endif
+#define LG_PROF_SAMPLE_DEFAULT 19
+#define LG_PROF_INTERVAL_DEFAULT -1
+
+/*
+ * Hard limit on stack backtrace depth.  The version of prof_backtrace() that
+ * is based on __builtin_return_address() necessarily has a hard-coded number
+ * of backtrace frame handlers, and should be kept in sync with this setting.
+ */
+#ifdef JEMALLOC_PROF_GCC
+#	define PROF_BT_MAX_LIMIT 256
+#else
+#	define PROF_BT_MAX_LIMIT UINT_MAX
+#endif
+#define PROF_BT_MAX_DEFAULT 128
+
+/* Initial hash table size. */
+#define PROF_CKH_MINITEMS 64
+
+/* Size of memory buffer to use when writing dump files. */
+#ifndef JEMALLOC_PROF
+/* Minimize memory bloat for non-prof builds. */
+#	define PROF_DUMP_BUFSIZE 1
+#elif defined(JEMALLOC_DEBUG)
+/* Use a small buffer size in debug build, mainly to facilitate testing. */
+#	define PROF_DUMP_BUFSIZE 16
+#else
+#	define PROF_DUMP_BUFSIZE 65536
+#endif
+
+/* Size of size class related tables */
+#ifdef JEMALLOC_PROF
+#	define PROF_SC_NSIZES SC_NSIZES
+#else
+/* Minimize memory bloat for non-prof builds. */
+#	define PROF_SC_NSIZES 1
+#endif
+
+/* Size of stack-allocated buffer used by prof_printf(). */
+#define PROF_PRINTF_BUFSIZE 128
+
+/*
+ * Number of mutexes shared among all gctx's.  No space is allocated for these
+ * unless profiling is enabled, so it's okay to over-provision.
+ */
+#define PROF_NCTX_LOCKS 1024
+
+/*
+ * Number of mutexes shared among all tdata's.  No space is allocated for these
+ * unless profiling is enabled, so it's okay to over-provision.
+ */
+#define PROF_NTDATA_LOCKS 256
+
+/* Minimize memory bloat for non-prof builds. */
+#ifdef JEMALLOC_PROF
+#	define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
+#else
+#	define PROF_DUMP_FILENAME_LEN 1
+#endif
+
+/* Default number of recent allocations to record. */
+#define PROF_RECENT_ALLOC_MAX_DEFAULT 0
+
+/* Thread name storage size limit. */
+#define PROF_THREAD_NAME_MAX_LEN 16
+
+/*
+ * Minimum required alignment for sampled allocations. Over-aligning sampled
+ * allocations allows us to quickly identify them on the dalloc path without
+ * resorting to metadata lookup.
+ */
+#define PROF_SAMPLE_ALIGNMENT PAGE
+#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK
+
+/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
+#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U))
+
+/******************************************************************************/
+/* STRUCTS */
+/******************************************************************************/
+
+struct prof_bt_s {
+	/* Backtrace, stored as len program counters. */
+	void   **vec;
+	unsigned len;
+};
+
+#ifdef JEMALLOC_PROF_LIBGCC
+/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
+typedef struct {
+	void    **vec;
+	unsigned *len;
+	unsigned  max;
+} prof_unwind_data_t;
+#endif
+
+struct prof_cnt_s {
+	/* Profiling counters. */
+	uint64_t curobjs;
+	uint64_t curobjs_shifted_unbiased;
+	uint64_t curbytes;
+	uint64_t curbytes_unbiased;
+	uint64_t accumobjs;
+	uint64_t accumobjs_shifted_unbiased;
+	uint64_t accumbytes;
+	uint64_t accumbytes_unbiased;
+};
+
+typedef enum {
+	prof_tctx_state_initializing,
+	prof_tctx_state_nominal,
+	prof_tctx_state_dumping,
+	prof_tctx_state_purgatory /* Dumper must finish destroying. */
+} prof_tctx_state_t;
+
+struct prof_tctx_s {
+	/* Thread data for thread that performed the allocation. */
+	prof_tdata_t *tdata;
+
+	/*
+	 * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
+	 * defunct during teardown.
+	 */
+	uint64_t thr_uid;
+	uint64_t thr_discrim;
+
+	/*
+	 * Reference count of how many times this tctx object is referenced in
+	 * recent allocation / deallocation records, protected by tdata->lock.
+	 */
+	uint64_t recent_count;
+
+	/* Profiling counters, protected by tdata->lock. */
+	prof_cnt_t cnts;
+
+	/* Associated global context. */
+	prof_gctx_t *gctx;
+
+	/*
+	 * UID that distinguishes multiple tctx's created by the same thread,
+	 * but coexisting in gctx->tctxs.  There are two ways that such
+	 * coexistence can occur:
+	 * - A dumper thread can cause a tctx to be retained in the purgatory
+	 *   state.
+	 * - Although a single "producer" thread must create all tctx's which
+	 *   share the same thr_uid, multiple "consumers" can each concurrently
+	 *   execute portions of prof_tctx_destroy().  prof_tctx_destroy() only
+	 *   gets called once each time cnts.cur{objs,bytes} drop to 0, but this
+	 *   threshold can be hit again before the first consumer finishes
+	 *   executing prof_tctx_destroy().
+	 */
+	uint64_t tctx_uid;
+
+	/* Linkage into gctx's tctxs. */
+	rb_node(prof_tctx_t) tctx_link;
+
+	/*
+	 * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
+	 * sample vs destroy race.
+	 */
+	bool prepared;
+
+	/* Current dump-related state, protected by gctx->lock. */
+	prof_tctx_state_t state;
+
+	/*
+	 * Copy of cnts snapshotted during early dump phase, protected by
+	 * dump_mtx.
+	 */
+	prof_cnt_t dump_cnts;
+};
+typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
+
+struct prof_info_s {
+	/* Time when the allocation was made. */
+	nstime_t alloc_time;
+	/* Points to the prof_tctx_t corresponding to the allocation. */
+	prof_tctx_t *alloc_tctx;
+	/* Allocation request size. */
+	size_t alloc_size;
+};
+
+struct prof_gctx_s {
+	/* Protects nlimbo, cnt_summed, and tctxs. */
+	malloc_mutex_t *lock;
+
+	/*
+	 * Number of threads that currently cause this gctx to be in a state of
+	 * limbo due to one of:
+	 *   - Initializing this gctx.
+	 *   - Initializing per thread counters associated with this gctx.
+	 *   - Preparing to destroy this gctx.
+	 *   - Dumping a heap profile that includes this gctx.
+	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
+	 * gctx.
+	 */
+	unsigned nlimbo;
+
+	/*
+	 * Tree of profile counters, one for each thread that has allocated in
+	 * this context.
+	 */
+	prof_tctx_tree_t tctxs;
+
+	/* Linkage for tree of contexts to be dumped. */
+	rb_node(prof_gctx_t) dump_link;
+
+	/* Temporary storage for summation during dump. */
+	prof_cnt_t cnt_summed;
+
+	/* Associated backtrace. */
+	prof_bt_t bt;
+
+	/* Backtrace vector, variable size, referred to by bt. */
+	void *vec[1];
+};
+typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
+
+struct prof_tdata_s {
+	malloc_mutex_t *lock;
+
+	/* Monotonically increasing unique thread identifier. */
+	uint64_t thr_uid;
+
+	/*
+	 * Monotonically increasing discriminator among tdata structures
+	 * associated with the same thr_uid.
+	 */
+	uint64_t thr_discrim;
+
+	rb_node(prof_tdata_t) tdata_link;
+
+	/*
+	 * Counter used to initialize prof_tctx_t's tctx_uid.  No locking is
+	 * necessary when incrementing this field, because only one thread ever
+	 * does so.
+	 */
+	uint64_t tctx_uid_next;
+
+	/*
+	 * Hash of (prof_bt_t *)-->(prof_tctx_t *).  Each thread tracks
+	 * backtraces for which it has non-zero allocation/deallocation counters
+	 * associated with thread-specific prof_tctx_t objects.  Other threads
+	 * may write to prof_tctx_t contents when freeing associated objects.
+	 */
+	ckh_t bt2tctx;
+
+	/* Included in heap profile dumps if has content. */
+	char thread_name[PROF_THREAD_NAME_MAX_LEN];
+
+	/* State used to avoid dumping while operating on prof internals. */
+	bool enq;
+	bool enq_idump;
+	bool enq_gdump;
+
+	/*
+	 * Set to true during an early dump phase for tdata's which are
+	 * currently being dumped.  New threads' tdata's have this initialized
+	 * to false so that they aren't accidentally included in later dump
+	 * phases.
+	 */
+	bool dumping;
+
+	/*
+	 * True if profiling is active for this tdata's thread
+	 * (thread.prof.active mallctl).
+	 */
+	bool active;
+
+	bool attached;
+	bool expired;
+
+	/* Temporary storage for summation during dump. */
+	prof_cnt_t cnt_summed;
+
+	/* Backtrace vector, used for calls to prof_backtrace(). */
+	void **vec;
+};
+typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
+
+struct prof_recent_s {
+	nstime_t alloc_time;
+	nstime_t dalloc_time;
+
+	ql_elm(prof_recent_t) link;
+	size_t       size;
+	size_t       usize;
+	atomic_p_t   alloc_edata; /* NULL means allocation has been freed. */
+	prof_tctx_t *alloc_tctx;
+	prof_tctx_t *dalloc_tctx;
+};
+
+/******************************************************************************/
+/* EXTERNS */
+/******************************************************************************/
+
+extern bool     opt_prof;
+extern bool     opt_prof_active;
+extern bool     opt_prof_thread_active_init;
+extern unsigned opt_prof_bt_max;
+extern size_t   opt_lg_prof_sample; /* Mean bytes between samples. */
+extern ssize_t opt_lg_prof_interval;    /* lg(prof_interval). */
+extern bool    opt_prof_gdump;          /* High-water memory dumping. */
+extern bool    opt_prof_final;          /* Final profile dumping. */
+extern bool    opt_prof_leak;           /* Dump leak summary at exit. */
+extern bool    opt_prof_leak_error; /* Exit with error code if memory leaked */
+extern bool    opt_prof_accum;      /* Report cumulative bytes. */
+extern bool    opt_prof_log;        /* Turn logging on at boot. */
+extern char    opt_prof_prefix[
+/* Minimize memory bloat for non-prof builds. */
+#ifdef JEMALLOC_PROF
+    PATH_MAX +
+#endif
+    1];
+extern bool opt_prof_unbias;
+
+/* Include pid namespace in profile file names. */
+extern bool opt_prof_pid_namespace;
+
+/* For recording recent allocations */
+extern ssize_t opt_prof_recent_alloc_max;
+
+/* Whether to use thread name provided by the system or by mallctl. */
+extern bool opt_prof_sys_thread_name;
+
+/* Whether to record per size class counts and request size totals. */
+extern bool opt_prof_stats;
+
+/* Accessed via prof_active_[gs]et{_unlocked,}(). */
+extern bool prof_active_state;
+
+/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
+extern bool prof_gdump_val;
+
+/* Profile dump interval, measured in bytes allocated. */
+extern uint64_t prof_interval;
+
+/*
+ * Initialized as opt_lg_prof_sample, and potentially modified during profiling
+ * resets.
+ */
+extern size_t lg_prof_sample;
+
+extern bool prof_booted;
+
+void                  prof_backtrace_hook_set(prof_backtrace_hook_t hook);
+prof_backtrace_hook_t prof_backtrace_hook_get(void);
+
+void             prof_dump_hook_set(prof_dump_hook_t hook);
+prof_dump_hook_t prof_dump_hook_get(void);
+
+void               prof_sample_hook_set(prof_sample_hook_t hook);
+prof_sample_hook_t prof_sample_hook_get(void);
+
+void                    prof_sample_free_hook_set(prof_sample_free_hook_t hook);
+prof_sample_free_hook_t prof_sample_free_hook_get(void);
+
+/* Functions only accessed in prof_inlines.h */
+prof_tdata_t *prof_tdata_init(tsd_t *tsd);
+prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
+
+void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
+void prof_malloc_sample_object(
+    tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
+void prof_free_sampled_object(
+    tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
+prof_tctx_t *prof_tctx_create(tsd_t *tsd);
+void         prof_idump(tsdn_t *tsdn);
+bool         prof_mdump(tsd_t *tsd, const char *filename);
+void         prof_gdump(tsdn_t *tsdn);
+
+void        prof_tdata_cleanup(tsd_t *tsd);
+bool        prof_active_get(tsdn_t *tsdn);
+bool        prof_active_set(tsdn_t *tsdn, bool active);
+const char *prof_thread_name_get(tsd_t *tsd);
+int         prof_thread_name_set(tsd_t *tsd, const char *thread_name);
+bool        prof_thread_active_get(tsd_t *tsd);
+bool        prof_thread_active_set(tsd_t *tsd, bool active);
+bool        prof_thread_active_init_get(tsdn_t *tsdn);
+bool        prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
+bool        prof_gdump_get(tsdn_t *tsdn);
+bool        prof_gdump_set(tsdn_t *tsdn, bool active);
+void        prof_boot0(void);
+void        prof_boot1(void);
+bool        prof_boot2(tsd_t *tsd, base_t *base);
+void        prof_prefork0(tsdn_t *tsdn);
+void        prof_prefork1(tsdn_t *tsdn);
+void        prof_postfork_parent(tsdn_t *tsdn);
+void        prof_postfork_child(tsdn_t *tsdn);
+
+uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
+
+extern te_base_cb_t prof_sample_te_handler;
+
+#endif /* JEMALLOC_INTERNAL_PROF_H */

include/jemalloc/internal/prof_data.h

@@ -29,9 +29,4 @@ void          prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata);
 void          prof_reset(tsd_t *tsd, size_t lg_sample);
 void          prof_tctx_try_destroy(tsd_t *tsd, prof_tctx_t *tctx);
 
-/* Used in unit tests. */
-size_t prof_tdata_count(void);
-size_t prof_bt_count(void);
-void   prof_cnt_all(prof_cnt_t *cnt_all);
-
 #endif /* JEMALLOC_INTERNAL_PROF_DATA_H */

include/jemalloc/internal/prof_externs.h

@@ -1,165 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PROF_EXTERNS_H
-#define JEMALLOC_INTERNAL_PROF_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/base.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/prof_hook.h"
-#include "jemalloc/internal/thread_event_registry.h"
-
-extern bool     opt_prof;
-extern bool     opt_prof_active;
-extern bool     opt_prof_thread_active_init;
-extern unsigned opt_prof_bt_max;
-extern size_t   opt_lg_prof_sample; /* Mean bytes between samples. */
-extern ssize_t opt_lg_prof_interval;    /* lg(prof_interval). */
-extern bool    opt_prof_gdump;          /* High-water memory dumping. */
-extern bool    opt_prof_final;          /* Final profile dumping. */
-extern bool    opt_prof_leak;           /* Dump leak summary at exit. */
-extern bool    opt_prof_leak_error; /* Exit with error code if memory leaked */
-extern bool    opt_prof_accum;      /* Report cumulative bytes. */
-extern bool    opt_prof_log;        /* Turn logging on at boot. */
-extern char    opt_prof_prefix[
-/* Minimize memory bloat for non-prof builds. */
-#ifdef JEMALLOC_PROF
-    PATH_MAX +
-#endif
-    1];
-extern bool opt_prof_unbias;
-
-/* Include pid namespace in profile file names. */
-extern bool opt_prof_pid_namespace;
-
-/* For recording recent allocations */
-extern ssize_t opt_prof_recent_alloc_max;
-
-/* Whether to use thread name provided by the system or by mallctl. */
-extern bool opt_prof_sys_thread_name;
-
-/* Whether to record per size class counts and request size totals. */
-extern bool opt_prof_stats;
-
-/* Accessed via prof_active_[gs]et{_unlocked,}(). */
-extern bool prof_active_state;
-
-/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */
-extern bool prof_gdump_val;
-
-/* Profile dump interval, measured in bytes allocated. */
-extern uint64_t prof_interval;
-
-/*
- * Initialized as opt_lg_prof_sample, and potentially modified during profiling
- * resets.
- */
-extern size_t lg_prof_sample;
-
-extern bool prof_booted;
-
-void                  prof_backtrace_hook_set(prof_backtrace_hook_t hook);
-prof_backtrace_hook_t prof_backtrace_hook_get(void);
-
-void             prof_dump_hook_set(prof_dump_hook_t hook);
-prof_dump_hook_t prof_dump_hook_get(void);
-
-void               prof_sample_hook_set(prof_sample_hook_t hook);
-prof_sample_hook_t prof_sample_hook_get(void);
-
-void                    prof_sample_free_hook_set(prof_sample_free_hook_t hook);
-prof_sample_free_hook_t prof_sample_free_hook_get(void);
-
-/* Functions only accessed in prof_inlines.h */
-prof_tdata_t *prof_tdata_init(tsd_t *tsd);
-prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
-
-void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx);
-void prof_malloc_sample_object(
-    tsd_t *tsd, const void *ptr, size_t size, size_t usize, prof_tctx_t *tctx);
-void prof_free_sampled_object(
-    tsd_t *tsd, const void *ptr, size_t usize, prof_info_t *prof_info);
-prof_tctx_t *prof_tctx_create(tsd_t *tsd);
-void         prof_idump(tsdn_t *tsdn);
-bool         prof_mdump(tsd_t *tsd, const char *filename);
-void         prof_gdump(tsdn_t *tsdn);
-
-void        prof_tdata_cleanup(tsd_t *tsd);
-bool        prof_active_get(tsdn_t *tsdn);
-bool        prof_active_set(tsdn_t *tsdn, bool active);
-const char *prof_thread_name_get(tsd_t *tsd);
-int         prof_thread_name_set(tsd_t *tsd, const char *thread_name);
-bool        prof_thread_active_get(tsd_t *tsd);
-bool        prof_thread_active_set(tsd_t *tsd, bool active);
-bool        prof_thread_active_init_get(tsdn_t *tsdn);
-bool        prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
-bool        prof_gdump_get(tsdn_t *tsdn);
-bool        prof_gdump_set(tsdn_t *tsdn, bool active);
-void        prof_boot0(void);
-void        prof_boot1(void);
-bool        prof_boot2(tsd_t *tsd, base_t *base);
-void        prof_prefork0(tsdn_t *tsdn);
-void        prof_prefork1(tsdn_t *tsdn);
-void        prof_postfork_parent(tsdn_t *tsdn);
-void        prof_postfork_child(tsdn_t *tsdn);
-
-uint64_t prof_sample_new_event_wait(tsd_t *tsd);
-uint64_t tsd_prof_sample_event_wait_get(tsd_t *tsd);
-
-/*
- * The lookahead functionality facilitates events to be able to lookahead, i.e.
- * without touching the event counters, to determine whether an event would be
- * triggered.  The event counters are not advanced until the end of the
- * allocation / deallocation calls, so the lookahead can be useful if some
- * preparation work for some event must be done early in the allocation /
- * deallocation calls.
- *
- * Currently only the profiling sampling event needs the lookahead
- * functionality, so we don't yet define general purpose lookahead functions.
- *
- * Surplus is a terminology referring to the amount of bytes beyond what's
- * needed for triggering an event, which can be a useful quantity to have in
- * general when lookahead is being called.
- *
- * This function returns true if allocation of usize would go above the next
- * trigger for prof event, and false otherwise.
- * If function returns true surplus will contain number of bytes beyond that
- * trigger.
- */
-
-JEMALLOC_ALWAYS_INLINE bool
-te_prof_sample_event_lookahead_surplus(
-    tsd_t *tsd, size_t usize, size_t *surplus) {
-	if (surplus != NULL) {
-		/*
-		 * This is a dead store: the surplus will be overwritten before
-		 * any read.  The initialization suppresses compiler warnings.
-		 * Meanwhile, using SIZE_MAX to initialize is good for
-		 * debugging purpose, because a valid surplus value is strictly
-		 * less than usize, which is at most SIZE_MAX.
-		 */
-		*surplus = SIZE_MAX;
-	}
-	if (unlikely(!tsd_nominal(tsd) || tsd_reentrancy_level_get(tsd) > 0)) {
-		return false;
-	}
-	/* The subtraction is intentionally susceptible to underflow. */
-	uint64_t accumbytes = tsd_thread_allocated_get(tsd) + usize
-	    - tsd_thread_allocated_last_event_get(tsd);
-	uint64_t sample_wait = tsd_prof_sample_event_wait_get(tsd);
-	if (accumbytes < sample_wait) {
-		return false;
-	}
-	assert(accumbytes - sample_wait < (uint64_t)usize);
-	if (surplus != NULL) {
-		*surplus = (size_t)(accumbytes - sample_wait);
-	}
-	return true;
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-te_prof_sample_event_lookahead(tsd_t *tsd, size_t usize) {
-	return te_prof_sample_event_lookahead_surplus(tsd, usize, NULL);
-}
-
-extern te_base_cb_t prof_sample_te_handler;
-
-#endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */

include/jemalloc/internal/prof_inlines.h

@@ -2,10 +2,9 @@
 #define JEMALLOC_INTERNAL_PROF_INLINES_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_c.h"
-#include "jemalloc/internal/prof_externs.h"
-#include "jemalloc/internal/prof_structs.h"
+#include "jemalloc/internal/prof.h"
 #include "jemalloc/internal/safety_check.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/thread_event.h"

include/jemalloc/internal/prof_log.h

@@ -9,14 +9,6 @@ extern malloc_mutex_t log_mtx;
 void prof_try_log(tsd_t *tsd, size_t usize, prof_info_t *prof_info);
 bool prof_log_init(tsd_t *tsdn);
 
-/* Used in unit tests. */
-size_t prof_log_bt_count(void);
-size_t prof_log_alloc_count(void);
-size_t prof_log_thr_count(void);
-bool   prof_log_is_logging(void);
-bool   prof_log_rep_check(void);
-void   prof_log_dummy_set(bool new_value);
-
 bool prof_log_start(tsdn_t *tsdn, const char *filename);
 bool prof_log_stop(tsdn_t *tsdn);
 

include/jemalloc/internal/prof_recent.h

@@ -17,8 +17,6 @@ void edata_prof_recent_alloc_init(edata_t *edata);
 /* Used in unit tests. */
 typedef ql_head(prof_recent_t) prof_recent_list_t;
 extern prof_recent_list_t prof_recent_alloc_list;
-edata_t *prof_recent_alloc_edata_get_no_lock_test(const prof_recent_t *node);
-prof_recent_t *edata_prof_recent_alloc_get_no_lock_test(const edata_t *edata);
 
 ssize_t prof_recent_alloc_max_ctl_read(void);
 ssize_t prof_recent_alloc_max_ctl_write(tsd_t *tsd, ssize_t max);

include/jemalloc/internal/prof_structs.h

@@ -1,222 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PROF_STRUCTS_H
-#define JEMALLOC_INTERNAL_PROF_STRUCTS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/ckh.h"
-#include "jemalloc/internal/edata.h"
-#include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/prng.h"
-#include "jemalloc/internal/rb.h"
-
-struct prof_bt_s {
-	/* Backtrace, stored as len program counters. */
-	void   **vec;
-	unsigned len;
-};
-
-#ifdef JEMALLOC_PROF_LIBGCC
-/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */
-typedef struct {
-	void    **vec;
-	unsigned *len;
-	unsigned  max;
-} prof_unwind_data_t;
-#endif
-
-struct prof_cnt_s {
-	/* Profiling counters. */
-	uint64_t curobjs;
-	uint64_t curobjs_shifted_unbiased;
-	uint64_t curbytes;
-	uint64_t curbytes_unbiased;
-	uint64_t accumobjs;
-	uint64_t accumobjs_shifted_unbiased;
-	uint64_t accumbytes;
-	uint64_t accumbytes_unbiased;
-};
-
-typedef enum {
-	prof_tctx_state_initializing,
-	prof_tctx_state_nominal,
-	prof_tctx_state_dumping,
-	prof_tctx_state_purgatory /* Dumper must finish destroying. */
-} prof_tctx_state_t;
-
-struct prof_tctx_s {
-	/* Thread data for thread that performed the allocation. */
-	prof_tdata_t *tdata;
-
-	/*
-	 * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be
-	 * defunct during teardown.
-	 */
-	uint64_t thr_uid;
-	uint64_t thr_discrim;
-
-	/*
-	 * Reference count of how many times this tctx object is referenced in
-	 * recent allocation / deallocation records, protected by tdata->lock.
-	 */
-	uint64_t recent_count;
-
-	/* Profiling counters, protected by tdata->lock. */
-	prof_cnt_t cnts;
-
-	/* Associated global context. */
-	prof_gctx_t *gctx;
-
-	/*
-	 * UID that distinguishes multiple tctx's created by the same thread,
-	 * but coexisting in gctx->tctxs.  There are two ways that such
-	 * coexistence can occur:
-	 * - A dumper thread can cause a tctx to be retained in the purgatory
-	 *   state.
-	 * - Although a single "producer" thread must create all tctx's which
-	 *   share the same thr_uid, multiple "consumers" can each concurrently
-	 *   execute portions of prof_tctx_destroy().  prof_tctx_destroy() only
-	 *   gets called once each time cnts.cur{objs,bytes} drop to 0, but this
-	 *   threshold can be hit again before the first consumer finishes
-	 *   executing prof_tctx_destroy().
-	 */
-	uint64_t tctx_uid;
-
-	/* Linkage into gctx's tctxs. */
-	rb_node(prof_tctx_t) tctx_link;
-
-	/*
-	 * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents
-	 * sample vs destroy race.
-	 */
-	bool prepared;
-
-	/* Current dump-related state, protected by gctx->lock. */
-	prof_tctx_state_t state;
-
-	/*
-	 * Copy of cnts snapshotted during early dump phase, protected by
-	 * dump_mtx.
-	 */
-	prof_cnt_t dump_cnts;
-};
-typedef rb_tree(prof_tctx_t) prof_tctx_tree_t;
-
-struct prof_info_s {
-	/* Time when the allocation was made. */
-	nstime_t alloc_time;
-	/* Points to the prof_tctx_t corresponding to the allocation. */
-	prof_tctx_t *alloc_tctx;
-	/* Allocation request size. */
-	size_t alloc_size;
-};
-
-struct prof_gctx_s {
-	/* Protects nlimbo, cnt_summed, and tctxs. */
-	malloc_mutex_t *lock;
-
-	/*
-	 * Number of threads that currently cause this gctx to be in a state of
-	 * limbo due to one of:
-	 *   - Initializing this gctx.
-	 *   - Initializing per thread counters associated with this gctx.
-	 *   - Preparing to destroy this gctx.
-	 *   - Dumping a heap profile that includes this gctx.
-	 * nlimbo must be 1 (single destroyer) in order to safely destroy the
-	 * gctx.
-	 */
-	unsigned nlimbo;
-
-	/*
-	 * Tree of profile counters, one for each thread that has allocated in
-	 * this context.
-	 */
-	prof_tctx_tree_t tctxs;
-
-	/* Linkage for tree of contexts to be dumped. */
-	rb_node(prof_gctx_t) dump_link;
-
-	/* Temporary storage for summation during dump. */
-	prof_cnt_t cnt_summed;
-
-	/* Associated backtrace. */
-	prof_bt_t bt;
-
-	/* Backtrace vector, variable size, referred to by bt. */
-	void *vec[1];
-};
-typedef rb_tree(prof_gctx_t) prof_gctx_tree_t;
-
-struct prof_tdata_s {
-	malloc_mutex_t *lock;
-
-	/* Monotonically increasing unique thread identifier. */
-	uint64_t thr_uid;
-
-	/*
-	 * Monotonically increasing discriminator among tdata structures
-	 * associated with the same thr_uid.
-	 */
-	uint64_t thr_discrim;
-
-	rb_node(prof_tdata_t) tdata_link;
-
-	/*
-	 * Counter used to initialize prof_tctx_t's tctx_uid.  No locking is
-	 * necessary when incrementing this field, because only one thread ever
-	 * does so.
-	 */
-	uint64_t tctx_uid_next;
-
-	/*
-	 * Hash of (prof_bt_t *)-->(prof_tctx_t *).  Each thread tracks
-	 * backtraces for which it has non-zero allocation/deallocation counters
-	 * associated with thread-specific prof_tctx_t objects.  Other threads
-	 * may write to prof_tctx_t contents when freeing associated objects.
-	 */
-	ckh_t bt2tctx;
-
-	/* Included in heap profile dumps if has content. */
-	char thread_name[PROF_THREAD_NAME_MAX_LEN];
-
-	/* State used to avoid dumping while operating on prof internals. */
-	bool enq;
-	bool enq_idump;
-	bool enq_gdump;
-
-	/*
-	 * Set to true during an early dump phase for tdata's which are
-	 * currently being dumped.  New threads' tdata's have this initialized
-	 * to false so that they aren't accidentally included in later dump
-	 * phases.
-	 */
-	bool dumping;
-
-	/*
-	 * True if profiling is active for this tdata's thread
-	 * (thread.prof.active mallctl).
-	 */
-	bool active;
-
-	bool attached;
-	bool expired;
-
-	/* Temporary storage for summation during dump. */
-	prof_cnt_t cnt_summed;
-
-	/* Backtrace vector, used for calls to prof_backtrace(). */
-	void **vec;
-};
-typedef rb_tree(prof_tdata_t) prof_tdata_tree_t;
-
-struct prof_recent_s {
-	nstime_t alloc_time;
-	nstime_t dalloc_time;
-
-	ql_elm(prof_recent_t) link;
-	size_t       size;
-	size_t       usize;
-	atomic_p_t   alloc_edata; /* NULL means allocation has been freed. */
-	prof_tctx_t *alloc_tctx;
-	prof_tctx_t *dalloc_tctx;
-};
-
-#endif /* JEMALLOC_INTERNAL_PROF_STRUCTS_H */

include/jemalloc/internal/prof_sys.h

@@ -4,6 +4,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/prof.h"
 
 extern malloc_mutex_t prof_dump_filename_mtx;
 extern base_t        *prof_base;

include/jemalloc/internal/prof_types.h

@@ -1,94 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_PROF_TYPES_H
-#define JEMALLOC_INTERNAL_PROF_TYPES_H
-
-typedef struct prof_bt_s     prof_bt_t;
-typedef struct prof_cnt_s    prof_cnt_t;
-typedef struct prof_tctx_s   prof_tctx_t;
-typedef struct prof_info_s   prof_info_t;
-typedef struct prof_gctx_s   prof_gctx_t;
-typedef struct prof_tdata_s  prof_tdata_t;
-typedef struct prof_recent_s prof_recent_t;
-
-/* Option defaults. */
-#ifdef JEMALLOC_PROF
-#	define PROF_PREFIX_DEFAULT "jeprof"
-#else
-#	define PROF_PREFIX_DEFAULT ""
-#endif
-#define LG_PROF_SAMPLE_DEFAULT 19
-#define LG_PROF_INTERVAL_DEFAULT -1
-
-/*
- * Hard limit on stack backtrace depth.  The version of prof_backtrace() that
- * is based on __builtin_return_address() necessarily has a hard-coded number
- * of backtrace frame handlers, and should be kept in sync with this setting.
- */
-#ifdef JEMALLOC_PROF_GCC
-#	define PROF_BT_MAX_LIMIT 256
-#else
-#	define PROF_BT_MAX_LIMIT UINT_MAX
-#endif
-#define PROF_BT_MAX_DEFAULT 128
-
-/* Initial hash table size. */
-#define PROF_CKH_MINITEMS 64
-
-/* Size of memory buffer to use when writing dump files. */
-#ifndef JEMALLOC_PROF
-/* Minimize memory bloat for non-prof builds. */
-#	define PROF_DUMP_BUFSIZE 1
-#elif defined(JEMALLOC_DEBUG)
-/* Use a small buffer size in debug build, mainly to facilitate testing. */
-#	define PROF_DUMP_BUFSIZE 16
-#else
-#	define PROF_DUMP_BUFSIZE 65536
-#endif
-
-/* Size of size class related tables */
-#ifdef JEMALLOC_PROF
-#	define PROF_SC_NSIZES SC_NSIZES
-#else
-/* Minimize memory bloat for non-prof builds. */
-#	define PROF_SC_NSIZES 1
-#endif
-
-/* Size of stack-allocated buffer used by prof_printf(). */
-#define PROF_PRINTF_BUFSIZE 128
-
-/*
- * Number of mutexes shared among all gctx's.  No space is allocated for these
- * unless profiling is enabled, so it's okay to over-provision.
- */
-#define PROF_NCTX_LOCKS 1024
-
-/*
- * Number of mutexes shared among all tdata's.  No space is allocated for these
- * unless profiling is enabled, so it's okay to over-provision.
- */
-#define PROF_NTDATA_LOCKS 256
-
-/* Minimize memory bloat for non-prof builds. */
-#ifdef JEMALLOC_PROF
-#	define PROF_DUMP_FILENAME_LEN (PATH_MAX + 1)
-#else
-#	define PROF_DUMP_FILENAME_LEN 1
-#endif
-
-/* Default number of recent allocations to record. */
-#define PROF_RECENT_ALLOC_MAX_DEFAULT 0
-
-/* Thread name storage size limit. */
-#define PROF_THREAD_NAME_MAX_LEN 16
-
-/*
- * Minimum required alignment for sampled allocations. Over-aligning sampled
- * allocations allows us to quickly identify them on the dalloc path without
- * resorting to metadata lookup.
- */
-#define PROF_SAMPLE_ALIGNMENT PAGE
-#define PROF_SAMPLE_ALIGNMENT_MASK PAGE_MASK
-
-/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
-#define PROF_TCTX_SENTINEL ((prof_tctx_t *)((uintptr_t)1U))
-
-#endif /* JEMALLOC_INTERNAL_PROF_TYPES_H */

include/jemalloc/internal/psset.h

@@ -134,17 +134,17 @@ void psset_insert(psset_t *psset, hpdata_t *ps);
 void psset_remove(psset_t *psset, hpdata_t *ps);
 
 static inline size_t
-psset_npageslabs(psset_t *psset) {
+psset_npageslabs(const psset_t *psset) {
 	return psset->stats.merged.npageslabs;
 }
 
 static inline size_t
-psset_nactive(psset_t *psset) {
+psset_nactive(const psset_t *psset) {
 	return psset->stats.merged.nactive;
 }
 
 static inline size_t
-psset_ndirty(psset_t *psset) {
+psset_ndirty(const psset_t *psset) {
 	return psset->stats.merged.ndirty;
 }
 

include/jemalloc/internal/safety_check.h

@@ -11,11 +11,6 @@ void safety_check_fail_sized_dealloc(
     bool current_dealloc, const void *ptr, size_t true_size, size_t input_size);
 void safety_check_fail(const char *format, ...);
 
-typedef void (*safety_check_abort_hook_t)(const char *message);
-
-/* Can set to NULL for a default. */
-void safety_check_set_abort(safety_check_abort_hook_t abort_fn);
-
 #define REDZONE_SIZE ((size_t)32)
 #define REDZONE_FILL_VALUE 0xBC
 

include/jemalloc/internal/sc.h

@@ -268,12 +268,17 @@
 #define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP))
 #define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP)
 
-/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */
-#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2))
-#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP))
-
-/* The largest size class supported. */
-#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA)
+/*
+ * The largest size class supported.  Spell this out directly to avoid
+ * expanding subtractive arithmetic at every use site.
+ */
+#if LG_SIZEOF_PTR == 3
+#	define SC_LARGE_MAXCLASS 0x7000000000000000ULL
+#elif LG_SIZEOF_PTR == 2
+#	define SC_LARGE_MAXCLASS 0x70000000ULL
+#else
+#	error "Unsupported pointer size"
+#endif
 
 /* Maximum number of regions in one slab. */
 #ifndef CONFIG_LG_SLAB_MAXREGS
@@ -363,7 +368,6 @@ struct sc_data_s {
 	sc_t sc[SC_NSIZES];
 };
 
-size_t reg_size_compute(int lg_base, int lg_delta, int ndelta);
 void   sc_data_init(sc_data_t *data);
 /*
  * Updates slab sizes in [begin, end] to be pgs pages in length, if possible.

include/jemalloc/internal/sec.h

@@ -4,9 +4,10 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/edata.h"
 #include "jemalloc/internal/mutex.h"
-#include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/sec_opts.h"
+#include "jemalloc/internal/tsd_types.h"
 
 /*
  * Small extent cache.
@@ -34,19 +35,20 @@ typedef struct sec_stats_s sec_stats_t;
 struct sec_stats_s {
 	/* Sum of bytes_cur across all shards. */
 	size_t bytes;
+	/* Subset of bytes that are pinned. */
+	size_t bytes_pinned;
 
 	/* Totals of bin_stats. */
 	sec_bin_stats_t total;
 };
 
-static inline void
-sec_bin_stats_init(sec_bin_stats_t *stats) {
-	stats->ndalloc_flush = 0;
-	stats->nmisses = 0;
-	stats->nhits = 0;
-	stats->ndalloc_noflush = 0;
-	stats->noverfills = 0;
-}
+typedef struct sec_pszind_stats_s sec_pszind_stats_t;
+struct sec_pszind_stats_s {
+	size_t nextents;
+	size_t bytes;
+	size_t nextents_pinned;
+	size_t bytes_pinned;
+};
 
 static inline void
 sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
@@ -60,6 +62,7 @@ sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
 static inline void
 sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 	dst->bytes += src->bytes;
+	dst->bytes_pinned += src->bytes_pinned;
 	sec_bin_stats_accum(&dst->total, &src->total);
 }
 
@@ -67,16 +70,21 @@ sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 typedef struct sec_bin_s sec_bin_t;
 struct sec_bin_s {
 	/*
-	 * Protects the data members of the bin.
+	 * Protects the freelist and synchronizes counter updates.
 	 */
 	malloc_mutex_t mtx;
 
 	/*
 	 * Number of bytes in this particular bin.
 	 */
-	size_t              bytes_cur;
+	atomic_zu_t         bytes_cur;
+	atomic_zu_t         bytes_pinned_cur;
 	edata_list_active_t freelist;
-	sec_bin_stats_t     stats;
+	atomic_zu_t         nmisses;
+	atomic_zu_t         nhits;
+	atomic_zu_t         ndalloc_flush;
+	atomic_zu_t         ndalloc_noflush;
+	atomic_zu_t         noverfills;
 };
 
 typedef struct sec_s sec_t;
@@ -87,21 +95,49 @@ struct sec_s {
 };
 
 static inline bool
-sec_is_used(sec_t *sec) {
+sec_is_used(const sec_t *sec) {
 	return sec->opts.nshards != 0;
 }
 
 static inline bool
-sec_size_supported(sec_t *sec, size_t size) {
-	return sec_is_used(sec) && size <= sec->opts.max_alloc;
+sec_size_supported(const sec_t *sec, size_t size) {
+	return size <= sec->opts.max_alloc;
 }
 
-/* If sec does not have extent available, it will return NULL. */
-edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
-void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
-        edata_list_active_t *result, size_t nallocs);
+/* Min number of extents we will allocate when expanding the SEC. */
+#define SEC_MIN_NALLOCS 2
+
+/* Max number of extents we will allocate out of a single huge page. */
+#define SEC_MAX_NALLOCS 8
+
+/* Attempt to fill the SEC up to max_bytes / SEC_MAX_BYTES_DIV */
+#define SEC_MAX_BYTES_DIV 4
 
 /*
+ * Calculate the min and max number of extents we will try to allocate
+ * when expanding the SEC. We will attempt to allocate at least min
+ * extents and up to max extents depending on whether we can allocate
+ * them out of a huge page we have already allocated out of. Both
+ * min and max should the in the range [1, SEC_MAX_NALLOCS].
+ */
+void sec_calc_nallocs_for_size(
+    sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
+
+/*
+ * Lazily picks (and caches in *idxp) a shard for the calling thread.  Different
+ * SEC instances pass independent per-thread uint8_t slots, initialized to
+ * (uint8_t)-1.
+ */
+uint8_t sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp);
+
+/* Callers must ensure sec_size_supported(sec, size). */
+edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard);
+void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
+        edata_list_active_t *result, size_t nallocs, uint8_t shard);
+
+/*
+ * Callers must ensure sec_size_supported(sec, edata_size).
+ *
  * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty
  * if there are extents that need to be flushed from cache.  Please note, that
  * if we need to flush, extent(s) returned in the list to be deallocated
@@ -109,7 +145,8 @@ void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
  * considered "hot" and preserved in the cache, while "colder" ones are
  * returned).
  */
-void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list);
+void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
+    uint8_t shard);
 
 bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts);
 
@@ -122,7 +159,10 @@ void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
  * lets them fit easily into the pa_shard stats framework (which also has this
  * split), which simplifies the stats management.
  */
-void sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge_pszind(
+    tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
+    sec_pszind_stats_t *stats);
 void sec_mutex_stats_read(
     tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data);
 

include/jemalloc/internal/sec_opts.h

@@ -27,16 +27,9 @@ struct sec_opts_s {
 	 * until we are 1/4 below max_bytes.
 	 */
 	size_t max_bytes;
-	/*
-	 * When we can't satisfy an allocation out of the SEC because there are
-	 * no available ones cached, allocator will allocate a batch with extra
-	 * batch_fill_extra extents of the same size.
-	 */
-	size_t batch_fill_extra;
 };
 
 #define SEC_OPTS_NSHARDS_DEFAULT 2
-#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3
 #define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024))
 #define SEC_OPTS_MAX_BYTES_DEFAULT                                             \
 	((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT)                       \
@@ -45,6 +38,6 @@ struct sec_opts_s {
 
 #define SEC_OPTS_DEFAULT                                                       \
 	{SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT,                 \
-	    SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT}
+	    SEC_OPTS_MAX_BYTES_DEFAULT}
 
 #endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

include/jemalloc/internal/seq.h

@@ -1,58 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_SEQ_H
-#define JEMALLOC_INTERNAL_SEQ_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/atomic.h"
-
-/*
- * A simple seqlock implementation.
- */
-
-/* clang-format off */
-#define seq_define(type, short_type)					\
-typedef struct {							\
-	atomic_zu_t seq;						\
-	atomic_zu_t data[						\
-	    (sizeof(type) + sizeof(size_t) - 1) / sizeof(size_t)];	\
-} seq_##short_type##_t;							\
-									\
-/*									\
- * No internal synchronization -- the caller must ensure that there's	\
- * only a single writer at a time.					\
- */									\
-static inline void							\
-seq_store_##short_type(seq_##short_type##_t *dst, type *src) {		\
-	size_t buf[sizeof(dst->data) / sizeof(size_t)];			\
-	buf[sizeof(buf) / sizeof(size_t) - 1] = 0;			\
-	memcpy(buf, src, sizeof(type));					\
-	size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED);	\
-	atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED);	\
-	atomic_fence(ATOMIC_RELEASE);					\
-	for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) {	\
-		atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED);	\
-	}								\
-	atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE);	\
-}									\
-									\
-/* Returns whether or not the read was consistent. */			\
-static inline bool							\
-seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) {	\
-	size_t buf[sizeof(src->data) / sizeof(size_t)];			\
-	size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE);	\
-	if (seq1 % 2 != 0) {						\
-		return false;						\
-	}								\
-	for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) {	\
-		buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED);	\
-	}								\
-	atomic_fence(ATOMIC_ACQUIRE);					\
-	size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED);	\
-	if (seq1 != seq2) {						\
-		return false;						\
-	}								\
-	memcpy(dst, buf, sizeof(type));					\
-	return true;							\
-}
-/* clang-format on */
-
-#endif /* JEMALLOC_INTERNAL_SEQ_H */

include/jemalloc/internal/stats.h

@@ -43,6 +43,9 @@ extern char    opt_stats_interval_opts[stats_print_tot_num_options + 1];
 #define STATS_INTERVAL_ACCUM_LG_BATCH_SIZE 6
 #define STATS_INTERVAL_ACCUM_BATCH_MAX (4 << 20)
 
+/* Per thread batch accum size for stats_interval; read by thread event. */
+extern uint64_t stats_interval_accum_batch;
+
 /* Only accessed by thread event. */
 extern te_base_cb_t stats_interval_te_handler;
 

include/jemalloc/internal/sz.h

@@ -3,6 +3,7 @@
 
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/bit_util.h"
+#include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/pages.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/util.h"
@@ -314,13 +315,23 @@ sz_size2index_usize_fastpath(size_t size, szind_t *ind, size_t *usize) {
 
 JEMALLOC_ALWAYS_INLINE size_t
 sz_s2u_compute_using_delta(size_t size) {
+	if (unlikely(size > SC_LARGE_MAXCLASS)) {
+		return 0;
+	}
+
 	size_t x = lg_floor((size << 1) - 1);
 	size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1)
 	    ? LG_QUANTUM
 	    : x - SC_LG_NGROUP - 1;
 	size_t delta = ZU(1) << lg_delta;
 	size_t delta_mask = delta - 1;
-	size_t usize = (size + delta_mask) & ~delta_mask;
+
+	size_t usize = size + delta_mask;
+	if (unlikely(usize < size)) {
+		/* size_t overflow. */
+		return 0;
+	}
+	usize &= ~delta_mask;
 	return usize;
 }
 
@@ -350,7 +361,12 @@ sz_s2u_compute(size_t size) {
 		 * multiple of PAGE to minimize the memory overhead, especially
 		 * when using hugepages.
 		 */
-		size_t usize = PAGE_CEILING(size);
+		size_t usize = size + PAGE_MASK;
+		if (unlikely(usize < size)) {
+			/* size_t overflow. */
+			return 0;
+		}
+		usize &= ~PAGE_MASK;
 		assert(usize - size < PAGE);
 		return usize;
 	}
@@ -385,7 +401,11 @@ JEMALLOC_ALWAYS_INLINE size_t
 sz_sa2u(size_t size, size_t alignment) {
 	size_t usize;
 
-	assert(alignment != 0 && ((alignment - 1) & alignment) == 0);
+	if (unlikely(alignment == 0)) {
+		return 0;
+	}
+	size_t alignment_mask = alignment - 1;
+	assert((alignment_mask & alignment) == 0);
 
 	/* Try for a small size class. */
 	if (size <= SC_SMALL_MAXCLASS && alignment <= PAGE) {
@@ -403,7 +423,12 @@ sz_sa2u(size_t size, size_t alignment) {
 		 *    144 | 10100000 |  32
 		 *    192 | 11000000 |  64
 		 */
-		usize = sz_s2u(ALIGNMENT_CEILING(size, alignment));
+		size_t rounded = size + alignment_mask;
+		if (unlikely(rounded < size)) {
+			/* size_t overflow. */
+			return 0;
+		}
+		usize = sz_s2u(rounded & ~alignment_mask);
 		if (usize < SC_LARGE_MINCLASS) {
 			return usize;
 		}

include/jemalloc/internal/tcache.h

@@ -0,0 +1,199 @@
+#ifndef JEMALLOC_INTERNAL_TCACHE_H
+#define JEMALLOC_INTERNAL_TCACHE_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/cache_bin.h"
+#include "jemalloc/internal/ql.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/thread_event_registry.h"
+#include "jemalloc/internal/ticker.h"
+
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s arena_t;
+typedef struct base_s  base_t;
+
+/******************************************************************************/
+/* TYPES */
+/******************************************************************************/
+
+typedef struct tcache_slow_s tcache_slow_t;
+typedef struct tcache_s      tcache_t;
+typedef struct tcaches_s     tcaches_t;
+
+/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
+#define TCACHE_ZERO_INITIALIZER                                                \
+	{ 0 }
+#define TCACHE_SLOW_ZERO_INITIALIZER                                           \
+	{                                                                      \
+		{ 0 }                                                          \
+	}
+
+/* Used in TSD static initializer only. Will be initialized to opt_tcache. */
+#define TCACHE_ENABLED_ZERO_INITIALIZER false
+
+/* Used for explicit tcache only. Means flushed but not destroyed. */
+/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
+#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1)
+
+#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
+#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
+#define TCACHE_NBINS_MAX                                                       \
+	(SC_NBINS                                                              \
+	    + SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS)    \
+	    + 1)
+#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21)       /* 2M */
+#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
+#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)
+#define TCACHE_GC_LARGE_NBINS_MAX 1
+
+/******************************************************************************/
+/* STRUCTS */
+/******************************************************************************/
+
+/*
+ * The tcache state is split into the slow and hot path data.  Each has a
+ * pointer to the other, and the data always comes in pairs.  The layout of each
+ * of them varies in practice; tcache_slow lives in the TSD for the automatic
+ * tcache, and as part of a dynamic allocation for manual allocations.  Keeping
+ * a pointer to tcache_slow lets us treat these cases uniformly, rather than
+ * splitting up the tcache [de]allocation code into those paths called with the
+ * TSD tcache and those called with a manual tcache.
+ */
+
+struct tcache_slow_s {
+	/*
+	 * The descriptor lets the arena find our cache bins without seeing the
+	 * tcache definition.  This enables arenas to aggregate stats across
+	 * tcaches without having a tcache dependency.
+	 */
+	cache_bin_array_descriptor_t cache_bin_array_descriptor;
+
+	/* The arena this tcache is associated with. */
+	arena_t *arena;
+	/* The number of bins activated in the tcache. */
+	unsigned tcache_nbins;
+	/* Last time GC has been performed.  */
+	nstime_t last_gc_time;
+	/* Next bin to GC. */
+	szind_t next_gc_bin;
+	szind_t next_gc_bin_small;
+	szind_t next_gc_bin_large;
+	/* For small bins, help determine how many items to fill at a time. */
+	cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS];
+	/* For small bins, whether has been refilled since last GC. */
+	bool bin_refilled[SC_NBINS];
+	/*
+	 * For small bins, the number of items we can pretend to flush before
+	 * actually flushing.
+	 */
+	uint8_t bin_flush_delay_items[SC_NBINS];
+	/*
+	 * The start of the allocation containing the dynamic allocation for
+	 * either the cache bins alone, or the cache bin memory as well as this
+	 * tcache_slow_t and its associated tcache_t.
+	 */
+	void *dyn_alloc;
+
+	/* The associated bins. */
+	tcache_t *tcache;
+};
+
+struct tcache_s {
+	tcache_slow_t *tcache_slow;
+	cache_bin_t    bins[TCACHE_NBINS_MAX];
+};
+
+/* Linkage for list of available (previously used) explicit tcache IDs. */
+struct tcaches_s {
+	union {
+		tcache_t  *tcache;
+		tcaches_t *next;
+	};
+};
+
+/******************************************************************************/
+/* EXTERNS */
+/******************************************************************************/
+
+extern bool     opt_tcache;
+extern size_t   opt_tcache_max;
+extern ssize_t  opt_lg_tcache_nslots_mul;
+extern unsigned opt_tcache_nslots_small_min;
+extern unsigned opt_tcache_nslots_small_max;
+extern unsigned opt_tcache_nslots_large;
+extern ssize_t  opt_lg_tcache_shift;
+extern size_t   opt_tcache_gc_incr_bytes;
+extern size_t   opt_tcache_gc_delay_bytes;
+extern unsigned opt_lg_tcache_flush_small_div;
+extern unsigned opt_lg_tcache_flush_large_div;
+
+/*
+ * Number of tcache bins.  There are SC_NBINS small-object bins, plus 0 or more
+ * large-object bins.  This is only used during threads initialization and
+ * changing it will not reflect on initialized threads as expected.  Thus,
+ * it should not be changed on the fly.  To change the number of tcache bins
+ * in use, refer to tcache_nbins of each tcache.
+ */
+extern unsigned global_do_not_change_tcache_nbins;
+
+/*
+ * Maximum cached size class.  Same as above, this is only used during threads
+ * initialization and should not be changed.  To change the maximum cached size
+ * class, refer to tcache_max of each tcache.
+ */
+extern size_t global_do_not_change_tcache_maxclass;
+
+/*
+ * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
+ * usable via the MALLOCX_TCACHE() flag.  The automatic per thread tcaches are
+ * completely disjoint from this data structure.  tcaches starts off as a sparse
+ * array, so it has no physical memory footprint until individual pages are
+ * touched.  This allows the entire array to be allocated the first time an
+ * explicit tcache is created without a disproportionate impact on memory usage.
+ */
+extern tcaches_t *tcaches;
+
+size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
+void  *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
+     cache_bin_t *cache_bin, szind_t binind, bool *tcache_success);
+
+void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache,
+    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
+void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
+    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
+void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
+    cache_bin_t *cache_bin, szind_t binind, bool is_small);
+bool tcache_bin_info_default_init(
+    const char *bin_settings_segment_cur, size_t len_left);
+bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
+bool tcache_bin_ncached_max_read(
+    tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
+void tcache_arena_reassociate(
+    tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
+tcache_t *tcache_create_explicit(tsd_t *tsd);
+bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
+void      tcache_cleanup(tsd_t *tsd);
+bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
+void      tcaches_flush(tsd_t *tsd, unsigned ind);
+void      tcaches_destroy(tsd_t *tsd, unsigned ind);
+bool      tcache_boot(tsdn_t *tsdn, base_t *base);
+void      tcache_arena_associate(
+         tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena);
+cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor(
+         tsdn_t *tsdn, arena_t *arena);
+void tcache_prefork(tsdn_t *tsdn);
+void tcache_postfork_parent(tsdn_t *tsdn);
+void tcache_postfork_child(tsdn_t *tsdn);
+void tcache_flush(tsd_t *tsd);
+bool tsd_tcache_enabled_data_init(tsd_t *tsd);
+void tcache_enabled_set(tsd_t *tsd, bool enabled);
+
+extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size,
+    size_t alignment);
+
+void tcache_assert_initialized(tcache_t *tcache);
+
+extern te_base_cb_t tcache_gc_te_handler;
+
+#endif /* JEMALLOC_INTERNAL_TCACHE_H */

include/jemalloc/internal/tcache_externs.h

@@ -1,90 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
-#define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/base.h"
-#include "jemalloc/internal/cache_bin.h"
-#include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_types.h"
-#include "jemalloc/internal/thread_event_registry.h"
-
-extern bool     opt_tcache;
-extern size_t   opt_tcache_max;
-extern ssize_t  opt_lg_tcache_nslots_mul;
-extern unsigned opt_tcache_nslots_small_min;
-extern unsigned opt_tcache_nslots_small_max;
-extern unsigned opt_tcache_nslots_large;
-extern ssize_t  opt_lg_tcache_shift;
-extern size_t   opt_tcache_gc_incr_bytes;
-extern size_t   opt_tcache_gc_delay_bytes;
-extern unsigned opt_lg_tcache_flush_small_div;
-extern unsigned opt_lg_tcache_flush_large_div;
-
-/*
- * Number of tcache bins.  There are SC_NBINS small-object bins, plus 0 or more
- * large-object bins.  This is only used during threads initialization and
- * changing it will not reflect on initialized threads as expected.  Thus,
- * it should not be changed on the fly.  To change the number of tcache bins
- * in use, refer to tcache_nbins of each tcache.
- */
-extern unsigned global_do_not_change_tcache_nbins;
-
-/*
- * Maximum cached size class.  Same as above, this is only used during threads
- * initialization and should not be changed.  To change the maximum cached size
- * class, refer to tcache_max of each tcache.
- */
-extern size_t global_do_not_change_tcache_maxclass;
-
-/*
- * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and
- * usable via the MALLOCX_TCACHE() flag.  The automatic per thread tcaches are
- * completely disjoint from this data structure.  tcaches starts off as a sparse
- * array, so it has no physical memory footprint until individual pages are
- * touched.  This allows the entire array to be allocated the first time an
- * explicit tcache is created without a disproportionate impact on memory usage.
- */
-extern tcaches_t *tcaches;
-
-size_t tcache_salloc(tsdn_t *tsdn, const void *ptr);
-void  *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
-     cache_bin_t *cache_bin, szind_t binind, bool *tcache_success);
-
-void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache,
-    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
-void tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache,
-    cache_bin_t *cache_bin, szind_t binind, unsigned rem);
-void tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache,
-    cache_bin_t *cache_bin, szind_t binind, bool is_small);
-bool tcache_bin_info_default_init(
-    const char *bin_settings_segment_cur, size_t len_left);
-bool tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len);
-bool tcache_bin_ncached_max_read(
-    tsd_t *tsd, size_t bin_size, cache_bin_sz_t *ncached_max);
-void tcache_arena_reassociate(
-    tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
-tcache_t *tcache_create_explicit(tsd_t *tsd);
-bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
-void      tcache_cleanup(tsd_t *tsd);
-void      tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
-bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
-void      tcaches_flush(tsd_t *tsd, unsigned ind);
-void      tcaches_destroy(tsd_t *tsd, unsigned ind);
-bool      tcache_boot(tsdn_t *tsdn, base_t *base);
-void      tcache_arena_associate(
-         tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_t *tcache, arena_t *arena);
-void tcache_prefork(tsdn_t *tsdn);
-void tcache_postfork_parent(tsdn_t *tsdn);
-void tcache_postfork_child(tsdn_t *tsdn);
-void tcache_flush(tsd_t *tsd);
-bool tsd_tcache_enabled_data_init(tsd_t *tsd);
-void tcache_enabled_set(tsd_t *tsd, bool enabled);
-
-extern void *(*JET_MUTABLE tcache_stack_alloc)(tsdn_t *tsdn, size_t size,
-    size_t alignment);
-
-void tcache_assert_initialized(tcache_t *tcache);
-
-extern te_base_cb_t tcache_gc_te_handler;
-
-#endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

include/jemalloc/internal/tcache_inlines.h

@@ -2,15 +2,15 @@
 #define JEMALLOC_INTERNAL_TCACHE_INLINES_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_externs.h"
+#include "jemalloc/internal/arena.h"
 #include "jemalloc/internal/bin.h"
-#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
+#include "jemalloc/internal/arena_inlines.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
-#include "jemalloc/internal/large_externs.h"
+#include "jemalloc/internal/large.h"
 #include "jemalloc/internal/san.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_externs.h"
+#include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/util.h"
 
 static inline bool
@@ -19,7 +19,7 @@ tcache_enabled_get(tsd_t *tsd) {
 }
 
 static inline unsigned
-tcache_nbins_get(tcache_slow_t *tcache_slow) {
+tcache_nbins_get(const tcache_slow_t *tcache_slow) {
 	assert(tcache_slow != NULL);
 	unsigned nbins = tcache_slow->tcache_nbins;
 	assert(nbins <= TCACHE_NBINS_MAX);
@@ -27,7 +27,7 @@ tcache_nbins_get(tcache_slow_t *tcache_slow) {
 }
 
 static inline size_t
-tcache_max_get(tcache_slow_t *tcache_slow) {
+tcache_max_get(const tcache_slow_t *tcache_slow) {
 	assert(tcache_slow != NULL);
 	size_t tcache_max = sz_index2size(tcache_nbins_get(tcache_slow) - 1);
 	assert(tcache_max <= TCACHE_MAXCLASS_LIMIT);
@@ -86,6 +86,13 @@ tcache_bin_disabled(szind_t ind, cache_bin_t *bin, tcache_slow_t *tcache_slow) {
 	return disabled;
 }
 
+JEMALLOC_ALWAYS_INLINE bool
+tcache_can_cache_large(tcache_t *tcache, szind_t ind) {
+	return ind < tcache_nbins_get(tcache->tcache_slow)
+	    && !tcache_bin_disabled(ind, &tcache->bins[ind],
+	        tcache->tcache_slow);
+}
+
 JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size,
     szind_t binind, bool zero, bool slow_path) {

include/jemalloc/internal/tcache_structs.h

@@ -1,75 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H
-#define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/cache_bin.h"
-#include "jemalloc/internal/ql.h"
-#include "jemalloc/internal/sc.h"
-#include "jemalloc/internal/tcache_types.h"
-#include "jemalloc/internal/ticker.h"
-
-/*
- * The tcache state is split into the slow and hot path data.  Each has a
- * pointer to the other, and the data always comes in pairs.  The layout of each
- * of them varies in practice; tcache_slow lives in the TSD for the automatic
- * tcache, and as part of a dynamic allocation for manual allocations.  Keeping
- * a pointer to tcache_slow lets us treat these cases uniformly, rather than
- * splitting up the tcache [de]allocation code into those paths called with the
- * TSD tcache and those called with a manual tcache.
- */
-
-struct tcache_slow_s {
-	/* Lets us track all the tcaches in an arena. */
-	ql_elm(tcache_slow_t) link;
-
-	/*
-	 * The descriptor lets the arena find our cache bins without seeing the
-	 * tcache definition.  This enables arenas to aggregate stats across
-	 * tcaches without having a tcache dependency.
-	 */
-	cache_bin_array_descriptor_t cache_bin_array_descriptor;
-
-	/* The arena this tcache is associated with. */
-	arena_t *arena;
-	/* The number of bins activated in the tcache. */
-	unsigned tcache_nbins;
-	/* Last time GC has been performed.  */
-	nstime_t last_gc_time;
-	/* Next bin to GC. */
-	szind_t next_gc_bin;
-	szind_t next_gc_bin_small;
-	szind_t next_gc_bin_large;
-	/* For small bins, help determine how many items to fill at a time. */
-	cache_bin_fill_ctl_t bin_fill_ctl_do_not_access_directly[SC_NBINS];
-	/* For small bins, whether has been refilled since last GC. */
-	bool bin_refilled[SC_NBINS];
-	/*
-	 * For small bins, the number of items we can pretend to flush before
-	 * actually flushing.
-	 */
-	uint8_t bin_flush_delay_items[SC_NBINS];
-	/*
-	 * The start of the allocation containing the dynamic allocation for
-	 * either the cache bins alone, or the cache bin memory as well as this
-	 * tcache_slow_t and its associated tcache_t.
-	 */
-	void *dyn_alloc;
-
-	/* The associated bins. */
-	tcache_t *tcache;
-};
-
-struct tcache_s {
-	tcache_slow_t *tcache_slow;
-	cache_bin_t    bins[TCACHE_NBINS_MAX];
-};
-
-/* Linkage for list of available (previously used) explicit tcache IDs. */
-struct tcaches_s {
-	union {
-		tcache_t  *tcache;
-		tcaches_t *next;
-	};
-};
-
-#endif /* JEMALLOC_INTERNAL_TCACHE_STRUCTS_H */

include/jemalloc/internal/tcache_types.h

@@ -1,37 +0,0 @@
-#ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H
-#define JEMALLOC_INTERNAL_TCACHE_TYPES_H
-
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/sc.h"
-
-typedef struct tcache_slow_s tcache_slow_t;
-typedef struct tcache_s      tcache_t;
-typedef struct tcaches_s     tcaches_t;
-
-/* Used in TSD static initializer only. Real init in tsd_tcache_data_init(). */
-#define TCACHE_ZERO_INITIALIZER                                                \
-	{ 0 }
-#define TCACHE_SLOW_ZERO_INITIALIZER                                           \
-	{                                                                      \
-		{ 0 }                                                          \
-	}
-
-/* Used in TSD static initializer only. Will be initialized to opt_tcache. */
-#define TCACHE_ENABLED_ZERO_INITIALIZER false
-
-/* Used for explicit tcache only. Means flushed but not destroyed. */
-/* NOLINTNEXTLINE(performance-no-int-to-ptr) */
-#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1)
-
-#define TCACHE_LG_MAXCLASS_LIMIT LG_USIZE_GROW_SLOW_THRESHOLD
-#define TCACHE_MAXCLASS_LIMIT ((size_t)1 << TCACHE_LG_MAXCLASS_LIMIT)
-#define TCACHE_NBINS_MAX                                                       \
-	(SC_NBINS                                                              \
-	    + SC_NGROUP * (TCACHE_LG_MAXCLASS_LIMIT - SC_LG_LARGE_MINCLASS)    \
-	    + 1)
-#define TCACHE_GC_NEIGHBOR_LIMIT ((uintptr_t)1 << 21)       /* 2M */
-#define TCACHE_GC_INTERVAL_NS ((uint64_t)10 * KQU(1000000)) /* 10ms */
-#define TCACHE_GC_SMALL_NBINS_MAX ((SC_NBINS > 8) ? (SC_NBINS >> 3) : 1)
-#define TCACHE_GC_LARGE_NBINS_MAX 1
-
-#endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */

include/jemalloc/internal/test_hooks.h

@@ -1,10 +1,12 @@
 #ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H
 #define JEMALLOC_INTERNAL_TEST_HOOKS_H
 
-#include "jemalloc/internal/jemalloc_preamble.h"
-
 extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(void);
 extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(void);
+extern JEMALLOC_EXPORT void (*test_hooks_safety_check_abort)(const char *);
+#if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
+extern JEMALLOC_EXPORT void (*test_hooks_tsd_bootstrap_hook)(void);
+#endif
 
 #if defined(JEMALLOC_JET) || defined(JEMALLOC_UNIT_TEST)
 #	define JEMALLOC_TEST_HOOK(fn, hook)                                   \

include/jemalloc/internal/tsd.h

@@ -37,13 +37,12 @@ TSD_DATA_SLOWER
 JEMALLOC_ALWAYS_INLINE t *						\
 tsd_##n##p_get(tsd_t *tsd) {						\
 	/*								\
-	 * Because the state might change asynchronously if it's	\
-	 * nominal, we need to make sure that we only read it once.	\
+	 * Read the state once, so that a transition between		\
+	 * nominal states does not confuse this assertion.		\
 	 */								\
 	uint8_t state = tsd_state_get(tsd);				\
 	assert(state == tsd_state_nominal ||				\
 	    state == tsd_state_nominal_slow ||				\
-	    state == tsd_state_nominal_recompute ||			\
 	    state == tsd_state_reincarnated ||				\
 	    state == tsd_state_minimal_initialized);			\
 	return tsd_##n##p_get_unsafe(tsd);				\
@@ -95,11 +94,6 @@ TSD_DATA_SLOWER
 
 JEMALLOC_ALWAYS_INLINE void
 tsd_assert_fast(tsd_t *tsd) {
-	/*
-	 * Note that our fastness assertion does *not* include global slowness
-	 * counters; it's not in general possible to ensure that they won't
-	 * change asynchronously from underneath us.
-	 */
 	assert(!malloc_slow && tsd_tcache_enabled_get(tsd)
 	    && tsd_reentrancy_level_get(tsd) == 0);
 }

include/jemalloc/internal/tsd_binshards.h

@@ -0,0 +1,24 @@
+#ifndef JEMALLOC_INTERNAL_TSD_BINSHARDS_H
+#define JEMALLOC_INTERNAL_TSD_BINSHARDS_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/sc.h"
+
+/*
+ * Per-thread cache of bin-shard assignments.  This lives in its own header
+ * (separate from bin.h) so that tsd_internals.h can pull it in for X-macro
+ * expansion without dragging in mutex.h, which itself depends on TSD machinery
+ * and would form an include-order dependency cycle.
+ */
+
+#define TSD_BINSHARDS_ZERO_INITIALIZER                                         \
+	{                                                                      \
+		{ UINT8_MAX }                                                  \
+	}
+
+typedef struct tsd_binshards_s tsd_binshards_t;
+struct tsd_binshards_s {
+	uint8_t binshard[SC_NBINS];
+};
+
+#endif /* JEMALLOC_INTERNAL_TSD_BINSHARDS_H */

include/jemalloc/internal/tsd_internals.h

@@ -4,21 +4,27 @@
 #define JEMALLOC_INTERNAL_TSD_INTERNALS_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/arena_types.h"
+#include "jemalloc/internal/arena_decay_constants.h"
 #include "jemalloc/internal/assert.h"
-#include "jemalloc/internal/bin_types.h"
+#include "jemalloc/internal/tsd_binshards.h"
 #include "jemalloc/internal/jemalloc_internal_externs.h"
 #include "jemalloc/internal/peak.h"
-#include "jemalloc/internal/prof_types.h"
-#include "jemalloc/internal/ql.h"
 #include "jemalloc/internal/rtree_tsd.h"
-#include "jemalloc/internal/tcache_structs.h"
-#include "jemalloc/internal/tcache_types.h"
+#include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/thread_event_registry.h"
 #include "jemalloc/internal/tsd_types.h"
 #include "jemalloc/internal/util.h"
 #include "jemalloc/internal/witness.h"
 
+/*
+ * Forward decls.  tsd_internals.h cannot include arena.h / prof.h directly:
+ * those headers' STRUCTS-section includes trigger mutex.h -> tsd.h ->
+ * tsd_generic.h, which would re-enter this file before its body finishes.
+ * Each consumer here only uses these as pointer types.
+ */
+typedef struct arena_s      arena_t;
+typedef struct prof_tdata_s prof_tdata_t;
+
 /*
  * Thread-Specific-Data layout
  *
@@ -57,8 +63,6 @@ typedef void (*test_callback_t)(int *);
 #	define MALLOC_TEST_TSD_INITIALIZER
 #endif
 
-typedef ql_elm(tsd_t) tsd_link_t;
-
 /*  O(name,			type,			nullable type) */
 #define TSD_DATA_SLOW                                                          \
 	O(tcache_enabled, bool, bool)                                          \
@@ -79,9 +83,8 @@ typedef ql_elm(tsd_t) tsd_link_t;
 	O(arena, arena_t *, arena_t *)                                         \
 	O(arena_decay_ticker, ticker_geom_t, ticker_geom_t)                    \
 	O(sec_shard, uint8_t, uint8_t)                                         \
+	O(pac_sec_shard, uint8_t, uint8_t)                                     \
 	O(binshards, tsd_binshards_t, tsd_binshards_t)                         \
-	O(tsd_link, tsd_link_t, tsd_link_t)                                    \
-	O(in_hook, bool, bool)                                                 \
 	O(peak, peak_t, peak_t)                                                \
 	O(tcache_slow, tcache_slow_t, tcache_slow_t)                           \
 	O(rtree_ctx, rtree_ctx_t, rtree_ctx_t)
@@ -100,10 +103,10 @@ typedef ql_elm(tsd_t) tsd_link_t;
 	    /* arena */ NULL, /* arena_decay_ticker */                         \
 	    TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),                   \
 	    /* sec_shard */ (uint8_t) - 1,                                     \
+	    /* pac_sec_shard */ (uint8_t) - 1,                                 \
 	    /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER,                    \
-	    /* tsd_link */ {NULL}, /* in_hook */ false,                        \
-	    /* peak */ PEAK_INITIALIZER,                                       \
-	    /* tcache_slow */ TCACHE_SLOW_ZERO_INITIALIZER,                    \
+	    /* peak */ PEAK_INITIALIZER, /* tcache_slow */                     \
+	    TCACHE_SLOW_ZERO_INITIALIZER,                                      \
 	    /* rtree_ctx */ RTREE_CTX_INITIALIZER,
 
 /*  O(name,			type,			nullable type) */
@@ -132,7 +135,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
 #define TSD_INITIALIZER                                                        \
 	{                                                                      \
 		TSD_DATA_SLOW_INITIALIZER                                      \
-		/* state */ ATOMIC_INIT(tsd_state_uninitialized),              \
+		/* state */ tsd_state_uninitialized,                           \
 		    TSD_DATA_FAST_INITIALIZER TSD_DATA_SLOWER_INITIALIZER      \
 	}
 
@@ -148,17 +151,6 @@ void   tsd_cleanup(void *arg);
 tsd_t *tsd_fetch_slow(tsd_t *tsd, bool minimal);
 void   tsd_state_set(tsd_t *tsd, uint8_t new_state);
 void   tsd_slow_update(tsd_t *tsd);
-void   tsd_prefork(tsd_t *tsd);
-void   tsd_postfork_parent(tsd_t *tsd);
-void   tsd_postfork_child(tsd_t *tsd);
-
-/*
- * Call ..._inc when your module wants to take all threads down the slow paths,
- * and ..._dec when it no longer needs to.
- */
-void tsd_global_slow_inc(tsdn_t *tsdn);
-void tsd_global_slow_dec(tsdn_t *tsdn);
-bool tsd_global_slow(void);
 
 #define TSD_MIN_INIT_STATE_MAX_FETCHED (128)
 
@@ -167,39 +159,29 @@ enum {
 	tsd_state_nominal = 0,
 	/* Initialized but on slow path. */
 	tsd_state_nominal_slow = 1,
-	/*
-	 * Some thread has changed global state in such a way that all nominal
-	 * threads need to recompute their fast / slow status the next time they
-	 * get a chance.
-	 *
-	 * Any thread can change another thread's status *to* recompute, but
-	 * threads are the only ones who can change their status *from*
-	 * recompute.
-	 */
-	tsd_state_nominal_recompute = 2,
 	/*
 	 * The above nominal states should be lower values.  We use
 	 * tsd_nominal_max to separate nominal states from threads in the
 	 * process of being born / dying.
 	 */
-	tsd_state_nominal_max = 2,
+	tsd_state_nominal_max = 1,
 
 	/*
 	 * A thread might free() during its death as its only allocator action;
 	 * in such scenarios, we need tsd, but set up in such a way that no
 	 * cleanup is necessary.
 	 */
-	tsd_state_minimal_initialized = 3,
+	tsd_state_minimal_initialized = 2,
 	/* States during which we know we're in thread death. */
-	tsd_state_purgatory = 4,
-	tsd_state_reincarnated = 5,
+	tsd_state_purgatory = 3,
+	tsd_state_reincarnated = 4,
 	/*
 	 * What it says on the tin; tsd that hasn't been initialized.  Note
 	 * that even when the tsd struct lives in TLS, when need to keep track
 	 * of stuff like whether or not our pthread destructors have been
 	 * scheduled, so this really truly is different than the nominal state.
 	 */
-	tsd_state_uninitialized = 6
+	tsd_state_uninitialized = 5
 };
 
 /*
@@ -209,18 +191,6 @@ enum {
  */
 #define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n
 
-#ifdef JEMALLOC_U8_ATOMICS
-#	define tsd_state_t atomic_u8_t
-#	define tsd_atomic_load atomic_load_u8
-#	define tsd_atomic_store atomic_store_u8
-#	define tsd_atomic_exchange atomic_exchange_u8
-#else
-#	define tsd_state_t atomic_u32_t
-#	define tsd_atomic_load atomic_load_u32
-#	define tsd_atomic_store atomic_store_u32
-#	define tsd_atomic_exchange atomic_exchange_u32
-#endif
-
 /* The actual tsd. */
 struct tsd_s {
 	/*
@@ -232,11 +202,8 @@ struct tsd_s {
 #define O(n, t, nt) t TSD_MANGLE(n);
 
 	TSD_DATA_SLOW
-	/*
-	 * We manually limit the state to just a single byte.  Unless the 8-bit
-	 * atomics are unavailable (which is rare).
-	 */
-	tsd_state_t state;
+	/* Encodes one of tsd_state_*; mutated only by the owning thread. */
+	uint8_t state;
 	TSD_DATA_FAST
 	TSD_DATA_SLOWER
 #undef O
@@ -244,13 +211,7 @@ struct tsd_s {
 
 JEMALLOC_ALWAYS_INLINE uint8_t
 tsd_state_get(tsd_t *tsd) {
-	/*
-	 * This should be atomic.  Unfortunately, compilers right now can't tell
-	 * that this can be done as a memory comparison, and forces a load into
-	 * a register that hurts fast-path performance.
-	 */
-	/* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */
-	return *(uint8_t *)&tsd->state;
+	return tsd->state;
 }
 
 /*

include/jemalloc/internal/util.h

@@ -6,6 +6,18 @@
 
 #define UTIL_INLINE static inline
 
+JEMALLOC_ALWAYS_INLINE
+size_t
+min_zu(size_t a, size_t b) {
+	return (a < b) ? a : b;
+}
+
+JEMALLOC_ALWAYS_INLINE
+size_t
+max_zu(size_t a, size_t b) {
+	return (a > b) ? a : b;
+}
+
 /* Junk fill patterns. */
 #ifndef JEMALLOC_ALLOC_JUNK
 #	define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)

include/jemalloc/internal/witness.h

@@ -44,7 +44,7 @@ enum witness_rank_e {
 	 */
 	WITNESS_RANK_CORE,
 	WITNESS_RANK_DECAY = WITNESS_RANK_CORE,
-	WITNESS_RANK_TCACHE_QL,
+	WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
 
 	WITNESS_RANK_SEC_BIN,
 

include/jemalloc/jemalloc_typedefs.h.in

@@ -4,6 +4,36 @@ extern "C" {
 
 typedef struct extent_hooks_s extent_hooks_t;
 
+/*
+ * Extent alloc flags.  A custom extent_alloc hook may OR these into the
+ * returned pointer; jemalloc strips the low bits before use.  Safe because
+ * returned addresses are at least page-aligned (PAGE >= 256).
+ *
+ * EXTENT_ALLOC_FLAG_PINNED: backing memory is non-reclaimable.
+ * Pinned extents are excluded from decay/purging and cached separately for
+ * preferential reuse.  A hook returning this flag must also set *commit to
+ * true: pinned memory bypasses jemalloc's commit/decommit machinery.
+ *
+ * The pinned attribute is per-extent: a single hook may return pinned and
+ * non-pinned extents in different calls.  Pinned and non-pinned extents are
+ * never merged together (the merge would change the reclamation policy of
+ * one half), so pinned-ness is set at allocation and inherited through
+ * splits, but never changes after that.
+ *
+ * Example (HugeTLB alloc hook):
+ *   void *my_alloc(extent_hooks_t *h, void *new_addr, size_t size,
+ *       size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
+ *       void *addr = mmap(NULL, size, PROT_READ|PROT_WRITE,
+ *           MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0);
+ *       if (addr == MAP_FAILED) return NULL;
+ *       *zero = true;
+ *       *commit = true;
+ *       return (void *)((uintptr_t)addr | EXTENT_ALLOC_FLAG_PINNED);
+ *   }
+ */
+#define EXTENT_ALLOC_FLAG_PINNED    0x1U
+#define EXTENT_ALLOC_FLAG_MASK      0xFFU
+
 /*
  * void *
  * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -36,6 +36,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
+    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -44,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
+    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -67,8 +69,11 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -31,6 +31,9 @@
     <ClCompile Include="..\..\..\..\src\ckh.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\conf.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\counter.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -91,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>
@@ -202,5 +208,14 @@
     <ClCompile Include="..\..\..\..\src\san_bump.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\arenas_management.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
 </Project>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -36,6 +36,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\..\src\arena.c" />
+    <ClCompile Include="..\..\..\..\src\arenas_management.c" />
     <ClCompile Include="..\..\..\..\src\background_thread.c" />
     <ClCompile Include="..\..\..\..\src\base.c" />
     <ClCompile Include="..\..\..\..\src\bin.c" />
@@ -44,6 +45,7 @@
     <ClCompile Include="..\..\..\..\src\buf_writer.c" />
     <ClCompile Include="..\..\..\..\src\cache_bin.c" />
     <ClCompile Include="..\..\..\..\src\ckh.c" />
+    <ClCompile Include="..\..\..\..\src\conf.c" />
     <ClCompile Include="..\..\..\..\src\counter.c" />
     <ClCompile Include="..\..\..\..\src\ctl.c" />
     <ClCompile Include="..\..\..\..\src\decay.c" />
@@ -67,8 +69,11 @@
     <ClCompile Include="..\..\..\..\src\hpdata.c" />
     <ClCompile Include="..\..\..\..\src\inspect.c" />
     <ClCompile Include="..\..\..\..\src\jemalloc.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_fork.c" />
+    <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />