Move malloc routing into new malloc_dispatch module

Pull the tcache-aware routing helpers out of arena into a layer that
sits directly below the public malloc interface:

  arena_malloc           -> malloc_dispatch_malloc
  arena_palloc           -> malloc_dispatch_palloc
  arena_ralloc           -> malloc_dispatch_ralloc
  arena_dalloc*          -> malloc_dispatch_dalloc*
  arena_sdalloc*         -> malloc_dispatch_sdalloc*
  arena_dalloc_promoted  -> malloc_dispatch_dalloc_promoted

The new module (malloc_dispatch.h, malloc_dispatch_inlines.h,
src/malloc_dispatch.c) owns the tcache-vs-fall-through decision; the
only consumer is jemalloc_internal_inlines_c.h.  arena keeps a narrower
arena_prof_demote() for the sampled-allocation demotion path.

Makefile.in

@@ -132,6 +132,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/inspect.c \
 	$(srcroot)src/large.c \
 	$(srcroot)src/log.c \
+	$(srcroot)src/malloc_dispatch.c \
 	$(srcroot)src/malloc_io.c \
 	$(srcroot)src/conf.c \
 	$(srcroot)src/mutex.c \
@@ -260,6 +261,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/log.c \
 	$(srcroot)test/unit/mallctl.c \
 	$(srcroot)test/unit/malloc_conf_2.c \
+	$(srcroot)test/unit/malloc_dispatch.c \
 	$(srcroot)test/unit/malloc_io.c \
 	$(srcroot)test/unit/math.c \
 	$(srcroot)test/unit/mpsc_queue.c \

include/jemalloc/internal/arena_externs.h

@@ -70,12 +70,9 @@ cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
 
 void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
     bool zero, bool slab);
-void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
-    bool zero, bool slab, tcache_t *tcache);
 void  arena_prof_promote(
      tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
-void arena_dalloc_promoted(
+size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
-    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
 void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
 
 void  arena_dalloc_small(tsdn_t *tsdn, void *ptr);
@@ -84,8 +81,6 @@ void  arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
      arena_t *stats_arena, cache_bin_stats_t merge_stats);
 bool  arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
      size_t extra, bool zero, size_t *newsize);
-void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache);
 dss_prec_t      arena_dss_prec_get(const arena_t *arena);
 ehooks_t       *arena_get_ehooks(const arena_t *arena);
 extent_hooks_t *arena_set_extent_hooks(

include/jemalloc/internal/arena_inlines_b.h

@@ -17,7 +17,6 @@
 #include "jemalloc/internal/safety_check.h"
 #include "jemalloc/internal/sc.h"
 #include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_inlines.h"
 #include "jemalloc/internal/ticker.h"
 
 static inline arena_t *
@@ -193,26 +192,6 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
 	arena_decay_ticks(tsdn, arena, 1);
 }
 
-JEMALLOC_ALWAYS_INLINE void *
-arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
-    bool slab, tcache_t *tcache, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-
-	if (likely(tcache != NULL)) {
-		if (likely(slab)) {
-			assert(sz_can_use_slab(size));
-			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
-			    size, ind, zero, slow_path);
-		} else if (likely(tcache_can_cache_large(tcache, ind))) {
-			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
-			    size, ind, zero, slow_path);
-		}
-		/* (size > tcache_max) case falls through. */
-	}
-
-	return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
-}
-
 JEMALLOC_ALWAYS_INLINE arena_t *
 arena_aalloc(tsdn_t *tsdn, const void *ptr) {
 	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
@@ -261,244 +240,6 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
 	return edata_usize_get(full_alloc_ctx.edata);
 }
 
-static inline void
-arena_dalloc_large_no_tcache(
-    tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
-	/*
-	 * szind is still needed in this function mainly becuase
-	 * szind < SC_NBINS determines not only if this is a small alloc,
-	 * but also if szind is valid (an inactive extent would have
-	 * szind == SC_NSIZES).
-	 */
-	if (config_prof && unlikely(szind < SC_NBINS)) {
-		arena_dalloc_promoted(tsdn, ptr, NULL, true);
-	} else {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		if (large_dalloc_safety_checks(edata, ptr, usize)) {
-			/* See the comment in isfree. */
-			return;
-		}
-		large_dalloc(tsdn, edata);
-	}
-}
-
-static inline void
-arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
-	assert(ptr != NULL);
-
-	emap_alloc_ctx_t alloc_ctx;
-	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.szind < SC_NSIZES);
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		arena_dalloc_small(tsdn, ptr);
-	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx));
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
-    size_t usize, bool slow_path) {
-	assert(!tsdn_null(tsdn) && tcache != NULL);
-	bool is_sample_promoted = config_prof && szind < SC_NBINS;
-	if (unlikely(is_sample_promoted)) {
-		arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
-	} else {
-		if (tcache_can_cache_large(tcache, szind)) {
-			tcache_dalloc_large(
-			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
-		} else {
-			edata_t *edata = emap_edata_lookup(
-			    tsdn, &arena_emap_global, ptr);
-			if (large_dalloc_safety_checks(edata, ptr, usize)) {
-				/* See the comment in isfree. */
-				return;
-			}
-			large_dalloc(tsdn, edata);
-		}
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
-	if (!config_debug) {
-		return false;
-	}
-	edata_t   *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
-	szind_t    binind = edata_szind_get(edata);
-	div_info_t div_info = arena_binind_div_info[binind];
-	/*
-	 * Calls the internal function bin_slab_regind_impl because the
-	 * safety check does not require a lock.
-	 */
-	size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
-	slab_data_t      *slab_data = edata_slab_data_get(edata);
-	const bin_info_t *bin_info = &bin_infos[binind];
-	assert(edata_nfree_get(edata) < bin_info->nregs);
-	if (unlikely(!bitmap_get(
-	        slab_data->bitmap, &bin_info->bitmap_info, regind))) {
-		safety_check_fail(
-		    "Invalid deallocation detected: the pointer being freed (%p) not "
-		    "currently active, possibly caused by double free bugs.\n",
-		    ptr);
-		return true;
-	}
-	return false;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
-    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-	assert(ptr != NULL);
-
-	if (unlikely(tcache == NULL)) {
-		arena_dalloc_no_tcache(tsdn, ptr);
-		return;
-	}
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (caller_alloc_ctx != NULL) {
-		alloc_ctx = *caller_alloc_ctx;
-	} else {
-		util_assume(tsdn != NULL);
-		emap_alloc_ctx_lookup(
-		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-	}
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.szind < SC_NSIZES);
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-			return;
-		}
-		tcache_dalloc_small(
-		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
-	} else {
-		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
-	}
-}
-
-static inline void
-arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
-	assert(ptr != NULL);
-	assert(size <= SC_LARGE_MAXCLASS);
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (!config_prof || !opt_prof) {
-		/*
-		 * There is no risk of being confused by a promoted sampled
-		 * object, so base szind and slab on the given size.
-		 */
-		szind_t szind = sz_size2index(size);
-		emap_alloc_ctx_init(
-		    &alloc_ctx, szind, (szind < SC_NBINS), size);
-	}
-
-	if ((config_prof && opt_prof) || config_debug) {
-		emap_alloc_ctx_lookup(
-		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
-		assert(alloc_ctx.szind == sz_size2index(size));
-		assert((config_prof && opt_prof)
-		    || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
-
-		if (config_debug) {
-			edata_t *edata = emap_edata_lookup(
-			    tsdn, &arena_emap_global, ptr);
-			assert(alloc_ctx.szind == edata_szind_get(edata));
-			assert(alloc_ctx.slab == edata_slab_get(edata));
-		}
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		arena_dalloc_small(tsdn, ptr);
-	} else {
-		arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
-		    emap_alloc_ctx_usize_get(&alloc_ctx));
-	}
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
-    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
-	assert(!tsdn_null(tsdn) || tcache == NULL);
-	assert(ptr != NULL);
-	assert(size <= SC_LARGE_MAXCLASS);
-
-	if (unlikely(tcache == NULL)) {
-		arena_sdalloc_no_tcache(tsdn, ptr, size);
-		return;
-	}
-
-	emap_alloc_ctx_t alloc_ctx;
-	if (config_prof && opt_prof) {
-		if (caller_alloc_ctx == NULL) {
-			/* Uncommon case and should be a static check. */
-			emap_alloc_ctx_lookup(
-			    tsdn, &arena_emap_global, ptr, &alloc_ctx);
-			assert(alloc_ctx.szind == sz_size2index(size));
-			assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
-		} else {
-			alloc_ctx = *caller_alloc_ctx;
-		}
-	} else {
-		/*
-		 * There is no risk of being confused by a promoted sampled
-		 * object, so base szind and slab on the given size.
-		 */
-		alloc_ctx.szind = sz_size2index(size);
-		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
-	}
-
-	if (config_debug) {
-		edata_t *edata = emap_edata_lookup(
-		    tsdn, &arena_emap_global, ptr);
-		assert(alloc_ctx.szind == edata_szind_get(edata));
-		assert(alloc_ctx.slab == edata_slab_get(edata));
-		emap_alloc_ctx_init(
-		    &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
-		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
-		    == edata_usize_get(edata));
-	}
-
-	if (likely(alloc_ctx.slab)) {
-		/* Small allocation. */
-		if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
-			return;
-		}
-		tcache_dalloc_small(
-		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
-	} else {
-		arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
-		    sz_s2u(size), slow_path);
-	}
-}
-
 static inline void
 arena_cache_oblivious_randomize(
     tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {

include/jemalloc/internal/jemalloc_internal_includes.h

@@ -61,6 +61,7 @@
 #include "jemalloc/internal/arena_externs.h"
 #include "jemalloc/internal/large_externs.h"
 #include "jemalloc/internal/tcache_externs.h"
+#include "jemalloc/internal/malloc_dispatch.h"
 #include "jemalloc/internal/prof_externs.h"
 #include "jemalloc/internal/background_thread_externs.h"
 
@@ -77,6 +78,7 @@
 #include "jemalloc/internal/jemalloc_internal_inlines_b.h"
 #include "jemalloc/internal/tcache_inlines.h"
 #include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/malloc_dispatch_inlines.h"
 #include "jemalloc/internal/jemalloc_internal_inlines_c.h"
 #include "jemalloc/internal/prof_inlines.h"
 #include "jemalloc/internal/background_thread_inlines.h"

include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -8,6 +8,7 @@
 #include "jemalloc/internal/jemalloc_init.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/log.h"
+#include "jemalloc/internal/malloc_dispatch_inlines.h"
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/thread_event.h"
 #include "jemalloc/internal/witness.h"
@@ -67,7 +68,7 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
 		    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	}
 
-	ret = arena_malloc(
+	ret = malloc_dispatch_malloc(
 	    tsdn, arena, size, ind, zero, slab, tcache, slow_path);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -102,7 +103,8 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 
-	ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
+	ret = malloc_dispatch_palloc(
+	    tsdn, arena, usize, alignment, zero, slab, tcache);
 	assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
 	if (config_stats && is_internal && likely(ret != NULL)) {
 		arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -156,7 +158,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
 	    && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
 		assert(tcache == NULL);
 	}
-	arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
+	malloc_dispatch_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -169,7 +171,7 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
 	witness_assert_depth_to_rank(
 	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-	arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
+	malloc_dispatch_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
@@ -217,8 +219,8 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 		    zero, slab, tcache, arena);
 	}
 
-	return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
+	return malloc_dispatch_ralloc(
-	    slab, tcache);
+	    tsdn, arena, ptr, oldsize, size, alignment, zero, slab, tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void *

include/jemalloc/internal/malloc_dispatch.h

@@ -0,0 +1,19 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s  arena_t;
+typedef struct tcache_s tcache_t;
+
+void *malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache);
+void  malloc_dispatch_dalloc_promoted(
+     tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
+void *malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr,
+    size_t oldsize, size_t size, size_t alignment, bool zero, bool slab,
+    tcache_t *tcache);
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_H */

include/jemalloc/internal/malloc_dispatch_inlines.h

@@ -0,0 +1,277 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_externs.h"
+#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/bin_inlines.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/large_externs.h"
+#include "jemalloc/internal/malloc_dispatch.h"
+#include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache_inlines.h"
+
+JEMALLOC_ALWAYS_INLINE void *
+malloc_dispatch_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
+    bool zero, bool slab, tcache_t *tcache, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+
+	if (likely(tcache != NULL)) {
+		if (likely(slab)) {
+			assert(sz_can_use_slab(size));
+			return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
+		} else if (likely(tcache_can_cache_large(tcache, ind))) {
+			return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
+			    size, ind, zero, slow_path);
+		}
+		/* (size > tcache_max) case falls through. */
+	}
+
+	return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
+}
+
+static inline void
+malloc_dispatch_dalloc_large_no_tcache(
+    tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
+	/*
+	 * szind both classifies small vs large and validates the extent --
+	 * inactive extents have szind == SC_NSIZES.
+	 */
+	if (config_prof && unlikely(szind < SC_NBINS)) {
+		malloc_dispatch_dalloc_promoted(tsdn, ptr, NULL, true);
+	} else {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		if (large_dalloc_safety_checks(edata, ptr, usize)) {
+			/* See the comment in isfree. */
+			return;
+		}
+		large_dalloc(tsdn, edata);
+	}
+}
+
+static inline void
+malloc_dispatch_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
+	assert(ptr != NULL);
+
+	emap_alloc_ctx_t alloc_ctx;
+	emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.szind < SC_NSIZES);
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		malloc_dispatch_dalloc_large_no_tcache(
+		    tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    szind_t szind, size_t usize, bool slow_path) {
+	assert(!tsdn_null(tsdn) && tcache != NULL);
+	bool is_sample_promoted = config_prof && szind < SC_NBINS;
+	if (unlikely(is_sample_promoted)) {
+		malloc_dispatch_dalloc_promoted(tsdn, ptr, tcache, slow_path);
+	} else {
+		if (tcache_can_cache_large(tcache, szind)) {
+			tcache_dalloc_large(
+			    tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
+		} else {
+			edata_t *edata = emap_edata_lookup(
+			    tsdn, &arena_emap_global, ptr);
+			if (large_dalloc_safety_checks(edata, ptr, usize)) {
+				/* See the comment in isfree. */
+				return;
+			}
+			large_dalloc(tsdn, edata);
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_dispatch_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
+	if (!config_debug) {
+		return false;
+	}
+	edata_t   *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	szind_t    binind = edata_szind_get(edata);
+	div_info_t div_info = arena_binind_div_info[binind];
+	/*
+	 * Calls the internal function bin_slab_regind_impl because the
+	 * safety check does not require a lock.
+	 */
+	size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
+	slab_data_t      *slab_data = edata_slab_data_get(edata);
+	const bin_info_t *bin_info = &bin_infos[binind];
+	assert(edata_nfree_get(edata) < bin_info->nregs);
+	if (unlikely(!bitmap_get(
+	        slab_data->bitmap, &bin_info->bitmap_info, regind))) {
+		safety_check_fail(
+		    "Invalid deallocation detected: the pointer being freed (%p) not "
+		    "currently active, possibly caused by double free bugs.\n",
+		    ptr);
+		return true;
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+
+	if (unlikely(tcache == NULL)) {
+		malloc_dispatch_dalloc_no_tcache(tsdn, ptr);
+		return;
+	}
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (caller_alloc_ctx != NULL) {
+		alloc_ctx = *caller_alloc_ctx;
+	} else {
+		util_assume(tsdn != NULL);
+		emap_alloc_ctx_lookup(
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+	}
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.szind < SC_NSIZES);
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+	} else {
+		malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
+	}
+}
+
+static inline void
+malloc_dispatch_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
+	assert(ptr != NULL);
+	assert(size <= SC_LARGE_MAXCLASS);
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (!config_prof || !opt_prof) {
+		/*
+		 * There is no risk of being confused by a promoted sampled
+		 * object, so base szind and slab on the given size.
+		 */
+		szind_t szind = sz_size2index(size);
+		emap_alloc_ctx_init(
+		    &alloc_ctx, szind, (szind < SC_NBINS), size);
+	}
+
+	if ((config_prof && opt_prof) || config_debug) {
+		emap_alloc_ctx_lookup(
+		    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+		assert(alloc_ctx.szind == sz_size2index(size));
+		assert((config_prof && opt_prof)
+		    || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
+
+		if (config_debug) {
+			edata_t *edata = emap_edata_lookup(
+			    tsdn, &arena_emap_global, ptr);
+			assert(alloc_ctx.szind == edata_szind_get(edata));
+			assert(alloc_ctx.slab == edata_slab_get(edata));
+		}
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		arena_dalloc_small(tsdn, ptr);
+	} else {
+		malloc_dispatch_dalloc_large_no_tcache(
+		    tsdn, ptr, alloc_ctx.szind,
+		    emap_alloc_ctx_usize_get(&alloc_ctx));
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+    emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+	assert(!tsdn_null(tsdn) || tcache == NULL);
+	assert(ptr != NULL);
+	assert(size <= SC_LARGE_MAXCLASS);
+
+	if (unlikely(tcache == NULL)) {
+		malloc_dispatch_sdalloc_no_tcache(tsdn, ptr, size);
+		return;
+	}
+
+	emap_alloc_ctx_t alloc_ctx;
+	if (config_prof && opt_prof) {
+		if (caller_alloc_ctx == NULL) {
+			/* Uncommon case and should be a static check. */
+			emap_alloc_ctx_lookup(
+			    tsdn, &arena_emap_global, ptr, &alloc_ctx);
+			assert(alloc_ctx.szind == sz_size2index(size));
+			assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
+		} else {
+			alloc_ctx = *caller_alloc_ctx;
+		}
+	} else {
+		/*
+		 * There is no risk of being confused by a promoted sampled
+		 * object, so base szind and slab on the given size.
+		 */
+		alloc_ctx.szind = sz_size2index(size);
+		alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
+	}
+
+	if (config_debug) {
+		edata_t *edata = emap_edata_lookup(
+		    tsdn, &arena_emap_global, ptr);
+		assert(alloc_ctx.szind == edata_szind_get(edata));
+		assert(alloc_ctx.slab == edata_slab_get(edata));
+		emap_alloc_ctx_init(
+		    &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
+		assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+		    == edata_usize_get(edata));
+	}
+
+	if (likely(alloc_ctx.slab)) {
+		/* Small allocation. */
+		if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+			return;
+		}
+		tcache_dalloc_small(
+		    tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+	} else {
+		malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+		    sz_s2u(size), slow_path);
+	}
+}
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H */

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj

@@ -73,6 +73,7 @@
     <ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
     <ClCompile Include="..\..\..\..\src\large.c" />
     <ClCompile Include="..\..\..\..\src\log.c" />
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
     <ClCompile Include="..\..\..\..\src\malloc_io.c" />
     <ClCompile Include="..\..\..\..\src\mutex.c" />
     <ClCompile Include="..\..\..\..\src\nstime.c" />

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters

@@ -94,6 +94,9 @@
     <ClCompile Include="..\..\..\..\src\log.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\malloc_io.c">
       <Filter>Source Files</Filter>
     </ClCompile>

src/arena.c

@@ -694,11 +694,13 @@ arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize) {
 	assert(isalloc(tsdn, ptr) == usize);
 }
 
-static size_t
+size_t
 arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
 	cassert(config_prof);
+	assert(opt_prof);
 	assert(ptr != NULL);
-	size_t usize = isalloc(tsdn, ptr);
+	size_t usize = edata_usize_get(edata);
+	assert(isalloc(tsdn, ptr) == usize);
 	size_t bumped_usize = sz_sa2u(usize, PROF_SAMPLE_ALIGNMENT);
 	assert(bumped_usize <= SC_LARGE_MINCLASS
 	    && PAGE_CEILING(bumped_usize) == bumped_usize);
@@ -710,17 +712,6 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
 
 	assert(isalloc(tsdn, ptr) == bumped_usize);
 
-	return bumped_usize;
-}
-
-static void
-arena_dalloc_promoted_impl(
-    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path, edata_t *edata) {
-	cassert(config_prof);
-	assert(opt_prof);
-
-	size_t usize = edata_usize_get(edata);
-	size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
 	if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) {
 		/*
 		 * Currently, we only do redzoning for small sampled
@@ -728,21 +719,8 @@ arena_dalloc_promoted_impl(
 		 */
 		safety_check_verify_redzone(ptr, usize, bumped_usize);
 	}
-	szind_t bumped_ind = sz_size2index(bumped_usize);
-	if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
-	    && tcache_can_cache_large(tcache, bumped_ind)) {
-		tcache_dalloc_large(
-		    tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
-	} else {
-		large_dalloc(tsdn, edata);
-	}
-}
 
-void
+	return bumped_usize;
-arena_dalloc_promoted(
-    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
-	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
-	arena_dalloc_promoted_impl(tsdn, ptr, tcache, slow_path, edata);
 }
 
 void
@@ -784,8 +762,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
 			prof_free(tsd, ptr, usize, &alloc_ctx);
 		}
 		if (config_prof && opt_prof && alloc_ctx.szind < SC_NBINS) {
-			arena_dalloc_promoted_impl(tsd_tsdn(tsd), ptr,
+			arena_prof_demote(tsd_tsdn(tsd), edata, ptr);
-			    /* tcache */ NULL, /* slow_path */ true, edata);
+			large_dalloc(tsd_tsdn(tsd), edata);
 		} else {
 			large_dalloc(tsd_tsdn(tsd), edata);
 		}
@@ -1154,33 +1132,6 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
 	}
 }
 
-void *
-arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
-    bool zero, bool slab, tcache_t *tcache) {
-	if (slab) {
-		assert(sz_can_use_slab(usize));
-		/* Small; alignment doesn't require special slab placement. */
-
-		/* usize should be a result of sz_sa2u() */
-		assert((usize & (alignment - 1)) == 0);
-
-		/*
-		 * Small usize can't come from an alignment larger than a page.
-		 */
-		assert(alignment <= PAGE);
-
-		return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
-		    zero, slab, tcache, true);
-	} else {
-		if (likely(alignment <= CACHELINE)) {
-			return large_malloc(tsdn, arena, usize, zero);
-		} else {
-			return large_palloc(
-			    tsdn, arena, usize, alignment, zero);
-		}
-	}
-}
-
 static void
 arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) {
 	szind_t  binind = edata_szind_get(edata);
@@ -1607,64 +1558,6 @@ done:
 	return ret;
 }
 
-static void *
-arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
-    size_t alignment, bool zero, bool slab, tcache_t *tcache) {
-	if (alignment == 0) {
-		return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
-		    zero, slab, tcache, true);
-	}
-	usize = sz_sa2u(usize, alignment);
-	if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
-		return NULL;
-	}
-	return ipalloct_explicit_slab(
-	    tsdn, usize, alignment, zero, slab, tcache, arena);
-}
-
-void *
-arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
-	size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
-	if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
-		return NULL;
-	}
-
-	if (likely(slab)) {
-		assert(sz_can_use_slab(usize));
-		/* Try to avoid moving the allocation. */
-		UNUSED size_t newsize;
-		if (!arena_ralloc_no_move(
-		        tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
-			return ptr;
-		}
-	}
-
-	if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
-		return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
-		    tcache);
-	}
-
-	/*
-	 * size and oldsize are different enough that we need to move the
-	 * object.  In that case, fall back to allocating new space and copying.
-	 */
-	void *ret = arena_ralloc_move_helper(
-	    tsdn, arena, usize, alignment, zero, slab, tcache);
-	if (ret == NULL) {
-		return NULL;
-	}
-
-	/*
-	 * Junk/zero-filling were already done by
-	 * ipalloc()/arena_malloc().
-	 */
-	size_t copysize = (usize < oldsize) ? usize : oldsize;
-	memcpy(ret, ptr, copysize);
-	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
-	return ret;
-}
-
 ehooks_t *
 arena_get_ehooks(const arena_t *arena) {
 	return base_ehooks_get(arena->base);

src/malloc_dispatch.c

@@ -0,0 +1,109 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/emap.h"
+
+/******************************************************************************/
+
+void
+malloc_dispatch_dalloc_promoted(
+    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
+	cassert(config_prof);
+	assert(opt_prof);
+
+	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
+	szind_t bumped_ind = sz_size2index(bumped_usize);
+	if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
+	    && tcache_can_cache_large(tcache, bumped_ind)) {
+		tcache_dalloc_large(
+		    tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
+	} else {
+		large_dalloc(tsdn, edata);
+	}
+}
+
+void *
+malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+	if (slab) {
+		assert(sz_can_use_slab(usize));
+		/* Small; alignment doesn't require special slab placement. */
+
+		/* usize should be a result of sz_sa2u() */
+		assert((usize & (alignment - 1)) == 0);
+
+		/*
+		 * Small usize can't come from an alignment larger than a page.
+		 */
+		assert(alignment <= PAGE);
+
+		return malloc_dispatch_malloc(tsdn, arena, usize,
+		    sz_size2index(usize), zero, slab, tcache, true);
+	} else {
+		if (likely(alignment <= CACHELINE)) {
+			return large_malloc(tsdn, arena, usize, zero);
+		} else {
+			return large_palloc(
+			    tsdn, arena, usize, alignment, zero);
+		}
+	}
+}
+
+static void *
+malloc_dispatch_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+	if (alignment == 0) {
+		return malloc_dispatch_malloc(tsdn, arena, usize,
+		    sz_size2index(usize), zero, slab, tcache, true);
+	}
+	usize = sz_sa2u(usize, alignment);
+	if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
+		return NULL;
+	}
+	return ipalloct_explicit_slab(
+	    tsdn, usize, alignment, zero, slab, tcache, arena);
+}
+
+void *
+malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
+    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+	size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
+	if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
+		return NULL;
+	}
+
+	if (likely(slab)) {
+		assert(sz_can_use_slab(usize));
+		/* Try to avoid moving the allocation. */
+		UNUSED size_t newsize;
+		if (!arena_ralloc_no_move(
+		        tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
+			return ptr;
+		}
+	}
+
+	if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
+		return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
+		    tcache);
+	}
+
+	/*
+	 * size and oldsize are different enough that we need to move the
+	 * object.  In that case, fall back to allocating new space and copying.
+	 */
+	void *ret = malloc_dispatch_ralloc_move_helper(
+	    tsdn, arena, usize, alignment, zero, slab, tcache);
+	if (ret == NULL) {
+		return NULL;
+	}
+
+	/*
+	 * Junk/zero-filling were already done by ipalloc() / dispatch alloc.
+	 */
+	size_t copysize = (usize < oldsize) ? usize : oldsize;
+	memcpy(ret, ptr, copysize);
+	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
+	return ret;
+}

test/unit/malloc_dispatch.c

@@ -0,0 +1,208 @@
+#include "test/jemalloc_test.h"
+
+/*
+ * Targeted coverage for the malloc_dispatch routing layer.  Integration
+ * tests exercise the public API end-to-end but don't assert which branch
+ * was taken for a given (size, tcache) input.  These tests make the
+ * routing observable through per-arena stats counters:
+ *
+ *   1. small + tcache != NULL          -> tcache_alloc_small
+ *   2. small + tcache == NULL          -> arena_malloc_hard + arena_dalloc_small
+ *   3. large <= tcache_max + tcache    -> tcache_alloc_large
+ *   4. large >  tcache_max + tcache    -> arena_malloc_hard + large_dalloc
+ */
+
+static unsigned
+create_fresh_arena(void) {
+	unsigned arena_ind;
+	size_t   sz = sizeof(arena_ind);
+	expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "arenas.create failed");
+	return arena_ind;
+}
+
+static void
+refresh_stats(void) {
+	uint64_t epoch = 1;
+	expect_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
+	    "epoch refresh failed");
+}
+
+static uint64_t
+read_u64_mallctl(const char *cmd) {
+	uint64_t v;
+	size_t   sz = sizeof(v);
+	expect_d_eq(mallctl(cmd, (void *)&v, &sz, NULL, 0), 0,
+	    "mallctl read failed");
+	return v;
+}
+
+static uint64_t
+arena_bin0_nmalloc(unsigned arena_ind) {
+	char cmd[128];
+	(void)snprintf(
+	    cmd, sizeof(cmd), "stats.arenas.%u.bins.0.nmalloc", arena_ind);
+	return read_u64_mallctl(cmd);
+}
+
+static uint64_t
+arena_bin0_ndalloc(unsigned arena_ind) {
+	char cmd[128];
+	(void)snprintf(
+	    cmd, sizeof(cmd), "stats.arenas.%u.bins.0.ndalloc", arena_ind);
+	return read_u64_mallctl(cmd);
+}
+
+static uint64_t
+arena_bin0_nfills(unsigned arena_ind) {
+	char cmd[128];
+	(void)snprintf(
+	    cmd, sizeof(cmd), "stats.arenas.%u.bins.0.nfills", arena_ind);
+	return read_u64_mallctl(cmd);
+}
+
+static uint64_t
+arena_large_nmalloc(unsigned arena_ind) {
+	char cmd[128];
+	(void)snprintf(
+	    cmd, sizeof(cmd), "stats.arenas.%u.large.nmalloc", arena_ind);
+	return read_u64_mallctl(cmd);
+}
+
+static uint64_t
+arena_large_ndalloc(unsigned arena_ind) {
+	char cmd[128];
+	(void)snprintf(
+	    cmd, sizeof(cmd), "stats.arenas.%u.large.ndalloc", arena_ind);
+	return read_u64_mallctl(cmd);
+}
+
+/*
+ * Branch 2: small alloc/dalloc with MALLOCX_TCACHE_NONE must increment the
+ * arena bin counters immediately (no tcache to absorb the call).
+ */
+TEST_BEGIN(test_dispatch_small_no_tcache) {
+	test_skip_if(!config_stats);
+	test_skip_if(opt_prof);
+
+	unsigned arena_ind = create_fresh_arena();
+	int      flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
+	size_t   sz = bin_infos[0].reg_size;
+
+	refresh_stats();
+	uint64_t nmalloc_before = arena_bin0_nmalloc(arena_ind);
+	uint64_t ndalloc_before = arena_bin0_ndalloc(arena_ind);
+	uint64_t nfills_before = arena_bin0_nfills(arena_ind);
+
+	void *p = mallocx(sz, flags);
+	expect_ptr_not_null(p, "mallocx failed");
+	dallocx(p, flags);
+
+	refresh_stats();
+	expect_u64_eq(arena_bin0_nmalloc(arena_ind), nmalloc_before + 1,
+	    "small no-tcache alloc must increment arena bin nmalloc");
+	expect_u64_eq(arena_bin0_ndalloc(arena_ind), ndalloc_before + 1,
+	    "small no-tcache dalloc must increment arena bin ndalloc");
+	expect_u64_eq(arena_bin0_nfills(arena_ind), nfills_before,
+	    "no-tcache path must not trigger any tcache fill");
+}
+TEST_END
+
+/*
+ * Branch 1: small alloc with tcache must NOT increment the arena bin nmalloc
+ * counter directly (the tcache absorbs the alloc); a refill (nfills > 0) is
+ * the routing observable instead.  Bound the test to a single tcache slot to
+ * make the refill predictable.
+ */
+TEST_BEGIN(test_dispatch_small_with_tcache) {
+	test_skip_if(!config_stats);
+	test_skip_if(!opt_tcache);
+	test_skip_if(opt_prof);
+
+	unsigned arena_ind = create_fresh_arena();
+	int      flags = MALLOCX_ARENA(arena_ind);
+	size_t   sz = bin_infos[0].reg_size;
+
+	/* Flush any per-thread tcache so this arena's tcache slot starts cold. */
+	expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0,
+	    "thread.tcache.flush failed");
+
+	refresh_stats();
+	uint64_t nfills_before = arena_bin0_nfills(arena_ind);
+
+	void *p = mallocx(sz, flags);
+	expect_ptr_not_null(p, "mallocx failed");
+
+	refresh_stats();
+	expect_u64_gt(arena_bin0_nfills(arena_ind), nfills_before,
+	    "small alloc with tcache must trigger a tcache fill");
+	dallocx(p, flags);
+}
+TEST_END
+
+/*
+ * Branches 3 & 4: large alloc routing pivots on tcache_max.  Use a fresh
+ * thread.tcache.max so we can compute a size that is just above it; that size
+ * must be routed through arena_malloc_hard (and not the tcache).  Allocations
+ * below tcache_max routed through the same call site must not touch the arena
+ * large counters on the malloc/free pair (they ride the tcache).
+ */
+TEST_BEGIN(test_dispatch_large_routes_on_tcache_max) {
+	test_skip_if(!config_stats);
+	test_skip_if(!opt_tcache);
+	test_skip_if(opt_prof);
+	test_skip_if(san_uaf_detection_enabled());
+
+	unsigned arena_ind = create_fresh_arena();
+	expect_d_eq(
+	    mallctl("thread.arena", NULL, NULL, &arena_ind, sizeof(arena_ind)),
+	    0, "thread.arena bind failed");
+
+	/* Pin tcache_max to a known small-large boundary. */
+	size_t small_large;
+	size_t sz = sizeof(small_large);
+	expect_d_eq(mallctl("arenas.lextent.0.size", (void *)&small_large, &sz,
+	                NULL, 0),
+	    0, "arenas.lextent.0.size lookup failed");
+	expect_d_eq(mallctl("thread.tcache.max", NULL, NULL, &small_large,
+	                sizeof(small_large)),
+	    0, "thread.tcache.max set failed");
+	expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0,
+	    "thread.tcache.flush failed");
+
+	int flags = MALLOCX_ARENA(arena_ind);
+
+	/* Above tcache_max: must hit arena_malloc_hard + large_dalloc. */
+	size_t too_big;
+	expect_d_eq(mallctl("arenas.lextent.1.size", (void *)&too_big, &sz, NULL,
+	                0),
+	    0, "arenas.lextent.1.size lookup failed");
+	expect_zu_gt(too_big, small_large,
+	    "lextent.1 must exceed tcache_max boundary");
+
+	refresh_stats();
+	uint64_t nmalloc_before = arena_large_nmalloc(arena_ind);
+	uint64_t ndalloc_before = arena_large_ndalloc(arena_ind);
+
+	void *big = mallocx(too_big, flags);
+	expect_ptr_not_null(big, "mallocx failed");
+
+	refresh_stats();
+	expect_u64_eq(arena_large_nmalloc(arena_ind), nmalloc_before + 1,
+	    "large alloc above tcache_max must bypass tcache "
+	    "and increment arena large nmalloc");
+
+	dallocx(big, flags);
+	refresh_stats();
+	expect_u64_eq(arena_large_ndalloc(arena_ind), ndalloc_before + 1,
+	    "large dalloc above tcache_max must bypass tcache "
+	    "and increment arena large ndalloc");
+}
+TEST_END
+
+int
+main(void) {
+	return test(test_dispatch_small_no_tcache,
+	    test_dispatch_small_with_tcache,
+	    test_dispatch_large_routes_on_tcache_max);
+}