Move tcache_stats_merge into arena as arena_cache_bins_stats_merge

include/jemalloc/internal/arena_externs.h

@@ -118,6 +118,7 @@ void   arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
        cache_bin_array_descriptor_t *desc);
 void   arena_cache_bin_array_postfork_child(arena_t *arena,
        cache_bin_array_descriptor_t *desc_or_null);
+void   arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork0(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork1(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork2(tsdn_t *tsdn, arena_t *arena);

include/jemalloc/internal/tcache_externs.h

@@ -66,8 +66,6 @@ void tcache_arena_reassociate(
 tcache_t *tcache_create_explicit(tsd_t *tsd);
 bool      thread_tcache_max_set(tsd_t *tsd, size_t tcache_max);
 void      tcache_cleanup(tsd_t *tsd);
-void      tcache_stats_merge(tsdn_t *tsdn,
-         cache_bin_array_descriptor_t *desc, arena_t *arena);
 bool      tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind);
 void      tcaches_flush(tsd_t *tsd, unsigned ind);
 void      tcaches_destroy(tsd_t *tsd, unsigned ind);

src/arena.c

@@ -228,6 +228,27 @@ arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena,
 	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 }
 
+static void
+arena_cache_bin_stats_flush(tsdn_t *tsdn, arena_t *arena,
+    cache_bin_array_descriptor_t *desc) {
+	cassert(config_stats);
+	for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
+		cache_bin_t *cache_bin = &desc->bins[i];
+		if (cache_bin_disabled(cache_bin)) {
+			continue;
+		}
+		if (i < SC_NBINS) {
+			bin_t *bin = bin_choose(tsdn, arena, i, NULL);
+			bin_stats_nrequests_add(tsdn, bin,
+			    cache_bin->tstats.nrequests);
+		} else {
+			arena_stats_large_flush_nrequests_add(tsdn,
+			    &arena->stats, i, cache_bin->tstats.nrequests);
+		}
+		cache_bin->tstats.nrequests = 0;
+	}
+}
+
 void
 arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
     cache_bin_array_descriptor_t *desc) {
@@ -245,13 +266,7 @@ arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena,
 		assert(in_ql);
 	}
 	ql_remove(&arena->cache_bin_array_descriptor_ql, desc, link);
-	/*
+	arena_cache_bin_stats_flush(tsdn, arena, desc);
-	 * Flush this descriptor's per-cache_bin request counts up to the
-	 * arena's bin/large stats before the owner forgets which arena
-	 * accumulated them. (Step 2 of this refactor will relocate
-	 * tcache_stats_merge into cache_bin.c.)
-	 */
-	tcache_stats_merge(tsdn, desc, arena);
 	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 }
 
@@ -272,6 +287,17 @@ arena_cache_bin_array_postfork_child(arena_t *arena,
 	}
 }
 
+void
+arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
+	cassert(config_stats);
+	malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+	cache_bin_array_descriptor_t *desc;
+	ql_foreach (desc, &arena->cache_bin_array_descriptor_ql, link) {
+		arena_cache_bin_stats_flush(tsdn, arena, desc);
+	}
+	malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
+}
+
 static void
 arena_background_thread_inactivity_check(
     tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {

src/jemalloc_init.c

@@ -305,16 +305,7 @@ stats_print_atexit(void) {
 		for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
 			arena_t *arena = arena_get(tsdn, i, false);
 			if (arena != NULL) {
-				cache_bin_array_descriptor_t *desc;
+				arena_cache_bins_stats_merge(tsdn, arena);
-
-				malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
-				ql_foreach (desc,
-				    &arena->cache_bin_array_descriptor_ql,
-				    link) {
-					tcache_stats_merge(tsdn, desc, arena);
-				}
-				malloc_mutex_unlock(
-				    tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
 			}
 		}
 	}

src/tcache.c

@@ -1248,28 +1248,6 @@ tcache_cleanup(tsd_t *tsd) {
 	memset(tcache->bins, 0, sizeof(cache_bin_t) * TCACHE_NBINS_MAX);
 }
 
-void
-tcache_stats_merge(tsdn_t *tsdn, cache_bin_array_descriptor_t *desc,
-    arena_t *arena) {
-	cassert(config_stats);
-
-	for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
-		cache_bin_t *cache_bin = &desc->bins[i];
-		if (cache_bin_disabled(cache_bin)) {
-			continue;
-		}
-		if (i < SC_NBINS) {
-			bin_t *bin = bin_choose(tsdn, arena, i, NULL);
-			bin_stats_nrequests_add(tsdn, bin,
-			    cache_bin->tstats.nrequests);
-		} else {
-			arena_stats_large_flush_nrequests_add(tsdn,
-			    &arena->stats, i, cache_bin->tstats.nrequests);
-		}
-		cache_bin->tstats.nrequests = 0;
-	}
-}
-
 static bool
 tcaches_create_prep(tsd_t *tsd, base_t *base) {
 	bool err;