Merge branch 'dev'

When a thread cache flushes objects to their arenas due to an abundance
of cached objects, it merges the allocation request count for the
associated size class, and increments a flush counter.  If none of the
flushed objects came from the thread's assigned arena, then the merging
wouldn't happen (though the counter would typically eventually be
merged), nor would the flush counter be incremented (a hard bug).  Fix
this via extra conditional code just after the flush loop.

jemalloc/ChangeLog

@@ -6,6 +6,14 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
+* 2.1.3 (March 14, 2011)
+
+  Bug fixes:
+  - Fix a cpp logic regression (due to the "thread.{de,}allocatedp" mallctl fix
+    for OS X in 2.1.2).
+  - Fix a "thread.arena" mallctl bug.
+  - Fix a thread cache stats merging bug.
+
 * 2.1.2 (March 2, 2011)
 
   Bug fixes:

jemalloc/include/jemalloc/internal/jemalloc_internal.h.in

@@ -369,7 +369,7 @@ size_t	s2u(size_t size);
 size_t	sa2u(size_t size, size_t alignment, size_t *run_size_p);
 void	malloc_write(const char *s);
 arena_t	*choose_arena(void);
-#  ifdef NO_TLS
+#  if (defined(JEMALLOC_STATS) && defined(NO_TLS))
 thread_allocated_t	*thread_allocated_get(void);
 #  endif
 #endif
@@ -533,7 +533,7 @@ choose_arena(void)
 	return (ret);
 }
 
-#ifdef NO_TLS
+#if (defined(JEMALLOC_STATS) && defined(NO_TLS))
 JEMALLOC_INLINE thread_allocated_t *
 thread_allocated_get(void)
 {

jemalloc/src/ctl.c

@@ -1114,8 +1114,8 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 	unsigned newind, oldind;
 
 	newind = oldind = choose_arena()->ind;
-	WRITE(oldind, unsigned);
+	WRITE(newind, unsigned);
-	READ(newind, unsigned);
+	READ(oldind, unsigned);
 	if (newind != oldind) {
 		arena_t *arena;
 

jemalloc/src/tcache.c

@@ -58,6 +58,9 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 	void *flush, *deferred, *ptr;
 	unsigned i, nflush, ndeferred;
 	bool first_pass;
+#ifdef JEMALLOC_STATS
+	bool merged_stats = false;
+#endif
 
 	assert(binind < nbins);
 	assert(rem <= tbin->ncached);
@@ -82,6 +85,8 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 		malloc_mutex_lock(&bin->lock);
 #ifdef JEMALLOC_STATS
 		if (arena == tcache->arena) {
+			assert(merged_stats == false);
+			merged_stats = true;
 			bin->stats.nflushes++;
 			bin->stats.nrequests += tbin->tstats.nrequests;
 			tbin->tstats.nrequests = 0;
@@ -119,6 +124,20 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 			first_pass = false;
 		}
 	}
+#ifdef JEMALLOC_STATS
+	if (merged_stats == false) {
+		/*
+		 * The flush loop didn't happen to flush to this thread's
+		 * arena, so the stats didn't get merged.  Manually do so now.
+		 */
+		arena_bin_t *bin = &tcache->arena->bins[binind];
+		malloc_mutex_lock(&bin->lock);
+		bin->stats.nflushes++;
+		bin->stats.nrequests += tbin->tstats.nrequests;
+		tbin->tstats.nrequests = 0;
+		malloc_mutex_unlock(&bin->lock);
+	}
+#endif
 
 	tbin->ncached = rem;
 	if (tbin->ncached < tbin->low_water)

jemalloc/test/thread_arena.c

@@ -2,10 +2,13 @@
 #include <stdlib.h>
 #include <pthread.h>
 #include <string.h>
+#include <assert.h>
 
 #define	JEMALLOC_MANGLE
 #include "jemalloc_test.h"
 
+#define NTHREADS 10
+
 void *
 thread_start(void *arg)
 {
@@ -29,6 +32,15 @@ thread_start(void *arg)
 		return (void *)1;
 	}
 
+	size = sizeof(arena_ind);
+	if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL,
+	    0))) {
+		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
+		    strerror(err));
+		return (void *)1;
+	}
+	assert(arena_ind == main_arena_ind);
+
 	return (NULL);
 }
 
@@ -40,7 +52,8 @@ main(void)
 	unsigned arena_ind;
 	size_t size;
 	int err;
-	pthread_t thread;
+	pthread_t threads[NTHREADS];
+	unsigned i;
 
 	fprintf(stderr, "Test begin\n");
 
@@ -60,21 +73,18 @@ main(void)
 		goto RETURN;
 	}
 
-	if (pthread_create(&thread, NULL, thread_start, (void *)&arena_ind)
+	for (i = 0; i < NTHREADS; i++) {
-	    != 0) {
+		if (pthread_create(&threads[i], NULL, thread_start,
-		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
+		    (void *)&arena_ind) != 0) {
-		ret = 1;
+			fprintf(stderr, "%s(): Error in pthread_create()\n",
-		goto RETURN;
+			    __func__);
+			ret = 1;
+			goto RETURN;
+		}
 	}
-	pthread_join(thread, (void *)&ret);
 
-	if (pthread_create(&thread, NULL, thread_start, (void *)&arena_ind)
+	for (i = 0; i < NTHREADS; i++)
-	    != 0) {
+		pthread_join(threads[i], (void *)&ret);
-		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
-		ret = 1;
-		goto RETURN;
-	}
-	pthread_join(thread, (void *)&ret);
 
 RETURN:
 	fprintf(stderr, "Test end\n");