Merge branch 'dev'

When a thread cache flushes objects to their arenas due to an abundance
of cached objects, it merges the allocation request count for the
associated size class, and increments a flush counter.  If none of the
flushed objects came from the thread's assigned arena, then the merging
wouldn't happen (though the counter would typically eventually be
merged), nor would the flush counter be incremented (a hard bug).  Fix
this via extra conditional code just after the flush loop.

jemalloc/ChangeLog

@@ -6,7 +6,29 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
-* 2.1.0
+* 2.1.3 (March 14, 2011)
+
+  Bug fixes:
+  - Fix a cpp logic regression (due to the "thread.{de,}allocatedp" mallctl fix
+    for OS X in 2.1.2).
+  - Fix a "thread.arena" mallctl bug.
+  - Fix a thread cache stats merging bug.
+
+* 2.1.2 (March 2, 2011)
+
+  Bug fixes:
+  - Fix "thread.{de,}allocatedp" mallctl for OS X.
+  - Add missing jemalloc.a to build system.
+
+* 2.1.1 (January 31, 2011)
+
+  Bug fixes:
+  - Fix aligned huge reallocation (affected allocm()).
+  - Fix the ALLOCM_LG_ALIGN macro definition.
+  - Fix a heap dumping deadlock.
+  - Fix a "thread.arena" mallctl bug.
+
+* 2.1.0 (December 3, 2010)
 
   This version incorporates some optimizations that can't quite be considered
   bug fixes.
@@ -23,16 +45,16 @@ found in the git revision history:
     --enable-debug --enable-dss configurations.
   - Fix a minor statistics bug for mallctl("swap.avail", ...).
 
-* 2.0.1
+* 2.0.1 (October 29, 2010)
 
   Bug fixes:
   - Fix a race condition in heap profiling that could cause undefined behavior
-    if opt.prof_accum were disabled.
+    if "opt.prof_accum" were disabled.
   - Add missing mutex unlocks for some OOM error paths in the heap profiling
     code.
   - Fix a compilation error for non-C99 builds.
 
-* 2.0.0
+* 2.0.0 (October 24, 2010)
 
   This version focuses on the experimental *allocm() API, and on improved
   run-time configuration/introspection.  Nonetheless, numerous performance
@@ -84,7 +106,7 @@ found in the git revision history:
   - Fix a heap profiling bug due to sometimes losing track of requested object
     size for sampled objects.
 
-* 1.0.3
+* 1.0.3 (August 12, 2010)
 
   Bug fixes:
   - Fix the libunwind-based implementation of stack backtracing (used for heap
@@ -93,7 +115,7 @@ found in the git revision history:
     threads raced to initialize malloc, some of them could end up permanently
     blocked.
 
-* 1.0.2
+* 1.0.2 (May 11, 2010)
 
   Bug fixes:
   - Fix junk filling of large objects, which could cause memory corruption.
@@ -101,7 +123,7 @@ found in the git revision history:
     memory limits could cause swap file configuration to fail.  Contributed by
     Jordan DeLong.
 
-* 1.0.1
+* 1.0.1 (April 14, 2010)
 
   Bug fixes:
   - Fix compilation when --enable-fill is specified.
@@ -110,7 +132,7 @@ found in the git revision history:
   - Fix dirty page purging race conditions that could cause crashes.
   - Fix crash in tcache flushing code during thread destruction.
 
-* 1.0.0
+* 1.0.0 (April 11, 2010)
 
   This release focuses on speed and run-time introspection.  Numerous
   algorithmic improvements make this release substantially faster than its
@@ -146,7 +168,7 @@ found in the git revision history:
   - Fix a chunk leak.  The leaked chunks were never touched, so this impacted
     virtual memory usage, but not physical memory usage.
 
-* linux_20080828a, linux_20080827a
+* linux_2008082[78]a (August 27/28, 2008)
 
   These snapshot releases are the simple result of incorporating Linux-specific
   support into the FreeBSD malloc sources.

jemalloc/INSTALL

@@ -28,7 +28,7 @@ any of the following arguments (not a definitive list) to 'configure':
 
 --with-jemalloc-prefix=<prefix>
     Prefix all public APIs with <prefix>.  For example, if <prefix> is
-    "prefix_", the API changes like the following occur:
+    "prefix_", API changes like the following occur:
 
       malloc()         --> prefix_malloc()
       malloc_conf      --> prefix_malloc_conf
@@ -48,9 +48,9 @@ any of the following arguments (not a definitive list) to 'configure':
     example, libjemalloc.so.0 becomes libjemalloc<suffix>.so.0.
 
 --enable-cc-silence
-    Enable code that silences unuseful compiler warnings.  This is helpful when
-    trying to tell serious warnings from those due to compiler limitations, but
-    it potentially incurs a performance penalty.
+    Enable code that silences non-useful compiler warnings.  This is helpful
+    when trying to tell serious warnings from those due to compiler
+    limitations, but it potentially incurs a performance penalty.
 
 --enable-debug
     Enable assertions and validation code.  This incurs a substantial
@@ -62,7 +62,7 @@ any of the following arguments (not a definitive list) to 'configure':
 
 --enable-prof
     Enable heap profiling and leak detection functionality.  See the "opt.prof"
-    option documention for usage details.
+    option documentation for usage details.
 
 --disable-prof-libgcc
     Disable the use of libgcc's backtracing functionality.  Ordinarily, libgcc's
@@ -89,7 +89,7 @@ any of the following arguments (not a definitive list) to 'configure':
 --disable-tcache
     Disable thread-specific caches for small objects.  Objects are cached and
     released in bulk, thus reducing the total number of mutex operations.  See
-    the "opt.tcache" option for suage details.
+    the "opt.tcache" option for usage details.
 
 --enable-swap
     Enable mmap()ed swap file support.  When this feature is built in, it is
@@ -198,8 +198,8 @@ MANDIR="?"
     Use this as the installation prefix for man pages.
 
 DESTDIR="?"
-    Prepend DESTDIR to INCLUDEDIR, LIBDIR, and MANDIR.  This is useful when
-    installing to a different path than was specified via --prefix.
+    Prepend DESTDIR to INCLUDEDIR, LIBDIR, DATADIR, and MANDIR.  This is useful
+    when installing to a different path than was specified via --prefix.
 
 CC="?"
     Use this to invoke the C compiler.

jemalloc/Makefile.in

@@ -20,7 +20,7 @@ MANDIR := $(DESTDIR)@MANDIR@
 
 # Build parameters.
 CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include
-CFLAGS := @CFLAGS@ -fPIC -DPIC
+CFLAGS := @CFLAGS@
 ifeq (macho, @abi@)
 CFLAGS += -dynamic
 endif
@@ -55,6 +55,7 @@ CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/base.c \
 ifeq (macho, @abi@)
 CSRCS += @srcroot@src/zone.c
 endif
+STATIC_LIBS :=	@objroot@lib/libjemalloc@install_suffix@.a
 DSOS := @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) \
 	@objroot@lib/libjemalloc@install_suffix@.$(SO) \
 	@objroot@lib/libjemalloc@install_suffix@_pic.a
@@ -72,8 +73,10 @@ CTESTS := @srcroot@test/allocated.c @srcroot@test/allocm.c \
 .PHONY: install_html install_man install_doc install
 .PHONY: tests check clean distclean relclean
 
+.SECONDARY : $(CTESTS:@srcroot@%.c=@objroot@%.o)
+
 # Default target.
-all: $(DSOS)
+all: $(DSOS) $(STATIC_LIBS)
 
 dist: doc
 
@@ -97,15 +100,24 @@ doc: $(DOCS)
 	$(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $<
 	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)"
 
+@objroot@src/%.pic.o: @srcroot@src/%.c
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS) -fPIC -DPIC -c $(CPPFLAGS) -o $@ $<
+	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)"
+
 %.$(SO) : %.$(SO).$(REV)
 	@mkdir -p $(@D)
 	ln -sf $(<F) $@
 
-@objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) : $(CSRCS:@srcroot@%.c=@objroot@%.o)
+@objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) : $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
 	@mkdir -p $(@D)
 	$(CC) -shared -Wl,-$(WL_SONAME),$(@F) $(RPATH_EXTRA:%=@RPATH@%) -o $@ $+ $(LDFLAGS) $(LIBS)
 
-@objroot@lib/libjemalloc@install_suffix@_pic.a : $(CSRCS:@srcroot@%.c=@objroot@%.o)
+@objroot@lib/libjemalloc@install_suffix@_pic.a : $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
+	@mkdir -p $(@D)
+	ar crus $@ $+
+
+@objroot@lib/libjemalloc@install_suffix@.a : $(CSRCS:@srcroot@%.c=@objroot@%.o)
 	@mkdir -p $(@D)
 	ar crus $@ $+
 
@@ -137,11 +149,12 @@ install_include:
 	install -m 644 $$h $(INCLUDEDIR)/jemalloc; \
 done
 
-install_lib: $(DSOS)
+install_lib: $(DSOS) $(STATIC_LIBS)
 	install -d $(LIBDIR)
 	install -m 755 @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)
 	ln -sf libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)/libjemalloc@install_suffix@.$(SO)
 	install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR)
+	install -m 755 @objroot@lib/libjemalloc@install_suffix@.a $(LIBDIR)
 
 install_html:
 	install -d $(DATADIR)/doc/jemalloc@install_suffix@
@@ -193,12 +206,14 @@ check: tests
 
 clean:
 	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.o)
+	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
 	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.d)
+	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.d)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.o)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.d)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.out)
-	rm -f $(DSOS)
+	rm -f $(DSOS) $(STATIC_LIBS)
 
 distclean: clean
 	rm -rf @objroot@autom4te.cache

jemalloc/include/jemalloc/internal/arena.h

@@ -45,9 +45,10 @@
  * point is implicitly RUN_BFP bits to the left.
  *
  * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
- * honored for some/all object sizes, since there is one bit of header overhead
- * per object (plus a constant).  This constraint is relaxed (ignored) for runs
- * that are so small that the per-region overhead is greater than:
+ * honored for some/all object sizes, since when heap profiling is enabled
+ * there is one pointer of header overhead per object (plus a constant).  This
+ * constraint is relaxed (ignored) for runs that are so small that the
+ * per-region overhead is greater than:
  *
  *   (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
  */
@@ -105,7 +106,7 @@ struct arena_chunk_map_s {
 	 * Run address (or size) and various flags are stored together.  The bit
 	 * layout looks like (assuming 32-bit system):
 	 *
-	 *   ???????? ???????? ????---- ----dzla
+	 *   ???????? ???????? ????---- ----dula
 	 *
 	 * ? : Unallocated: Run address for first/last pages, unset for internal
 	 *                  pages.
@@ -113,7 +114,7 @@ struct arena_chunk_map_s {
 	 *     Large: Run size for first page, unset for trailing pages.
 	 * - : Unused.
 	 * d : dirty?
-	 * z : zeroed?
+	 * u : unzeroed?
 	 * l : large?
 	 * a : allocated?
 	 *
@@ -129,30 +130,30 @@ struct arena_chunk_map_s {
 	 * [dula] : bit unset
 	 *
 	 *   Unallocated (clean):
-	 *     ssssssss ssssssss ssss---- ----du--
+	 *     ssssssss ssssssss ssss---- ----du-a
 	 *     xxxxxxxx xxxxxxxx xxxx---- -----Uxx
-	 *     ssssssss ssssssss ssss---- ----dU--
+	 *     ssssssss ssssssss ssss---- ----dU-a
 	 *
 	 *   Unallocated (dirty):
-	 *     ssssssss ssssssss ssss---- ----D---
+	 *     ssssssss ssssssss ssss---- ----D--a
 	 *     xxxxxxxx xxxxxxxx xxxx---- ----xxxx
-	 *     ssssssss ssssssss ssss---- ----D---
+	 *     ssssssss ssssssss ssss---- ----D--a
 	 *
 	 *   Small:
-	 *     pppppppp pppppppp pppp---- ----d--a
-	 *     pppppppp pppppppp pppp---- -------a
-	 *     pppppppp pppppppp pppp---- ----d--a
+	 *     pppppppp pppppppp pppp---- ----d--A
+	 *     pppppppp pppppppp pppp---- -------A
+	 *     pppppppp pppppppp pppp---- ----d--A
 	 *
 	 *   Large:
-	 *     ssssssss ssssssss ssss---- ----D-la
+	 *     ssssssss ssssssss ssss---- ----D-LA
 	 *     xxxxxxxx xxxxxxxx xxxx---- ----xxxx
-	 *     -------- -------- -------- ----D-la
+	 *     -------- -------- -------- ----D-LA
 	 *
 	 *   Large (sampled, size <= PAGE_SIZE):
-	 *     ssssssss ssssssss sssscccc ccccD-la
+	 *     ssssssss ssssssss sssscccc ccccD-LA
 	 *
 	 *   Large (not sampled, size == PAGE_SIZE):
-	 *     ssssssss ssssssss ssss---- ----D-la
+	 *     ssssssss ssssssss ssss---- ----D-LA
 	 */
 	size_t				bits;
 #ifdef JEMALLOC_PROF
@@ -347,45 +348,35 @@ struct arena_s {
 
 	/*
 	 * bins is used to store trees of free regions of the following sizes,
-	 * assuming a 16-byte quantum, 4 KiB page size, and default
-	 * JEMALLOC_OPTIONS.
+	 * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and
+	 * default MALLOC_CONF.
 	 *
 	 *   bins[i] |   size |
 	 *   --------+--------+
-	 *        0  |      2 |
-	 *        1  |      4 |
-	 *        2  |      8 |
+	 *        0  |      8 |
 	 *   --------+--------+
-	 *        3  |     16 |
-	 *        4  |     32 |
-	 *        5  |     48 |
+	 *        1  |     16 |
+	 *        2  |     32 |
+	 *        3  |     48 |
 	 *           :        :
-	 *        8  |     96 |
-	 *        9  |    112 |
-	 *       10  |    128 |
+	 *        6  |     96 |
+	 *        7  |    112 |
+	 *        8  |    128 |
 	 *   --------+--------+
-	 *       11  |    192 |
-	 *       12  |    256 |
-	 *       13  |    320 |
-	 *       14  |    384 |
-	 *       15  |    448 |
-	 *       16  |    512 |
+	 *        9  |    192 |
+	 *       10  |    256 |
+	 *       11  |    320 |
+	 *       12  |    384 |
+	 *       13  |    448 |
+	 *       14  |    512 |
 	 *   --------+--------+
-	 *       17  |    768 |
-	 *       18  |   1024 |
-	 *       19  |   1280 |
+	 *       15  |    768 |
+	 *       16  |   1024 |
+	 *       17  |   1280 |
 	 *           :        :
-	 *       27  |   3328 |
-	 *       28  |   3584 |
-	 *       29  |   3840 |
-	 *   --------+--------+
-	 *       30  |  4 KiB |
-	 *       31  |  6 KiB |
-	 *       33  |  8 KiB |
-	 *           :        :
-	 *       43  | 28 KiB |
-	 *       44  | 30 KiB |
-	 *       45  | 32 KiB |
+	 *       25  |   3328 |
+	 *       26  |   3584 |
+	 *       27  |   3840 |
 	 *   --------+--------+
 	 */
 	arena_bin_t		bins[1]; /* Dynamically sized. */

jemalloc/include/jemalloc/internal/ckh.h

@@ -31,7 +31,7 @@ struct ckhc_s {
 
 struct ckh_s {
 #ifdef JEMALLOC_DEBUG
-#define	CKH_MAGIG	0x3af2489d
+#define	CKH_MAGIC	0x3af2489d
 	uint32_t	magic;
 #endif
 

jemalloc/include/jemalloc/internal/hash.h

@@ -62,7 +62,7 @@ hash(const void *key, size_t len, uint64_t seed)
 	h *= m;
 	h ^= h >> r;
 
-	return h;
+	return (h);
 }
 #endif
 

jemalloc/include/jemalloc/internal/jemalloc_internal.h.in

@@ -148,11 +148,6 @@ extern void	(*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s);
 
 #define	SIZEOF_PTR		(1U << LG_SIZEOF_PTR)
 
-/* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
-#if (!defined(PIC) && !defined(NO_TLS))
-#  define NO_TLS
-#endif
-
 /*
  * Maximum size of L1 cache line.  This is used to avoid cache line aliasing.
  * In addition, this controls the spacing of cacheline-spaced size classes.
@@ -240,6 +235,13 @@ extern void	(*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s);
 #endif
 #include "jemalloc/internal/prof.h"
 
+#ifdef JEMALLOC_STATS
+typedef struct {
+	uint64_t	allocated;
+	uint64_t	deallocated;
+} thread_allocated_t;
+#endif
+
 #undef JEMALLOC_H_STRUCTS
 /******************************************************************************/
 #define JEMALLOC_H_EXTERNS
@@ -295,45 +297,28 @@ extern arena_t		**arenas;
 extern unsigned		narenas;
 
 #ifdef JEMALLOC_STATS
-typedef struct {
-	uint64_t	allocated;
-	uint64_t	deallocated;
-} thread_allocated_t;
 #  ifndef NO_TLS
 extern __thread thread_allocated_t	thread_allocated_tls;
-#    define ALLOCATED_GET() thread_allocated_tls.allocated
-#    define DEALLOCATED_GET() thread_allocated_tls.deallocated
+#    define ALLOCATED_GET() (thread_allocated_tls.allocated)
+#    define ALLOCATEDP_GET() (&thread_allocated_tls.allocated)
+#    define DEALLOCATED_GET() (thread_allocated_tls.deallocated)
+#    define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated)
 #    define ALLOCATED_ADD(a, d) do {					\
 	thread_allocated_tls.allocated += a;				\
 	thread_allocated_tls.deallocated += d;				\
 } while (0)
 #  else
 extern pthread_key_t	thread_allocated_tsd;
-#    define ALLOCATED_GET()						\
-	(uint64_t)((pthread_getspecific(thread_allocated_tsd) != NULL)	\
-	    ? ((thread_allocated_t *)					\
-	    pthread_getspecific(thread_allocated_tsd))->allocated : 0)
-#    define DEALLOCATED_GET()						\
-	(uint64_t)((pthread_getspecific(thread_allocated_tsd) != NULL)	\
-	    ? ((thread_allocated_t					\
-	    *)pthread_getspecific(thread_allocated_tsd))->deallocated :	\
-	    0)
+thread_allocated_t	*thread_allocated_get_hard(void);
+
+#    define ALLOCATED_GET() (thread_allocated_get()->allocated)
+#    define ALLOCATEDP_GET() (&thread_allocated_get()->allocated)
+#    define DEALLOCATED_GET() (thread_allocated_get()->deallocated)
+#    define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated)
 #    define ALLOCATED_ADD(a, d) do {					\
-	thread_allocated_t *thread_allocated = (thread_allocated_t *)	\
-	    pthread_getspecific(thread_allocated_tsd);			\
-	if (thread_allocated != NULL) {					\
-		thread_allocated->allocated += (a);			\
-		thread_allocated->deallocated += (d);			\
-	} else {							\
-		thread_allocated = (thread_allocated_t *)		\
-		    imalloc(sizeof(thread_allocated_t));		\
-		if (thread_allocated != NULL) {				\
-			pthread_setspecific(thread_allocated_tsd,	\
-			    thread_allocated);				\
-			thread_allocated->allocated = (a);		\
-			thread_allocated->deallocated = (d);		\
-		}							\
-	}								\
+	thread_allocated_t *thread_allocated = thread_allocated_get();	\
+	thread_allocated->allocated += (a);				\
+	thread_allocated->deallocated += (d);				\
 } while (0)
 #  endif
 #endif
@@ -384,6 +369,9 @@ size_t	s2u(size_t size);
 size_t	sa2u(size_t size, size_t alignment, size_t *run_size_p);
 void	malloc_write(const char *s);
 arena_t	*choose_arena(void);
+#  if (defined(JEMALLOC_STATS) && defined(NO_TLS))
+thread_allocated_t	*thread_allocated_get(void);
+#  endif
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
@@ -414,10 +402,10 @@ s2u(size_t size)
 {
 
 	if (size <= small_maxclass)
-		return arenas[0]->bins[small_size2bin[size]].reg_size;
+		return (arenas[0]->bins[small_size2bin[size]].reg_size);
 	if (size <= arena_maxclass)
-		return PAGE_CEILING(size);
-	return CHUNK_CEILING(size);
+		return (PAGE_CEILING(size));
+	return (CHUNK_CEILING(size));
 }
 
 /*
@@ -544,6 +532,19 @@ choose_arena(void)
 
 	return (ret);
 }
+
+#if (defined(JEMALLOC_STATS) && defined(NO_TLS))
+JEMALLOC_INLINE thread_allocated_t *
+thread_allocated_get(void)
+{
+	thread_allocated_t *thread_allocated = (thread_allocated_t *)
+	    pthread_getspecific(thread_allocated_tsd);
+
+	if (thread_allocated == NULL)
+		return (thread_allocated_get_hard());
+	return (thread_allocated);
+}
+#endif
 #endif
 
 #include "jemalloc/internal/rtree.h"

jemalloc/include/jemalloc/internal/tcache.h

@@ -37,7 +37,6 @@ struct tcache_bin_s {
 	tcache_bin_stats_t tstats;
 #  endif
 	unsigned	low_water;	/* Min # cached since last GC. */
-	unsigned	high_water;	/* Max # cached since last GC. */
 	unsigned	ncached;	/* # of cached objects. */
 	unsigned	ncached_max;	/* Upper limit on ncached. */
 	void		*avail;		/* Chain of available objects. */
@@ -194,7 +193,6 @@ tcache_event(tcache_t *tcache)
 			}
 		}
 		tbin->low_water = tbin->ncached;
-		tbin->high_water = tbin->ncached;
 
 		tcache->next_gc_bin++;
 		if (tcache->next_gc_bin == nhbins)
@@ -348,8 +346,6 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
 	*(void **)ptr = tbin->avail;
 	tbin->avail = ptr;
 	tbin->ncached++;
-	if (tbin->ncached > tbin->high_water)
-		tbin->high_water = tbin->ncached;
 
 	tcache_event(tcache);
 }
@@ -388,8 +384,6 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 	*(void **)ptr = tbin->avail;
 	tbin->avail = ptr;
 	tbin->ncached++;
-	if (tbin->ncached > tbin->high_water)
-		tbin->high_water = tbin->ncached;
 
 	tcache_event(tcache);
 }

jemalloc/include/jemalloc/jemalloc.h.in

@@ -19,7 +19,7 @@ extern "C" {
 #  define JEMALLOC_P(s) s
 #endif
 
-#define	ALLOCM_LG_ALIGN	((int)0x3f)
+#define	ALLOCM_LG_ALIGN(la)	(la)
 #if LG_SIZEOF_PTR == 2
 #define	ALLOCM_ALIGN(a)	(ffs(a)-1)
 #else

jemalloc/src/arena.c

@@ -1358,8 +1358,6 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind
 #endif
 	malloc_mutex_unlock(&bin->lock);
 	tbin->ncached = i;
-	if (tbin->ncached > tbin->high_water)
-		tbin->high_water = tbin->ncached;
 }
 #endif
 
@@ -1369,7 +1367,6 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind
  *   *) bin->run_size >= min_run_size
  *   *) bin->run_size <= arena_maxclass
  *   *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
- *   *) run header size < PAGE_SIZE
  *
  * bin->nregs and bin->reg0_offset are also calculated here, since these
  * settings are all interdependent.
@@ -1455,8 +1452,7 @@ arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
 	} while (try_run_size <= arena_maxclass
 	    && try_run_size <= arena_maxclass
 	    && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
-	    && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size
-	    && try_hdr_size < PAGE_SIZE);
+	    && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
 
 	assert(good_hdr_size <= good_reg0_offset);
 

jemalloc/src/chunk_mmap.c

@@ -206,13 +206,15 @@ chunk_alloc_mmap_internal(size_t size, bool noreserve)
 void *
 chunk_alloc_mmap(size_t size)
 {
-	return chunk_alloc_mmap_internal(size, false);
+
+	return (chunk_alloc_mmap_internal(size, false));
 }
 
 void *
 chunk_alloc_mmap_noreserve(size_t size)
 {
-	return chunk_alloc_mmap_internal(size, true);
+
+	return (chunk_alloc_mmap_internal(size, true));
 }
 
 void

jemalloc/src/ckh.c

@@ -73,7 +73,7 @@ ckh_isearch(ckh_t *ckh, const void *key)
 	size_t hash1, hash2, bucket, cell;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
 
@@ -383,7 +383,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
 	}
 
 #ifdef JEMALLOC_DEBUG
-	ckh->magic = CKH_MAGIG;
+	ckh->magic = CKH_MAGIC;
 #endif
 
 	ret = false;
@@ -396,7 +396,7 @@ ckh_delete(ckh_t *ckh)
 {
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 #ifdef CKH_VERBOSE
 	malloc_printf(
@@ -421,7 +421,7 @@ ckh_count(ckh_t *ckh)
 {
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	return (ckh->count);
 }
@@ -452,7 +452,7 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data)
 	bool ret;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 	assert(ckh_search(ckh, key, NULL, NULL));
 
 #ifdef CKH_COUNT
@@ -477,7 +477,7 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
 	size_t cell;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	cell = ckh_isearch(ckh, searchkey);
 	if (cell != SIZE_T_MAX) {
@@ -509,7 +509,7 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data)
 	size_t cell;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	cell = ckh_isearch(ckh, searchkey);
 	if (cell != SIZE_T_MAX) {

jemalloc/src/ctl.c

@@ -1114,8 +1114,8 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 	unsigned newind, oldind;
 
 	newind = oldind = choose_arena()->ind;
-	WRITE(oldind, unsigned);
-	READ(newind, unsigned);
+	WRITE(newind, unsigned);
+	READ(oldind, unsigned);
 	if (newind != oldind) {
 		arena_t *arena;
 
@@ -1137,6 +1137,11 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 
 		/* Set new arena association. */
 		ARENA_SET(arena);
+		{
+			tcache_t *tcache = TCACHE_GET();
+			if (tcache != NULL)
+				tcache->arena = arena;
+		}
 	}
 
 	ret = 0;
@@ -1146,9 +1151,9 @@ RETURN:
 
 #ifdef JEMALLOC_STATS
 CTL_RO_NL_GEN(thread_allocated, ALLOCATED_GET(), uint64_t);
-CTL_RO_NL_GEN(thread_allocatedp, &ALLOCATED_GET(), uint64_t *);
+CTL_RO_NL_GEN(thread_allocatedp, ALLOCATEDP_GET(), uint64_t *);
 CTL_RO_NL_GEN(thread_deallocated, DEALLOCATED_GET(), uint64_t);
-CTL_RO_NL_GEN(thread_deallocatedp, &DEALLOCATED_GET(), uint64_t *);
+CTL_RO_NL_GEN(thread_deallocatedp, DEALLOCATEDP_GET(), uint64_t *);
 #endif
 
 /******************************************************************************/

jemalloc/src/huge.c

@@ -83,7 +83,7 @@ huge_palloc(size_t size, size_t alignment, bool zero)
 	 * alignment, in order to assure the alignment can be achieved, then
 	 * unmap leading and trailing chunks.
 	 */
-	assert(alignment >= chunksize);
+	assert(alignment > chunksize);
 
 	chunk_size = CHUNK_CEILING(size);
 
@@ -192,7 +192,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 	 * different size class.  In that case, fall back to allocating new
 	 * space and copying.
 	 */
-	if (alignment != 0)
+	if (alignment > chunksize)
 		ret = huge_palloc(size + extra, alignment, zero);
 	else
 		ret = huge_malloc(size + extra, zero);
@@ -201,7 +201,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 		if (extra == 0)
 			return (NULL);
 		/* Try again, this time without extra. */
-		if (alignment != 0)
+		if (alignment > chunksize)
 			ret = huge_palloc(size, alignment, zero);
 		else
 			ret = huge_malloc(size, zero);

jemalloc/src/jemalloc.c

@@ -213,6 +213,28 @@ stats_print_atexit(void)
 	JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL);
 }
 
+#if (defined(JEMALLOC_STATS) && defined(NO_TLS))
+thread_allocated_t *
+thread_allocated_get_hard(void)
+{
+	thread_allocated_t *thread_allocated = (thread_allocated_t *)
+	    imalloc(sizeof(thread_allocated_t));
+	if (thread_allocated == NULL) {
+		static thread_allocated_t static_thread_allocated = {0, 0};
+		malloc_write("<jemalloc>: Error allocating TSD;"
+		    " mallctl(\"thread.{de,}allocated[p]\", ...)"
+		    " will be inaccurate\n");
+		if (opt_abort)
+			abort();
+		return (&static_thread_allocated);
+	}
+	pthread_setspecific(thread_allocated_tsd, thread_allocated);
+	thread_allocated->allocated = 0;
+	thread_allocated->deallocated = 0;
+	return (thread_allocated);
+}
+#endif
+
 /*
  * End miscellaneous support functions.
  */
@@ -421,8 +443,8 @@ malloc_conf_init(void)
 			if ((opts = getenv(envname)) != NULL) {
 				/*
 				 * Do nothing; opts is already initialized to
-				 * the value of the JEMALLOC_OPTIONS
-				 * environment variable.
+				 * the value of the MALLOC_CONF environment
+				 * variable.
 				 */
 			} else {
 				/* No configuration specified. */

jemalloc/src/prof.c

@@ -432,6 +432,7 @@ prof_lookup(prof_bt_t *bt)
 			prof_ctx_t	*p;
 			void		*v;
 		} ctx;
+		bool new_ctx;
 
 		/*
 		 * This thread's cache lacks bt.  Look for it in the global
@@ -468,12 +469,14 @@ prof_lookup(prof_bt_t *bt)
 				idalloc(ctx.v);
 				return (NULL);
 			}
-		}
-		/*
-		 * Acquire ctx's lock before releasing bt2ctx_mtx, in order to
-		 * avoid a race condition with prof_ctx_destroy().
-		 */
-		malloc_mutex_lock(&ctx.p->lock);
+			/*
+			 * Artificially raise curobjs, in order to avoid a race
+			 * condition with prof_ctx_merge()/prof_ctx_destroy().
+			 */
+			ctx.p->cnt_merged.curobjs++;
+			new_ctx = true;
+		} else
+			new_ctx = false;
 		prof_leave();
 
 		/* Link a prof_thd_cnt_t into ctx for this thread. */
@@ -498,7 +501,11 @@ prof_lookup(prof_bt_t *bt)
 			/* Allocate and partially initialize a new cnt. */
 			ret.v = imalloc(sizeof(prof_thr_cnt_t));
 			if (ret.p == NULL) {
-				malloc_mutex_unlock(&ctx.p->lock);
+				if (new_ctx) {
+					malloc_mutex_lock(&ctx.p->lock);
+					ctx.p->cnt_merged.curobjs--;
+					malloc_mutex_unlock(&ctx.p->lock);
+				}
 				return (NULL);
 			}
 			ql_elm_new(ret.p, cnts_link);
@@ -509,12 +516,19 @@ prof_lookup(prof_bt_t *bt)
 		ret.p->epoch = 0;
 		memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
 		if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) {
-			malloc_mutex_unlock(&ctx.p->lock);
+			if (new_ctx) {
+				malloc_mutex_lock(&ctx.p->lock);
+				ctx.p->cnt_merged.curobjs--;
+				malloc_mutex_unlock(&ctx.p->lock);
+			}
 			idalloc(ret.v);
 			return (NULL);
 		}
 		ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link);
+		malloc_mutex_lock(&ctx.p->lock);
 		ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link);
+		if (new_ctx)
+			ctx.p->cnt_merged.curobjs--;
 		malloc_mutex_unlock(&ctx.p->lock);
 	} else {
 		/* Move ret to the front of the LRU. */

jemalloc/src/tcache.c

@@ -58,6 +58,9 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 	void *flush, *deferred, *ptr;
 	unsigned i, nflush, ndeferred;
 	bool first_pass;
+#ifdef JEMALLOC_STATS
+	bool merged_stats = false;
+#endif
 
 	assert(binind < nbins);
 	assert(rem <= tbin->ncached);
@@ -82,6 +85,8 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 		malloc_mutex_lock(&bin->lock);
 #ifdef JEMALLOC_STATS
 		if (arena == tcache->arena) {
+			assert(merged_stats == false);
+			merged_stats = true;
 			bin->stats.nflushes++;
 			bin->stats.nrequests += tbin->tstats.nrequests;
 			tbin->tstats.nrequests = 0;
@@ -119,6 +124,20 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 			first_pass = false;
 		}
 	}
+#ifdef JEMALLOC_STATS
+	if (merged_stats == false) {
+		/*
+		 * The flush loop didn't happen to flush to this thread's
+		 * arena, so the stats didn't get merged.  Manually do so now.
+		 */
+		arena_bin_t *bin = &tcache->arena->bins[binind];
+		malloc_mutex_lock(&bin->lock);
+		bin->stats.nflushes++;
+		bin->stats.nrequests += tbin->tstats.nrequests;
+		tbin->tstats.nrequests = 0;
+		malloc_mutex_unlock(&bin->lock);
+	}
+#endif
 
 	tbin->ncached = rem;
 	if (tbin->ncached < tbin->low_water)

jemalloc/test/allocated.c

@@ -16,6 +16,7 @@ thread_start(void *arg)
 	int err;
 	void *p;
 	uint64_t a0, a1, d0, d1;
+	uint64_t *ap0, *ap1, *dp0, *dp1;
 	size_t sz, usize;
 
 	sz = sizeof(a0);
@@ -31,6 +32,20 @@ thread_start(void *arg)
 		    strerror(err));
 		exit(1);
 	}
+	sz = sizeof(ap0);
+	if ((err = JEMALLOC_P(mallctl)("thread.allocatedp", &ap0, &sz, NULL,
+	    0))) {
+		if (err == ENOENT) {
+#ifdef JEMALLOC_STATS
+			assert(false);
+#endif
+			goto RETURN;
+		}
+		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
+		    strerror(err));
+		exit(1);
+	}
+	assert(*ap0 == a0);
 
 	sz = sizeof(d0);
 	if ((err = JEMALLOC_P(mallctl)("thread.deallocated", &d0, &sz, NULL,
@@ -45,6 +60,20 @@ thread_start(void *arg)
 		    strerror(err));
 		exit(1);
 	}
+	sz = sizeof(dp0);
+	if ((err = JEMALLOC_P(mallctl)("thread.deallocatedp", &dp0, &sz, NULL,
+	    0))) {
+		if (err == ENOENT) {
+#ifdef JEMALLOC_STATS
+			assert(false);
+#endif
+			goto RETURN;
+		}
+		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
+		    strerror(err));
+		exit(1);
+	}
+	assert(*dp0 == d0);
 
 	p = JEMALLOC_P(malloc)(1);
 	if (p == NULL) {
@@ -54,6 +83,10 @@ thread_start(void *arg)
 
 	sz = sizeof(a1);
 	JEMALLOC_P(mallctl)("thread.allocated", &a1, &sz, NULL, 0);
+	sz = sizeof(ap1);
+	JEMALLOC_P(mallctl)("thread.allocatedp", &ap1, &sz, NULL, 0);
+	assert(*ap1 == a1);
+	assert(ap0 == ap1);
 
 	usize = JEMALLOC_P(malloc_usable_size)(p);
 	assert(a0 + usize <= a1);
@@ -62,6 +95,10 @@ thread_start(void *arg)
 
 	sz = sizeof(d1);
 	JEMALLOC_P(mallctl)("thread.deallocated", &d1, &sz, NULL, 0);
+	sz = sizeof(dp1);
+	JEMALLOC_P(mallctl)("thread.deallocatedp", &dp1, &sz, NULL, 0);
+	assert(*dp1 == d1);
+	assert(dp0 == dp1);
 
 	assert(d0 + usize <= d1);
 

jemalloc/test/thread_arena.c

@@ -2,10 +2,13 @@
 #include <stdlib.h>
 #include <pthread.h>
 #include <string.h>
+#include <assert.h>
 
 #define	JEMALLOC_MANGLE
 #include "jemalloc_test.h"
 
+#define NTHREADS 10
+
 void *
 thread_start(void *arg)
 {
@@ -29,6 +32,15 @@ thread_start(void *arg)
 		return (void *)1;
 	}
 
+	size = sizeof(arena_ind);
+	if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL,
+	    0))) {
+		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
+		    strerror(err));
+		return (void *)1;
+	}
+	assert(arena_ind == main_arena_ind);
+
 	return (NULL);
 }
 
@@ -40,7 +52,8 @@ main(void)
 	unsigned arena_ind;
 	size_t size;
 	int err;
-	pthread_t thread;
+	pthread_t threads[NTHREADS];
+	unsigned i;
 
 	fprintf(stderr, "Test begin\n");
 
@@ -60,21 +73,18 @@ main(void)
 		goto RETURN;
 	}
 
-	if (pthread_create(&thread, NULL, thread_start, (void *)&arena_ind)
-	    != 0) {
-		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
-		ret = 1;
-		goto RETURN;
+	for (i = 0; i < NTHREADS; i++) {
+		if (pthread_create(&threads[i], NULL, thread_start,
+		    (void *)&arena_ind) != 0) {
+			fprintf(stderr, "%s(): Error in pthread_create()\n",
+			    __func__);
+			ret = 1;
+			goto RETURN;
+		}
 	}
-	pthread_join(thread, (void *)&ret);
 
-	if (pthread_create(&thread, NULL, thread_start, (void *)&arena_ind)
-	    != 0) {
-		fprintf(stderr, "%s(): Error in pthread_create()\n", __func__);
-		ret = 1;
-		goto RETURN;
-	}
-	pthread_join(thread, (void *)&ret);
+	for (i = 0; i < NTHREADS; i++)
+		pthread_join(threads[i], (void *)&ret);
 
 RETURN:
 	fprintf(stderr, "Test end\n");