Merge branch 'dev'

When jemalloc is linked into an executable (as opposed to a shared
library), compiling with -fno-pic can have significant advantages,
mainly because we don't have to go throught the GOT (global offset
table).

Users who want to link jemalloc into a shared library that could
be dlopened need to link with libjemalloc_pic.a or libjemalloc.so.

jemalloc/ChangeLog

@@ -6,7 +6,21 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
-* 2.1.0
+* 2.1.2 (March 2, 2011)
+
+  Bug fixes:
+  - Fix "thread.{de,}allocatedp" mallctl for OS X.
+  - Add missing jemalloc.a to build system.
+
+* 2.1.1 (January 31, 2011)
+
+  Bug fixes:
+  - Fix aligned huge reallocation (affected allocm()).
+  - Fix the ALLOCM_LG_ALIGN macro definition.
+  - Fix a heap dumping deadlock.
+  - Fix a "thread.arena" mallctl bug.
+
+* 2.1.0 (December 3, 2010)
 
   This version incorporates some optimizations that can't quite be considered
   bug fixes.
@@ -23,16 +37,16 @@ found in the git revision history:
     --enable-debug --enable-dss configurations.
   - Fix a minor statistics bug for mallctl("swap.avail", ...).
 
-* 2.0.1
+* 2.0.1 (October 29, 2010)
 
   Bug fixes:
   - Fix a race condition in heap profiling that could cause undefined behavior
-    if opt.prof_accum were disabled.
+    if "opt.prof_accum" were disabled.
   - Add missing mutex unlocks for some OOM error paths in the heap profiling
     code.
   - Fix a compilation error for non-C99 builds.
 
-* 2.0.0
+* 2.0.0 (October 24, 2010)
 
   This version focuses on the experimental *allocm() API, and on improved
   run-time configuration/introspection.  Nonetheless, numerous performance
@@ -84,7 +98,7 @@ found in the git revision history:
   - Fix a heap profiling bug due to sometimes losing track of requested object
     size for sampled objects.
 
-* 1.0.3
+* 1.0.3 (August 12, 2010)
 
   Bug fixes:
   - Fix the libunwind-based implementation of stack backtracing (used for heap
@@ -93,7 +107,7 @@ found in the git revision history:
     threads raced to initialize malloc, some of them could end up permanently
     blocked.
 
-* 1.0.2
+* 1.0.2 (May 11, 2010)
 
   Bug fixes:
   - Fix junk filling of large objects, which could cause memory corruption.
@@ -101,7 +115,7 @@ found in the git revision history:
     memory limits could cause swap file configuration to fail.  Contributed by
     Jordan DeLong.
 
-* 1.0.1
+* 1.0.1 (April 14, 2010)
 
   Bug fixes:
   - Fix compilation when --enable-fill is specified.
@@ -110,7 +124,7 @@ found in the git revision history:
   - Fix dirty page purging race conditions that could cause crashes.
   - Fix crash in tcache flushing code during thread destruction.
 
-* 1.0.0
+* 1.0.0 (April 11, 2010)
 
   This release focuses on speed and run-time introspection.  Numerous
   algorithmic improvements make this release substantially faster than its
@@ -146,7 +160,7 @@ found in the git revision history:
   - Fix a chunk leak.  The leaked chunks were never touched, so this impacted
     virtual memory usage, but not physical memory usage.
 
-* linux_20080828a, linux_20080827a
+* linux_2008082[78]a (August 27/28, 2008)
 
   These snapshot releases are the simple result of incorporating Linux-specific
   support into the FreeBSD malloc sources.

jemalloc/INSTALL

@@ -28,7 +28,7 @@ any of the following arguments (not a definitive list) to 'configure':
 
 --with-jemalloc-prefix=<prefix>
     Prefix all public APIs with <prefix>.  For example, if <prefix> is
-    "prefix_", the API changes like the following occur:
+    "prefix_", API changes like the following occur:
 
       malloc()         --> prefix_malloc()
       malloc_conf      --> prefix_malloc_conf
@@ -48,9 +48,9 @@ any of the following arguments (not a definitive list) to 'configure':
     example, libjemalloc.so.0 becomes libjemalloc<suffix>.so.0.
 
 --enable-cc-silence
-    Enable code that silences unuseful compiler warnings.  This is helpful when
+    Enable code that silences non-useful compiler warnings.  This is helpful
-    trying to tell serious warnings from those due to compiler limitations, but
+    when trying to tell serious warnings from those due to compiler
-    it potentially incurs a performance penalty.
+    limitations, but it potentially incurs a performance penalty.
 
 --enable-debug
     Enable assertions and validation code.  This incurs a substantial
@@ -62,7 +62,7 @@ any of the following arguments (not a definitive list) to 'configure':
 
 --enable-prof
     Enable heap profiling and leak detection functionality.  See the "opt.prof"
-    option documention for usage details.
+    option documentation for usage details.
 
 --disable-prof-libgcc
     Disable the use of libgcc's backtracing functionality.  Ordinarily, libgcc's
@@ -89,7 +89,7 @@ any of the following arguments (not a definitive list) to 'configure':
 --disable-tcache
     Disable thread-specific caches for small objects.  Objects are cached and
     released in bulk, thus reducing the total number of mutex operations.  See
-    the "opt.tcache" option for suage details.
+    the "opt.tcache" option for usage details.
 
 --enable-swap
     Enable mmap()ed swap file support.  When this feature is built in, it is
@@ -198,8 +198,8 @@ MANDIR="?"
     Use this as the installation prefix for man pages.
 
 DESTDIR="?"
-    Prepend DESTDIR to INCLUDEDIR, LIBDIR, and MANDIR.  This is useful when
+    Prepend DESTDIR to INCLUDEDIR, LIBDIR, DATADIR, and MANDIR.  This is useful
-    installing to a different path than was specified via --prefix.
+    when installing to a different path than was specified via --prefix.
 
 CC="?"
     Use this to invoke the C compiler.

jemalloc/Makefile.in

@@ -20,7 +20,7 @@ MANDIR := $(DESTDIR)@MANDIR@
 
 # Build parameters.
 CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include
-CFLAGS := @CFLAGS@ -fPIC -DPIC
+CFLAGS := @CFLAGS@
 ifeq (macho, @abi@)
 CFLAGS += -dynamic
 endif
@@ -55,6 +55,7 @@ CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/base.c \
 ifeq (macho, @abi@)
 CSRCS += @srcroot@src/zone.c
 endif
+STATIC_LIBS :=	@objroot@lib/libjemalloc@install_suffix@.a
 DSOS := @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) \
 	@objroot@lib/libjemalloc@install_suffix@.$(SO) \
 	@objroot@lib/libjemalloc@install_suffix@_pic.a
@@ -72,8 +73,10 @@ CTESTS := @srcroot@test/allocated.c @srcroot@test/allocm.c \
 .PHONY: install_html install_man install_doc install
 .PHONY: tests check clean distclean relclean
 
+.SECONDARY : $(CTESTS:@srcroot@%.c=@objroot@%.o)
+
 # Default target.
-all: $(DSOS)
+all: $(DSOS) $(STATIC_LIBS)
 
 dist: doc
 
@@ -97,15 +100,24 @@ doc: $(DOCS)
 	$(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $<
 	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)"
 
+@objroot@src/%.pic.o: @srcroot@src/%.c
+	@mkdir -p $(@D)
+	$(CC) $(CFLAGS) -fPIC -DPIC -c $(CPPFLAGS) -o $@ $<
+	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)"
+
 %.$(SO) : %.$(SO).$(REV)
 	@mkdir -p $(@D)
 	ln -sf $(<F) $@
 
-@objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) : $(CSRCS:@srcroot@%.c=@objroot@%.o)
+@objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) : $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
 	@mkdir -p $(@D)
 	$(CC) -shared -Wl,-$(WL_SONAME),$(@F) $(RPATH_EXTRA:%=@RPATH@%) -o $@ $+ $(LDFLAGS) $(LIBS)
 
-@objroot@lib/libjemalloc@install_suffix@_pic.a : $(CSRCS:@srcroot@%.c=@objroot@%.o)
+@objroot@lib/libjemalloc@install_suffix@_pic.a : $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
+	@mkdir -p $(@D)
+	ar crus $@ $+
+
+@objroot@lib/libjemalloc@install_suffix@.a : $(CSRCS:@srcroot@%.c=@objroot@%.o)
 	@mkdir -p $(@D)
 	ar crus $@ $+
 
@@ -137,11 +149,12 @@ install_include:
 	install -m 644 $$h $(INCLUDEDIR)/jemalloc; \
 done
 
-install_lib: $(DSOS)
+install_lib: $(DSOS) $(STATIC_LIBS)
 	install -d $(LIBDIR)
 	install -m 755 @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)
 	ln -sf libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)/libjemalloc@install_suffix@.$(SO)
 	install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR)
+	install -m 755 @objroot@lib/libjemalloc@install_suffix@.a $(LIBDIR)
 
 install_html:
 	install -d $(DATADIR)/doc/jemalloc@install_suffix@
@@ -193,12 +206,14 @@ check: tests
 
 clean:
 	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.o)
+	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.o)
 	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.d)
+	rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.d)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.o)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.d)
 	rm -f $(CTESTS:@srcroot@%.c=@objroot@%.out)
-	rm -f $(DSOS)
+	rm -f $(DSOS) $(STATIC_LIBS)
 
 distclean: clean
 	rm -rf @objroot@autom4te.cache

jemalloc/include/jemalloc/internal/arena.h

@@ -45,9 +45,10 @@
  * point is implicitly RUN_BFP bits to the left.
  *
  * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
- * honored for some/all object sizes, since there is one bit of header overhead
+ * honored for some/all object sizes, since when heap profiling is enabled
- * per object (plus a constant).  This constraint is relaxed (ignored) for runs
+ * there is one pointer of header overhead per object (plus a constant).  This
- * that are so small that the per-region overhead is greater than:
+ * constraint is relaxed (ignored) for runs that are so small that the
+ * per-region overhead is greater than:
  *
  *   (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
  */
@@ -105,7 +106,7 @@ struct arena_chunk_map_s {
 	 * Run address (or size) and various flags are stored together.  The bit
 	 * layout looks like (assuming 32-bit system):
 	 *
-	 *   ???????? ???????? ????---- ----dzla
+	 *   ???????? ???????? ????---- ----dula
 	 *
 	 * ? : Unallocated: Run address for first/last pages, unset for internal
 	 *                  pages.
@@ -113,7 +114,7 @@ struct arena_chunk_map_s {
 	 *     Large: Run size for first page, unset for trailing pages.
 	 * - : Unused.
 	 * d : dirty?
-	 * z : zeroed?
+	 * u : unzeroed?
 	 * l : large?
 	 * a : allocated?
 	 *
@@ -129,30 +130,30 @@ struct arena_chunk_map_s {
 	 * [dula] : bit unset
 	 *
 	 *   Unallocated (clean):
-	 *     ssssssss ssssssss ssss---- ----du--
+	 *     ssssssss ssssssss ssss---- ----du-a
 	 *     xxxxxxxx xxxxxxxx xxxx---- -----Uxx
-	 *     ssssssss ssssssss ssss---- ----dU--
+	 *     ssssssss ssssssss ssss---- ----dU-a
 	 *
 	 *   Unallocated (dirty):
-	 *     ssssssss ssssssss ssss---- ----D---
+	 *     ssssssss ssssssss ssss---- ----D--a
 	 *     xxxxxxxx xxxxxxxx xxxx---- ----xxxx
-	 *     ssssssss ssssssss ssss---- ----D---
+	 *     ssssssss ssssssss ssss---- ----D--a
 	 *
 	 *   Small:
-	 *     pppppppp pppppppp pppp---- ----d--a
+	 *     pppppppp pppppppp pppp---- ----d--A
-	 *     pppppppp pppppppp pppp---- -------a
+	 *     pppppppp pppppppp pppp---- -------A
-	 *     pppppppp pppppppp pppp---- ----d--a
+	 *     pppppppp pppppppp pppp---- ----d--A
 	 *
 	 *   Large:
-	 *     ssssssss ssssssss ssss---- ----D-la
+	 *     ssssssss ssssssss ssss---- ----D-LA
 	 *     xxxxxxxx xxxxxxxx xxxx---- ----xxxx
-	 *     -------- -------- -------- ----D-la
+	 *     -------- -------- -------- ----D-LA
 	 *
 	 *   Large (sampled, size <= PAGE_SIZE):
-	 *     ssssssss ssssssss sssscccc ccccD-la
+	 *     ssssssss ssssssss sssscccc ccccD-LA
 	 *
 	 *   Large (not sampled, size == PAGE_SIZE):
-	 *     ssssssss ssssssss ssss---- ----D-la
+	 *     ssssssss ssssssss ssss---- ----D-LA
 	 */
 	size_t				bits;
 #ifdef JEMALLOC_PROF
@@ -347,45 +348,35 @@ struct arena_s {
 
 	/*
 	 * bins is used to store trees of free regions of the following sizes,
-	 * assuming a 16-byte quantum, 4 KiB page size, and default
+	 * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and
-	 * JEMALLOC_OPTIONS.
+	 * default MALLOC_CONF.
 	 *
 	 *   bins[i] |   size |
 	 *   --------+--------+
-	 *        0  |      2 |
+	 *        0  |      8 |
-	 *        1  |      4 |
-	 *        2  |      8 |
 	 *   --------+--------+
-	 *        3  |     16 |
+	 *        1  |     16 |
-	 *        4  |     32 |
+	 *        2  |     32 |
-	 *        5  |     48 |
+	 *        3  |     48 |
 	 *           :        :
-	 *        8  |     96 |
+	 *        6  |     96 |
-	 *        9  |    112 |
+	 *        7  |    112 |
-	 *       10  |    128 |
+	 *        8  |    128 |
 	 *   --------+--------+
-	 *       11  |    192 |
+	 *        9  |    192 |
-	 *       12  |    256 |
+	 *       10  |    256 |
-	 *       13  |    320 |
+	 *       11  |    320 |
-	 *       14  |    384 |
+	 *       12  |    384 |
-	 *       15  |    448 |
+	 *       13  |    448 |
-	 *       16  |    512 |
+	 *       14  |    512 |
 	 *   --------+--------+
-	 *       17  |    768 |
+	 *       15  |    768 |
-	 *       18  |   1024 |
+	 *       16  |   1024 |
-	 *       19  |   1280 |
+	 *       17  |   1280 |
 	 *           :        :
-	 *       27  |   3328 |
+	 *       25  |   3328 |
-	 *       28  |   3584 |
+	 *       26  |   3584 |
-	 *       29  |   3840 |
+	 *       27  |   3840 |
-	 *   --------+--------+
-	 *       30  |  4 KiB |
-	 *       31  |  6 KiB |
-	 *       33  |  8 KiB |
-	 *           :        :
-	 *       43  | 28 KiB |
-	 *       44  | 30 KiB |
-	 *       45  | 32 KiB |
 	 *   --------+--------+
 	 */
 	arena_bin_t		bins[1]; /* Dynamically sized. */

jemalloc/include/jemalloc/internal/ckh.h

@@ -31,7 +31,7 @@ struct ckhc_s {
 
 struct ckh_s {
 #ifdef JEMALLOC_DEBUG
-#define	CKH_MAGIG	0x3af2489d
+#define	CKH_MAGIC	0x3af2489d
 	uint32_t	magic;
 #endif
 

jemalloc/include/jemalloc/internal/hash.h

@@ -62,7 +62,7 @@ hash(const void *key, size_t len, uint64_t seed)
 	h *= m;
 	h ^= h >> r;
 
-	return h;
+	return (h);
 }
 #endif
 

jemalloc/include/jemalloc/internal/jemalloc_internal.h.in

@@ -148,11 +148,6 @@ extern void	(*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s);
 
 #define	SIZEOF_PTR		(1U << LG_SIZEOF_PTR)
 
-/* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
-#if (!defined(PIC) && !defined(NO_TLS))
-#  define NO_TLS
-#endif
-
 /*
  * Maximum size of L1 cache line.  This is used to avoid cache line aliasing.
  * In addition, this controls the spacing of cacheline-spaced size classes.
@@ -240,6 +235,13 @@ extern void	(*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s);
 #endif
 #include "jemalloc/internal/prof.h"
 
+#ifdef JEMALLOC_STATS
+typedef struct {
+	uint64_t	allocated;
+	uint64_t	deallocated;
+} thread_allocated_t;
+#endif
+
 #undef JEMALLOC_H_STRUCTS
 /******************************************************************************/
 #define JEMALLOC_H_EXTERNS
@@ -295,45 +297,28 @@ extern arena_t		**arenas;
 extern unsigned		narenas;
 
 #ifdef JEMALLOC_STATS
-typedef struct {
-	uint64_t	allocated;
-	uint64_t	deallocated;
-} thread_allocated_t;
 #  ifndef NO_TLS
 extern __thread thread_allocated_t	thread_allocated_tls;
-#    define ALLOCATED_GET() thread_allocated_tls.allocated
+#    define ALLOCATED_GET() (thread_allocated_tls.allocated)
-#    define DEALLOCATED_GET() thread_allocated_tls.deallocated
+#    define ALLOCATEDP_GET() (&thread_allocated_tls.allocated)
+#    define DEALLOCATED_GET() (thread_allocated_tls.deallocated)
+#    define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated)
 #    define ALLOCATED_ADD(a, d) do {					\
 	thread_allocated_tls.allocated += a;				\
 	thread_allocated_tls.deallocated += d;				\
 } while (0)
 #  else
 extern pthread_key_t	thread_allocated_tsd;
-#    define ALLOCATED_GET()						\
+thread_allocated_t	*thread_allocated_get_hard(void);
-	(uint64_t)((pthread_getspecific(thread_allocated_tsd) != NULL)	\
+
-	    ? ((thread_allocated_t *)					\
+#    define ALLOCATED_GET() (thread_allocated_get()->allocated)
-	    pthread_getspecific(thread_allocated_tsd))->allocated : 0)
+#    define ALLOCATEDP_GET() (&thread_allocated_get()->allocated)
-#    define DEALLOCATED_GET()						\
+#    define DEALLOCATED_GET() (thread_allocated_get()->deallocated)
-	(uint64_t)((pthread_getspecific(thread_allocated_tsd) != NULL)	\
+#    define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated)
-	    ? ((thread_allocated_t					\
-	    *)pthread_getspecific(thread_allocated_tsd))->deallocated :	\
-	    0)
 #    define ALLOCATED_ADD(a, d) do {					\
-	thread_allocated_t *thread_allocated = (thread_allocated_t *)	\
+	thread_allocated_t *thread_allocated = thread_allocated_get();	\
-	    pthread_getspecific(thread_allocated_tsd);			\
+	thread_allocated->allocated += (a);				\
-	if (thread_allocated != NULL) {					\
+	thread_allocated->deallocated += (d);				\
-		thread_allocated->allocated += (a);			\
-		thread_allocated->deallocated += (d);			\
-	} else {							\
-		thread_allocated = (thread_allocated_t *)		\
-		    imalloc(sizeof(thread_allocated_t));		\
-		if (thread_allocated != NULL) {				\
-			pthread_setspecific(thread_allocated_tsd,	\
-			    thread_allocated);				\
-			thread_allocated->allocated = (a);		\
-			thread_allocated->deallocated = (d);		\
-		}							\
-	}								\
 } while (0)
 #  endif
 #endif
@@ -384,6 +369,9 @@ size_t	s2u(size_t size);
 size_t	sa2u(size_t size, size_t alignment, size_t *run_size_p);
 void	malloc_write(const char *s);
 arena_t	*choose_arena(void);
+#  ifdef NO_TLS
+thread_allocated_t	*thread_allocated_get(void);
+#  endif
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
@@ -414,10 +402,10 @@ s2u(size_t size)
 {
 
 	if (size <= small_maxclass)
-		return arenas[0]->bins[small_size2bin[size]].reg_size;
+		return (arenas[0]->bins[small_size2bin[size]].reg_size);
 	if (size <= arena_maxclass)
-		return PAGE_CEILING(size);
+		return (PAGE_CEILING(size));
-	return CHUNK_CEILING(size);
+	return (CHUNK_CEILING(size));
 }
 
 /*
@@ -544,6 +532,19 @@ choose_arena(void)
 
 	return (ret);
 }
+
+#ifdef NO_TLS
+JEMALLOC_INLINE thread_allocated_t *
+thread_allocated_get(void)
+{
+	thread_allocated_t *thread_allocated = (thread_allocated_t *)
+	    pthread_getspecific(thread_allocated_tsd);
+
+	if (thread_allocated == NULL)
+		return (thread_allocated_get_hard());
+	return (thread_allocated);
+}
+#endif
 #endif
 
 #include "jemalloc/internal/rtree.h"

jemalloc/include/jemalloc/internal/tcache.h

@@ -37,7 +37,6 @@ struct tcache_bin_s {
 	tcache_bin_stats_t tstats;
 #  endif
 	unsigned	low_water;	/* Min # cached since last GC. */
-	unsigned	high_water;	/* Max # cached since last GC. */
 	unsigned	ncached;	/* # of cached objects. */
 	unsigned	ncached_max;	/* Upper limit on ncached. */
 	void		*avail;		/* Chain of available objects. */
@@ -194,7 +193,6 @@ tcache_event(tcache_t *tcache)
 			}
 		}
 		tbin->low_water = tbin->ncached;
-		tbin->high_water = tbin->ncached;
 
 		tcache->next_gc_bin++;
 		if (tcache->next_gc_bin == nhbins)
@@ -348,8 +346,6 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr)
 	*(void **)ptr = tbin->avail;
 	tbin->avail = ptr;
 	tbin->ncached++;
-	if (tbin->ncached > tbin->high_water)
-		tbin->high_water = tbin->ncached;
 
 	tcache_event(tcache);
 }
@@ -388,8 +384,6 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 	*(void **)ptr = tbin->avail;
 	tbin->avail = ptr;
 	tbin->ncached++;
-	if (tbin->ncached > tbin->high_water)
-		tbin->high_water = tbin->ncached;
 
 	tcache_event(tcache);
 }

jemalloc/include/jemalloc/jemalloc.h.in

@@ -19,7 +19,7 @@ extern "C" {
 #  define JEMALLOC_P(s) s
 #endif
 
-#define	ALLOCM_LG_ALIGN	((int)0x3f)
+#define	ALLOCM_LG_ALIGN(la)	(la)
 #if LG_SIZEOF_PTR == 2
 #define	ALLOCM_ALIGN(a)	(ffs(a)-1)
 #else

jemalloc/src/arena.c

@@ -1358,8 +1358,6 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind
 #endif
 	malloc_mutex_unlock(&bin->lock);
 	tbin->ncached = i;
-	if (tbin->ncached > tbin->high_water)
-		tbin->high_water = tbin->ncached;
 }
 #endif
 
@@ -1369,7 +1367,6 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind
  *   *) bin->run_size >= min_run_size
  *   *) bin->run_size <= arena_maxclass
  *   *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
- *   *) run header size < PAGE_SIZE
  *
  * bin->nregs and bin->reg0_offset are also calculated here, since these
  * settings are all interdependent.
@@ -1455,8 +1452,7 @@ arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
 	} while (try_run_size <= arena_maxclass
 	    && try_run_size <= arena_maxclass
 	    && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
-	    && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size
+	    && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
-	    && try_hdr_size < PAGE_SIZE);
 
 	assert(good_hdr_size <= good_reg0_offset);
 

jemalloc/src/chunk_mmap.c

@@ -206,13 +206,15 @@ chunk_alloc_mmap_internal(size_t size, bool noreserve)
 void *
 chunk_alloc_mmap(size_t size)
 {
-	return chunk_alloc_mmap_internal(size, false);
+
+	return (chunk_alloc_mmap_internal(size, false));
 }
 
 void *
 chunk_alloc_mmap_noreserve(size_t size)
 {
-	return chunk_alloc_mmap_internal(size, true);
+
+	return (chunk_alloc_mmap_internal(size, true));
 }
 
 void

jemalloc/src/ckh.c

@@ -73,7 +73,7 @@ ckh_isearch(ckh_t *ckh, const void *key)
 	size_t hash1, hash2, bucket, cell;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
 
@@ -383,7 +383,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp)
 	}
 
 #ifdef JEMALLOC_DEBUG
-	ckh->magic = CKH_MAGIG;
+	ckh->magic = CKH_MAGIC;
 #endif
 
 	ret = false;
@@ -396,7 +396,7 @@ ckh_delete(ckh_t *ckh)
 {
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 #ifdef CKH_VERBOSE
 	malloc_printf(
@@ -421,7 +421,7 @@ ckh_count(ckh_t *ckh)
 {
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	return (ckh->count);
 }
@@ -452,7 +452,7 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data)
 	bool ret;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 	assert(ckh_search(ckh, key, NULL, NULL));
 
 #ifdef CKH_COUNT
@@ -477,7 +477,7 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data)
 	size_t cell;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	cell = ckh_isearch(ckh, searchkey);
 	if (cell != SIZE_T_MAX) {
@@ -509,7 +509,7 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data)
 	size_t cell;
 
 	assert(ckh != NULL);
-	assert(ckh->magic = CKH_MAGIG);
+	assert(ckh->magic == CKH_MAGIC);
 
 	cell = ckh_isearch(ckh, searchkey);
 	if (cell != SIZE_T_MAX) {

jemalloc/src/ctl.c

@@ -1137,6 +1137,11 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 
 		/* Set new arena association. */
 		ARENA_SET(arena);
+		{
+			tcache_t *tcache = TCACHE_GET();
+			if (tcache != NULL)
+				tcache->arena = arena;
+		}
 	}
 
 	ret = 0;
@@ -1146,9 +1151,9 @@ RETURN:
 
 #ifdef JEMALLOC_STATS
 CTL_RO_NL_GEN(thread_allocated, ALLOCATED_GET(), uint64_t);
-CTL_RO_NL_GEN(thread_allocatedp, &ALLOCATED_GET(), uint64_t *);
+CTL_RO_NL_GEN(thread_allocatedp, ALLOCATEDP_GET(), uint64_t *);
 CTL_RO_NL_GEN(thread_deallocated, DEALLOCATED_GET(), uint64_t);
-CTL_RO_NL_GEN(thread_deallocatedp, &DEALLOCATED_GET(), uint64_t *);
+CTL_RO_NL_GEN(thread_deallocatedp, DEALLOCATEDP_GET(), uint64_t *);
 #endif
 
 /******************************************************************************/

jemalloc/src/huge.c

@@ -83,7 +83,7 @@ huge_palloc(size_t size, size_t alignment, bool zero)
 	 * alignment, in order to assure the alignment can be achieved, then
 	 * unmap leading and trailing chunks.
 	 */
-	assert(alignment >= chunksize);
+	assert(alignment > chunksize);
 
 	chunk_size = CHUNK_CEILING(size);
 
@@ -192,7 +192,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 	 * different size class.  In that case, fall back to allocating new
 	 * space and copying.
 	 */
-	if (alignment != 0)
+	if (alignment > chunksize)
 		ret = huge_palloc(size + extra, alignment, zero);
 	else
 		ret = huge_malloc(size + extra, zero);
@@ -201,7 +201,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
 		if (extra == 0)
 			return (NULL);
 		/* Try again, this time without extra. */
-		if (alignment != 0)
+		if (alignment > chunksize)
 			ret = huge_palloc(size, alignment, zero);
 		else
 			ret = huge_malloc(size, zero);

jemalloc/src/jemalloc.c

@@ -213,6 +213,28 @@ stats_print_atexit(void)
 	JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL);
 }
 
+#if (defined(JEMALLOC_STATS) && defined(NO_TLS))
+thread_allocated_t *
+thread_allocated_get_hard(void)
+{
+	thread_allocated_t *thread_allocated = (thread_allocated_t *)
+	    imalloc(sizeof(thread_allocated_t));
+	if (thread_allocated == NULL) {
+		static thread_allocated_t static_thread_allocated = {0, 0};
+		malloc_write("<jemalloc>: Error allocating TSD;"
+		    " mallctl(\"thread.{de,}allocated[p]\", ...)"
+		    " will be inaccurate\n");
+		if (opt_abort)
+			abort();
+		return (&static_thread_allocated);
+	}
+	pthread_setspecific(thread_allocated_tsd, thread_allocated);
+	thread_allocated->allocated = 0;
+	thread_allocated->deallocated = 0;
+	return (thread_allocated);
+}
+#endif
+
 /*
  * End miscellaneous support functions.
  */
@@ -421,8 +443,8 @@ malloc_conf_init(void)
 			if ((opts = getenv(envname)) != NULL) {
 				/*
 				 * Do nothing; opts is already initialized to
-				 * the value of the JEMALLOC_OPTIONS
+				 * the value of the MALLOC_CONF environment
-				 * environment variable.
+				 * variable.
 				 */
 			} else {
 				/* No configuration specified. */

jemalloc/src/prof.c

@@ -432,6 +432,7 @@ prof_lookup(prof_bt_t *bt)
 			prof_ctx_t	*p;
 			void		*v;
 		} ctx;
+		bool new_ctx;
 
 		/*
 		 * This thread's cache lacks bt.  Look for it in the global
@@ -468,12 +469,14 @@ prof_lookup(prof_bt_t *bt)
 				idalloc(ctx.v);
 				return (NULL);
 			}
-		}
+			/*
-		/*
+			 * Artificially raise curobjs, in order to avoid a race
-		 * Acquire ctx's lock before releasing bt2ctx_mtx, in order to
+			 * condition with prof_ctx_merge()/prof_ctx_destroy().
-		 * avoid a race condition with prof_ctx_destroy().
+			 */
-		 */
+			ctx.p->cnt_merged.curobjs++;
-		malloc_mutex_lock(&ctx.p->lock);
+			new_ctx = true;
+		} else
+			new_ctx = false;
 		prof_leave();
 
 		/* Link a prof_thd_cnt_t into ctx for this thread. */
@@ -498,7 +501,11 @@ prof_lookup(prof_bt_t *bt)
 			/* Allocate and partially initialize a new cnt. */
 			ret.v = imalloc(sizeof(prof_thr_cnt_t));
 			if (ret.p == NULL) {
-				malloc_mutex_unlock(&ctx.p->lock);
+				if (new_ctx) {
+					malloc_mutex_lock(&ctx.p->lock);
+					ctx.p->cnt_merged.curobjs--;
+					malloc_mutex_unlock(&ctx.p->lock);
+				}
 				return (NULL);
 			}
 			ql_elm_new(ret.p, cnts_link);
@@ -509,12 +516,19 @@ prof_lookup(prof_bt_t *bt)
 		ret.p->epoch = 0;
 		memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
 		if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) {
-			malloc_mutex_unlock(&ctx.p->lock);
+			if (new_ctx) {
+				malloc_mutex_lock(&ctx.p->lock);
+				ctx.p->cnt_merged.curobjs--;
+				malloc_mutex_unlock(&ctx.p->lock);
+			}
 			idalloc(ret.v);
 			return (NULL);
 		}
 		ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link);
+		malloc_mutex_lock(&ctx.p->lock);
 		ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link);
+		if (new_ctx)
+			ctx.p->cnt_merged.curobjs--;
 		malloc_mutex_unlock(&ctx.p->lock);
 	} else {
 		/* Move ret to the front of the LRU. */

jemalloc/test/allocated.c

@@ -16,6 +16,7 @@ thread_start(void *arg)
 	int err;
 	void *p;
 	uint64_t a0, a1, d0, d1;
+	uint64_t *ap0, *ap1, *dp0, *dp1;
 	size_t sz, usize;
 
 	sz = sizeof(a0);
@@ -31,6 +32,20 @@ thread_start(void *arg)
 		    strerror(err));
 		exit(1);
 	}
+	sz = sizeof(ap0);
+	if ((err = JEMALLOC_P(mallctl)("thread.allocatedp", &ap0, &sz, NULL,
+	    0))) {
+		if (err == ENOENT) {
+#ifdef JEMALLOC_STATS
+			assert(false);
+#endif
+			goto RETURN;
+		}
+		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
+		    strerror(err));
+		exit(1);
+	}
+	assert(*ap0 == a0);
 
 	sz = sizeof(d0);
 	if ((err = JEMALLOC_P(mallctl)("thread.deallocated", &d0, &sz, NULL,
@@ -45,6 +60,20 @@ thread_start(void *arg)
 		    strerror(err));
 		exit(1);
 	}
+	sz = sizeof(dp0);
+	if ((err = JEMALLOC_P(mallctl)("thread.deallocatedp", &dp0, &sz, NULL,
+	    0))) {
+		if (err == ENOENT) {
+#ifdef JEMALLOC_STATS
+			assert(false);
+#endif
+			goto RETURN;
+		}
+		fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__,
+		    strerror(err));
+		exit(1);
+	}
+	assert(*dp0 == d0);
 
 	p = JEMALLOC_P(malloc)(1);
 	if (p == NULL) {
@@ -54,6 +83,10 @@ thread_start(void *arg)
 
 	sz = sizeof(a1);
 	JEMALLOC_P(mallctl)("thread.allocated", &a1, &sz, NULL, 0);
+	sz = sizeof(ap1);
+	JEMALLOC_P(mallctl)("thread.allocatedp", &ap1, &sz, NULL, 0);
+	assert(*ap1 == a1);
+	assert(ap0 == ap1);
 
 	usize = JEMALLOC_P(malloc_usable_size)(p);
 	assert(a0 + usize <= a1);
@@ -62,6 +95,10 @@ thread_start(void *arg)
 
 	sz = sizeof(d1);
 	JEMALLOC_P(mallctl)("thread.deallocated", &d1, &sz, NULL, 0);
+	sz = sizeof(dp1);
+	JEMALLOC_P(mallctl)("thread.deallocatedp", &dp1, &sz, NULL, 0);
+	assert(*dp1 == d1);
+	assert(dp0 == dp1);
 
 	assert(d0 + usize <= d1);