Merge branch 'dev'

Fix a compiler warning in chunk_record() that was due to reading node
rather than xnode.  In practice this did not cause any correctness
issue, but dataflow analysis in some compilers cannot tell that node and
xnode are always equal in cases that the read is reached.

ChangeLog

@@ -6,6 +6,34 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
+* 3.4.1 (October 20, 2013)
+
+  Bug fixes:
+  - Fix a race in the "arenas.extend" mallctl that could cause memory corruption
+    of internal data structures and subsequent crashes.
+  - Fix Valgrind integration flaws that caused Valgrind warnings about reads of
+    uninitialized memory in:
+    + arena chunk headers
+    + internal zero-initialized data structures (relevant to tcache and prof
+      code)
+  - Preserve errno during the first allocation.  A readlink(2) call during
+    initialization fails unless /etc/malloc.conf exists, so errno was typically
+    set during the first allocation prior to this fix.
+  - Fix compilation warnings reported by gcc 4.8.1.
+
+* 3.4.0 (June 2, 2013)
+
+  This version is essentially a small bugfix release, but the addition of
+  aarch64 support requires that the minor version be incremented.
+
+  Bug fixes:
+  - Fix race-triggered deadlocks in chunk_record().  These deadlocks were
+    typically triggered by multiple threads concurrently deallocating huge
+    objects.
+
+  New features:
+  - Add support for the aarch64 architecture.
+
 * 3.3.1 (March 6, 2013)
 
   This version fixes bugs that are typically encountered only when utilizing
@@ -15,7 +43,7 @@ found in the git revision history:
   - Fix a locking order bug that could cause deadlock during fork if heap
     profiling were enabled.
   - Fix a chunk recycling bug that could cause the allocator to lose track of
-    whether a chunk was zeroed.   On FreeBSD, NetBSD, and OS X, it could cause
+    whether a chunk was zeroed.  On FreeBSD, NetBSD, and OS X, it could cause
     corruption if allocating via sbrk(2) (unlikely unless running with the
     "dss:primary" option specified).  This was completely harmless on Linux
     unless using mlockall(2) (and unlikely even then, unless the
@@ -47,7 +75,7 @@ found in the git revision history:
 
   Bug fixes:
   - Fix "arenas.extend" mallctl to output the number of arenas.
-  - Fix chunk_recycyle() to unconditionally inform Valgrind that returned memory
+  - Fix chunk_recycle() to unconditionally inform Valgrind that returned memory
     is undefined.
   - Fix build break on FreeBSD related to alloca.h.
 

Makefile.in

@@ -55,6 +55,8 @@ PIC_CFLAGS = @PIC_CFLAGS@
 CTARGET = @CTARGET@
 LDTARGET = @LDTARGET@
 MKLIB = @MKLIB@
+AR = @AR@
+ARFLAGS = @ARFLAGS@
 CC_MM = @CC_MM@
 
 ifeq (macho, $(ABI))
@@ -185,7 +187,7 @@ $(objroot)lib/$(LIBJEMALLOC)_s.$(A) : $(COBJS)
 
 $(STATIC_LIBS):
 	@mkdir -p $(@D)
-	$(MKLIB) $+
+	$(AR) $(ARFLAGS)@AROUT@ $+
 
 $(objroot)test/bitmap$(EXE): $(objroot)src/bitmap.$(O)
 

README

@@ -1,10 +1,14 @@
-jemalloc is a general-purpose scalable concurrent malloc(3) implementation.
+jemalloc is a general purpose malloc(3) implementation that emphasizes
-This distribution is a "portable" implementation that currently targets
+fragmentation avoidance and scalable concurrency support.  jemalloc first came
-FreeBSD, Linux, Apple OS X, and MinGW.  jemalloc is included as the default
+into use as the FreeBSD libc allocator in 2005, and since then it has found its
-allocator in the FreeBSD and NetBSD operating systems, and it is used by the
+way into numerous applications that rely on its predictable behavior.  In 2010
-Mozilla Firefox web browser on Microsoft Windows-related platforms.  Depending
+jemalloc development efforts broadened to include developer support features
-on your needs, one of the other divergent versions may suit your needs better
+such as heap profiling, Valgrind integration, and extensive monitoring/tuning
-than this distribution.
+hooks.  Modern jemalloc releases continue to be integrated back into FreeBSD,
+and therefore versatility remains critical.  Ongoing development efforts trend
+toward making jemalloc among the best allocators for a broad range of demanding
+applications, and eliminating/mitigating weaknesses that have practical
+repercussions for real world applications.
 
 The COPYING file contains copyright and licensing information.
 

configure.ac

@@ -226,9 +226,15 @@ PIC_CFLAGS='-fPIC -DPIC'
 CTARGET='-o $@'
 LDTARGET='-o $@'
 EXTRA_LDFLAGS=
-MKLIB='ar crus $@'
+ARFLAGS='crus'
+AROUT=' $@'
 CC_MM=1
 
+AN_MAKEVAR([AR], [AC_PROG_AR])
+AN_PROGRAM([ar], [AC_PROG_AR])
+AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL(AR, ar, :)])
+AC_PROG_AR
+
 dnl Platform-specific settings.  abi and RPATH can probably be determined
 dnl programmatically, but doing so is error-prone, which makes it generally
 dnl not worth the trouble.
@@ -310,7 +316,9 @@ case "${host}" in
 	  EXTRA_LDFLAGS="-link -DEBUG"
 	  CTARGET='-Fo$@'
 	  LDTARGET='-Fe$@'
-	  MKLIB='lib -nologo -out:$@'
+	  AR='lib'
+	  ARFLAGS='-nologo -out:'
+	  AROUT='$@'
 	  CC_MM=
         else
 	  importlib="${so}"
@@ -343,6 +351,8 @@ AC_SUBST([PIC_CFLAGS])
 AC_SUBST([CTARGET])
 AC_SUBST([LDTARGET])
 AC_SUBST([MKLIB])
+AC_SUBST([ARFLAGS])
+AC_SUBST([AROUT])
 AC_SUBST([CC_MM])
 
 if test "x$abi" != "xpecoff"; then
@@ -403,7 +413,6 @@ AC_SUBST([enable_autogen])
 
 AC_PROG_INSTALL
 AC_PROG_RANLIB
-AC_PATH_PROG([AR], [ar], [false], [$PATH])
 AC_PATH_PROG([LD], [ld], [false], [$PATH])
 AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH])
 

doc/jemalloc.xml.in

@@ -432,7 +432,14 @@ for (i = 0; i < nbins; i++) {
     referenced by the symbolic link named <filename
     class="symlink">/etc/malloc.conf</filename>, and the value of the
     environment variable <envar>MALLOC_CONF</envar>, will be interpreted, in
-    that order, from left to right as options.</para>
+    that order, from left to right as options.  Note that
+    <varname>malloc_conf</varname> may be read before
+    <function>main<parameter/></function> is entered, so the declaration of
+    <varname>malloc_conf</varname> should specify an initializer that contains
+    the final value to be read by jemalloc.  <varname>malloc_conf</varname> is
+    a compile-time setting, whereas <filename
+    class="symlink">/etc/malloc.conf</filename> and <envar>MALLOC_CONF</envar>
+    can be safely set any time prior to program invocation.</para>
 
     <para>An options string is a comma-separated list of option:value pairs.
     There is one key corresponding to each <link

include/jemalloc/internal/arena.h

@@ -441,6 +441,7 @@ void	arena_postfork_child(arena_t *arena);
 #ifndef JEMALLOC_ENABLE_INLINE
 arena_chunk_map_t	*arena_mapp_get(arena_chunk_t *chunk, size_t pageind);
 size_t	*arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind);
+size_t	arena_mapbitsp_read(size_t *mapbitsp);
 size_t	arena_mapbits_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_unallocated_size_get(arena_chunk_t *chunk,
     size_t pageind);
@@ -451,6 +452,7 @@ size_t	arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind);
 size_t	arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind);
+void	arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits);
 void	arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind,
     size_t size, size_t flags);
 void	arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
@@ -497,11 +499,18 @@ arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind)
 	return (&arena_mapp_get(chunk, pageind)->bits);
 }
 
+JEMALLOC_ALWAYS_INLINE size_t
+arena_mapbitsp_read(size_t *mapbitsp)
+{
+
+	return (*mapbitsp);
+}
+
 JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_get(arena_chunk_t *chunk, size_t pageind)
 {
 
-	return (*arena_mapbitsp_get(chunk, pageind));
+	return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind)));
 }
 
 JEMALLOC_ALWAYS_INLINE size_t
@@ -584,83 +593,90 @@ arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits & CHUNK_MAP_ALLOCATED);
 }
 
+JEMALLOC_ALWAYS_INLINE void
+arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits)
+{
+
+	*mapbitsp = mapbits;
+}
+
 JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
     size_t flags)
 {
-	size_t *mapbitsp;
+	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);
 
-	mapbitsp = arena_mapbitsp_get(chunk, pageind);
 	assert((size & PAGE_MASK) == 0);
 	assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0);
 	assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags);
-	*mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags;
+	arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
     size_t size)
 {
-	size_t *mapbitsp;
+	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);
+	size_t mapbits = arena_mapbitsp_read(mapbitsp);
 
-	mapbitsp = arena_mapbitsp_get(chunk, pageind);
 	assert((size & PAGE_MASK) == 0);
-	assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
+	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
-	*mapbitsp = size | (*mapbitsp & PAGE_MASK);
+	arena_mapbitsp_write(mapbitsp, size | (mapbits & PAGE_MASK));
 }
 
 JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
     size_t flags)
 {
-	size_t *mapbitsp;
+	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);
+	size_t mapbits = arena_mapbitsp_read(mapbitsp);
 	size_t unzeroed;
 
-	mapbitsp = arena_mapbitsp_get(chunk, pageind);
 	assert((size & PAGE_MASK) == 0);
 	assert((flags & CHUNK_MAP_DIRTY) == flags);
-	unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
+	unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
-	*mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | unzeroed |
+	arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags
-	    CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
+	    | unzeroed | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
     size_t binind)
 {
-	size_t *mapbitsp;
+	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);
+	size_t mapbits = arena_mapbitsp_read(mapbitsp);
 
 	assert(binind <= BININD_INVALID);
-	mapbitsp = arena_mapbitsp_get(chunk, pageind);
 	assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE);
-	*mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind <<
+	arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) |
-	    CHUNK_MAP_BININD_SHIFT);
+	    (binind << CHUNK_MAP_BININD_SHIFT));
 }
 
 JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
     size_t binind, size_t flags)
 {
-	size_t *mapbitsp;
+	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);
+	size_t mapbits = arena_mapbitsp_read(mapbitsp);
 	size_t unzeroed;
 
 	assert(binind < BININD_INVALID);
-	mapbitsp = arena_mapbitsp_get(chunk, pageind);
 	assert(pageind - runind >= map_bias);
 	assert((flags & CHUNK_MAP_DIRTY) == flags);
-	unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
+	unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */
-	*mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) |
+	arena_mapbitsp_write(mapbitsp, (runind << LG_PAGE) | (binind <<
-	    flags | unzeroed | CHUNK_MAP_ALLOCATED;
+	    CHUNK_MAP_BININD_SHIFT) | flags | unzeroed | CHUNK_MAP_ALLOCATED);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
     size_t unzeroed)
 {
-	size_t *mapbitsp;
+	size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind);
+	size_t mapbits = arena_mapbitsp_read(mapbitsp);
 
-	mapbitsp = arena_mapbitsp_get(chunk, pageind);
+	arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_UNZEROED) |
-	*mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed;
+	    unzeroed);
 }
 
 JEMALLOC_INLINE bool

include/jemalloc/internal/jemalloc_internal.h.in

@@ -232,9 +232,18 @@ static const bool config_ivsalloc =
 #  define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
 #endif
 
+/*
+ * JEMALLOC_ALWAYS_INLINE is used within header files for functions that are
+ * static inline functions if inlining is enabled, and single-definition
+ * library-private functions if inlining is disabled.
+ *
+ * JEMALLOC_ALWAYS_INLINE_C is for use in .c files, in which case the denoted
+ * functions are always static, regardless of whether inlining is enabled.
+ */
 #ifdef JEMALLOC_DEBUG
    /* Disable inlining to make debugging easier. */
 #  define JEMALLOC_ALWAYS_INLINE
+#  define JEMALLOC_ALWAYS_INLINE_C static
 #  define JEMALLOC_INLINE
 #  define inline
 #else
@@ -242,8 +251,11 @@ static const bool config_ivsalloc =
 #  ifdef JEMALLOC_HAVE_ATTR
 #    define JEMALLOC_ALWAYS_INLINE \
 	 static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline)
+#    define JEMALLOC_ALWAYS_INLINE_C \
+	 static inline JEMALLOC_ATTR(always_inline)
 #  else
 #    define JEMALLOC_ALWAYS_INLINE static inline
+#    define JEMALLOC_ALWAYS_INLINE_C static inline
 #  endif
 #  define JEMALLOC_INLINE static inline
 #  ifdef _MSC_VER
@@ -278,6 +290,9 @@ static const bool config_ivsalloc =
 #  ifdef __arm__
 #    define LG_QUANTUM		3
 #  endif
+#  ifdef __aarch64__
+#    define LG_QUANTUM		4
+#  endif
 #  ifdef __hppa__
 #    define LG_QUANTUM		4
 #  endif

include/jemalloc/internal/private_namespace.h

@@ -33,6 +33,8 @@
 #define	arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get)
 #define	arena_mapbits_unzeroed_set JEMALLOC_N(arena_mapbits_unzeroed_set)
 #define	arena_mapbitsp_get JEMALLOC_N(arena_mapbitsp_get)
+#define	arena_mapbitsp_read JEMALLOC_N(arena_mapbitsp_read)
+#define	arena_mapbitsp_write JEMALLOC_N(arena_mapbitsp_write)
 #define	arena_mapp_get JEMALLOC_N(arena_mapp_get)
 #define	arena_maxclass JEMALLOC_N(arena_maxclass)
 #define	arena_new JEMALLOC_N(arena_new)

include/jemalloc/internal/tcache.h

@@ -313,6 +313,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 			} else if (opt_zero)
 				memset(ret, 0, size);
 		}
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	} else {
 		if (config_fill && opt_junk) {
 			arena_alloc_junk_small(ret, &arena_bin_info[binind],
@@ -321,7 +322,6 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
 	}
-	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 	if (config_stats)
 		tbin->tstats.nrequests++;
@@ -368,11 +368,11 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 				else if (opt_zero)
 					memset(ret, 0, size);
 			}
+			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		} else {
 			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 			memset(ret, 0, size);
 		}
-		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 		if (config_stats)
 			tbin->tstats.nrequests++;

src/arena.c

@@ -368,14 +368,21 @@ arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)
 	    (npages << LG_PAGE));
 }
 
+static inline void
+arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind)
+{
+
+	VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind <<
+	    LG_PAGE)), PAGE);
+}
+
 static inline void
 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
 {
 	size_t i;
 	UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));
 
-	VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind <<
+	arena_run_page_mark_zeroed(chunk, run_ind);
-	    LG_PAGE)), PAGE);
 	for (i = 0; i < PAGE / sizeof(size_t); i++)
 		assert(p[i] == 0);
 }
@@ -458,6 +465,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 					} else if (config_debug) {
 						arena_run_page_validate_zeroed(
 						    chunk, run_ind+i);
+					} else {
+						arena_run_page_mark_zeroed(
+						    chunk, run_ind+i);
 					}
 				}
 			} else {
@@ -467,6 +477,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 				 */
 				arena_run_zero(chunk, run_ind, need_pages);
 			}
+		} else {
+			VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
+			    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
 		}
 
 		/*
@@ -508,9 +521,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 			arena_run_page_validate_zeroed(chunk,
 			    run_ind+need_pages-1);
 		}
+		VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
+		    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
 	}
-	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
-	    LG_PAGE)), (need_pages << LG_PAGE));
 }
 
 static arena_chunk_t *
@@ -569,17 +582,24 @@ arena_chunk_alloc(arena_t *arena)
 		 * unless the chunk is not zeroed.
 		 */
 		if (zero == false) {
+			VALGRIND_MAKE_MEM_UNDEFINED(
+			    (void *)arena_mapp_get(chunk, map_bias+1),
+			    (size_t)((uintptr_t) arena_mapp_get(chunk,
+			    chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk,
+			    map_bias+1)));
 			for (i = map_bias+1; i < chunk_npages-1; i++)
 				arena_mapbits_unzeroed_set(chunk, i, unzeroed);
-		} else if (config_debug) {
+		} else {
 			VALGRIND_MAKE_MEM_DEFINED(
 			    (void *)arena_mapp_get(chunk, map_bias+1),
-			    (void *)((uintptr_t)
+			    (size_t)((uintptr_t) arena_mapp_get(chunk,
-			    arena_mapp_get(chunk, chunk_npages-1)
+			    chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk,
-			    - (uintptr_t)arena_mapp_get(chunk, map_bias+1)));
+			    map_bias+1)));
-			for (i = map_bias+1; i < chunk_npages-1; i++) {
+			if (config_debug) {
-				assert(arena_mapbits_unzeroed_get(chunk, i) ==
+				for (i = map_bias+1; i < chunk_npages-1; i++) {
-				    unzeroed);
+					assert(arena_mapbits_unzeroed_get(chunk,
+					    i) == unzeroed);
+				}
 			}
 		}
 		arena_mapbits_unallocated_set(chunk, chunk_npages-1,
@@ -1458,6 +1478,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 			} else if (opt_zero)
 				memset(ret, 0, size);
 		}
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	} else {
 		if (config_fill && opt_junk) {
 			arena_alloc_junk_small(ret, &arena_bin_info[binind],
@@ -1466,7 +1487,6 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
 	}
-	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 	return (ret);
 }

src/chunk.c

@@ -214,7 +214,7 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
     size_t size)
 {
 	bool unzeroed;
-	extent_node_t *xnode, *node, *prev, key;
+	extent_node_t *xnode, *node, *prev, *xprev, key;
 
 	unzeroed = pages_purge(chunk, size);
 	VALGRIND_MAKE_MEM_NOACCESS(chunk, size);
@@ -226,6 +226,8 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 	 * held.
 	 */
 	xnode = base_node_alloc();
+	/* Use xprev to implement conditional deferred deallocation of prev. */
+	xprev = NULL;
 
 	malloc_mutex_lock(&chunks_mtx);
 	key.addr = (void *)((uintptr_t)chunk + size);
@@ -242,8 +244,6 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 		node->size += size;
 		node->zeroed = (node->zeroed && (unzeroed == false));
 		extent_tree_szad_insert(chunks_szad, node);
-		if (xnode != NULL)
-			base_node_dealloc(xnode);
 	} else {
 		/* Coalescing forward failed, so insert a new node. */
 		if (xnode == NULL) {
@@ -253,10 +253,10 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 			 * already been purged, so this is only a virtual
 			 * memory leak.
 			 */
-			malloc_mutex_unlock(&chunks_mtx);
+			goto label_return;
-			return;
 		}
 		node = xnode;
+		xnode = NULL; /* Prevent deallocation below. */
 		node->addr = chunk;
 		node->size = size;
 		node->zeroed = (unzeroed == false);
@@ -282,9 +282,19 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 		node->zeroed = (node->zeroed && prev->zeroed);
 		extent_tree_szad_insert(chunks_szad, node);
 
-		base_node_dealloc(prev);
+		xprev = prev;
 	}
+
+label_return:
 	malloc_mutex_unlock(&chunks_mtx);
+	/*
+	 * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to
+	 * avoid potential deadlock.
+	 */
+	if (xnode != NULL)
+		base_node_dealloc(xnode);
+	if (xprev != NULL)
+		base_node_dealloc(xprev);
 }
 
 void

src/ctl.c

@@ -546,43 +546,30 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
 static bool
 ctl_grow(void)
 {
-	size_t astats_size;
 	ctl_arena_stats_t *astats;
 	arena_t **tarenas;
 
-	/* Extend arena stats and arenas arrays. */
+	/* Allocate extended arena stats and arenas arrays. */
-	astats_size = (ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t);
+	astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) *
-	if (ctl_stats.narenas == narenas_auto) {
+	    sizeof(ctl_arena_stats_t));
-		/* ctl_stats.arenas and arenas came from base_alloc(). */
+	if (astats == NULL)
-		astats = (ctl_arena_stats_t *)imalloc(astats_size);
-		if (astats == NULL)
-			return (true);
-		memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
-		    sizeof(ctl_arena_stats_t));
-
-		tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) *
-		    sizeof(arena_t *));
-		if (tarenas == NULL) {
-			idalloc(astats);
-			return (true);
-		}
-		memcpy(tarenas, arenas, ctl_stats.narenas * sizeof(arena_t *));
-	} else {
-		astats = (ctl_arena_stats_t *)iralloc(ctl_stats.arenas,
-		    astats_size, 0, 0, false, false);
-		if (astats == NULL)
-			return (true);
-
-		tarenas = (arena_t **)iralloc(arenas, (ctl_stats.narenas + 1) *
-		    sizeof(arena_t *), 0, 0, false, false);
-		if (tarenas == NULL)
-			return (true);
-	}
-	/* Initialize the new astats and arenas elements. */
-	memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
-	if (ctl_arena_init(&astats[ctl_stats.narenas + 1]))
 		return (true);
-	tarenas[ctl_stats.narenas] = NULL;
+	tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) *
+	    sizeof(arena_t *));
+	if (tarenas == NULL) {
+		idalloc(astats);
+		return (true);
+	}
+
+	/* Initialize the new astats element. */
+	memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
+	    sizeof(ctl_arena_stats_t));
+	memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
+	if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) {
+		idalloc(tarenas);
+		idalloc(astats);
+		return (true);
+	}
 	/* Swap merged stats to their new location. */
 	{
 		ctl_arena_stats_t tstats;
@@ -593,13 +580,34 @@ ctl_grow(void)
 		memcpy(&astats[ctl_stats.narenas + 1], &tstats,
 		    sizeof(ctl_arena_stats_t));
 	}
+	/* Initialize the new arenas element. */
+	tarenas[ctl_stats.narenas] = NULL;
+	{
+		arena_t **arenas_old = arenas;
+		/*
+		 * Swap extended arenas array into place.  Although ctl_mtx
+		 * protects this function from other threads extending the
+		 * array, it does not protect from other threads mutating it
+		 * (i.e. initializing arenas and setting array elements to
+		 * point to them).  Therefore, array copying must happen under
+		 * the protection of arenas_lock.
+		 */
+		malloc_mutex_lock(&arenas_lock);
+		arenas = tarenas;
+		memcpy(arenas, arenas_old, ctl_stats.narenas *
+		    sizeof(arena_t *));
+		narenas_total++;
+		arenas_extend(narenas_total - 1);
+		malloc_mutex_unlock(&arenas_lock);
+		/*
+		 * Deallocate arenas_old only if it came from imalloc() (not
+		 * base_alloc()).
+		 */
+		if (ctl_stats.narenas != narenas_auto)
+			idalloc(arenas_old);
+	}
 	ctl_stats.arenas = astats;
 	ctl_stats.narenas++;
-	malloc_mutex_lock(&arenas_lock);
-	arenas = tarenas;
-	narenas_total++;
-	arenas_extend(narenas_total - 1);
-	malloc_mutex_unlock(&arenas_lock);
 
 	return (false);
 }
@@ -1109,7 +1117,7 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
     void *newp, size_t newlen)
 {
 	int ret;
-	uint64_t newval;
+	UNUSED uint64_t newval;
 
 	malloc_mutex_lock(&ctl_mtx);
 	WRITE(newval, uint64_t);

src/jemalloc.c

@@ -282,7 +282,7 @@ arenas_cleanup(void *arg)
 	malloc_mutex_unlock(&arenas_lock);
 }
 
-static JEMALLOC_ATTR(always_inline) void
+JEMALLOC_ALWAYS_INLINE_C void
 malloc_thread_init(void)
 {
 
@@ -299,7 +299,7 @@ malloc_thread_init(void)
 		quarantine_alloc_hook();
 }
 
-static JEMALLOC_ATTR(always_inline) bool
+JEMALLOC_ALWAYS_INLINE_C bool
 malloc_init(void)
 {
 
@@ -436,8 +436,9 @@ malloc_conf_init(void)
 			}
 			break;
 		case 1: {
+			int linklen = 0;
 #ifndef _WIN32
-			int linklen;
+			int saved_errno = errno;
 			const char *linkname =
 #  ifdef JEMALLOC_PREFIX
 			    "/etc/"JEMALLOC_PREFIX"malloc.conf"
@@ -446,21 +447,20 @@ malloc_conf_init(void)
 #  endif
 			    ;
 
-			if ((linklen = readlink(linkname, buf,
+			/*
-			    sizeof(buf) - 1)) != -1) {
+			 * Try to use the contents of the "/etc/malloc.conf"
-				/*
+			 * symbolic link's name.
-				 * Use the contents of the "/etc/malloc.conf"
+			 */
-				 * symbolic link's name.
+			linklen = readlink(linkname, buf, sizeof(buf) - 1);
-				 */
+			if (linklen == -1) {
-				buf[linklen] = '\0';
-				opts = buf;
-			} else
-#endif
-			{
 				/* No configuration specified. */
-				buf[0] = '\0';
+				linklen = 0;
-				opts = buf;
+				/* restore errno */
+				set_errno(saved_errno);
 			}
+#endif
+			buf[linklen] = '\0';
+			opts = buf;
 			break;
 		} case 2: {
 			const char *envname =
@@ -1402,7 +1402,7 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
  */
 #ifdef JEMALLOC_EXPERIMENTAL
 
-static JEMALLOC_ATTR(always_inline) void *
+JEMALLOC_ALWAYS_INLINE_C void *
 iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache,
     arena_t *arena)
 {

test/ALLOCM_ARENA.c

@@ -23,7 +23,8 @@ je_thread_start(void *arg)
 		size_t mib[3];
 		size_t miblen = sizeof(mib) / sizeof(size_t);
 		const char *dss_precs[] = {"disabled", "primary", "secondary"};
-		const char *dss = dss_precs[thread_ind % 4];
+		const char *dss = dss_precs[thread_ind %
+		    (sizeof(dss_precs)/sizeof(char*))];
 		if (mallctlnametomib("arena.0.dss", mib, &miblen) != 0) {
 			malloc_printf("Error in mallctlnametomib()\n");
 			abort();

test/allocated.c

@@ -104,12 +104,12 @@ main(void)
 	je_thread_start(NULL);
 
 	je_thread_create(&thread, je_thread_start, NULL);
-	je_thread_join(thread, (void *)&ret);
+	je_thread_join(thread, NULL);
 
 	je_thread_start(NULL);
 
 	je_thread_create(&thread, je_thread_start, NULL);
-	je_thread_join(thread, (void *)&ret);
+	je_thread_join(thread, NULL);
 
 	je_thread_start(NULL);
 

test/thread_arena.c

@@ -72,8 +72,12 @@ main(void)
 		    (void *)&arena_ind);
 	}
 
-	for (i = 0; i < NTHREADS; i++)
+	for (i = 0; i < NTHREADS; i++) {
-		je_thread_join(threads[i], (void *)&ret);
+		intptr_t join_ret;
+		je_thread_join(threads[i], (void *)&join_ret);
+		if (join_ret != 0)
+			ret = 1;
+	}
 
 label_return:
 	malloc_printf("Test end\n");

test/thread_tcache_enabled.c

@@ -77,12 +77,12 @@ main(void)
 	je_thread_start(NULL);
 
 	je_thread_create(&thread, je_thread_start, NULL);
-	je_thread_join(thread, (void *)&ret);
+	je_thread_join(thread, NULL);
 
 	je_thread_start(NULL);
 
 	je_thread_create(&thread, je_thread_start, NULL);
-	je_thread_join(thread, (void *)&ret);
+	je_thread_join(thread, NULL);
 
 	je_thread_start(NULL);