Merge branch 'dev'

Signed-off-by: Riku Voipio <riku.voipio@linaro.org>

COPYING

@@ -1,10 +1,10 @@
 Unless otherwise specified, files in the jemalloc source distribution are
 subject to the following license:
 --------------------------------------------------------------------------------
-Copyright (C) 2002-2012 Jason Evans <jasone@canonware.com>.
+Copyright (C) 2002-2013 Jason Evans <jasone@canonware.com>.
 All rights reserved.
 Copyright (C) 2007-2012 Mozilla Foundation.  All rights reserved.
-Copyright (C) 2009-2012 Facebook, Inc.  All rights reserved.
+Copyright (C) 2009-2013 Facebook, Inc.  All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

ChangeLog

@@ -6,6 +6,79 @@ found in the git revision history:
     http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
     git://canonware.com/jemalloc.git
 
+* 3.4.0 (June 2, 2013)
+
+  This version is essentially a small bugfix release, but the addition of
+  aarch64 support requires that the minor version be incremented.
+
+  Bug fixes:
+  - Fix race-triggered deadlocks in chunk_record().  These deadlocks were
+    typically triggered by multiple threads concurrently deallocating huge
+    objects.
+
+  New features:
+  - Add support for the aarch64 architecture.
+
+* 3.3.1 (March 6, 2013)
+
+  This version fixes bugs that are typically encountered only when utilizing
+  custom run-time options.
+
+  Bug fixes:
+  - Fix a locking order bug that could cause deadlock during fork if heap
+    profiling were enabled.
+  - Fix a chunk recycling bug that could cause the allocator to lose track of
+    whether a chunk was zeroed.  On FreeBSD, NetBSD, and OS X, it could cause
+    corruption if allocating via sbrk(2) (unlikely unless running with the
+    "dss:primary" option specified).  This was completely harmless on Linux
+    unless using mlockall(2) (and unlikely even then, unless the
+    --disable-munmap configure option or the "dss:primary" option was
+    specified).  This regression was introduced in 3.1.0 by the
+    mlockall(2)/madvise(2) interaction fix.
+  - Fix TLS-related memory corruption that could occur during thread exit if the
+    thread never allocated memory.  Only the quarantine and prof facilities were
+    susceptible.
+  - Fix two quarantine bugs:
+    + Internal reallocation of the quarantined object array leaked the old
+      array.
+    + Reallocation failure for internal reallocation of the quarantined object
+      array (very unlikely) resulted in memory corruption.
+  - Fix Valgrind integration to annotate all internally allocated memory in a
+    way that keeps Valgrind happy about internal data structure access.
+  - Fix building for s390 systems.
+
+* 3.3.0 (January 23, 2013)
+
+  This version includes a few minor performance improvements in addition to the
+  listed new features and bug fixes.
+
+  New features:
+  - Add clipping support to lg_chunk option processing.
+  - Add the --enable-ivsalloc option.
+  - Add the --without-export option.
+  - Add the --disable-zone-allocator option.
+
+  Bug fixes:
+  - Fix "arenas.extend" mallctl to output the number of arenas.
+  - Fix chunk_recycyle() to unconditionally inform Valgrind that returned memory
+    is undefined.
+  - Fix build break on FreeBSD related to alloca.h.
+
+* 3.2.0 (November 9, 2012)
+
+  In addition to a couple of bug fixes, this version modifies page run
+  allocation and dirty page purging algorithms in order to better control
+  page-level virtual memory fragmentation.
+
+  Incompatible changes:
+  - Change the "opt.lg_dirty_mult" default from 5 to 3 (32:1 to 8:1).
+
+  Bug fixes:
+  - Fix dss/mmap allocation precedence code to use recyclable mmap memory only
+    after primary dss allocation fails.
+  - Fix deadlock in the "arenas.purge" mallctl.  This regression was introduced
+    in 3.1.0 by the addition of the "arena.<i>.purge" mallctl.
+
 * 3.1.0 (October 16, 2012)
 
   New features:

INSTALL

@@ -55,6 +55,11 @@ any of the following arguments (not a definitive list) to 'configure':
     jemalloc overlays the default malloc zone, but makes no attempt to actually
     replace the "malloc", "calloc", etc. symbols.
 
+--without-export
+    Don't export public APIs. This can be useful when building jemalloc as a
+    static library, or to avoid exporting public APIs when using the zone
+    allocator on OSX.
+
 --with-private-namespace=<prefix>
     Prefix all library-private APIs with <prefix>.  For shared libraries,
     symbol visibility mechanisms prevent these symbols from being exported, but
@@ -74,6 +79,12 @@ any of the following arguments (not a definitive list) to 'configure':
 --enable-debug
     Enable assertions and validation code.  This incurs a substantial
     performance hit, but is very useful during application development.
+    Implies --enable-ivsalloc.
+
+--enable-ivsalloc
+    Enable validation code, which verifies that pointers reside within
+    jemalloc-owned chunks before dereferencing them. This incurs a substantial
+    performance hit.
 
 --disable-stats
     Disable statistics gathering functionality.  See the "opt.stats_print"
@@ -136,6 +147,10 @@ any of the following arguments (not a definitive list) to 'configure':
 --disable-experimental
     Disable support for the experimental API (*allocm()).
 
+--disable-zone-allocator
+    Disable zone allocator for Darwin. This means jemalloc won't be hooked as
+    the default allocator on OSX/iOS.
+
 --enable-utrace
     Enable utrace(2)-based allocation tracing.  This feature is not broadly
     portable (FreeBSD has it, but Linux and OS X do not).

Makefile.in

@@ -48,6 +48,7 @@ cfgoutputs_in := @cfgoutputs_in@
 cfgoutputs_out := @cfgoutputs_out@
 enable_autogen := @enable_autogen@
 enable_experimental := @enable_experimental@
+enable_zone_allocator := @enable_zone_allocator@
 DSO_LDFLAGS = @DSO_LDFLAGS@
 SOREV = @SOREV@
 PIC_CFLAGS = @PIC_CFLAGS@
@@ -80,7 +81,7 @@ CSRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c $(srcroot)src/atomic.c \
 	$(srcroot)src/mutex.c $(srcroot)src/prof.c $(srcroot)src/quarantine.c \
 	$(srcroot)src/rtree.c $(srcroot)src/stats.c $(srcroot)src/tcache.c \
 	$(srcroot)src/util.c $(srcroot)src/tsd.c
-ifeq (macho, $(ABI))
+ifeq ($(enable_zone_allocator), 1)
 CSRCS += $(srcroot)src/zone.c
 endif
 ifeq ($(IMPORTLIB),$(SO))
@@ -112,9 +113,9 @@ COBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.$(O))
 CPICOBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.pic.$(O))
 CTESTOBJS := $(CTESTS:$(srcroot)%.c=$(objroot)%.$(O))
 
-.PHONY: all dist doc_html doc_man doc
+.PHONY: all dist build_doc_html build_doc_man build_doc
 .PHONY: install_bin install_include install_lib
-.PHONY: install_html install_man install_doc install
+.PHONY: install_doc_html install_doc_man install_doc install
 .PHONY: tests check clean distclean relclean
 
 .SECONDARY : $(CTESTOBJS)

configure.ac

@@ -86,7 +86,7 @@ MANDIR=`eval echo $MANDIR`
 AC_SUBST([MANDIR])
 
 dnl Support for building documentation.
-AC_PATH_PROG([XSLTPROC], [xsltproc], , [$PATH])
+AC_PATH_PROG([XSLTPROC], [xsltproc], [false], [$PATH])
 if test -d "/usr/share/xml/docbook/stylesheet/docbook-xsl" ; then
   DEFAULT_XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl"
 elif test -d "/usr/share/sgml/docbook/xsl-stylesheets" ; then
@@ -261,6 +261,7 @@ case "${host}" in
 	CFLAGS="$CFLAGS"
 	CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE"
 	abi="elf"
+	AC_DEFINE([JEMALLOC_HAS_ALLOCA_H])
 	AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
 	AC_DEFINE([JEMALLOC_THREADED_INIT], [ ])
 	JEMALLOC_USABLE_SIZE_CONST=""
@@ -402,9 +403,9 @@ AC_SUBST([enable_autogen])
 
 AC_PROG_INSTALL
 AC_PROG_RANLIB
-AC_PATH_PROG([AR], [ar], , [$PATH])
+AC_PATH_PROG([AR], [ar], [false], [$PATH])
-AC_PATH_PROG([LD], [ld], , [$PATH])
+AC_PATH_PROG([LD], [ld], [false], [$PATH])
-AC_PATH_PROG([AUTOCONF], [autoconf], , [$PATH])
+AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH])
 
 public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free malloc_usable_size malloc_stats_print mallctl mallctlnametomib mallctlbymib"
 
@@ -471,6 +472,13 @@ for stem in ${public_syms}; do
   AC_DEFINE_UNQUOTED([${n}], [${m}])
 done
 
+AC_ARG_WITH([export],
+  [AS_HELP_STRING([--without-export], [disable exporting jemalloc public APIs])],
+  [if test "x$with_export" = "xno"; then
+  AC_DEFINE([JEMALLOC_EXPORT],[])
+fi]
+)
+
 dnl Do not mangle library-private APIs by default.
 AC_ARG_WITH([private_namespace],
   [AS_HELP_STRING([--with-private-namespace=<prefix>], [Prefix to prepend to all library-private APIs])],
@@ -544,7 +552,7 @@ fi
 
 dnl Do not compile with debugging by default.
 AC_ARG_ENABLE([debug],
-  [AS_HELP_STRING([--enable-debug], [Build debugging code])],
+  [AS_HELP_STRING([--enable-debug], [Build debugging code (implies --enable-ivsalloc)])],
 [if test "x$enable_debug" = "xno" ; then
   enable_debug="0"
 else
@@ -555,10 +563,25 @@ fi
 )
 if test "x$enable_debug" = "x1" ; then
   AC_DEFINE([JEMALLOC_DEBUG], [ ])
-  AC_DEFINE([JEMALLOC_IVSALLOC], [ ])
+  enable_ivsalloc="1"
 fi
 AC_SUBST([enable_debug])
 
+dnl Do not validate pointers by default.
+AC_ARG_ENABLE([ivsalloc],
+  [AS_HELP_STRING([--enable-ivsalloc], [Validate pointers passed through the public API])],
+[if test "x$enable_ivsalloc" = "xno" ; then
+  enable_ivsalloc="0"
+else
+  enable_ivsalloc="1"
+fi
+],
+[enable_ivsalloc="0"]
+)
+if test "x$enable_ivsalloc" = "x1" ; then
+  AC_DEFINE([JEMALLOC_IVSALLOC], [ ])
+fi
+
 dnl Only optimize if not debugging.
 if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then
   dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS.
@@ -906,7 +929,7 @@ AC_CACHE_CHECK([STATIC_PAGE_SHIFT],
 #include <stdio.h>
 ]],
 [[
-    long result;
+    int result;
     FILE *f;
 
 #ifdef _WIN32
@@ -925,7 +948,7 @@ AC_CACHE_CHECK([STATIC_PAGE_SHIFT],
     if (f == NULL) {
 	return 1;
     }
-    fprintf(f, "%u\n", result);
+    fprintf(f, "%d\n", result);
     fclose(f);
 
     return 0;
@@ -1070,11 +1093,13 @@ dnl Check for ffsl(3), and fail if not found.  This function exists on all
 dnl platforms that jemalloc currently has a chance of functioning on without
 dnl modification.
 JE_COMPILABLE([a program using ffsl], [
+#include <stdio.h>
 #include <strings.h>
 #include <string.h>
 ], [
 	{
 		int rv = ffsl(0x08);
+		printf("%d\n", rv);
 	}
 ], [je_cv_function_ffsl])
 if test "x${je_cv_function_ffsl}" != "xyes" ; then
@@ -1178,7 +1203,26 @@ fi
 dnl ============================================================================
 dnl Darwin-related configuration.
 
-if test "x${abi}" = "xmacho" ; then
+AC_ARG_ENABLE([zone-allocator],
+  [AS_HELP_STRING([--disable-zone-allocator],
+                  [Disable zone allocator for Darwin])],
+[if test "x$enable_zone_allocator" = "xno" ; then
+  enable_zone_allocator="0"
+else
+  enable_zone_allocator="1"
+fi
+],
+[if test "x${abi}" = "xmacho"; then
+  enable_zone_allocator="1"
+fi
+]
+)
+AC_SUBST([enable_zone_allocator])
+
+if test "x${enable_zone_allocator}" = "x1" ; then
+  if test "x${abi}" != "xmacho"; then
+    AC_MSG_ERROR([--enable-zone-allocator is only supported on Darwin])
+  fi
   AC_DEFINE([JEMALLOC_IVSALLOC], [ ])
   AC_DEFINE([JEMALLOC_ZONE], [ ])
 

doc/jemalloc.xml.in

@@ -432,7 +432,14 @@ for (i = 0; i < nbins; i++) {
     referenced by the symbolic link named <filename
     class="symlink">/etc/malloc.conf</filename>, and the value of the
     environment variable <envar>MALLOC_CONF</envar>, will be interpreted, in
-    that order, from left to right as options.</para>
+    that order, from left to right as options.  Note that
+    <varname>malloc_conf</varname> may be read before
+    <function>main<parameter/></function> is entered, so the declaration of
+    <varname>malloc_conf</varname> should specify an initializer that contains
+    the final value to be read by jemalloc.  <varname>malloc_conf</varname> is
+    a compile-time setting, whereas <filename
+    class="symlink">/etc/malloc.conf</filename> and <envar>MALLOC_CONF</envar>
+    can be safely set any time prior to program invocation.</para>
 
     <para>An options string is a comma-separated list of option:value pairs.
     There is one key corresponding to each <link
@@ -790,8 +797,11 @@ for (i = 0; i < nbins; i++) {
           (<type>size_t</type>)
           <literal>r-</literal>
         </term>
-        <listitem><para>Virtual memory chunk size (log base 2).  The default
+        <listitem><para>Virtual memory chunk size (log base 2).  If a chunk
-        chunk size is 4 MiB (2^22).</para></listitem>
+        size outside the supported size range is specified, the size is
+        silently clipped to the minimum/maximum supported size.  The default
+        chunk size is 4 MiB (2^22).
+        </para></listitem>
       </varlistentry>
 
       <varlistentry id="opt.dss">
@@ -833,7 +843,7 @@ for (i = 0; i < nbins; i++) {
         <manvolnum>2</manvolnum></citerefentry> or a similar system call.  This
         provides the kernel with sufficient information to recycle dirty pages
         if physical memory becomes scarce and the pages remain unused.  The
-        default minimum ratio is 32:1 (2^5:1); an option value of -1 will
+        default minimum ratio is 8:1 (2^3:1); an option value of -1 will
         disable dirty page purging.</para></listitem>
       </varlistentry>
 
@@ -1512,7 +1522,9 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         application.  This is a multiple of the page size, and greater than or
         equal to <link
         linkend="stats.allocated"><mallctl>stats.allocated</mallctl></link>.
-        </para></listitem>
+        This does not include <link linkend="stats.arenas.i.pdirty">
+        <mallctl>stats.arenas.&lt;i&gt;.pdirty</mallctl></link> and pages
+        entirely devoted to allocator metadata.</para></listitem>
       </varlistentry>
 
       <varlistentry>
@@ -1628,7 +1640,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         <listitem><para>Number of pages in active runs.</para></listitem>
       </varlistentry>
 
-      <varlistentry>
+      <varlistentry id="stats.arenas.i.pdirty">
         <term>
           <mallctl>stats.arenas.&lt;i&gt;.pdirty</mallctl>
           (<type>size_t</type>)

include/jemalloc/internal/arena.h

@@ -38,10 +38,10 @@
  *
  *   (nactive >> opt_lg_dirty_mult) >= ndirty
  *
- * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32
+ * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times
- * times as many active pages as dirty pages.
+ * as many active pages as dirty pages.
  */
-#define	LG_DIRTY_MULT_DEFAULT	5
+#define	LG_DIRTY_MULT_DEFAULT	3
 
 typedef struct arena_chunk_map_s arena_chunk_map_t;
 typedef struct arena_chunk_s arena_chunk_t;
@@ -69,7 +69,7 @@ struct arena_chunk_map_s {
 		/*
 		 * Linkage for run trees.  There are two disjoint uses:
 		 *
-		 * 1) arena_t's runs_avail_{clean,dirty} trees.
+		 * 1) arena_t's runs_avail tree.
 		 * 2) arena_run_t conceptually uses this linkage for in-use
 		 *    non-full runs, rather than directly embedding linkage.
 		 */
@@ -162,20 +162,24 @@ typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
 /* Arena chunk header. */
 struct arena_chunk_s {
 	/* Arena that owns the chunk. */
-	arena_t		*arena;
+	arena_t			*arena;
 
-	/* Linkage for the arena's chunks_dirty list. */
+	/* Linkage for tree of arena chunks that contain dirty runs. */
-	ql_elm(arena_chunk_t) link_dirty;
+	rb_node(arena_chunk_t)	dirty_link;
-
-	/*
-	 * True if the chunk is currently in the chunks_dirty list, due to
-	 * having at some point contained one or more dirty pages.  Removal
-	 * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible.
-	 */
-	bool		dirtied;
 
 	/* Number of dirty pages. */
-	size_t		ndirty;
+	size_t			ndirty;
+
+	/* Number of available runs. */
+	size_t			nruns_avail;
+
+	/*
+	 * Number of available run adjacencies.  Clean and dirty available runs
+	 * are not coalesced, which causes virtual memory fragmentation.  The
+	 * ratio of (nruns_avail-nruns_adjac):nruns_adjac is used for tracking
+	 * this fragmentation.
+	 * */
+	size_t			nruns_adjac;
 
 	/*
 	 * Map of pages within chunk that keeps track of free/large/small.  The
@@ -183,7 +187,7 @@ struct arena_chunk_s {
 	 * need to be tracked in the map.  This omission saves a header page
 	 * for common chunk sizes (e.g. 4 MiB).
 	 */
-	arena_chunk_map_t map[1]; /* Dynamically sized. */
+	arena_chunk_map_t	map[1]; /* Dynamically sized. */
 };
 typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
 
@@ -333,8 +337,8 @@ struct arena_s {
 
 	dss_prec_t		dss_prec;
 
-	/* List of dirty-page-containing chunks this arena manages. */
+	/* Tree of dirty-page-containing chunks this arena manages. */
-	ql_head(arena_chunk_t)	chunks_dirty;
+	arena_chunk_tree_t	chunks_dirty;
 
 	/*
 	 * In order to avoid rapid chunk allocation/deallocation when an arena
@@ -369,18 +373,9 @@ struct arena_s {
 
 	/*
 	 * Size/address-ordered trees of this arena's available runs.  The trees
-	 * are used for first-best-fit run allocation.  The dirty tree contains
+	 * are used for first-best-fit run allocation.
-	 * runs with dirty pages (i.e. very likely to have been touched and
-	 * therefore have associated physical pages), whereas the clean tree
-	 * contains runs with pages that either have no associated physical
-	 * pages, or have pages that the kernel may recycle at any time due to
-	 * previous madvise(2) calls.  The dirty tree is used in preference to
-	 * the clean tree for allocations, because using dirty pages reduces
-	 * the amount of dirty purging necessary to keep the active:dirty page
-	 * ratio below the purge threshold.
 	 */
-	arena_avail_tree_t	runs_avail_clean;
+	arena_avail_tree_t	runs_avail;
-	arena_avail_tree_t	runs_avail_dirty;
 
 	/* bins is used to store trees of free regions. */
 	arena_bin_t		bins[NBINS];
@@ -405,7 +400,6 @@ extern arena_bin_info_t	arena_bin_info[NBINS];
 #define			nlclasses (chunk_npages - map_bias)
 
 void	arena_purge_all(arena_t *arena);
-void	arena_prof_accum(arena_t *arena, uint64_t accumbytes);
 void	arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
     size_t binind, uint64_t prof_accumbytes);
 void	arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info,
@@ -469,6 +463,9 @@ void	arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
     size_t runind, size_t binind, size_t flags);
 void	arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
     size_t unzeroed);
+bool	arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
+bool	arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
+bool	arena_prof_accum(arena_t *arena, uint64_t accumbytes);
 size_t	arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
 size_t	arena_bin_index(arena_t *arena, arena_bin_t *bin);
 unsigned	arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
@@ -483,7 +480,7 @@ void	arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
 #  ifdef JEMALLOC_ARENA_INLINE_A
-JEMALLOC_INLINE arena_chunk_map_t *
+JEMALLOC_ALWAYS_INLINE arena_chunk_map_t *
 arena_mapp_get(arena_chunk_t *chunk, size_t pageind)
 {
 
@@ -493,21 +490,21 @@ arena_mapp_get(arena_chunk_t *chunk, size_t pageind)
 	return (&chunk->map[pageind-map_bias]);
 }
 
-JEMALLOC_INLINE size_t *
+JEMALLOC_ALWAYS_INLINE size_t *
 arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind)
 {
 
 	return (&arena_mapp_get(chunk, pageind)->bits);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_get(arena_chunk_t *chunk, size_t pageind)
 {
 
 	return (*arena_mapbitsp_get(chunk, pageind));
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -517,7 +514,7 @@ arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits & ~PAGE_MASK);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -528,7 +525,7 @@ arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits & ~PAGE_MASK);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -539,7 +536,7 @@ arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits >> LG_PAGE);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -551,7 +548,7 @@ arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind)
 	return (binind);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -560,7 +557,7 @@ arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits & CHUNK_MAP_DIRTY);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -569,7 +566,7 @@ arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits & CHUNK_MAP_UNZEROED);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -578,7 +575,7 @@ arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits & CHUNK_MAP_LARGE);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind)
 {
 	size_t mapbits;
@@ -587,7 +584,7 @@ arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind)
 	return (mapbits & CHUNK_MAP_ALLOCATED);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
     size_t flags)
 {
@@ -600,7 +597,7 @@ arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
 	*mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags;
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
     size_t size)
 {
@@ -612,7 +609,7 @@ arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
 	*mapbitsp = size | (*mapbitsp & PAGE_MASK);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
     size_t flags)
 {
@@ -627,7 +624,7 @@ arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
 	    CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
     size_t binind)
 {
@@ -640,7 +637,7 @@ arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
 	    CHUNK_MAP_BININD_SHIFT);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
     size_t binind, size_t flags)
 {
@@ -656,7 +653,7 @@ arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
 	    flags | unzeroed | CHUNK_MAP_ALLOCATED;
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
     size_t unzeroed)
 {
@@ -666,7 +663,52 @@ arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
 	*mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed;
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_INLINE bool
+arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes)
+{
+
+	cassert(config_prof);
+	assert(prof_interval != 0);
+
+	arena->prof_accumbytes += accumbytes;
+	if (arena->prof_accumbytes >= prof_interval) {
+		arena->prof_accumbytes -= prof_interval;
+		return (true);
+	}
+	return (false);
+}
+
+JEMALLOC_INLINE bool
+arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes)
+{
+
+	cassert(config_prof);
+
+	if (prof_interval == 0)
+		return (false);
+	return (arena_prof_accum_impl(arena, accumbytes));
+}
+
+JEMALLOC_INLINE bool
+arena_prof_accum(arena_t *arena, uint64_t accumbytes)
+{
+
+	cassert(config_prof);
+
+	if (prof_interval == 0)
+		return (false);
+
+	{
+		bool ret;
+
+		malloc_mutex_lock(&arena->lock);
+		ret = arena_prof_accum_impl(arena, accumbytes);
+		malloc_mutex_unlock(&arena->lock);
+		return (ret);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE size_t
 arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
 {
 	size_t binind;
@@ -861,7 +903,7 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
 		arena_mapp_get(chunk, pageind)->prof_ctx = ctx;
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
 {
 	tcache_t *tcache;
@@ -892,7 +934,7 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
 }
 
 /* Return the size of the allocation pointed to by ptr. */
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 arena_salloc(const void *ptr, bool demote)
 {
 	size_t ret;
@@ -938,7 +980,7 @@ arena_salloc(const void *ptr, bool demote)
 	return (ret);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
 {
 	size_t pageind, mapbits;

include/jemalloc/internal/ckh.h

@@ -5,7 +5,7 @@ typedef struct ckh_s ckh_t;
 typedef struct ckhc_s ckhc_t;
 
 /* Typedefs to allow easy function pointer passing. */
-typedef void ckh_hash_t (const void *, unsigned, size_t *, size_t *);
+typedef void ckh_hash_t (const void *, size_t[2]);
 typedef bool ckh_keycomp_t (const void *, const void *);
 
 /* Maintain counters used to get an idea of performance. */
@@ -75,11 +75,9 @@ bool	ckh_insert(ckh_t *ckh, const void *key, const void *data);
 bool	ckh_remove(ckh_t *ckh, const void *searchkey, void **key,
     void **data);
 bool	ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data);
-void	ckh_string_hash(const void *key, unsigned minbits, size_t *hash1,
+void	ckh_string_hash(const void *key, size_t r_hash[2]);
-    size_t *hash2);
 bool	ckh_string_keycomp(const void *k1, const void *k2);
-void	ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1,
+void	ckh_pointer_hash(const void *key, size_t r_hash[2]);
-    size_t *hash2);
 bool	ckh_pointer_keycomp(const void *k1, const void *k2);
 
 #endif /* JEMALLOC_H_EXTERNS */

include/jemalloc/internal/hash.h

@@ -1,3 +1,8 @@
+/*
+ * The following hash function is based on MurmurHash3, placed into the public
+ * domain by Austin Appleby.  See http://code.google.com/p/smhasher/ for
+ * details.
+ */
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
@@ -14,55 +19,311 @@
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-uint64_t	hash(const void *key, size_t len, uint64_t seed);
+void	hash(const void *key, size_t len, const uint32_t seed,
+    size_t r_hash[2]);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_))
-/*
+/******************************************************************************/
- * The following hash function is based on MurmurHash64A(), placed into the
+/* Internal implementation. */
- * public domain by Austin Appleby.  See http://murmurhash.googlepages.com/ for
+JEMALLOC_INLINE uint32_t
- * details.
+hash_rotl_32(uint32_t x, int8_t r)
- */
-JEMALLOC_INLINE uint64_t
-hash(const void *key, size_t len, uint64_t seed)
 {
-	const uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
-	const int r = 47;
-	uint64_t h = seed ^ (len * m);
-	const uint64_t *data = (const uint64_t *)key;
-	const uint64_t *end = data + (len/8);
-	const unsigned char *data2;
 
-	assert(((uintptr_t)key & 0x7) == 0);
+	return (x << r) | (x >> (32 - r));
+}
 
-	while(data != end) {
+JEMALLOC_INLINE uint64_t
-		uint64_t k = *data++;
+hash_rotl_64(uint64_t x, int8_t r)
+{
+	return (x << r) | (x >> (64 - r));
+}
 
-		k *= m;
+JEMALLOC_INLINE uint32_t
-		k ^= k >> r;
+hash_get_block_32(const uint32_t *p, int i)
-		k *= m;
+{
 
-		h ^= k;
+	return p[i];
-		h *= m;
+}
+
+JEMALLOC_INLINE uint64_t
+hash_get_block_64(const uint64_t *p, int i)
+{
+
+	return p[i];
+}
+
+JEMALLOC_INLINE uint32_t
+hash_fmix_32(uint32_t h)
+{
+
+	h ^= h >> 16;
+	h *= 0x85ebca6b;
+	h ^= h >> 13;
+	h *= 0xc2b2ae35;
+	h ^= h >> 16;
+
+	return h;
+}
+
+JEMALLOC_INLINE uint64_t
+hash_fmix_64(uint64_t k)
+{
+
+	k ^= k >> 33;
+	k *= QU(0xff51afd7ed558ccdLLU);
+	k ^= k >> 33;
+	k *= QU(0xc4ceb9fe1a85ec53LLU);
+	k ^= k >> 33;
+
+	return k;
+}
+
+JEMALLOC_INLINE uint32_t
+hash_x86_32(const void *key, int len, uint32_t seed)
+{
+	const uint8_t *data = (const uint8_t *) key;
+	const int nblocks = len / 4;
+
+	uint32_t h1 = seed;
+
+	const uint32_t c1 = 0xcc9e2d51;
+	const uint32_t c2 = 0x1b873593;
+
+	/* body */
+	{
+		const uint32_t *blocks = (const uint32_t *) (data + nblocks*4);
+		int i;
+
+		for (i = -nblocks; i; i++) {
+			uint32_t k1 = hash_get_block_32(blocks, i);
+
+			k1 *= c1;
+			k1 = hash_rotl_32(k1, 15);
+			k1 *= c2;
+
+			h1 ^= k1;
+			h1 = hash_rotl_32(h1, 13);
+			h1 = h1*5 + 0xe6546b64;
+		}
 	}
 
-	data2 = (const unsigned char *)data;
+	/* tail */
-	switch(len & 7) {
+	{
-	case 7: h ^= ((uint64_t)(data2[6])) << 48;
+		const uint8_t *tail = (const uint8_t *) (data + nblocks*4);
-	case 6: h ^= ((uint64_t)(data2[5])) << 40;
+
-	case 5: h ^= ((uint64_t)(data2[4])) << 32;
+		uint32_t k1 = 0;
-	case 4: h ^= ((uint64_t)(data2[3])) << 24;
+
-	case 3: h ^= ((uint64_t)(data2[2])) << 16;
+		switch (len & 3) {
-	case 2: h ^= ((uint64_t)(data2[1])) << 8;
+		case 3: k1 ^= tail[2] << 16;
-	case 1: h ^= ((uint64_t)(data2[0]));
+		case 2: k1 ^= tail[1] << 8;
-		h *= m;
+		case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15);
+			k1 *= c2; h1 ^= k1;
+		}
 	}
 
-	h ^= h >> r;
+	/* finalization */
-	h *= m;
+	h1 ^= len;
-	h ^= h >> r;
 
-	return (h);
+	h1 = hash_fmix_32(h1);
+
+	return h1;
+}
+
+UNUSED JEMALLOC_INLINE void
+hash_x86_128(const void *key, const int len, uint32_t seed,
+  uint64_t r_out[2])
+{
+	const uint8_t * data = (const uint8_t *) key;
+	const int nblocks = len / 16;
+
+	uint32_t h1 = seed;
+	uint32_t h2 = seed;
+	uint32_t h3 = seed;
+	uint32_t h4 = seed;
+
+	const uint32_t c1 = 0x239b961b;
+	const uint32_t c2 = 0xab0e9789;
+	const uint32_t c3 = 0x38b34ae5;
+	const uint32_t c4 = 0xa1e38b93;
+
+	/* body */
+	{
+		const uint32_t *blocks = (const uint32_t *) (data + nblocks*16);
+		int i;
+
+		for (i = -nblocks; i; i++) {
+			uint32_t k1 = hash_get_block_32(blocks, i*4 + 0);
+			uint32_t k2 = hash_get_block_32(blocks, i*4 + 1);
+			uint32_t k3 = hash_get_block_32(blocks, i*4 + 2);
+			uint32_t k4 = hash_get_block_32(blocks, i*4 + 3);
+
+			k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+
+			h1 = hash_rotl_32(h1, 19); h1 += h2;
+			h1 = h1*5 + 0x561ccd1b;
+
+			k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
+
+			h2 = hash_rotl_32(h2, 17); h2 += h3;
+			h2 = h2*5 + 0x0bcaa747;
+
+			k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
+
+			h3 = hash_rotl_32(h3, 15); h3 += h4;
+			h3 = h3*5 + 0x96cd1c35;
+
+			k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
+
+			h4 = hash_rotl_32(h4, 13); h4 += h1;
+			h4 = h4*5 + 0x32ac3b17;
+		}
+	}
+
+	/* tail */
+	{
+		const uint8_t *tail = (const uint8_t *) (data + nblocks*16);
+		uint32_t k1 = 0;
+		uint32_t k2 = 0;
+		uint32_t k3 = 0;
+		uint32_t k4 = 0;
+
+		switch (len & 15) {
+		case 15: k4 ^= tail[14] << 16;
+		case 14: k4 ^= tail[13] << 8;
+		case 13: k4 ^= tail[12] << 0;
+			k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4;
+
+		case 12: k3 ^= tail[11] << 24;
+		case 11: k3 ^= tail[10] << 16;
+		case 10: k3 ^= tail[ 9] << 8;
+		case  9: k3 ^= tail[ 8] << 0;
+		     k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3;
+
+		case  8: k2 ^= tail[ 7] << 24;
+		case  7: k2 ^= tail[ 6] << 16;
+		case  6: k2 ^= tail[ 5] << 8;
+		case  5: k2 ^= tail[ 4] << 0;
+			k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2;
+
+		case  4: k1 ^= tail[ 3] << 24;
+		case  3: k1 ^= tail[ 2] << 16;
+		case  2: k1 ^= tail[ 1] << 8;
+		case  1: k1 ^= tail[ 0] << 0;
+			k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1;
+		}
+	}
+
+	/* finalization */
+	h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
+
+	h1 += h2; h1 += h3; h1 += h4;
+	h2 += h1; h3 += h1; h4 += h1;
+
+	h1 = hash_fmix_32(h1);
+	h2 = hash_fmix_32(h2);
+	h3 = hash_fmix_32(h3);
+	h4 = hash_fmix_32(h4);
+
+	h1 += h2; h1 += h3; h1 += h4;
+	h2 += h1; h3 += h1; h4 += h1;
+
+	r_out[0] = (((uint64_t) h2) << 32) | h1;
+	r_out[1] = (((uint64_t) h4) << 32) | h3;
+}
+
+UNUSED JEMALLOC_INLINE void
+hash_x64_128(const void *key, const int len, const uint32_t seed,
+  uint64_t r_out[2])
+{
+	const uint8_t *data = (const uint8_t *) key;
+	const int nblocks = len / 16;
+
+	uint64_t h1 = seed;
+	uint64_t h2 = seed;
+
+	const uint64_t c1 = QU(0x87c37b91114253d5LLU);
+	const uint64_t c2 = QU(0x4cf5ad432745937fLLU);
+
+	/* body */
+	{
+		const uint64_t *blocks = (const uint64_t *) (data);
+		int i;
+
+		for (i = 0; i < nblocks; i++) {
+			uint64_t k1 = hash_get_block_64(blocks, i*2 + 0);
+			uint64_t k2 = hash_get_block_64(blocks, i*2 + 1);
+
+			k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+
+			h1 = hash_rotl_64(h1, 27); h1 += h2;
+			h1 = h1*5 + 0x52dce729;
+
+			k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
+
+			h2 = hash_rotl_64(h2, 31); h2 += h1;
+			h2 = h2*5 + 0x38495ab5;
+		}
+	}
+
+	/* tail */
+	{
+		const uint8_t *tail = (const uint8_t*)(data + nblocks*16);
+		uint64_t k1 = 0;
+		uint64_t k2 = 0;
+
+		switch (len & 15) {
+		case 15: k2 ^= ((uint64_t)(tail[14])) << 48;
+		case 14: k2 ^= ((uint64_t)(tail[13])) << 40;
+		case 13: k2 ^= ((uint64_t)(tail[12])) << 32;
+		case 12: k2 ^= ((uint64_t)(tail[11])) << 24;
+		case 11: k2 ^= ((uint64_t)(tail[10])) << 16;
+		case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8;
+		case  9: k2 ^= ((uint64_t)(tail[ 8])) << 0;
+			k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2;
+
+		case  8: k1 ^= ((uint64_t)(tail[ 7])) << 56;
+		case  7: k1 ^= ((uint64_t)(tail[ 6])) << 48;
+		case  6: k1 ^= ((uint64_t)(tail[ 5])) << 40;
+		case  5: k1 ^= ((uint64_t)(tail[ 4])) << 32;
+		case  4: k1 ^= ((uint64_t)(tail[ 3])) << 24;
+		case  3: k1 ^= ((uint64_t)(tail[ 2])) << 16;
+		case  2: k1 ^= ((uint64_t)(tail[ 1])) << 8;
+		case  1: k1 ^= ((uint64_t)(tail[ 0])) << 0;
+			k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1;
+		}
+	}
+
+	/* finalization */
+	h1 ^= len; h2 ^= len;
+
+	h1 += h2;
+	h2 += h1;
+
+	h1 = hash_fmix_64(h1);
+	h2 = hash_fmix_64(h2);
+
+	h1 += h2;
+	h2 += h1;
+
+	r_out[0] = h1;
+	r_out[1] = h2;
+}
+
+
+/******************************************************************************/
+/* API. */
+JEMALLOC_INLINE void
+hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2])
+{
+#if (LG_SIZEOF_PTR == 3)
+	hash_x64_128(key, len, seed, (uint64_t *)r_hash);
+#else
+	uint64_t hashes[2];
+	hash_x86_128(key, len, seed, hashes);
+	r_hash[0] = (size_t)hashes[0];
+	r_hash[1] = (size_t)hashes[1];
+#endif
 }
 #endif
 

include/jemalloc/internal/jemalloc_internal.h.in

@@ -226,6 +226,7 @@ static const bool config_ivsalloc =
 #define	ALLOCM_LG_ALIGN_MASK	((int)0x3f)
 
 #define	ZU(z)	((size_t)z)
+#define	QU(q)	((uint64_t)q)
 
 #ifndef __DECONST
 #  define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
@@ -233,10 +234,17 @@ static const bool config_ivsalloc =
 
 #ifdef JEMALLOC_DEBUG
    /* Disable inlining to make debugging easier. */
+#  define JEMALLOC_ALWAYS_INLINE
 #  define JEMALLOC_INLINE
 #  define inline
 #else
 #  define JEMALLOC_ENABLE_INLINE
+#  ifdef JEMALLOC_HAVE_ATTR
+#    define JEMALLOC_ALWAYS_INLINE \
+	 static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline)
+#  else
+#    define JEMALLOC_ALWAYS_INLINE static inline
+#  endif
 #  define JEMALLOC_INLINE static inline
 #  ifdef _MSC_VER
 #    define inline _inline
@@ -270,6 +278,9 @@ static const bool config_ivsalloc =
 #  ifdef __arm__
 #    define LG_QUANTUM		3
 #  endif
+#  ifdef __aarch64__
+#    define LG_QUANTUM		4
+#  endif
 #  ifdef __hppa__
 #    define LG_QUANTUM		4
 #  endif
@@ -279,7 +290,7 @@ static const bool config_ivsalloc =
 #  ifdef __powerpc__
 #    define LG_QUANTUM		4
 #  endif
-#  ifdef __s390x__
+#  ifdef __s390__
 #    define LG_QUANTUM		4
 #  endif
 #  ifdef __SH4__
@@ -359,7 +370,11 @@ static const bool config_ivsalloc =
 #    include <malloc.h>
 #    define alloca _alloca
 #  else
-#    include <alloca.h>
+#    ifdef JEMALLOC_HAS_ALLOCA_H
+#      include <alloca.h>
+#    else
+#      include <stdlib.h>
+#    endif
 #  endif
 #  define VARIABLE_ARRAY(type, name, count) \
 	type *name = alloca(sizeof(type) * count)
@@ -428,15 +443,18 @@ static const bool config_ivsalloc =
 } while (0)
 #else
 #define	RUNNING_ON_VALGRIND	((unsigned)0)
-#define	VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
+#define	VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \
-#define	VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)
+    do {} while (0)
-#define	VALGRIND_FREELIKE_BLOCK(addr, rzB)
+#define	VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \
-#define	VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len)
+    do {} while (0)
-#define	VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len)
+#define	VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0)
-#define	JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero)
+#define	VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0)
+#define	VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0)
+#define	VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0)
+#define	JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0)
 #define	JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize,	\
-    old_rzsize, zero)
+    old_rzsize, zero) do {} while (0)
-#define	JEMALLOC_VALGRIND_FREE(ptr, rzsize)
+#define	JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0)
 #endif
 
 #include "jemalloc/internal/util.h"
@@ -591,13 +609,14 @@ arena_t	*choose_arena(arena_t *arena);
  * for allocations.
  */
 malloc_tsd_externs(arenas, arena_t *)
-malloc_tsd_funcs(JEMALLOC_INLINE, arenas, arena_t *, NULL, arenas_cleanup)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL,
+    arenas_cleanup)
 
 /*
  * Compute usable size that would result from allocating an object with the
  * specified size.
  */
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 s2u(size_t size)
 {
 
@@ -612,7 +631,7 @@ s2u(size_t size)
  * Compute usable size that would result from allocating an object with the
  * specified size and alignment.
  */
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 sa2u(size_t size, size_t alignment)
 {
 	size_t usize;
@@ -757,7 +776,7 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 imallocx(size_t size, bool try_tcache, arena_t *arena)
 {
 
@@ -769,14 +788,14 @@ imallocx(size_t size, bool try_tcache, arena_t *arena)
 		return (huge_malloc(size, false));
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 imalloc(size_t size)
 {
 
 	return (imallocx(size, true, NULL));
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 icallocx(size_t size, bool try_tcache, arena_t *arena)
 {
 
@@ -786,14 +805,14 @@ icallocx(size_t size, bool try_tcache, arena_t *arena)
 		return (huge_malloc(size, true));
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 icalloc(size_t size)
 {
 
 	return (icallocx(size, true, NULL));
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
     arena_t *arena)
 {
@@ -818,7 +837,7 @@ ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
 	return (ret);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 ipalloc(size_t usize, size_t alignment, bool zero)
 {
 
@@ -830,7 +849,7 @@ ipalloc(size_t usize, size_t alignment, bool zero)
  *   void *ptr = [...]
  *   size_t sz = isalloc(ptr, config_prof);
  */
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 isalloc(const void *ptr, bool demote)
 {
 	size_t ret;
@@ -849,7 +868,7 @@ isalloc(const void *ptr, bool demote)
 	return (ret);
 }
 
-JEMALLOC_INLINE size_t
+JEMALLOC_ALWAYS_INLINE size_t
 ivsalloc(const void *ptr, bool demote)
 {
 
@@ -882,7 +901,7 @@ p2rz(const void *ptr)
 	return (u2rz(usize));
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 idallocx(void *ptr, bool try_tcache)
 {
 	arena_chunk_t *chunk;
@@ -896,14 +915,14 @@ idallocx(void *ptr, bool try_tcache)
 		huge_dalloc(ptr, true);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 idalloc(void *ptr)
 {
 
 	idallocx(ptr, true);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 iqallocx(void *ptr, bool try_tcache)
 {
 
@@ -913,14 +932,14 @@ iqallocx(void *ptr, bool try_tcache)
 		idallocx(ptr, try_tcache);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 iqalloc(void *ptr)
 {
 
 	iqallocx(ptr, true);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
     bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
 {
@@ -989,7 +1008,7 @@ irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 	}
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
     bool no_move)
 {
@@ -999,7 +1018,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 }
 
 malloc_tsd_externs(thread_allocated, thread_allocated_t)
-malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t,
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t,
     THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
 #endif
 

include/jemalloc/internal/private_namespace.h

@@ -41,6 +41,8 @@
 #define	arena_postfork_parent JEMALLOC_N(arena_postfork_parent)
 #define	arena_prefork JEMALLOC_N(arena_prefork)
 #define	arena_prof_accum JEMALLOC_N(arena_prof_accum)
+#define	arena_prof_accum_impl JEMALLOC_N(arena_prof_accum_impl)
+#define	arena_prof_accum_locked JEMALLOC_N(arena_prof_accum_locked)
 #define	arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get)
 #define	arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set)
 #define	arena_prof_promoted JEMALLOC_N(arena_prof_promoted)
@@ -63,6 +65,7 @@
 #define	arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot)
 #define	arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper)
 #define	arenas_tsd_get JEMALLOC_N(arenas_tsd_get)
+#define	arenas_tsd_get_wrapper JEMALLOC_N(arenas_tsd_get_wrapper)
 #define	arenas_tsd_set JEMALLOC_N(arenas_tsd_set)
 #define	atomic_add_u JEMALLOC_N(atomic_add_u)
 #define	atomic_add_uint32 JEMALLOC_N(atomic_add_uint32)
@@ -174,6 +177,15 @@
 #define	extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search)
 #define	get_errno JEMALLOC_N(get_errno)
 #define	hash JEMALLOC_N(hash)
+#define	hash_fmix_32 JEMALLOC_N(hash_fmix_32)
+#define	hash_fmix_64 JEMALLOC_N(hash_fmix_64)
+#define	hash_get_block_32 JEMALLOC_N(hash_get_block_32)
+#define	hash_get_block_64 JEMALLOC_N(hash_get_block_64)
+#define	hash_rotl_32 JEMALLOC_N(hash_rotl_32)
+#define	hash_rotl_64 JEMALLOC_N(hash_rotl_64)
+#define	hash_x64_128 JEMALLOC_N(hash_x64_128)
+#define	hash_x86_128 JEMALLOC_N(hash_x86_128)
+#define	hash_x86_32 JEMALLOC_N(hash_x86_32)
 #define	huge_allocated JEMALLOC_N(huge_allocated)
 #define	huge_boot JEMALLOC_N(huge_boot)
 #define	huge_dalloc JEMALLOC_N(huge_dalloc)
@@ -291,12 +303,20 @@
 #define	prof_tdata_tsd_boot JEMALLOC_N(prof_tdata_tsd_boot)
 #define	prof_tdata_tsd_cleanup_wrapper JEMALLOC_N(prof_tdata_tsd_cleanup_wrapper)
 #define	prof_tdata_tsd_get JEMALLOC_N(prof_tdata_tsd_get)
+#define	prof_tdata_tsd_get_wrapper JEMALLOC_N(prof_tdata_tsd_get_wrapper)
 #define	prof_tdata_tsd_set JEMALLOC_N(prof_tdata_tsd_set)
 #define	quarantine JEMALLOC_N(quarantine)
+#define	quarantine_alloc_hook JEMALLOC_N(quarantine_alloc_hook)
 #define	quarantine_boot JEMALLOC_N(quarantine_boot)
+#define	quarantine_booted JEMALLOC_N(quarantine_booted)
+#define	quarantine_cleanup JEMALLOC_N(quarantine_cleanup)
+#define	quarantine_init JEMALLOC_N(quarantine_init)
+#define	quarantine_tls JEMALLOC_N(quarantine_tls)
+#define	quarantine_tsd JEMALLOC_N(quarantine_tsd)
 #define	quarantine_tsd_boot JEMALLOC_N(quarantine_tsd_boot)
 #define	quarantine_tsd_cleanup_wrapper JEMALLOC_N(quarantine_tsd_cleanup_wrapper)
 #define	quarantine_tsd_get JEMALLOC_N(quarantine_tsd_get)
+#define	quarantine_tsd_get_wrapper JEMALLOC_N(quarantine_tsd_get_wrapper)
 #define	quarantine_tsd_set JEMALLOC_N(quarantine_tsd_set)
 #define	register_zone JEMALLOC_N(register_zone)
 #define	rtree_get JEMALLOC_N(rtree_get)
@@ -340,6 +360,7 @@
 #define	tcache_enabled_tsd_boot JEMALLOC_N(tcache_enabled_tsd_boot)
 #define	tcache_enabled_tsd_cleanup_wrapper JEMALLOC_N(tcache_enabled_tsd_cleanup_wrapper)
 #define	tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get)
+#define	tcache_enabled_tsd_get_wrapper JEMALLOC_N(tcache_enabled_tsd_get_wrapper)
 #define	tcache_enabled_tsd_set JEMALLOC_N(tcache_enabled_tsd_set)
 #define	tcache_event JEMALLOC_N(tcache_event)
 #define	tcache_event_hard JEMALLOC_N(tcache_event_hard)
@@ -355,6 +376,7 @@
 #define	tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot)
 #define	tcache_tsd_cleanup_wrapper JEMALLOC_N(tcache_tsd_cleanup_wrapper)
 #define	tcache_tsd_get JEMALLOC_N(tcache_tsd_get)
+#define	tcache_tsd_get_wrapper JEMALLOC_N(tcache_tsd_get_wrapper)
 #define	tcache_tsd_set JEMALLOC_N(tcache_tsd_set)
 #define	thread_allocated_booted JEMALLOC_N(thread_allocated_booted)
 #define	thread_allocated_initialized JEMALLOC_N(thread_allocated_initialized)
@@ -363,5 +385,6 @@
 #define	thread_allocated_tsd_boot JEMALLOC_N(thread_allocated_tsd_boot)
 #define	thread_allocated_tsd_cleanup_wrapper JEMALLOC_N(thread_allocated_tsd_cleanup_wrapper)
 #define	thread_allocated_tsd_get JEMALLOC_N(thread_allocated_tsd_get)
+#define	thread_allocated_tsd_get_wrapper JEMALLOC_N(thread_allocated_tsd_get_wrapper)
 #define	thread_allocated_tsd_set JEMALLOC_N(thread_allocated_tsd_set)
 #define	u2rz JEMALLOC_N(u2rz)

include/jemalloc/internal/prof.h

@@ -237,7 +237,7 @@ void	prof_postfork_child(void);
 									\
 	assert(size == s2u(size));					\
 									\
-	prof_tdata = prof_tdata_get();					\
+	prof_tdata = prof_tdata_get(true);				\
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {	\
 		if (prof_tdata != NULL)					\
 			ret = (prof_thr_cnt_t *)(uintptr_t)1U;		\
@@ -286,7 +286,7 @@ void	prof_postfork_child(void);
 #ifndef JEMALLOC_ENABLE_INLINE
 malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
 
-prof_tdata_t	*prof_tdata_get(void);
+prof_tdata_t	*prof_tdata_get(bool create);
 void	prof_sample_threshold_update(prof_tdata_t *prof_tdata);
 prof_ctx_t	*prof_ctx_get(const void *ptr);
 void	prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
@@ -304,17 +304,15 @@ malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
     prof_tdata_cleanup)
 
 JEMALLOC_INLINE prof_tdata_t *
-prof_tdata_get(void)
+prof_tdata_get(bool create)
 {
 	prof_tdata_t *prof_tdata;
 
 	cassert(config_prof);
 
 	prof_tdata = *prof_tdata_tsd_get();
-	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {
+	if (create && prof_tdata == NULL)
-		if (prof_tdata == NULL)
+		prof_tdata = prof_tdata_init();
-			prof_tdata = prof_tdata_init();
-	}
 
 	return (prof_tdata);
 }
@@ -397,7 +395,7 @@ prof_sample_accum_update(size_t size)
 	/* Sampling logic is unnecessary if the interval is 1. */
 	assert(opt_lg_prof_sample != 0);
 
-	prof_tdata = *prof_tdata_tsd_get();
+	prof_tdata = prof_tdata_get(false);
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return (true);
 

include/jemalloc/internal/quarantine.h

@@ -1,6 +1,9 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
+typedef struct quarantine_obj_s quarantine_obj_t;
+typedef struct quarantine_s quarantine_t;
+
 /* Default per thread quarantine size if valgrind is enabled. */
 #define	JEMALLOC_VALGRIND_QUARANTINE_DEFAULT	(ZU(1) << 24)
 
@@ -8,17 +11,57 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+struct quarantine_obj_s {
+	void	*ptr;
+	size_t	usize;
+};
+
+struct quarantine_s {
+	size_t			curbytes;
+	size_t			curobjs;
+	size_t			first;
+#define	LG_MAXOBJS_INIT 10
+	size_t			lg_maxobjs;
+	quarantine_obj_t	objs[1]; /* Dynamically sized ring buffer. */
+};
+
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
+quarantine_t	*quarantine_init(size_t lg_maxobjs);
 void	quarantine(void *ptr);
+void	quarantine_cleanup(void *arg);
 bool	quarantine_boot(void);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_INLINES
 
+#ifndef JEMALLOC_ENABLE_INLINE
+malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *)
+
+void	quarantine_alloc_hook(void);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_))
+malloc_tsd_externs(quarantine, quarantine_t *)
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL,
+    quarantine_cleanup)
+
+JEMALLOC_ALWAYS_INLINE void
+quarantine_alloc_hook(void)
+{
+	quarantine_t *quarantine;
+
+	assert(config_fill && opt_quarantine);
+
+	quarantine = *quarantine_tsd_get();
+	if (quarantine == NULL)
+		quarantine_init(LG_MAXOBJS_INIT);
+}
+#endif
+
 #endif /* JEMALLOC_H_INLINES */
 /******************************************************************************/
 

include/jemalloc/internal/tcache.h

@@ -140,11 +140,11 @@ void	tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
 /* Map of thread-specific caches. */
 malloc_tsd_externs(tcache, tcache_t *)
-malloc_tsd_funcs(JEMALLOC_INLINE, tcache, tcache_t *, NULL,
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL,
     tcache_thread_cleanup)
 /* Per thread flag that allows thread caches to be disabled. */
 malloc_tsd_externs(tcache_enabled, tcache_enabled_t)
-malloc_tsd_funcs(JEMALLOC_INLINE, tcache_enabled, tcache_enabled_t,
+malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t,
     tcache_enabled_default, malloc_tsd_no_cleanup)
 
 JEMALLOC_INLINE void
@@ -206,7 +206,7 @@ tcache_enabled_set(bool enabled)
 	}
 }
 
-JEMALLOC_INLINE tcache_t *
+JEMALLOC_ALWAYS_INLINE tcache_t *
 tcache_get(bool create)
 {
 	tcache_t *tcache;
@@ -258,7 +258,7 @@ tcache_get(bool create)
 	return (tcache);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 tcache_event(tcache_t *tcache)
 {
 
@@ -271,7 +271,7 @@ tcache_event(tcache_t *tcache)
 		tcache_event_hard(tcache);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_easy(tcache_bin_t *tbin)
 {
 	void *ret;
@@ -287,7 +287,7 @@ tcache_alloc_easy(tcache_bin_t *tbin)
 	return (ret);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 {
 	void *ret;
@@ -321,6 +321,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
 	}
+	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 	if (config_stats)
 		tbin->tstats.nrequests++;
@@ -330,7 +331,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 	return (ret);
 }
 
-JEMALLOC_INLINE void *
+JEMALLOC_ALWAYS_INLINE void *
 tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 {
 	void *ret;
@@ -371,6 +372,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 			memset(ret, 0, size);
 		}
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 		if (config_stats)
 			tbin->tstats.nrequests++;
@@ -382,7 +384,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
 	return (ret);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind)
 {
 	tcache_bin_t *tbin;
@@ -406,7 +408,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind)
 	tcache_event(tcache);
 }
 
-JEMALLOC_INLINE void
+JEMALLOC_ALWAYS_INLINE void
 tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 {
 	size_t binind;

include/jemalloc/jemalloc_defs.h.in

@@ -249,6 +249,11 @@
 #undef JEMALLOC_PURGE_MADVISE_DONTNEED
 #undef JEMALLOC_PURGE_MADVISE_FREE
 
+/*
+ * Define if operating system has alloca.h header.
+ */
+#undef JEMALLOC_HAS_ALLOCA_H
+
 /* sizeof(void *) == 2^LG_SIZEOF_PTR. */
 #undef LG_SIZEOF_PTR
 

src/arena.c

@@ -40,6 +40,12 @@ const uint8_t	small_size2bin[] = {
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
+static void	arena_avail_insert(arena_t *arena, arena_chunk_t *chunk,
+    size_t pageind, size_t npages, bool maybe_adjac_pred,
+    bool maybe_adjac_succ);
+static void	arena_avail_remove(arena_t *arena, arena_chunk_t *chunk,
+    size_t pageind, size_t npages, bool maybe_adjac_pred,
+    bool maybe_adjac_succ);
 static void	arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
     bool large, size_t binind, bool zero);
 static arena_chunk_t *arena_chunk_alloc(arena_t *arena);
@@ -48,8 +54,11 @@ static arena_run_t	*arena_run_alloc_helper(arena_t *arena, size_t size,
     bool large, size_t binind, bool zero);
 static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large,
     size_t binind, bool zero);
+static arena_chunk_t	*chunks_dirty_iter_cb(arena_chunk_tree_t *tree,
+    arena_chunk_t *chunk, void *arg);
 static void	arena_purge(arena_t *arena, bool all);
-static void	arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
+static void	arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty,
+    bool cleaned);
 static void	arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
     arena_run_t *run, size_t oldsize, size_t newsize);
 static void	arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
@@ -101,9 +110,6 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
 	size_t a_size = a->bits & ~PAGE_MASK;
 	size_t b_size = b->bits & ~PAGE_MASK;
 
-	assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits &
-	    CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY));
-
 	ret = (a_size > b_size) - (a_size < b_size);
 	if (ret == 0) {
 		uintptr_t a_mapelm, b_mapelm;
@@ -129,6 +135,182 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
 rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t,
     u.rb_link, arena_avail_comp)
 
+static inline int
+arena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b)
+{
+
+	assert(a != NULL);
+	assert(b != NULL);
+
+	/*
+	 * Short-circuit for self comparison.  The following comparison code
+	 * would come to the same result, but at the cost of executing the slow
+	 * path.
+	 */
+	if (a == b)
+		return (0);
+
+	/*
+	 * Order such that chunks with higher fragmentation are "less than"
+	 * those with lower fragmentation -- purging order is from "least" to
+	 * "greatest".  Fragmentation is measured as:
+	 *
+	 *     mean current avail run size
+	 *   --------------------------------
+	 *   mean defragmented avail run size
+	 *
+	 *            navail
+	 *         -----------
+	 *         nruns_avail           nruns_avail-nruns_adjac
+	 * = ========================= = -----------------------
+	 *            navail                  nruns_avail
+	 *    -----------------------
+	 *    nruns_avail-nruns_adjac
+	 *
+	 * The following code multiplies away the denominator prior to
+	 * comparison, in order to avoid division.
+	 *
+	 */
+	{
+		size_t a_val = (a->nruns_avail - a->nruns_adjac) *
+		    b->nruns_avail;
+		size_t b_val = (b->nruns_avail - b->nruns_adjac) *
+		    a->nruns_avail;
+
+		if (a_val < b_val)
+			return (1);
+		if (a_val > b_val)
+			return (-1);
+	}
+	/*
+	 * Break ties by chunk address.  For fragmented chunks, report lower
+	 * addresses as "lower", so that fragmentation reduction happens first
+	 * at lower addresses.  However, use the opposite ordering for
+	 * unfragmented chunks, in order to increase the chances of
+	 * re-allocating dirty runs.
+	 */
+	{
+		uintptr_t a_chunk = (uintptr_t)a;
+		uintptr_t b_chunk = (uintptr_t)b;
+		int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk));
+		if (a->nruns_adjac == 0) {
+			assert(b->nruns_adjac == 0);
+			ret = -ret;
+		}
+		return (ret);
+	}
+}
+
+/* Generate red-black tree functions. */
+rb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t,
+    dirty_link, arena_chunk_dirty_comp)
+
+static inline bool
+arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind)
+{
+	bool ret;
+
+	if (pageind-1 < map_bias)
+		ret = false;
+	else {
+		ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0);
+		assert(ret == false || arena_mapbits_dirty_get(chunk,
+		    pageind-1) != arena_mapbits_dirty_get(chunk, pageind));
+	}
+	return (ret);
+}
+
+static inline bool
+arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages)
+{
+	bool ret;
+
+	if (pageind+npages == chunk_npages)
+		ret = false;
+	else {
+		assert(pageind+npages < chunk_npages);
+		ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0);
+		assert(ret == false || arena_mapbits_dirty_get(chunk, pageind)
+		    != arena_mapbits_dirty_get(chunk, pageind+npages));
+	}
+	return (ret);
+}
+
+static inline bool
+arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages)
+{
+
+	return (arena_avail_adjac_pred(chunk, pageind) ||
+	    arena_avail_adjac_succ(chunk, pageind, npages));
+}
+
+static void
+arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
+    size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ)
+{
+
+	assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
+	    LG_PAGE));
+
+	/*
+	 * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be
+	 * removed and reinserted even if the run to be inserted is clean.
+	 */
+	if (chunk->ndirty != 0)
+		arena_chunk_dirty_remove(&arena->chunks_dirty, chunk);
+
+	if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind))
+		chunk->nruns_adjac++;
+	if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages))
+		chunk->nruns_adjac++;
+	chunk->nruns_avail++;
+	assert(chunk->nruns_avail > chunk->nruns_adjac);
+
+	if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
+		arena->ndirty += npages;
+		chunk->ndirty += npages;
+	}
+	if (chunk->ndirty != 0)
+		arena_chunk_dirty_insert(&arena->chunks_dirty, chunk);
+
+	arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk,
+	    pageind));
+}
+
+static void
+arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
+    size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ)
+{
+
+	assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
+	    LG_PAGE));
+
+	/*
+	 * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be
+	 * removed and reinserted even if the run to be removed is clean.
+	 */
+	if (chunk->ndirty != 0)
+		arena_chunk_dirty_remove(&arena->chunks_dirty, chunk);
+
+	if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind))
+		chunk->nruns_adjac--;
+	if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages))
+		chunk->nruns_adjac--;
+	chunk->nruns_avail--;
+	assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail
+	    == 0 && chunk->nruns_adjac == 0));
+
+	if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
+		arena->ndirty -= npages;
+		chunk->ndirty -= npages;
+	}
+	if (chunk->ndirty != 0)
+		arena_chunk_dirty_insert(&arena->chunks_dirty, chunk);
+
+	arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk,
+	    pageind));
+}
+
 static inline void *
 arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info)
 {
@@ -177,11 +359,23 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr)
 }
 
 static inline void
-arena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
+arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)
+{
+
+	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
+	    LG_PAGE)), (npages << LG_PAGE));
+	memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0,
+	    (npages << LG_PAGE));
+}
+
+static inline void
+arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
 {
 	size_t i;
 	UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));
 
+	VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind <<
+	    LG_PAGE)), PAGE);
 	for (i = 0; i < PAGE / sizeof(size_t); i++)
 		assert(p[i] == 0);
 }
@@ -193,7 +387,6 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 	arena_chunk_t *chunk;
 	size_t run_ind, total_pages, need_pages, rem_pages, i;
 	size_t flag_dirty;
-	arena_avail_tree_t *runs_avail;
 
 	assert((large && binind == BININD_INVALID) || (large == false && binind
 	    != BININD_INVALID));
@@ -201,8 +394,6 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
 	run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
 	flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
-	runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty :
-	    &arena->runs_avail_clean;
 	total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>
 	    LG_PAGE;
 	assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) ==
@@ -212,7 +403,7 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 	assert(need_pages <= total_pages);
 	rem_pages = total_pages - need_pages;
 
-	arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, run_ind));
+	arena_avail_remove(arena, chunk, run_ind, total_pages, true, true);
 	if (config_stats) {
 		/*
 		 * Update stats_cactive if nactive is crossing a chunk
@@ -244,14 +435,8 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 			    arena_mapbits_unzeroed_get(chunk,
 			    run_ind+total_pages-1));
 		}
-		arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk,
+		arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages,
-		    run_ind+need_pages));
+		    false, true);
-	}
-
-	/* Update dirty page accounting. */
-	if (flag_dirty != 0) {
-		chunk->ndirty -= need_pages;
-		arena->ndirty -= need_pages;
 	}
 
 	/*
@@ -268,19 +453,10 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 				for (i = 0; i < need_pages; i++) {
 					if (arena_mapbits_unzeroed_get(chunk,
 					    run_ind+i) != 0) {
-						VALGRIND_MAKE_MEM_UNDEFINED(
+						arena_run_zero(chunk, run_ind+i,
-						    (void *)((uintptr_t)
+						    1);
-						    chunk + ((run_ind+i) <<
-						    LG_PAGE)), PAGE);
-						memset((void *)((uintptr_t)
-						    chunk + ((run_ind+i) <<
-						    LG_PAGE)), 0, PAGE);
 					} else if (config_debug) {
-						VALGRIND_MAKE_MEM_DEFINED(
+						arena_run_page_validate_zeroed(
-						    (void *)((uintptr_t)
-						    chunk + ((run_ind+i) <<
-						    LG_PAGE)), PAGE);
-						arena_chunk_validate_zeroed(
 						    chunk, run_ind+i);
 					}
 				}
@@ -289,11 +465,7 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 				 * The run is dirty, so all pages must be
 				 * zeroed.
 				 */
-				VALGRIND_MAKE_MEM_UNDEFINED((void
+				arena_run_zero(chunk, run_ind, need_pages);
-				    *)((uintptr_t)chunk + (run_ind <<
-				    LG_PAGE)), (need_pages << LG_PAGE));
-				memset((void *)((uintptr_t)chunk + (run_ind <<
-				    LG_PAGE)), 0, (need_pages << LG_PAGE));
 			}
 		}
 
@@ -319,22 +491,26 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 		 */
 		if (config_debug && flag_dirty == 0 &&
 		    arena_mapbits_unzeroed_get(chunk, run_ind) == 0)
-			arena_chunk_validate_zeroed(chunk, run_ind);
+			arena_run_page_validate_zeroed(chunk, run_ind);
 		for (i = 1; i < need_pages - 1; i++) {
 			arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0);
 			if (config_debug && flag_dirty == 0 &&
-			    arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0)
+			    arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) {
-				arena_chunk_validate_zeroed(chunk, run_ind+i);
+				arena_run_page_validate_zeroed(chunk,
+				    run_ind+i);
+			}
 		}
 		arena_mapbits_small_set(chunk, run_ind+need_pages-1,
 		    need_pages-1, binind, flag_dirty);
 		if (config_debug && flag_dirty == 0 &&
 		    arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) ==
 		    0) {
-			arena_chunk_validate_zeroed(chunk,
+			arena_run_page_validate_zeroed(chunk,
 			    run_ind+need_pages-1);
 		}
 	}
+	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
+	    LG_PAGE)), (need_pages << LG_PAGE));
 }
 
 static arena_chunk_t *
@@ -344,8 +520,6 @@ arena_chunk_alloc(arena_t *arena)
 	size_t i;
 
 	if (arena->spare != NULL) {
-		arena_avail_tree_t *runs_avail;
-
 		chunk = arena->spare;
 		arena->spare = NULL;
 
@@ -357,14 +531,6 @@ arena_chunk_alloc(arena_t *arena)
 		    chunk_npages-1) == arena_maxclass);
 		assert(arena_mapbits_dirty_get(chunk, map_bias) ==
 		    arena_mapbits_dirty_get(chunk, chunk_npages-1));
-
-		/* Insert the run into the appropriate runs_avail_* tree. */
-		if (arena_mapbits_dirty_get(chunk, map_bias) == 0)
-			runs_avail = &arena->runs_avail_clean;
-		else
-			runs_avail = &arena->runs_avail_dirty;
-		arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk,
-		    map_bias));
 	} else {
 		bool zero;
 		size_t unzeroed;
@@ -380,8 +546,6 @@ arena_chunk_alloc(arena_t *arena)
 			arena->stats.mapped += chunksize;
 
 		chunk->arena = arena;
-		ql_elm_new(chunk, link_dirty);
-		chunk->dirtied = false;
 
 		/*
 		 * Claim that no pages are in use, since the header is merely
@@ -389,6 +553,9 @@ arena_chunk_alloc(arena_t *arena)
 		 */
 		chunk->ndirty = 0;
 
+		chunk->nruns_avail = 0;
+		chunk->nruns_adjac = 0;
+
 		/*
 		 * Initialize the map to contain one maximal free untouched run.
 		 * Mark the pages as zeroed iff chunk_alloc() returned a zeroed
@@ -405,6 +572,11 @@ arena_chunk_alloc(arena_t *arena)
 			for (i = map_bias+1; i < chunk_npages-1; i++)
 				arena_mapbits_unzeroed_set(chunk, i, unzeroed);
 		} else if (config_debug) {
+			VALGRIND_MAKE_MEM_DEFINED(
+			    (void *)arena_mapp_get(chunk, map_bias+1),
+			    (void *)((uintptr_t)
+			    arena_mapp_get(chunk, chunk_npages-1)
+			    - (uintptr_t)arena_mapp_get(chunk, map_bias+1)));
 			for (i = map_bias+1; i < chunk_npages-1; i++) {
 				assert(arena_mapbits_unzeroed_get(chunk, i) ==
 				    unzeroed);
@@ -412,20 +584,18 @@ arena_chunk_alloc(arena_t *arena)
 		}
 		arena_mapbits_unallocated_set(chunk, chunk_npages-1,
 		    arena_maxclass, unzeroed);
-
-		/* Insert the run into the runs_avail_clean tree. */
-		arena_avail_tree_insert(&arena->runs_avail_clean,
-		    arena_mapp_get(chunk, map_bias));
 	}
 
+	/* Insert the run into the runs_avail tree. */
+	arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias,
+	    false, false);
+
 	return (chunk);
 }
 
 static void
 arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
 {
-	arena_avail_tree_t *runs_avail;
-
 	assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
 	assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
 	assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
@@ -436,24 +606,16 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
 	    arena_mapbits_dirty_get(chunk, chunk_npages-1));
 
 	/*
-	 * Remove run from the appropriate runs_avail_* tree, so that the arena
+	 * Remove run from the runs_avail tree, so that the arena does not use
-	 * does not use it.
+	 * it.
 	 */
-	if (arena_mapbits_dirty_get(chunk, map_bias) == 0)
+	arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias,
-		runs_avail = &arena->runs_avail_clean;
+	    false, false);
-	else
-		runs_avail = &arena->runs_avail_dirty;
-	arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, map_bias));
 
 	if (arena->spare != NULL) {
 		arena_chunk_t *spare = arena->spare;
 
 		arena->spare = chunk;
-		if (spare->dirtied) {
-			ql_remove(&chunk->arena->chunks_dirty, spare,
-			    link_dirty);
-			arena->ndirty -= spare->ndirty;
-		}
 		malloc_mutex_unlock(&arena->lock);
 		chunk_dealloc((void *)spare, chunksize, true);
 		malloc_mutex_lock(&arena->lock);
@@ -471,19 +633,7 @@ arena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind,
 	arena_chunk_map_t *mapelm, key;
 
 	key.bits = size | CHUNK_MAP_KEY;
-	mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key);
+	mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
-	if (mapelm != NULL) {
-		arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm);
-		size_t pageind = (((uintptr_t)mapelm -
-		    (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t))
-		    + map_bias;
-
-		run = (arena_run_t *)((uintptr_t)run_chunk + (pageind <<
-		    LG_PAGE));
-		arena_run_split(arena, run, size, large, binind, zero);
-		return (run);
-	}
-	mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key);
 	if (mapelm != NULL) {
 		arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm);
 		size_t pageind = (((uintptr_t)mapelm -
@@ -537,29 +687,40 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind,
 static inline void
 arena_maybe_purge(arena_t *arena)
 {
+	size_t npurgeable, threshold;
 
-	/* Enforce opt_lg_dirty_mult. */
+	/* Don't purge if the option is disabled. */
-	if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory &&
+	if (opt_lg_dirty_mult < 0)
-	    (arena->ndirty - arena->npurgatory) > chunk_npages &&
+		return;
-	    (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty -
+	/* Don't purge if all dirty pages are already being purged. */
-	    arena->npurgatory))
+	if (arena->ndirty <= arena->npurgatory)
-		arena_purge(arena, false);
+		return;
+	npurgeable = arena->ndirty - arena->npurgatory;
+	threshold = (arena->nactive >> opt_lg_dirty_mult);
+	/*
+	 * Don't purge unless the number of purgeable pages exceeds the
+	 * threshold.
+	 */
+	if (npurgeable <= threshold)
+		return;
+
+	arena_purge(arena, false);
 }
 
-static inline void
+static inline size_t
-arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
+arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all)
 {
+	size_t npurged;
 	ql_head(arena_chunk_map_t) mapelms;
 	arena_chunk_map_t *mapelm;
-	size_t pageind;
+	size_t pageind, npages;
-	size_t ndirty;
 	size_t nmadvise;
 
 	ql_new(&mapelms);
 
 	/*
 	 * If chunk is the spare, temporarily re-allocate it, 1) so that its
-	 * run is reinserted into runs_avail_dirty, and 2) so that it cannot be
+	 * run is reinserted into runs_avail, and 2) so that it cannot be
 	 * completely discarded by another thread while arena->lock is dropped
 	 * by this thread.  Note that the arena_run_dalloc() call will
 	 * implicitly deallocate the chunk, so no explicit action is required
@@ -579,54 +740,50 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
 		arena_chunk_alloc(arena);
 	}
 
-	/* Temporarily allocate all free dirty runs within chunk. */
+	if (config_stats)
-	for (pageind = map_bias; pageind < chunk_npages;) {
+		arena->stats.purged += chunk->ndirty;
+
+	/*
+	 * Operate on all dirty runs if there is no clean/dirty run
+	 * fragmentation.
+	 */
+	if (chunk->nruns_adjac == 0)
+		all = true;
+
+	/*
+	 * Temporarily allocate free dirty runs within chunk.  If all is false,
+	 * only operate on dirty runs that are fragments; otherwise operate on
+	 * all dirty runs.
+	 */
+	for (pageind = map_bias; pageind < chunk_npages; pageind += npages) {
 		mapelm = arena_mapp_get(chunk, pageind);
 		if (arena_mapbits_allocated_get(chunk, pageind) == 0) {
-			size_t npages;
+			size_t run_size =
+			    arena_mapbits_unallocated_size_get(chunk, pageind);
 
-			npages = arena_mapbits_unallocated_size_get(chunk,
+			npages = run_size >> LG_PAGE;
-			    pageind) >> LG_PAGE;
 			assert(pageind + npages <= chunk_npages);
 			assert(arena_mapbits_dirty_get(chunk, pageind) ==
 			    arena_mapbits_dirty_get(chunk, pageind+npages-1));
-			if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
-				arena_avail_tree_remove(
-				    &arena->runs_avail_dirty, mapelm);
 
-				arena_mapbits_large_set(chunk, pageind,
+			if (arena_mapbits_dirty_get(chunk, pageind) != 0 &&
-				    (npages << LG_PAGE), 0);
+			    (all || arena_avail_adjac(chunk, pageind,
-				if (npages > 1) {
+			    npages))) {
-					arena_mapbits_large_set(chunk,
+				arena_run_t *run = (arena_run_t *)((uintptr_t)
-					    pageind+npages-1, 0, 0);
+				    chunk + (uintptr_t)(pageind << LG_PAGE));
-				}
 
-				if (config_stats) {
+				arena_run_split(arena, run, run_size, true,
-					/*
+				    BININD_INVALID, false);
-					 * Update stats_cactive if nactive is
-					 * crossing a chunk multiple.
-					 */
-					size_t cactive_diff =
-					    CHUNK_CEILING((arena->nactive +
-					    npages) << LG_PAGE) -
-					    CHUNK_CEILING(arena->nactive <<
-					    LG_PAGE);
-					if (cactive_diff != 0)
-						stats_cactive_add(cactive_diff);
-				}
-				arena->nactive += npages;
 				/* Append to list for later processing. */
 				ql_elm_new(mapelm, u.ql_link);
 				ql_tail_insert(&mapelms, mapelm, u.ql_link);
 			}
-
-			pageind += npages;
 		} else {
-			/* Skip allocated run. */
+			/* Skip run. */
-			if (arena_mapbits_large_get(chunk, pageind))
+			if (arena_mapbits_large_get(chunk, pageind) != 0) {
-				pageind += arena_mapbits_large_size_get(chunk,
+				npages = arena_mapbits_large_size_get(chunk,
 				    pageind) >> LG_PAGE;
-			else {
+			} else {
 				size_t binind;
 				arena_bin_info_t *bin_info;
 				arena_run_t *run = (arena_run_t *)((uintptr_t)
@@ -636,36 +793,27 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
 				    pageind) == 0);
 				binind = arena_bin_index(arena, run->bin);
 				bin_info = &arena_bin_info[binind];
-				pageind += bin_info->run_size >> LG_PAGE;
+				npages = bin_info->run_size >> LG_PAGE;
 			}
 		}
 	}
 	assert(pageind == chunk_npages);
-
+	assert(chunk->ndirty == 0 || all == false);
-	if (config_debug)
+	assert(chunk->nruns_adjac == 0);
-		ndirty = chunk->ndirty;
-	if (config_stats)
-		arena->stats.purged += chunk->ndirty;
-	arena->ndirty -= chunk->ndirty;
-	chunk->ndirty = 0;
-	ql_remove(&arena->chunks_dirty, chunk, link_dirty);
-	chunk->dirtied = false;
 
 	malloc_mutex_unlock(&arena->lock);
 	if (config_stats)
 		nmadvise = 0;
+	npurged = 0;
 	ql_foreach(mapelm, &mapelms, u.ql_link) {
-		size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) /
-		    sizeof(arena_chunk_map_t)) + map_bias;
-		size_t npages = arena_mapbits_large_size_get(chunk, pageind) >>
-		    LG_PAGE;
 		bool unzeroed;
 		size_t flag_unzeroed, i;
 
+		pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) /
+		    sizeof(arena_chunk_map_t)) + map_bias;
+		npages = arena_mapbits_large_size_get(chunk, pageind) >>
+		    LG_PAGE;
 		assert(pageind + npages <= chunk_npages);
-		assert(ndirty >= npages);
-		if (config_debug)
-			ndirty -= npages;
 		unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind <<
 		    LG_PAGE)), (npages << LG_PAGE));
 		flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0;
@@ -683,10 +831,10 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
 			arena_mapbits_unzeroed_set(chunk, pageind+i,
 			    flag_unzeroed);
 		}
+		npurged += npages;
 		if (config_stats)
 			nmadvise++;
 	}
-	assert(ndirty == 0);
 	malloc_mutex_lock(&arena->lock);
 	if (config_stats)
 		arena->stats.nmadvise += nmadvise;
@@ -694,14 +842,27 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
 	/* Deallocate runs. */
 	for (mapelm = ql_first(&mapelms); mapelm != NULL;
 	    mapelm = ql_first(&mapelms)) {
-		size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) /
+		arena_run_t *run;
-		    sizeof(arena_chunk_map_t)) + map_bias;
-		arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
-		    (uintptr_t)(pageind << LG_PAGE));
 
+		pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) /
+		    sizeof(arena_chunk_map_t)) + map_bias;
+		run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind <<
+		    LG_PAGE));
 		ql_remove(&mapelms, mapelm, u.ql_link);
-		arena_run_dalloc(arena, run, false);
+		arena_run_dalloc(arena, run, false, true);
 	}
+
+	return (npurged);
+}
+
+static arena_chunk_t *
+chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg)
+{
+       size_t *ndirty = (size_t *)arg;
+
+       assert(chunk->ndirty != 0);
+       *ndirty += chunk->ndirty;
+       return (NULL);
 }
 
 static void
@@ -712,14 +873,11 @@ arena_purge(arena_t *arena, bool all)
 	if (config_debug) {
 		size_t ndirty = 0;
 
-		ql_foreach(chunk, &arena->chunks_dirty, link_dirty) {
+		arena_chunk_dirty_iter(&arena->chunks_dirty, NULL,
-		    assert(chunk->dirtied);
+		    chunks_dirty_iter_cb, (void *)&ndirty);
-		    ndirty += chunk->ndirty;
-		}
 		assert(ndirty == arena->ndirty);
 	}
 	assert(arena->ndirty > arena->npurgatory || all);
-	assert(arena->ndirty - arena->npurgatory > chunk_npages || all);
 	assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty -
 	    arena->npurgatory) || all);
 
@@ -731,16 +889,24 @@ arena_purge(arena_t *arena, bool all)
 	 * purge, and add the result to arena->npurgatory.  This will keep
 	 * multiple threads from racing to reduce ndirty below the threshold.
 	 */
-	npurgatory = arena->ndirty - arena->npurgatory;
+	{
-	if (all == false) {
+		size_t npurgeable = arena->ndirty - arena->npurgatory;
-		assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult);
+
-		npurgatory -= arena->nactive >> opt_lg_dirty_mult;
+		if (all == false) {
+			size_t threshold = (arena->nactive >>
+			    opt_lg_dirty_mult);
+
+			npurgatory = npurgeable - threshold;
+		} else
+			npurgatory = npurgeable;
 	}
 	arena->npurgatory += npurgatory;
 
 	while (npurgatory > 0) {
+		size_t npurgeable, npurged, nunpurged;
+
 		/* Get next chunk with dirty pages. */
-		chunk = ql_first(&arena->chunks_dirty);
+		chunk = arena_chunk_dirty_first(&arena->chunks_dirty);
 		if (chunk == NULL) {
 			/*
 			 * This thread was unable to purge as many pages as
@@ -751,23 +917,15 @@ arena_purge(arena_t *arena, bool all)
 			arena->npurgatory -= npurgatory;
 			return;
 		}
-		while (chunk->ndirty == 0) {
+		npurgeable = chunk->ndirty;
-			ql_remove(&arena->chunks_dirty, chunk, link_dirty);
+		assert(npurgeable != 0);
-			chunk->dirtied = false;
-			chunk = ql_first(&arena->chunks_dirty);
-			if (chunk == NULL) {
-				/* Same logic as for above. */
-				arena->npurgatory -= npurgatory;
-				return;
-			}
-		}
 
-		if (chunk->ndirty > npurgatory) {
+		if (npurgeable > npurgatory && chunk->nruns_adjac == 0) {
 			/*
-			 * This thread will, at a minimum, purge all the dirty
+			 * This thread will purge all the dirty pages in chunk,
-			 * pages in chunk, so set npurgatory to reflect this
+			 * so set npurgatory to reflect this thread's intent to
-			 * thread's commitment to purge the pages.  This tends
+			 * purge the pages.  This tends to reduce the chances
-			 * to reduce the chances of the following scenario:
+			 * of the following scenario:
 			 *
 			 * 1) This thread sets arena->npurgatory such that
 			 *    (arena->ndirty - arena->npurgatory) is at the
@@ -781,13 +939,20 @@ arena_purge(arena_t *arena, bool all)
 			 * because all of the purging work being done really
 			 * needs to happen.
 			 */
-			arena->npurgatory += chunk->ndirty - npurgatory;
+			arena->npurgatory += npurgeable - npurgatory;
-			npurgatory = chunk->ndirty;
+			npurgatory = npurgeable;
 		}
 
-		arena->npurgatory -= chunk->ndirty;
+		/*
-		npurgatory -= chunk->ndirty;
+		 * Keep track of how many pages are purgeable, versus how many
-		arena_chunk_purge(arena, chunk);
+		 * actually get purged, and adjust counters accordingly.
+		 */
+		arena->npurgatory -= npurgeable;
+		npurgatory -= npurgeable;
+		npurged = arena_chunk_purge(arena, chunk, all);
+		nunpurged = npurgeable - npurged;
+		arena->npurgatory += nunpurged;
+		npurgatory += nunpurged;
 	}
 }
 
@@ -801,11 +966,10 @@ arena_purge_all(arena_t *arena)
 }
 
 static void
-arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
+arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned)
 {
 	arena_chunk_t *chunk;
 	size_t size, run_ind, run_pages, flag_dirty;
-	arena_avail_tree_t *runs_avail;
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
 	run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
@@ -836,15 +1000,14 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
 
 	/*
 	 * The run is dirty if the caller claims to have dirtied it, as well as
-	 * if it was already dirty before being allocated.
+	 * if it was already dirty before being allocated and the caller
+	 * doesn't claim to have cleaned it.
 	 */
 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==
 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
-	if (arena_mapbits_dirty_get(chunk, run_ind) != 0)
+	if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0)
 		dirty = true;
 	flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0;
-	runs_avail = dirty ? &arena->runs_avail_dirty :
-	    &arena->runs_avail_clean;
 
 	/* Mark pages as unallocated in the chunk map. */
 	if (dirty) {
@@ -852,9 +1015,6 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
 		    CHUNK_MAP_DIRTY);
 		arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
 		    CHUNK_MAP_DIRTY);
-
-		chunk->ndirty += run_pages;
-		arena->ndirty += run_pages;
 	} else {
 		arena_mapbits_unallocated_set(chunk, run_ind, size,
 		    arena_mapbits_unzeroed_get(chunk, run_ind));
@@ -878,8 +1038,8 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
 		    run_ind+run_pages+nrun_pages-1) == nrun_size);
 		assert(arena_mapbits_dirty_get(chunk,
 		    run_ind+run_pages+nrun_pages-1) == flag_dirty);
-		arena_avail_tree_remove(runs_avail,
+		arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages,
-		    arena_mapp_get(chunk, run_ind+run_pages));
+		    false, true);
 
 		size += nrun_size;
 		run_pages += nrun_pages;
@@ -905,8 +1065,8 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
 		assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
 		    prun_size);
 		assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty);
-		arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk,
+		arena_avail_remove(arena, chunk, run_ind, prun_pages, true,
-		    run_ind));
+		    false);
 
 		size += prun_size;
 		run_pages += prun_pages;
@@ -921,19 +1081,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
 	    arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1));
 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==
 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
-	arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, run_ind));
+	arena_avail_insert(arena, chunk, run_ind, run_pages, true, true);
-
-	if (dirty) {
-		/*
-		 * Insert into chunks_dirty before potentially calling
-		 * arena_chunk_dealloc(), so that chunks_dirty and
-		 * arena->ndirty are consistent.
-		 */
-		if (chunk->dirtied == false) {
-			ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty);
-			chunk->dirtied = true;
-		}
-	}
 
 	/* Deallocate chunk if it is now completely unused. */
 	if (size == arena_maxclass) {
@@ -982,7 +1130,7 @@ arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
 	arena_mapbits_large_set(chunk, pageind+head_npages, newsize,
 	    flag_dirty);
 
-	arena_run_dalloc(arena, run, false);
+	arena_run_dalloc(arena, run, false, false);
 }
 
 static void
@@ -1015,7 +1163,7 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
 	    flag_dirty);
 
 	arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
-	    dirty);
+	    dirty, false);
 }
 
 static arena_run_t *
@@ -1101,8 +1249,6 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
 		    (uintptr_t)bin_info->bitmap_offset);
 
 		/* Initialize run internals. */
-		VALGRIND_MAKE_MEM_UNDEFINED(run, bin_info->reg0_offset -
-		    bin_info->redzone_size);
 		run->bin = bin;
 		run->nextind = 0;
 		run->nfree = bin_info->nregs;
@@ -1181,21 +1327,6 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
 	return (arena_run_reg_alloc(bin->runcur, bin_info));
 }
 
-void
-arena_prof_accum(arena_t *arena, uint64_t accumbytes)
-{
-
-	cassert(config_prof);
-
-	if (config_prof && prof_interval != 0) {
-		arena->prof_accumbytes += accumbytes;
-		if (arena->prof_accumbytes >= prof_interval) {
-			prof_idump();
-			arena->prof_accumbytes -= prof_interval;
-		}
-	}
-}
-
 void
 arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind,
     uint64_t prof_accumbytes)
@@ -1207,11 +1338,8 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind,
 
 	assert(tbin->ncached == 0);
 
-	if (config_prof) {
+	if (config_prof && arena_prof_accum(arena, prof_accumbytes))
-		malloc_mutex_lock(&arena->lock);
+		prof_idump();
-		arena_prof_accum(arena, prof_accumbytes);
-		malloc_mutex_unlock(&arena->lock);
-	}
 	bin = &arena->bins[binind];
 	malloc_mutex_lock(&bin->lock);
 	for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >>
@@ -1319,11 +1447,8 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 		bin->stats.nrequests++;
 	}
 	malloc_mutex_unlock(&bin->lock);
-	if (config_prof && isthreaded == false) {
+	if (config_prof && isthreaded == false && arena_prof_accum(arena, size))
-		malloc_mutex_lock(&arena->lock);
+		prof_idump();
-		arena_prof_accum(arena, size);
-		malloc_mutex_unlock(&arena->lock);
-	}
 
 	if (zero == false) {
 		if (config_fill) {
@@ -1341,6 +1466,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
 	}
+	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 	return (ret);
 }
@@ -1349,6 +1475,7 @@ void *
 arena_malloc_large(arena_t *arena, size_t size, bool zero)
 {
 	void *ret;
+	UNUSED bool idump;
 
 	/* Large allocation. */
 	size = PAGE_CEILING(size);
@@ -1367,8 +1494,10 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero)
 		arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++;
 	}
 	if (config_prof)
-		arena_prof_accum(arena, size);
+		idump = arena_prof_accum_locked(arena, size);
 	malloc_mutex_unlock(&arena->lock);
+	if (config_prof && idump)
+		prof_idump();
 
 	if (zero == false) {
 		if (config_fill) {
@@ -1526,7 +1655,7 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
 		    ((past - run_ind) << LG_PAGE), false);
 		/* npages = past - run_ind; */
 	}
-	arena_run_dalloc(arena, run, true);
+	arena_run_dalloc(arena, run, true, false);
 	malloc_mutex_unlock(&arena->lock);
 	/****************************/
 	malloc_mutex_lock(&bin->lock);
@@ -1638,7 +1767,7 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 		}
 	}
 
-	arena_run_dalloc(arena, (arena_run_t *)ptr, true);
+	arena_run_dalloc(arena, (arena_run_t *)ptr, true, false);
 }
 
 void
@@ -1985,15 +2114,14 @@ arena_new(arena_t *arena, unsigned ind)
 	arena->dss_prec = chunk_dss_prec_get();
 
 	/* Initialize chunks. */
-	ql_new(&arena->chunks_dirty);
+	arena_chunk_dirty_new(&arena->chunks_dirty);
 	arena->spare = NULL;
 
 	arena->nactive = 0;
 	arena->ndirty = 0;
 	arena->npurgatory = 0;
 
-	arena_avail_tree_new(&arena->runs_avail_clean);
+	arena_avail_tree_new(&arena->runs_avail);
-	arena_avail_tree_new(&arena->runs_avail_dirty);
 
 	/* Initialize bins. */
 	for (i = 0; i < NBINS; i++) {

src/base.c

@@ -63,6 +63,7 @@ base_alloc(size_t size)
 	ret = base_next_addr;
 	base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
 	malloc_mutex_unlock(&base_mtx);
+	VALGRIND_MAKE_MEM_UNDEFINED(ret, csize);
 
 	return (ret);
 }
@@ -88,6 +89,7 @@ base_node_alloc(void)
 		ret = base_nodes;
 		base_nodes = *(extent_node_t **)ret;
 		malloc_mutex_unlock(&base_mtx);
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, sizeof(extent_node_t));
 	} else {
 		malloc_mutex_unlock(&base_mtx);
 		ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
@@ -100,6 +102,7 @@ void
 base_node_dealloc(extent_node_t *node)
 {
 
+	VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t));
 	malloc_mutex_lock(&base_mtx);
 	*(extent_node_t **)node = base_nodes;
 	base_nodes = node;

src/chunk.c

@@ -78,6 +78,9 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
 	assert(node->size >= leadsize + size);
 	trailsize = node->size - leadsize - size;
 	ret = (void *)((uintptr_t)node->addr + leadsize);
+	zeroed = node->zeroed;
+	if (zeroed)
+	    *zero = true;
 	/* Remove node from the tree. */
 	extent_tree_szad_remove(chunks_szad, node);
 	extent_tree_ad_remove(chunks_ad, node);
@@ -108,23 +111,26 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
 		}
 		node->addr = (void *)((uintptr_t)(ret) + size);
 		node->size = trailsize;
+		node->zeroed = zeroed;
 		extent_tree_szad_insert(chunks_szad, node);
 		extent_tree_ad_insert(chunks_ad, node);
 		node = NULL;
 	}
 	malloc_mutex_unlock(&chunks_mtx);
 
-	zeroed = false;
+	if (node != NULL)
-	if (node != NULL) {
-		if (node->zeroed) {
-			zeroed = true;
-			*zero = true;
-		}
 		base_node_dealloc(node);
-	}
+	if (*zero) {
-	if (zeroed == false && *zero) {
+		if (zeroed == false)
-		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
+			memset(ret, 0, size);
-		memset(ret, 0, size);
+		else if (config_debug) {
+			size_t i;
+			size_t *p = (size_t *)(uintptr_t)ret;
+
+			VALGRIND_MAKE_MEM_DEFINED(ret, size);
+			for (i = 0; i < size / sizeof(size_t); i++)
+				assert(p[i] == 0);
+		}
 	}
 	return (ret);
 }
@@ -146,73 +152,58 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
 	assert(alignment != 0);
 	assert((alignment & chunksize_mask) == 0);
 
-	/*
-	 * Try to recycle an existing mapping.
-	 */
-
 	/* "primary" dss. */
-	if (config_dss && dss_prec == dss_prec_primary && (ret =
+	if (config_dss && dss_prec == dss_prec_primary) {
-	    chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment,
+		if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size,
-	    base, zero)) != NULL)
+		    alignment, base, zero)) != NULL)
-		goto label_return;
+			goto label_return;
+		if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL)
+			goto label_return;
+	}
 	/* mmap. */
 	if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size,
 	    alignment, base, zero)) != NULL)
 		goto label_return;
-	/* "secondary" dss. */
-	if (config_dss && dss_prec == dss_prec_secondary && (ret =
-	    chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment,
-	    base, zero)) != NULL)
-		goto label_return;
-
-	/*
-	 * Try to allocate a new mapping.
-	 */
-
-	/* "primary" dss. */
-	if (config_dss && dss_prec == dss_prec_primary && (ret =
-	    chunk_alloc_dss(size, alignment, zero)) != NULL)
-		goto label_return;
-	/* mmap. */
 	if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL)
 		goto label_return;
 	/* "secondary" dss. */
-	if (config_dss && dss_prec == dss_prec_secondary && (ret =
+	if (config_dss && dss_prec == dss_prec_secondary) {
-	    chunk_alloc_dss(size, alignment, zero)) != NULL)
+		if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size,
-		goto label_return;
+		    alignment, base, zero)) != NULL)
+			goto label_return;
+		if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL)
+			goto label_return;
+	}
 
 	/* All strategies for allocation failed. */
 	ret = NULL;
 label_return:
-	if (config_ivsalloc && base == false && ret != NULL) {
+	if (ret != NULL) {
-		if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) {
+		if (config_ivsalloc && base == false) {
-			chunk_dealloc(ret, size, true);
+			if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) {
-			return (NULL);
+				chunk_dealloc(ret, size, true);
+				return (NULL);
+			}
 		}
-	}
+		if (config_stats || config_prof) {
-	if ((config_stats || config_prof) && ret != NULL) {
+			bool gdump;
-		bool gdump;
+			malloc_mutex_lock(&chunks_mtx);
-		malloc_mutex_lock(&chunks_mtx);
+			if (config_stats)
-		if (config_stats)
+				stats_chunks.nchunks += (size / chunksize);
-			stats_chunks.nchunks += (size / chunksize);
+			stats_chunks.curchunks += (size / chunksize);
-		stats_chunks.curchunks += (size / chunksize);
+			if (stats_chunks.curchunks > stats_chunks.highchunks) {
-		if (stats_chunks.curchunks > stats_chunks.highchunks) {
+				stats_chunks.highchunks =
-			stats_chunks.highchunks = stats_chunks.curchunks;
+				    stats_chunks.curchunks;
-			if (config_prof)
+				if (config_prof)
-				gdump = true;
+					gdump = true;
-		} else if (config_prof)
+			} else if (config_prof)
-			gdump = false;
+				gdump = false;
-		malloc_mutex_unlock(&chunks_mtx);
+			malloc_mutex_unlock(&chunks_mtx);
-		if (config_prof && opt_prof && opt_prof_gdump && gdump)
+			if (config_prof && opt_prof && opt_prof_gdump && gdump)
-			prof_gdump();
+				prof_gdump();
-	}
+		}
-	if (config_debug && *zero && ret != NULL) {
+		if (config_valgrind)
-		size_t i;
+			VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
-		size_t *p = (size_t *)(uintptr_t)ret;
-
-		VALGRIND_MAKE_MEM_DEFINED(ret, size);
-		for (i = 0; i < size / sizeof(size_t); i++)
-			assert(p[i] == 0);
 	}
 	assert(CHUNK_ADDR2BASE(ret) == ret);
 	return (ret);
@@ -223,9 +214,10 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
     size_t size)
 {
 	bool unzeroed;
-	extent_node_t *xnode, *node, *prev, key;
+	extent_node_t *xnode, *node, *prev, *xprev, key;
 
 	unzeroed = pages_purge(chunk, size);
+	VALGRIND_MAKE_MEM_NOACCESS(chunk, size);
 
 	/*
 	 * Allocate a node before acquiring chunks_mtx even though it might not
@@ -234,6 +226,8 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 	 * held.
 	 */
 	xnode = base_node_alloc();
+	/* Use xprev to implement conditional deferred deallocation of prev. */
+	xprev = NULL;
 
 	malloc_mutex_lock(&chunks_mtx);
 	key.addr = (void *)((uintptr_t)chunk + size);
@@ -250,8 +244,6 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 		node->size += size;
 		node->zeroed = (node->zeroed && (unzeroed == false));
 		extent_tree_szad_insert(chunks_szad, node);
-		if (xnode != NULL)
-			base_node_dealloc(xnode);
 	} else {
 		/* Coalescing forward failed, so insert a new node. */
 		if (xnode == NULL) {
@@ -261,10 +253,10 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 			 * already been purged, so this is only a virtual
 			 * memory leak.
 			 */
-			malloc_mutex_unlock(&chunks_mtx);
+			goto label_return;
-			return;
 		}
 		node = xnode;
+		xnode = NULL; /* Prevent deallocation below. */
 		node->addr = chunk;
 		node->size = size;
 		node->zeroed = (unzeroed == false);
@@ -290,9 +282,19 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
 		node->zeroed = (node->zeroed && prev->zeroed);
 		extent_tree_szad_insert(chunks_szad, node);
 
-		base_node_dealloc(prev);
+		xprev = prev;
 	}
+
+label_return:
 	malloc_mutex_unlock(&chunks_mtx);
+	/*
+	 * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to
+	 * avoid potential deadlock.
+	 */
+	if (xnode != NULL)
+		base_node_dealloc(xnode);
+	if (xprev != NULL)
+		base_node_dealloc(prev);
 }
 
 void

src/ckh.c

@@ -70,20 +70,20 @@ ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key)
 JEMALLOC_INLINE size_t
 ckh_isearch(ckh_t *ckh, const void *key)
 {
-	size_t hash1, hash2, bucket, cell;
+	size_t hashes[2], bucket, cell;
 
 	assert(ckh != NULL);
 
-	ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
+	ckh->hash(key, hashes);
 
 	/* Search primary bucket. */
-	bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	cell = ckh_bucket_search(ckh, bucket, key);
 	if (cell != SIZE_T_MAX)
 		return (cell);
 
 	/* Search secondary bucket. */
-	bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	cell = ckh_bucket_search(ckh, bucket, key);
 	return (cell);
 }
@@ -126,7 +126,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 {
 	const void *key, *data, *tkey, *tdata;
 	ckhc_t *cell;
-	size_t hash1, hash2, bucket, tbucket;
+	size_t hashes[2], bucket, tbucket;
 	unsigned i;
 
 	bucket = argbucket;
@@ -155,10 +155,11 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 #endif
 
 		/* Find the alternate bucket for the evicted item. */
-		ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
+		ckh->hash(key, hashes);
-		tbucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+		tbucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 		if (tbucket == bucket) {
-			tbucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+			tbucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets)
+			    - 1);
 			/*
 			 * It may be that (tbucket == bucket) still, if the
 			 * item's hashes both indicate this bucket.  However,
@@ -192,19 +193,19 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
 JEMALLOC_INLINE bool
 ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata)
 {
-	size_t hash1, hash2, bucket;
+	size_t hashes[2], bucket;
 	const void *key = *argkey;
 	const void *data = *argdata;
 
-	ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2);
+	ckh->hash(key, hashes);
 
 	/* Try to insert in primary bucket. */
-	bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	if (ckh_try_bucket_insert(ckh, bucket, key, data) == false)
 		return (false);
 
 	/* Try to insert in secondary bucket. */
-	bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1);
+	bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1);
 	if (ckh_try_bucket_insert(ckh, bucket, key, data) == false)
 		return (false);
 
@@ -417,9 +418,8 @@ ckh_delete(ckh_t *ckh)
 #endif
 
 	idalloc(ckh->tab);
-#ifdef JEMALLOC_DEBUG
+	if (config_debug)
-	memset(ckh, 0x5a, sizeof(ckh_t));
+		memset(ckh, 0x5a, sizeof(ckh_t));
-#endif
 }
 
 size_t
@@ -526,31 +526,10 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data)
 }
 
 void
-ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2)
+ckh_string_hash(const void *key, size_t r_hash[2])
 {
-	size_t ret1, ret2;
-	uint64_t h;
 
-	assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64));
+	hash(key, strlen((const char *)key), 0x94122f33U, r_hash);
-	assert(hash1 != NULL);
-	assert(hash2 != NULL);
-
-	h = hash(key, strlen((const char *)key), UINT64_C(0x94122f335b332aea));
-	if (minbits <= 32) {
-		/*
-		 * Avoid doing multiple hashes, since a single hash provides
-		 * enough bits.
-		 */
-		ret1 = h & ZU(0xffffffffU);
-		ret2 = h >> 32;
-	} else {
-		ret1 = h;
-		ret2 = hash(key, strlen((const char *)key),
-		    UINT64_C(0x8432a476666bbc13));
-	}
-
-	*hash1 = ret1;
-	*hash2 = ret2;
 }
 
 bool
@@ -564,41 +543,16 @@ ckh_string_keycomp(const void *k1, const void *k2)
 }
 
 void
-ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1,
+ckh_pointer_hash(const void *key, size_t r_hash[2])
-    size_t *hash2)
 {
-	size_t ret1, ret2;
-	uint64_t h;
 	union {
 		const void	*v;
-		uint64_t	i;
+		size_t		i;
 	} u;
 
-	assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64));
-	assert(hash1 != NULL);
-	assert(hash2 != NULL);
-
 	assert(sizeof(u.v) == sizeof(u.i));
-#if (LG_SIZEOF_PTR != LG_SIZEOF_INT)
-	u.i = 0;
-#endif
 	u.v = key;
-	h = hash(&u.i, sizeof(u.i), UINT64_C(0xd983396e68886082));
+	hash(&u.i, sizeof(u.i), 0xd983396eU, r_hash);
-	if (minbits <= 32) {
-		/*
-		 * Avoid doing multiple hashes, since a single hash provides
-		 * enough bits.
-		 */
-		ret1 = h & ZU(0xffffffffU);
-		ret2 = h >> 32;
-	} else {
-		assert(SIZEOF_PTR == 8);
-		ret1 = h;
-		ret2 = hash(&u.i, sizeof(u.i), UINT64_C(0x5e2be9aff8709a5d));
-	}
-
-	*hash1 = ret1;
-	*hash2 = ret2;
 }
 
 bool

src/ctl.c

@@ -113,7 +113,7 @@ CTL_PROTO(opt_prof_final)
 CTL_PROTO(opt_prof_leak)
 CTL_PROTO(opt_prof_accum)
 CTL_PROTO(arena_i_purge)
-static int	arena_purge(unsigned arena_ind);
+static void	arena_purge(unsigned arena_ind);
 CTL_PROTO(arena_i_dss)
 INDEX_PROTO(arena_i)
 CTL_PROTO(arenas_bin_i_size)
@@ -960,11 +960,11 @@ ctl_postfork_child(void)
 		if (*oldlenp != sizeof(t)) {				\
 			size_t	copylen = (sizeof(t) <= *oldlenp)	\
 			    ? sizeof(t) : *oldlenp;			\
-			memcpy(oldp, (void *)&v, copylen);		\
+			memcpy(oldp, (void *)&(v), copylen);		\
 			ret = EINVAL;					\
 			goto label_return;				\
 		} else							\
-			*(t *)oldp = v;					\
+			*(t *)oldp = (v);				\
 	}								\
 } while (0)
 
@@ -974,7 +974,7 @@ ctl_postfork_child(void)
 			ret = EINVAL;					\
 			goto label_return;				\
 		}							\
-		v = *(t *)newp;						\
+		(v) = *(t *)newp;					\
 	}								\
 } while (0)
 
@@ -995,7 +995,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,	\
 	if (l)								\
 		malloc_mutex_lock(&ctl_mtx);				\
 	READONLY();							\
-	oldval = v;							\
+	oldval = (v);							\
 	READ(oldval, t);						\
 									\
 	ret = 0;							\
@@ -1017,7 +1017,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,	\
 		return (ENOENT);					\
 	malloc_mutex_lock(&ctl_mtx);					\
 	READONLY();							\
-	oldval = v;							\
+	oldval = (v);							\
 	READ(oldval, t);						\
 									\
 	ret = 0;							\
@@ -1036,7 +1036,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,	\
 									\
 	malloc_mutex_lock(&ctl_mtx);					\
 	READONLY();							\
-	oldval = v;							\
+	oldval = (v);							\
 	READ(oldval, t);						\
 									\
 	ret = 0;							\
@@ -1060,7 +1060,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,	\
 	if ((c) == false)						\
 		return (ENOENT);					\
 	READONLY();							\
-	oldval = v;							\
+	oldval = (v);							\
 	READ(oldval, t);						\
 									\
 	ret = 0;							\
@@ -1077,7 +1077,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,	\
 	t oldval;							\
 									\
 	READONLY();							\
-	oldval = v;							\
+	oldval = (v);							\
 	READ(oldval, t);						\
 									\
 	ret = 0;							\
@@ -1274,35 +1274,27 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
 
 /******************************************************************************/
 
-static int
+/* ctl_mutex must be held during execution of this function. */
+static void
 arena_purge(unsigned arena_ind)
 {
-	int ret;
+	VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
 
-	malloc_mutex_lock(&ctl_mtx);
+	malloc_mutex_lock(&arenas_lock);
-	{
+	memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas);
-		VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
+	malloc_mutex_unlock(&arenas_lock);
 
-		malloc_mutex_lock(&arenas_lock);
+	if (arena_ind == ctl_stats.narenas) {
-		memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas);
+		unsigned i;
-		malloc_mutex_unlock(&arenas_lock);
+		for (i = 0; i < ctl_stats.narenas; i++) {
-
+			if (tarenas[i] != NULL)
-		if (arena_ind == ctl_stats.narenas) {
+				arena_purge_all(tarenas[i]);
-			unsigned i;
-			for (i = 0; i < ctl_stats.narenas; i++) {
-				if (tarenas[i] != NULL)
-					arena_purge_all(tarenas[i]);
-			}
-		} else {
-			assert(arena_ind < ctl_stats.narenas);
-			if (tarenas[arena_ind] != NULL)
-				arena_purge_all(tarenas[arena_ind]);
 		}
+	} else {
+		assert(arena_ind < ctl_stats.narenas);
+		if (tarenas[arena_ind] != NULL)
+			arena_purge_all(tarenas[arena_ind]);
 	}
-
-	ret = 0;
-	malloc_mutex_unlock(&ctl_mtx);
-	return (ret);
 }
 
 static int
@@ -1313,8 +1305,11 @@ arena_i_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 
 	READONLY();
 	WRITEONLY();
-	ret = arena_purge(mib[1]);
+	malloc_mutex_lock(&ctl_mtx);
+	arena_purge(mib[1]);
+	malloc_mutex_unlock(&ctl_mtx);
 
+	ret = 0;
 label_return:
 	return (ret);
 }
@@ -1483,7 +1478,8 @@ arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 	else {
 		if (arena_ind == UINT_MAX)
 			arena_ind = ctl_stats.narenas;
-		ret = arena_purge(arena_ind);
+		arena_purge(arena_ind);
+		ret = 0;
 	}
 
 label_return:
@@ -1496,6 +1492,7 @@ arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
     void *newp, size_t newlen)
 {
 	int ret;
+	unsigned narenas;
 
 	malloc_mutex_lock(&ctl_mtx);
 	READONLY();
@@ -1503,7 +1500,8 @@ arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 		ret = EAGAIN;
 		goto label_return;
 	}
-	READ(ctl_stats.narenas - 1, unsigned);
+	narenas = ctl_stats.narenas - 1;
+	READ(narenas, unsigned);
 
 	ret = 0;
 label_return:

src/jemalloc.c

@@ -10,17 +10,20 @@ malloc_tsd_data(, thread_allocated, thread_allocated_t,
 
 /* Runtime configuration options. */
 const char	*je_malloc_conf;
+bool	opt_abort =
 #ifdef JEMALLOC_DEBUG
-bool	opt_abort = true;
+    true
-#  ifdef JEMALLOC_FILL
-bool	opt_junk = true;
-#  else
-bool	opt_junk = false;
-#  endif
 #else
-bool	opt_abort = false;
+    false
-bool	opt_junk = false;
 #endif
+    ;
+bool	opt_junk =
+#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL))
+    true
+#else
+    false
+#endif
+    ;
 size_t	opt_quarantine = ZU(0);
 bool	opt_redzone = false;
 bool	opt_utrace = false;
@@ -83,11 +86,13 @@ typedef struct {
 #ifdef JEMALLOC_UTRACE
 #  define UTRACE(a, b, c) do {						\
 	if (opt_utrace) {						\
+		int utrace_serrno = errno;				\
 		malloc_utrace_t ut;					\
 		ut.p = (a);						\
 		ut.s = (b);						\
 		ut.r = (c);						\
 		utrace(&ut, sizeof(ut));				\
+		errno = utrace_serrno;					\
 	}								\
 } while (0)
 #else
@@ -277,12 +282,30 @@ arenas_cleanup(void *arg)
 	malloc_mutex_unlock(&arenas_lock);
 }
 
-static inline bool
+static JEMALLOC_ATTR(always_inline) void
+malloc_thread_init(void)
+{
+
+	/*
+	 * TSD initialization can't be safely done as a side effect of
+	 * deallocation, because it is possible for a thread to do nothing but
+	 * deallocate its TLS data via free(), in which case writing to TLS
+	 * would cause write-after-free memory corruption.  The quarantine
+	 * facility *only* gets used as a side effect of deallocation, so make
+	 * a best effort attempt at initializing its TSD by hooking all
+	 * allocation events.
+	 */
+	if (config_fill && opt_quarantine)
+		quarantine_alloc_hook();
+}
+
+static JEMALLOC_ATTR(always_inline) bool
 malloc_init(void)
 {
 
-	if (malloc_initialized == false)
+	if (malloc_initialized == false && malloc_init_hard())
-		return (malloc_init_hard());
+		return (true);
+	malloc_thread_init();
 
 	return (false);
 }
@@ -469,7 +492,7 @@ malloc_conf_init(void)
 
 		while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v,
 		    &vlen) == false) {
-#define	CONF_HANDLE_BOOL_HIT(o, n, hit)					\
+#define	CONF_HANDLE_BOOL(o, n)						\
 			if (sizeof(n)-1 == klen && strncmp(n, k,	\
 			    klen) == 0) {				\
 				if (strncmp("true", v, vlen) == 0 &&	\
@@ -483,16 +506,9 @@ malloc_conf_init(void)
 					    "Invalid conf value",	\
 					    k, klen, v, vlen);		\
 				}					\
-				hit = true;				\
-			} else						\
-				hit = false;
-#define	CONF_HANDLE_BOOL(o, n) {					\
-			bool hit;					\
-			CONF_HANDLE_BOOL_HIT(o, n, hit);		\
-			if (hit)					\
 				continue;				\
-}
+			}
-#define	CONF_HANDLE_SIZE_T(o, n, min, max)				\
+#define	CONF_HANDLE_SIZE_T(o, n, min, max, clip)			\
 			if (sizeof(n)-1 == klen && strncmp(n, k,	\
 			    klen) == 0) {				\
 				uintmax_t um;				\
@@ -505,12 +521,22 @@ malloc_conf_init(void)
 					malloc_conf_error(		\
 					    "Invalid conf value",	\
 					    k, klen, v, vlen);		\
-				} else if (um < min || um > max) {	\
+				} else if (clip) {			\
-					malloc_conf_error(		\
+					if (um < min)			\
-					    "Out-of-range conf value",	\
+						o = min;		\
-					    k, klen, v, vlen);		\
+					else if (um > max)		\
-				} else					\
+						o = max;		\
-					o = um;				\
+					else				\
+						o = um;			\
+				} else {				\
+					if (um < min || um > max) {	\
+						malloc_conf_error(	\
+						    "Out-of-range "	\
+						    "conf value",	\
+						    k, klen, v, vlen);	\
+					} else				\
+						o = um;			\
+				}					\
 				continue;				\
 			}
 #define	CONF_HANDLE_SSIZE_T(o, n, min, max)				\
@@ -555,7 +581,8 @@ malloc_conf_init(void)
 			 * config_fill.
 			 */
 			CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
-			    (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1)
+			    (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1,
+			    true)
 			if (strncmp("dss", k, klen) == 0) {
 				int i;
 				bool match = false;
@@ -581,14 +608,14 @@ malloc_conf_init(void)
 				continue;
 			}
 			CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
-			    SIZE_T_MAX)
+			    SIZE_T_MAX, false)
 			CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
 			    -1, (sizeof(size_t) << 3) - 1)
 			CONF_HANDLE_BOOL(opt_stats_print, "stats_print")
 			if (config_fill) {
 				CONF_HANDLE_BOOL(opt_junk, "junk")
 				CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
-				    0, SIZE_T_MAX)
+				    0, SIZE_T_MAX, false)
 				CONF_HANDLE_BOOL(opt_redzone, "redzone")
 				CONF_HANDLE_BOOL(opt_zero, "zero")
 			}
@@ -886,7 +913,7 @@ JEMALLOC_ATTR(nonnull(1))
  * Avoid any uncertainty as to how many backtrace frames to ignore in
  * PROF_ALLOC_PREP().
  */
-JEMALLOC_ATTR(noinline)
+JEMALLOC_NOINLINE
 #endif
 static int
 imemalign(void **memptr, size_t alignment, size_t size,
@@ -1086,6 +1113,7 @@ je_realloc(void *ptr, size_t size)
 	if (size == 0) {
 		if (ptr != NULL) {
 			/* realloc(ptr, 0) is equivalent to free(p). */
+			assert(malloc_initialized || IS_INITIALIZER);
 			if (config_prof) {
 				old_size = isalloc(ptr, true);
 				if (config_valgrind && opt_valgrind)
@@ -1111,6 +1139,7 @@ je_realloc(void *ptr, size_t size)
 
 	if (ptr != NULL) {
 		assert(malloc_initialized || IS_INITIALIZER);
+		malloc_thread_init();
 
 		if (config_prof) {
 			old_size = isalloc(ptr, true);
@@ -1314,6 +1343,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
 	size_t ret;
 
 	assert(malloc_initialized || IS_INITIALIZER);
+	malloc_thread_init();
 
 	if (config_ivsalloc)
 		ret = ivsalloc(ptr, config_prof);
@@ -1372,7 +1402,7 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
  */
 #ifdef JEMALLOC_EXPERIMENTAL
 
-JEMALLOC_INLINE void *
+static JEMALLOC_ATTR(always_inline) void *
 iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache,
     arena_t *arena)
 {
@@ -1488,6 +1518,7 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
 	assert(size != 0);
 	assert(SIZE_T_MAX - size >= extra);
 	assert(malloc_initialized || IS_INITIALIZER);
+	malloc_thread_init();
 
 	if (arena_ind != UINT_MAX) {
 		arena_chunk_t *chunk;
@@ -1602,6 +1633,7 @@ je_sallocm(const void *ptr, size_t *rsize, int flags)
 	size_t sz;
 
 	assert(malloc_initialized || IS_INITIALIZER);
+	malloc_thread_init();
 
 	if (config_ivsalloc)
 		sz = ivsalloc(ptr, config_prof);
@@ -1721,12 +1753,12 @@ _malloc_prefork(void)
 
 	/* Acquire all mutexes in a safe order. */
 	ctl_prefork();
+	prof_prefork();
 	malloc_mutex_prefork(&arenas_lock);
 	for (i = 0; i < narenas_total; i++) {
 		if (arenas[i] != NULL)
 			arena_prefork(arenas[i]);
 	}
-	prof_prefork();
 	chunk_prefork();
 	base_prefork();
 	huge_prefork();
@@ -1752,12 +1784,12 @@ _malloc_postfork(void)
 	huge_postfork_parent();
 	base_postfork_parent();
 	chunk_postfork_parent();
-	prof_postfork_parent();
 	for (i = 0; i < narenas_total; i++) {
 		if (arenas[i] != NULL)
 			arena_postfork_parent(arenas[i]);
 	}
 	malloc_mutex_postfork_parent(&arenas_lock);
+	prof_postfork_parent();
 	ctl_postfork_parent();
 }
 
@@ -1772,12 +1804,12 @@ jemalloc_postfork_child(void)
 	huge_postfork_child();
 	base_postfork_child();
 	chunk_postfork_child();
-	prof_postfork_child();
 	for (i = 0; i < narenas_total; i++) {
 		if (arenas[i] != NULL)
 			arena_postfork_child(arenas[i]);
 	}
 	malloc_mutex_postfork_child(&arenas_lock);
+	prof_postfork_child();
 	ctl_postfork_child();
 }
 

src/prof.c

@@ -26,7 +26,7 @@ bool		opt_prof_leak = false;
 bool		opt_prof_accum = false;
 char		opt_prof_prefix[PATH_MAX + 1];
 
-uint64_t	prof_interval;
+uint64_t	prof_interval = 0;
 bool		prof_promote;
 
 /*
@@ -90,8 +90,7 @@ static bool	prof_dump(bool propagate_err, const char *filename,
     bool leakcheck);
 static void	prof_dump_filename(char *filename, char v, int64_t vseq);
 static void	prof_fdump(void);
-static void	prof_bt_hash(const void *key, unsigned minbits, size_t *hash1,
+static void	prof_bt_hash(const void *key, size_t r_hash[2]);
-    size_t *hash2);
 static bool	prof_bt_keycomp(const void *k1, const void *k2);
 static malloc_mutex_t	*prof_ctx_mutex_choose(void);
 
@@ -439,7 +438,7 @@ prof_lookup(prof_bt_t *bt)
 
 	cassert(config_prof);
 
-	prof_tdata = prof_tdata_get();
+	prof_tdata = prof_tdata_get(false);
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return (NULL);
 
@@ -685,7 +684,7 @@ prof_ctx_destroy(prof_ctx_t *ctx)
 	 * avoid a race between the main body of prof_ctx_merge() and entry
 	 * into this function.
 	 */
-	prof_tdata = *prof_tdata_tsd_get();
+	prof_tdata = prof_tdata_get(false);
 	assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX);
 	prof_enter(prof_tdata);
 	malloc_mutex_lock(ctx->lock);
@@ -845,7 +844,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
 
 	cassert(config_prof);
 
-	prof_tdata = prof_tdata_get();
+	prof_tdata = prof_tdata_get(false);
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return (true);
 	prof_enter(prof_tdata);
@@ -967,11 +966,7 @@ prof_idump(void)
 
 	if (prof_booted == false)
 		return;
-	/*
+	prof_tdata = prof_tdata_get(false);
-	 * Don't call prof_tdata_get() here, because it could cause recursive
-	 * allocation.
-	 */
-	prof_tdata = *prof_tdata_tsd_get();
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return;
 	if (prof_tdata->enq) {
@@ -1021,11 +1016,7 @@ prof_gdump(void)
 
 	if (prof_booted == false)
 		return;
-	/*
+	prof_tdata = prof_tdata_get(false);
-	 * Don't call prof_tdata_get() here, because it could cause recursive
-	 * allocation.
-	 */
-	prof_tdata = *prof_tdata_tsd_get();
 	if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
 		return;
 	if (prof_tdata->enq) {
@@ -1043,34 +1034,13 @@ prof_gdump(void)
 }
 
 static void
-prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2)
+prof_bt_hash(const void *key, size_t r_hash[2])
 {
-	size_t ret1, ret2;
-	uint64_t h;
 	prof_bt_t *bt = (prof_bt_t *)key;
 
 	cassert(config_prof);
-	assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64));
-	assert(hash1 != NULL);
-	assert(hash2 != NULL);
 
-	h = hash(bt->vec, bt->len * sizeof(void *),
+	hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash);
-	    UINT64_C(0x94122f335b332aea));
-	if (minbits <= 32) {
-		/*
-		 * Avoid doing multiple hashes, since a single hash provides
-		 * enough bits.
-		 */
-		ret1 = h & ZU(0xffffffffU);
-		ret2 = h >> 32;
-	} else {
-		ret1 = h;
-		ret2 = hash(bt->vec, bt->len * sizeof(void *),
-		    UINT64_C(0x8432a476666bbc13));
-	}
-
-	*hash1 = ret1;
-	*hash2 = ret2;
 }
 
 static bool
@@ -1206,13 +1176,11 @@ prof_boot1(void)
 		 */
 		opt_prof = true;
 		opt_prof_gdump = false;
-		prof_interval = 0;
 	} else if (opt_prof) {
 		if (opt_lg_prof_interval >= 0) {
 			prof_interval = (((uint64_t)1U) <<
 			    opt_lg_prof_interval);
-		} else
+		}
-			prof_interval = 0;
 	}
 
 	prof_promote = (opt_prof && opt_lg_prof_sample > LG_PAGE);

src/quarantine.c

@@ -1,3 +1,4 @@
+#define	JEMALLOC_QUARANTINE_C_
 #include "jemalloc/internal/jemalloc_internal.h"
 
 /*
@@ -11,39 +12,18 @@
 /******************************************************************************/
 /* Data. */
 
-typedef struct quarantine_obj_s quarantine_obj_t;
+malloc_tsd_data(, quarantine, quarantine_t *, NULL)
-typedef struct quarantine_s quarantine_t;
-
-struct quarantine_obj_s {
-	void	*ptr;
-	size_t	usize;
-};
-
-struct quarantine_s {
-	size_t			curbytes;
-	size_t			curobjs;
-	size_t			first;
-#define	LG_MAXOBJS_INIT 10
-	size_t			lg_maxobjs;
-	quarantine_obj_t	objs[1]; /* Dynamically sized ring buffer. */
-};
-
-static void	quarantine_cleanup(void *arg);
-
-malloc_tsd_data(static, quarantine, quarantine_t *, NULL)
-malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL,
-    quarantine_cleanup)
 
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static quarantine_t	*quarantine_init(size_t lg_maxobjs);
 static quarantine_t	*quarantine_grow(quarantine_t *quarantine);
+static void	quarantine_drain_one(quarantine_t *quarantine);
 static void	quarantine_drain(quarantine_t *quarantine, size_t upper_bound);
 
 /******************************************************************************/
 
-static quarantine_t *
+quarantine_t *
 quarantine_init(size_t lg_maxobjs)
 {
 	quarantine_t *quarantine;
@@ -68,8 +48,10 @@ quarantine_grow(quarantine_t *quarantine)
 	quarantine_t *ret;
 
 	ret = quarantine_init(quarantine->lg_maxobjs + 1);
-	if (ret == NULL)
+	if (ret == NULL) {
+		quarantine_drain_one(quarantine);
 		return (quarantine);
+	}
 
 	ret->curbytes = quarantine->curbytes;
 	ret->curobjs = quarantine->curobjs;
@@ -89,23 +71,29 @@ quarantine_grow(quarantine_t *quarantine)
 		memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
 		    sizeof(quarantine_obj_t));
 	}
+	idalloc(quarantine);
 
 	return (ret);
 }
 
+static void
+quarantine_drain_one(quarantine_t *quarantine)
+{
+	quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
+	assert(obj->usize == isalloc(obj->ptr, config_prof));
+	idalloc(obj->ptr);
+	quarantine->curbytes -= obj->usize;
+	quarantine->curobjs--;
+	quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
+	    quarantine->lg_maxobjs) - 1);
+}
+
 static void
 quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
 {
 
-	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) {
+	while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0)
-		quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
+		quarantine_drain_one(quarantine);
-		assert(obj->usize == isalloc(obj->ptr, config_prof));
-		idalloc(obj->ptr);
-		quarantine->curbytes -= obj->usize;
-		quarantine->curobjs--;
-		quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
-		    quarantine->lg_maxobjs) - 1);
-	}
 }
 
 void
@@ -119,24 +107,16 @@ quarantine(void *ptr)
 
 	quarantine = *quarantine_tsd_get();
 	if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
-		if (quarantine == NULL) {
+		if (quarantine == QUARANTINE_STATE_PURGATORY) {
-			if ((quarantine = quarantine_init(LG_MAXOBJS_INIT)) ==
+			/*
-			    NULL) {
+			 * Make a note that quarantine() was called after
-				idalloc(ptr);
+			 * quarantine_cleanup() was called.
-				return;
+			 */
-			}
+			quarantine = QUARANTINE_STATE_REINCARNATED;
-		} else {
+			quarantine_tsd_set(&quarantine);
-			if (quarantine == QUARANTINE_STATE_PURGATORY) {
-				/*
-				 * Make a note that quarantine() was called
-				 * after quarantine_cleanup() was called.
-				 */
-				quarantine = QUARANTINE_STATE_REINCARNATED;
-				quarantine_tsd_set(&quarantine);
-			}
-			idalloc(ptr);
-			return;
 		}
+		idalloc(ptr);
+		return;
 	}
 	/*
 	 * Drain one or more objects if the quarantine size limit would be
@@ -169,7 +149,7 @@ quarantine(void *ptr)
 	}
 }
 
-static void
+void
 quarantine_cleanup(void *arg)
 {
 	quarantine_t *quarantine = *(quarantine_t **)arg;

src/tcache.c

@@ -97,9 +97,8 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
 		arena_bin_t *bin = &arena->bins[binind];
 
 		if (config_prof && arena == tcache->arena) {
-			malloc_mutex_lock(&arena->lock);
+			if (arena_prof_accum(arena, tcache->prof_accumbytes))
-			arena_prof_accum(arena, tcache->prof_accumbytes);
+				prof_idump();
-			malloc_mutex_unlock(&arena->lock);
 			tcache->prof_accumbytes = 0;
 		}
 
@@ -176,11 +175,14 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
 		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
 		    tbin->avail[0]);
 		arena_t *arena = chunk->arena;
+		UNUSED bool idump;
 
+		if (config_prof)
+			idump = false;
 		malloc_mutex_lock(&arena->lock);
 		if ((config_prof || config_stats) && arena == tcache->arena) {
 			if (config_prof) {
-				arena_prof_accum(arena,
+				idump = arena_prof_accum_locked(arena,
 				    tcache->prof_accumbytes);
 				tcache->prof_accumbytes = 0;
 			}
@@ -212,6 +214,8 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
 			}
 		}
 		malloc_mutex_unlock(&arena->lock);
+		if (config_prof && idump)
+			prof_idump();
 	}
 	if (config_stats && merged_stats == false) {
 		/*
@@ -343,11 +347,9 @@ tcache_destroy(tcache_t *tcache)
 		}
 	}
 
-	if (config_prof && tcache->prof_accumbytes > 0) {
+	if (config_prof && tcache->prof_accumbytes > 0 &&
-		malloc_mutex_lock(&tcache->arena->lock);
+	    arena_prof_accum(tcache->arena, tcache->prof_accumbytes))
-		arena_prof_accum(tcache->arena, tcache->prof_accumbytes);
+		prof_idump();
-		malloc_mutex_unlock(&tcache->arena->lock);
-	}
 
 	tcache_size = arena_salloc(tcache, false);
 	if (tcache_size <= SMALL_MAXCLASS) {

src/zone.c

@@ -171,6 +171,16 @@ void
 register_zone(void)
 {
 
+	/*
+	 * If something else replaced the system default zone allocator, don't
+	 * register jemalloc's.
+	 */
+	malloc_zone_t *default_zone = malloc_default_zone();
+	if (!default_zone->zone_name ||
+	    strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) {
+		return;
+	}
+
 	zone.size = (void *)zone_size;
 	zone.malloc = (void *)zone_malloc;
 	zone.calloc = (void *)zone_calloc;
@@ -241,7 +251,7 @@ register_zone(void)
 	 * then becomes the default.
 	 */
 	do {
-		malloc_zone_t *default_zone = malloc_default_zone();
+		default_zone = malloc_default_zone();
 		malloc_zone_unregister(default_zone);
 		malloc_zone_register(default_zone);
 	} while (malloc_default_zone() != &zone);

test/ALLOCM_ARENA.c

@@ -41,6 +41,7 @@ je_thread_start(void *arg)
 		malloc_printf("Unexpected allocm() error\n");
 		abort();
 	}
+	dallocm(p, 0);
 
 	return (NULL);
 }

test/thread_arena.c

@@ -17,6 +17,7 @@ je_thread_start(void *arg)
 		malloc_printf("%s(): Error in malloc()\n", __func__);
 		return (void *)1;
 	}
+	free(p);
 
 	size = sizeof(arena_ind);
 	if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind,