Merge branch 'rc-4.5.0'

This avoids signed/unsigned comparison warnings when specifying integer
constants as inputs.

Clean up whitespace and add clarifying parentheses for
CONF_HANDLE_SIZE_T(opt_lg_chunk, ...).

.appveyor.yml

@@ -12,6 +12,20 @@ environment:
     CPU: x86_64
   - MSYSTEM: MINGW32
     CPU: i686
+  - MSYSTEM: MINGW64
+    CPU: x86_64
+    MSVC: amd64
+    CONFIG_FLAGS: --enable-debug
+  - MSYSTEM: MINGW32
+    CPU: i686
+    MSVC: x86
+    CONFIG_FLAGS: --enable-debug
+  - MSYSTEM: MINGW64
+    CPU: x86_64
+    CONFIG_FLAGS: --enable-debug
+  - MSYSTEM: MINGW32
+    CPU: i686
+    CONFIG_FLAGS: --enable-debug
 
 install:
   - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH%
@@ -21,7 +35,7 @@ install:
 
 build_script:
   - bash -c "autoconf"
-  - bash -c "./configure"
+  - bash -c "./configure $CONFIG_FLAGS"
   - mingw32-make -j3
   - file lib/jemalloc.dll
   - mingw32-make -j3 tests

.travis.yml

@@ -1,29 +1,95 @@
-language: c
+language: generic
 
 matrix:
   include:
     - os: linux
-      compiler: gcc
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS=""
+    - os: osx
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS=""
     - os: linux
-      compiler: gcc
-      env:
-        - EXTRA_FLAGS=-m32
+      env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS=""
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS=""
       addons:
         apt:
           packages:
-          - gcc-multilib
+            - gcc-multilib
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache"
     - os: osx
-      compiler: clang
+      env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS=""
     - os: osx
-      compiler: clang
-      env:
-        - EXTRA_FLAGS=-m32
+      env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS=""
+    - os: osx
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug"
+    - os: osx
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats"
+    - os: osx
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache"
+    - os: linux
+      env: CC=clang COMPILER_FLAGS="-m32" CONFIGURE_FLAGS=""
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+    - os: linux
+      env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug"
+    - os: linux
+      env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof"
+    - os: linux
+      env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats"
+    - os: linux
+      env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug"
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof"
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats"
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-tcache"
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-tcache"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-tcache"
+    - os: linux
+      env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-tcache"
+
 
 before_script:
   - autoconf
-  - ./configure${EXTRA_FLAGS:+ CC="$CC $EXTRA_FLAGS"}
+  - ./configure ${COMPILER_FLAGS:+       CC="$CC $COMPILER_FLAGS" }       $CONFIGURE_FLAGS
   - make -j3
   - make -j3 tests
 
 script:
   - make check
+

ChangeLog

@@ -4,6 +4,68 @@ brevity.  Much more detail can be found in the git revision history:
 
     https://github.com/jemalloc/jemalloc
 
+* 4.5.0 (February 28, 2017)
+
+  This is the first release to benefit from much broader continuous integration
+  testing, thanks to @davidtgoldblatt.  Had we had this testing infrastructure
+  in place for prior releases, it would have caught all of the most serious
+  regressions fixed by this release.
+
+  New features:
+  - Add --disable-thp and the opt.thp to provide opt-out mechanisms for
+    transparent huge page integration.  (@jasone)
+  - Update zone allocator integration to work with macOS 10.12.  (@glandium)
+  - Restructure *CFLAGS configuration, so that CFLAGS behaves typically, and
+    EXTRA_CFLAGS provides a way to specify e.g. -Werror during building, but not
+    during configuration.  (@jasone, @ronawho)
+
+  Bug fixes:
+  - Fix DSS (sbrk(2)-based) allocation.  This regression was first released in
+    4.3.0.  (@jasone)
+  - Handle race in per size class utilization computation.  This functionality
+    was first released in 4.0.0.  (@interwq)
+  - Fix lock order reversal during gdump.  (@jasone)
+  - Fix-refactor tcache synchronization.  This regression was first released in
+    4.0.0.  (@jasone)
+  - Fix various JSON-formatted malloc_stats_print() bugs.  This functionality
+    was first released in 4.3.0.  (@jasone)
+  - Fix huge-aligned allocation.  This regression was first released in 4.4.0.
+    (@jasone)
+  - When transparent huge page integration is enabled, detect what state pages
+    start in according to the kernel's current operating mode, and only convert
+    arena chunks to non-huge during purging if that is not their initial state.
+    This functionality was first released in 4.4.0.  (@jasone)
+  - Fix lg_chunk clamping for the --enable-cache-oblivious --disable-fill case.
+    This regression was first released in 4.0.0.  (@jasone, @428desmo)
+  - Properly detect sparc64 when building for Linux.  (@glaubitz)
+
+* 4.4.0 (December 3, 2016)
+
+  New features:
+  - Add configure support for *-*-linux-android.  (@cferris1000, @jasone)
+  - Add the --disable-syscall configure option, for use on systems that place
+    security-motivated limitations on syscall(2).  (@jasone)
+  - Add support for Debian GNU/kFreeBSD.  (@thesam)
+
+  Optimizations:
+  - Add extent serial numbers and use them where appropriate as a sort key that
+    is higher priority than address, so that the allocation policy prefers older
+    extents.  This tends to improve locality (decrease fragmentation) when
+    memory grows downward.  (@jasone)
+  - Refactor madvise(2) configuration so that MADV_FREE is detected and utilized
+    on Linux 4.5 and newer.  (@jasone)
+  - Mark partially purged arena chunks as non-huge-page.  This improves
+    interaction with Linux's transparent huge page functionality.  (@jasone)
+
+  Bug fixes:
+  - Fix size class computations for edge conditions involving extremely large
+    allocations.  This regression was first released in 4.0.0.  (@jasone,
+    @ingvarha)
+  - Remove overly restrictive assertions related to the cactive statistic.  This
+    regression was first released in 4.1.0.  (@jasone)
+  - Implement a more reliable detection scheme for os_unfair_lock on macOS.
+    (@jszakmeister)
+
 * 4.3.1 (November 7, 2016)
 
   Bug fixes:

INSTALL

@@ -157,6 +157,13 @@ any of the following arguments (not a definitive list) to 'configure':
     released in bulk, thus reducing the total number of mutex operations.  See
     the "opt.tcache" option for usage details.
 
+--disable-thp
+    Disable transparent huge page (THP) integration.  On systems with THP
+    support, THPs are explicitly disabled as a side effect of unused dirty page
+    purging for chunks that back small and/or large allocations, because such
+    chunks typically comprise active, unused dirty, and untouched clean
+    pages.
+
 --disable-munmap
     Disable virtual memory deallocation via munmap(2); instead keep track of
     the virtual memory for later use.  munmap() is disabled by default (i.e.
@@ -206,6 +213,11 @@ any of the following arguments (not a definitive list) to 'configure':
     most extreme case increases physical memory usage for the 16 KiB size class
     to 20 KiB.
 
+--disable-syscall
+    Disable use of syscall(2) rather than {open,read,write,close}(2).  This is
+    intended as a workaround for systems that place security limitations on
+    syscall(2).
+
 --with-xslroot=<path>
     Specify where to find DocBook XSL stylesheets when building the
     documentation.
@@ -301,17 +313,16 @@ The following environment variables (not a definitive list) impact configure's
 behavior:
 
 CFLAGS="?"
-    Pass these flags to the compiler.  You probably shouldn't define this unless
-    you know what you are doing.  (Use EXTRA_CFLAGS instead.)
+    Pass these flags to the C compiler.  Any flags set by the configure script
+    are prepended, which means explicitly set flags generally take precedence.
+    Take care when specifying flags such as -Werror, because configure tests may
+    be affected in undesirable ways.
 
 EXTRA_CFLAGS="?"
-    Append these flags to CFLAGS.  This makes it possible to add flags such as
-    -Werror, while allowing the configure script to determine what other flags
-    are appropriate for the specified configuration.
-
-    The configure script specifically checks whether an optimization flag (-O*)
-    is specified in EXTRA_CFLAGS, and refrains from specifying an optimization
-    level if it finds that one has already been specified.
+    Append these flags to CFLAGS, without passing them to the compiler during
+    configuration.  This makes it possible to add flags such as -Werror, while
+    allowing the configure script to determine what other flags are appropriate
+    for the specified configuration.
 
 CPPFLAGS="?"
     Pass these flags to the C preprocessor.  Note that CFLAGS is not passed to
@@ -327,6 +338,15 @@ LDFLAGS="?"
 PATH="?"
     'configure' uses this to find programs.
 
+In some cases it may be necessary to work around configuration results that do
+not match reality.  For example, Linux 4.5 added support for the MADV_FREE flag
+to madvise(2), which can cause problems if building on a host with MADV_FREE
+support and deploying to a target without.  To work around this, use a cache
+file to override the relevant configuration variable defined in configure.ac,
+e.g.:
+
+    echo "je_cv_madv_free=no" > config.cache && ./configure -C
+
 === Advanced compilation =======================================================
 
 To build only parts of jemalloc, use the following targets:

Makefile.in

@@ -24,8 +24,10 @@ abs_objroot := @abs_objroot@
 
 # Build parameters.
 CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include
+CONFIGURE_CFLAGS := @CONFIGURE_CFLAGS@
+SPECIFIED_CFLAGS := @SPECIFIED_CFLAGS@
 EXTRA_CFLAGS := @EXTRA_CFLAGS@
-CFLAGS := @CFLAGS@ $(EXTRA_CFLAGS)
+CFLAGS := $(strip $(CONFIGURE_CFLAGS) $(SPECIFIED_CFLAGS) $(EXTRA_CFLAGS))
 LDFLAGS := @LDFLAGS@
 EXTRA_LDFLAGS := @EXTRA_LDFLAGS@
 LIBS := @LIBS@
@@ -156,6 +158,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/bitmap.c \
 	$(srcroot)test/unit/ckh.c \
 	$(srcroot)test/unit/decay.c \
+	$(srcroot)test/unit/extent_quantize.c \
 	$(srcroot)test/unit/fork.c \
 	$(srcroot)test/unit/hash.c \
 	$(srcroot)test/unit/junk.c \
@@ -166,6 +169,8 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/math.c \
 	$(srcroot)test/unit/mq.c \
 	$(srcroot)test/unit/mtx.c \
+	$(srcroot)test/unit/pack.c \
+	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/ph.c \
 	$(srcroot)test/unit/prng.c \
 	$(srcroot)test/unit/prof_accum.c \
@@ -184,6 +189,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/size_classes.c \
 	$(srcroot)test/unit/smoothstep.c \
 	$(srcroot)test/unit/stats.c \
+	$(srcroot)test/unit/stats_print.c \
 	$(srcroot)test/unit/ticker.c \
 	$(srcroot)test/unit/nstime.c \
 	$(srcroot)test/unit/tsd.c \

build-aux/config.guess

@@ -1,8 +1,8 @@
 #! /bin/sh
 # Attempt to guess a canonical system name.
-#   Copyright 1992-2014 Free Software Foundation, Inc.
+#   Copyright 1992-2016 Free Software Foundation, Inc.
 
-timestamp='2014-03-23'
+timestamp='2016-10-02'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -24,12 +24,12 @@ timestamp='2014-03-23'
 # program.  This Exception is an additional permission under section 7
 # of the GNU General Public License, version 3 ("GPLv3").
 #
-# Originally written by Per Bothner.
+# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
 #
 # You can get the latest version of this script from:
-# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
 #
-# Please send patches with a ChangeLog entry to config-patches@gnu.org.
+# Please send patches to <config-patches@gnu.org>.
 
 
 me=`echo "$0" | sed -e 's,.*/,,'`
@@ -50,7 +50,7 @@ version="\
 GNU config.guess ($timestamp)
 
 Originally written by Per Bothner.
-Copyright 1992-2014 Free Software Foundation, Inc.
+Copyright 1992-2016 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -168,19 +168,29 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	# Note: NetBSD doesn't particularly care about the vendor
 	# portion of the name.  We always set it to "unknown".
 	sysctl="sysctl -n hw.machine_arch"
-	UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
-	    /usr/sbin/$sysctl 2>/dev/null || echo unknown)`
+	UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
+	    /sbin/$sysctl 2>/dev/null || \
+	    /usr/sbin/$sysctl 2>/dev/null || \
+	    echo unknown)`
 	case "${UNAME_MACHINE_ARCH}" in
 	    armeb) machine=armeb-unknown ;;
 	    arm*) machine=arm-unknown ;;
 	    sh3el) machine=shl-unknown ;;
 	    sh3eb) machine=sh-unknown ;;
 	    sh5el) machine=sh5le-unknown ;;
+	    earmv*)
+		arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
+		endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'`
+		machine=${arch}${endian}-unknown
+		;;
 	    *) machine=${UNAME_MACHINE_ARCH}-unknown ;;
 	esac
 	# The Operating System including object format, if it has switched
-	# to ELF recently, or will in the future.
+	# to ELF recently (or will in the future) and ABI.
 	case "${UNAME_MACHINE_ARCH}" in
+	    earm*)
+		os=netbsdelf
+		;;
 	    arm*|i386|m68k|ns32k|sh3*|sparc|vax)
 		eval $set_cc_for_build
 		if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
@@ -197,6 +207,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 		os=netbsd
 		;;
 	esac
+	# Determine ABI tags.
+	case "${UNAME_MACHINE_ARCH}" in
+	    earm*)
+		expr='s/^earmv[0-9]/-eabi/;s/eb$//'
+		abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"`
+		;;
+	esac
 	# The OS release
 	# Debian GNU/NetBSD machines have a different userland, and
 	# thus, need a distinct triplet. However, they do not need
@@ -207,13 +224,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 		release='-gnu'
 		;;
 	    *)
-		release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+		release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2`
 		;;
 	esac
 	# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
 	# contains redundant information, the shorter form:
 	# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
-	echo "${machine}-${os}${release}"
+	echo "${machine}-${os}${release}${abi}"
 	exit ;;
     *:Bitrig:*:*)
 	UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
@@ -223,6 +240,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
 	echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE}
 	exit ;;
+    *:LibertyBSD:*:*)
+	UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'`
+	echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE}
+	exit ;;
     *:ekkoBSD:*:*)
 	echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE}
 	exit ;;
@@ -235,6 +256,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
     *:MirBSD:*:*)
 	echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
 	exit ;;
+    *:Sortix:*:*)
+	echo ${UNAME_MACHINE}-unknown-sortix
+	exit ;;
     alpha:OSF1:*:*)
 	case $UNAME_RELEASE in
 	*4.0)
@@ -251,42 +275,42 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^  The alpha \(.*\) processor.*$/\1/p' | head -n 1`
 	case "$ALPHA_CPU_TYPE" in
 	    "EV4 (21064)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "EV4.5 (21064)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "LCA4 (21066/21068)")
-		UNAME_MACHINE="alpha" ;;
+		UNAME_MACHINE=alpha ;;
 	    "EV5 (21164)")
-		UNAME_MACHINE="alphaev5" ;;
+		UNAME_MACHINE=alphaev5 ;;
 	    "EV5.6 (21164A)")
-		UNAME_MACHINE="alphaev56" ;;
+		UNAME_MACHINE=alphaev56 ;;
 	    "EV5.6 (21164PC)")
-		UNAME_MACHINE="alphapca56" ;;
+		UNAME_MACHINE=alphapca56 ;;
 	    "EV5.7 (21164PC)")
-		UNAME_MACHINE="alphapca57" ;;
+		UNAME_MACHINE=alphapca57 ;;
 	    "EV6 (21264)")
-		UNAME_MACHINE="alphaev6" ;;
+		UNAME_MACHINE=alphaev6 ;;
 	    "EV6.7 (21264A)")
-		UNAME_MACHINE="alphaev67" ;;
+		UNAME_MACHINE=alphaev67 ;;
 	    "EV6.8CB (21264C)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.8AL (21264B)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.8CX (21264D)")
-		UNAME_MACHINE="alphaev68" ;;
+		UNAME_MACHINE=alphaev68 ;;
 	    "EV6.9A (21264/EV69A)")
-		UNAME_MACHINE="alphaev69" ;;
+		UNAME_MACHINE=alphaev69 ;;
 	    "EV7 (21364)")
-		UNAME_MACHINE="alphaev7" ;;
+		UNAME_MACHINE=alphaev7 ;;
 	    "EV7.9 (21364A)")
-		UNAME_MACHINE="alphaev79" ;;
+		UNAME_MACHINE=alphaev79 ;;
 	esac
 	# A Pn.n version is a patched version.
 	# A Vn.n version is a released version.
 	# A Tn.n version is a released field test version.
 	# A Xn.n version is an unreleased experimental baselevel.
 	# 1.2 uses "1.2" for uname -r.
-	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
+	echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
 	# Reset EXIT trap before exiting to avoid spurious non-zero exit code.
 	exitcode=$?
 	trap '' 0
@@ -359,16 +383,16 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	exit ;;
     i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*)
 	eval $set_cc_for_build
-	SUN_ARCH="i386"
+	SUN_ARCH=i386
 	# If there is a compiler, see if it is configured for 64-bit objects.
 	# Note that the Sun cc does not turn __LP64__ into 1 like gcc does.
 	# This test works for both compilers.
-	if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
+	if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
 	    if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \
-		(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
+		(CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
 		grep IS_64BIT_ARCH >/dev/null
 	    then
-		SUN_ARCH="x86_64"
+		SUN_ARCH=x86_64
 	    fi
 	fi
 	echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
@@ -393,7 +417,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
 	exit ;;
     sun*:*:4.2BSD:*)
 	UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null`
-	test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3
+	test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3
 	case "`/bin/arch`" in
 	    sun3)
 		echo m68k-sun-sunos${UNAME_RELEASE}
@@ -579,8 +603,9 @@ EOF
 	else
 		IBM_ARCH=powerpc
 	fi
-	if [ -x /usr/bin/oslevel ] ; then
-		IBM_REV=`/usr/bin/oslevel`
+	if [ -x /usr/bin/lslpp ] ; then
+		IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
+			   awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
 	else
 		IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
 	fi
@@ -617,13 +642,13 @@ EOF
 		    sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
 		    sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
 		    case "${sc_cpu_version}" in
-		      523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
-		      528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
+		      523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0
+		      528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1
 		      532)                      # CPU_PA_RISC2_0
 			case "${sc_kernel_bits}" in
-			  32) HP_ARCH="hppa2.0n" ;;
-			  64) HP_ARCH="hppa2.0w" ;;
-			  '') HP_ARCH="hppa2.0" ;;   # HP-UX 10.20
+			  32) HP_ARCH=hppa2.0n ;;
+			  64) HP_ARCH=hppa2.0w ;;
+			  '') HP_ARCH=hppa2.0 ;;   # HP-UX 10.20
 			esac ;;
 		    esac
 		fi
@@ -662,11 +687,11 @@ EOF
 		    exit (0);
 		}
 EOF
-		    (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
+		    (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy`
 		    test -z "$HP_ARCH" && HP_ARCH=hppa
 		fi ;;
 	esac
-	if [ ${HP_ARCH} = "hppa2.0w" ]
+	if [ ${HP_ARCH} = hppa2.0w ]
 	then
 	    eval $set_cc_for_build
 
@@ -679,12 +704,12 @@ EOF
 	    # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess
 	    # => hppa64-hp-hpux11.23
 
-	    if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) |
+	    if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) |
 		grep -q __LP64__
 	    then
-		HP_ARCH="hppa2.0w"
+		HP_ARCH=hppa2.0w
 	    else
-		HP_ARCH="hppa64"
+		HP_ARCH=hppa64
 	    fi
 	fi
 	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
@@ -789,14 +814,14 @@ EOF
 	echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
 	exit ;;
     F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
-	FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
-	FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+	FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`
+	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
 	FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
 	echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
 	exit ;;
     5000:UNIX_System_V:4.*:*)
-	FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
-	FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
+	FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'`
+	FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'`
 	echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
 	exit ;;
     i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
@@ -878,7 +903,7 @@ EOF
 	exit ;;
     *:GNU/*:*:*)
 	# other systems with GNU libc and userland
-	echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
+	echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
 	exit ;;
     i*86:Minix:*:*)
 	echo ${UNAME_MACHINE}-pc-minix
@@ -901,7 +926,7 @@ EOF
 	  EV68*) UNAME_MACHINE=alphaev68 ;;
 	esac
 	objdump --private-headers /bin/sh | grep -q ld.so.1
-	if test "$?" = 0 ; then LIBC="gnulibc1" ; fi
+	if test "$?" = 0 ; then LIBC=gnulibc1 ; fi
 	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
     arc:Linux:*:* | arceb:Linux:*:*)
@@ -932,6 +957,9 @@ EOF
     crisv32:Linux:*:*)
 	echo ${UNAME_MACHINE}-axis-linux-${LIBC}
 	exit ;;
+    e2k:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
     frv:Linux:*:*)
 	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
@@ -944,6 +972,9 @@ EOF
     ia64:Linux:*:*)
 	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
+    k1om:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
     m32r*:Linux:*:*)
 	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
 	exit ;;
@@ -969,6 +1000,9 @@ EOF
 	eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
 	test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; }
 	;;
+    mips64el:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
     openrisc*:Linux:*:*)
 	echo or1k-unknown-linux-${LIBC}
 	exit ;;
@@ -1001,6 +1035,9 @@ EOF
     ppcle:Linux:*:*)
 	echo powerpcle-unknown-linux-${LIBC}
 	exit ;;
+    riscv32:Linux:*:* | riscv64:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	exit ;;
     s390:Linux:*:* | s390x:Linux:*:*)
 	echo ${UNAME_MACHINE}-ibm-linux-${LIBC}
 	exit ;;
@@ -1020,7 +1057,7 @@ EOF
 	echo ${UNAME_MACHINE}-dec-linux-${LIBC}
 	exit ;;
     x86_64:Linux:*:*)
-	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+	echo ${UNAME_MACHINE}-pc-linux-${LIBC}
 	exit ;;
     xtensa*:Linux:*:*)
 	echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
@@ -1099,7 +1136,7 @@ EOF
 	# uname -m prints for DJGPP always 'pc', but it prints nothing about
 	# the processor, so we play safe by assuming i586.
 	# Note: whatever this is, it MUST be the same as what config.sub
-	# prints for the "djgpp" host, or else GDB configury will decide that
+	# prints for the "djgpp" host, or else GDB configure will decide that
 	# this is a cross-build.
 	echo i586-pc-msdosdjgpp
 	exit ;;
@@ -1248,6 +1285,9 @@ EOF
     SX-8R:SUPER-UX:*:*)
 	echo sx8r-nec-superux${UNAME_RELEASE}
 	exit ;;
+    SX-ACE:SUPER-UX:*:*)
+	echo sxace-nec-superux${UNAME_RELEASE}
+	exit ;;
     Power*:Rhapsody:*:*)
 	echo powerpc-apple-rhapsody${UNAME_RELEASE}
 	exit ;;
@@ -1261,9 +1301,9 @@ EOF
 	    UNAME_PROCESSOR=powerpc
 	fi
 	if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then
-	    if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
+	    if [ "$CC_FOR_BUILD" != no_compiler_found ]; then
 		if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
-		    (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
+		    (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \
 		    grep IS_64BIT_ARCH >/dev/null
 		then
 		    case $UNAME_PROCESSOR in
@@ -1285,7 +1325,7 @@ EOF
 	exit ;;
     *:procnto*:*:* | *:QNX:[0123456789]*:*)
 	UNAME_PROCESSOR=`uname -p`
-	if test "$UNAME_PROCESSOR" = "x86"; then
+	if test "$UNAME_PROCESSOR" = x86; then
 		UNAME_PROCESSOR=i386
 		UNAME_MACHINE=pc
 	fi
@@ -1316,7 +1356,7 @@ EOF
 	# "uname -m" is not consistent, so use $cputype instead. 386
 	# is converted to i386 for consistency with other x86
 	# operating systems.
-	if test "$cputype" = "386"; then
+	if test "$cputype" = 386; then
 	    UNAME_MACHINE=i386
 	else
 	    UNAME_MACHINE="$cputype"
@@ -1358,7 +1398,7 @@ EOF
 	echo i386-pc-xenix
 	exit ;;
     i*86:skyos:*:*)
-	echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//'
+	echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'`
 	exit ;;
     i*86:rdos:*:*)
 	echo ${UNAME_MACHINE}-pc-rdos
@@ -1369,23 +1409,25 @@ EOF
     x86_64:VMkernel:*:*)
 	echo ${UNAME_MACHINE}-unknown-esx
 	exit ;;
+    amd64:Isilon\ OneFS:*:*)
+	echo x86_64-unknown-onefs
+	exit ;;
 esac
 
 cat >&2 <<EOF
 $0: unable to guess system type
 
-This script, last modified $timestamp, has failed to recognize
-the operating system you are using. It is advised that you
-download the most up to date version of the config scripts from
+This script (version $timestamp), has failed to recognize the
+operating system you are using. If your script is old, overwrite
+config.guess and config.sub with the latest versions from:
 
-  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
+  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess
 and
-  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+  http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
 
-If the version you run ($0) is already up to date, please
-send the following data and any information you think might be
-pertinent to <config-patches@gnu.org> in order to provide the needed
-information to handle your system.
+If $0 has already been updated, send the following data and any
+information you think might be pertinent to config-patches@gnu.org to
+provide the necessary information to handle your system.
 
 config.guess timestamp = $timestamp
 

build-aux/config.sub

@@ -1,8 +1,8 @@
 #! /bin/sh
 # Configuration validation subroutine script.
-#   Copyright 1992-2014 Free Software Foundation, Inc.
+#   Copyright 1992-2016 Free Software Foundation, Inc.
 
-timestamp='2014-05-01'
+timestamp='2016-11-04'
 
 # This file is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -25,7 +25,7 @@ timestamp='2014-05-01'
 # of the GNU General Public License, version 3 ("GPLv3").
 
 
-# Please send patches with a ChangeLog entry to config-patches@gnu.org.
+# Please send patches to <config-patches@gnu.org>.
 #
 # Configuration subroutine to validate and canonicalize a configuration type.
 # Supply the specified configuration type as an argument.
@@ -33,7 +33,7 @@ timestamp='2014-05-01'
 # Otherwise, we print the canonical config type on stdout and succeed.
 
 # You can get the latest version of this script from:
-# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD
+# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub
 
 # This file is supposed to be the same for all GNU packages
 # and recognize all the CPU types, system types and aliases
@@ -53,8 +53,7 @@ timestamp='2014-05-01'
 me=`echo "$0" | sed -e 's,.*/,,'`
 
 usage="\
-Usage: $0 [OPTION] CPU-MFR-OPSYS
-       $0 [OPTION] ALIAS
+Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS
 
 Canonicalize a configuration name.
 
@@ -68,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>."
 version="\
 GNU config.sub ($timestamp)
 
-Copyright 1992-2014 Free Software Foundation, Inc.
+Copyright 1992-2016 Free Software Foundation, Inc.
 
 This is free software; see the source for copying conditions.  There is NO
 warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -117,8 +116,8 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
 case $maybe_os in
   nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
   linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
-  knetbsd*-gnu* | netbsd*-gnu* | \
-  kopensolaris*-gnu* | \
+  knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
+  kopensolaris*-gnu* | cloudabi*-eabi* | \
   storm-chaos* | os2-emx* | rtmk-nova*)
     os=-$maybe_os
     basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
@@ -255,12 +254,13 @@ case $basic_machine in
 	| arc | arceb \
 	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
 	| avr | avr32 \
+	| ba \
 	| be32 | be64 \
 	| bfin \
 	| c4x | c8051 | clipper \
 	| d10v | d30v | dlx | dsp16xx \
-	| epiphany \
-	| fido | fr30 | frv \
+	| e2k | epiphany \
+	| fido | fr30 | frv | ft32 \
 	| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
 	| hexagon \
 	| i370 | i860 | i960 | ia64 \
@@ -301,10 +301,12 @@ case $basic_machine in
 	| open8 | or1k | or1knd | or32 \
 	| pdp10 | pdp11 | pj | pjl \
 	| powerpc | powerpc64 | powerpc64le | powerpcle \
+	| pru \
 	| pyramid \
+	| riscv32 | riscv64 \
 	| rl78 | rx \
 	| score \
-	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
+	| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
 	| sh64 | sh64le \
 	| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \
 	| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
@@ -312,6 +314,7 @@ case $basic_machine in
 	| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
 	| ubicom32 \
 	| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
+	| visium \
 	| we32k \
 	| x86 | xc16x | xstormy16 | xtensa \
 	| z8k | z80)
@@ -326,6 +329,9 @@ case $basic_machine in
 	c6x)
 		basic_machine=tic6x-unknown
 		;;
+	leon|leon[3-9])
+		basic_machine=sparc-$basic_machine
+		;;
 	m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
 		basic_machine=$basic_machine-unknown
 		os=-none
@@ -371,12 +377,13 @@ case $basic_machine in
 	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
 	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
 	| avr-* | avr32-* \
+	| ba-* \
 	| be32-* | be64-* \
 	| bfin-* | bs2000-* \
 	| c[123]* | c30-* | [cjt]90-* | c4x-* \
 	| c8051-* | clipper-* | craynv-* | cydra-* \
 	| d10v-* | d30v-* | dlx-* \
-	| elxsi-* \
+	| e2k-* | elxsi-* \
 	| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
 	| h8300-* | h8500-* \
 	| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
@@ -422,13 +429,15 @@ case $basic_machine in
 	| orion-* \
 	| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
 	| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
+	| pru-* \
 	| pyramid-* \
+	| riscv32-* | riscv64-* \
 	| rl78-* | romp-* | rs6000-* | rx-* \
 	| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \
 	| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
 	| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \
 	| sparclite-* \
-	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
+	| sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \
 	| tahoe-* \
 	| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
 	| tile*-* \
@@ -436,6 +445,7 @@ case $basic_machine in
 	| ubicom32-* \
 	| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
 	| vax-* \
+	| visium-* \
 	| we32k-* \
 	| x86-* | x86_64-* | xc16x-* | xps100-* \
 	| xstormy16-* | xtensa*-* \
@@ -512,6 +522,9 @@ case $basic_machine in
 		basic_machine=i386-pc
 		os=-aros
 		;;
+	asmjs)
+		basic_machine=asmjs-unknown
+		;;
 	aux)
 		basic_machine=m68k-apple
 		os=-aux
@@ -632,6 +645,14 @@ case $basic_machine in
 		basic_machine=m68k-bull
 		os=-sysv3
 		;;
+	e500v[12])
+		basic_machine=powerpc-unknown
+		os=$os"spe"
+		;;
+	e500v[12]-*)
+		basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+		os=$os"spe"
+		;;
 	ebmon29k)
 		basic_machine=a29k-amd
 		os=-ebmon
@@ -773,6 +794,9 @@ case $basic_machine in
 		basic_machine=m68k-isi
 		os=-sysv
 		;;
+	leon-*|leon[3-9]-*)
+		basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'`
+		;;
 	m68knommu)
 		basic_machine=m68k-unknown
 		os=-linux
@@ -828,6 +852,10 @@ case $basic_machine in
 		basic_machine=powerpc-unknown
 		os=-morphos
 		;;
+	moxiebox)
+		basic_machine=moxie-unknown
+		os=-moxiebox
+		;;
 	msdos)
 		basic_machine=i386-pc
 		os=-msdos
@@ -1004,7 +1032,7 @@ case $basic_machine in
 	ppc-* | ppcbe-*)
 		basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
-	ppcle | powerpclittle | ppc-le | powerpc-little)
+	ppcle | powerpclittle)
 		basic_machine=powerpcle-unknown
 		;;
 	ppcle-* | powerpclittle-*)
@@ -1014,7 +1042,7 @@ case $basic_machine in
 		;;
 	ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'`
 		;;
-	ppc64le | powerpc64little | ppc64-le | powerpc64-little)
+	ppc64le | powerpc64little)
 		basic_machine=powerpc64le-unknown
 		;;
 	ppc64le-* | powerpc64little-*)
@@ -1360,27 +1388,28 @@ case $os in
 	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
 	      | -sym* | -kopensolaris* | -plan9* \
 	      | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
-	      | -aos* | -aros* \
+	      | -aos* | -aros* | -cloudabi* | -sortix* \
 	      | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
 	      | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
 	      | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
-	      | -bitrig* | -openbsd* | -solidbsd* \
+	      | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \
 	      | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
 	      | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
 	      | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
 	      | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
 	      | -chorusos* | -chorusrdb* | -cegcc* \
 	      | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
-	      | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
+	      | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
 	      | -linux-newlib* | -linux-musl* | -linux-uclibc* \
-	      | -uxpv* | -beos* | -mpeix* | -udk* \
+	      | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \
 	      | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
 	      | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
 	      | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
 	      | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
 	      | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
 	      | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
-	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*)
+	      | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \
+	      | -onefs* | -tirtos* | -phoenix* | -fuchsia*)
 	# Remember, each alternative MUST END IN *, to match a version number.
 		;;
 	-qnx*)
@@ -1404,9 +1433,6 @@ case $os in
 	-mac*)
 		os=`echo $os | sed -e 's|mac|macos|'`
 		;;
-	# Apple iOS
-	-ios*)
-		;;
 	-linux-dietlibc)
 		os=-linux-dietlibc
 		;;
@@ -1515,6 +1541,8 @@ case $os in
 		;;
 	-nacl*)
 		;;
+	-ios)
+		;;
 	-none)
 		;;
 	*)

configure.ac

@@ -6,29 +6,66 @@ AC_CONFIG_AUX_DIR([build-aux])
 dnl ============================================================================
 dnl Custom macro definitions.
 
-dnl JE_CFLAGS_APPEND(cflag)
-AC_DEFUN([JE_CFLAGS_APPEND],
+dnl JE_CONCAT_VVV(r, a, b)
+dnl 
+dnl Set $r to the concatenation of $a and $b, with a space separating them iff
+dnl both $a and $b are non-emty.
+AC_DEFUN([JE_CONCAT_VVV],
+if test "x[$]{$2}" = "x" -o "x[$]{$3}" = "x" ; then
+  $1="[$]{$2}[$]{$3}"
+else
+  $1="[$]{$2} [$]{$3}"
+fi
+)
+
+dnl JE_APPEND_VS(a, b)
+dnl 
+dnl Set $a to the concatenation of $a and b, with a space separating them iff
+dnl both $a and b are non-empty.
+AC_DEFUN([JE_APPEND_VS],
+  T_APPEND_V=$2
+  JE_CONCAT_VVV($1, $1, T_APPEND_V)
+)
+
+CONFIGURE_CFLAGS=
+SPECIFIED_CFLAGS="${CFLAGS}"
+dnl JE_CFLAGS_ADD(cflag)
+dnl 
+dnl CFLAGS is the concatenation of CONFIGURE_CFLAGS and SPECIFIED_CFLAGS
+dnl (ignoring EXTRA_CFLAGS, which does not impact configure tests.  This macro
+dnl appends to CONFIGURE_CFLAGS and regenerates CFLAGS.
+AC_DEFUN([JE_CFLAGS_ADD],
 [
 AC_MSG_CHECKING([whether compiler supports $1])
-TCFLAGS="${CFLAGS}"
-if test "x${CFLAGS}" = "x" ; then
-  CFLAGS="$1"
-else
-  CFLAGS="${CFLAGS} $1"
-fi
+T_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}"
+JE_APPEND_VS(CONFIGURE_CFLAGS, $1)
+JE_CONCAT_VVV(CFLAGS, CONFIGURE_CFLAGS, SPECIFIED_CFLAGS)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
 [[
 ]], [[
     return 0;
 ]])],
-              [je_cv_cflags_appended=$1]
+              [je_cv_cflags_added=$1]
               AC_MSG_RESULT([yes]),
-              [je_cv_cflags_appended=]
+              [je_cv_cflags_added=]
               AC_MSG_RESULT([no])
-              [CFLAGS="${TCFLAGS}"]
+              [CONFIGURE_CFLAGS="${T_CONFIGURE_CFLAGS}"]
 )
+JE_CONCAT_VVV(CFLAGS, CONFIGURE_CFLAGS, SPECIFIED_CFLAGS)
 ])
 
+dnl JE_CFLAGS_SAVE()
+dnl JE_CFLAGS_RESTORE()
+dnl 
+dnl Save/restore CFLAGS.  Nesting is not supported.
+AC_DEFUN([JE_CFLAGS_SAVE],
+SAVED_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}"
+)
+AC_DEFUN([JE_CFLAGS_RESTORE],
+CONFIGURE_CFLAGS="${SAVED_CONFIGURE_CFLAGS}"
+JE_CONCAT_VVV(CFLAGS, CONFIGURE_CFLAGS, SPECIFIED_CFLAGS)
+)
+
 dnl JE_COMPILABLE(label, hcode, mcode, rvar)
 dnl 
 dnl Use AC_LINK_IFELSE() rather than AC_COMPILE_IFELSE() so that linker errors
@@ -168,47 +205,45 @@ if test "x${je_cv_cray}" = "xyes" ; then
                               [je_cv_cray_84=no])])
 fi
 
-if test "x$CFLAGS" = "x" ; then
-  no_CFLAGS="yes"
-  if test "x$GCC" = "xyes" ; then
-dnl    JE_CFLAGS_APPEND([-std=gnu99])
-    JE_CFLAGS_APPEND([-std=gnu11])
-    if test "x$je_cv_cflags_appended" = "x-std=gnu11" ; then
+if test "x$GCC" = "xyes" ; then
+  JE_CFLAGS_ADD([-std=gnu11])
+  if test "x$je_cv_cflags_added" = "x-std=gnu11" ; then
+    AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT])
+  else
+    JE_CFLAGS_ADD([-std=gnu99])
+    if test "x$je_cv_cflags_added" = "x-std=gnu99" ; then
       AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT])
-    else
-      JE_CFLAGS_APPEND([-std=gnu99])
-      if test "x$je_cv_cflags_appended" = "x-std=gnu99" ; then
-        AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT])
-      fi
     fi
-    JE_CFLAGS_APPEND([-Wall])
-    JE_CFLAGS_APPEND([-Werror=declaration-after-statement])
-    JE_CFLAGS_APPEND([-Wshorten-64-to-32])
-    JE_CFLAGS_APPEND([-Wsign-compare])
-    JE_CFLAGS_APPEND([-pipe])
-    JE_CFLAGS_APPEND([-g3])
-  elif test "x$je_cv_msvc" = "xyes" ; then
-    CC="$CC -nologo"
-    JE_CFLAGS_APPEND([-Zi])
-    JE_CFLAGS_APPEND([-MT])
-    JE_CFLAGS_APPEND([-W3])
-    JE_CFLAGS_APPEND([-FS])
-    CPPFLAGS="$CPPFLAGS -I${srcdir}/include/msvc_compat"
   fi
-  if test "x$je_cv_cray" = "xyes" ; then
-    dnl cray compiler 8.4 has an inlining bug
-    if test "x$je_cv_cray_84" = "xyes" ; then
-      JE_CFLAGS_APPEND([-hipa2])
-      JE_CFLAGS_APPEND([-hnognu])
-    fi
-    if test "x$enable_cc_silence" != "xno" ; then
-      dnl ignore unreachable code warning
-      JE_CFLAGS_APPEND([-hnomessage=128])
-      dnl ignore redefinition of "malloc", "free", etc warning
-      JE_CFLAGS_APPEND([-hnomessage=1357])
-    fi
+  JE_CFLAGS_ADD([-Wall])
+  JE_CFLAGS_ADD([-Werror=declaration-after-statement])
+  JE_CFLAGS_ADD([-Wshorten-64-to-32])
+  JE_CFLAGS_ADD([-Wsign-compare])
+  JE_CFLAGS_ADD([-pipe])
+  JE_CFLAGS_ADD([-g3])
+elif test "x$je_cv_msvc" = "xyes" ; then
+  CC="$CC -nologo"
+  JE_CFLAGS_ADD([-Zi])
+  JE_CFLAGS_ADD([-MT])
+  JE_CFLAGS_ADD([-W3])
+  JE_CFLAGS_ADD([-FS])
+  JE_APPEND_VS(CPPFLAGS, -I${srcdir}/include/msvc_compat)
+fi
+if test "x$je_cv_cray" = "xyes" ; then
+  dnl cray compiler 8.4 has an inlining bug
+  if test "x$je_cv_cray_84" = "xyes" ; then
+    JE_CFLAGS_ADD([-hipa2])
+    JE_CFLAGS_ADD([-hnognu])
+  fi
+  if test "x$enable_cc_silence" != "xno" ; then
+    dnl ignore unreachable code warning
+    JE_CFLAGS_ADD([-hnomessage=128])
+    dnl ignore redefinition of "malloc", "free", etc warning
+    JE_CFLAGS_ADD([-hnomessage=1357])
   fi
 fi
+AC_SUBST([CONFIGURE_CFLAGS])
+AC_SUBST([SPECIFIED_CFLAGS])
 AC_SUBST([EXTRA_CFLAGS])
 AC_PROG_CPP
 
@@ -218,7 +253,7 @@ if test "x${ac_cv_big_endian}" = "x1" ; then
 fi
 
 if test "x${je_cv_msvc}" = "xyes" -a "x${ac_cv_header_inttypes_h}" = "xno"; then
-  CPPFLAGS="$CPPFLAGS -I${srcdir}/include/msvc_compat/C99"
+  JE_APPEND_VS(CPPFLAGS, -I${srcdir}/include/msvc_compat/C99)
 fi
 
 if test "x${je_cv_msvc}" = "xyes" ; then
@@ -349,13 +384,11 @@ dnl
 dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the
 dnl definitions need to be seen before any headers are included, which is a pain
 dnl to make happen otherwise.
-CFLAGS="$CFLAGS"
 default_munmap="1"
 maps_coalesce="1"
 case "${host}" in
   *-*-darwin* | *-*-ios*)
 	abi="macho"
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
 	RPATH=""
 	LD_PRELOAD_VAR="DYLD_INSERT_LIBRARIES"
 	so="dylib"
@@ -368,29 +401,35 @@ case "${host}" in
   *-*-freebsd*)
 	abi="elf"
 	AC_DEFINE([JEMALLOC_SYSCTL_VM_OVERCOMMIT], [ ])
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
 	force_lazy_lock="1"
 	;;
   *-*-dragonfly*)
 	abi="elf"
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
 	;;
   *-*-openbsd*)
 	abi="elf"
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
 	force_tls="0"
 	;;
   *-*-bitrig*)
 	abi="elf"
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
 	;;
-  *-*-linux*)
+  *-*-linux-android)
 	dnl syscall(2) and secure_getenv(3) are exposed by _GNU_SOURCE.
-	CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE"
+	JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE)
+	abi="elf"
+	AC_DEFINE([JEMALLOC_HAS_ALLOCA_H])
+	AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ])
+	AC_DEFINE([JEMALLOC_THREADED_INIT], [ ])
+	AC_DEFINE([JEMALLOC_C11ATOMICS])
+	force_tls="0"
+	default_munmap="0"
+	;;
+  *-*-linux* | *-*-kfreebsd*)
+	dnl syscall(2) and secure_getenv(3) are exposed by _GNU_SOURCE.
+	JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE)
 	abi="elf"
 	AC_DEFINE([JEMALLOC_HAS_ALLOCA_H])
 	AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ])
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
 	AC_DEFINE([JEMALLOC_THREADED_INIT], [ ])
 	AC_DEFINE([JEMALLOC_USE_CXX_THROW], [ ])
 	default_munmap="0"
@@ -407,15 +446,13 @@ case "${host}" in
                           [abi="elf"],
                           [abi="aout"])
 	AC_MSG_RESULT([$abi])
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
 	;;
   *-*-solaris2*)
 	abi="elf"
-	AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
 	RPATH='-Wl,-R,$(1)'
 	dnl Solaris needs this for sigwait().
-	CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS"
-	LIBS="$LIBS -lposix4 -lsocket -lnsl"
+	JE_APPEND_VS(CPPFLAGS, -D_POSIX_PTHREAD_SEMANTICS)
+	JE_APPEND_VS(LIBS, -lposix4 -lsocket -lnsl)
 	;;
   *-ibm-aix*)
 	if "$LG_SIZEOF_PTR" = "8"; then
@@ -513,19 +550,19 @@ JE_COMPILABLE([__attribute__ syntax],
 if test "x${je_cv_attribute}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_HAVE_ATTR], [ ])
   if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then
-    JE_CFLAGS_APPEND([-fvisibility=hidden])
+    JE_CFLAGS_ADD([-fvisibility=hidden])
   fi
 fi
 dnl Check for tls_model attribute support (clang 3.0 still lacks support).
-SAVED_CFLAGS="${CFLAGS}"
-JE_CFLAGS_APPEND([-Werror])
-JE_CFLAGS_APPEND([-herror_on_warning])
+JE_CFLAGS_SAVE()
+JE_CFLAGS_ADD([-Werror])
+JE_CFLAGS_ADD([-herror_on_warning])
 JE_COMPILABLE([tls_model attribute], [],
               [static __thread int
                __attribute__((tls_model("initial-exec"), unused)) foo;
                foo = 0;],
               [je_cv_tls_model])
-CFLAGS="${SAVED_CFLAGS}"
+JE_CFLAGS_RESTORE()
 if test "x${je_cv_tls_model}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_TLS_MODEL],
             [__attribute__((tls_model("initial-exec")))])
@@ -533,35 +570,35 @@ else
   AC_DEFINE([JEMALLOC_TLS_MODEL], [ ])
 fi
 dnl Check for alloc_size attribute support.
-SAVED_CFLAGS="${CFLAGS}"
-JE_CFLAGS_APPEND([-Werror])
-JE_CFLAGS_APPEND([-herror_on_warning])
+JE_CFLAGS_SAVE()
+JE_CFLAGS_ADD([-Werror])
+JE_CFLAGS_ADD([-herror_on_warning])
 JE_COMPILABLE([alloc_size attribute], [#include <stdlib.h>],
               [void *foo(size_t size) __attribute__((alloc_size(1)));],
               [je_cv_alloc_size])
-CFLAGS="${SAVED_CFLAGS}"
+JE_CFLAGS_RESTORE()
 if test "x${je_cv_alloc_size}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_HAVE_ATTR_ALLOC_SIZE], [ ])
 fi
 dnl Check for format(gnu_printf, ...) attribute support.
-SAVED_CFLAGS="${CFLAGS}"
-JE_CFLAGS_APPEND([-Werror])
-JE_CFLAGS_APPEND([-herror_on_warning])
+JE_CFLAGS_SAVE()
+JE_CFLAGS_ADD([-Werror])
+JE_CFLAGS_ADD([-herror_on_warning])
 JE_COMPILABLE([format(gnu_printf, ...) attribute], [#include <stdlib.h>],
               [void *foo(const char *format, ...) __attribute__((format(gnu_printf, 1, 2)));],
               [je_cv_format_gnu_printf])
-CFLAGS="${SAVED_CFLAGS}"
+JE_CFLAGS_RESTORE()
 if test "x${je_cv_format_gnu_printf}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF], [ ])
 fi
 dnl Check for format(printf, ...) attribute support.
-SAVED_CFLAGS="${CFLAGS}"
-JE_CFLAGS_APPEND([-Werror])
-JE_CFLAGS_APPEND([-herror_on_warning])
+JE_CFLAGS_SAVE()
+JE_CFLAGS_ADD([-Werror])
+JE_CFLAGS_ADD([-herror_on_warning])
 JE_COMPILABLE([format(printf, ...) attribute], [#include <stdlib.h>],
               [void *foo(const char *format, ...) __attribute__((format(printf, 1, 2)));],
               [je_cv_format_printf])
-CFLAGS="${SAVED_CFLAGS}"
+JE_CFLAGS_RESTORE()
 if test "x${je_cv_format_printf}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_PRINTF], [ ])
 fi
@@ -623,9 +660,9 @@ if test "x$enable_code_coverage" = "x1" ; then
   deoptimize="no"
   echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || deoptimize="yes"
   if test "x${deoptimize}" = "xyes" ; then
-    JE_CFLAGS_APPEND([-O0])
+    JE_CFLAGS_ADD([-O0])
   fi
-  JE_CFLAGS_APPEND([-fprofile-arcs -ftest-coverage])
+  JE_CFLAGS_ADD([-fprofile-arcs -ftest-coverage])
   EXTRA_LDFLAGS="$EXTRA_LDFLAGS -fprofile-arcs -ftest-coverage"
   AC_DEFINE([JEMALLOC_CODE_COVERAGE], [ ])
 fi
@@ -815,19 +852,14 @@ if test "x$enable_ivsalloc" = "x1" ; then
 fi
 
 dnl Only optimize if not debugging.
-if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then
-  dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS.
-  optimize="no"
-  echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || optimize="yes"
-  if test "x${optimize}" = "xyes" ; then
-    if test "x$GCC" = "xyes" ; then
-      JE_CFLAGS_APPEND([-O3])
-      JE_CFLAGS_APPEND([-funroll-loops])
-    elif test "x$je_cv_msvc" = "xyes" ; then
-      JE_CFLAGS_APPEND([-O2])
-    else
-      JE_CFLAGS_APPEND([-O])
-    fi
+if test "x$enable_debug" = "x0" ; then
+  if test "x$GCC" = "xyes" ; then
+    JE_CFLAGS_ADD([-O3])
+    JE_CFLAGS_ADD([-funroll-loops])
+  elif test "x$je_cv_msvc" = "xyes" ; then
+    JE_CFLAGS_ADD([-O2])
+  else
+    JE_CFLAGS_ADD([-O])
   fi
 fi
 
@@ -891,10 +923,10 @@ fi,
 if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then
   AC_CHECK_HEADERS([libunwind.h], , [enable_prof_libunwind="0"])
   if test "x$LUNWIND" = "x-lunwind" ; then
-    AC_CHECK_LIB([unwind], [unw_backtrace], [LIBS="$LIBS $LUNWIND"],
+    AC_CHECK_LIB([unwind], [unw_backtrace], [JE_APPEND_VS(LIBS, $LUNWIND)],
                  [enable_prof_libunwind="0"])
   else
-    LIBS="$LIBS $LUNWIND"
+    JE_APPEND_VS(LIBS, $LUNWIND)
   fi
   if test "x${enable_prof_libunwind}" = "x1" ; then
     backtrace_method="libunwind"
@@ -916,7 +948,7 @@ fi
 if test "x$backtrace_method" = "x" -a "x$enable_prof_libgcc" = "x1" \
      -a "x$GCC" = "xyes" ; then
   AC_CHECK_HEADERS([unwind.h], , [enable_prof_libgcc="0"])
-  AC_CHECK_LIB([gcc], [_Unwind_Backtrace], [LIBS="$LIBS -lgcc"], [enable_prof_libgcc="0"])
+  AC_CHECK_LIB([gcc], [_Unwind_Backtrace], [JE_APPEND_VS(LIBS, -lgcc)], [enable_prof_libgcc="0"])
   if test "x${enable_prof_libgcc}" = "x1" ; then
     backtrace_method="libgcc"
     AC_DEFINE([JEMALLOC_PROF_LIBGCC], [ ])
@@ -938,7 +970,7 @@ fi
 )
 if test "x$backtrace_method" = "x" -a "x$enable_prof_gcc" = "x1" \
      -a "x$GCC" = "xyes" ; then
-  JE_CFLAGS_APPEND([-fno-omit-frame-pointer])
+  JE_CFLAGS_ADD([-fno-omit-frame-pointer])
   backtrace_method="gcc intrinsics"
   AC_DEFINE([JEMALLOC_PROF_GCC], [ ])
 else
@@ -953,9 +985,7 @@ AC_MSG_CHECKING([configured backtracing method])
 AC_MSG_RESULT([$backtrace_method])
 if test "x$enable_prof" = "x1" ; then
   dnl Heap profiling uses the log(3) function.
-  if test "x$LM" != "x" ; then
-    LIBS="$LIBS $LM"
-  fi
+  JE_APPEND_VS(LIBS, $LM)
 
   AC_DEFINE([JEMALLOC_PROF], [ ])
 fi
@@ -1324,12 +1354,20 @@ if test "x$abi" != "xpecoff" ; then
   AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])])
   dnl Some systems may embed pthreads functionality in libc; check for libpthread
   dnl first, but try libc too before failing.
-  AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"],
+  AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -lpthread)],
                [AC_SEARCH_LIBS([pthread_create], , ,
                                AC_MSG_ERROR([libpthread is missing]))])
+  JE_COMPILABLE([pthread_atfork(3)], [
+#include <pthread.h>
+], [
+  pthread_atfork((void *)0, (void *)0, (void *)0);
+], [je_cv_pthread_atfork])
+  if test "x${je_cv_pthread_atfork}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_HAVE_PTHREAD_ATFORK], [ ])
+  fi
 fi
 
-CPPFLAGS="$CPPFLAGS -D_REENTRANT"
+JE_APPEND_VS(CPPFLAGS, -D_REENTRANT)
 
 dnl Check whether clock_gettime(2) is in libc or librt.
 AC_SEARCH_LIBS([clock_gettime], [rt])
@@ -1338,13 +1376,13 @@ dnl Cray wrapper compiler often adds `-lrt` when using `-static`. Check with
 dnl `-dynamic` as well in case a user tries to dynamically link in jemalloc
 if test "x$je_cv_cray_prgenv_wrapper" = "xyes" ; then
   if test "$ac_cv_search_clock_gettime" != "-lrt"; then
-    SAVED_CFLAGS="${CFLAGS}"
+    JE_CFLAGS_SAVE()
 
     unset ac_cv_search_clock_gettime
-    JE_CFLAGS_APPEND([-dynamic])
+    JE_CFLAGS_ADD([-dynamic])
     AC_SEARCH_LIBS([clock_gettime], [rt])
 
-    CFLAGS="${SAVED_CFLAGS}"
+    JE_CFLAGS_RESTORE()
   fi
 fi
 
@@ -1386,20 +1424,33 @@ if test "x${je_cv_mach_absolute_time}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_HAVE_MACH_ABSOLUTE_TIME])
 fi
 
-dnl Check if syscall(2) is usable.  Treat warnings as errors, so that e.g. OS X
-dnl 10.12's deprecation warning prevents use.
-SAVED_CFLAGS="${CFLAGS}"
-JE_CFLAGS_APPEND([-Werror])
-JE_COMPILABLE([syscall(2)], [
+dnl Use syscall(2) (if available) by default.
+AC_ARG_ENABLE([syscall],
+  [AS_HELP_STRING([--disable-syscall], [Disable use of syscall(2)])],
+[if test "x$enable_syscall" = "xno" ; then
+  enable_syscall="0"
+else
+  enable_syscall="1"
+fi
+],
+[enable_syscall="1"]
+)
+if test "x$enable_syscall" = "x1" ; then
+  dnl Check if syscall(2) is usable.  Treat warnings as errors, so that e.g. OS
+  dnl X 10.12's deprecation warning prevents use.
+  JE_CFLAGS_SAVE()
+  JE_CFLAGS_ADD([-Werror])
+  JE_COMPILABLE([syscall(2)], [
 #include <sys/syscall.h>
 #include <unistd.h>
 ], [
 	syscall(SYS_write, 2, "hello", 5);
 ],
-              [je_cv_syscall])
-CFLAGS="${SAVED_CFLAGS}"
-if test "x$je_cv_syscall" = "xyes" ; then
-  AC_DEFINE([JEMALLOC_HAVE_SYSCALL], [ ])
+                [je_cv_syscall])
+  JE_CFLAGS_RESTORE()
+  if test "x$je_cv_syscall" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_USE_SYSCALL], [ ])
+  fi
 fi
 
 dnl Check if the GNU-specific secure_getenv function exists.
@@ -1472,7 +1523,7 @@ if test "x$enable_lazy_lock" = "x1" ; then
   if test "x$abi" != "xpecoff" ; then
     AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])])
     AC_CHECK_FUNC([dlsym], [],
-      [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"],
+      [AC_CHECK_LIB([dl], [dlsym], [JE_APPEND_VS(LIBS, -ldl)],
                     [AC_MSG_ERROR([libdl is missing])])
       ])
   fi
@@ -1599,14 +1650,64 @@ dnl Check for madvise(2).
 JE_COMPILABLE([madvise(2)], [
 #include <sys/mman.h>
 ], [
-	{
-		madvise((void *)0, 0, 0);
-	}
+	madvise((void *)0, 0, 0);
 ], [je_cv_madvise])
 if test "x${je_cv_madvise}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ])
+
+  dnl Check for madvise(..., MADV_FREE).
+  JE_COMPILABLE([madvise(..., MADV_FREE)], [
+#include <sys/mman.h>
+], [
+	madvise((void *)0, 0, MADV_FREE);
+], [je_cv_madv_free])
+  if test "x${je_cv_madv_free}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ])
+  fi
+
+  dnl Check for madvise(..., MADV_DONTNEED).
+  JE_COMPILABLE([madvise(..., MADV_DONTNEED)], [
+#include <sys/mman.h>
+], [
+	madvise((void *)0, 0, MADV_DONTNEED);
+], [je_cv_madv_dontneed])
+  if test "x${je_cv_madv_dontneed}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
+  fi
+
+  dnl Check for madvise(..., MADV_[NO]HUGEPAGE).
+  JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [
+#include <sys/mman.h>
+], [
+	madvise((void *)0, 0, MADV_HUGEPAGE);
+	madvise((void *)0, 0, MADV_NOHUGEPAGE);
+], [je_cv_thp])
+  if test "x${je_cv_thp}" = "xyes" ; then
+    AC_DEFINE([JEMALLOC_HAVE_MADVISE_HUGE], [ ])
+  fi
 fi
 
+dnl Enable transparent huge page support by default.
+AC_ARG_ENABLE([thp],
+  [AS_HELP_STRING([--disable-thp],
+                  [Disable transparent huge page supprot])],
+[if test "x$enable_thp" = "xno" -o "x${je_cv_thp}" != "xyes" ; then
+  enable_thp="0"
+else
+  enable_thp="1"
+fi
+],
+[if test "x${je_cv_thp}" = "xyes" ; then
+  enable_thp="1"
+else
+  enable_thp="0"
+fi
+])
+if test "x$enable_thp" = "x1" ; then
+  AC_DEFINE([JEMALLOC_THP], [ ])
+fi
+AC_SUBST([enable_thp])
+
 dnl ============================================================================
 dnl Check whether __sync_{add,sub}_and_fetch() are available despite
 dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined.
@@ -1669,10 +1770,15 @@ dnl Check for os_unfair_lock operations as provided on Darwin.
 
 JE_COMPILABLE([Darwin os_unfair_lock_*()], [
 #include <os/lock.h>
+#include <AvailabilityMacros.h>
 ], [
+	#if MAC_OS_X_VERSION_MIN_REQUIRED < 101200
+	#error "os_unfair_lock is not supported"
+	#else
 	os_unfair_lock lock = OS_UNFAIR_LOCK_INIT;
 	os_unfair_lock_lock(&lock);
 	os_unfair_lock_unlock(&lock);
+	#endif
 ], [je_cv_os_unfair_lock])
 if test "x${je_cv_os_unfair_lock}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_OS_UNFAIR_LOCK], [ ])
@@ -1717,37 +1823,6 @@ if test "x${enable_zone_allocator}" = "x1" ; then
     AC_MSG_ERROR([--enable-zone-allocator is only supported on Darwin])
   fi
   AC_DEFINE([JEMALLOC_ZONE], [ ])
-
-  dnl The szone version jumped from 3 to 6 between the OS X 10.5.x and 10.6
-  dnl releases.  malloc_zone_t and malloc_introspection_t have new fields in
-  dnl 10.6, which is the only source-level indication of the change.
-  AC_MSG_CHECKING([malloc zone version])
-  AC_DEFUN([JE_ZONE_PROGRAM],
-    [AC_LANG_PROGRAM(
-      [#include <malloc/malloc.h>],
-      [static int foo[[sizeof($1) $2 sizeof(void *) * $3 ? 1 : -1]]]
-    )])
-
-  AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,14)],[JEMALLOC_ZONE_VERSION=3],[
-  AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,15)],[JEMALLOC_ZONE_VERSION=5],[
-  AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,16)],[
-    AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,9)],[JEMALLOC_ZONE_VERSION=6],[
-    AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,13)],[JEMALLOC_ZONE_VERSION=7],[JEMALLOC_ZONE_VERSION=]
-  )])],[
-  AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,17)],[JEMALLOC_ZONE_VERSION=8],[
-  AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,>,17)],[JEMALLOC_ZONE_VERSION=9],[JEMALLOC_ZONE_VERSION=]
-  )])])])])
-  if test "x${JEMALLOC_ZONE_VERSION}" = "x"; then
-    AC_MSG_RESULT([unsupported])
-    AC_MSG_ERROR([Unsupported malloc zone version])
-  fi
-  if test "${JEMALLOC_ZONE_VERSION}" = 9; then
-    JEMALLOC_ZONE_VERSION=8
-    AC_MSG_RESULT([> 8])
-  else
-    AC_MSG_RESULT([$JEMALLOC_ZONE_VERSION])
-  fi
-  AC_DEFINE_UNQUOTED(JEMALLOC_ZONE_VERSION, [$JEMALLOC_ZONE_VERSION])
 fi
 
 dnl ============================================================================
@@ -1921,7 +1996,8 @@ AC_MSG_RESULT([library revision   : ${rev}])
 AC_MSG_RESULT([])
 AC_MSG_RESULT([CONFIG             : ${CONFIG}])
 AC_MSG_RESULT([CC                 : ${CC}])
-AC_MSG_RESULT([CFLAGS             : ${CFLAGS}])
+AC_MSG_RESULT([CONFIGURE_CFLAGS   : ${CONFIGURE_CFLAGS}])
+AC_MSG_RESULT([SPECIFIED_CFLAGS   : ${SPECIFIED_CFLAGS}])
 AC_MSG_RESULT([EXTRA_CFLAGS       : ${EXTRA_CFLAGS}])
 AC_MSG_RESULT([CPPFLAGS           : ${CPPFLAGS}])
 AC_MSG_RESULT([LDFLAGS            : ${LDFLAGS}])
@@ -1959,6 +2035,7 @@ AC_MSG_RESULT([prof-libunwind     : ${enable_prof_libunwind}])
 AC_MSG_RESULT([prof-libgcc        : ${enable_prof_libgcc}])
 AC_MSG_RESULT([prof-gcc           : ${enable_prof_gcc}])
 AC_MSG_RESULT([tcache             : ${enable_tcache}])
+AC_MSG_RESULT([thp                : ${enable_thp}])
 AC_MSG_RESULT([fill               : ${enable_fill}])
 AC_MSG_RESULT([utrace             : ${enable_utrace}])
 AC_MSG_RESULT([valgrind           : ${enable_valgrind}])

doc/jemalloc.xml.in

@@ -406,7 +406,7 @@ for (i = 0; i < nbins; i++) {
 
 	mib[2] = i;
 	len = sizeof(bin_size);
-	mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0);
+	mallctlbymib(mib, miblen, (void *)&bin_size, &len, NULL, 0);
 	/* Do something with bin_size... */
 }]]></programlisting></para>
 
@@ -850,6 +850,17 @@ for (i = 0; i < nbins; i++) {
         during build configuration.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="config.thp">
+        <term>
+          <mallctl>config.thp</mallctl>
+          (<type>bool</type>)
+          <literal>r-</literal>
+        </term>
+        <listitem><para><option>--disable-thp</option> was not specified
+        during build configuration, and the system supports transparent huge
+        page manipulation.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="config.tls">
         <term>
           <mallctl>config.tls</mallctl>
@@ -1162,6 +1173,21 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         forcefully disabled.</para></listitem>
       </varlistentry>
 
+      <varlistentry id="opt.thp">
+        <term>
+          <mallctl>opt.thp</mallctl>
+          (<type>bool</type>)
+          <literal>r-</literal>
+          [<option>--enable-thp</option>]
+        </term>
+        <listitem><para>Transparent huge page (THP) integration
+        enabled/disabled.  When enabled, THPs are explicitly disabled as a side
+        effect of unused dirty page purging for chunks that back small and/or
+        large allocations, because such chunks typically comprise active,
+        unused dirty, and untouched clean pages.  This option is enabled by
+        default.</para></listitem>
+      </varlistentry>
+
       <varlistentry id="opt.lg_tcache_max">
         <term>
           <mallctl>opt.lg_tcache_max</mallctl>

include/jemalloc/internal/arena.h

@@ -190,6 +190,14 @@ struct arena_chunk_s {
 	 */
 	extent_node_t		node;
 
+	/*
+	 * True if memory could be backed by transparent huge pages.  This is
+	 * only directly relevant to Linux, since it is the only supported
+	 * platform on which jemalloc interacts with explicit transparent huge
+	 * page controls.
+	 */
+	bool			hugepage;
+
 	/*
 	 * Map of pages within chunk that keeps track of free/large/small.  The
 	 * first map_bias entries are omitted, since the chunk header does not
@@ -374,10 +382,12 @@ struct arena_s {
 
 	dss_prec_t		dss_prec;
 
-
 	/* Extant arena chunks. */
 	ql_head(extent_node_t)	achunks;
 
+	/* Extent serial number generator state. */
+	size_t			extent_sn_next;
+
 	/*
 	 * In order to avoid rapid chunk allocation/deallocation when an arena
 	 * oscillates right on the cusp of needing a new chunk, cache the most
@@ -453,9 +463,9 @@ struct arena_s {
 	 * orderings are needed, which is why there are two trees with the same
 	 * contents.
 	 */
-	extent_tree_t		chunks_szad_cached;
+	extent_tree_t		chunks_szsnad_cached;
 	extent_tree_t		chunks_ad_cached;
-	extent_tree_t		chunks_szad_retained;
+	extent_tree_t		chunks_szsnad_retained;
 	extent_tree_t		chunks_ad_retained;
 
 	malloc_mutex_t		chunks_mtx;
@@ -496,6 +506,7 @@ static const size_t	large_pad =
 #endif
     ;
 
+extern bool		opt_thp;
 extern purge_mode_t	opt_purge;
 extern const char	*purge_mode_names[];
 extern ssize_t		opt_lg_dirty_mult;
@@ -522,13 +533,13 @@ void	arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node,
 extent_node_t	*arena_node_alloc(tsdn_t *tsdn, arena_t *arena);
 void	arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node);
 void	*arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
-    size_t alignment, bool *zero);
+    size_t alignment, size_t *sn, bool *zero);
 void	arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk,
-    size_t usize);
+    size_t usize, size_t sn);
 void	arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena,
     void *chunk, size_t oldsize, size_t usize);
 void	arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena,
-    void *chunk, size_t oldsize, size_t usize);
+    void *chunk, size_t oldsize, size_t usize, size_t sn);
 bool	arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena,
     void *chunk, size_t oldsize, size_t usize, bool *zero);
 ssize_t	arena_lg_dirty_mult_get(tsdn_t *tsdn, arena_t *arena);
@@ -601,6 +612,7 @@ void	arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 unsigned	arena_nthreads_get(arena_t *arena, bool internal);
 void	arena_nthreads_inc(arena_t *arena, bool internal);
 void	arena_nthreads_dec(arena_t *arena, bool internal);
+size_t	arena_extent_sn_next(arena_t *arena);
 arena_t	*arena_new(tsdn_t *tsdn, unsigned ind);
 void	arena_boot(void);
 void	arena_prefork0(tsdn_t *tsdn, arena_t *arena);

include/jemalloc/internal/chunk.h

@@ -52,21 +52,22 @@ chunk_hooks_t	chunk_hooks_get(tsdn_t *tsdn, arena_t *arena);
 chunk_hooks_t	chunk_hooks_set(tsdn_t *tsdn, arena_t *arena,
     const chunk_hooks_t *chunk_hooks);
 
-bool	chunk_register(tsdn_t *tsdn, const void *chunk,
-    const extent_node_t *node);
+bool	chunk_register(const void *chunk, const extent_node_t *node,
+    bool *gdump);
 void	chunk_deregister(const void *chunk, const extent_node_t *node);
 void	*chunk_alloc_base(size_t size);
 void	*chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
     chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
-    bool *zero, bool *commit, bool dalloc_node);
+    size_t *sn, bool *zero, bool *commit, bool dalloc_node);
 void	*chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
     chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
-    bool *zero, bool *commit);
+    size_t *sn, bool *zero, bool *commit);
 void	chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
-    chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool committed);
-void	chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
-    chunk_hooks_t *chunk_hooks, void *chunk, size_t size, bool zeroed,
+    chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn,
     bool committed);
+void	chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
+    chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t sn,
+    bool zeroed, bool committed);
 bool	chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
     chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset,
     size_t length);

include/jemalloc/internal/extent.h

@@ -18,6 +18,20 @@ struct extent_node_s {
 	/* Total region size. */
 	size_t			en_size;
 
+	/*
+	 * Serial number (potentially non-unique).
+	 *
+	 * In principle serial numbers can wrap around on 32-bit systems if
+	 * JEMALLOC_MUNMAP is defined, but as long as comparison functions fall
+	 * back on address comparison for equal serial numbers, stable (if
+	 * imperfect) ordering is maintained.
+	 *
+	 * Serial numbers may not be unique even in the absence of wrap-around,
+	 * e.g. when splitting an extent and assigning the same serial number to
+	 * both resulting adjacent extents.
+	 */
+	size_t			en_sn;
+
 	/*
 	 * The zeroed flag is used by chunk recycling code to track whether
 	 * memory is zero-filled.
@@ -45,8 +59,8 @@ struct extent_node_s {
 	qr(extent_node_t)	cc_link;
 
 	union {
-		/* Linkage for the size/address-ordered tree. */
-		rb_node(extent_node_t)	szad_link;
+		/* Linkage for the size/sn/address-ordered tree. */
+		rb_node(extent_node_t)	szsnad_link;
 
 		/* Linkage for arena's achunks, huge, and node_cache lists. */
 		ql_elm(extent_node_t)	ql_link;
@@ -61,7 +75,12 @@ typedef rb_tree(extent_node_t) extent_tree_t;
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t)
+#ifdef JEMALLOC_JET
+size_t	extent_size_quantize_floor(size_t size);
+#endif
+size_t	extent_size_quantize_ceil(size_t size);
+
+rb_proto(, extent_tree_szsnad_, extent_tree_t, extent_node_t)
 
 rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
 
@@ -73,6 +92,7 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t)
 arena_t	*extent_node_arena_get(const extent_node_t *node);
 void	*extent_node_addr_get(const extent_node_t *node);
 size_t	extent_node_size_get(const extent_node_t *node);
+size_t	extent_node_sn_get(const extent_node_t *node);
 bool	extent_node_zeroed_get(const extent_node_t *node);
 bool	extent_node_committed_get(const extent_node_t *node);
 bool	extent_node_achunk_get(const extent_node_t *node);
@@ -80,12 +100,13 @@ prof_tctx_t	*extent_node_prof_tctx_get(const extent_node_t *node);
 void	extent_node_arena_set(extent_node_t *node, arena_t *arena);
 void	extent_node_addr_set(extent_node_t *node, void *addr);
 void	extent_node_size_set(extent_node_t *node, size_t size);
+void	extent_node_sn_set(extent_node_t *node, size_t sn);
 void	extent_node_zeroed_set(extent_node_t *node, bool zeroed);
 void	extent_node_committed_set(extent_node_t *node, bool committed);
 void	extent_node_achunk_set(extent_node_t *node, bool achunk);
 void	extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx);
 void	extent_node_init(extent_node_t *node, arena_t *arena, void *addr,
-    size_t size, bool zeroed, bool committed);
+    size_t size, size_t sn, bool zeroed, bool committed);
 void	extent_node_dirty_linkage_init(extent_node_t *node);
 void	extent_node_dirty_insert(extent_node_t *node,
     arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty);
@@ -114,6 +135,13 @@ extent_node_size_get(const extent_node_t *node)
 	return (node->en_size);
 }
 
+JEMALLOC_INLINE size_t
+extent_node_sn_get(const extent_node_t *node)
+{
+
+	return (node->en_sn);
+}
+
 JEMALLOC_INLINE bool
 extent_node_zeroed_get(const extent_node_t *node)
 {
@@ -164,6 +192,13 @@ extent_node_size_set(extent_node_t *node, size_t size)
 	node->en_size = size;
 }
 
+JEMALLOC_INLINE void
+extent_node_sn_set(extent_node_t *node, size_t sn)
+{
+
+	node->en_sn = sn;
+}
+
 JEMALLOC_INLINE void
 extent_node_zeroed_set(extent_node_t *node, bool zeroed)
 {
@@ -194,12 +229,13 @@ extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx)
 
 JEMALLOC_INLINE void
 extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size,
-    bool zeroed, bool committed)
+    size_t sn, bool zeroed, bool committed)
 {
 
 	extent_node_arena_set(node, arena);
 	extent_node_addr_set(node, addr);
 	extent_node_size_set(node, size);
+	extent_node_sn_set(node, sn);
 	extent_node_zeroed_set(node, zeroed);
 	extent_node_committed_set(node, committed);
 	extent_node_achunk_set(node, false);

include/jemalloc/internal/jemalloc_internal.h.in

@@ -99,6 +99,13 @@ static const bool config_tcache =
     false
 #endif
     ;
+static const bool config_thp =
+#ifdef JEMALLOC_THP
+    true
+#else
+    false
+#endif
+    ;
 static const bool config_tls =
 #ifdef JEMALLOC_TLS
     true
@@ -158,7 +165,6 @@ static const bool config_cache_oblivious =
 #include <mach/mach_error.h>
 #include <mach/mach_init.h>
 #include <mach/vm_map.h>
-#include <malloc/malloc.h>
 #endif
 
 #include "jemalloc/internal/ph.h"
@@ -337,7 +343,7 @@ typedef unsigned szind_t;
 
 /* Return the nearest aligned address at or below a. */
 #define	ALIGNMENT_ADDR2BASE(a, alignment)				\
-	((void *)((uintptr_t)(a) & (-(alignment))))
+	((void *)((uintptr_t)(a) & ((~(alignment)) + 1)))
 
 /* Return the offset between a and the nearest aligned address at or below a. */
 #define	ALIGNMENT_ADDR2OFFSET(a, alignment)				\
@@ -345,7 +351,7 @@ typedef unsigned szind_t;
 
 /* Return the smallest alignment multiple that is >= s. */
 #define	ALIGNMENT_CEILING(s, alignment)					\
-	(((s) + (alignment - 1)) & (-(alignment)))
+	(((s) + (alignment - 1)) & ((~(alignment)) + 1))
 
 /* Declare a variable-length array. */
 #if __STDC_VERSION__ < 199901L

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -55,11 +55,6 @@
  */
 #undef JEMALLOC_HAVE_BUILTIN_CLZ
 
-/*
- * Defined if madvise(2) is available.
- */
-#undef JEMALLOC_HAVE_MADVISE
-
 /*
  * Defined if os_unfair_lock_*() functions are available, as provided by Darwin.
  */
@@ -71,8 +66,8 @@
  */
 #undef JEMALLOC_OSSPIN
 
-/* Defined if syscall(2) is available. */
-#undef JEMALLOC_HAVE_SYSCALL
+/* Defined if syscall(2) is usable. */
+#undef JEMALLOC_USE_SYSCALL
 
 /*
  * Defined if secure_getenv(3) is available.
@@ -84,6 +79,9 @@
  */
 #undef JEMALLOC_HAVE_ISSETUGID
 
+/* Defined if pthread_atfork(3) is available. */
+#undef JEMALLOC_HAVE_PTHREAD_ATFORK
+
 /*
  * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
  */
@@ -241,7 +239,6 @@
  * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
  */
 #undef JEMALLOC_ZONE
-#undef JEMALLOC_ZONE_VERSION
 
 /*
  * Methods for determining whether the OS overcommits.
@@ -252,18 +249,29 @@
 #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT
 #undef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
 
+/* Defined if madvise(2) is available. */
+#undef JEMALLOC_HAVE_MADVISE
+
+/*
+ * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE
+ * arguments to madvise(2).
+ */
+#undef JEMALLOC_HAVE_MADVISE_HUGE
+
 /*
  * Methods for purging unused pages differ between operating systems.
  *
- *   madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages,
- *                                 such that new pages will be demand-zeroed if
- *                                 the address region is later touched.
- *   madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being
- *                             unused, such that they will be discarded rather
- *                             than swapped out.
+ *   madvise(..., MADV_FREE) : This marks pages as being unused, such that they
+ *                             will be discarded rather than swapped out.
+ *   madvise(..., MADV_DONTNEED) : This immediately discards pages, such that
+ *                                 new pages will be demand-zeroed if the
+ *                                 address region is later touched.
  */
-#undef JEMALLOC_PURGE_MADVISE_DONTNEED
 #undef JEMALLOC_PURGE_MADVISE_FREE
+#undef JEMALLOC_PURGE_MADVISE_DONTNEED
+
+/* Defined if transparent huge page support is enabled. */
+#undef JEMALLOC_THP
 
 /* Define if operating system has alloca.h header. */
 #undef JEMALLOC_HAS_ALLOCA_H

include/jemalloc/internal/mb.h

@@ -76,7 +76,7 @@ mb_write(void)
 	    : "memory" /* Clobbers. */
 	    );
 }
-#elif defined(__sparc64__)
+#elif defined(__sparc__) && defined(__arch64__)
 JEMALLOC_INLINE void
 mb_write(void)
 {

include/jemalloc/internal/mutex.h

@@ -85,8 +85,8 @@ JEMALLOC_INLINE void
 malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
+	witness_assert_not_owner(tsdn, &mutex->witness);
 	if (isthreaded) {
-		witness_assert_not_owner(tsdn, &mutex->witness);
 #ifdef _WIN32
 #  if _WIN32_WINNT >= 0x0600
 		AcquireSRWLockExclusive(&mutex->lock);
@@ -100,16 +100,16 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
 #else
 		pthread_mutex_lock(&mutex->lock);
 #endif
-		witness_lock(tsdn, &mutex->witness);
 	}
+	witness_lock(tsdn, &mutex->witness);
 }
 
 JEMALLOC_INLINE void
 malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
+	witness_unlock(tsdn, &mutex->witness);
 	if (isthreaded) {
-		witness_unlock(tsdn, &mutex->witness);
 #ifdef _WIN32
 #  if _WIN32_WINNT >= 0x0600
 		ReleaseSRWLockExclusive(&mutex->lock);
@@ -130,16 +130,14 @@ JEMALLOC_INLINE void
 malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
-	if (isthreaded)
-		witness_assert_owner(tsdn, &mutex->witness);
+	witness_assert_owner(tsdn, &mutex->witness);
 }
 
 JEMALLOC_INLINE void
 malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
 {
 
-	if (isthreaded)
-		witness_assert_not_owner(tsdn, &mutex->witness);
+	witness_assert_not_owner(tsdn, &mutex->witness);
 }
 #endif
 

include/jemalloc/internal/pages.h

@@ -16,6 +16,8 @@ void	*pages_trim(void *addr, size_t alloc_size, size_t leadsize,
 bool	pages_commit(void *addr, size_t size);
 bool	pages_decommit(void *addr, size_t size);
 bool	pages_purge(void *addr, size_t size);
+bool	pages_huge(void *addr, size_t size);
+bool	pages_nohuge(void *addr, size_t size);
 void	pages_boot(void);
 
 #endif /* JEMALLOC_H_EXTERNS */

include/jemalloc/internal/private_symbols.txt

@@ -36,6 +36,7 @@ arena_decay_time_get
 arena_decay_time_set
 arena_dss_prec_get
 arena_dss_prec_set
+arena_extent_sn_next
 arena_get
 arena_ichoose
 arena_init
@@ -218,8 +219,12 @@ extent_node_prof_tctx_get
 extent_node_prof_tctx_set
 extent_node_size_get
 extent_node_size_set
+extent_node_sn_get
+extent_node_sn_set
 extent_node_zeroed_get
 extent_node_zeroed_set
+extent_size_quantize_ceil
+extent_size_quantize_floor
 extent_tree_ad_destroy
 extent_tree_ad_destroy_recurse
 extent_tree_ad_empty
@@ -239,25 +244,25 @@ extent_tree_ad_reverse_iter
 extent_tree_ad_reverse_iter_recurse
 extent_tree_ad_reverse_iter_start
 extent_tree_ad_search
-extent_tree_szad_destroy
-extent_tree_szad_destroy_recurse
-extent_tree_szad_empty
-extent_tree_szad_first
-extent_tree_szad_insert
-extent_tree_szad_iter
-extent_tree_szad_iter_recurse
-extent_tree_szad_iter_start
-extent_tree_szad_last
-extent_tree_szad_new
-extent_tree_szad_next
-extent_tree_szad_nsearch
-extent_tree_szad_prev
-extent_tree_szad_psearch
-extent_tree_szad_remove
-extent_tree_szad_reverse_iter
-extent_tree_szad_reverse_iter_recurse
-extent_tree_szad_reverse_iter_start
-extent_tree_szad_search
+extent_tree_szsnad_destroy
+extent_tree_szsnad_destroy_recurse
+extent_tree_szsnad_empty
+extent_tree_szsnad_first
+extent_tree_szsnad_insert
+extent_tree_szsnad_iter
+extent_tree_szsnad_iter_recurse
+extent_tree_szsnad_iter_start
+extent_tree_szsnad_last
+extent_tree_szsnad_new
+extent_tree_szsnad_next
+extent_tree_szsnad_nsearch
+extent_tree_szsnad_prev
+extent_tree_szsnad_psearch
+extent_tree_szsnad_remove
+extent_tree_szsnad_reverse_iter
+extent_tree_szsnad_reverse_iter_recurse
+extent_tree_szsnad_reverse_iter_start
+extent_tree_szsnad_search
 ffs_llu
 ffs_lu
 ffs_u
@@ -387,6 +392,7 @@ opt_quarantine
 opt_redzone
 opt_stats_print
 opt_tcache
+opt_thp
 opt_utrace
 opt_xmalloc
 opt_zero
@@ -394,7 +400,9 @@ p2rz
 pages_boot
 pages_commit
 pages_decommit
+pages_huge
 pages_map
+pages_nohuge
 pages_purge
 pages_trim
 pages_unmap
@@ -524,6 +532,9 @@ tcache_flush
 tcache_get
 tcache_get_hard
 tcache_maxclass
+tcache_postfork_child
+tcache_postfork_parent
+tcache_prefork
 tcache_salloc
 tcache_stats_merge
 tcaches
@@ -607,14 +618,16 @@ valgrind_freelike_block
 valgrind_make_mem_defined
 valgrind_make_mem_noaccess
 valgrind_make_mem_undefined
+witness_assert_depth
+witness_assert_depth_to_rank
 witness_assert_lockless
 witness_assert_not_owner
 witness_assert_owner
+witness_depth_error
 witness_fork_cleanup
 witness_init
 witness_lock
 witness_lock_error
-witness_lockless_error
 witness_not_owner_error
 witness_owner
 witness_owner_error

include/jemalloc/internal/stats.h

@@ -175,25 +175,21 @@ stats_cactive_get(void)
 JEMALLOC_INLINE void
 stats_cactive_add(size_t size)
 {
-	UNUSED size_t cactive;
 
 	assert(size > 0);
 	assert((size & chunksize_mask) == 0);
 
-	cactive = atomic_add_z(&stats_cactive, size);
-	assert(cactive - size < cactive);
+	atomic_add_z(&stats_cactive, size);
 }
 
 JEMALLOC_INLINE void
 stats_cactive_sub(size_t size)
 {
-	UNUSED size_t cactive;
 
 	assert(size > 0);
 	assert((size & chunksize_mask) == 0);
 
-	cactive = atomic_sub_z(&stats_cactive, size);
-	assert(cactive + size > cactive);
+	atomic_sub_z(&stats_cactive, size);
 }
 #endif
 

include/jemalloc/internal/tcache.h

@@ -149,6 +149,9 @@ bool	tcaches_create(tsd_t *tsd, unsigned *r_ind);
 void	tcaches_flush(tsd_t *tsd, unsigned ind);
 void	tcaches_destroy(tsd_t *tsd, unsigned ind);
 bool	tcache_boot(tsdn_t *tsdn);
+void tcache_prefork(tsdn_t *tsdn);
+void tcache_postfork_parent(tsdn_t *tsdn);
+void tcache_postfork_child(tsdn_t *tsdn);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

include/jemalloc/internal/tsd.h

@@ -479,13 +479,14 @@ a_name##tsd_wrapper_get(bool init)					\
 									\
 	if (init && unlikely(wrapper == NULL)) {			\
 		tsd_init_block_t block;					\
-		wrapper = tsd_init_check_recursion(			\
-		    &a_name##tsd_init_head, &block);			\
+		wrapper = (a_name##tsd_wrapper_t *)			\
+		    tsd_init_check_recursion(&a_name##tsd_init_head,	\
+		    &block);						\
 		if (wrapper)						\
 		    return (wrapper);					\
 		wrapper = (a_name##tsd_wrapper_t *)			\
 		    malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t));	\
-		block.data = wrapper;					\
+		block.data = (void *)wrapper;				\
 		if (wrapper == NULL) {					\
 			malloc_write("<jemalloc>: Error allocating"	\
 			    " TSD for "#a_name"\n");			\

include/jemalloc/internal/util.h

@@ -41,8 +41,12 @@
 #define	MALLOC_PRINTF_BUFSIZE	4096
 
 /* Junk fill patterns. */
-#define	JEMALLOC_ALLOC_JUNK	((uint8_t)0xa5)
-#define	JEMALLOC_FREE_JUNK	((uint8_t)0x5a)
+#ifndef JEMALLOC_ALLOC_JUNK
+#  define JEMALLOC_ALLOC_JUNK	((uint8_t)0xa5)
+#endif
+#ifndef JEMALLOC_FREE_JUNK
+#  define JEMALLOC_FREE_JUNK	((uint8_t)0x5a)
+#endif
 
 /*
  * Wrap a cpp argument that contains commas such that it isn't broken up into

include/jemalloc/internal/valgrind.h

@@ -36,13 +36,25 @@
 		    zero);						\
 	}								\
 } while (0)
-#define	JEMALLOC_VALGRIND_REALLOC(maybe_moved, tsdn, ptr, usize,	\
-    ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null,	\
-    zero) do {								\
+#define	JEMALLOC_VALGRIND_REALLOC_MOVED_no(ptr, old_ptr)		\
+    (false)
+#define	JEMALLOC_VALGRIND_REALLOC_MOVED_maybe(ptr, old_ptr)		\
+    ((ptr) != (old_ptr))
+#define	JEMALLOC_VALGRIND_REALLOC_PTR_NULL_no(ptr)			\
+    (false)
+#define	JEMALLOC_VALGRIND_REALLOC_PTR_NULL_maybe(ptr)			\
+    (ptr == NULL)
+#define	JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_no(old_ptr)		\
+    (false)
+#define	JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_maybe(old_ptr)		\
+    (old_ptr == NULL)
+#define	JEMALLOC_VALGRIND_REALLOC(moved, tsdn, ptr, usize, ptr_null,	\
+    old_ptr, old_usize, old_rzsize, old_ptr_null, zero) do {		\
 	if (unlikely(in_valgrind)) {					\
 		size_t rzsize = p2rz(tsdn, ptr);			\
 									\
-		if (!maybe_moved || ptr == old_ptr) {			\
+		if (!JEMALLOC_VALGRIND_REALLOC_MOVED_##moved(ptr,	\
+		    old_ptr)) {						\
 			VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize,	\
 			    usize, rzsize);				\
 			if (zero && old_usize < usize) {		\
@@ -51,11 +63,13 @@
 				    old_usize), usize - old_usize);	\
 			}						\
 		} else {						\
-			if (!old_ptr_maybe_null || old_ptr != NULL) {	\
+			if (!JEMALLOC_VALGRIND_REALLOC_OLD_PTR_NULL_##	\
+			    old_ptr_null(old_ptr)) {			\
 				valgrind_freelike_block(old_ptr,	\
 				    old_rzsize);			\
 			}						\
-			if (!ptr_maybe_null || ptr != NULL) {		\
+			if (!JEMALLOC_VALGRIND_REALLOC_PTR_NULL_##	\
+			    ptr_null(ptr)) {				\
 				size_t copy_size = (old_usize < usize)	\
 				    ?  old_usize : usize;		\
 				size_t tail_size = usize - copy_size;	\

include/jemalloc/internal/witness.h

@@ -12,21 +12,32 @@ typedef int witness_comp_t (const witness_t *, const witness_t *);
  */
 #define	WITNESS_RANK_OMIT		0U
 
+#define WITNESS_RANK_MIN		1U
+
 #define	WITNESS_RANK_INIT		1U
 #define	WITNESS_RANK_CTL		1U
-#define	WITNESS_RANK_ARENAS		2U
+#define WITNESS_RANK_TCACHES		2U
+#define	WITNESS_RANK_ARENAS		3U
 
-#define	WITNESS_RANK_PROF_DUMP		3U
-#define	WITNESS_RANK_PROF_BT2GCTX	4U
-#define	WITNESS_RANK_PROF_TDATAS	5U
-#define	WITNESS_RANK_PROF_TDATA		6U
-#define	WITNESS_RANK_PROF_GCTX		7U
+#define	WITNESS_RANK_PROF_DUMP		4U
+#define	WITNESS_RANK_PROF_BT2GCTX	5U
+#define	WITNESS_RANK_PROF_TDATAS	6U
+#define	WITNESS_RANK_PROF_TDATA		7U
+#define	WITNESS_RANK_PROF_GCTX		8U
 
-#define	WITNESS_RANK_ARENA		8U
-#define	WITNESS_RANK_ARENA_CHUNKS	9U
-#define	WITNESS_RANK_ARENA_NODE_CACHE	10
+/*
+ * Used as an argument to witness_assert_depth_to_rank() in order to validate
+ * depth excluding non-core locks with lower ranks.  Since the rank argument to
+ * witness_assert_depth_to_rank() is inclusive rather than exclusive, this
+ * definition can have the same value as the minimally ranked core lock.
+ */
+#define WITNESS_RANK_CORE		9U
 
-#define	WITNESS_RANK_BASE		11U
+#define	WITNESS_RANK_ARENA		9U
+#define	WITNESS_RANK_ARENA_CHUNKS	10U
+#define	WITNESS_RANK_ARENA_NODE_CACHE	11U
+
+#define	WITNESS_RANK_BASE		12U
 
 #define	WITNESS_RANK_LEAF		0xffffffffU
 #define	WITNESS_RANK_ARENA_BIN		WITNESS_RANK_LEAF
@@ -91,10 +102,12 @@ extern witness_not_owner_error_t *witness_not_owner_error;
 void	witness_not_owner_error(const witness_t *witness);
 #endif
 #ifdef JEMALLOC_JET
-typedef void (witness_lockless_error_t)(const witness_list_t *);
-extern witness_lockless_error_t *witness_lockless_error;
+typedef void (witness_depth_error_t)(const witness_list_t *,
+    witness_rank_t rank_inclusive, unsigned depth);
+extern witness_depth_error_t *witness_depth_error;
 #else
-void	witness_lockless_error(const witness_list_t *witnesses);
+void	witness_depth_error(const witness_list_t *witnesses,
+    witness_rank_t rank_inclusive, unsigned depth);
 #endif
 
 void	witnesses_cleanup(tsd_t *tsd);
@@ -111,6 +124,9 @@ void	witness_postfork_child(tsd_t *tsd);
 bool	witness_owner(tsd_t *tsd, const witness_t *witness);
 void	witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
 void	witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
+void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
+    unsigned depth);
+void witness_assert_depth(tsdn_t *tsdn, unsigned depth);
 void	witness_assert_lockless(tsdn_t *tsdn);
 void	witness_lock(tsdn_t *tsdn, witness_t *witness);
 void	witness_unlock(tsdn_t *tsdn, witness_t *witness);
@@ -123,6 +139,8 @@ witness_owner(tsd_t *tsd, const witness_t *witness)
 	witness_list_t *witnesses;
 	witness_t *w;
 
+	cassert(config_debug);
+
 	witnesses = tsd_witnessesp_get(tsd);
 	ql_foreach(w, witnesses, link) {
 		if (w == witness)
@@ -175,9 +193,10 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
 }
 
 JEMALLOC_INLINE void
-witness_assert_lockless(tsdn_t *tsdn)
-{
+witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
+    unsigned depth) {
 	tsd_t *tsd;
+	unsigned d;
 	witness_list_t *witnesses;
 	witness_t *w;
 
@@ -188,10 +207,29 @@ witness_assert_lockless(tsdn_t *tsdn)
 		return;
 	tsd = tsdn_tsd(tsdn);
 
+	d = 0;
 	witnesses = tsd_witnessesp_get(tsd);
 	w = ql_last(witnesses, link);
-	if (w != NULL)
-		witness_lockless_error(witnesses);
+	if (w != NULL) {
+		ql_reverse_foreach(w, witnesses, link) {
+			if (w->rank < rank_inclusive) {
+				break;
+			}
+			d++;
+		}
+	}
+	if (d != depth)
+		witness_depth_error(witnesses, rank_inclusive, depth);
+}
+
+JEMALLOC_INLINE void
+witness_assert_depth(tsdn_t *tsdn, unsigned depth) {
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth);
+}
+
+JEMALLOC_INLINE void
+witness_assert_lockless(tsdn_t *tsdn) {
+	witness_assert_depth(tsdn, 0);
 }
 
 JEMALLOC_INLINE void

msvc/projects/vc2015/test_threads/test_threads.cpp

@@ -21,7 +21,7 @@ int test_threads()
   je_malloc_conf = "narenas:3";
   int narenas = 0;
   size_t sz = sizeof(narenas);
-  je_mallctl("opt.narenas", &narenas, &sz, NULL, 0);
+  je_mallctl("opt.narenas", (void *)&narenas, &sz, NULL, 0);
   if (narenas != 3) {
     printf("Error: unexpected number of arenas: %d\n", narenas);
     return 1;
@@ -33,7 +33,7 @@ int test_threads()
   je_malloc_stats_print(NULL, NULL, NULL);
   size_t allocated1;
   size_t sz1 = sizeof(allocated1);
-  je_mallctl("stats.active", &allocated1, &sz1, NULL, 0);
+  je_mallctl("stats.active", (void *)&allocated1, &sz1, NULL, 0);
   printf("\nPress Enter to start threads...\n");
   getchar();
   printf("Starting %d threads x %d x %d iterations...\n", numThreads, numIter1, numIter2);
@@ -78,7 +78,7 @@ int test_threads()
   }
   je_malloc_stats_print(NULL, NULL, NULL);
   size_t allocated2;
-  je_mallctl("stats.active", &allocated2, &sz1, NULL, 0);
+  je_mallctl("stats.active", (void *)&allocated2, &sz1, NULL, 0);
   size_t leaked = allocated2 - allocated1;
   printf("\nDone. Leaked: %zd bytes\n", leaked);
   bool failed = leaked > 65536; // in case C++ runtime allocated something (e.g. iostream locale or facet)

scripts/gen_travis.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+
+from itertools import combinations
+
+travis_template = """\
+language: generic
+
+matrix:
+  include:
+%s
+
+before_script:
+  - autoconf
+  - ./configure ${COMPILER_FLAGS:+ \
+      CC="$CC $COMPILER_FLAGS" } \
+      $CONFIGURE_FLAGS
+  - make -j3
+  - make -j3 tests
+
+script:
+  - make check
+"""
+
+# The 'default' configuration is gcc, on linux, with no compiler or configure
+# flags.  We also test with clang, -m32, --enable-debug, --enable-prof,
+# --disable-stats, and --disable-tcache.  To avoid abusing travis though, we
+# don't test all 2**7 = 128 possible combinations of these; instead, we only
+# test combinations of up to 2 'unusual' settings, under the hope that bugs
+# involving interactions of such settings are rare.
+# things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29
+MAX_UNUSUAL_OPTIONS = 2
+
+os_default = 'linux'
+os_unusual = 'osx'
+
+compilers_default = 'CC=gcc'
+compilers_unusual = 'CC=clang'
+
+compiler_flag_unusuals = ['-m32']
+
+configure_flag_unusuals = [
+    '--enable-debug', '--enable-prof', '--disable-stats', '--disable-tcache',
+]
+
+all_unusuals = (
+    [os_unusual] + [compilers_unusual] + compiler_flag_unusuals
+    + configure_flag_unusuals
+)
+
+unusual_combinations_to_test = []
+for i in xrange(MAX_UNUSUAL_OPTIONS + 1):
+    unusual_combinations_to_test += combinations(all_unusuals, i)
+
+include_rows = ""
+for unusual_combination in unusual_combinations_to_test:
+    os = os_default
+    if os_unusual in unusual_combination:
+        os = os_unusual
+
+    compilers = compilers_default
+    if compilers_unusual in unusual_combination:
+        compilers = compilers_unusual
+
+    compiler_flags = [
+        x for x in unusual_combination if x in compiler_flag_unusuals]
+
+    configure_flags = [
+        x for x in unusual_combination if x in configure_flag_unusuals]
+
+    # Filter out an unsupported configuration - heap profiling on OS X.
+    if os == 'osx' and '--enable-prof' in configure_flags:
+        continue
+
+    env_string = '{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}"'.format(
+        compilers, " ".join(compiler_flags), " ".join(configure_flags))
+
+    include_rows += '    - os: %s\n' % os
+    include_rows += '      env: %s\n' % env_string
+    if '-m32' in unusual_combination and os == 'linux':
+        include_rows += '      addons:\n'
+	include_rows += '        apt:\n'
+	include_rows += '          packages:\n'
+	include_rows += '            - gcc-multilib\n'
+
+print travis_template % include_rows

src/arena.c

@@ -4,6 +4,8 @@
 /******************************************************************************/
 /* Data. */
 
+bool		opt_thp = true;
+static bool	thp_initially_huge;
 purge_mode_t	opt_purge = PURGE_DEFAULT;
 const char	*purge_mode_names[] = {
 	"ratio",
@@ -38,8 +40,8 @@ static void	arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run,
     bool dirty, bool cleaned, bool decommitted);
 static void	arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena,
     arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin);
-static void	arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
-    arena_run_t *run, arena_bin_t *bin);
+static void	arena_bin_lower_run(arena_t *arena, arena_run_t *run,
+    arena_bin_t *bin);
 
 /******************************************************************************/
 
@@ -55,8 +57,31 @@ arena_miscelm_size_get(const arena_chunk_map_misc_t *miscelm)
 	return (arena_mapbits_size_decode(mapbits));
 }
 
+JEMALLOC_INLINE_C const extent_node_t *
+arena_miscelm_extent_get(const arena_chunk_map_misc_t *miscelm)
+{
+	arena_chunk_t *chunk;
+
+	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm);
+	return (&chunk->node);
+}
+
 JEMALLOC_INLINE_C int
-arena_run_addr_comp(const arena_chunk_map_misc_t *a,
+arena_sn_comp(const arena_chunk_map_misc_t *a, const arena_chunk_map_misc_t *b)
+{
+	size_t a_sn, b_sn;
+
+	assert(a != NULL);
+	assert(b != NULL);
+
+	a_sn = extent_node_sn_get(arena_miscelm_extent_get(a));
+	b_sn = extent_node_sn_get(arena_miscelm_extent_get(b));
+
+	return ((a_sn > b_sn) - (a_sn < b_sn));
+}
+
+JEMALLOC_INLINE_C int
+arena_ad_comp(const arena_chunk_map_misc_t *a,
     const arena_chunk_map_misc_t *b)
 {
 	uintptr_t a_miscelm = (uintptr_t)a;
@@ -68,9 +93,26 @@ arena_run_addr_comp(const arena_chunk_map_misc_t *a,
 	return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm));
 }
 
+JEMALLOC_INLINE_C int
+arena_snad_comp(const arena_chunk_map_misc_t *a,
+    const arena_chunk_map_misc_t *b)
+{
+	int ret;
+
+	assert(a != NULL);
+	assert(b != NULL);
+
+	ret = arena_sn_comp(a, b);
+	if (ret != 0)
+		return (ret);
+
+	ret = arena_ad_comp(a, b);
+	return (ret);
+}
+
 /* Generate pairing heap functions. */
 ph_gen(static UNUSED, arena_run_heap_, arena_run_heap_t, arena_chunk_map_misc_t,
-    ph_link, arena_run_addr_comp)
+    ph_link, arena_snad_comp)
 
 #ifdef JEMALLOC_JET
 #undef run_quantize_floor
@@ -528,8 +570,8 @@ arena_chunk_init_spare(arena_t *arena)
 }
 
 static bool
-arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
-    bool zero)
+arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, size_t sn, bool zero,
+    bool *gdump)
 {
 
 	/*
@@ -538,9 +580,9 @@ arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
 	 * of runs is tracked individually, and upon chunk deallocation the
 	 * entire chunk is in a consistent commit state.
 	 */
-	extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true);
+	extent_node_init(&chunk->node, arena, chunk, chunksize, sn, zero, true);
 	extent_node_achunk_set(&chunk->node, true);
-	return (chunk_register(tsdn, chunk, &chunk->node));
+	return (chunk_register(chunk, &chunk->node, gdump));
 }
 
 static arena_chunk_t *
@@ -548,29 +590,37 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena,
     chunk_hooks_t *chunk_hooks, bool *zero, bool *commit)
 {
 	arena_chunk_t *chunk;
+	size_t sn;
 
 	malloc_mutex_unlock(tsdn, &arena->lock);
+	/* prof_gdump() requirement. */
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
 
 	chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks,
-	    NULL, chunksize, chunksize, zero, commit);
+	    NULL, chunksize, chunksize, &sn, zero, commit);
 	if (chunk != NULL && !*commit) {
 		/* Commit header. */
 		if (chunk_hooks->commit(chunk, chunksize, 0, map_bias <<
 		    LG_PAGE, arena->ind)) {
 			chunk_dalloc_wrapper(tsdn, arena, chunk_hooks,
-			    (void *)chunk, chunksize, *zero, *commit);
+			    (void *)chunk, chunksize, sn, *zero, *commit);
 			chunk = NULL;
 		}
 	}
-	if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, *zero)) {
-		if (!*commit) {
-			/* Undo commit of header. */
-			chunk_hooks->decommit(chunk, chunksize, 0, map_bias <<
-			    LG_PAGE, arena->ind);
+	if (chunk != NULL) {
+		bool gdump;
+		if (arena_chunk_register(arena, chunk, sn, *zero, &gdump)) {
+			if (!*commit) {
+				/* Undo commit of header. */
+				chunk_hooks->decommit(chunk, chunksize, 0,
+				    map_bias << LG_PAGE, arena->ind);
+			}
+			chunk_dalloc_wrapper(tsdn, arena, chunk_hooks,
+			    (void *)chunk, chunksize, sn, *zero, *commit);
+			chunk = NULL;
 		}
-		chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk,
-		    chunksize, *zero, *commit);
-		chunk = NULL;
+		if (config_prof && opt_prof && gdump)
+			prof_gdump(tsdn);
 	}
 
 	malloc_mutex_lock(tsdn, &arena->lock);
@@ -583,15 +633,26 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero,
 {
 	arena_chunk_t *chunk;
 	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
+	size_t sn;
+
+	/* prof_gdump() requirement. */
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1);
+	malloc_mutex_assert_owner(tsdn, &arena->lock);
 
 	chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize,
-	    chunksize, zero, commit, true);
+	    chunksize, &sn, zero, commit, true);
 	if (chunk != NULL) {
-		if (arena_chunk_register(tsdn, arena, chunk, *zero)) {
+		bool gdump;
+		if (arena_chunk_register(arena, chunk, sn, *zero, &gdump)) {
 			chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk,
-			    chunksize, true);
+			    chunksize, sn, true);
 			return (NULL);
 		}
+		if (config_prof && opt_prof && gdump) {
+			malloc_mutex_unlock(tsdn, &arena->lock);
+			prof_gdump(tsdn);
+			malloc_mutex_lock(tsdn, &arena->lock);
+		}
 	}
 	if (chunk == NULL) {
 		chunk = arena_chunk_alloc_internal_hard(tsdn, arena,
@@ -621,6 +682,10 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena)
 	if (chunk == NULL)
 		return (NULL);
 
+	if (config_thp && opt_thp) {
+		chunk->hugepage = thp_initially_huge;
+	}
+
 	/*
 	 * Initialize the map to contain one maximal free untouched run.  Mark
 	 * the pages as zeroed if arena_chunk_alloc_internal() returned a zeroed
@@ -684,11 +749,17 @@ arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena)
 static void
 arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
 {
+	size_t sn;
+	UNUSED bool hugepage JEMALLOC_CC_SILENCE_INIT(false);
 	bool committed;
 	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
 
 	chunk_deregister(chunk, &chunk->node);
 
+	sn = extent_node_sn_get(&chunk->node);
+	if (config_thp && opt_thp) {
+		hugepage = chunk->hugepage;
+	}
 	committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0);
 	if (!committed) {
 		/*
@@ -701,9 +772,20 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
 		chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE,
 		    arena->ind);
 	}
+	if (config_thp && opt_thp && hugepage != thp_initially_huge) {
+		/*
+		 * Convert chunk back to initial THP state, so that all
+		 * subsequent chunk allocations start out in a consistent state.
+		 */
+		if (thp_initially_huge) {
+			pages_huge(chunk, chunksize);
+		} else {
+			pages_nohuge(chunk, chunksize);
+		}
+	}
 
 	chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize,
-	    committed);
+	    sn, committed);
 
 	if (config_stats) {
 		arena->stats.mapped -= chunksize;
@@ -859,14 +941,14 @@ arena_node_dalloc(tsdn_t *tsdn, arena_t *arena, extent_node_t *node)
 
 static void *
 arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
-    chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero,
-    size_t csize)
+    chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, size_t *sn,
+    bool *zero, size_t csize)
 {
 	void *ret;
 	bool commit = true;
 
 	ret = chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, csize,
-	    alignment, zero, &commit);
+	    alignment, sn, zero, &commit);
 	if (ret == NULL) {
 		/* Revert optimistic stats updates. */
 		malloc_mutex_lock(tsdn, &arena->lock);
@@ -883,7 +965,7 @@ arena_chunk_alloc_huge_hard(tsdn_t *tsdn, arena_t *arena,
 
 void *
 arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
-    size_t alignment, bool *zero)
+    size_t alignment, size_t *sn, bool *zero)
 {
 	void *ret;
 	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
@@ -900,18 +982,19 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
 	arena_nactive_add(arena, usize >> LG_PAGE);
 
 	ret = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, csize,
-	    alignment, zero, &commit, true);
+	    alignment, sn, zero, &commit, true);
 	malloc_mutex_unlock(tsdn, &arena->lock);
 	if (ret == NULL) {
 		ret = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks,
-		    usize, alignment, zero, csize);
+		    usize, alignment, sn, zero, csize);
 	}
 
 	return (ret);
 }
 
 void
-arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize)
+arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize,
+    size_t sn)
 {
 	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
 	size_t csize;
@@ -924,7 +1007,7 @@ arena_chunk_dalloc_huge(tsdn_t *tsdn, arena_t *arena, void *chunk, size_t usize)
 	}
 	arena_nactive_sub(arena, usize >> LG_PAGE);
 
-	chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, true);
+	chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, csize, sn, true);
 	malloc_mutex_unlock(tsdn, &arena->lock);
 }
 
@@ -948,7 +1031,7 @@ arena_chunk_ralloc_huge_similar(tsdn_t *tsdn, arena_t *arena, void *chunk,
 
 void
 arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk,
-    size_t oldsize, size_t usize)
+    size_t oldsize, size_t usize, size_t sn)
 {
 	size_t udiff = oldsize - usize;
 	size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
@@ -967,7 +1050,7 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk,
 		    CHUNK_CEILING(usize));
 
 		chunk_dalloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff,
-		    true);
+		    sn, true);
 	}
 	malloc_mutex_unlock(tsdn, &arena->lock);
 }
@@ -975,13 +1058,13 @@ arena_chunk_ralloc_huge_shrink(tsdn_t *tsdn, arena_t *arena, void *chunk,
 static bool
 arena_chunk_ralloc_huge_expand_hard(tsdn_t *tsdn, arena_t *arena,
     chunk_hooks_t *chunk_hooks, void *chunk, size_t oldsize, size_t usize,
-    bool *zero, void *nchunk, size_t udiff, size_t cdiff)
+    size_t *sn, bool *zero, void *nchunk, size_t udiff, size_t cdiff)
 {
 	bool err;
 	bool commit = true;
 
 	err = (chunk_alloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff,
-	    chunksize, zero, &commit) == NULL);
+	    chunksize, sn, zero, &commit) == NULL);
 	if (err) {
 		/* Revert optimistic stats updates. */
 		malloc_mutex_lock(tsdn, &arena->lock);
@@ -995,7 +1078,7 @@ arena_chunk_ralloc_huge_expand_hard(tsdn_t *tsdn, arena_t *arena,
 	} else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk,
 	    cdiff, true, arena->ind)) {
 		chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, nchunk, cdiff,
-		    *zero, true);
+		    *sn, *zero, true);
 		err = true;
 	}
 	return (err);
@@ -1010,6 +1093,7 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk,
 	void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize));
 	size_t udiff = usize - oldsize;
 	size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
+	size_t sn;
 	bool commit = true;
 
 	malloc_mutex_lock(tsdn, &arena->lock);
@@ -1022,16 +1106,16 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk,
 	arena_nactive_add(arena, udiff >> LG_PAGE);
 
 	err = (chunk_alloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff,
-	    chunksize, zero, &commit, true) == NULL);
+	    chunksize, &sn, zero, &commit, true) == NULL);
 	malloc_mutex_unlock(tsdn, &arena->lock);
 	if (err) {
 		err = arena_chunk_ralloc_huge_expand_hard(tsdn, arena,
-		    &chunk_hooks, chunk, oldsize, usize, zero, nchunk, udiff,
-		    cdiff);
+		    &chunk_hooks, chunk, oldsize, usize, &sn, zero, nchunk,
+		    udiff, cdiff);
 	} else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk,
 	    cdiff, true, arena->ind)) {
 		chunk_dalloc_wrapper(tsdn, arena, &chunk_hooks, nchunk, cdiff,
-		    *zero, true);
+		    sn, *zero, true);
 		err = true;
 	}
 
@@ -1519,6 +1603,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 
 		if (rdelm == &chunkselm->rd) {
 			extent_node_t *chunkselm_next;
+			size_t sn;
 			bool zero, commit;
 			UNUSED void *chunk;
 
@@ -1536,8 +1621,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 			commit = false;
 			chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks,
 			    extent_node_addr_get(chunkselm),
-			    extent_node_size_get(chunkselm), chunksize, &zero,
-			    &commit, false);
+			    extent_node_size_get(chunkselm), chunksize, &sn,
+			    &zero, &commit, false);
 			assert(chunk == extent_node_addr_get(chunkselm));
 			assert(zero == extent_node_zeroed_get(chunkselm));
 			extent_node_dirty_insert(chunkselm, purge_runs_sentinel,
@@ -1634,6 +1719,17 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 			run_size = arena_mapbits_large_size_get(chunk, pageind);
 			npages = run_size >> LG_PAGE;
 
+			/*
+			 * If this is the first run purged within chunk, mark
+			 * the chunk as non-THP-capable.  This will prevent all
+			 * use of THPs for this chunk until the chunk as a whole
+			 * is deallocated.
+			 */
+			if (config_thp && opt_thp && chunk->hugepage) {
+				chunk->hugepage = pages_nohuge(chunk,
+				    chunksize);
+			}
+
 			assert(pageind + npages <= chunk_npages);
 			assert(!arena_mapbits_decommitted_get(chunk, pageind));
 			assert(!arena_mapbits_decommitted_get(chunk,
@@ -1703,13 +1799,14 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 			    cc_link);
 			void *addr = extent_node_addr_get(chunkselm);
 			size_t size = extent_node_size_get(chunkselm);
+			size_t sn = extent_node_sn_get(chunkselm);
 			bool zeroed = extent_node_zeroed_get(chunkselm);
 			bool committed = extent_node_committed_get(chunkselm);
 			extent_node_dirty_remove(chunkselm);
 			arena_node_dalloc(tsdn, arena, chunkselm);
 			chunkselm = chunkselm_next;
 			chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr,
-			    size, zeroed, committed);
+			    size, sn, zeroed, committed);
 		} else {
 			arena_chunk_t *chunk =
 			    (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
@@ -2315,7 +2412,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin)
 				arena_dalloc_bin_run(tsdn, arena, chunk, run,
 				    bin);
 			} else
-				arena_bin_lower_run(arena, chunk, run, bin);
+				arena_bin_lower_run(arena, run, bin);
 		}
 		return (ret);
 	}
@@ -2623,6 +2720,7 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
 		return (arena_malloc_small(tsdn, arena, ind, zero));
 	if (likely(size <= large_maxclass))
 		return (arena_malloc_large(tsdn, arena, ind, zero));
+	assert(index2size(ind) >= chunksize);
 	return (huge_malloc(tsdn, arena, index2size(ind), zero));
 }
 
@@ -2820,16 +2918,18 @@ arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
 }
 
 static void
-arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
-    arena_bin_t *bin)
+arena_bin_lower_run(arena_t *arena, arena_run_t *run, arena_bin_t *bin)
 {
 
 	/*
-	 * Make sure that if bin->runcur is non-NULL, it refers to the lowest
-	 * non-full run.  It is okay to NULL runcur out rather than proactively
-	 * keeping it pointing at the lowest non-full run.
+	 * Make sure that if bin->runcur is non-NULL, it refers to the
+	 * oldest/lowest non-full run.  It is okay to NULL runcur out rather
+	 * than proactively keeping it pointing at the oldest/lowest non-full
+	 * run.
 	 */
-	if ((uintptr_t)run < (uintptr_t)bin->runcur) {
+	if (bin->runcur != NULL &&
+	    arena_snad_comp(arena_run_to_miscelm(bin->runcur),
+	    arena_run_to_miscelm(run)) > 0) {
 		/* Switch runcur. */
 		if (bin->runcur->nfree > 0)
 			arena_bin_runs_insert(bin, bin->runcur);
@@ -2865,7 +2965,7 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
 		arena_dissociate_bin_run(chunk, run, bin);
 		arena_dalloc_bin_run(tsdn, arena, chunk, run, bin);
 	} else if (run->nfree == 1 && run != bin->runcur)
-		arena_bin_lower_run(arena, chunk, run, bin);
+		arena_bin_lower_run(arena, run, bin);
 
 	if (config_stats) {
 		bin->stats.ndalloc++;
@@ -3452,6 +3552,13 @@ arena_nthreads_dec(arena_t *arena, bool internal)
 	atomic_sub_u(&arena->nthreads[internal], 1);
 }
 
+size_t
+arena_extent_sn_next(arena_t *arena)
+{
+
+	return (atomic_add_z(&arena->extent_sn_next, 1) - 1);
+}
+
 arena_t *
 arena_new(tsdn_t *tsdn, unsigned ind)
 {
@@ -3511,6 +3618,8 @@ arena_new(tsdn_t *tsdn, unsigned ind)
 
 	ql_new(&arena->achunks);
 
+	arena->extent_sn_next = 0;
+
 	arena->spare = NULL;
 
 	arena->lg_dirty_mult = arena_lg_dirty_mult_default_get();
@@ -3532,9 +3641,9 @@ arena_new(tsdn_t *tsdn, unsigned ind)
 	    WITNESS_RANK_ARENA_HUGE))
 		return (NULL);
 
-	extent_tree_szad_new(&arena->chunks_szad_cached);
+	extent_tree_szsnad_new(&arena->chunks_szsnad_cached);
 	extent_tree_ad_new(&arena->chunks_ad_cached);
-	extent_tree_szad_new(&arena->chunks_szad_retained);
+	extent_tree_szsnad_new(&arena->chunks_szsnad_retained);
 	extent_tree_ad_new(&arena->chunks_ad_retained);
 	if (malloc_mutex_init(&arena->chunks_mtx, "arena_chunks",
 	    WITNESS_RANK_ARENA_CHUNKS))
@@ -3673,11 +3782,78 @@ bin_info_init(void)
 #undef SC
 }
 
+static void
+init_thp_initially_huge(void) {
+	int fd;
+	char buf[sizeof("[always] madvise never\n")];
+	ssize_t nread;
+	static const char *enabled_states[] = {
+		"[always] madvise never\n",
+		"always [madvise] never\n",
+		"always madvise [never]\n"
+	};
+	static const bool thp_initially_huge_states[] = {
+		true,
+		false,
+		false
+	};
+	unsigned i;
+
+	if (config_debug) {
+		for (i = 0; i < sizeof(enabled_states)/sizeof(const char *);
+		    i++) {
+			assert(sizeof(buf) > strlen(enabled_states[i]));
+		}
+	}
+	assert(sizeof(enabled_states)/sizeof(const char *) ==
+	    sizeof(thp_initially_huge_states)/sizeof(bool));
+
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
+	fd = (int)syscall(SYS_open,
+	    "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+#else
+	fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+#endif
+	if (fd == -1) {
+		goto label_error;
+	}
+
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
+	nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
+#else
+	nread = read(fd, &buf, sizeof(buf));
+#endif
+
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
+	syscall(SYS_close, fd);
+#else
+	close(fd);
+#endif
+
+	if (nread < 1) {
+		goto label_error;
+	}
+	for (i = 0; i < sizeof(enabled_states)/sizeof(const char *);
+	    i++) {
+		if (strncmp(buf, enabled_states[i], (size_t)nread) == 0) {
+			thp_initially_huge = thp_initially_huge_states[i];
+			return;
+		}
+	}
+
+label_error:
+	thp_initially_huge = false;
+}
+
 void
 arena_boot(void)
 {
 	unsigned i;
 
+	if (config_thp && opt_thp) {
+		init_thp_initially_huge();
+	}
+
 	arena_lg_dirty_mult_default_set(opt_lg_dirty_mult);
 	arena_decay_time_default_set(opt_decay_time);
 
@@ -3708,15 +3884,8 @@ arena_boot(void)
 	arena_maxrun = chunksize - (map_bias << LG_PAGE);
 	assert(arena_maxrun > 0);
 	large_maxclass = index2size(size2index(chunksize)-1);
-	if (large_maxclass > arena_maxrun) {
-		/*
-		 * For small chunk sizes it's possible for there to be fewer
-		 * non-header pages available than are necessary to serve the
-		 * size classes just below chunksize.
-		 */
-		large_maxclass = arena_maxrun;
-	}
 	assert(large_maxclass > 0);
+	assert(large_maxclass + large_pad <= arena_maxrun);
 	nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS);
 	nhclasses = NSIZES - nlclasses - NBINS;
 

src/base.c

@@ -5,7 +5,8 @@
 /* Data. */
 
 static malloc_mutex_t	base_mtx;
-static extent_tree_t	base_avail_szad;
+static size_t		base_extent_sn_next;
+static extent_tree_t	base_avail_szsnad;
 static extent_node_t	*base_nodes;
 static size_t		base_allocated;
 static size_t		base_resident;
@@ -39,6 +40,14 @@ base_node_dalloc(tsdn_t *tsdn, extent_node_t *node)
 	base_nodes = node;
 }
 
+static void
+base_extent_node_init(extent_node_t *node, void *addr, size_t size)
+{
+	size_t sn = atomic_add_z(&base_extent_sn_next, 1) - 1;
+
+	extent_node_init(node, NULL, addr, size, sn, true, true);
+}
+
 static extent_node_t *
 base_chunk_alloc(tsdn_t *tsdn, size_t minsize)
 {
@@ -68,7 +77,7 @@ base_chunk_alloc(tsdn_t *tsdn, size_t minsize)
 			base_resident += PAGE_CEILING(nsize);
 		}
 	}
-	extent_node_init(node, NULL, addr, csize, true, true);
+	base_extent_node_init(node, addr, csize);
 	return (node);
 }
 
@@ -92,12 +101,12 @@ base_alloc(tsdn_t *tsdn, size_t size)
 	csize = CACHELINE_CEILING(size);
 
 	usize = s2u(csize);
-	extent_node_init(&key, NULL, NULL, usize, false, false);
+	extent_node_init(&key, NULL, NULL, usize, 0, false, false);
 	malloc_mutex_lock(tsdn, &base_mtx);
-	node = extent_tree_szad_nsearch(&base_avail_szad, &key);
+	node = extent_tree_szsnad_nsearch(&base_avail_szsnad, &key);
 	if (node != NULL) {
 		/* Use existing space. */
-		extent_tree_szad_remove(&base_avail_szad, node);
+		extent_tree_szsnad_remove(&base_avail_szsnad, node);
 	} else {
 		/* Try to allocate more space. */
 		node = base_chunk_alloc(tsdn, csize);
@@ -111,7 +120,7 @@ base_alloc(tsdn_t *tsdn, size_t size)
 	if (extent_node_size_get(node) > csize) {
 		extent_node_addr_set(node, (void *)((uintptr_t)ret + csize));
 		extent_node_size_set(node, extent_node_size_get(node) - csize);
-		extent_tree_szad_insert(&base_avail_szad, node);
+		extent_tree_szsnad_insert(&base_avail_szsnad, node);
 	} else
 		base_node_dalloc(tsdn, node);
 	if (config_stats) {
@@ -149,7 +158,8 @@ base_boot(void)
 
 	if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE))
 		return (true);
-	extent_tree_szad_new(&base_avail_szad);
+	base_extent_sn_next = 0;
+	extent_tree_szsnad_new(&base_avail_szsnad);
 	base_nodes = NULL;
 
 	return (false);

src/chunk.c

@@ -50,9 +50,9 @@ const chunk_hooks_t	chunk_hooks_default = {
  */
 
 static void	chunk_record(tsdn_t *tsdn, arena_t *arena,
-    chunk_hooks_t *chunk_hooks, extent_tree_t *chunks_szad,
-    extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, bool zeroed,
-    bool committed);
+    chunk_hooks_t *chunk_hooks, extent_tree_t *chunks_szsnad,
+    extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, size_t sn,
+    bool zeroed, bool committed);
 
 /******************************************************************************/
 
@@ -141,7 +141,7 @@ chunk_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
 }
 
 bool
-chunk_register(tsdn_t *tsdn, const void *chunk, const extent_node_t *node)
+chunk_register(const void *chunk, const extent_node_t *node, bool *gdump)
 {
 
 	assert(extent_node_addr_get(node) == chunk);
@@ -160,8 +160,7 @@ chunk_register(tsdn_t *tsdn, const void *chunk, const extent_node_t *node)
 			 */
 			high = atomic_read_z(&highchunks);
 		}
-		if (cur > high && prof_gdump_get_unlocked())
-			prof_gdump(tsdn);
+		*gdump = (cur > high && prof_gdump_get_unlocked());
 	}
 
 	return (false);
@@ -183,33 +182,40 @@ chunk_deregister(const void *chunk, const extent_node_t *node)
 }
 
 /*
- * Do first-best-fit chunk selection, i.e. select the lowest chunk that best
- * fits.
+ * Do first-best-fit chunk selection, i.e. select the oldest/lowest chunk that
+ * best fits.
  */
 static extent_node_t *
-chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szad,
-    extent_tree_t *chunks_ad, size_t size)
+chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szsnad, size_t size)
 {
+	extent_node_t *node;
+	size_t qsize;
 	extent_node_t key;
 
 	assert(size == CHUNK_CEILING(size));
 
-	extent_node_init(&key, arena, NULL, size, false, false);
-	return (extent_tree_szad_nsearch(chunks_szad, &key));
+	qsize = extent_size_quantize_ceil(size);
+	extent_node_init(&key, arena, NULL, qsize, 0, false, false);
+	node = extent_tree_szsnad_nsearch(chunks_szsnad, &key);
+	assert(node == NULL || extent_node_size_get(node) >= size);
+	return node;
 }
 
 static void *
 chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
-    extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache,
-    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
-    bool dalloc_node)
+    extent_tree_t *chunks_szsnad, extent_tree_t *chunks_ad, bool cache,
+    void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero,
+    bool *commit, bool dalloc_node)
 {
 	void *ret;
 	extent_node_t *node;
 	size_t alloc_size, leadsize, trailsize;
 	bool zeroed, committed;
 
+	assert(CHUNK_CEILING(size) == size);
+	assert(alignment > 0);
 	assert(new_addr == NULL || alignment == chunksize);
+	assert(CHUNK_ADDR2BASE(new_addr) == new_addr);
 	/*
 	 * Cached chunks use the node linkage embedded in their headers, in
 	 * which case dalloc_node is true, and new_addr is non-NULL because
@@ -217,7 +223,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 	 */
 	assert(dalloc_node || new_addr != NULL);
 
-	alloc_size = CHUNK_CEILING(s2u(size + alignment - chunksize));
+	alloc_size = size + CHUNK_CEILING(alignment) - chunksize;
 	/* Beware size_t wrap-around. */
 	if (alloc_size < size)
 		return (NULL);
@@ -225,12 +231,11 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 	chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks);
 	if (new_addr != NULL) {
 		extent_node_t key;
-		extent_node_init(&key, arena, new_addr, alloc_size, false,
+		extent_node_init(&key, arena, new_addr, alloc_size, 0, false,
 		    false);
 		node = extent_tree_ad_search(chunks_ad, &key);
 	} else {
-		node = chunk_first_best_fit(arena, chunks_szad, chunks_ad,
-		    alloc_size);
+		node = chunk_first_best_fit(arena, chunks_szsnad, alloc_size);
 	}
 	if (node == NULL || (new_addr != NULL && extent_node_size_get(node) <
 	    size)) {
@@ -243,6 +248,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 	assert(extent_node_size_get(node) >= leadsize + size);
 	trailsize = extent_node_size_get(node) - leadsize - size;
 	ret = (void *)((uintptr_t)extent_node_addr_get(node) + leadsize);
+	*sn = extent_node_sn_get(node);
 	zeroed = extent_node_zeroed_get(node);
 	if (zeroed)
 		*zero = true;
@@ -257,13 +263,13 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 		return (NULL);
 	}
 	/* Remove node from the tree. */
-	extent_tree_szad_remove(chunks_szad, node);
+	extent_tree_szsnad_remove(chunks_szsnad, node);
 	extent_tree_ad_remove(chunks_ad, node);
 	arena_chunk_cache_maybe_remove(arena, node, cache);
 	if (leadsize != 0) {
 		/* Insert the leading space as a smaller chunk. */
 		extent_node_size_set(node, leadsize);
-		extent_tree_szad_insert(chunks_szad, node);
+		extent_tree_szsnad_insert(chunks_szsnad, node);
 		extent_tree_ad_insert(chunks_ad, node);
 		arena_chunk_cache_maybe_insert(arena, node, cache);
 		node = NULL;
@@ -275,9 +281,9 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 			if (dalloc_node && node != NULL)
 				arena_node_dalloc(tsdn, arena, node);
 			malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
-			chunk_record(tsdn, arena, chunk_hooks, chunks_szad,
-			    chunks_ad, cache, ret, size + trailsize, zeroed,
-			    committed);
+			chunk_record(tsdn, arena, chunk_hooks, chunks_szsnad,
+			    chunks_ad, cache, ret, size + trailsize, *sn,
+			    zeroed, committed);
 			return (NULL);
 		}
 		/* Insert the trailing space as a smaller chunk. */
@@ -286,22 +292,22 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 			if (node == NULL) {
 				malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
 				chunk_record(tsdn, arena, chunk_hooks,
-				    chunks_szad, chunks_ad, cache, ret, size +
-				    trailsize, zeroed, committed);
+				    chunks_szsnad, chunks_ad, cache, ret, size
+				    + trailsize, *sn, zeroed, committed);
 				return (NULL);
 			}
 		}
 		extent_node_init(node, arena, (void *)((uintptr_t)(ret) + size),
-		    trailsize, zeroed, committed);
-		extent_tree_szad_insert(chunks_szad, node);
+		    trailsize, *sn, zeroed, committed);
+		extent_tree_szsnad_insert(chunks_szsnad, node);
 		extent_tree_ad_insert(chunks_ad, node);
 		arena_chunk_cache_maybe_insert(arena, node, cache);
 		node = NULL;
 	}
 	if (!committed && chunk_hooks->commit(ret, size, 0, size, arena->ind)) {
 		malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
-		chunk_record(tsdn, arena, chunk_hooks, chunks_szad, chunks_ad,
-		    cache, ret, size, zeroed, committed);
+		chunk_record(tsdn, arena, chunk_hooks, chunks_szsnad, chunks_ad,
+		    cache, ret, size, *sn, zeroed, committed);
 		return (NULL);
 	}
 	malloc_mutex_unlock(tsdn, &arena->chunks_mtx);
@@ -385,8 +391,8 @@ chunk_alloc_base(size_t size)
 
 void *
 chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
-    bool dalloc_node)
+    void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero,
+    bool *commit, bool dalloc_node)
 {
 	void *ret;
 
@@ -396,8 +402,8 @@ chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 	assert((alignment & chunksize_mask) == 0);
 
 	ret = chunk_recycle(tsdn, arena, chunk_hooks,
-	    &arena->chunks_szad_cached, &arena->chunks_ad_cached, true,
-	    new_addr, size, alignment, zero, commit, dalloc_node);
+	    &arena->chunks_szsnad_cached, &arena->chunks_ad_cached, true,
+	    new_addr, size, alignment, sn, zero, commit, dalloc_node);
 	if (ret == NULL)
 		return (NULL);
 	if (config_valgrind)
@@ -451,7 +457,8 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
 
 static void *
 chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
+    void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero,
+    bool *commit)
 {
 	void *ret;
 
@@ -461,8 +468,8 @@ chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 	assert((alignment & chunksize_mask) == 0);
 
 	ret = chunk_recycle(tsdn, arena, chunk_hooks,
-	    &arena->chunks_szad_retained, &arena->chunks_ad_retained, false,
-	    new_addr, size, alignment, zero, commit, true);
+	    &arena->chunks_szsnad_retained, &arena->chunks_ad_retained, false,
+	    new_addr, size, alignment, sn, zero, commit, true);
 
 	if (config_stats && ret != NULL)
 		arena->stats.retained -= size;
@@ -472,14 +479,15 @@ chunk_alloc_retained(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 
 void *
 chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
+    void *new_addr, size_t size, size_t alignment, size_t *sn, bool *zero,
+    bool *commit)
 {
 	void *ret;
 
 	chunk_hooks_assure_initialized(tsdn, arena, chunk_hooks);
 
 	ret = chunk_alloc_retained(tsdn, arena, chunk_hooks, new_addr, size,
-	    alignment, zero, commit);
+	    alignment, sn, zero, commit);
 	if (ret == NULL) {
 		if (chunk_hooks->alloc == chunk_alloc_default) {
 			/* Call directly to propagate tsdn. */
@@ -493,6 +501,8 @@ chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 		if (ret == NULL)
 			return (NULL);
 
+		*sn = arena_extent_sn_next(arena);
+
 		if (config_valgrind && chunk_hooks->alloc !=
 		    chunk_alloc_default)
 			JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, chunksize);
@@ -503,8 +513,8 @@ chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 
 static void
 chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
-    extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache,
-    void *chunk, size_t size, bool zeroed, bool committed)
+    extent_tree_t *chunks_szsnad, extent_tree_t *chunks_ad, bool cache,
+    void *chunk, size_t size, size_t sn, bool zeroed, bool committed)
 {
 	bool unzeroed;
 	extent_node_t *node, *prev;
@@ -516,7 +526,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 
 	malloc_mutex_lock(tsdn, &arena->chunks_mtx);
 	chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks);
-	extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0,
+	extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0, 0,
 	    false, false);
 	node = extent_tree_ad_nsearch(chunks_ad, &key);
 	/* Try to coalesce forward. */
@@ -528,15 +538,17 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 		/*
 		 * Coalesce chunk with the following address range.  This does
 		 * not change the position within chunks_ad, so only
-		 * remove/insert from/into chunks_szad.
+		 * remove/insert from/into chunks_szsnad.
 		 */
-		extent_tree_szad_remove(chunks_szad, node);
+		extent_tree_szsnad_remove(chunks_szsnad, node);
 		arena_chunk_cache_maybe_remove(arena, node, cache);
 		extent_node_addr_set(node, chunk);
 		extent_node_size_set(node, size + extent_node_size_get(node));
+		if (sn < extent_node_sn_get(node))
+			extent_node_sn_set(node, sn);
 		extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
 		    !unzeroed);
-		extent_tree_szad_insert(chunks_szad, node);
+		extent_tree_szsnad_insert(chunks_szsnad, node);
 		arena_chunk_cache_maybe_insert(arena, node, cache);
 	} else {
 		/* Coalescing forward failed, so insert a new node. */
@@ -554,10 +566,10 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 			}
 			goto label_return;
 		}
-		extent_node_init(node, arena, chunk, size, !unzeroed,
+		extent_node_init(node, arena, chunk, size, sn, !unzeroed,
 		    committed);
 		extent_tree_ad_insert(chunks_ad, node);
-		extent_tree_szad_insert(chunks_szad, node);
+		extent_tree_szsnad_insert(chunks_szsnad, node);
 		arena_chunk_cache_maybe_insert(arena, node, cache);
 	}
 
@@ -571,19 +583,21 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 		/*
 		 * Coalesce chunk with the previous address range.  This does
 		 * not change the position within chunks_ad, so only
-		 * remove/insert node from/into chunks_szad.
+		 * remove/insert node from/into chunks_szsnad.
 		 */
-		extent_tree_szad_remove(chunks_szad, prev);
+		extent_tree_szsnad_remove(chunks_szsnad, prev);
 		extent_tree_ad_remove(chunks_ad, prev);
 		arena_chunk_cache_maybe_remove(arena, prev, cache);
-		extent_tree_szad_remove(chunks_szad, node);
+		extent_tree_szsnad_remove(chunks_szsnad, node);
 		arena_chunk_cache_maybe_remove(arena, node, cache);
 		extent_node_addr_set(node, extent_node_addr_get(prev));
 		extent_node_size_set(node, extent_node_size_get(prev) +
 		    extent_node_size_get(node));
+		if (extent_node_sn_get(prev) < extent_node_sn_get(node))
+			extent_node_sn_set(node, extent_node_sn_get(prev));
 		extent_node_zeroed_set(node, extent_node_zeroed_get(prev) &&
 		    extent_node_zeroed_get(node));
-		extent_tree_szad_insert(chunks_szad, node);
+		extent_tree_szsnad_insert(chunks_szsnad, node);
 		arena_chunk_cache_maybe_insert(arena, node, cache);
 
 		arena_node_dalloc(tsdn, arena, prev);
@@ -595,7 +609,7 @@ label_return:
 
 void
 chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *chunk, size_t size, bool committed)
+    void *chunk, size_t size, size_t sn, bool committed)
 {
 
 	assert(chunk != NULL);
@@ -603,8 +617,9 @@ chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 	assert(size != 0);
 	assert((size & chunksize_mask) == 0);
 
-	chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szad_cached,
-	    &arena->chunks_ad_cached, true, chunk, size, false, committed);
+	chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szsnad_cached,
+	    &arena->chunks_ad_cached, true, chunk, size, sn, false,
+	    committed);
 	arena_maybe_purge(tsdn, arena);
 }
 
@@ -627,7 +642,7 @@ chunk_dalloc_default(void *chunk, size_t size, bool committed,
 
 void
 chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
-    void *chunk, size_t size, bool zeroed, bool committed)
+    void *chunk, size_t size, size_t sn, bool zeroed, bool committed)
 {
 	bool err;
 
@@ -653,8 +668,9 @@ chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
 	}
 	zeroed = !committed || !chunk_hooks->purge(chunk, size, 0, size,
 	    arena->ind);
-	chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szad_retained,
-	    &arena->chunks_ad_retained, false, chunk, size, zeroed, committed);
+	chunk_record(tsdn, arena, chunk_hooks, &arena->chunks_szsnad_retained,
+	    &arena->chunks_ad_retained, false, chunk, size, sn, zeroed,
+	    committed);
 
 	if (config_stats)
 		arena->stats.retained += size;

src/chunk_dss.c

@@ -115,8 +115,9 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 		 * malloc.
 		 */
 		while (true) {
-			void *ret, *cpad, *max_cur, *dss_next, *dss_prev;
-			size_t gap_size, cpad_size;
+			void *ret, *max_cur, *dss_next, *dss_prev;
+			void *gap_addr_chunk, *gap_addr_subchunk;
+			size_t gap_size_chunk, gap_size_subchunk;
 			intptr_t incr;
 
 			max_cur = chunk_dss_max_update(new_addr);
@@ -124,25 +125,32 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 				goto label_oom;
 
 			/*
-			 * Calculate how much padding is necessary to
-			 * chunk-align the end of the DSS.
-			 */
-			gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) &
-			    chunksize_mask;
-			/*
-			 * Compute how much chunk-aligned pad space (if any) is
+			 * Compute how much chunk-aligned gap space (if any) is
 			 * necessary to satisfy alignment.  This space can be
 			 * recycled for later use.
 			 */
-			cpad = (void *)((uintptr_t)dss_max + gap_size);
-			ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max,
-			    alignment);
-			cpad_size = (uintptr_t)ret - (uintptr_t)cpad;
+			gap_addr_chunk = (void *)(CHUNK_CEILING(
+			    (uintptr_t)max_cur));
+			ret = (void *)ALIGNMENT_CEILING(
+			    (uintptr_t)gap_addr_chunk, alignment);
+			gap_size_chunk = (uintptr_t)ret -
+			    (uintptr_t)gap_addr_chunk;
+			/*
+			 * Compute the address just past the end of the desired
+			 * allocation space.
+			 */
 			dss_next = (void *)((uintptr_t)ret + size);
-			if ((uintptr_t)ret < (uintptr_t)dss_max ||
-			    (uintptr_t)dss_next < (uintptr_t)dss_max)
+			if ((uintptr_t)ret < (uintptr_t)max_cur ||
+			    (uintptr_t)dss_next < (uintptr_t)max_cur)
 				goto label_oom; /* Wrap-around. */
-			incr = gap_size + cpad_size + size;
+			/* Compute the increment, including subchunk bytes. */
+			gap_addr_subchunk = max_cur;
+			gap_size_subchunk = (uintptr_t)ret -
+			    (uintptr_t)gap_addr_subchunk;
+			incr = gap_size_subchunk + size;
+
+			assert((uintptr_t)max_cur + incr == (uintptr_t)ret +
+			    size);
 
 			/*
 			 * Optimistically update dss_max, and roll back below if
@@ -157,12 +165,14 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
 			dss_prev = chunk_dss_sbrk(incr);
 			if (dss_prev == max_cur) {
 				/* Success. */
-				if (cpad_size != 0) {
+				if (gap_size_chunk != 0) {
 					chunk_hooks_t chunk_hooks =
 					    CHUNK_HOOKS_INITIALIZER;
 					chunk_dalloc_wrapper(tsdn, arena,
-					    &chunk_hooks, cpad, cpad_size,
-					    false, true);
+					    &chunk_hooks, gap_addr_chunk,
+					    gap_size_chunk,
+					    arena_extent_sn_next(arena), false,
+					    true);
 				}
 				if (*zero) {
 					JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(

src/ctl.c

@@ -84,6 +84,7 @@ CTL_PROTO(config_prof_libgcc)
 CTL_PROTO(config_prof_libunwind)
 CTL_PROTO(config_stats)
 CTL_PROTO(config_tcache)
+CTL_PROTO(config_thp)
 CTL_PROTO(config_tls)
 CTL_PROTO(config_utrace)
 CTL_PROTO(config_valgrind)
@@ -104,6 +105,7 @@ CTL_PROTO(opt_utrace)
 CTL_PROTO(opt_xmalloc)
 CTL_PROTO(opt_tcache)
 CTL_PROTO(opt_lg_tcache_max)
+CTL_PROTO(opt_thp)
 CTL_PROTO(opt_prof)
 CTL_PROTO(opt_prof_prefix)
 CTL_PROTO(opt_prof_active)
@@ -258,6 +260,7 @@ static const ctl_named_node_t	config_node[] = {
 	{NAME("prof_libunwind"), CTL(config_prof_libunwind)},
 	{NAME("stats"),		CTL(config_stats)},
 	{NAME("tcache"),	CTL(config_tcache)},
+	{NAME("thp"),		CTL(config_thp)},
 	{NAME("tls"),		CTL(config_tls)},
 	{NAME("utrace"),	CTL(config_utrace)},
 	{NAME("valgrind"),	CTL(config_valgrind)},
@@ -281,6 +284,7 @@ static const ctl_named_node_t opt_node[] = {
 	{NAME("xmalloc"),	CTL(opt_xmalloc)},
 	{NAME("tcache"),	CTL(opt_tcache)},
 	{NAME("lg_tcache_max"),	CTL(opt_lg_tcache_max)},
+	{NAME("thp"),		CTL(opt_thp)},
 	{NAME("prof"),		CTL(opt_prof)},
 	{NAME("prof_prefix"),	CTL(opt_prof_prefix)},
 	{NAME("prof_active"),	CTL(opt_prof_active)},
@@ -1268,6 +1272,7 @@ CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
 CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
 CTL_RO_CONFIG_GEN(config_stats, bool)
 CTL_RO_CONFIG_GEN(config_tcache, bool)
+CTL_RO_CONFIG_GEN(config_thp, bool)
 CTL_RO_CONFIG_GEN(config_tls, bool)
 CTL_RO_CONFIG_GEN(config_utrace, bool)
 CTL_RO_CONFIG_GEN(config_valgrind, bool)
@@ -1291,6 +1296,7 @@ CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
 CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
 CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
 CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
+CTL_RO_NL_CGEN(config_thp, opt_thp, opt_thp, bool)
 CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
 CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
 CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool)
@@ -1476,7 +1482,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 	if (!config_tcache)
 		return (ENOENT);
 
-	malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
 	READONLY();
 	if (tcaches_create(tsd, &tcache_ind)) {
 		ret = EFAULT;
@@ -1486,8 +1491,7 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
 
 	ret = 0;
 label_return:
-	malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
-	return (ret);
+	return ret;
 }
 
 static int

src/extent.c

@@ -3,42 +3,64 @@
 
 /******************************************************************************/
 
-JEMALLOC_INLINE_C size_t
-extent_quantize(size_t size)
-{
+#ifndef JEMALLOC_JET
+static
+#endif
+size_t
+extent_size_quantize_floor(size_t size) {
+	size_t ret;
+	szind_t ind;
 
-	/*
-	 * Round down to the nearest chunk size that can actually be requested
-	 * during normal huge allocation.
-	 */
-	return (index2size(size2index(size + 1) - 1));
-}
+	assert(size > 0);
 
-JEMALLOC_INLINE_C int
-extent_szad_comp(const extent_node_t *a, const extent_node_t *b)
-{
-	int ret;
-	size_t a_qsize = extent_quantize(extent_node_size_get(a));
-	size_t b_qsize = extent_quantize(extent_node_size_get(b));
-
-	/*
-	 * Compare based on quantized size rather than size, in order to sort
-	 * equally useful extents only by address.
-	 */
-	ret = (a_qsize > b_qsize) - (a_qsize < b_qsize);
-	if (ret == 0) {
-		uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a);
-		uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b);
-
-		ret = (a_addr > b_addr) - (a_addr < b_addr);
+	ind = size2index(size + 1);
+	if (ind == 0) {
+		/* Avoid underflow. */
+		return (index2size(0));
 	}
-
+	ret = index2size(ind - 1);
+	assert(ret <= size);
 	return (ret);
 }
 
-/* Generate red-black tree functions. */
-rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, szad_link,
-    extent_szad_comp)
+size_t
+extent_size_quantize_ceil(size_t size) {
+	size_t ret;
+
+	assert(size > 0);
+
+	ret = extent_size_quantize_floor(size);
+	if (ret < size) {
+		/*
+		 * Skip a quantization that may have an adequately large extent,
+		 * because under-sized extents may be mixed in.  This only
+		 * happens when an unusual size is requested, i.e. for aligned
+		 * allocation, and is just one of several places where linear
+		 * search would potentially find sufficiently aligned available
+		 * memory somewhere lower.
+		 */
+		ret = index2size(size2index(ret  + 1));
+	}
+	return ret;
+}
+
+JEMALLOC_INLINE_C int
+extent_sz_comp(const extent_node_t *a, const extent_node_t *b)
+{
+	size_t a_qsize = extent_size_quantize_floor(extent_node_size_get(a));
+	size_t b_qsize = extent_size_quantize_floor(extent_node_size_get(b));
+
+	return ((a_qsize > b_qsize) - (a_qsize < b_qsize));
+}
+
+JEMALLOC_INLINE_C int
+extent_sn_comp(const extent_node_t *a, const extent_node_t *b)
+{
+	size_t a_sn = extent_node_sn_get(a);
+	size_t b_sn = extent_node_sn_get(b);
+
+	return ((a_sn > b_sn) - (a_sn < b_sn));
+}
 
 JEMALLOC_INLINE_C int
 extent_ad_comp(const extent_node_t *a, const extent_node_t *b)
@@ -49,5 +71,26 @@ extent_ad_comp(const extent_node_t *a, const extent_node_t *b)
 	return ((a_addr > b_addr) - (a_addr < b_addr));
 }
 
+JEMALLOC_INLINE_C int
+extent_szsnad_comp(const extent_node_t *a, const extent_node_t *b)
+{
+	int ret;
+
+	ret = extent_sz_comp(a, b);
+	if (ret != 0)
+		return (ret);
+
+	ret = extent_sn_comp(a, b);
+	if (ret != 0)
+		return (ret);
+
+	ret = extent_ad_comp(a, b);
+	return (ret);
+}
+
+/* Generate red-black tree functions. */
+rb_gen(, extent_tree_szsnad_, extent_tree_t, extent_node_t, szsnad_link,
+    extent_szsnad_comp)
+
 /* Generate red-black tree functions. */
 rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, ad_link, extent_ad_comp)

src/huge.c

@@ -15,20 +15,20 @@ huge_node_get(const void *ptr)
 }
 
 static bool
-huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
+huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node, bool *gdump)
 {
 
 	assert(extent_node_addr_get(node) == ptr);
 	assert(!extent_node_achunk_get(node));
-	return (chunk_register(tsdn, ptr, node));
+	return (chunk_register(ptr, node, gdump));
 }
 
 static void
-huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
+huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node, bool *gdump)
 {
 	bool err;
 
-	err = huge_node_set(tsdn, ptr, node);
+	err = huge_node_set(tsdn, ptr, node, gdump);
 	assert(!err);
 }
 
@@ -56,11 +56,14 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
 	size_t ausize;
 	arena_t *iarena;
 	extent_node_t *node;
-	bool is_zeroed;
+	size_t sn;
+	bool is_zeroed, gdump;
 
 	/* Allocate one or more contiguous chunks for this request. */
 
 	assert(!tsdn_null(tsdn) || arena != NULL);
+	/* prof_gdump() requirement. */
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
 
 	ausize = sa2u(usize, alignment);
 	if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
@@ -68,7 +71,8 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
 	assert(ausize >= chunksize);
 
 	/* Allocate an extent node with which to track the chunk. */
-	iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) : a0get();
+	iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) :
+	    a0get();
 	node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
 	    CACHELINE, false, NULL, true, iarena);
 	if (node == NULL)
@@ -82,18 +86,20 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
 	if (likely(!tsdn_null(tsdn)))
 		arena = arena_choose(tsdn_tsd(tsdn), arena);
 	if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
-	    arena, usize, alignment, &is_zeroed)) == NULL) {
+	    arena, usize, alignment, &sn, &is_zeroed)) == NULL) {
 		idalloctm(tsdn, node, NULL, true, true);
 		return (NULL);
 	}
 
-	extent_node_init(node, arena, ret, usize, is_zeroed, true);
+	extent_node_init(node, arena, ret, usize, sn, is_zeroed, true);
 
-	if (huge_node_set(tsdn, ret, node)) {
-		arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
+	if (huge_node_set(tsdn, ret, node, &gdump)) {
+		arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn);
 		idalloctm(tsdn, node, NULL, true, true);
 		return (NULL);
 	}
+	if (config_prof && opt_prof && gdump)
+		prof_gdump(tsdn);
 
 	/* Insert node into huge. */
 	malloc_mutex_lock(tsdn, &arena->huge_mtx);
@@ -142,7 +148,10 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
 	extent_node_t *node;
 	arena_t *arena;
 	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
-	bool pre_zeroed, post_zeroed;
+	bool pre_zeroed, post_zeroed, gdump;
+
+	/* prof_gdump() requirement. */
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
 
 	/* Increase usize to incorporate extra. */
 	for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
@@ -176,10 +185,13 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
 	huge_node_unset(ptr, node);
 	assert(extent_node_size_get(node) != usize);
 	extent_node_size_set(node, usize);
-	huge_node_reset(tsdn, ptr, node);
+	huge_node_reset(tsdn, ptr, node, &gdump);
 	/* Update zeroed. */
 	extent_node_zeroed_set(node, post_zeroed);
 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+	/* gdump without any locks held. */
+	if (config_prof && opt_prof && gdump)
+		prof_gdump(tsdn);
 
 	arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
 
@@ -205,7 +217,7 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
 	arena_t *arena;
 	chunk_hooks_t chunk_hooks;
 	size_t cdiff;
-	bool pre_zeroed, post_zeroed;
+	bool pre_zeroed, post_zeroed, gdump;
 
 	node = huge_node_get(ptr);
 	arena = extent_node_arena_get(node);
@@ -213,6 +225,8 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
 	chunk_hooks = chunk_hooks_get(tsdn, arena);
 
 	assert(oldsize > usize);
+	/* prof_gdump() requirement. */
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
 
 	/* Split excess chunks. */
 	cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
@@ -239,13 +253,17 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
 	/* Update the size of the huge allocation. */
 	huge_node_unset(ptr, node);
 	extent_node_size_set(node, usize);
-	huge_node_reset(tsdn, ptr, node);
+	huge_node_reset(tsdn, ptr, node, &gdump);
 	/* Update zeroed. */
 	extent_node_zeroed_set(node, post_zeroed);
 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+	/* gdump without any locks held. */
+	if (config_prof && opt_prof && gdump)
+		prof_gdump(tsdn);
 
 	/* Zap the excess chunks. */
-	arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
+	arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize,
+	    extent_node_sn_get(node));
 
 	return (false);
 }
@@ -255,7 +273,7 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
     size_t usize, bool zero) {
 	extent_node_t *node;
 	arena_t *arena;
-	bool is_zeroed_subchunk, is_zeroed_chunk;
+	bool is_zeroed_subchunk, is_zeroed_chunk, gdump;
 
 	node = huge_node_get(ptr);
 	arena = extent_node_arena_get(node);
@@ -263,6 +281,9 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
 	is_zeroed_subchunk = extent_node_zeroed_get(node);
 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
 
+	/* prof_gdump() requirement. */
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
+
 	/*
 	 * Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
 	 * update extent's zeroed field, and zero as necessary.
@@ -277,8 +298,11 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
 	extent_node_size_set(node, usize);
 	extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
 	    is_zeroed_chunk);
-	huge_node_reset(tsdn, ptr, node);
+	huge_node_reset(tsdn, ptr, node, &gdump);
 	malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+	/* gdump without any locks held. */
+	if (config_prof && opt_prof && gdump)
+		prof_gdump(tsdn);
 
 	if (zero || (config_fill && unlikely(opt_zero))) {
 		if (!is_zeroed_subchunk) {
@@ -407,7 +431,8 @@ huge_dalloc(tsdn_t *tsdn, void *ptr)
 	huge_dalloc_junk(extent_node_addr_get(node),
 	    extent_node_size_get(node));
 	arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
-	    extent_node_addr_get(node), extent_node_size_get(node));
+	    extent_node_addr_get(node), extent_node_size_get(node),
+	    extent_node_sn_get(node));
 	idalloctm(tsdn, node, NULL, true, true);
 
 	arena_decay_tick(tsdn, arena);

src/jemalloc.c

@@ -790,18 +790,19 @@ stats_print_atexit(void)
  * Begin initialization functions.
  */
 
-#ifndef JEMALLOC_HAVE_SECURE_GETENV
 static char *
-secure_getenv(const char *name)
+jemalloc_secure_getenv(const char *name)
 {
-
+#ifdef JEMALLOC_HAVE_SECURE_GETENV
+	return secure_getenv(name);
+#else
 #  ifdef JEMALLOC_HAVE_ISSETUGID
 	if (issetugid() != 0)
 		return (NULL);
 #  endif
 	return (getenv(name));
-}
 #endif
+}
 
 static unsigned
 malloc_ncpus(void)
@@ -1018,7 +1019,7 @@ malloc_conf_init(void)
 #endif
 			    ;
 
-			if ((opts = secure_getenv(envname)) != NULL) {
+			if ((opts = jemalloc_secure_getenv(envname)) != NULL) {
 				/*
 				 * Do nothing; opts is already initialized to
 				 * the value of the MALLOC_CONF environment
@@ -1056,7 +1057,11 @@ malloc_conf_init(void)
 				if (cont)				\
 					continue;			\
 			}
-#define	CONF_HANDLE_T_U(t, o, n, min, max, clip)			\
+#define	CONF_MIN_no(um, min)	false
+#define	CONF_MIN_yes(um, min)	((um) < (min))
+#define	CONF_MAX_no(um, max)	false
+#define	CONF_MAX_yes(um, max)	((um) > (max))
+#define	CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip)	\
 			if (CONF_MATCH(n)) {				\
 				uintmax_t um;				\
 				char *end;				\
@@ -1069,15 +1074,19 @@ malloc_conf_init(void)
 					    "Invalid conf value",	\
 					    k, klen, v, vlen);		\
 				} else if (clip) {			\
-					if ((min) != 0 && um < (min))	\
+					if (CONF_MIN_##check_min(um,	\
+					    (t)(min)))			\
 						o = (t)(min);		\
-					else if (um > (max))		\
+					else if (CONF_MAX_##check_max(	\
+					    um, (t)(max)))		\
 						o = (t)(max);		\
 					else				\
 						o = (t)um;		\
 				} else {				\
-					if (((min) != 0 && um < (min))	\
-					    || um > (max)) {		\
+					if (CONF_MIN_##check_min(um,	\
+					    (t)(min)) ||		\
+					    CONF_MAX_##check_max(um,	\
+					    (t)(max))) {		\
 						malloc_conf_error(	\
 						    "Out-of-range "	\
 						    "conf value",	\
@@ -1087,10 +1096,13 @@ malloc_conf_init(void)
 				}					\
 				continue;				\
 			}
-#define	CONF_HANDLE_UNSIGNED(o, n, min, max, clip)			\
-			CONF_HANDLE_T_U(unsigned, o, n, min, max, clip)
-#define	CONF_HANDLE_SIZE_T(o, n, min, max, clip)			\
-			CONF_HANDLE_T_U(size_t, o, n, min, max, clip)
+#define	CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max,	\
+    clip)								\
+			CONF_HANDLE_T_U(unsigned, o, n, min, max,	\
+			    check_min, check_max, clip)
+#define	CONF_HANDLE_SIZE_T(o, n, min, max, check_min, check_max, clip)	\
+			CONF_HANDLE_T_U(size_t, o, n, min, max,		\
+			    check_min, check_max, clip)
 #define	CONF_HANDLE_SSIZE_T(o, n, min, max)				\
 			if (CONF_MATCH(n)) {				\
 				long l;					\
@@ -1124,16 +1136,18 @@ malloc_conf_init(void)
 
 			CONF_HANDLE_BOOL(opt_abort, "abort", true)
 			/*
-			 * Chunks always require at least one header page,
-			 * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and
-			 * possibly an additional page in the presence of
-			 * redzones.  In order to simplify options processing,
-			 * use a conservative bound that accommodates all these
-			 * constraints.
+			 * Chunks always require at least one header page, as
+			 * many as 2^(LG_SIZE_CLASS_GROUP+1) data pages (plus an
+			 * additional page in the presence of cache-oblivious
+			 * large), and possibly an additional page in the
+			 * presence of redzones.  In order to simplify options
+			 * processing, use a conservative bound that
+			 * accommodates all these constraints.
 			 */
 			CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
-			    LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1),
-			    (sizeof(size_t) << 3) - 1, true)
+			    LG_SIZE_CLASS_GROUP + 1 + ((config_cache_oblivious
+			    || config_fill) ? 1 : 0), (sizeof(size_t) << 3) - 1,
+			    yes, yes, true)
 			if (strncmp("dss", k, klen) == 0) {
 				int i;
 				bool match = false;
@@ -1159,7 +1173,7 @@ malloc_conf_init(void)
 				continue;
 			}
 			CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1,
-			    UINT_MAX, false)
+			    UINT_MAX, yes, no, false)
 			if (strncmp("purge", k, klen) == 0) {
 				int i;
 				bool match = false;
@@ -1230,7 +1244,7 @@ malloc_conf_init(void)
 					continue;
 				}
 				CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine",
-				    0, SIZE_T_MAX, false)
+				    0, SIZE_T_MAX, no, no, false)
 				CONF_HANDLE_BOOL(opt_redzone, "redzone", true)
 				CONF_HANDLE_BOOL(opt_zero, "zero", true)
 			}
@@ -1258,6 +1272,9 @@ malloc_conf_init(void)
 				    "lg_tcache_max", -1,
 				    (sizeof(size_t) << 3) - 1)
 			}
+			if (config_thp) {
+				CONF_HANDLE_BOOL(opt_thp, "thp", true)
+			}
 			if (config_prof) {
 				CONF_HANDLE_BOOL(opt_prof, "prof", true)
 				CONF_HANDLE_CHAR_P(opt_prof_prefix,
@@ -1267,8 +1284,8 @@ malloc_conf_init(void)
 				CONF_HANDLE_BOOL(opt_prof_thread_active_init,
 				    "prof_thread_active_init", true)
 				CONF_HANDLE_SIZE_T(opt_lg_prof_sample,
-				    "lg_prof_sample", 0,
-				    (sizeof(uint64_t) << 3) - 1, true)
+				    "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
+				    - 1, no, yes, true)
 				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum",
 				    true)
 				CONF_HANDLE_SSIZE_T(opt_lg_prof_interval,
@@ -1284,7 +1301,14 @@ malloc_conf_init(void)
 			malloc_conf_error("Invalid conf pair", k, klen, v,
 			    vlen);
 #undef CONF_MATCH
+#undef CONF_MATCH_VALUE
 #undef CONF_HANDLE_BOOL
+#undef CONF_MIN_no
+#undef CONF_MIN_yes
+#undef CONF_MAX_no
+#undef CONF_MAX_yes
+#undef CONF_HANDLE_T_U
+#undef CONF_HANDLE_UNSIGNED
 #undef CONF_HANDLE_SIZE_T
 #undef CONF_HANDLE_SSIZE_T
 #undef CONF_HANDLE_CHAR_P
@@ -1393,8 +1417,9 @@ malloc_init_hard_recursible(void)
 
 	ncpus = malloc_ncpus();
 
-#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
-    && !defined(_WIN32) && !defined(__native_client__))
+#if (defined(JEMALLOC_HAVE_PTHREAD_ATFORK) && !defined(JEMALLOC_MUTEX_INIT_CB) \
+    && !defined(JEMALLOC_ZONE) && !defined(_WIN32) && \
+    !defined(__native_client__))
 	/* LinuxThreads' pthread_atfork() allocates. */
 	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
 	    jemalloc_postfork_child) != 0) {
@@ -1973,8 +1998,8 @@ je_realloc(void *ptr, size_t size)
 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
 	}
 	UTRACE(ptr, size, ret);
-	JEMALLOC_VALGRIND_REALLOC(true, tsdn, ret, usize, true, ptr, old_usize,
-	    old_rzsize, true, false);
+	JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr,
+	    old_usize, old_rzsize, maybe, false);
 	witness_assert_lockless(tsdn);
 	return (ret);
 }
@@ -2400,8 +2425,8 @@ je_rallocx(void *ptr, size_t size, int flags)
 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
 	}
 	UTRACE(ptr, size, p);
-	JEMALLOC_VALGRIND_REALLOC(true, tsd_tsdn(tsd), p, usize, false, ptr,
-	    old_usize, old_rzsize, false, zero);
+	JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr,
+	    old_usize, old_rzsize, no, zero);
 	witness_assert_lockless(tsd_tsdn(tsd));
 	return (p);
 label_oom:
@@ -2543,8 +2568,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
 		*tsd_thread_allocatedp_get(tsd) += usize;
 		*tsd_thread_deallocatedp_get(tsd) += old_usize;
 	}
-	JEMALLOC_VALGRIND_REALLOC(false, tsd_tsdn(tsd), ptr, usize, false, ptr,
-	    old_usize, old_rzsize, false, zero);
+	JEMALLOC_VALGRIND_REALLOC(no, tsd_tsdn(tsd), ptr, usize, no, ptr,
+	    old_usize, old_rzsize, no, zero);
 label_not_resized:
 	UTRACE(ptr, size, ptr);
 	witness_assert_lockless(tsd_tsdn(tsd));
@@ -2808,6 +2833,7 @@ _malloc_prefork(void)
 	witness_prefork(tsd);
 	/* Acquire all mutexes in a safe order. */
 	ctl_prefork(tsd_tsdn(tsd));
+	tcache_prefork(tsd_tsdn(tsd));
 	malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
 	prof_prefork0(tsd_tsdn(tsd));
 	for (i = 0; i < 3; i++) {
@@ -2867,6 +2893,7 @@ _malloc_postfork(void)
 	}
 	prof_postfork_parent(tsd_tsdn(tsd));
 	malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
+	tcache_postfork_parent(tsd_tsdn(tsd));
 	ctl_postfork_parent(tsd_tsdn(tsd));
 }
 
@@ -2891,6 +2918,7 @@ jemalloc_postfork_child(void)
 	}
 	prof_postfork_child(tsd_tsdn(tsd));
 	malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
+	tcache_postfork_child(tsd_tsdn(tsd));
 	ctl_postfork_child(tsd_tsdn(tsd));
 }
 

src/pages.c

@@ -170,15 +170,16 @@ pages_purge(void *addr, size_t size)
 #ifdef _WIN32
 	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
 	unzeroed = true;
-#elif defined(JEMALLOC_HAVE_MADVISE)
-#  ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
-#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
-#    define JEMALLOC_MADV_ZEROS true
-#  elif defined(JEMALLOC_PURGE_MADVISE_FREE)
+#elif (defined(JEMALLOC_PURGE_MADVISE_FREE) || \
+    defined(JEMALLOC_PURGE_MADVISE_DONTNEED))
+#  if defined(JEMALLOC_PURGE_MADVISE_FREE)
 #    define JEMALLOC_MADV_PURGE MADV_FREE
 #    define JEMALLOC_MADV_ZEROS false
+#  elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)
+#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
+#    define JEMALLOC_MADV_ZEROS true
 #  else
-#    error "No madvise(2) flag defined for purging unused dirty pages."
+#    error No madvise(2) flag defined for purging unused dirty pages
 #  endif
 	int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
 	unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
@@ -191,6 +192,34 @@ pages_purge(void *addr, size_t size)
 	return (unzeroed);
 }
 
+bool
+pages_huge(void *addr, size_t size)
+{
+
+	assert(PAGE_ADDR2BASE(addr) == addr);
+	assert(PAGE_CEILING(size) == size);
+
+#ifdef JEMALLOC_HAVE_MADVISE_HUGE
+	return (madvise(addr, size, MADV_HUGEPAGE) != 0);
+#else
+	return (false);
+#endif
+}
+
+bool
+pages_nohuge(void *addr, size_t size)
+{
+
+	assert(PAGE_ADDR2BASE(addr) == addr);
+	assert(PAGE_CEILING(size) == size);
+
+#ifdef JEMALLOC_HAVE_MADVISE_HUGE
+	return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
+#else
+	return (false);
+#endif
+}
+
 #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
 static bool
 os_overcommits_sysctl(void)
@@ -219,7 +248,7 @@ os_overcommits_proc(void)
 	char buf[1];
 	ssize_t nread;
 
-#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_open)
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
 	fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY);
 #else
 	fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
@@ -227,13 +256,13 @@ os_overcommits_proc(void)
 	if (fd == -1)
 		return (false); /* Error. */
 
-#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_read)
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
 	nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
 #else
 	nread = read(fd, &buf, sizeof(buf));
 #endif
 
-#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_close)
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
 	syscall(SYS_close, fd);
 #else
 	close(fd);

src/stats.c

@@ -3,7 +3,7 @@
 
 #define	CTL_GET(n, v, t) do {						\
 	size_t sz = sizeof(t);						\
-	xmallctl(n, v, &sz, NULL, 0);					\
+	xmallctl(n, (void *)v, &sz, NULL, 0);				\
 } while (0)
 
 #define	CTL_M2_GET(n, i, v, t) do {					\
@@ -12,7 +12,7 @@
 	size_t sz = sizeof(t);						\
 	xmallctlnametomib(n, mib, &miblen);				\
 	mib[2] = (i);							\
-	xmallctlbymib(mib, miblen, v, &sz, NULL, 0);			\
+	xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0);		\
 } while (0)
 
 #define	CTL_M2_M4_GET(n, i, j, v, t) do {				\
@@ -22,7 +22,7 @@
 	xmallctlnametomib(n, mib, &miblen);				\
 	mib[2] = (i);							\
 	mib[4] = (j);							\
-	xmallctlbymib(mib, miblen, v, &sz, NULL, 0);			\
+	xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0);		\
 } while (0)
 
 /******************************************************************************/
@@ -39,7 +39,7 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
     bool json, bool large, bool huge, unsigned i)
 {
 	size_t page;
-	bool config_tcache, in_gap, in_gap_prev;
+	bool in_gap, in_gap_prev;
 	unsigned nbins, j;
 
 	CTL_GET("arenas.page", &page, size_t);
@@ -49,7 +49,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		malloc_cprintf(write_cb, cbopaque,
 		    "\t\t\t\t\"bins\": [\n");
 	} else {
-		CTL_GET("config.tcache", &config_tcache, bool);
 		if (config_tcache) {
 			malloc_cprintf(write_cb, cbopaque,
 			    "bins:           size ind    allocated      nmalloc"
@@ -137,8 +136,16 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			availregs = nregs * curruns;
 			milli = (availregs != 0) ? (1000 * curregs) / availregs
 			    : 1000;
-			assert(milli <= 1000);
-			if (milli < 10) {
+
+			if (milli > 1000) {
+				/*
+				 * Race detected: the counters were read in
+				 * separate mallctl calls and concurrent
+				 * operations happened in between. In this case
+				 * no meaningful utilization can be computed.
+				 */
+				malloc_snprintf(util, sizeof(util), " race");
+			} else if (milli < 10) {
 				malloc_snprintf(util, sizeof(util),
 				    "0.00%zu", milli);
 			} else if (milli < 100) {
@@ -147,8 +154,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			} else if (milli < 1000) {
 				malloc_snprintf(util, sizeof(util), "0.%zu",
 				    milli);
-			} else
+			} else {
+				assert(milli == 1000);
 				malloc_snprintf(util, sizeof(util), "1");
+			}
 
 			if (config_tcache) {
 				malloc_cprintf(write_cb, cbopaque,
@@ -536,7 +545,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		    "\t\t\t\t\t\"allocated\": %zu\n", metadata_allocated);
 
 		malloc_cprintf(write_cb, cbopaque,
-		    "\t\t\t\t},\n");
+		    "\t\t\t\t}%s\n", (bins || large || huge) ? "," : "");
 	} else {
 		malloc_cprintf(write_cb, cbopaque,
 		    "metadata: mapped: %zu, allocated: %zu\n",
@@ -555,7 +564,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
 
 static void
 stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
-    bool json, bool merged, bool unmerged)
+    bool json, bool more)
 {
 	const char *cpv;
 	bool bv;
@@ -647,7 +656,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
 #define	OPT_WRITE_BOOL_MUTABLE(n, m, c) {				\
 	bool bv2;							\
 	if (je_mallctl("opt."#n, (void *)&bv, &bsz, NULL, 0) == 0 &&	\
-	    je_mallctl(#m, &bv2, &bsz, NULL, 0) == 0) {			\
+	    je_mallctl(#m, &bv2, (void *)&bsz, NULL, 0) == 0) {		\
 		if (json) {						\
 			malloc_cprintf(write_cb, cbopaque,		\
 			    "\t\t\t\""#n"\": %s%s\n", bv ? "true" :	\
@@ -692,7 +701,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
 #define	OPT_WRITE_SSIZE_T_MUTABLE(n, m, c) {				\
 	ssize_t ssv2;							\
 	if (je_mallctl("opt."#n, (void *)&ssv, &sssz, NULL, 0) == 0 &&	\
-	    je_mallctl(#m, &ssv2, &sssz, NULL, 0) == 0) {		\
+	    je_mallctl(#m, (void *)&ssv2, &sssz, NULL, 0) == 0) {	\
 		if (json) {						\
 			malloc_cprintf(write_cb, cbopaque,		\
 			    "\t\t\t\""#n"\": %zd%s\n", ssv, (c));	\
@@ -741,6 +750,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	OPT_WRITE_BOOL(xmalloc, ",")
 	OPT_WRITE_BOOL(tcache, ",")
 	OPT_WRITE_SSIZE_T(lg_tcache_max, ",")
+	OPT_WRITE_BOOL(thp, ",")
 	OPT_WRITE_BOOL(prof, ",")
 	OPT_WRITE_CHAR_P(prof_prefix, ",")
 	OPT_WRITE_BOOL_MUTABLE(prof_active, prof.active, ",")
@@ -838,9 +848,11 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		malloc_cprintf(write_cb, cbopaque,
 		    "\t\t\t\"nbins\": %u,\n", nbins);
 
-		CTL_GET("arenas.nhbins", &uv, unsigned);
-		malloc_cprintf(write_cb, cbopaque,
-		    "\t\t\t\"nhbins\": %u,\n", uv);
+		if (config_tcache) {
+			CTL_GET("arenas.nhbins", &uv, unsigned);
+			malloc_cprintf(write_cb, cbopaque,
+			    "\t\t\t\"nhbins\": %u,\n", uv);
+		}
 
 		malloc_cprintf(write_cb, cbopaque,
 		    "\t\t\t\"bin\": [\n");
@@ -907,11 +919,11 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		    "\t\t\t]\n");
 
 		malloc_cprintf(write_cb, cbopaque,
-		    "\t\t},\n");
+		    "\t\t}%s\n", (config_prof || more) ? "," : "");
 	}
 
 	/* prof. */
-	if (json) {
+	if (config_prof && json) {
 		malloc_cprintf(write_cb, cbopaque,
 		    "\t\t\"prof\": {\n");
 
@@ -937,8 +949,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		    "\t\t\t\"lg_sample\": %zd\n", ssv);
 
 		malloc_cprintf(write_cb, cbopaque,
-		    "\t\t}%s\n", (config_stats || merged || unmerged) ? "," :
-		    "");
+		    "\t\t}%s\n", more ? "," : "");
 	}
 }
 
@@ -1023,31 +1034,37 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
 				    narenas, bins, large, huge);
 				if (json) {
 					malloc_cprintf(write_cb, cbopaque,
-					    "\t\t\t}%s\n", (ninitialized > 1) ?
-					    "," : "");
+					    "\t\t\t}%s\n", unmerged ?  "," :
+					    "");
 				}
 			}
 
 			/* Unmerged stats. */
-			for (i = j = 0; i < narenas; i++) {
-				if (initialized[i]) {
-					if (json) {
-						j++;
-						malloc_cprintf(write_cb,
-						    cbopaque,
-						    "\t\t\t\"%u\": {\n", i);
-					} else {
-						malloc_cprintf(write_cb,
-						    cbopaque, "\narenas[%u]:\n",
-						    i);
-					}
-					stats_arena_print(write_cb, cbopaque,
-					    json, i, bins, large, huge);
-					if (json) {
-						malloc_cprintf(write_cb,
-						    cbopaque,
-						    "\t\t\t}%s\n", (j <
-						    ninitialized) ? "," : "");
+			if (unmerged) {
+				for (i = j = 0; i < narenas; i++) {
+					if (initialized[i]) {
+						if (json) {
+							j++;
+							malloc_cprintf(write_cb,
+							    cbopaque,
+							    "\t\t\t\"%u\": {\n",
+							    i);
+						} else {
+							malloc_cprintf(write_cb,
+							    cbopaque,
+							    "\narenas[%u]:\n",
+							    i);
+						}
+						stats_arena_print(write_cb,
+						    cbopaque, json, i, bins,
+						    large, huge);
+						if (json) {
+							malloc_cprintf(write_cb,
+							    cbopaque,
+							    "\t\t\t}%s\n", (j <
+							    ninitialized) ? ","
+							    : "");
+						}
 					}
 				}
 			}
@@ -1069,8 +1086,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	size_t u64sz;
 	bool json = false;
 	bool general = true;
-	bool merged = true;
-	bool unmerged = true;
+	bool merged = config_stats;
+	bool unmerged = config_stats;
 	bool bins = true;
 	bool large = true;
 	bool huge = true;
@@ -1084,7 +1101,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	 * */
 	epoch = 1;
 	u64sz = sizeof(uint64_t);
-	err = je_mallctl("epoch", &epoch, &u64sz, &epoch, sizeof(uint64_t));
+	err = je_mallctl("epoch", (void *)&epoch, &u64sz, (void *)&epoch,
+	    sizeof(uint64_t));
 	if (err != 0) {
 		if (err == EAGAIN) {
 			malloc_write("<jemalloc>: Memory allocation failure in "
@@ -1136,8 +1154,10 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		    "___ Begin jemalloc statistics ___\n");
 	}
 
-	if (general)
-		stats_general_print(write_cb, cbopaque, json, merged, unmerged);
+	if (general) {
+		bool more = (merged || unmerged);
+		stats_general_print(write_cb, cbopaque, json, more);
+	}
 	if (config_stats) {
 		stats_print_helper(write_cb, cbopaque, json, merged, unmerged,
 		    bins, large, huge);

src/tcache.c

@@ -21,6 +21,9 @@ static unsigned		tcaches_past;
 /* Head of singly linked list tracking available tcaches elements. */
 static tcaches_t	*tcaches_avail;
 
+/* Protects tcaches{,_past,_avail}. */
+static malloc_mutex_t	tcaches_mtx;
+
 /******************************************************************************/
 
 size_t
@@ -444,29 +447,56 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
 	}
 }
 
-bool
-tcaches_create(tsd_t *tsd, unsigned *r_ind)
-{
-	arena_t *arena;
-	tcache_t *tcache;
-	tcaches_t *elm;
+static bool
+tcaches_create_prep(tsd_t *tsd) {
+	bool err;
+
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
 
 	if (tcaches == NULL) {
 		tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) *
 		    (MALLOCX_TCACHE_MAX+1));
-		if (tcaches == NULL)
-			return (true);
+		if (tcaches == NULL) {
+			err = true;
+			goto label_return;
+		}
 	}
 
-	if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
-		return (true);
-	arena = arena_ichoose(tsd, NULL);
-	if (unlikely(arena == NULL))
-		return (true);
-	tcache = tcache_create(tsd_tsdn(tsd), arena);
-	if (tcache == NULL)
-		return (true);
+	if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
+		err = true;
+		goto label_return;
+	}
 
+	err = false;
+label_return:
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
+	return err;
+}
+
+bool
+tcaches_create(tsd_t *tsd, unsigned *r_ind) {
+	bool err;
+	arena_t *arena;
+	tcache_t *tcache;
+	tcaches_t *elm;
+
+	if (tcaches_create_prep(tsd)) {
+		err = true;
+		goto label_return;
+	}
+
+	arena = arena_ichoose(tsd, NULL);
+	if (unlikely(arena == NULL)) {
+		err = true;
+		goto label_return;
+	}
+	tcache = tcache_create(tsd_tsdn(tsd), arena);
+	if (tcache == NULL) {
+		err = true;
+		goto label_return;
+	}
+
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
 	if (tcaches_avail != NULL) {
 		elm = tcaches_avail;
 		tcaches_avail = tcaches_avail->next;
@@ -478,51 +508,64 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
 		*r_ind = tcaches_past;
 		tcaches_past++;
 	}
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
 
-	return (false);
+	err = false;
+label_return:
+	malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &tcaches_mtx);
+	return err;
 }
 
 static void
-tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm)
-{
+tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
+	malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
 
-	if (elm->tcache == NULL)
+	if (elm->tcache == NULL) {
 		return;
+	}
 	tcache_destroy(tsd, elm->tcache);
 	elm->tcache = NULL;
 }
 
 void
-tcaches_flush(tsd_t *tsd, unsigned ind)
-{
-
+tcaches_flush(tsd_t *tsd, unsigned ind) {
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
 	tcaches_elm_flush(tsd, &tcaches[ind]);
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
 }
 
 void
-tcaches_destroy(tsd_t *tsd, unsigned ind)
-{
-	tcaches_t *elm = &tcaches[ind];
+tcaches_destroy(tsd_t *tsd, unsigned ind) {
+	tcaches_t *elm;
+
+	malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
+	elm = &tcaches[ind];
 	tcaches_elm_flush(tsd, elm);
 	elm->next = tcaches_avail;
 	tcaches_avail = elm;
+	malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
 }
 
 bool
-tcache_boot(tsdn_t *tsdn)
-{
+tcache_boot(tsdn_t *tsdn) {
 	unsigned i;
 
+	cassert(config_tcache);
+
 	/*
 	 * If necessary, clamp opt_lg_tcache_max, now that large_maxclass is
 	 * known.
 	 */
-	if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS)
+	if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < SMALL_MAXCLASS)
 		tcache_maxclass = SMALL_MAXCLASS;
-	else if ((1U << opt_lg_tcache_max) > large_maxclass)
+	else if ((ZU(1) << opt_lg_tcache_max) > large_maxclass)
 		tcache_maxclass = large_maxclass;
 	else
-		tcache_maxclass = (1U << opt_lg_tcache_max);
+		tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
+
+	if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) {
+		return true;
+	}
 
 	nhbins = size2index(tcache_maxclass) + 1;
 
@@ -553,3 +596,24 @@ tcache_boot(tsdn_t *tsdn)
 
 	return (false);
 }
+
+void
+tcache_prefork(tsdn_t *tsdn) {
+	if (!config_prof && opt_tcache) {
+		malloc_mutex_prefork(tsdn, &tcaches_mtx);
+	}
+}
+
+void
+tcache_postfork_parent(tsdn_t *tsdn) {
+	if (!config_prof && opt_tcache) {
+		malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
+	}
+}
+
+void
+tcache_postfork_child(tsdn_t *tsdn) {
+	if (!config_prof && opt_tcache) {
+		malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
+	}
+}

src/util.c

@@ -49,7 +49,7 @@ static void
 wrtmessage(void *cbopaque, const char *s)
 {
 
-#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_write)
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write)
 	/*
 	 * Use syscall(2) rather than write(2) when possible in order to avoid
 	 * the possibility of memory allocation within libc.  This is necessary
@@ -200,7 +200,7 @@ malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base)
 		p++;
 	}
 	if (neg)
-		ret = -ret;
+		ret = (uintmax_t)(-((intmax_t)ret));
 
 	if (p == ns) {
 		/* No conversion performed. */

src/witness.c

@@ -71,15 +71,16 @@ witness_not_owner_error_t *witness_not_owner_error =
 #endif
 
 #ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define	witness_lockless_error JEMALLOC_N(n_witness_lockless_error)
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(n_witness_depth_error)
 #endif
 void
-witness_lockless_error(const witness_list_t *witnesses)
-{
+witness_depth_error(const witness_list_t *witnesses,
+    witness_rank_t rank_inclusive, unsigned depth) {
 	witness_t *w;
 
-	malloc_printf("<jemalloc>: Should not own any locks:");
+	malloc_printf("<jemalloc>: Should own %u lock%s of rank >= %u:", depth,
+	    (depth != 1) ?  "s" : "", rank_inclusive);
 	ql_foreach(w, witnesses, link) {
 		malloc_printf(" %s(%u)", w->name, w->rank);
 	}
@@ -87,10 +88,9 @@ witness_lockless_error(const witness_list_t *witnesses)
 	abort();
 }
 #ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define	witness_lockless_error JEMALLOC_N(witness_lockless_error)
-witness_lockless_error_t *witness_lockless_error =
-    JEMALLOC_N(n_witness_lockless_error);
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(witness_depth_error)
+witness_depth_error_t *witness_depth_error = JEMALLOC_N(n_witness_depth_error);
 #endif
 
 void

src/zone.c

@@ -3,6 +3,75 @@
 #  error "This source file is for zones on Darwin (OS X)."
 #endif
 
+/* Definitions of the following structs in malloc/malloc.h might be too old
+ * for the built binary to run on newer versions of OSX. So use the newest
+ * possible version of those structs.
+ */
+typedef struct _malloc_zone_t {
+	void *reserved1;
+	void *reserved2;
+	size_t (*size)(struct _malloc_zone_t *, const void *);
+	void *(*malloc)(struct _malloc_zone_t *, size_t);
+	void *(*calloc)(struct _malloc_zone_t *, size_t, size_t);
+	void *(*valloc)(struct _malloc_zone_t *, size_t);
+	void (*free)(struct _malloc_zone_t *, void *);
+	void *(*realloc)(struct _malloc_zone_t *, void *, size_t);
+	void (*destroy)(struct _malloc_zone_t *);
+	const char *zone_name;
+	unsigned (*batch_malloc)(struct _malloc_zone_t *, size_t, void **, unsigned);
+	void (*batch_free)(struct _malloc_zone_t *, void **, unsigned);
+	struct malloc_introspection_t *introspect;
+	unsigned version;
+	void *(*memalign)(struct _malloc_zone_t *, size_t, size_t);
+	void (*free_definite_size)(struct _malloc_zone_t *, void *, size_t);
+	size_t (*pressure_relief)(struct _malloc_zone_t *, size_t);
+} malloc_zone_t;
+
+typedef struct {
+	vm_address_t address;
+	vm_size_t size;
+} vm_range_t;
+
+typedef struct malloc_statistics_t {
+	unsigned blocks_in_use;
+	size_t size_in_use;
+	size_t max_size_in_use;
+	size_t size_allocated;
+} malloc_statistics_t;
+
+typedef kern_return_t memory_reader_t(task_t, vm_address_t, vm_size_t, void **);
+
+typedef void vm_range_recorder_t(task_t, void *, unsigned type, vm_range_t *, unsigned);
+
+typedef struct malloc_introspection_t {
+	kern_return_t (*enumerator)(task_t, void *, unsigned, vm_address_t, memory_reader_t, vm_range_recorder_t);
+	size_t (*good_size)(malloc_zone_t *, size_t);
+	boolean_t (*check)(malloc_zone_t *);
+	void (*print)(malloc_zone_t *, boolean_t);
+	void (*log)(malloc_zone_t *, void *);
+	void (*force_lock)(malloc_zone_t *);
+	void (*force_unlock)(malloc_zone_t *);
+	void (*statistics)(malloc_zone_t *, malloc_statistics_t *);
+	boolean_t (*zone_locked)(malloc_zone_t *);
+	boolean_t (*enable_discharge_checking)(malloc_zone_t *);
+	boolean_t (*disable_discharge_checking)(malloc_zone_t *);
+	void (*discharge)(malloc_zone_t *, void *);
+#ifdef __BLOCKS__
+	void (*enumerate_discharged_pointers)(malloc_zone_t *, void (^)(void *, void *));
+#else
+	void *enumerate_unavailable_without_blocks;
+#endif
+	void (*reinit_lock)(malloc_zone_t *);
+} malloc_introspection_t;
+
+extern kern_return_t malloc_get_all_zones(task_t, memory_reader_t, vm_address_t **, unsigned *);
+
+extern malloc_zone_t *malloc_default_zone(void);
+
+extern void malloc_zone_register(malloc_zone_t *zone);
+
+extern void malloc_zone_unregister(malloc_zone_t *zone);
+
 /*
  * The malloc_default_purgeable_zone() function is only available on >= 10.6.
  * We need to check whether it is present at runtime, thus the weak_import.
@@ -20,24 +89,35 @@ static struct malloc_introspection_t jemalloc_zone_introspect;
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static size_t	zone_size(malloc_zone_t *zone, void *ptr);
+static size_t	zone_size(malloc_zone_t *zone, const void *ptr);
 static void	*zone_malloc(malloc_zone_t *zone, size_t size);
 static void	*zone_calloc(malloc_zone_t *zone, size_t num, size_t size);
 static void	*zone_valloc(malloc_zone_t *zone, size_t size);
 static void	zone_free(malloc_zone_t *zone, void *ptr);
 static void	*zone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
-#if (JEMALLOC_ZONE_VERSION >= 5)
 static void	*zone_memalign(malloc_zone_t *zone, size_t alignment,
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 6)
     size_t size);
 static void	zone_free_definite_size(malloc_zone_t *zone, void *ptr,
     size_t size);
-#endif
-static void	*zone_destroy(malloc_zone_t *zone);
+static void	zone_destroy(malloc_zone_t *zone);
+static unsigned	zone_batch_malloc(struct _malloc_zone_t *zone, size_t size,
+    void **results, unsigned num_requested);
+static void	zone_batch_free(struct _malloc_zone_t *zone,
+    void **to_be_freed, unsigned num_to_be_freed);
+static size_t	zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal);
 static size_t	zone_good_size(malloc_zone_t *zone, size_t size);
+static kern_return_t	zone_enumerator(task_t task, void *data, unsigned type_mask,
+    vm_address_t zone_address, memory_reader_t reader,
+    vm_range_recorder_t recorder);
+static boolean_t	zone_check(malloc_zone_t *zone);
+static void	zone_print(malloc_zone_t *zone, boolean_t verbose);
+static void	zone_log(malloc_zone_t *zone, void *address);
 static void	zone_force_lock(malloc_zone_t *zone);
 static void	zone_force_unlock(malloc_zone_t *zone);
+static void	zone_statistics(malloc_zone_t *zone,
+    malloc_statistics_t *stats);
+static boolean_t	zone_locked(malloc_zone_t *zone);
+static void	zone_reinit_lock(malloc_zone_t *zone);
 
 /******************************************************************************/
 /*
@@ -45,7 +125,7 @@ static void	zone_force_unlock(malloc_zone_t *zone);
  */
 
 static size_t
-zone_size(malloc_zone_t *zone, void *ptr)
+zone_size(malloc_zone_t *zone, const void *ptr)
 {
 
 	/*
@@ -106,7 +186,6 @@ zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
 	return (realloc(ptr, size));
 }
 
-#if (JEMALLOC_ZONE_VERSION >= 5)
 static void *
 zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
 {
@@ -116,9 +195,7 @@ zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
 
 	return (ret);
 }
-#endif
 
-#if (JEMALLOC_ZONE_VERSION >= 6)
 static void
 zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
 {
@@ -133,15 +210,46 @@ zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
 
 	free(ptr);
 }
-#endif
 
-static void *
+static void
 zone_destroy(malloc_zone_t *zone)
 {
 
 	/* This function should never be called. */
 	not_reached();
-	return (NULL);
+}
+
+static unsigned
+zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results,
+    unsigned num_requested)
+{
+	unsigned i;
+
+	for (i = 0; i < num_requested; i++) {
+		results[i] = je_malloc(size);
+		if (!results[i])
+			break;
+	}
+
+	return i;
+}
+
+static void
+zone_batch_free(struct _malloc_zone_t *zone, void **to_be_freed,
+    unsigned num_to_be_freed)
+{
+	unsigned i;
+
+	for (i = 0; i < num_to_be_freed; i++) {
+		zone_free(zone, to_be_freed[i]);
+		to_be_freed[i] = NULL;
+	}
+}
+
+static size_t
+zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal)
+{
+	return 0;
 }
 
 static size_t
@@ -153,6 +261,30 @@ zone_good_size(malloc_zone_t *zone, size_t size)
 	return (s2u(size));
 }
 
+static kern_return_t
+zone_enumerator(task_t task, void *data, unsigned type_mask,
+    vm_address_t zone_address, memory_reader_t reader,
+    vm_range_recorder_t recorder)
+{
+	return KERN_SUCCESS;
+}
+
+static boolean_t
+zone_check(malloc_zone_t *zone)
+{
+	return true;
+}
+
+static void
+zone_print(malloc_zone_t *zone, boolean_t verbose)
+{
+}
+
+static void
+zone_log(malloc_zone_t *zone, void *address)
+{
+}
+
 static void
 zone_force_lock(malloc_zone_t *zone)
 {
@@ -176,53 +308,69 @@ zone_force_unlock(malloc_zone_t *zone)
 		jemalloc_postfork_child();
 }
 
+static void
+zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats)
+{
+	/* We make no effort to actually fill the values */
+	stats->blocks_in_use = 0;
+	stats->size_in_use = 0;
+	stats->max_size_in_use = 0;
+	stats->size_allocated = 0;
+}
+
+static boolean_t
+zone_locked(malloc_zone_t *zone)
+{
+	/* Pretend no lock is being held */
+	return false;
+}
+
+static void
+zone_reinit_lock(malloc_zone_t *zone)
+{
+	/* As of OSX 10.12, this function is only used when force_unlock would
+	 * be used if the zone version were < 9. So just use force_unlock. */
+	zone_force_unlock(zone);
+}
+
 static void
 zone_init(void)
 {
 
-	jemalloc_zone.size = (void *)zone_size;
-	jemalloc_zone.malloc = (void *)zone_malloc;
-	jemalloc_zone.calloc = (void *)zone_calloc;
-	jemalloc_zone.valloc = (void *)zone_valloc;
-	jemalloc_zone.free = (void *)zone_free;
-	jemalloc_zone.realloc = (void *)zone_realloc;
-	jemalloc_zone.destroy = (void *)zone_destroy;
+	jemalloc_zone.size = zone_size;
+	jemalloc_zone.malloc = zone_malloc;
+	jemalloc_zone.calloc = zone_calloc;
+	jemalloc_zone.valloc = zone_valloc;
+	jemalloc_zone.free = zone_free;
+	jemalloc_zone.realloc = zone_realloc;
+	jemalloc_zone.destroy = zone_destroy;
 	jemalloc_zone.zone_name = "jemalloc_zone";
-	jemalloc_zone.batch_malloc = NULL;
-	jemalloc_zone.batch_free = NULL;
+	jemalloc_zone.batch_malloc = zone_batch_malloc;
+	jemalloc_zone.batch_free = zone_batch_free;
 	jemalloc_zone.introspect = &jemalloc_zone_introspect;
-	jemalloc_zone.version = JEMALLOC_ZONE_VERSION;
-#if (JEMALLOC_ZONE_VERSION >= 5)
+	jemalloc_zone.version = 9;
 	jemalloc_zone.memalign = zone_memalign;
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 6)
 	jemalloc_zone.free_definite_size = zone_free_definite_size;
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 8)
-	jemalloc_zone.pressure_relief = NULL;
-#endif
+	jemalloc_zone.pressure_relief = zone_pressure_relief;
 
-	jemalloc_zone_introspect.enumerator = NULL;
-	jemalloc_zone_introspect.good_size = (void *)zone_good_size;
-	jemalloc_zone_introspect.check = NULL;
-	jemalloc_zone_introspect.print = NULL;
-	jemalloc_zone_introspect.log = NULL;
-	jemalloc_zone_introspect.force_lock = (void *)zone_force_lock;
-	jemalloc_zone_introspect.force_unlock = (void *)zone_force_unlock;
-	jemalloc_zone_introspect.statistics = NULL;
-#if (JEMALLOC_ZONE_VERSION >= 6)
-	jemalloc_zone_introspect.zone_locked = NULL;
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 7)
+	jemalloc_zone_introspect.enumerator = zone_enumerator;
+	jemalloc_zone_introspect.good_size = zone_good_size;
+	jemalloc_zone_introspect.check = zone_check;
+	jemalloc_zone_introspect.print = zone_print;
+	jemalloc_zone_introspect.log = zone_log;
+	jemalloc_zone_introspect.force_lock = zone_force_lock;
+	jemalloc_zone_introspect.force_unlock = zone_force_unlock;
+	jemalloc_zone_introspect.statistics = zone_statistics;
+	jemalloc_zone_introspect.zone_locked = zone_locked;
 	jemalloc_zone_introspect.enable_discharge_checking = NULL;
 	jemalloc_zone_introspect.disable_discharge_checking = NULL;
 	jemalloc_zone_introspect.discharge = NULL;
-#  ifdef __BLOCKS__
+#ifdef __BLOCKS__
 	jemalloc_zone_introspect.enumerate_discharged_pointers = NULL;
-#  else
+#else
 	jemalloc_zone_introspect.enumerate_unavailable_without_blocks = NULL;
-#  endif
 #endif
+	jemalloc_zone_introspect.reinit_lock = zone_reinit_lock;
 }
 
 static malloc_zone_t *

test/integration/MALLOCX_ARENA.c

@@ -19,8 +19,8 @@ thd_start(void *arg)
 	size_t sz;
 
 	sz = sizeof(arena_ind);
-	assert_d_eq(mallctl("arenas.extend", &arena_ind, &sz, NULL, 0), 0,
-	    "Error in arenas.extend");
+	assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Error in arenas.extend");
 
 	if (thread_ind % 4 != 3) {
 		size_t mib[3];

test/integration/allocated.c

@@ -18,14 +18,14 @@ thd_start(void *arg)
 	size_t sz, usize;
 
 	sz = sizeof(a0);
-	if ((err = mallctl("thread.allocated", &a0, &sz, NULL, 0))) {
+	if ((err = mallctl("thread.allocated", (void *)&a0, &sz, NULL, 0))) {
 		if (err == ENOENT)
 			goto label_ENOENT;
 		test_fail("%s(): Error in mallctl(): %s", __func__,
 		    strerror(err));
 	}
 	sz = sizeof(ap0);
-	if ((err = mallctl("thread.allocatedp", &ap0, &sz, NULL, 0))) {
+	if ((err = mallctl("thread.allocatedp", (void *)&ap0, &sz, NULL, 0))) {
 		if (err == ENOENT)
 			goto label_ENOENT;
 		test_fail("%s(): Error in mallctl(): %s", __func__,
@@ -36,14 +36,15 @@ thd_start(void *arg)
 	    "storage");
 
 	sz = sizeof(d0);
-	if ((err = mallctl("thread.deallocated", &d0, &sz, NULL, 0))) {
+	if ((err = mallctl("thread.deallocated", (void *)&d0, &sz, NULL, 0))) {
 		if (err == ENOENT)
 			goto label_ENOENT;
 		test_fail("%s(): Error in mallctl(): %s", __func__,
 		    strerror(err));
 	}
 	sz = sizeof(dp0);
-	if ((err = mallctl("thread.deallocatedp", &dp0, &sz, NULL, 0))) {
+	if ((err = mallctl("thread.deallocatedp", (void *)&dp0, &sz, NULL,
+	    0))) {
 		if (err == ENOENT)
 			goto label_ENOENT;
 		test_fail("%s(): Error in mallctl(): %s", __func__,
@@ -57,9 +58,9 @@ thd_start(void *arg)
 	assert_ptr_not_null(p, "Unexpected malloc() error");
 
 	sz = sizeof(a1);
-	mallctl("thread.allocated", &a1, &sz, NULL, 0);
+	mallctl("thread.allocated", (void *)&a1, &sz, NULL, 0);
 	sz = sizeof(ap1);
-	mallctl("thread.allocatedp", &ap1, &sz, NULL, 0);
+	mallctl("thread.allocatedp", (void *)&ap1, &sz, NULL, 0);
 	assert_u64_eq(*ap1, a1,
 	    "Dereferenced \"thread.allocatedp\" value should equal "
 	    "\"thread.allocated\" value");
@@ -74,9 +75,9 @@ thd_start(void *arg)
 	free(p);
 
 	sz = sizeof(d1);
-	mallctl("thread.deallocated", &d1, &sz, NULL, 0);
+	mallctl("thread.deallocated", (void *)&d1, &sz, NULL, 0);
 	sz = sizeof(dp1);
-	mallctl("thread.deallocatedp", &dp1, &sz, NULL, 0);
+	mallctl("thread.deallocatedp", (void *)&dp1, &sz, NULL, 0);
 	assert_u64_eq(*dp1, d1,
 	    "Dereferenced \"thread.deallocatedp\" value should equal "
 	    "\"thread.deallocated\" value");

test/integration/chunk.c

@@ -1,9 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_FILL
-const char *malloc_conf = "junk:false";
-#endif
-
 static chunk_hooks_t orig_hooks;
 static chunk_hooks_t old_hooks;
 
@@ -137,8 +133,8 @@ TEST_BEGIN(test_chunk)
 	bool xallocx_success_a, xallocx_success_b, xallocx_success_c;
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.extend", &arena_ind, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Unexpected mallctl() failure");
 	flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
 
 	/* Install custom chunk hooks. */
@@ -148,8 +144,9 @@ TEST_BEGIN(test_chunk)
 	hooks_mib[1] = (size_t)arena_ind;
 	old_size = sizeof(chunk_hooks_t);
 	new_size = sizeof(chunk_hooks_t);
-	assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
-	    &new_hooks, new_size), 0, "Unexpected chunk_hooks error");
+	assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, (void *)&old_hooks,
+	    &old_size, (void *)&new_hooks, new_size), 0,
+	    "Unexpected chunk_hooks error");
 	orig_hooks = old_hooks;
 	assert_ptr_ne(old_hooks.alloc, chunk_alloc, "Unexpected alloc error");
 	assert_ptr_ne(old_hooks.dalloc, chunk_dalloc,
@@ -164,18 +161,18 @@ TEST_BEGIN(test_chunk)
 
 	/* Get large size classes. */
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("arenas.lrun.0.size", &large0, &sz, NULL, 0), 0,
-	    "Unexpected arenas.lrun.0.size failure");
-	assert_d_eq(mallctl("arenas.lrun.1.size", &large1, &sz, NULL, 0), 0,
-	    "Unexpected arenas.lrun.1.size failure");
+	assert_d_eq(mallctl("arenas.lrun.0.size", (void *)&large0, &sz, NULL,
+	    0), 0, "Unexpected arenas.lrun.0.size failure");
+	assert_d_eq(mallctl("arenas.lrun.1.size", (void *)&large1, &sz, NULL,
+	    0), 0, "Unexpected arenas.lrun.1.size failure");
 
 	/* Get huge size classes. */
-	assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0,
-	    "Unexpected arenas.hchunk.0.size failure");
-	assert_d_eq(mallctl("arenas.hchunk.1.size", &huge1, &sz, NULL, 0), 0,
-	    "Unexpected arenas.hchunk.1.size failure");
-	assert_d_eq(mallctl("arenas.hchunk.2.size", &huge2, &sz, NULL, 0), 0,
-	    "Unexpected arenas.hchunk.2.size failure");
+	assert_d_eq(mallctl("arenas.hchunk.0.size", (void *)&huge0, &sz, NULL,
+	    0), 0, "Unexpected arenas.hchunk.0.size failure");
+	assert_d_eq(mallctl("arenas.hchunk.1.size", (void *)&huge1, &sz, NULL,
+	    0), 0, "Unexpected arenas.hchunk.1.size failure");
+	assert_d_eq(mallctl("arenas.hchunk.2.size", (void *)&huge2, &sz, NULL,
+	    0), 0, "Unexpected arenas.hchunk.2.size failure");
 
 	/* Test dalloc/decommit/purge cascade. */
 	purge_miblen = sizeof(purge_mib)/sizeof(size_t);
@@ -265,9 +262,9 @@ TEST_BEGIN(test_chunk)
 
 	/* Restore chunk hooks. */
 	assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, NULL, NULL,
-	    &old_hooks, new_size), 0, "Unexpected chunk_hooks error");
-	assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, &old_hooks, &old_size,
-	    NULL, 0), 0, "Unexpected chunk_hooks error");
+	    (void *)&old_hooks, new_size), 0, "Unexpected chunk_hooks error");
+	assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, (void *)&old_hooks,
+	    &old_size, NULL, 0), 0, "Unexpected chunk_hooks error");
 	assert_ptr_eq(old_hooks.alloc, orig_hooks.alloc,
 	    "Unexpected alloc error");
 	assert_ptr_eq(old_hooks.dalloc, orig_hooks.dalloc,

test/integration/chunk.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="junk:false"
+fi

test/integration/mallocx.c

@@ -1,9 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_FILL
-const char *malloc_conf = "junk:false";
-#endif
-
 static unsigned
 get_nsizes_impl(const char *cmd)
 {
@@ -11,7 +7,7 @@ get_nsizes_impl(const char *cmd)
 	size_t z;
 
 	z = sizeof(unsigned);
-	assert_d_eq(mallctl(cmd, &ret, &z, NULL, 0), 0,
+	assert_d_eq(mallctl(cmd, (void *)&ret, &z, NULL, 0), 0,
 	    "Unexpected mallctl(\"%s\", ...) failure", cmd);
 
 	return (ret);
@@ -37,7 +33,7 @@ get_size_impl(const char *cmd, size_t ind)
 	    0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd);
 	mib[2] = ind;
 	z = sizeof(size_t);
-	assert_d_eq(mallctlbymib(mib, miblen, &ret, &z, NULL, 0),
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&ret, &z, NULL, 0),
 	    0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind);
 
 	return (ret);

test/integration/mallocx.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="junk:false"
+fi

test/integration/overflow.c

@@ -8,8 +8,8 @@ TEST_BEGIN(test_overflow)
 	void *p;
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("arenas.nhchunks", (void *)&nhchunks, &sz, NULL, 0),
+	    0, "Unexpected mallctl() error");
 
 	miblen = sizeof(mib) / sizeof(size_t);
 	assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0,
@@ -17,8 +17,8 @@ TEST_BEGIN(test_overflow)
 	mib[2] = nhchunks - 1;
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,
-	    "Unexpected mallctlbymib() error");
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&max_size_class, &sz,
+	    NULL, 0), 0, "Unexpected mallctlbymib() error");
 
 	assert_ptr_null(malloc(max_size_class + 1),
 	    "Expected OOM due to over-sized allocation request");

test/integration/rallocx.c

@@ -7,7 +7,7 @@ get_nsizes_impl(const char *cmd)
 	size_t z;
 
 	z = sizeof(unsigned);
-	assert_d_eq(mallctl(cmd, &ret, &z, NULL, 0), 0,
+	assert_d_eq(mallctl(cmd, (void *)&ret, &z, NULL, 0), 0,
 	    "Unexpected mallctl(\"%s\", ...) failure", cmd);
 
 	return (ret);
@@ -33,7 +33,7 @@ get_size_impl(const char *cmd, size_t ind)
 	    0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd);
 	mib[2] = ind;
 	z = sizeof(size_t);
-	assert_d_eq(mallctlbymib(mib, miblen, &ret, &z, NULL, 0),
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&ret, &z, NULL, 0),
 	    0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind);
 
 	return (ret);

test/integration/sdallocx.c

@@ -1,7 +1,7 @@
 #include "test/jemalloc_test.h"
 
-#define	MAXALIGN (((size_t)1) << 25)
-#define	NITER 4
+#define	MAXALIGN (((size_t)1) << 22)
+#define	NITER 3
 
 TEST_BEGIN(test_basic)
 {

test/integration/thread_arena.c

@@ -16,8 +16,8 @@ thd_start(void *arg)
 	free(p);
 
 	size = sizeof(arena_ind);
-	if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind,
-	    sizeof(main_arena_ind)))) {
+	if ((err = mallctl("thread.arena", (void *)&arena_ind, &size,
+	    (void *)&main_arena_ind, sizeof(main_arena_ind)))) {
 		char buf[BUFERROR_BUF];
 
 		buferror(err, buf, sizeof(buf));
@@ -25,7 +25,8 @@ thd_start(void *arg)
 	}
 
 	size = sizeof(arena_ind);
-	if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) {
+	if ((err = mallctl("thread.arena", (void *)&arena_ind, &size, NULL,
+	    0))) {
 		char buf[BUFERROR_BUF];
 
 		buferror(err, buf, sizeof(buf));
@@ -50,7 +51,8 @@ TEST_BEGIN(test_thread_arena)
 	assert_ptr_not_null(p, "Error in malloc()");
 
 	size = sizeof(arena_ind);
-	if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) {
+	if ((err = mallctl("thread.arena", (void *)&arena_ind, &size, NULL,
+	    0))) {
 		char buf[BUFERROR_BUF];
 
 		buferror(err, buf, sizeof(buf));

test/integration/thread_tcache_enabled.c

@@ -16,7 +16,8 @@ thd_start(void *arg)
 	bool e0, e1;
 
 	sz = sizeof(bool);
-	if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) {
+	if ((err = mallctl("thread.tcache.enabled", (void *)&e0, &sz, NULL,
+	    0))) {
 		if (err == ENOENT) {
 			assert_false(config_tcache,
 			    "ENOENT should only be returned if tcache is "
@@ -27,53 +28,53 @@ thd_start(void *arg)
 
 	if (e0) {
 		e1 = false;
-		assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz),
-		    0, "Unexpected mallctl() error");
+		assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+		    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 		assert_true(e0, "tcache should be enabled");
 	}
 
 	e1 = true;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_false(e0, "tcache should be disabled");
 
 	e1 = true;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_true(e0, "tcache should be enabled");
 
 	e1 = false;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_true(e0, "tcache should be enabled");
 
 	e1 = false;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_false(e0, "tcache should be disabled");
 
 	free(malloc(1));
 	e1 = true;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_false(e0, "tcache should be disabled");
 
 	free(malloc(1));
 	e1 = true;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_true(e0, "tcache should be enabled");
 
 	free(malloc(1));
 	e1 = false;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_true(e0, "tcache should be enabled");
 
 	free(malloc(1));
 	e1 = false;
-	assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz,
+	    (void *)&e1, sz), 0, "Unexpected mallctl() error");
 	assert_false(e0, "tcache should be disabled");
 
 	free(malloc(1));

test/integration/xallocx.c

@@ -1,9 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_FILL
-const char *malloc_conf = "junk:false";
-#endif
-
 /*
  * Use a separate arena for xallocx() extension/contraction tests so that
  * internal allocation e.g. by heap profiling can't interpose allocations where
@@ -16,8 +12,8 @@ arena_ind(void)
 
 	if (ind == 0) {
 		size_t sz = sizeof(ind);
-		assert_d_eq(mallctl("arenas.extend", &ind, &sz, NULL, 0), 0,
-		    "Unexpected mallctl failure creating arena");
+		assert_d_eq(mallctl("arenas.extend", (void *)&ind, &sz, NULL,
+		    0), 0, "Unexpected mallctl failure creating arena");
 	}
 
 	return (ind);
@@ -78,7 +74,7 @@ get_nsizes_impl(const char *cmd)
 	size_t z;
 
 	z = sizeof(unsigned);
-	assert_d_eq(mallctl(cmd, &ret, &z, NULL, 0), 0,
+	assert_d_eq(mallctl(cmd, (void *)&ret, &z, NULL, 0), 0,
 	    "Unexpected mallctl(\"%s\", ...) failure", cmd);
 
 	return (ret);
@@ -118,7 +114,7 @@ get_size_impl(const char *cmd, size_t ind)
 	    0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd);
 	mib[2] = ind;
 	z = sizeof(size_t);
-	assert_d_eq(mallctlbymib(mib, miblen, &ret, &z, NULL, 0),
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&ret, &z, NULL, 0),
 	    0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind);
 
 	return (ret);

test/integration/xallocx.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="junk:false"
+fi

test/test.sh.in

@@ -11,6 +11,18 @@ case @abi@ in
     ;;
 esac
 
+# Make a copy of the @JEMALLOC_CPREFIX@MALLOC_CONF passed in to this script, so
+# it can be repeatedly concatenated with per test settings.
+export MALLOC_CONF_ALL=${@JEMALLOC_CPREFIX@MALLOC_CONF}
+# Concatenate the individual test's MALLOC_CONF and MALLOC_CONF_ALL.
+export_malloc_conf() {
+  if [ "x${MALLOC_CONF}" != "x" -a "x${MALLOC_CONF_ALL}" != "x" ] ; then
+    export @JEMALLOC_CPREFIX@MALLOC_CONF="${MALLOC_CONF},${MALLOC_CONF_ALL}"
+  else
+    export @JEMALLOC_CPREFIX@MALLOC_CONF="${MALLOC_CONF}${MALLOC_CONF_ALL}"
+  fi
+}
+
 # Corresponds to test_status_t.
 pass_code=0
 skip_code=1
@@ -24,7 +36,22 @@ for t in $@; do
     echo
   fi
   echo "=== ${t} ==="
-  ${t}@exe@ @abs_srcroot@ @abs_objroot@
+  if [ -e "@srcroot@${t}.sh" ] ; then
+    # Source the shell script corresponding to the test in a subshell and
+    # execute the test.  This allows the shell script to set MALLOC_CONF, which
+    # is then used to set @JEMALLOC_CPREFIX@MALLOC_CONF (thus allowing the
+    # per test shell script to ignore the @JEMALLOC_CPREFIX@ detail).
+    $(enable_fill=@enable_fill@ \
+      enable_prof=@enable_prof@ \
+      enable_tcache=@enable_tcache@ \
+      . @srcroot@${t}.sh && \
+      export_malloc_conf && \
+      ${t}@exe@ @abs_srcroot@ @abs_objroot@)
+  else
+    $(export MALLOC_CONF= && \
+      export_malloc_conf &&
+      ${t}@exe@ @abs_srcroot@ @abs_objroot@)
+  fi
   result_code=$?
   case ${result_code} in
     ${pass_code})

test/unit/arena_reset.c

@@ -1,9 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_PROF
-const char *malloc_conf = "prof:true,lg_prof_sample:0";
-#endif
-
 static unsigned
 get_nsizes_impl(const char *cmd)
 {
@@ -11,7 +7,7 @@ get_nsizes_impl(const char *cmd)
 	size_t z;
 
 	z = sizeof(unsigned);
-	assert_d_eq(mallctl(cmd, &ret, &z, NULL, 0), 0,
+	assert_d_eq(mallctl(cmd, (void *)&ret, &z, NULL, 0), 0,
 	    "Unexpected mallctl(\"%s\", ...) failure", cmd);
 
 	return (ret);
@@ -51,7 +47,7 @@ get_size_impl(const char *cmd, size_t ind)
 	    0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd);
 	mib[2] = ind;
 	z = sizeof(size_t);
-	assert_d_eq(mallctlbymib(mib, miblen, &ret, &z, NULL, 0),
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&ret, &z, NULL, 0),
 	    0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind);
 
 	return (ret);
@@ -92,8 +88,8 @@ TEST_BEGIN(test_arena_reset)
 	    && unlikely(opt_quarantine)));
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.extend", &arena_ind, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Unexpected mallctl() failure");
 
 	flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
 

test/unit/arena_reset.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,lg_prof_sample:0"
+fi

test/unit/decay.c

@@ -1,7 +1,5 @@
 #include "test/jemalloc_test.h"
 
-const char *malloc_conf = "purge:decay,decay_time:1";
-
 static nstime_monotonic_t *nstime_monotonic_orig;
 static nstime_update_t *nstime_update_orig;
 
@@ -40,10 +38,10 @@ TEST_BEGIN(test_decay_ticks)
 	    "Unexpected failure getting decay ticker");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0,
-	    "Unexpected mallctl failure");
-	assert_d_eq(mallctl("arenas.lrun.0.size", &large0, &sz, NULL, 0), 0,
-	    "Unexpected mallctl failure");
+	assert_d_eq(mallctl("arenas.hchunk.0.size", (void *)&huge0, &sz, NULL,
+	    0), 0, "Unexpected mallctl failure");
+	assert_d_eq(mallctl("arenas.lrun.0.size", (void *)&large0, &sz, NULL,
+	    0), 0, "Unexpected mallctl failure");
 
 	/*
 	 * Test the standard APIs using a huge size class, since we can't
@@ -175,8 +173,8 @@ TEST_BEGIN(test_decay_ticks)
 		tcache_sizes[1] = 1;
 
 		sz = sizeof(unsigned);
-		assert_d_eq(mallctl("tcache.create", &tcache_ind, &sz, NULL, 0),
-		    0, "Unexpected mallctl failure");
+		assert_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz,
+		    NULL, 0), 0, "Unexpected mallctl failure");
 
 		for (i = 0; i < sizeof(tcache_sizes) / sizeof(size_t); i++) {
 			sz = tcache_sizes[i];
@@ -193,7 +191,7 @@ TEST_BEGIN(test_decay_ticks)
 			dallocx(p, MALLOCX_TCACHE(tcache_ind));
 			tick0 = ticker_read(decay_ticker);
 			assert_d_eq(mallctl("tcache.flush", NULL, NULL,
-			    &tcache_ind, sizeof(unsigned)), 0,
+			    (void *)&tcache_ind, sizeof(unsigned)), 0,
 			    "Unexpected mallctl failure");
 			tick1 = ticker_read(decay_ticker);
 			assert_u32_ne(tick1, tick0,
@@ -228,22 +226,22 @@ TEST_BEGIN(test_decay_ticker)
 		size_t tcache_max;
 
 		sz = sizeof(size_t);
-		assert_d_eq(mallctl("arenas.tcache_max", &tcache_max, &sz, NULL,
-		    0), 0, "Unexpected mallctl failure");
+		assert_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max,
+		    &sz, NULL, 0), 0, "Unexpected mallctl failure");
 		large = nallocx(tcache_max + 1, flags);
 	}  else {
 		sz = sizeof(size_t);
-		assert_d_eq(mallctl("arenas.lrun.0.size", &large, &sz, NULL, 0),
-		    0, "Unexpected mallctl failure");
+		assert_d_eq(mallctl("arenas.lrun.0.size", (void *)&large, &sz,
+		    NULL, 0), 0, "Unexpected mallctl failure");
 	}
 
 	assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
 	    "Unexpected mallctl failure");
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(uint64_t)), 0,
-	    "Unexpected mallctl failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
+	    sizeof(uint64_t)), 0, "Unexpected mallctl failure");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge0, &sz, NULL, 0),
-	    config_stats ? 0 : ENOENT, "Unexpected mallctl result");
+	assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge0, &sz,
+	    NULL, 0), config_stats ? 0 : ENOENT, "Unexpected mallctl result");
 
 	for (i = 0; i < NPS; i++) {
 		ps[i] = mallocx(large, flags);
@@ -283,11 +281,11 @@ TEST_BEGIN(test_decay_ticker)
 			assert_ptr_not_null(p, "Unexpected mallocx() failure");
 			dallocx(p, flags);
 		}
-		assert_d_eq(mallctl("epoch", NULL, NULL, &epoch,
+		assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
 		    sizeof(uint64_t)), 0, "Unexpected mallctl failure");
 		sz = sizeof(uint64_t);
-		assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge1, &sz,
-		    NULL, 0), config_stats ? 0 : ENOENT,
+		assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge1,
+		    &sz, NULL, 0), config_stats ? 0 : ENOENT,
 		    "Unexpected mallctl result");
 
 		nstime_update(&time);
@@ -313,16 +311,16 @@ TEST_BEGIN(test_decay_nonmonotonic)
 	test_skip_if(opt_purge != purge_mode_decay);
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("arenas.lrun.0.size", &large0, &sz, NULL, 0), 0,
-	    "Unexpected mallctl failure");
+	assert_d_eq(mallctl("arenas.lrun.0.size", (void *)&large0, &sz, NULL,
+	    0), 0, "Unexpected mallctl failure");
 
 	assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
 	    "Unexpected mallctl failure");
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(uint64_t)), 0,
-	    "Unexpected mallctl failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
+	    sizeof(uint64_t)), 0, "Unexpected mallctl failure");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge0, &sz, NULL, 0),
-	    config_stats ? 0 : ENOENT, "Unexpected mallctl result");
+	assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge0, &sz,
+	    NULL, 0), config_stats ? 0 : ENOENT, "Unexpected mallctl result");
 
 	nupdates_mock = 0;
 	nstime_init(&time_mock, 0);
@@ -348,11 +346,11 @@ TEST_BEGIN(test_decay_nonmonotonic)
 		    "Expected nstime_update() to be called");
 	}
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(uint64_t)), 0,
-	    "Unexpected mallctl failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
+	    sizeof(uint64_t)), 0, "Unexpected mallctl failure");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge1, &sz, NULL, 0),
-	    config_stats ? 0 : ENOENT, "Unexpected mallctl result");
+	assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge1, &sz,
+	    NULL, 0), config_stats ? 0 : ENOENT, "Unexpected mallctl result");
 
 	if (config_stats)
 		assert_u64_eq(npurge0, npurge1, "Unexpected purging occurred");

test/unit/decay.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+export MALLOC_CONF="purge:decay,decay_time:1"

test/unit/extent_quantize.c

@@ -0,0 +1,98 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_huge_extent_size) {
+	unsigned nhchunks, i;
+	size_t sz, extent_size_prev, ceil_prev;
+	size_t mib[4];
+	size_t miblen = sizeof(mib) / sizeof(size_t);
+
+	/*
+	 * Iterate over all huge size classes, get their extent sizes, and
+	 * verify that the quantized size is the same as the extent size.
+	 */
+
+	sz = sizeof(unsigned);
+	assert_d_eq(mallctl("arenas.nhchunks", (void *)&nhchunks, &sz, NULL,
+	    0), 0, "Unexpected mallctl failure");
+
+	assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0,
+	    "Unexpected mallctlnametomib failure");
+	for (i = 0; i < nhchunks; i++) {
+		size_t extent_size, floor, ceil;
+
+
+		mib[2] = i;
+		sz = sizeof(size_t);
+		assert_d_eq(mallctlbymib(mib, miblen, (void *)&extent_size,
+		    &sz, NULL, 0), 0, "Unexpected mallctlbymib failure");
+		floor = extent_size_quantize_floor(extent_size);
+		ceil = extent_size_quantize_ceil(extent_size);
+
+		assert_zu_eq(extent_size, floor,
+		    "Extent quantization should be a no-op for precise size "
+		    "(extent_size=%zu)", extent_size);
+		assert_zu_eq(extent_size, ceil,
+		    "Extent quantization should be a no-op for precise size "
+		    "(extent_size=%zu)", extent_size);
+
+		if (i > 0) {
+			assert_zu_eq(extent_size_prev,
+			    extent_size_quantize_floor(extent_size - PAGE),
+			    "Floor should be a precise size");
+			if (extent_size_prev < ceil_prev) {
+				assert_zu_eq(ceil_prev, extent_size,
+				    "Ceiling should be a precise size "
+				    "(extent_size_prev=%zu, ceil_prev=%zu, "
+				    "extent_size=%zu)", extent_size_prev,
+				    ceil_prev, extent_size);
+			}
+		}
+		if (i + 1 < nhchunks) {
+			extent_size_prev = floor;
+			ceil_prev = extent_size_quantize_ceil(extent_size +
+			    PAGE);
+		}
+	}
+}
+TEST_END
+
+TEST_BEGIN(test_monotonic) {
+#define SZ_MAX	ZU(4 * 1024 * 1024)
+	unsigned i;
+	size_t floor_prev, ceil_prev;
+
+	floor_prev = 0;
+	ceil_prev = 0;
+	for (i = 1; i <= SZ_MAX >> LG_PAGE; i++) {
+		size_t extent_size, floor, ceil;
+
+		extent_size = i << LG_PAGE;
+		floor = extent_size_quantize_floor(extent_size);
+		ceil = extent_size_quantize_ceil(extent_size);
+
+		assert_zu_le(floor, extent_size,
+		    "Floor should be <= (floor=%zu, extent_size=%zu, ceil=%zu)",
+		    floor, extent_size, ceil);
+		assert_zu_ge(ceil, extent_size,
+		    "Ceiling should be >= (floor=%zu, extent_size=%zu, "
+		    "ceil=%zu)", floor, extent_size, ceil);
+
+		assert_zu_le(floor_prev, floor, "Floor should be monotonic "
+		    "(floor_prev=%zu, floor=%zu, extent_size=%zu, ceil=%zu)",
+		    floor_prev, floor, extent_size, ceil);
+		assert_zu_le(ceil_prev, ceil, "Ceiling should be monotonic "
+		    "(floor=%zu, extent_size=%zu, ceil_prev=%zu, ceil=%zu)",
+		    floor, extent_size, ceil_prev, ceil);
+
+		floor_prev = floor;
+		ceil_prev = ceil;
+	}
+}
+TEST_END
+
+int
+main(void) {
+	return test(
+	    test_huge_extent_size,
+	    test_monotonic);
+}

test/unit/junk.c

@@ -1,13 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_FILL
-#  ifndef JEMALLOC_TEST_JUNK_OPT
-#    define JEMALLOC_TEST_JUNK_OPT "junk:true"
-#  endif
-const char *malloc_conf =
-    "abort:false,zero:false,redzone:true,quarantine:0," JEMALLOC_TEST_JUNK_OPT;
-#endif
-
 static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig;
 static arena_dalloc_junk_large_t *arena_dalloc_junk_large_orig;
 static huge_dalloc_junk_t *huge_dalloc_junk_orig;

test/unit/junk.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="abort:false,zero:false,redzone:true,quarantine:0,junk:true"
+fi

test/unit/junk_alloc.c

@@ -1,3 +1 @@
-#define	JEMALLOC_TEST_JUNK_OPT "junk:alloc"
 #include "junk.c"
-#undef JEMALLOC_TEST_JUNK_OPT

test/unit/junk_alloc.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="abort:false,zero:false,redzone:true,quarantine:0,junk:alloc"
+fi

test/unit/junk_free.c

@@ -1,3 +1 @@
-#define	JEMALLOC_TEST_JUNK_OPT "junk:free"
 #include "junk.c"
-#undef JEMALLOC_TEST_JUNK_OPT

test/unit/junk_free.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="abort:false,zero:false,redzone:true,quarantine:0,junk:free"
+fi

test/unit/lg_chunk.c

@@ -1,12 +1,5 @@
 #include "test/jemalloc_test.h"
 
-/*
- * Make sure that opt.lg_chunk clamping is sufficient.  In practice, this test
- * program will fail a debug assertion during initialization and abort (rather
- * than the test soft-failing) if clamping is insufficient.
- */
-const char *malloc_conf = "lg_chunk:0";
-
 TEST_BEGIN(test_lg_chunk_clamp)
 {
 	void *p;

test/unit/lg_chunk.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# Make sure that opt.lg_chunk clamping is sufficient.  In practice, this test
+# program will fail a debug assertion during initialization and abort (rather
+# than the test soft-failing) if clamping is insufficient.
+export MALLOC_CONF="lg_chunk:0"

test/unit/mallctl.c

@@ -12,16 +12,18 @@ TEST_BEGIN(test_mallctl_errors)
 	    EPERM, "mallctl() should return EPERM on attempt to write "
 	    "read-only value");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)-1),
-	    EINVAL, "mallctl() should return EINVAL for input size mismatch");
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)+1),
-	    EINVAL, "mallctl() should return EINVAL for input size mismatch");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
+	    sizeof(epoch)-1), EINVAL,
+	    "mallctl() should return EINVAL for input size mismatch");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch,
+	    sizeof(epoch)+1), EINVAL,
+	    "mallctl() should return EINVAL for input size mismatch");
 
 	sz = sizeof(epoch)-1;
-	assert_d_eq(mallctl("epoch", &epoch, &sz, NULL, 0), EINVAL,
+	assert_d_eq(mallctl("epoch", (void *)&epoch, &sz, NULL, 0), EINVAL,
 	    "mallctl() should return EINVAL for output size mismatch");
 	sz = sizeof(epoch)+1;
-	assert_d_eq(mallctl("epoch", &epoch, &sz, NULL, 0), EINVAL,
+	assert_d_eq(mallctl("epoch", (void *)&epoch, &sz, NULL, 0), EINVAL,
 	    "mallctl() should return EINVAL for output size mismatch");
 }
 TEST_END
@@ -56,18 +58,20 @@ TEST_BEGIN(test_mallctlbymib_errors)
 	assert_d_eq(mallctlnametomib("epoch", mib, &miblen), 0,
 	    "Unexpected mallctlnametomib() failure");
 
-	assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &epoch,
+	assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&epoch,
 	    sizeof(epoch)-1), EINVAL,
 	    "mallctlbymib() should return EINVAL for input size mismatch");
-	assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &epoch,
+	assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&epoch,
 	    sizeof(epoch)+1), EINVAL,
 	    "mallctlbymib() should return EINVAL for input size mismatch");
 
 	sz = sizeof(epoch)-1;
-	assert_d_eq(mallctlbymib(mib, miblen, &epoch, &sz, NULL, 0), EINVAL,
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&epoch, &sz, NULL, 0),
+	    EINVAL,
 	    "mallctlbymib() should return EINVAL for output size mismatch");
 	sz = sizeof(epoch)+1;
-	assert_d_eq(mallctlbymib(mib, miblen, &epoch, &sz, NULL, 0), EINVAL,
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&epoch, &sz, NULL, 0),
+	    EINVAL,
 	    "mallctlbymib() should return EINVAL for output size mismatch");
 }
 TEST_END
@@ -83,18 +87,19 @@ TEST_BEGIN(test_mallctl_read_write)
 	assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size");
 
 	/* Read. */
-	assert_d_eq(mallctl("epoch", &old_epoch, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl("epoch", (void *)&old_epoch, &sz, NULL, 0), 0,
 	    "Unexpected mallctl() failure");
 	assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size");
 
 	/* Write. */
-	assert_d_eq(mallctl("epoch", NULL, NULL, &new_epoch, sizeof(new_epoch)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&new_epoch,
+	    sizeof(new_epoch)), 0, "Unexpected mallctl() failure");
 	assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size");
 
 	/* Read+write. */
-	assert_d_eq(mallctl("epoch", &old_epoch, &sz, &new_epoch,
-	    sizeof(new_epoch)), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", (void *)&old_epoch, &sz,
+	    (void *)&new_epoch, sizeof(new_epoch)), 0,
+	    "Unexpected mallctl() failure");
 	assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size");
 }
 TEST_END
@@ -120,8 +125,8 @@ TEST_BEGIN(test_mallctl_config)
 #define	TEST_MALLCTL_CONFIG(config, t) do {				\
 	t oldval;							\
 	size_t sz = sizeof(oldval);					\
-	assert_d_eq(mallctl("config."#config, &oldval, &sz, NULL, 0),	\
-	    0, "Unexpected mallctl() failure");				\
+	assert_d_eq(mallctl("config."#config, (void *)&oldval, &sz,	\
+	    NULL, 0), 0, "Unexpected mallctl() failure");		\
 	assert_b_eq(oldval, config_##config, "Incorrect config value");	\
 	assert_zu_eq(sz, sizeof(oldval), "Unexpected output size");	\
 } while (0)
@@ -137,6 +142,7 @@ TEST_BEGIN(test_mallctl_config)
 	TEST_MALLCTL_CONFIG(prof_libunwind, bool);
 	TEST_MALLCTL_CONFIG(stats, bool);
 	TEST_MALLCTL_CONFIG(tcache, bool);
+	TEST_MALLCTL_CONFIG(thp, bool);
 	TEST_MALLCTL_CONFIG(tls, bool);
 	TEST_MALLCTL_CONFIG(utrace, bool);
 	TEST_MALLCTL_CONFIG(valgrind, bool);
@@ -154,7 +160,8 @@ TEST_BEGIN(test_mallctl_opt)
 	t oldval;							\
 	size_t sz = sizeof(oldval);					\
 	int expected = config_##config ? 0 : ENOENT;			\
-	int result = mallctl("opt."#opt, &oldval, &sz, NULL, 0);	\
+	int result = mallctl("opt."#opt, (void *)&oldval, &sz, NULL,	\
+	    0);								\
 	assert_d_eq(result, expected,					\
 	    "Unexpected mallctl() result for opt."#opt);		\
 	assert_zu_eq(sz, sizeof(oldval), "Unexpected output size");	\
@@ -176,6 +183,7 @@ TEST_BEGIN(test_mallctl_opt)
 	TEST_MALLCTL_OPT(bool, xmalloc, xmalloc);
 	TEST_MALLCTL_OPT(bool, tcache, tcache);
 	TEST_MALLCTL_OPT(size_t, lg_tcache_max, tcache);
+	TEST_MALLCTL_OPT(bool, thp, thp);
 	TEST_MALLCTL_OPT(bool, prof, prof);
 	TEST_MALLCTL_OPT(const char *, prof_prefix, prof);
 	TEST_MALLCTL_OPT(bool, prof_active, prof);
@@ -197,7 +205,7 @@ TEST_BEGIN(test_manpage_example)
 	size_t len, miblen;
 
 	len = sizeof(nbins);
-	assert_d_eq(mallctl("arenas.nbins", &nbins, &len, NULL, 0), 0,
+	assert_d_eq(mallctl("arenas.nbins", (void *)&nbins, &len, NULL, 0), 0,
 	    "Unexpected mallctl() failure");
 
 	miblen = 4;
@@ -208,8 +216,8 @@ TEST_BEGIN(test_manpage_example)
 
 		mib[2] = i;
 		len = sizeof(bin_size);
-		assert_d_eq(mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0),
-		    0, "Unexpected mallctlbymib() failure");
+		assert_d_eq(mallctlbymib(mib, miblen, (void *)&bin_size, &len,
+		    NULL, 0), 0, "Unexpected mallctlbymib() failure");
 		/* Do something with bin_size... */
 	}
 }
@@ -258,25 +266,25 @@ TEST_BEGIN(test_tcache)
 	/* Create tcaches. */
 	for (i = 0; i < NTCACHES; i++) {
 		sz = sizeof(unsigned);
-		assert_d_eq(mallctl("tcache.create", &tis[i], &sz, NULL, 0), 0,
-		    "Unexpected mallctl() failure, i=%u", i);
+		assert_d_eq(mallctl("tcache.create", (void *)&tis[i], &sz, NULL,
+		    0), 0, "Unexpected mallctl() failure, i=%u", i);
 	}
 
 	/* Exercise tcache ID recycling. */
 	for (i = 0; i < NTCACHES; i++) {
-		assert_d_eq(mallctl("tcache.destroy", NULL, NULL, &tis[i],
-		    sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u",
-		    i);
+		assert_d_eq(mallctl("tcache.destroy", NULL, NULL,
+		    (void *)&tis[i], sizeof(unsigned)), 0,
+		    "Unexpected mallctl() failure, i=%u", i);
 	}
 	for (i = 0; i < NTCACHES; i++) {
 		sz = sizeof(unsigned);
-		assert_d_eq(mallctl("tcache.create", &tis[i], &sz, NULL, 0), 0,
-		    "Unexpected mallctl() failure, i=%u", i);
+		assert_d_eq(mallctl("tcache.create", (void *)&tis[i], &sz, NULL,
+		    0), 0, "Unexpected mallctl() failure, i=%u", i);
 	}
 
 	/* Flush empty tcaches. */
 	for (i = 0; i < NTCACHES; i++) {
-		assert_d_eq(mallctl("tcache.flush", NULL, NULL, &tis[i],
+		assert_d_eq(mallctl("tcache.flush", NULL, NULL, (void *)&tis[i],
 		    sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u",
 		    i);
 	}
@@ -321,16 +329,16 @@ TEST_BEGIN(test_tcache)
 
 	/* Flush some non-empty tcaches. */
 	for (i = 0; i < NTCACHES/2; i++) {
-		assert_d_eq(mallctl("tcache.flush", NULL, NULL, &tis[i],
+		assert_d_eq(mallctl("tcache.flush", NULL, NULL, (void *)&tis[i],
 		    sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u",
 		    i);
 	}
 
 	/* Destroy tcaches. */
 	for (i = 0; i < NTCACHES; i++) {
-		assert_d_eq(mallctl("tcache.destroy", NULL, NULL, &tis[i],
-		    sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u",
-		    i);
+		assert_d_eq(mallctl("tcache.destroy", NULL, NULL,
+		    (void *)&tis[i], sizeof(unsigned)), 0,
+		    "Unexpected mallctl() failure, i=%u", i);
 	}
 }
 TEST_END
@@ -340,15 +348,17 @@ TEST_BEGIN(test_thread_arena)
 	unsigned arena_old, arena_new, narenas;
 	size_t sz = sizeof(unsigned);
 
-	assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0),
+	    0, "Unexpected mallctl() failure");
 	assert_u_eq(narenas, opt_narenas, "Number of arenas incorrect");
 	arena_new = narenas - 1;
-	assert_d_eq(mallctl("thread.arena", &arena_old, &sz, &arena_new,
-	    sizeof(unsigned)), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", (void *)&arena_old, &sz,
+	    (void *)&arena_new, sizeof(unsigned)), 0,
+	    "Unexpected mallctl() failure");
 	arena_new = 0;
-	assert_d_eq(mallctl("thread.arena", &arena_old, &sz, &arena_new,
-	    sizeof(unsigned)), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", (void *)&arena_old, &sz,
+	    (void *)&arena_new, sizeof(unsigned)), 0,
+	    "Unexpected mallctl() failure");
 }
 TEST_END
 
@@ -359,17 +369,18 @@ TEST_BEGIN(test_arena_i_lg_dirty_mult)
 
 	test_skip_if(opt_purge != purge_mode_ratio);
 
-	assert_d_eq(mallctl("arena.0.lg_dirty_mult", &orig_lg_dirty_mult, &sz,
-	    NULL, 0), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arena.0.lg_dirty_mult",
+	    (void *)&orig_lg_dirty_mult, &sz, NULL, 0), 0,
+	    "Unexpected mallctl() failure");
 
 	lg_dirty_mult = -2;
 	assert_d_eq(mallctl("arena.0.lg_dirty_mult", NULL, NULL,
-	    &lg_dirty_mult, sizeof(ssize_t)), EFAULT,
+	    (void *)&lg_dirty_mult, sizeof(ssize_t)), EFAULT,
 	    "Unexpected mallctl() success");
 
 	lg_dirty_mult = (sizeof(size_t) << 3);
 	assert_d_eq(mallctl("arena.0.lg_dirty_mult", NULL, NULL,
-	    &lg_dirty_mult, sizeof(ssize_t)), EFAULT,
+	    (void *)&lg_dirty_mult, sizeof(ssize_t)), EFAULT,
 	    "Unexpected mallctl() success");
 
 	for (prev_lg_dirty_mult = orig_lg_dirty_mult, lg_dirty_mult = -1;
@@ -377,9 +388,9 @@ TEST_BEGIN(test_arena_i_lg_dirty_mult)
 	    = lg_dirty_mult, lg_dirty_mult++) {
 		ssize_t old_lg_dirty_mult;
 
-		assert_d_eq(mallctl("arena.0.lg_dirty_mult", &old_lg_dirty_mult,
-		    &sz, &lg_dirty_mult, sizeof(ssize_t)), 0,
-		    "Unexpected mallctl() failure");
+		assert_d_eq(mallctl("arena.0.lg_dirty_mult",
+		    (void *)&old_lg_dirty_mult, &sz, (void *)&lg_dirty_mult,
+		    sizeof(ssize_t)), 0, "Unexpected mallctl() failure");
 		assert_zd_eq(old_lg_dirty_mult, prev_lg_dirty_mult,
 		    "Unexpected old arena.0.lg_dirty_mult");
 	}
@@ -393,25 +404,25 @@ TEST_BEGIN(test_arena_i_decay_time)
 
 	test_skip_if(opt_purge != purge_mode_decay);
 
-	assert_d_eq(mallctl("arena.0.decay_time", &orig_decay_time, &sz,
+	assert_d_eq(mallctl("arena.0.decay_time", (void *)&orig_decay_time, &sz,
 	    NULL, 0), 0, "Unexpected mallctl() failure");
 
 	decay_time = -2;
 	assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL,
-	    &decay_time, sizeof(ssize_t)), EFAULT,
+	    (void *)&decay_time, sizeof(ssize_t)), EFAULT,
 	    "Unexpected mallctl() success");
 
 	decay_time = 0x7fffffff;
 	assert_d_eq(mallctl("arena.0.decay_time", NULL, NULL,
-	    &decay_time, sizeof(ssize_t)), 0,
+	    (void *)&decay_time, sizeof(ssize_t)), 0,
 	    "Unexpected mallctl() failure");
 
 	for (prev_decay_time = decay_time, decay_time = -1;
 	    decay_time < 20; prev_decay_time = decay_time, decay_time++) {
 		ssize_t old_decay_time;
 
-		assert_d_eq(mallctl("arena.0.decay_time", &old_decay_time,
-		    &sz, &decay_time, sizeof(ssize_t)), 0,
+		assert_d_eq(mallctl("arena.0.decay_time", (void *)&old_decay_time,
+		    &sz, (void *)&decay_time, sizeof(ssize_t)), 0,
 		    "Unexpected mallctl() failure");
 		assert_zd_eq(old_decay_time, prev_decay_time,
 		    "Unexpected old arena.0.decay_time");
@@ -429,8 +440,8 @@ TEST_BEGIN(test_arena_i_purge)
 	assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
 	    "Unexpected mallctl() failure");
 
-	assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0),
+	    0, "Unexpected mallctl() failure");
 	assert_d_eq(mallctlnametomib("arena.0.purge", mib, &miblen), 0,
 	    "Unexpected mallctlnametomib() failure");
 	mib[1] = narenas;
@@ -449,8 +460,8 @@ TEST_BEGIN(test_arena_i_decay)
 	assert_d_eq(mallctl("arena.0.decay", NULL, NULL, NULL, 0), 0,
 	    "Unexpected mallctl() failure");
 
-	assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0),
+	    0, "Unexpected mallctl() failure");
 	assert_d_eq(mallctlnametomib("arena.0.decay", mib, &miblen), 0,
 	    "Unexpected mallctlnametomib() failure");
 	mib[1] = narenas;
@@ -471,31 +482,35 @@ TEST_BEGIN(test_arena_i_dss)
 	    "Unexpected mallctlnametomib() error");
 
 	dss_prec_new = "disabled";
-	assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, &dss_prec_new,
-	    sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&dss_prec_old, &sz,
+	    (void *)&dss_prec_new, sizeof(dss_prec_new)), 0,
+	    "Unexpected mallctl() failure");
 	assert_str_ne(dss_prec_old, "primary",
 	    "Unexpected default for dss precedence");
 
-	assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_new, &sz, &dss_prec_old,
-	    sizeof(dss_prec_old)), 0, "Unexpected mallctl() failure");
-
-	assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, NULL, 0), 0,
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&dss_prec_new, &sz,
+	    (void *)&dss_prec_old, sizeof(dss_prec_old)), 0,
 	    "Unexpected mallctl() failure");
+
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&dss_prec_old, &sz, NULL,
+	    0), 0, "Unexpected mallctl() failure");
 	assert_str_ne(dss_prec_old, "primary",
 	    "Unexpected value for dss precedence");
 
 	mib[1] = narenas_total_get();
 	dss_prec_new = "disabled";
-	assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, &dss_prec_new,
-	    sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&dss_prec_old, &sz,
+	    (void *)&dss_prec_new, sizeof(dss_prec_new)), 0,
+	    "Unexpected mallctl() failure");
 	assert_str_ne(dss_prec_old, "primary",
 	    "Unexpected default for dss precedence");
 
-	assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_new, &sz, &dss_prec_old,
-	    sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure");
-
-	assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, NULL, 0), 0,
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&dss_prec_new, &sz,
+	    (void *)&dss_prec_old, sizeof(dss_prec_new)), 0,
 	    "Unexpected mallctl() failure");
+
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&dss_prec_old, &sz, NULL,
+	    0), 0, "Unexpected mallctl() failure");
 	assert_str_ne(dss_prec_old, "primary",
 	    "Unexpected value for dss precedence");
 }
@@ -506,14 +521,14 @@ TEST_BEGIN(test_arenas_initialized)
 	unsigned narenas;
 	size_t sz = sizeof(narenas);
 
-	assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0),
+	    0, "Unexpected mallctl() failure");
 	{
 		VARIABLE_ARRAY(bool, initialized, narenas);
 
 		sz = narenas * sizeof(bool);
-		assert_d_eq(mallctl("arenas.initialized", initialized, &sz,
-		    NULL, 0), 0, "Unexpected mallctl() failure");
+		assert_d_eq(mallctl("arenas.initialized", (void *)initialized,
+		    &sz, NULL, 0), 0, "Unexpected mallctl() failure");
 	}
 }
 TEST_END
@@ -525,17 +540,17 @@ TEST_BEGIN(test_arenas_lg_dirty_mult)
 
 	test_skip_if(opt_purge != purge_mode_ratio);
 
-	assert_d_eq(mallctl("arenas.lg_dirty_mult", &orig_lg_dirty_mult, &sz,
-	    NULL, 0), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.lg_dirty_mult", (void *)&orig_lg_dirty_mult,
+	    &sz, NULL, 0), 0, "Unexpected mallctl() failure");
 
 	lg_dirty_mult = -2;
 	assert_d_eq(mallctl("arenas.lg_dirty_mult", NULL, NULL,
-	    &lg_dirty_mult, sizeof(ssize_t)), EFAULT,
+	    (void *)&lg_dirty_mult, sizeof(ssize_t)), EFAULT,
 	    "Unexpected mallctl() success");
 
 	lg_dirty_mult = (sizeof(size_t) << 3);
 	assert_d_eq(mallctl("arenas.lg_dirty_mult", NULL, NULL,
-	    &lg_dirty_mult, sizeof(ssize_t)), EFAULT,
+	    (void *)&lg_dirty_mult, sizeof(ssize_t)), EFAULT,
 	    "Unexpected mallctl() success");
 
 	for (prev_lg_dirty_mult = orig_lg_dirty_mult, lg_dirty_mult = -1;
@@ -543,9 +558,9 @@ TEST_BEGIN(test_arenas_lg_dirty_mult)
 	    lg_dirty_mult, lg_dirty_mult++) {
 		ssize_t old_lg_dirty_mult;
 
-		assert_d_eq(mallctl("arenas.lg_dirty_mult", &old_lg_dirty_mult,
-		    &sz, &lg_dirty_mult, sizeof(ssize_t)), 0,
-		    "Unexpected mallctl() failure");
+		assert_d_eq(mallctl("arenas.lg_dirty_mult",
+		    (void *)&old_lg_dirty_mult, &sz, (void *)&lg_dirty_mult,
+		    sizeof(ssize_t)), 0, "Unexpected mallctl() failure");
 		assert_zd_eq(old_lg_dirty_mult, prev_lg_dirty_mult,
 		    "Unexpected old arenas.lg_dirty_mult");
 	}
@@ -559,26 +574,26 @@ TEST_BEGIN(test_arenas_decay_time)
 
 	test_skip_if(opt_purge != purge_mode_decay);
 
-	assert_d_eq(mallctl("arenas.decay_time", &orig_decay_time, &sz,
+	assert_d_eq(mallctl("arenas.decay_time", (void *)&orig_decay_time, &sz,
 	    NULL, 0), 0, "Unexpected mallctl() failure");
 
 	decay_time = -2;
 	assert_d_eq(mallctl("arenas.decay_time", NULL, NULL,
-	    &decay_time, sizeof(ssize_t)), EFAULT,
+	    (void *)&decay_time, sizeof(ssize_t)), EFAULT,
 	    "Unexpected mallctl() success");
 
 	decay_time = 0x7fffffff;
 	assert_d_eq(mallctl("arenas.decay_time", NULL, NULL,
-	    &decay_time, sizeof(ssize_t)), 0,
+	    (void *)&decay_time, sizeof(ssize_t)), 0,
 	    "Expected mallctl() failure");
 
 	for (prev_decay_time = decay_time, decay_time = -1;
 	    decay_time < 20; prev_decay_time = decay_time, decay_time++) {
 		ssize_t old_decay_time;
 
-		assert_d_eq(mallctl("arenas.decay_time", &old_decay_time,
-		    &sz, &decay_time, sizeof(ssize_t)), 0,
-		    "Unexpected mallctl() failure");
+		assert_d_eq(mallctl("arenas.decay_time",
+		    (void *)&old_decay_time, &sz, (void *)&decay_time,
+		    sizeof(ssize_t)), 0, "Unexpected mallctl() failure");
 		assert_zd_eq(old_decay_time, prev_decay_time,
 		    "Unexpected old arenas.decay_time");
 	}
@@ -591,8 +606,8 @@ TEST_BEGIN(test_arenas_constants)
 #define	TEST_ARENAS_CONSTANT(t, name, expected) do {			\
 	t name;								\
 	size_t sz = sizeof(t);						\
-	assert_d_eq(mallctl("arenas."#name, &name, &sz, NULL, 0), 0,	\
-	    "Unexpected mallctl() failure");				\
+	assert_d_eq(mallctl("arenas."#name, (void *)&name, &sz, NULL,	\
+	    0), 0, "Unexpected mallctl() failure");			\
 	assert_zu_eq(name, expected, "Incorrect "#name" size");		\
 } while (0)
 
@@ -612,8 +627,8 @@ TEST_BEGIN(test_arenas_bin_constants)
 #define	TEST_ARENAS_BIN_CONSTANT(t, name, expected) do {		\
 	t name;								\
 	size_t sz = sizeof(t);						\
-	assert_d_eq(mallctl("arenas.bin.0."#name, &name, &sz, NULL, 0),	\
-	    0, "Unexpected mallctl() failure");				\
+	assert_d_eq(mallctl("arenas.bin.0."#name, (void *)&name, &sz,	\
+	    NULL, 0), 0, "Unexpected mallctl() failure");		\
 	assert_zu_eq(name, expected, "Incorrect "#name" size");		\
 } while (0)
 
@@ -631,8 +646,8 @@ TEST_BEGIN(test_arenas_lrun_constants)
 #define	TEST_ARENAS_LRUN_CONSTANT(t, name, expected) do {		\
 	t name;								\
 	size_t sz = sizeof(t);						\
-	assert_d_eq(mallctl("arenas.lrun.0."#name, &name, &sz, NULL,	\
-	    0), 0, "Unexpected mallctl() failure");			\
+	assert_d_eq(mallctl("arenas.lrun.0."#name, (void *)&name, &sz,	\
+	    NULL, 0), 0, "Unexpected mallctl() failure");		\
 	assert_zu_eq(name, expected, "Incorrect "#name" size");		\
 } while (0)
 
@@ -648,8 +663,8 @@ TEST_BEGIN(test_arenas_hchunk_constants)
 #define	TEST_ARENAS_HCHUNK_CONSTANT(t, name, expected) do {		\
 	t name;								\
 	size_t sz = sizeof(t);						\
-	assert_d_eq(mallctl("arenas.hchunk.0."#name, &name, &sz, NULL,	\
-	    0), 0, "Unexpected mallctl() failure");			\
+	assert_d_eq(mallctl("arenas.hchunk.0."#name, (void *)&name,	\
+	    &sz, NULL, 0), 0, "Unexpected mallctl() failure");		\
 	assert_zu_eq(name, expected, "Incorrect "#name" size");		\
 } while (0)
 
@@ -664,12 +679,12 @@ TEST_BEGIN(test_arenas_extend)
 	unsigned narenas_before, arena, narenas_after;
 	size_t sz = sizeof(unsigned);
 
-	assert_d_eq(mallctl("arenas.narenas", &narenas_before, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
-	assert_d_eq(mallctl("arenas.extend", &arena, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() failure");
-	assert_d_eq(mallctl("arenas.narenas", &narenas_after, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl("arenas.narenas", (void *)&narenas_before, &sz,
+	    NULL, 0), 0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.extend", (void *)&arena, &sz, NULL, 0), 0,
 	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("arenas.narenas", (void *)&narenas_after, &sz, NULL,
+	    0), 0, "Unexpected mallctl() failure");
 
 	assert_u_eq(narenas_before+1, narenas_after,
 	    "Unexpected number of arenas before versus after extension");
@@ -683,8 +698,8 @@ TEST_BEGIN(test_stats_arenas)
 #define	TEST_STATS_ARENAS(t, name) do {					\
 	t name;								\
 	size_t sz = sizeof(t);						\
-	assert_d_eq(mallctl("stats.arenas.0."#name, &name, &sz, NULL,	\
-	    0), 0, "Unexpected mallctl() failure");			\
+	assert_d_eq(mallctl("stats.arenas.0."#name, (void *)&name, &sz,	\
+	    NULL, 0), 0, "Unexpected mallctl() failure");		\
 } while (0)
 
 	TEST_STATS_ARENAS(unsigned, nthreads);

test/unit/pack.c

@@ -0,0 +1,198 @@
+#include "test/jemalloc_test.h"
+
+/*
+ * Size class that is a divisor of the page size, ideally 4+ regions per run.
+ */
+#if LG_PAGE <= 14
+#define	SZ	(ZU(1) << (LG_PAGE - 2))
+#else
+#define	SZ	4096
+#endif
+
+/*
+ * Number of chunks to consume at high water mark.  Should be at least 2 so that
+ * if mmap()ed memory grows downward, downward growth of mmap()ed memory is
+ * tested.
+ */
+#define	NCHUNKS	8
+
+static unsigned
+binind_compute(void)
+{
+	size_t sz;
+	unsigned nbins, i;
+
+	sz = sizeof(nbins);
+	assert_d_eq(mallctl("arenas.nbins", (void *)&nbins, &sz, NULL, 0), 0,
+	    "Unexpected mallctl failure");
+
+	for (i = 0; i < nbins; i++) {
+		size_t mib[4];
+		size_t miblen = sizeof(mib)/sizeof(size_t);
+		size_t size;
+
+		assert_d_eq(mallctlnametomib("arenas.bin.0.size", mib,
+		    &miblen), 0, "Unexpected mallctlnametomb failure");
+		mib[2] = (size_t)i;
+
+		sz = sizeof(size);
+		assert_d_eq(mallctlbymib(mib, miblen, (void *)&size, &sz, NULL,
+		    0), 0, "Unexpected mallctlbymib failure");
+		if (size == SZ)
+			return (i);
+	}
+
+	test_fail("Unable to compute nregs_per_run");
+	return (0);
+}
+
+static size_t
+nregs_per_run_compute(void)
+{
+	uint32_t nregs;
+	size_t sz;
+	unsigned binind = binind_compute();
+	size_t mib[4];
+	size_t miblen = sizeof(mib)/sizeof(size_t);
+
+	assert_d_eq(mallctlnametomib("arenas.bin.0.nregs", mib, &miblen), 0,
+	    "Unexpected mallctlnametomb failure");
+	mib[2] = (size_t)binind;
+	sz = sizeof(nregs);
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&nregs, &sz, NULL,
+	    0), 0, "Unexpected mallctlbymib failure");
+	return (nregs);
+}
+
+static size_t
+npages_per_run_compute(void)
+{
+	size_t sz;
+	unsigned binind = binind_compute();
+	size_t mib[4];
+	size_t miblen = sizeof(mib)/sizeof(size_t);
+	size_t run_size;
+
+	assert_d_eq(mallctlnametomib("arenas.bin.0.run_size", mib, &miblen), 0,
+	    "Unexpected mallctlnametomb failure");
+	mib[2] = (size_t)binind;
+	sz = sizeof(run_size);
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&run_size, &sz, NULL,
+	    0), 0, "Unexpected mallctlbymib failure");
+	return (run_size >> LG_PAGE);
+}
+
+static size_t
+npages_per_chunk_compute(void)
+{
+
+	return ((chunksize >> LG_PAGE) - map_bias);
+}
+
+static size_t
+nruns_per_chunk_compute(void)
+{
+
+	return (npages_per_chunk_compute() / npages_per_run_compute());
+}
+
+static unsigned
+arenas_extend_mallctl(void)
+{
+	unsigned arena_ind;
+	size_t sz;
+
+	sz = sizeof(arena_ind);
+	assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Error in arenas.extend");
+
+	return (arena_ind);
+}
+
+static void
+arena_reset_mallctl(unsigned arena_ind)
+{
+	size_t mib[3];
+	size_t miblen = sizeof(mib)/sizeof(size_t);
+
+	assert_d_eq(mallctlnametomib("arena.0.reset", mib, &miblen), 0,
+	    "Unexpected mallctlnametomib() failure");
+	mib[1] = (size_t)arena_ind;
+	assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0,
+	    "Unexpected mallctlbymib() failure");
+}
+
+TEST_BEGIN(test_pack)
+{
+	unsigned arena_ind = arenas_extend_mallctl();
+	size_t nregs_per_run = nregs_per_run_compute();
+	size_t nruns_per_chunk = nruns_per_chunk_compute();
+	size_t nruns = nruns_per_chunk * NCHUNKS;
+	size_t nregs = nregs_per_run * nruns;
+	VARIABLE_ARRAY(void *, ptrs, nregs);
+	size_t i, j, offset;
+
+	/* Fill matrix. */
+	for (i = offset = 0; i < nruns; i++) {
+		for (j = 0; j < nregs_per_run; j++) {
+			void *p = mallocx(SZ, MALLOCX_ARENA(arena_ind) |
+			    MALLOCX_TCACHE_NONE);
+			assert_ptr_not_null(p,
+			    "Unexpected mallocx(%zu, MALLOCX_ARENA(%u) |"
+			    " MALLOCX_TCACHE_NONE) failure, run=%zu, reg=%zu",
+			    SZ, arena_ind, i, j);
+			ptrs[(i * nregs_per_run) + j] = p;
+		}
+	}
+
+	/*
+	 * Free all but one region of each run, but rotate which region is
+	 * preserved, so that subsequent allocations exercise the within-run
+	 * layout policy.
+	 */
+	offset = 0;
+	for (i = offset = 0;
+	    i < nruns;
+	    i++, offset = (offset + 1) % nregs_per_run) {
+		for (j = 0; j < nregs_per_run; j++) {
+			void *p = ptrs[(i * nregs_per_run) + j];
+			if (offset == j)
+				continue;
+			dallocx(p, MALLOCX_ARENA(arena_ind) |
+			    MALLOCX_TCACHE_NONE);
+		}
+	}
+
+	/*
+	 * Logically refill matrix, skipping preserved regions and verifying
+	 * that the matrix is unmodified.
+	 */
+	offset = 0;
+	for (i = offset = 0;
+	    i < nruns;
+	    i++, offset = (offset + 1) % nregs_per_run) {
+		for (j = 0; j < nregs_per_run; j++) {
+			void *p;
+
+			if (offset == j)
+				continue;
+			p = mallocx(SZ, MALLOCX_ARENA(arena_ind) |
+			    MALLOCX_TCACHE_NONE);
+			assert_ptr_eq(p, ptrs[(i * nregs_per_run) + j],
+			    "Unexpected refill discrepancy, run=%zu, reg=%zu\n",
+			    i, j);
+		}
+	}
+
+	/* Clean up. */
+	arena_reset_mallctl(arena_ind);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_pack));
+}

test/unit/pack.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Use smallest possible chunk size.  Immediately purge to minimize
+# fragmentation.
+export MALLOC_CONF="lg_chunk:0,lg_dirty_mult:-1,decay_time:-1"

test/unit/pages.c

@@ -0,0 +1,27 @@
+#include "test/jemalloc_test.h"
+
+TEST_BEGIN(test_pages_huge)
+{
+	bool commit;
+	void *pages;
+
+	commit = true;
+	pages = pages_map(NULL, PAGE, &commit);
+	assert_ptr_not_null(pages, "Unexpected pages_map() error");
+
+	assert_false(pages_huge(pages, PAGE),
+	    "Unexpected pages_huge() result");
+	assert_false(pages_nohuge(pages, PAGE),
+	    "Unexpected pages_nohuge() result");
+
+	pages_unmap(pages, PAGE);
+}
+TEST_END
+
+int
+main(void)
+{
+
+	return (test(
+	    test_pages_huge));
+}

test/unit/prof_accum.c

@@ -5,11 +5,6 @@
 #define	DUMP_INTERVAL		1
 #define	BT_COUNT_CHECK_INTERVAL	5
 
-#ifdef JEMALLOC_PROF
-const char *malloc_conf =
-    "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0";
-#endif
-
 static int
 prof_dump_open_intercept(bool propagate_err, const char *filename)
 {
@@ -68,8 +63,9 @@ TEST_BEGIN(test_idump)
 	test_skip_if(!config_prof);
 
 	active = true;
-	assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)),
-	    0, "Unexpected mallctl failure while activating profiling");
+	assert_d_eq(mallctl("prof.active", NULL, NULL, (void *)&active,
+	    sizeof(active)), 0,
+	    "Unexpected mallctl failure while activating profiling");
 
 	prof_dump_open = prof_dump_open_intercept;
 

test/unit/prof_accum.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0"
+fi

test/unit/prof_active.c

@@ -1,10 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_PROF
-const char *malloc_conf =
-    "prof:true,prof_thread_active_init:false,lg_prof_sample:0";
-#endif
-
 static void
 mallctl_bool_get(const char *name, bool expected, const char *func, int line)
 {
@@ -12,7 +7,7 @@ mallctl_bool_get(const char *name, bool expected, const char *func, int line)
 	size_t sz;
 
 	sz = sizeof(old);
-	assert_d_eq(mallctl(name, &old, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl(name, (void *)&old, &sz, NULL, 0), 0,
 	    "%s():%d: Unexpected mallctl failure reading %s", func, line, name);
 	assert_b_eq(old, expected, "%s():%d: Unexpected %s value", func, line,
 	    name);
@@ -26,7 +21,8 @@ mallctl_bool_set(const char *name, bool old_expected, bool val_new,
 	size_t sz;
 
 	sz = sizeof(old);
-	assert_d_eq(mallctl(name, &old, &sz, &val_new, sizeof(val_new)), 0,
+	assert_d_eq(mallctl(name, (void *)&old, &sz, (void *)&val_new,
+	    sizeof(val_new)), 0,
 	    "%s():%d: Unexpected mallctl failure reading/writing %s", func,
 	    line, name);
 	assert_b_eq(old, old_expected, "%s():%d: Unexpected %s value", func,

test/unit/prof_active.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,prof_thread_active_init:false,lg_prof_sample:0"
+fi

test/unit/prof_gdump.c

@@ -1,9 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_PROF
-const char *malloc_conf = "prof:true,prof_active:false,prof_gdump:true";
-#endif
-
 static bool did_prof_dump_open;
 
 static int
@@ -28,8 +24,9 @@ TEST_BEGIN(test_gdump)
 	test_skip_if(!config_prof);
 
 	active = true;
-	assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)),
-	    0, "Unexpected mallctl failure while activating profiling");
+	assert_d_eq(mallctl("prof.active", NULL, NULL, (void *)&active,
+	    sizeof(active)), 0,
+	    "Unexpected mallctl failure while activating profiling");
 
 	prof_dump_open = prof_dump_open_intercept;
 
@@ -45,8 +42,8 @@ TEST_BEGIN(test_gdump)
 
 	gdump = false;
 	sz = sizeof(gdump_old);
-	assert_d_eq(mallctl("prof.gdump", &gdump_old, &sz, &gdump,
-	    sizeof(gdump)), 0,
+	assert_d_eq(mallctl("prof.gdump", (void *)&gdump_old, &sz,
+	    (void *)&gdump, sizeof(gdump)), 0,
 	    "Unexpected mallctl failure while disabling prof.gdump");
 	assert(gdump_old);
 	did_prof_dump_open = false;
@@ -56,8 +53,8 @@ TEST_BEGIN(test_gdump)
 
 	gdump = true;
 	sz = sizeof(gdump_old);
-	assert_d_eq(mallctl("prof.gdump", &gdump_old, &sz, &gdump,
-	    sizeof(gdump)), 0,
+	assert_d_eq(mallctl("prof.gdump", (void *)&gdump_old, &sz,
+	    (void *)&gdump, sizeof(gdump)), 0,
 	    "Unexpected mallctl failure while enabling prof.gdump");
 	assert(!gdump_old);
 	did_prof_dump_open = false;

test/unit/prof_gdump.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,prof_active:false,prof_gdump:true"
+fi
+

test/unit/prof_idump.c

@@ -1,11 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_PROF
-const char *malloc_conf =
-    "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0,"
-    "lg_prof_interval:0";
-#endif
-
 static bool did_prof_dump_open;
 
 static int
@@ -29,8 +23,9 @@ TEST_BEGIN(test_idump)
 	test_skip_if(!config_prof);
 
 	active = true;
-	assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)),
-	    0, "Unexpected mallctl failure while activating profiling");
+	assert_d_eq(mallctl("prof.active", NULL, NULL, (void *)&active,
+	    sizeof(active)), 0,
+	    "Unexpected mallctl failure while activating profiling");
 
 	prof_dump_open = prof_dump_open_intercept;
 

test/unit/prof_idump.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0,lg_prof_interval:0"
+fi
+
+

test/unit/prof_reset.c

@@ -1,10 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_PROF
-const char *malloc_conf =
-    "prof:true,prof_active:false,lg_prof_sample:0";
-#endif
-
 static int
 prof_dump_open_intercept(bool propagate_err, const char *filename)
 {
@@ -20,8 +15,8 @@ static void
 set_prof_active(bool active)
 {
 
-	assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)),
-	    0, "Unexpected mallctl failure");
+	assert_d_eq(mallctl("prof.active", NULL, NULL, (void *)&active,
+	    sizeof(active)), 0, "Unexpected mallctl failure");
 }
 
 static size_t
@@ -30,7 +25,8 @@ get_lg_prof_sample(void)
 	size_t lg_prof_sample;
 	size_t sz = sizeof(size_t);
 
-	assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl("prof.lg_sample", (void *)&lg_prof_sample, &sz,
+	    NULL, 0), 0,
 	    "Unexpected mallctl failure while reading profiling sample rate");
 	return (lg_prof_sample);
 }
@@ -39,7 +35,7 @@ static void
 do_prof_reset(size_t lg_prof_sample)
 {
 	assert_d_eq(mallctl("prof.reset", NULL, NULL,
-	    &lg_prof_sample, sizeof(size_t)), 0,
+	    (void *)&lg_prof_sample, sizeof(size_t)), 0,
 	    "Unexpected mallctl failure while resetting profile data");
 	assert_zu_eq(lg_prof_sample, get_lg_prof_sample(),
 	    "Expected profile sample rate change");
@@ -54,8 +50,8 @@ TEST_BEGIN(test_prof_reset_basic)
 	test_skip_if(!config_prof);
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("opt.lg_prof_sample", &lg_prof_sample_orig, &sz,
-	    NULL, 0), 0,
+	assert_d_eq(mallctl("opt.lg_prof_sample", (void *)&lg_prof_sample_orig,
+	    &sz, NULL, 0), 0,
 	    "Unexpected mallctl failure while reading profiling sample rate");
 	assert_zu_eq(lg_prof_sample_orig, 0,
 	    "Unexpected profiling sample rate");

test/unit/prof_reset.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,prof_active:false,lg_prof_sample:0"
+fi

test/unit/prof_tctx.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,lg_prof_sample:0"
+fi

test/unit/prof_thread_name.c

@@ -1,9 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_PROF
-const char *malloc_conf = "prof:true,prof_active:false";
-#endif
-
 static void
 mallctl_thread_name_get_impl(const char *thread_name_expected, const char *func,
     int line)
@@ -12,8 +8,9 @@ mallctl_thread_name_get_impl(const char *thread_name_expected, const char *func,
 	size_t sz;
 
 	sz = sizeof(thread_name_old);
-	assert_d_eq(mallctl("thread.prof.name", &thread_name_old, &sz, NULL, 0),
-	    0, "%s():%d: Unexpected mallctl failure reading thread.prof.name",
+	assert_d_eq(mallctl("thread.prof.name", (void *)&thread_name_old, &sz,
+	    NULL, 0), 0,
+	    "%s():%d: Unexpected mallctl failure reading thread.prof.name",
 	    func, line);
 	assert_str_eq(thread_name_old, thread_name_expected,
 	    "%s():%d: Unexpected thread.prof.name value", func, line);
@@ -26,8 +23,8 @@ mallctl_thread_name_set_impl(const char *thread_name, const char *func,
     int line)
 {
 
-	assert_d_eq(mallctl("thread.prof.name", NULL, NULL, &thread_name,
-	    sizeof(thread_name)), 0,
+	assert_d_eq(mallctl("thread.prof.name", NULL, NULL,
+	    (void *)&thread_name, sizeof(thread_name)), 0,
 	    "%s():%d: Unexpected mallctl failure reading thread.prof.name",
 	    func, line);
 	mallctl_thread_name_get_impl(thread_name, func, line);
@@ -46,15 +43,15 @@ TEST_BEGIN(test_prof_thread_name_validation)
 
 	/* NULL input shouldn't be allowed. */
 	thread_name = NULL;
-	assert_d_eq(mallctl("thread.prof.name", NULL, NULL, &thread_name,
-	    sizeof(thread_name)), EFAULT,
+	assert_d_eq(mallctl("thread.prof.name", NULL, NULL,
+	    (void *)&thread_name, sizeof(thread_name)), EFAULT,
 	    "Unexpected mallctl result writing \"%s\" to thread.prof.name",
 	    thread_name);
 
 	/* '\n' shouldn't be allowed. */
 	thread_name = "hi\nthere";
-	assert_d_eq(mallctl("thread.prof.name", NULL, NULL, &thread_name,
-	    sizeof(thread_name)), EFAULT,
+	assert_d_eq(mallctl("thread.prof.name", NULL, NULL,
+	    (void *)&thread_name, sizeof(thread_name)), EFAULT,
 	    "Unexpected mallctl result writing \"%s\" to thread.prof.name",
 	    thread_name);
 
@@ -64,8 +61,9 @@ TEST_BEGIN(test_prof_thread_name_validation)
 		size_t sz;
 
 		sz = sizeof(thread_name_old);
-		assert_d_eq(mallctl("thread.prof.name", &thread_name_old, &sz,
-		    &thread_name, sizeof(thread_name)), EPERM,
+		assert_d_eq(mallctl("thread.prof.name",
+		    (void *)&thread_name_old, &sz, (void *)&thread_name,
+		    sizeof(thread_name)), EPERM,
 		    "Unexpected mallctl result writing \"%s\" to "
 		    "thread.prof.name", thread_name);
 	}

test/unit/prof_thread_name.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_prof}" = "x1" ] ; then
+  export MALLOC_CONF="prof:true,prof_active:false"
+fi

test/unit/quarantine.c

@@ -1,13 +1,7 @@
 #include "test/jemalloc_test.h"
 
+/* Keep in sync with definition in quarantine.sh. */
 #define	QUARANTINE_SIZE		8192
-#define	STRINGIFY_HELPER(x)	#x
-#define	STRINGIFY(x)		STRINGIFY_HELPER(x)
-
-#ifdef JEMALLOC_FILL
-const char *malloc_conf = "abort:false,junk:true,redzone:true,quarantine:"
-    STRINGIFY(QUARANTINE_SIZE);
-#endif
 
 void
 quarantine_clear(void)

test/unit/quarantine.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Keep in sync with definition in quarantine.c.
+export QUARANTINE_SIZE=8192
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="abort:false,junk:true,redzone:true,quarantine:${QUARANTINE_SIZE}"
+fi

test/unit/run_quantize.c

@@ -13,7 +13,7 @@ TEST_BEGIN(test_small_run_size)
 	 */
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.nbins", &nbins, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl("arenas.nbins", (void *)&nbins, &sz, NULL, 0), 0,
 	    "Unexpected mallctl failure");
 
 	assert_d_eq(mallctlnametomib("arenas.bin.0.run_size", mib, &miblen), 0,
@@ -21,8 +21,8 @@ TEST_BEGIN(test_small_run_size)
 	for (i = 0; i < nbins; i++) {
 		mib[2] = i;
 		sz = sizeof(size_t);
-		assert_d_eq(mallctlbymib(mib, miblen, &run_size, &sz, NULL, 0),
-		    0, "Unexpected mallctlbymib failure");
+		assert_d_eq(mallctlbymib(mib, miblen, (void *)&run_size, &sz,
+		    NULL, 0), 0, "Unexpected mallctlbymib failure");
 		assert_zu_eq(run_size, run_quantize_floor(run_size),
 		    "Small run quantization should be a no-op (run_size=%zu)",
 		    run_size);
@@ -47,11 +47,11 @@ TEST_BEGIN(test_large_run_size)
 	 */
 
 	sz = sizeof(bool);
-	assert_d_eq(mallctl("config.cache_oblivious", &cache_oblivious, &sz,
-	    NULL, 0), 0, "Unexpected mallctl failure");
+	assert_d_eq(mallctl("config.cache_oblivious", (void *)&cache_oblivious,
+	    &sz, NULL, 0), 0, "Unexpected mallctl failure");
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.nlruns", &nlruns, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl("arenas.nlruns", (void *)&nlruns, &sz, NULL, 0), 0,
 	    "Unexpected mallctl failure");
 
 	assert_d_eq(mallctlnametomib("arenas.lrun.0.size", mib, &miblen), 0,
@@ -61,8 +61,8 @@ TEST_BEGIN(test_large_run_size)
 
 		mib[2] = i;
 		sz = sizeof(size_t);
-		assert_d_eq(mallctlbymib(mib, miblen, &lrun_size, &sz, NULL, 0),
-		    0, "Unexpected mallctlbymib failure");
+		assert_d_eq(mallctlbymib(mib, miblen, (void *)&lrun_size, &sz,
+		    NULL, 0), 0, "Unexpected mallctlbymib failure");
 		run_size = cache_oblivious ? lrun_size + PAGE : lrun_size;
 		floor = run_quantize_floor(run_size);
 		ceil = run_quantize_ceil(run_size);
@@ -102,11 +102,11 @@ TEST_BEGIN(test_monotonic)
 	 */
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.nbins", &nbins, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl("arenas.nbins", (void *)&nbins, &sz, NULL, 0), 0,
 	    "Unexpected mallctl failure");
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.nlruns", &nlruns, &sz, NULL, 0), 0,
+	assert_d_eq(mallctl("arenas.nlruns", (void *)&nlruns, &sz, NULL, 0), 0,
 	    "Unexpected mallctl failure");
 
 	floor_prev = 0;

test/unit/size_classes.c

@@ -8,8 +8,8 @@ get_max_size_class(void)
 	size_t sz, miblen, max_size_class;
 
 	sz = sizeof(unsigned);
-	assert_d_eq(mallctl("arenas.nhchunks", &nhchunks, &sz, NULL, 0), 0,
-	    "Unexpected mallctl() error");
+	assert_d_eq(mallctl("arenas.nhchunks", (void *)&nhchunks, &sz, NULL, 0),
+	    0, "Unexpected mallctl() error");
 
 	miblen = sizeof(mib) / sizeof(size_t);
 	assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0,
@@ -17,8 +17,8 @@ get_max_size_class(void)
 	mib[2] = nhchunks - 1;
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctlbymib(mib, miblen, &max_size_class, &sz, NULL, 0), 0,
-	    "Unexpected mallctlbymib() error");
+	assert_d_eq(mallctlbymib(mib, miblen, (void *)&max_size_class, &sz,
+	    NULL, 0), 0, "Unexpected mallctlbymib() error");
 
 	return (max_size_class);
 }

test/unit/stats.c

@@ -7,18 +7,18 @@ TEST_BEGIN(test_stats_summary)
 	int expected = config_stats ? 0 : ENOENT;
 
 	sz = sizeof(cactive);
-	assert_d_eq(mallctl("stats.cactive", &cactive, &sz, NULL, 0), expected,
-	    "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.cactive", (void *)&cactive, &sz, NULL, 0),
+	    expected, "Unexpected mallctl() result");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.allocated", &allocated, &sz, NULL, 0),
+	assert_d_eq(mallctl("stats.allocated", (void *)&allocated, &sz, NULL,
+	    0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.active", (void *)&active, &sz, NULL, 0),
 	    expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.active", &active, &sz, NULL, 0), expected,
-	    "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.resident", &resident, &sz, NULL, 0),
+	assert_d_eq(mallctl("stats.resident", (void *)&resident, &sz, NULL, 0),
+	    expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.mapped", (void *)&mapped, &sz, NULL, 0),
 	    expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.mapped", &mapped, &sz, NULL, 0), expected,
-	    "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_zu_le(active, *cactive,
@@ -45,19 +45,19 @@ TEST_BEGIN(test_stats_huge)
 	p = mallocx(large_maxclass+1, 0);
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.huge.allocated", (void *)&allocated,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz, NULL,
-	    0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz, NULL,
-	    0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.huge.nrequests", &nrequests, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", (void *)&nmalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", (void *)&ndalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.huge.nrequests", (void *)&nrequests,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_zu_gt(allocated, 0,
@@ -83,8 +83,8 @@ TEST_BEGIN(test_stats_arenas_summary)
 	uint64_t npurge, nmadvise, purged;
 
 	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
+	    sizeof(arena)), 0, "Unexpected mallctl() failure");
 
 	little = mallocx(SMALL_MAXCLASS, 0);
 	assert_ptr_not_null(little, "Unexpected mallocx() failure");
@@ -100,19 +100,19 @@ TEST_BEGIN(test_stats_arenas_summary)
 	assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0,
 	    "Unexpected mallctl() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.mapped", &mapped, &sz, NULL, 0),
-	    expected, "Unexepected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.mapped", (void *)&mapped, &sz, NULL,
+	    0), expected, "Unexepected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge, &sz, NULL, 0),
-	    expected, "Unexepected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.nmadvise", &nmadvise, &sz, NULL, 0),
-	    expected, "Unexepected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.purged", &purged, &sz, NULL, 0),
-	    expected, "Unexepected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.npurge", (void *)&npurge, &sz, NULL,
+	    0), expected, "Unexepected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.nmadvise", (void *)&nmadvise, &sz,
+	    NULL, 0), expected, "Unexepected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.purged", (void *)&purged, &sz, NULL,
+	    0), expected, "Unexepected mallctl() result");
 
 	if (config_stats) {
 		assert_u64_gt(npurge, 0,
@@ -150,8 +150,8 @@ TEST_BEGIN(test_stats_arenas_small)
 	no_lazy_lock(); /* Lazy locking would dodge tcache testing. */
 
 	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
+	    sizeof(arena)), 0, "Unexpected mallctl() failure");
 
 	p = mallocx(SMALL_MAXCLASS, 0);
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
@@ -159,19 +159,21 @@ TEST_BEGIN(test_stats_arenas_small)
 	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
 	    config_tcache ? 0 : ENOENT, "Unexpected mallctl() result");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.small.allocated", &allocated, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.small.allocated",
+	    (void *)&allocated, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.small.nmalloc", &nmalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.small.ndalloc", &ndalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.small.nrequests", &nrequests, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.small.nmalloc", (void *)&nmalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.small.ndalloc", (void *)&ndalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.small.nrequests",
+	    (void *)&nrequests, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_zu_gt(allocated, 0,
@@ -197,25 +199,27 @@ TEST_BEGIN(test_stats_arenas_large)
 	int expected = config_stats ? 0 : ENOENT;
 
 	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
+	    sizeof(arena)), 0, "Unexpected mallctl() failure");
 
 	p = mallocx(large_maxclass, 0);
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.large.allocated", &allocated, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.large.allocated",
+	    (void *)&allocated, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", &nmalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", &ndalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.large.nrequests", &nrequests, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", (void *)&nmalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", (void *)&ndalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.large.nrequests",
+	    (void *)&nrequests, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_zu_gt(allocated, 0,
@@ -241,23 +245,23 @@ TEST_BEGIN(test_stats_arenas_huge)
 	int expected = config_stats ? 0 : ENOENT;
 
 	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
+	    sizeof(arena)), 0, "Unexpected mallctl() failure");
 
 	p = mallocx(chunksize, 0);
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.huge.allocated", (void *)&allocated,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", (void *)&nmalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", (void *)&ndalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_zu_gt(allocated, 0,
@@ -282,8 +286,8 @@ TEST_BEGIN(test_stats_arenas_bins)
 	int expected = config_stats ? 0 : ENOENT;
 
 	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
+	    sizeof(arena)), 0, "Unexpected mallctl() failure");
 
 	p = mallocx(arena_bin_info[0].reg_size, 0);
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
@@ -291,35 +295,36 @@ TEST_BEGIN(test_stats_arenas_bins)
 	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
 	    config_tcache ? 0 : ENOENT, "Unexpected mallctl() result");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nmalloc", &nmalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.ndalloc", &ndalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nrequests", &nrequests, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.nmalloc", (void *)&nmalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.ndalloc", (void *)&ndalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.nrequests",
+	    (void *)&nrequests, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.curregs", &curregs, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.curregs", (void *)&curregs,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nfills", &nfills, &sz,
-	    NULL, 0), config_tcache ? expected : ENOENT,
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.nfills", (void *)&nfills,
+	    &sz, NULL, 0), config_tcache ? expected : ENOENT,
 	    "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nflushes", &nflushes, &sz,
-	    NULL, 0), config_tcache ? expected : ENOENT,
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.nflushes", (void *)&nflushes,
+	    &sz, NULL, 0), config_tcache ? expected : ENOENT,
 	    "Unexpected mallctl() result");
 
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nruns", &nruns, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nreruns", &nreruns, &sz,
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.nruns", (void *)&nruns, &sz,
 	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.nreruns", (void *)&nreruns,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.curruns", &curruns, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.bins.0.curruns", (void *)&curruns,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_u64_gt(nmalloc, 0,
@@ -355,25 +360,26 @@ TEST_BEGIN(test_stats_arenas_lruns)
 	int expected = config_stats ? 0 : ENOENT;
 
 	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
+	    sizeof(arena)), 0, "Unexpected mallctl() failure");
 
 	p = mallocx(LARGE_MINCLASS, 0);
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.lruns.0.nmalloc", &nmalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.lruns.0.ndalloc", &ndalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.lruns.0.nrequests", &nrequests, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.lruns.0.nmalloc", (void *)&nmalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.lruns.0.ndalloc", (void *)&ndalloc,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.lruns.0.nrequests",
+	    (void *)&nrequests, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.lruns.0.curruns", &curruns, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.lruns.0.curruns", (void *)&curruns,
+	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_u64_gt(nmalloc, 0,
@@ -399,23 +405,26 @@ TEST_BEGIN(test_stats_arenas_hchunks)
 	int expected = config_stats ? 0 : ENOENT;
 
 	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)),
-	    0, "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
+	    sizeof(arena)), 0, "Unexpected mallctl() failure");
 
 	p = mallocx(chunksize, 0);
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
-	assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
-	    "Unexpected mallctl() failure");
+	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
+	    0, "Unexpected mallctl() failure");
 
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.hchunks.0.nmalloc", &nmalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.hchunks.0.ndalloc", &ndalloc, &sz,
-	    NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.hchunks.0.nmalloc",
+	    (void *)&nmalloc, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.hchunks.0.ndalloc",
+	    (void *)&ndalloc, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.hchunks.0.curhchunks", &curhchunks,
-	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	assert_d_eq(mallctl("stats.arenas.0.hchunks.0.curhchunks",
+	    (void *)&curhchunks, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_u64_gt(nmalloc, 0,

test/unit/tsd.c

@@ -79,7 +79,7 @@ thd_start(void *arg)
 TEST_BEGIN(test_tsd_main_thread)
 {
 
-	thd_start((void *) 0xa5f3e329);
+	thd_start((void *)(uintptr_t)0xa5f3e329);
 }
 TEST_END
 

test/unit/util.c

@@ -75,6 +75,7 @@ TEST_BEGIN(test_malloc_strtoumax)
 	};
 #define	ERR(e)		e, #e
 #define	KUMAX(x)	((uintmax_t)x##ULL)
+#define	KSMAX(x)	((uintmax_t)(intmax_t)x##LL)
 	struct test_s tests[] = {
 		{"0",		"0",	-1,	ERR(EINVAL),	UINTMAX_MAX},
 		{"0",		"0",	1,	ERR(EINVAL),	UINTMAX_MAX},
@@ -87,13 +88,13 @@ TEST_BEGIN(test_malloc_strtoumax)
 
 		{"42",		"",	0,	ERR(0),		KUMAX(42)},
 		{"+42",		"",	0,	ERR(0),		KUMAX(42)},
-		{"-42",		"",	0,	ERR(0),		KUMAX(-42)},
+		{"-42",		"",	0,	ERR(0),		KSMAX(-42)},
 		{"042",		"",	0,	ERR(0),		KUMAX(042)},
 		{"+042",	"",	0,	ERR(0),		KUMAX(042)},
-		{"-042",	"",	0,	ERR(0),		KUMAX(-042)},
+		{"-042",	"",	0,	ERR(0),		KSMAX(-042)},
 		{"0x42",	"",	0,	ERR(0),		KUMAX(0x42)},
 		{"+0x42",	"",	0,	ERR(0),		KUMAX(0x42)},
-		{"-0x42",	"",	0,	ERR(0),		KUMAX(-0x42)},
+		{"-0x42",	"",	0,	ERR(0),		KSMAX(-0x42)},
 
 		{"0",		"",	0,	ERR(0),		KUMAX(0)},
 		{"1",		"",	0,	ERR(0),		KUMAX(1)},
@@ -130,6 +131,7 @@ TEST_BEGIN(test_malloc_strtoumax)
 	};
 #undef ERR
 #undef KUMAX
+#undef KSMAX
 	unsigned i;
 
 	for (i = 0; i < sizeof(tests)/sizeof(struct test_s); i++) {

test/unit/witness.c

@@ -3,12 +3,12 @@
 static witness_lock_error_t *witness_lock_error_orig;
 static witness_owner_error_t *witness_owner_error_orig;
 static witness_not_owner_error_t *witness_not_owner_error_orig;
-static witness_lockless_error_t *witness_lockless_error_orig;
+static witness_depth_error_t *witness_depth_error_orig;
 
 static bool saw_lock_error;
 static bool saw_owner_error;
 static bool saw_not_owner_error;
-static bool saw_lockless_error;
+static bool saw_depth_error;
 
 static void
 witness_lock_error_intercept(const witness_list_t *witnesses,
@@ -33,10 +33,9 @@ witness_not_owner_error_intercept(const witness_t *witness)
 }
 
 static void
-witness_lockless_error_intercept(const witness_list_t *witnesses)
-{
-
-	saw_lockless_error = true;
+witness_depth_error_intercept(const witness_list_t *witnesses,
+    witness_rank_t rank_inclusive, unsigned depth) {
+	saw_depth_error = true;
 }
 
 static int
@@ -67,21 +66,36 @@ TEST_BEGIN(test_witness)
 	tsdn = tsdn_fetch();
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0);
 
 	witness_init(&a, "a", 1, NULL);
 	witness_assert_not_owner(tsdn, &a);
 	witness_lock(tsdn, &a);
 	witness_assert_owner(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 0);
 
 	witness_init(&b, "b", 2, NULL);
 	witness_assert_not_owner(tsdn, &b);
 	witness_lock(tsdn, &b);
 	witness_assert_owner(tsdn, &b);
+	witness_assert_depth(tsdn, 2);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 2);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0);
 
 	witness_unlock(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0);
 	witness_unlock(tsdn, &b);
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0);
 }
 TEST_END
 
@@ -100,12 +114,15 @@ TEST_BEGIN(test_witness_comp)
 	witness_assert_not_owner(tsdn, &a);
 	witness_lock(tsdn, &a);
 	witness_assert_owner(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
 
 	witness_init(&b, "b", 1, witness_comp);
 	witness_assert_not_owner(tsdn, &b);
 	witness_lock(tsdn, &b);
 	witness_assert_owner(tsdn, &b);
+	witness_assert_depth(tsdn, 2);
 	witness_unlock(tsdn, &b);
+	witness_assert_depth(tsdn, 1);
 
 	witness_lock_error_orig = witness_lock_error;
 	witness_lock_error = witness_lock_error_intercept;
@@ -117,6 +134,7 @@ TEST_BEGIN(test_witness_comp)
 	witness_lock(tsdn, &c);
 	assert_true(saw_lock_error, "Expected witness lock error");
 	witness_unlock(tsdn, &c);
+	witness_assert_depth(tsdn, 1);
 
 	saw_lock_error = false;
 
@@ -126,6 +144,7 @@ TEST_BEGIN(test_witness_comp)
 	witness_lock(tsdn, &d);
 	assert_true(saw_lock_error, "Expected witness lock error");
 	witness_unlock(tsdn, &d);
+	witness_assert_depth(tsdn, 1);
 
 	witness_unlock(tsdn, &a);
 
@@ -154,11 +173,13 @@ TEST_BEGIN(test_witness_reversal)
 	witness_init(&b, "b", 2, NULL);
 
 	witness_lock(tsdn, &b);
+	witness_assert_depth(tsdn, 1);
 	assert_false(saw_lock_error, "Unexpected witness lock error");
 	witness_lock(tsdn, &a);
 	assert_true(saw_lock_error, "Expected witness lock error");
 
 	witness_unlock(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
 	witness_unlock(tsdn, &b);
 
 	witness_assert_lockless(tsdn);
@@ -232,35 +253,38 @@ TEST_BEGIN(test_witness_unlock_not_owned)
 }
 TEST_END
 
-TEST_BEGIN(test_witness_lockful)
-{
+TEST_BEGIN(test_witness_depth) {
 	witness_t a;
 	tsdn_t *tsdn;
 
 	test_skip_if(!config_debug);
 
-	witness_lockless_error_orig = witness_lockless_error;
-	witness_lockless_error = witness_lockless_error_intercept;
-	saw_lockless_error = false;
+	witness_depth_error_orig = witness_depth_error;
+	witness_depth_error = witness_depth_error_intercept;
+	saw_depth_error = false;
 
 	tsdn = tsdn_fetch();
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
 
 	witness_init(&a, "a", 1, NULL);
 
-	assert_false(saw_lockless_error, "Unexpected lockless error");
+	assert_false(saw_depth_error, "Unexpected depth error");
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
 
 	witness_lock(tsdn, &a);
 	witness_assert_lockless(tsdn);
-	assert_true(saw_lockless_error, "Expected lockless error");
+	witness_assert_depth(tsdn, 0);
+	assert_true(saw_depth_error, "Expected depth error");
 
 	witness_unlock(tsdn, &a);
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
 
-	witness_lockless_error = witness_lockless_error_orig;
+	witness_depth_error = witness_depth_error_orig;
 }
 TEST_END
 
@@ -268,11 +292,11 @@ int
 main(void)
 {
 
-	return (test(
+	return test(
 	    test_witness,
 	    test_witness_comp,
 	    test_witness_reversal,
 	    test_witness_recursive,
 	    test_witness_unlock_not_owned,
-	    test_witness_lockful));
+	    test_witness_depth);
 }

test/unit/zero.c

@@ -1,10 +1,5 @@
 #include "test/jemalloc_test.h"
 
-#ifdef JEMALLOC_FILL
-const char *malloc_conf =
-    "abort:false,junk:false,zero:true,redzone:false,quarantine:0";
-#endif
-
 static void
 test_zero(size_t sz_min, size_t sz_max)
 {

test/unit/zero.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+if [ "x${enable_fill}" = "x1" ] ; then
+  export MALLOC_CONF="abort:false,junk:false,zero:true,redzone:false,quarantine:0"
+fi