Merge branch 'dev'

Modify xallocx() tests that expect to expand in place to use a separate
arena.  This avoids the potential for interposed internal allocations
from e.g. heap profile sampling to disrupt the tests.

This resolves #286.

ChangeLog

@@ -4,6 +4,45 @@ brevity.  Much more detail can be found in the git revision history:
 
     https://github.com/jemalloc/jemalloc
 
+* 4.0.4 (October 24, 2015)
+
+  This bugfix release fixes another xallocx() regression.  No other regressions
+  have come to light in over a month, so this is likely a good starting point
+  for people who prefer to wait for "dot one" releases with all the major issues
+  shaken out.
+
+  Bug fixes:
+  - Fix xallocx(..., MALLOCX_ZERO to zero the last full trailing page of large
+    allocations that have been randomly assigned an offset of 0 when
+    --enable-cache-oblivious configure option is enabled.
+
+* 4.0.3 (September 24, 2015)
+
+  This bugfix release continues the trend of xallocx() and heap profiling fixes.
+
+  Bug fixes:
+  - Fix xallocx(..., MALLOCX_ZERO) to zero all trailing bytes of large
+    allocations when --enable-cache-oblivious configure option is enabled.
+  - Fix xallocx(..., MALLOCX_ZERO) to zero trailing bytes of huge allocations
+    when resizing from/to a size class that is not a multiple of the chunk size.
+  - Fix prof_tctx_dump_iter() to filter out nodes that were created after heap
+    profile dumping started.
+  - Work around a potentially bad thread-specific data initialization
+    interaction with NPTL (glibc's pthreads implementation).
+
+* 4.0.2 (September 21, 2015)
+
+  This bugfix release addresses a few bugs specific to heap profiling.
+
+  Bug fixes:
+  - Fix ixallocx_prof_sample() to never modify nor create sampled small
+    allocations.  xallocx() is in general incapable of moving small allocations,
+    so this fix removes buggy code without loss of generality.
+  - Fix irallocx_prof_sample() to always allocate large regions, even when
+    alignment is non-zero.
+  - Fix prof_alloc_rollback() to read tdata from thread-specific data rather
+    than dereferencing a potentially invalid tctx.
+
 * 4.0.1 (September 15, 2015)
 
   This is a bugfix release that is somewhat high risk due to the amount of

Makefile.in

@@ -353,6 +353,7 @@ check_unit: tests_unit check_unit_dir
 check_integration_prof: tests_integration check_integration_dir
 ifeq ($(enable_prof), 1)
 	$(MALLOC_CONF)="prof:true" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
+	$(MALLOC_CONF)="prof:true,prof_active:false" $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)
 endif
 check_integration: tests_integration check_integration_dir
 	$(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%)

doc/jemalloc.xml.in

@@ -1418,8 +1418,8 @@ malloc_conf = "xmalloc:true";]]></programlisting>
         can cause asynchronous string deallocation.  Furthermore, each
         invocation of this interface can only read or write; simultaneous
         read/write is not supported due to string lifetime limitations.  The
-        name string must nil-terminated and comprised only of characters in the
+        name string must be nil-terminated and comprised only of characters in
-        sets recognized
+        the sets recognized
         by <citerefentry><refentrytitle>isgraph</refentrytitle>
         <manvolnum>3</manvolnum></citerefentry> and
         <citerefentry><refentrytitle>isblank</refentrytitle>

include/jemalloc/internal/jemalloc_internal.h.in

@@ -317,6 +317,10 @@ typedef unsigned szind_t;
 #define	PAGE		((size_t)(1U << LG_PAGE))
 #define	PAGE_MASK	((size_t)(PAGE - 1))
 
+/* Return the page base address for the page containing address a. */
+#define	PAGE_ADDR2BASE(a)						\
+	((void *)((uintptr_t)(a) & ~PAGE_MASK))
+
 /* Return the smallest pagesize multiple that is >= s. */
 #define	PAGE_CEILING(s)							\
 	(((s) + PAGE_MASK) & ~PAGE_MASK)

include/jemalloc/internal/tsd.h

@@ -190,7 +190,7 @@ a_name##tsd_boot0(void)							\
 	return (false);							\
 }									\
 a_attr void								\
-a_name##tsd_boot1()							\
+a_name##tsd_boot1(void)							\
 {									\
 									\
 	/* Do nothing. */						\
@@ -235,7 +235,7 @@ a_name##tsd_boot0(void)							\
 	return (false);							\
 }									\
 a_attr void								\
-a_name##tsd_boot1()							\
+a_name##tsd_boot1(void)							\
 {									\
 									\
 	/* Do nothing. */						\
@@ -345,7 +345,7 @@ a_name##tsd_boot0(void)							\
 	return (false);							\
 }									\
 a_attr void								\
-a_name##tsd_boot1()							\
+a_name##tsd_boot1(void)							\
 {									\
 	a_name##tsd_wrapper_t *wrapper;					\
 	wrapper = (a_name##tsd_wrapper_t *)				\
@@ -467,7 +467,7 @@ a_name##tsd_boot0(void)							\
 	return (false);							\
 }									\
 a_attr void								\
-a_name##tsd_boot1()							\
+a_name##tsd_boot1(void)							\
 {									\
 	a_name##tsd_wrapper_t *wrapper;					\
 	wrapper = (a_name##tsd_wrapper_t *)				\

src/arena.c

@@ -2560,7 +2560,7 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large =
     JEMALLOC_N(arena_dalloc_junk_large_impl);
 #endif
 
-void
+static void
 arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk,
     void *ptr, bool junked)
 {
@@ -2679,6 +2679,22 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 		if (arena_run_split_large(arena, run, splitsize, zero))
 			goto label_fail;
 
+		if (config_cache_oblivious && zero) {
+			/*
+			 * Zero the trailing bytes of the original allocation's
+			 * last page, since they are in an indeterminate state.
+			 * There will always be trailing bytes, because ptr's
+			 * offset from the beginning of the run is a multiple of
+			 * CACHELINE in [0 .. PAGE).
+			 */
+			void *zbase = (void *)((uintptr_t)ptr + oldsize);
+			void *zpast = PAGE_ADDR2BASE((void *)((uintptr_t)zbase +
+			    PAGE));
+			size_t nzero = (uintptr_t)zpast - (uintptr_t)zbase;
+			assert(nzero > 0);
+			memset(zbase, 0, nzero);
+		}
+
 		size = oldsize + splitsize;
 		npages = (size + large_pad) >> LG_PAGE;
 

src/huge.c

@@ -133,7 +133,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
 	extent_node_t *node;
 	arena_t *arena;
 	chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
-	bool zeroed;
+	bool pre_zeroed, post_zeroed;
 
 	/* Increase usize to incorporate extra. */
 	for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
@@ -145,26 +145,27 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
 
 	node = huge_node_get(ptr);
 	arena = extent_node_arena_get(node);
+	pre_zeroed = extent_node_zeroed_get(node);
 
 	/* Fill if necessary (shrinking). */
 	if (oldsize > usize) {
 		size_t sdiff = oldsize - usize;
 		if (config_fill && unlikely(opt_junk_free)) {
 			memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff);
-			zeroed = false;
+			post_zeroed = false;
 		} else {
-			zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr,
+			post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
-			    CHUNK_CEILING(oldsize), usize, sdiff);
+			    ptr, CHUNK_CEILING(oldsize), usize, sdiff);
 		}
 	} else
-		zeroed = true;
+		post_zeroed = pre_zeroed;
 
 	malloc_mutex_lock(&arena->huge_mtx);
 	/* Update the size of the huge allocation. */
 	assert(extent_node_size_get(node) != usize);
 	extent_node_size_set(node, usize);
-	/* Clear node's zeroed field if zeroing failed above. */
+	/* Update zeroed. */
-	extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed);
+	extent_node_zeroed_set(node, post_zeroed);
 	malloc_mutex_unlock(&arena->huge_mtx);
 
 	arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize);
@@ -172,7 +173,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
 	/* Fill if necessary (growing). */
 	if (oldsize < usize) {
 		if (zero || (config_fill && unlikely(opt_zero))) {
-			if (!zeroed) {
+			if (!pre_zeroed) {
 				memset((void *)((uintptr_t)ptr + oldsize), 0,
 				    usize - oldsize);
 			}
@@ -190,10 +191,11 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
 	arena_t *arena;
 	chunk_hooks_t chunk_hooks;
 	size_t cdiff;
-	bool zeroed;
+	bool pre_zeroed, post_zeroed;
 
 	node = huge_node_get(ptr);
 	arena = extent_node_arena_get(node);
+	pre_zeroed = extent_node_zeroed_get(node);
 	chunk_hooks = chunk_hooks_get(arena);
 
 	assert(oldsize > usize);
@@ -209,21 +211,21 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
 		if (config_fill && unlikely(opt_junk_free)) {
 			huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
 			    sdiff);
-			zeroed = false;
+			post_zeroed = false;
 		} else {
-			zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
+			post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
 			    CHUNK_ADDR2BASE((uintptr_t)ptr + usize),
 			    CHUNK_CEILING(oldsize),
 			    CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
 		}
 	} else
-		zeroed = true;
+		post_zeroed = pre_zeroed;
 
 	malloc_mutex_lock(&arena->huge_mtx);
 	/* Update the size of the huge allocation. */
 	extent_node_size_set(node, usize);
-	/* Clear node's zeroed field if zeroing failed above. */
+	/* Update zeroed. */
-	extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed);
+	extent_node_zeroed_set(node, post_zeroed);
 	malloc_mutex_unlock(&arena->huge_mtx);
 
 	/* Zap the excess chunks. */

src/jemalloc.c

@@ -1923,6 +1923,7 @@ imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize,
 		*alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags);
 		*usize = sa2u(size, *alignment);
 	}
+	assert(*usize != 0);
 	*zero = MALLOCX_ZERO_GET(flags);
 	if ((flags & MALLOCX_TCACHE_MASK) != 0) {
 		if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
@@ -1965,41 +1966,29 @@ imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, arena_t *arena)
 {
 
-	if (alignment != 0)
+	if (unlikely(alignment != 0))
 		return (ipalloct(tsd, usize, alignment, zero, tcache, arena));
-	if (zero)
+	if (unlikely(zero))
 		return (icalloct(tsd, usize, tcache, arena));
 	return (imalloct(tsd, usize, tcache, arena));
 }
 
-JEMALLOC_ALWAYS_INLINE_C void *
-imallocx_maybe_flags(tsd_t *tsd, size_t size, int flags, size_t usize,
-    size_t alignment, bool zero, tcache_t *tcache, arena_t *arena)
-{
-
-	if (likely(flags == 0))
-		return (imalloc(tsd, size));
-	return (imallocx_flags(tsd, usize, alignment, zero, tcache, arena));
-}
-
 static void *
-imallocx_prof_sample(tsd_t *tsd, size_t size, int flags, size_t usize,
+imallocx_prof_sample(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
-    size_t alignment, bool zero, tcache_t *tcache, arena_t *arena)
+    tcache_t *tcache, arena_t *arena)
 {
 	void *p;
 
 	if (usize <= SMALL_MAXCLASS) {
 		assert(((alignment == 0) ? s2u(LARGE_MINCLASS) :
 		    sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS);
-		p = imallocx_maybe_flags(tsd, LARGE_MINCLASS, flags,
+		p = imallocx_flags(tsd, LARGE_MINCLASS, alignment, zero, tcache,
-		    LARGE_MINCLASS, alignment, zero, tcache, arena);
+		    arena);
 		if (p == NULL)
 			return (NULL);
 		arena_prof_promoted(p, usize);
-	} else {
+	} else
-		p = imallocx_maybe_flags(tsd, size, flags, usize, alignment,
+		p = imallocx_flags(tsd, usize, alignment, zero, tcache, arena);
-		    zero, tcache, arena);
-	}
 
 	return (p);
 }
@@ -2018,12 +2007,11 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)
 	    &zero, &tcache, &arena)))
 		return (NULL);
 	tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true);
-	if (likely((uintptr_t)tctx == (uintptr_t)1U)) {
+	if (likely((uintptr_t)tctx == (uintptr_t)1U))
-		p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment,
+		p = imallocx_flags(tsd, *usize, alignment, zero, tcache, arena);
-		    zero, tcache, arena);
+	else if ((uintptr_t)tctx > (uintptr_t)1U) {
-	} else if ((uintptr_t)tctx > (uintptr_t)1U) {
+		p = imallocx_prof_sample(tsd, *usize, alignment, zero, tcache,
-		p = imallocx_prof_sample(tsd, size, flags, *usize, alignment,
+		    arena);
-		    zero, tcache, arena);
 	} else
 		p = NULL;
 	if (unlikely(p == NULL)) {
@@ -2098,8 +2086,8 @@ label_oom:
 }
 
 static void *
-irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
+irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize,
-    size_t alignment, size_t usize, bool zero, tcache_t *tcache, arena_t *arena,
+    size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
     prof_tctx_t *tctx)
 {
 	void *p;
@@ -2113,7 +2101,7 @@ irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
 			return (NULL);
 		arena_prof_promoted(p, usize);
 	} else {
-		p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero,
+		p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,
 		    tcache, arena);
 	}
 
@@ -2133,8 +2121,8 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
 	old_tctx = prof_tctx_get(old_ptr);
 	tctx = prof_alloc_prep(tsd, *usize, prof_active, true);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
-		p = irallocx_prof_sample(tsd, old_ptr, old_usize, size,
+		p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,
-		    alignment, *usize, zero, tcache, arena, tctx);
+		    alignment, zero, tcache, arena, tctx);
 	} else {
 		p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero,
 		    tcache, arena);
@@ -2251,26 +2239,13 @@ ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra,
 
 static size_t
 ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
-    size_t alignment, size_t usize_max, bool zero, prof_tctx_t *tctx)
+    size_t alignment, bool zero, prof_tctx_t *tctx)
 {
 	size_t usize;
 
 	if (tctx == NULL)
 		return (old_usize);
-	/* Use minimum usize to determine whether promotion may happen. */
+	usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero);
-	if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <=
-	    SMALL_MAXCLASS) {
-		if (ixalloc(ptr, old_usize, SMALL_MAXCLASS+1,
-		    (SMALL_MAXCLASS+1 >= size+extra) ? 0 : size+extra -
-		    (SMALL_MAXCLASS+1), alignment, zero))
-			return (old_usize);
-		usize = isalloc(ptr, config_prof);
-		if (usize_max < LARGE_MINCLASS)
-			arena_prof_promoted(ptr, usize);
-	} else {
-		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
-		    zero);
-	}
 
 	return (usize);
 }
@@ -2293,15 +2268,16 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
 	 */
 	usize_max = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra,
 	    alignment);
+	assert(usize_max != 0);
 	tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
 	if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
 		usize = ixallocx_prof_sample(ptr, old_usize, size, extra,
-		    alignment, usize_max, zero, tctx);
+		    alignment, zero, tctx);
 	} else {
 		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
 		    zero);
 	}
-	if (unlikely(usize == old_usize)) {
+	if (usize == old_usize) {
 		prof_alloc_rollback(tsd, tctx, false);
 		return (usize);
 	}

src/prof.c

@@ -209,7 +209,7 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
 		 */
 		tdata = prof_tdata_get(tsd, true);
 		if (tdata != NULL)
-			prof_sample_threshold_update(tctx->tdata);
+			prof_sample_threshold_update(tdata);
 	}
 
 	if ((uintptr_t)tctx > (uintptr_t)1U) {
@@ -1102,11 +1102,23 @@ prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
 {
 	bool propagate_err = *(bool *)arg;
 
+	switch (tctx->state) {
+	case prof_tctx_state_initializing:
+	case prof_tctx_state_nominal:
+		/* Not captured by this dump. */
+		break;
+	case prof_tctx_state_dumping:
+	case prof_tctx_state_purgatory:
 		if (prof_dump_printf(propagate_err,
-	    "  t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": %"FMTu64"]\n",
+		    "  t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
-	    tctx->thr_uid, tctx->dump_cnts.curobjs, tctx->dump_cnts.curbytes,
+		    "%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
-	    tctx->dump_cnts.accumobjs, tctx->dump_cnts.accumbytes))
+		    tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
+		    tctx->dump_cnts.accumbytes))
 			return (tctx);
+		break;
+	default:
+		not_reached();
+	}
 	return (NULL);
 }
 

src/tsd.c

@@ -73,6 +73,9 @@ tsd_cleanup(void *arg)
 	tsd_t *tsd = (tsd_t *)arg;
 
 	switch (tsd->state) {
+	case tsd_state_uninitialized:
+		/* Do nothing. */
+		break;
 	case tsd_state_nominal:
 #define O(n, t)								\
 		n##_cleanup(tsd);

test/integration/mallocx.c

@@ -1,5 +1,85 @@
 #include "test/jemalloc_test.h"
 
+static unsigned
+get_nsizes_impl(const char *cmd)
+{
+	unsigned ret;
+	size_t z;
+
+	z = sizeof(unsigned);
+	assert_d_eq(mallctl(cmd, &ret, &z, NULL, 0), 0,
+	    "Unexpected mallctl(\"%s\", ...) failure", cmd);
+
+	return (ret);
+}
+
+static unsigned
+get_nhuge(void)
+{
+
+	return (get_nsizes_impl("arenas.nhchunks"));
+}
+
+static size_t
+get_size_impl(const char *cmd, size_t ind)
+{
+	size_t ret;
+	size_t z;
+	size_t mib[4];
+	size_t miblen = 4;
+
+	z = sizeof(size_t);
+	assert_d_eq(mallctlnametomib(cmd, mib, &miblen),
+	    0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd);
+	mib[2] = ind;
+	z = sizeof(size_t);
+	assert_d_eq(mallctlbymib(mib, miblen, &ret, &z, NULL, 0),
+	    0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind);
+
+	return (ret);
+}
+
+static size_t
+get_huge_size(size_t ind)
+{
+
+	return (get_size_impl("arenas.hchunk.0.size", ind));
+}
+
+TEST_BEGIN(test_oom)
+{
+	size_t hugemax, size, alignment;
+
+	hugemax = get_huge_size(get_nhuge()-1);
+
+	/*
+	 * It should be impossible to allocate two objects that each consume
+	 * more than half the virtual address space.
+	 */
+	{
+		void *p;
+
+		p = mallocx(hugemax, 0);
+		if (p != NULL) {
+			assert_ptr_null(mallocx(hugemax, 0),
+			    "Expected OOM for mallocx(size=%#zx, 0)", hugemax);
+			dallocx(p, 0);
+		}
+	}
+
+#if LG_SIZEOF_PTR == 3
+	size      = ZU(0x8000000000000000);
+	alignment = ZU(0x8000000000000000);
+#else
+	size      = ZU(0x80000000);
+	alignment = ZU(0x80000000);
+#endif
+	assert_ptr_null(mallocx(size, MALLOCX_ALIGN(alignment)),
+	    "Expected OOM for mallocx(size=%#zx, MALLOCX_ALIGN(%#zx)", size,
+	    alignment);
+}
+TEST_END
+
 TEST_BEGIN(test_basic)
 {
 #define	MAXSZ (((size_t)1) << 26)
@@ -96,6 +176,7 @@ main(void)
 {
 
 	return (test(
+	    test_oom,
 	    test_basic,
 	    test_alignment_and_size));
 }

test/integration/xallocx.c

@@ -1,5 +1,24 @@
 #include "test/jemalloc_test.h"
 
+/*
+ * Use a separate arena for xallocx() extension/contraction tests so that
+ * internal allocation e.g. by heap profiling can't interpose allocations where
+ * xallocx() would ordinarily be able to extend.
+ */
+static unsigned
+arena_ind(void)
+{
+	static unsigned ind = 0;
+
+	if (ind == 0) {
+		size_t sz = sizeof(ind);
+		assert_d_eq(mallctl("arenas.extend", &ind, &sz, NULL, 0), 0,
+		    "Unexpected mallctl failure creating arena");
+	}
+
+	return (ind);
+}
+
 TEST_BEGIN(test_same_size)
 {
 	void *p;
@@ -218,6 +237,7 @@ TEST_END
 
 TEST_BEGIN(test_extra_large)
 {
+	int flags = MALLOCX_ARENA(arena_ind());
 	size_t smallmax, large0, large1, large2, huge0, hugemax;
 	void *p;
 
@@ -229,61 +249,62 @@ TEST_BEGIN(test_extra_large)
 	huge0 = get_huge_size(0);
 	hugemax = get_huge_size(get_nhuge()-1);
 
-	p = mallocx(large2, 0);
+	p = mallocx(large2, flags);
 	assert_ptr_not_null(p, "Unexpected mallocx() error");
 
-	assert_zu_eq(xallocx(p, large2, 0, 0), large2,
+	assert_zu_eq(xallocx(p, large2, 0, flags), large2,
 	    "Unexpected xallocx() behavior");
 	/* Test size decrease with zero extra. */
-	assert_zu_eq(xallocx(p, large0, 0, 0), large0,
+	assert_zu_eq(xallocx(p, large0, 0, flags), large0,
 	    "Unexpected xallocx() behavior");
-	assert_zu_eq(xallocx(p, smallmax, 0, 0), large0,
+	assert_zu_eq(xallocx(p, smallmax, 0, flags), large0,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_eq(xallocx(p, large2, 0, 0), large2,
+	assert_zu_eq(xallocx(p, large2, 0, flags), large2,
 	    "Unexpected xallocx() behavior");
 	/* Test size decrease with non-zero extra. */
-	assert_zu_eq(xallocx(p, large0, large2 - large0, 0), large2,
+	assert_zu_eq(xallocx(p, large0, large2 - large0, flags), large2,
 	    "Unexpected xallocx() behavior");
-	assert_zu_eq(xallocx(p, large1, large2 - large1, 0), large2,
+	assert_zu_eq(xallocx(p, large1, large2 - large1, flags), large2,
 	    "Unexpected xallocx() behavior");
-	assert_zu_eq(xallocx(p, large0, large1 - large0, 0), large1,
+	assert_zu_eq(xallocx(p, large0, large1 - large0, flags), large1,
 	    "Unexpected xallocx() behavior");
-	assert_zu_eq(xallocx(p, smallmax, large0 - smallmax, 0), large0,
+	assert_zu_eq(xallocx(p, smallmax, large0 - smallmax, flags), large0,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_eq(xallocx(p, large0, 0, 0), large0,
+	assert_zu_eq(xallocx(p, large0, 0, flags), large0,
 	    "Unexpected xallocx() behavior");
 	/* Test size increase with zero extra. */
-	assert_zu_eq(xallocx(p, large2, 0, 0), large2,
+	assert_zu_eq(xallocx(p, large2, 0, flags), large2,
 	    "Unexpected xallocx() behavior");
-	assert_zu_eq(xallocx(p, huge0, 0, 0), large2,
+	assert_zu_eq(xallocx(p, huge0, 0, flags), large2,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_eq(xallocx(p, large0, 0, 0), large0,
+	assert_zu_eq(xallocx(p, large0, 0, flags), large0,
 	    "Unexpected xallocx() behavior");
 	/* Test size increase with non-zero extra. */
-	assert_zu_lt(xallocx(p, large0, huge0 - large0, 0), huge0,
+	assert_zu_lt(xallocx(p, large0, huge0 - large0, flags), huge0,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_eq(xallocx(p, large0, 0, 0), large0,
+	assert_zu_eq(xallocx(p, large0, 0, flags), large0,
 	    "Unexpected xallocx() behavior");
 	/* Test size increase with non-zero extra. */
-	assert_zu_eq(xallocx(p, large0, large2 - large0, 0), large2,
+	assert_zu_eq(xallocx(p, large0, large2 - large0, flags), large2,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_eq(xallocx(p, large2, 0, 0), large2,
+	assert_zu_eq(xallocx(p, large2, 0, flags), large2,
 	    "Unexpected xallocx() behavior");
 	/* Test size+extra overflow. */
-	assert_zu_lt(xallocx(p, large2, hugemax - large2 + 1, 0), huge0,
+	assert_zu_lt(xallocx(p, large2, hugemax - large2 + 1, flags), huge0,
 	    "Unexpected xallocx() behavior");
 
-	dallocx(p, 0);
+	dallocx(p, flags);
 }
 TEST_END
 
 TEST_BEGIN(test_extra_huge)
 {
+	int flags = MALLOCX_ARENA(arena_ind());
 	size_t largemax, huge0, huge1, huge2, hugemax;
 	void *p;
 
@@ -294,56 +315,163 @@ TEST_BEGIN(test_extra_huge)
 	huge2 = get_huge_size(2);
 	hugemax = get_huge_size(get_nhuge()-1);
 
-	p = mallocx(huge2, 0);
+	p = mallocx(huge2, flags);
 	assert_ptr_not_null(p, "Unexpected mallocx() error");
 
-	assert_zu_eq(xallocx(p, huge2, 0, 0), huge2,
+	assert_zu_eq(xallocx(p, huge2, 0, flags), huge2,
 	    "Unexpected xallocx() behavior");
 	/* Test size decrease with zero extra. */
-	assert_zu_ge(xallocx(p, huge0, 0, 0), huge0,
+	assert_zu_ge(xallocx(p, huge0, 0, flags), huge0,
 	    "Unexpected xallocx() behavior");
-	assert_zu_ge(xallocx(p, largemax, 0, 0), huge0,
+	assert_zu_ge(xallocx(p, largemax, 0, flags), huge0,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_eq(xallocx(p, huge2, 0, 0), huge2,
+	assert_zu_eq(xallocx(p, huge2, 0, flags), huge2,
 	    "Unexpected xallocx() behavior");
 	/* Test size decrease with non-zero extra. */
-	assert_zu_eq(xallocx(p, huge0, huge2 - huge0, 0), huge2,
+	assert_zu_eq(xallocx(p, huge0, huge2 - huge0, flags), huge2,
 	    "Unexpected xallocx() behavior");
-	assert_zu_eq(xallocx(p, huge1, huge2 - huge1, 0), huge2,
+	assert_zu_eq(xallocx(p, huge1, huge2 - huge1, flags), huge2,
 	    "Unexpected xallocx() behavior");
-	assert_zu_eq(xallocx(p, huge0, huge1 - huge0, 0), huge1,
+	assert_zu_eq(xallocx(p, huge0, huge1 - huge0, flags), huge1,
 	    "Unexpected xallocx() behavior");
-	assert_zu_ge(xallocx(p, largemax, huge0 - largemax, 0), huge0,
+	assert_zu_ge(xallocx(p, largemax, huge0 - largemax, flags), huge0,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_ge(xallocx(p, huge0, 0, 0), huge0,
+	assert_zu_ge(xallocx(p, huge0, 0, flags), huge0,
 	    "Unexpected xallocx() behavior");
 	/* Test size increase with zero extra. */
-	assert_zu_le(xallocx(p, huge2, 0, 0), huge2,
+	assert_zu_le(xallocx(p, huge2, 0, flags), huge2,
 	    "Unexpected xallocx() behavior");
-	assert_zu_le(xallocx(p, hugemax+1, 0, 0), huge2,
+	assert_zu_le(xallocx(p, hugemax+1, 0, flags), huge2,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_ge(xallocx(p, huge0, 0, 0), huge0,
+	assert_zu_ge(xallocx(p, huge0, 0, flags), huge0,
 	    "Unexpected xallocx() behavior");
 	/* Test size increase with non-zero extra. */
-	assert_zu_le(xallocx(p, huge0, SIZE_T_MAX - huge0, 0), hugemax,
+	assert_zu_le(xallocx(p, huge0, SIZE_T_MAX - huge0, flags), hugemax,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_ge(xallocx(p, huge0, 0, 0), huge0,
+	assert_zu_ge(xallocx(p, huge0, 0, flags), huge0,
 	    "Unexpected xallocx() behavior");
 	/* Test size increase with non-zero extra. */
-	assert_zu_le(xallocx(p, huge0, huge2 - huge0, 0), huge2,
+	assert_zu_le(xallocx(p, huge0, huge2 - huge0, flags), huge2,
 	    "Unexpected xallocx() behavior");
 
-	assert_zu_eq(xallocx(p, huge2, 0, 0), huge2,
+	assert_zu_eq(xallocx(p, huge2, 0, flags), huge2,
 	    "Unexpected xallocx() behavior");
 	/* Test size+extra overflow. */
-	assert_zu_le(xallocx(p, huge2, hugemax - huge2 + 1, 0), hugemax,
+	assert_zu_le(xallocx(p, huge2, hugemax - huge2 + 1, flags), hugemax,
 	    "Unexpected xallocx() behavior");
 
-	dallocx(p, 0);
+	dallocx(p, flags);
+}
+TEST_END
+
+static void
+print_filled_extents(const void *p, uint8_t c, size_t len)
+{
+	const uint8_t *pc = (const uint8_t *)p;
+	size_t i, range0;
+	uint8_t c0;
+
+	malloc_printf("  p=%p, c=%#x, len=%zu:", p, c, len);
+	range0 = 0;
+	c0 = pc[0];
+	for (i = 0; i < len; i++) {
+		if (pc[i] != c0) {
+			malloc_printf(" %#x[%zu..%zu)", c0, range0, i);
+			range0 = i;
+			c0 = pc[i];
+		}
+	}
+	malloc_printf(" %#x[%zu..%zu)\n", c0, range0, i);
+}
+
+static bool
+validate_fill(const void *p, uint8_t c, size_t offset, size_t len)
+{
+	const uint8_t *pc = (const uint8_t *)p;
+	bool err;
+	size_t i;
+
+	for (i = offset, err = false; i < offset+len; i++) {
+		if (pc[i] != c)
+			err = true;
+	}
+
+	if (err)
+		print_filled_extents(p, c, offset + len);
+
+	return (err);
+}
+
+static void
+test_zero(size_t szmin, size_t szmax)
+{
+	int flags = MALLOCX_ARENA(arena_ind()) | MALLOCX_ZERO;
+	size_t sz, nsz;
+	void *p;
+#define	FILL_BYTE 0x7aU
+
+	sz = szmax;
+	p = mallocx(sz, flags);
+	assert_ptr_not_null(p, "Unexpected mallocx() error");
+	assert_false(validate_fill(p, 0x00, 0, sz), "Memory not filled: sz=%zu",
+	    sz);
+
+	/*
+	 * Fill with non-zero so that non-debug builds are more likely to detect
+	 * errors.
+	 */
+	memset(p, FILL_BYTE, sz);
+	assert_false(validate_fill(p, FILL_BYTE, 0, sz),
+	    "Memory not filled: sz=%zu", sz);
+
+	/* Shrink in place so that we can expect growing in place to succeed. */
+	sz = szmin;
+	assert_zu_eq(xallocx(p, sz, 0, flags), sz,
+	    "Unexpected xallocx() error");
+	assert_false(validate_fill(p, FILL_BYTE, 0, sz),
+	    "Memory not filled: sz=%zu", sz);
+
+	for (sz = szmin; sz < szmax; sz = nsz) {
+		nsz = nallocx(sz+1, flags);
+		assert_zu_eq(xallocx(p, sz+1, 0, flags), nsz,
+		    "Unexpected xallocx() failure");
+		assert_false(validate_fill(p, FILL_BYTE, 0, sz),
+		    "Memory not filled: sz=%zu", sz);
+		assert_false(validate_fill(p, 0x00, sz, nsz-sz),
+		    "Memory not filled: sz=%zu, nsz-sz=%zu", sz, nsz-sz);
+		memset((void *)((uintptr_t)p + sz), FILL_BYTE, nsz-sz);
+		assert_false(validate_fill(p, FILL_BYTE, 0, nsz),
+		    "Memory not filled: nsz=%zu", nsz);
+	}
+
+	dallocx(p, flags);
+}
+
+TEST_BEGIN(test_zero_large)
+{
+	size_t large0, largemax;
+
+	/* Get size classes. */
+	large0 = get_large_size(0);
+	largemax = get_large_size(get_nlarge()-1);
+
+	test_zero(large0, largemax);
+}
+TEST_END
+
+TEST_BEGIN(test_zero_huge)
+{
+	size_t huge0, huge1;
+
+	/* Get size classes. */
+	huge0 = get_huge_size(0);
+	huge1 = get_huge_size(1);
+
+	test_zero(huge1, huge0 * 2);
 }
 TEST_END
 
@@ -359,5 +487,7 @@ main(void)
 	    test_size_extra_overflow,
 	    test_extra_small,
 	    test_extra_large,
-	    test_extra_huge));
+	    test_extra_huge,
+	    test_zero_large,
+	    test_zero_huge));
 }