Merge branch 'dev'

Move assert() calls up in arena_run_reg_alloc(), so that a corrupt
pointer will likely be caught by an assertion *before* it is
dereferenced.

jemalloc/include/jemalloc/internal/chunk_mmap.h

@@ -10,6 +10,7 @@
 #ifdef JEMALLOC_H_EXTERNS
 
 void	*chunk_alloc_mmap(size_t size);
+void	*chunk_alloc_mmap_noreserve(size_t size);
 void	chunk_dealloc_mmap(void *chunk, size_t size);
 
 #endif /* JEMALLOC_H_EXTERNS */

jemalloc/include/jemalloc/internal/tcache.h

@@ -353,7 +353,7 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
 
 #ifdef JEMALLOC_FILL
 	if (opt_junk)
-		memset(ptr, 0x5a, arena->bins[binind].reg_size);
+		memset(ptr, 0x5a, size);
 #endif
 
 	tbin = &tcache->tbins[binind];

jemalloc/src/arena.c

@@ -254,7 +254,6 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
 	run->nfree--;
 	ret = run->avail;
 	if (ret != NULL) {
-		run->avail = *(void **)ret;
 		/* Double free can cause assertion failure.*/
 		assert(ret != NULL);
 		/* Write-after free can cause assertion failure. */
@@ -264,6 +263,7 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
 		assert(((uintptr_t)ret - ((uintptr_t)run +
 		    (uintptr_t)bin->reg0_offset)) % (uintptr_t)bin->reg_size ==
 		    0);
+		run->avail = *(void **)ret;
 		return (ret);
 	}
 	ret = run->next;

jemalloc/src/chunk_mmap.c

@@ -23,14 +23,15 @@ static
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static void	*pages_map(void *addr, size_t size);
+static void	*pages_map(void *addr, size_t size, bool noreserve);
 static void	pages_unmap(void *addr, size_t size);
-static void	*chunk_alloc_mmap_slow(size_t size, bool unaligned);
+static void	*chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve);
+static void	*chunk_alloc_mmap_internal(size_t size, bool noreserve);
 
 /******************************************************************************/
 
 static void *
-pages_map(void *addr, size_t size)
+pages_map(void *addr, size_t size, bool noreserve)
 {
 	void *ret;
 
@@ -38,8 +39,12 @@ pages_map(void *addr, size_t size)
 	 * We don't use MAP_FIXED here, because it can cause the *replacement*
 	 * of existing mappings, and we only want to create new mappings.
 	 */
-	ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON,
+	int flags = MAP_PRIVATE | MAP_ANON;
-	    -1, 0);
+#ifdef MAP_NORESERVE
+	if (noreserve)
+		flags |= MAP_NORESERVE;
+#endif
+	ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0);
 	assert(ret != NULL);
 
 	if (ret == MAP_FAILED)
@@ -83,7 +88,7 @@ pages_unmap(void *addr, size_t size)
 }
 
 static void *
-chunk_alloc_mmap_slow(size_t size, bool unaligned)
+chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve)
 {
 	void *ret;
 	size_t offset;
@@ -92,7 +97,7 @@ chunk_alloc_mmap_slow(size_t size, bool unaligned)
 	if (size + chunksize <= size)
 		return (NULL);
 
-	ret = pages_map(NULL, size + chunksize);
+	ret = pages_map(NULL, size + chunksize, noreserve);
 	if (ret == NULL)
 		return (NULL);
 
@@ -128,8 +133,8 @@ chunk_alloc_mmap_slow(size_t size, bool unaligned)
 	return (ret);
 }
 
-void *
+static void *
-chunk_alloc_mmap(size_t size)
+chunk_alloc_mmap_internal(size_t size, bool noreserve)
 {
 	void *ret;
 
@@ -164,7 +169,7 @@ chunk_alloc_mmap(size_t size)
 	if (mmap_unaligned == false) {
 		size_t offset;
 
-		ret = pages_map(NULL, size);
+		ret = pages_map(NULL, size, noreserve);
 		if (ret == NULL)
 			return (NULL);
 
@@ -173,13 +178,13 @@ chunk_alloc_mmap(size_t size)
 			mmap_unaligned = true;
 			/* Try to extend chunk boundary. */
 			if (pages_map((void *)((uintptr_t)ret + size),
-			    chunksize - offset) == NULL) {
+			    chunksize - offset, noreserve) == NULL) {
 				/*
 				 * Extension failed.  Clean up, then revert to
 				 * the reliable-but-expensive method.
 				 */
 				pages_unmap(ret, size);
-				ret = chunk_alloc_mmap_slow(size, true);
+				ret = chunk_alloc_mmap_slow(size, true, noreserve);
 			} else {
 				/* Clean up unneeded leading space. */
 				pages_unmap(ret, chunksize - offset);
@@ -188,11 +193,23 @@ chunk_alloc_mmap(size_t size)
 			}
 		}
 	} else
-		ret = chunk_alloc_mmap_slow(size, false);
+		ret = chunk_alloc_mmap_slow(size, false, noreserve);
 
 	return (ret);
 }
 
+void *
+chunk_alloc_mmap(size_t size)
+{
+	return chunk_alloc_mmap_internal(size, false);
+}
+
+void *
+chunk_alloc_mmap_noreserve(size_t size)
+{
+	return chunk_alloc_mmap_internal(size, true);
+}
+
 void
 chunk_dealloc_mmap(void *chunk, size_t size)
 {

jemalloc/src/chunk_swap.c

@@ -283,7 +283,7 @@ chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed)
 	 * Allocate a chunk-aligned region of anonymous memory, which will
 	 * be the final location for the memory-mapped files.
 	 */
-	vaddr = chunk_alloc_mmap(cumsize);
+	vaddr = chunk_alloc_mmap_noreserve(cumsize);
 	if (vaddr == NULL) {
 		ret = true;
 		goto RETURN;

jemalloc/src/jemalloc.c

@@ -324,6 +324,7 @@ malloc_init_hard(void)
 			CPU_SPINWAIT;
 			malloc_mutex_lock(&init_lock);
 		} while (malloc_initialized == false);
+		malloc_mutex_unlock(&init_lock);
 		return (false);
 	}
 
@@ -775,7 +776,7 @@ MALLOC_OUT:
 #endif
 
 #ifndef NO_TLS
-	next_arena = 0;
+	next_arena = (narenas > 0) ? 1 : 0;
 #endif
 
 	/* Allocate and initialize arenas. */

jemalloc/src/prof.c

@@ -239,16 +239,15 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max)
 	}
 
 	/*
-	 * Iterate over stack frames until there are no more.  Heap-allocate
+	 * Iterate over stack frames until there are no more, or until no space
-	 * and iteratively grow a larger bt if necessary.
+	 * remains in bt.
 	 */
 	for (i = 0; i < max; i++) {
 		unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]);
+		bt->len++;
 		err = unw_step(&cursor);
-		if (err <= 0) {
+		if (err <= 0)
-			bt->len = i;
 			break;
-		}
 	}
 }
 #else
@@ -623,13 +622,8 @@ static inline void
 prof_sample_accum_update(size_t size)
 {
 
-	if (opt_lg_prof_sample == 0) {
+	/* Sampling logic is unnecessary if the interval is 1. */
-		/*
+	assert(opt_lg_prof_sample != 0);
-		 * Don't bother with sampling logic, since sampling interval is
-		 * 1.
-		 */
-		return;
-	}
 
 	/* Take care to avoid integer overflow. */
 	if (size >= prof_sample_threshold - prof_sample_accum) {
@@ -647,11 +641,15 @@ prof_sample_accum_update(size_t size)
 void
 prof_malloc(const void *ptr, prof_thr_cnt_t *cnt)
 {
-	size_t size = isalloc(ptr);
+	size_t size;
 
 	assert(ptr != NULL);
 
-	prof_sample_accum_update(size);
+	if (opt_lg_prof_sample != 0) {
+		size = isalloc(ptr);
+		prof_sample_accum_update(size);
+	} else if ((uintptr_t)cnt > (uintptr_t)1U)
+		size = isalloc(ptr);
 
 	if ((uintptr_t)cnt > (uintptr_t)1U) {
 		prof_ctx_set(ptr, cnt->ctx);
@@ -679,11 +677,18 @@ void
 prof_realloc(const void *ptr, prof_thr_cnt_t *cnt, const void *old_ptr,
     size_t old_size, prof_ctx_t *old_ctx)
 {
-	size_t size = isalloc(ptr);
+	size_t size;
 	prof_thr_cnt_t *told_cnt;
 
-	if (ptr != NULL)
+	assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U);
-		prof_sample_accum_update(size);
+
+	if (ptr != NULL) {
+		if (opt_lg_prof_sample != 0) {
+			size = isalloc(ptr);
+			prof_sample_accum_update(size);
+		} else if ((uintptr_t)cnt > (uintptr_t)1U)
+			size = isalloc(ptr);
+	}
 
 	if ((uintptr_t)old_ctx > (uintptr_t)1U) {
 		told_cnt = prof_lookup(old_ctx->bt);