Update documentation.

Generalize ExtractSymbols to handle all cases of library address overlap
with the main binary.

.gitignore

@@ -1,7 +1,5 @@
 /jemalloc/autom4te.cache/
-/jemalloc/bin/
+/jemalloc/config.stamp
-/jemalloc/cfghdrs.stamp
-/jemalloc/cfgoutputs.stamp
 /jemalloc/config.log
 /jemalloc/config.status
 /jemalloc/configure

jemalloc/COPYING

@@ -23,7 +23,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 --------------------------------------------------------------------------------
-Copyright (C) 2009 Facebook, Inc.
+Copyright (C) 2009-2010 Facebook, Inc.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

jemalloc/INSTALL

@@ -45,8 +45,8 @@ any of the following arguments (not a definitive list) to 'configure':
     detailed allocation statistics at exit.
 
 --enable-prof
-    Enable heap profiling and leak detection functionality.  Use the 'B', 'F',
+    Enable heap profiling and leak detection functionality.  Use the 'B', 'E',
-    'I', 'L', and 'U' options to control these features.
+    'F', 'I', 'L', and 'U' options to control these features.
 
 --disable-prof-libgcc
     Disable the use of libgcc's backtracing functionality.  Ordinarily, libgcc's
@@ -59,6 +59,10 @@ any of the following arguments (not a definitive list) to 'configure':
     variety of system configurations than the default backtracing code, which is
     based on libgcc functionality or gcc intrinsics.
 
+--with-static-libunwind=<libunwind.a>
+    Statically link against the specified libunwind.a rather than dynamically
+    linking with -lunwind.
+
 --disable-tiny
     Disable tiny (sub-quantum-sized) object support.  Technically it is not
     legal for a malloc implementation to allocate objects with less than
@@ -67,9 +71,9 @@ any of the following arguments (not a definitive list) to 'configure':
     are 4-byte-aligned.
 
 --disable-tcache
-    Disable thread-specific caches for small and medium objects.  Objects are
+    Disable thread-specific caches for small objects.  Objects are cached and
-    cached and released in bulk, thus reducing the total number of mutex
+    released in bulk, thus reducing the total number of mutex operations.  Use
-    operations.  Use the 'H' and 'G' options to control thread-specific caching.
+    the 'H', 'G', and 'M' options to control thread-specific caching.
 
 --enable-swap
     Enable mmap()ed swap file support.  When this feature is built in, it is
@@ -112,7 +116,7 @@ any of the following arguments (not a definitive list) to 'configure':
 --disable-tls
     Disable thread-local storage (TLS), which allows for fast access to
     thread-local variables via the __thread keyword.  If TLS is available,
-    jemalloc uses it for several purposes.  Not that disabling TLS implies
+    jemalloc uses it for several purposes.  Note that disabling TLS implies
     --disable-tcache.
 
 The following environment variables (not a definitive list) impact configure's
@@ -147,9 +151,12 @@ PATH="?"
 
 === Advanced compilation =======================================================
 
-To run integrated regression tests, type:
+To install only parts of jemalloc, use the following targets:
 
-    make check
+    install_bin
+    install_include
+    install_lib
+    install_man
 
 To clean up build results to varying degrees, use the following make targets:
 
@@ -171,6 +178,10 @@ LIBDIR="?"
 MANDIR="?"
     Use this as the installation prefix for man pages.
 
+DESTDIR="?"
+    Prepend DESTDIR to INCLUDEDIR, LIBDIR, and MANDIR.  This is useful when
+    installing to a different path than was specified via --prefix.
+
 CC="?"
     Use this to invoke the C compiler.
 

jemalloc/Makefile.in

@@ -11,9 +11,11 @@ SHELL := /bin/sh
 CC := @CC@
 
 # Configuration parameters.
-INCLUDEDIR := @INCLUDEDIR@
+DESTDIR =
-LIBDIR := @LIBDIR@
+BINDIR := $(DESTDIR)@BINDIR@
-MANDIR := @MANDIR@
+INCLUDEDIR := $(DESTDIR)@INCLUDEDIR@
+LIBDIR := $(DESTDIR)@LIBDIR@
+MANDIR := $(DESTDIR)@MANDIR@
 
 # Build parameters.
 CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include
@@ -32,8 +34,9 @@ endif
 REV := 0
 
 # Lists of files.
-CHDRS := @objroot@include/jemalloc@install_suffix@.h \
+BINS := @srcroot@bin/pprof
-	@objroot@include/jemalloc_defs@install_suffix@.h
+CHDRS := @objroot@include/jemalloc/jemalloc@install_suffix@.h \
+	@objroot@include/jemalloc/jemalloc_defs@install_suffix@.h
 CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/base.c \
 	@srcroot@src/chunk.c @srcroot@src/chunk_dss.c \
 	@srcroot@src/chunk_mmap.c @srcroot@src/chunk_swap.c @srcroot@src/ckh.c \
@@ -60,28 +63,47 @@ all: $(DSOS)
 	$(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $<
 	@$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)"
 
-$(DSOS): $(CSRCS:@srcroot@%.c=@objroot@%.o)
+%.so : %.so.$(REV)
 	@mkdir -p $(@D)
-	$(CC) -shared -o $@ $+ $(LDFLAGS) $(LIBS)
+	ln -sf $(<F) $@
-	ln -sf libjemalloc@install_suffix@.so.$(REV) lib/libjemalloc@install_suffix@.so
-	ar crus @objroot@lib/libjemalloc@install_suffix@_pic.a $+
 
-install:
+@objroot@lib/libjemalloc@install_suffix@.so.$(REV) : $(CSRCS:@srcroot@%.c=@objroot@%.o)
+	@mkdir -p $(@D)
+	$(CC) -shared -Wl,-soname,$(@F) -o $@ $+ $(LDFLAGS) $(LIBS)
+
+@objroot@lib/libjemalloc@install_suffix@_pic.a : $(CSRCS:@srcroot@%.c=@objroot@%.o)
+	@mkdir -p $(@D)
+	ar crus $@ $+
+
+install_bin:
+	install -d $(BINDIR)
+	@for b in $(BINS); do \
+	echo "install -m 755 $$b $(BINDIR)"; \
+	install -m 755 $$b $(BINDIR); \
+done
+
+install_include:
 	install -d $(INCLUDEDIR)/jemalloc
 	@for h in $(CHDRS); do \
 	echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \
 	install -m 644 $$h $(INCLUDEDIR)/jemalloc; \
 done
+
+install_lib: $(DSOS)
 	install -d $(LIBDIR)
 	install -m 755 @objroot@lib/libjemalloc@install_suffix@.so.$(REV) $(LIBDIR)
 	ln -sf libjemalloc@install_suffix@.so.$(REV) $(LIBDIR)/libjemalloc@install_suffix@.so
 	install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR)
+
+install_man:
 	install -d $(MANDIR)/man3
 	@for m in $(MAN3); do \
 	echo "install -m 644 $$m $(MANDIR)/man3"; \
 	install -m 644 $$m $(MANDIR)/man3; \
 done
 
+install: install_bin install_include install_lib install_man
+
 check:
 
 clean:
@@ -112,26 +134,15 @@ ifeq (@enable_autogen@, 1)
 @objroot@config.status : @srcroot@configure
 	./@objroot@config.status --recheck
 
-# cfghdrs rules.
+@srcroot@config.stamp.in : @srcroot@configure.ac
-@srcroot@cfghdrs.stamp.in : @srcroot@configure.ac
+	echo stamp > @srcroot@config.stamp.in
-	echo stamp > @srcroot@cfghdrs.stamp.in
 
-@objroot@cfghdrs.stamp : @cfghdrs_in@ @srcroot@configure
+@objroot@config.stamp : @cfgoutputs_in@ @cfghdrs_in@ @srcroot@configure
-	./@objroot@config.status
-	@touch $@
-
-@cfghdrs@ : @objroot@cfghdrs.stamp
-
-# cfgoutputs rules.
-@srcroot@cfgoutputs.stamp.in : @srcroot@configure.ac
-	echo stamp > @srcroot@cfgoutputs.stamp.in
-
-@objroot@cfgoutputs.stamp : @cfgoutputs_in@ @srcroot@configure
 	./@objroot@config.status
 	@touch $@
 
 # There must be some action in order for make to re-read Makefile when it is
 # out of date.
-@cfgoutputs@ : @objroot@cfgoutputs.stamp
+@cfgoutputs_out@ @cfghdrs_out@ : @objroot@config.stamp
 	@true
 endif

jemalloc/README

@@ -1,4 +1,14 @@
 jemalloc is a general-purpose scalable concurrent malloc(3) implementation.
+This distribution is a stand-alone "portable" implementation that currently
+targets only Linux.  jemalloc is included as the default allocator in the
+FreeBSD and NetBSD operating systems, and it is used by the Mozilla Firefox web
+browser on Microsoft Windows-related platforms.  Depending on your needs, one
+of the other divergent versions may suit your needs better than this
+distribution.
+
+The COPYING file contains copyright and licensing information.
 
 The INSTALL file contains information on how to configure, build, and install
 jemalloc.
+
+URL: http://www.canonware.com/jemalloc/

jemalloc/configure.ac

@@ -86,11 +86,12 @@ CFLAGS=$CFLAGS
 AC_PROG_CC
 if test "x$CFLAGS" = "x" ; then
   no_CFLAGS="yes"
+  if test "x$GCC" = "xyes" ; then
     JE_CFLAGS_APPEND([-std=gnu99])
     JE_CFLAGS_APPEND([-Wall])
     JE_CFLAGS_APPEND([-pipe])
     JE_CFLAGS_APPEND([-g3])
-  JE_CFLAGS_APPEND([-march=native])
+  fi
 fi
 dnl Append EXTRA_CFLAGS to CFLAGS, if defined.
 if test "x$EXTRA_CFLAGS" != "x" ; then
@@ -262,9 +263,9 @@ AC_ARG_WITH([install_suffix],
 install_suffix="$INSTALL_SUFFIX"
 AC_SUBST([install_suffix])
 
-cfgoutputs_in="Makefile doc/jemalloc.3.in"
+cfgoutputs_in="${srcroot}Makefile.in ${srcroot}doc/jemalloc.3.in"
-cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc.h.in"
+cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc.h.in"
-cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_internal.h.in"
+cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal.h.in"
 
 cfgoutputs_out="Makefile doc/jemalloc${install_suffix}.3"
 cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc${install_suffix}.h"
@@ -274,7 +275,7 @@ cfgoutputs_tup="Makefile doc/jemalloc${install_suffix}.3:doc/jemalloc.3.in"
 cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc${install_suffix}.h:include/jemalloc/jemalloc.h.in"
 cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h"
 
-cfghdrs_in="include/jemalloc/jemalloc_defs.h.in"
+cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in"
 
 cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h"
 
@@ -305,7 +306,6 @@ if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then
     if test "x$GCC" = "xyes" ; then
       JE_CFLAGS_APPEND([-O3])
       JE_CFLAGS_APPEND([-funroll-loops])
-      JE_CFLAGS_APPEND([-fomit-frame-pointer])
     else
       JE_CFLAGS_APPEND([-O])
     fi
@@ -366,6 +366,19 @@ fi
 ],
 [enable_prof_libunwind="0"]
 )
+AC_ARG_WITH([static_libunwind],
+  [AS_HELP_STRING([--with-static-libunwind=<libunwind.a>],
+  [Path to static libunwind library; use rather than dynamically linking])],
+if test "x$with_static_libunwind" = "xno" ; then
+  LUNWIND="-lunwind"
+else
+  if test ! -f "$with_static_libunwind" ; then
+    AC_MSG_ERROR([Static libunwind not found: $with_static_libunwind])
+  fi
+  LUNWIND="$with_static_libunwind"
+fi,
+  LUNWIND="-lunwind"
+)
 dnl Finish prof-related definitions below, once TLS configuration is done.
 
 dnl Enable tiny allocations by default.
@@ -665,8 +678,12 @@ if test "x$enable_prof" = "x1" ; then
   AC_DEFINE([JEMALLOC_PROF], [ ])
   if test "x$enable_prof_libunwind" = "x1" ; then
     AC_CHECK_HEADERS([libunwind.h], , [enable_prof_libunwind="0"])
-    AC_CHECK_LIB([unwind], [backtrace], [LIBS="$LIBS -lunwind"],
+    if test "x$LUNWIND" = "x-lunwind" ; then
+      AC_CHECK_LIB([unwind], [backtrace], [LIBS="$LIBS $LUNWIND"],
                    [enable_prof_libunwind="0"])
+    else
+      LIBS="$LIBS $LUNWIND"
+    fi
     if test "x${enable_prof_libunwind}" = "x1" ; then
       AC_DEFINE([JEMALLOC_PROF_LIBUNWIND], [ ])
     fi
@@ -702,11 +719,11 @@ AC_HEADER_STDBOOL
 dnl Process .in files.
 AC_SUBST([cfghdrs_in])
 AC_SUBST([cfghdrs_out])
-AC_CONFIG_HEADERS([$cfghdrs_tup cfghdrs.stamp])
+AC_CONFIG_HEADERS([$cfghdrs_tup])
 
 dnl ============================================================================
 dnl Generate outputs.
-AC_CONFIG_FILES([$cfgoutputs_tup cfgoutputs.stamp])
+AC_CONFIG_FILES([$cfgoutputs_tup config.stamp])
 AC_SUBST([cfgoutputs_in])
 AC_SUBST([cfgoutputs_out])
 AC_OUTPUT

jemalloc/doc/jemalloc.3.in

@@ -38,7 +38,7 @@
 .\"     @(#)malloc.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD: head/lib/libc/stdlib/malloc.3 182225 2008-08-27 02:00:53Z jasone $
 .\"
-.Dd March 1, 2010
+.Dd April 2, 2010
 .Dt JEMALLOC 3
 .Os
 .Sh NAME
@@ -71,7 +71,7 @@
 .Ft size_t
 .Fn @jemalloc_prefix@malloc_usable_size "const void *ptr"
 .Ft void
-.Fn @jemalloc_prefix@malloc_stats_print "void (*write4)(void *" "const char *" "const char *" "const char *" "const char *)" "const char *opts"
+.Fn @jemalloc_prefix@malloc_stats_print "void (*write_cb)(void *" "const char *)" "void *cbopaque" "const char *opts"
 .Ft int
 .Fn @jemalloc_prefix@mallctl "const char *name" "void *oldp" "size_t *oldlenp" "void *newp" "size_t newlen"
 .Ft int
@@ -81,7 +81,7 @@
 .Ft const char *
 .Va @jemalloc_prefix@malloc_options ;
 .Ft void
-.Fn \*(lp*@jemalloc_prefix@malloc_message\*(rp "void *w4opaque" "const char *p1" "const char *p2" "const char *p3" "const char *p4"
+.Fn \*(lp*@jemalloc_prefix@malloc_message\*(rp "void *cbopaque" "const char *s"
 .Sh DESCRIPTION
 The
 .Fn @jemalloc_prefix@malloc
@@ -178,15 +178,15 @@ implementation-dependent.
 The
 .Fn @jemalloc_prefix@malloc_stats_print
 function writes human-readable summary statistics via the
-.Fa write4
+.Fa write_cb
 callback function pointer and
-.Fa w4opaque
+.Fa cbopaque
 data passed to
-.Fn write4 ,
+.Fn write_cb ,
 or
 .Fn @jemalloc_prefix@malloc_message
 if
-.Fa write4
+.Fa write_cb
 is
 .Dv NULL .
 This function can be called repeatedly.
@@ -333,13 +333,24 @@ The default value is 512 bytes.
 .It D
 Halve/double the per-arena minimum ratio of active to dirty pages.
 Some dirty unused pages may be allowed to accumulate, within the limit set by
-the ratio, before informing the kernel about at least half of those pages via
+the ratio (or one chunk worth of dirty pages, whichever is greater), before
+informing the kernel about some of those pages via
 .Xr madvise 2 .
 This provides the kernel with sufficient information to recycle dirty pages if
 physical memory becomes scarce and the pages remain unused.
 The default minimum ratio is 32:1;
 .Ev JEMALLOC_OPTIONS=6D
 will disable dirty page purging.
+@roff_prof@.It E
+@roff_prof@Activate/deactivate profiling.
+@roff_prof@This is a secondary control mechanism that makes it possible to
+@roff_prof@start the application with profiling enabled (see the
+@roff_prof@.Dq F
+@roff_prof@option) but inactive, then toggle profiling at any time during
+@roff_prof@program execution with the
+@roff_prof@.Dq prof.active
+@roff_prof@mallctl.
+@roff_prof@This option is enabled by default.
 @roff_prof@.It F
 @roff_prof@Profile memory allocation activity, and use an
 @roff_prof@.Xr atexit 3
@@ -355,6 +366,9 @@ will disable dirty page purging.
 @roff_prof@.Dq B
 @roff_prof@option for backtrace depth control.
 @roff_prof@See the
+@roff_prof@.Dq E
+@roff_prof@option for on-the-fly activation/deactivation.
+@roff_prof@See the
 @roff_prof@.Dq S
 @roff_prof@option for probabilistic sampling control.
 @roff_prof@See the
@@ -362,8 +376,8 @@ will disable dirty page purging.
 @roff_prof@option for information on interval-triggered profile dumping, and the
 @roff_prof@.Dq U
 @roff_prof@option for information on high-water-triggered profile dumping.
-@roff_prof@Profile output is compatible with the pprof Perl script, which is
+@roff_prof@Profile output is compatible with the included pprof Perl script,
-@roff_prof@part of the google-perftools package
+@roff_prof@which originates from the google-perftools package
 @roff_prof@(http://code.google.com/p/google-perftools/).
 @roff_tcache@.It G
 @roff_tcache@Double/halve the approximate interval (counted in terms of
@@ -375,21 +389,18 @@ will disable dirty page purging.
 @roff_tcache@.Ev JEMALLOC_OPTIONS=14g
 @roff_tcache@will disable garbage collection.
 @roff_tcache@.It H
-@roff_tcache@Double/halve the number of thread-specific cache slots per size
+@roff_tcache@Enable/disable thread-specific caching.
-@roff_tcache@class.
 @roff_tcache@When there are multiple threads, each thread uses a
-@roff_tcache@thread-specific cache for small and medium objects.
+@roff_tcache@thread-specific cache for objects up to a certain size.
 @roff_tcache@Thread-specific caching allows many allocations to be satisfied
 @roff_tcache@without performing any thread synchronization, at the cost of
 @roff_tcache@increased memory use.
 @roff_tcache@See the
 @roff_tcache@.Dq G
-@roff_tcache@option for related tuning information.
+@roff_tcache@and
-@roff_tcache@The default number of cache slots is 128;
+@roff_tcache@.Dq M
-@roff_tcache@.Ev JEMALLOC_OPTIONS=7h
+@roff_tcache@options for related tuning information.
-@roff_tcache@will disable thread-specific caching.
+@roff_tcache@This option is enabled by default.
-@roff_tcache@Note that one cache slot per size class is not a valid
-@roff_tcache@configuration due to implementation details.
 @roff_prof@.It I
 @roff_prof@Double/halve the average interval between memory profile dumps, as
 @roff_prof@measured in bytes of allocation activity.
@@ -403,7 +414,9 @@ will disable dirty page purging.
 @roff_prof@is controlled by the
 @roff_prof@JEMALLOC_PROF_PREFIX
 @roff_prof@environment variable.
-@roff_prof@The default average interval is 1 GiB.
+@roff_prof@The default average interval is 1 GiB;
+@roff_prof@.Ev JEMALLOC_OPTIONS=31i
+@roff_prof@will disable interval-triggered profile dumping.
 @roff_fill@.It J
 @roff_fill@Each byte of new memory allocated by
 @roff_fill@.Fn @jemalloc_prefix@malloc
@@ -430,16 +443,15 @@ The default chunk size is 4 MiB.
 @roff_prof@See the
 @roff_prof@.Dq F option for information on analyzing heap profile output.
 @roff_prof@This option is disabled by default.
-.It M
+@roff_tcache@.It M
-Double/halve the size of the maximum medium size class.
+@roff_tcache@Double/halve the maximum size class to cache.
-The valid range is from one page to one half chunk.
+@roff_tcache@At a minimum, all small size classes are cached, and at a maximum
-The default value is 32 KiB.
+@roff_tcache@all large size classes are cached.
+@roff_tcache@The default maximum is 32 KiB.
 .It N
 Double/halve the number of arenas.
-The default number of arenas is
+The default number of arenas is four times the number of CPUs, or one if there
-@roff_tcache@two
+is a single CPU.
-@roff_no_tcache@four
-times the number of CPUs, or one if there is a single CPU.
 @roff_swap@.It O
 @roff_swap@Over-commit memory as a side effect of using anonymous
 @roff_swap@.Xr mmap 2
@@ -474,8 +486,6 @@ The default value is 128 bytes.
 @roff_prof@also decreases the computational overhead.
 @roff_prof@The default sample interval is one (i.e. all allocations are
 @roff_prof@sampled).
-@roff_prof@A sample interval greater than one implicitly disables leak
-@roff_prof@reporting.
 @roff_prof@.It U
 @roff_prof@Trigger a memory profile dump every time the total virtual memory
 @roff_prof@exceeds the previous maximum.
@@ -554,9 +564,9 @@ However, it may make sense to reduce the number of arenas if an application
 does not make much use of the allocation functions.
 .Pp
 @roff_tcache@In addition to multiple arenas, this allocator supports
-@roff_tcache@thread-specific caching for small and medium objects, in order to
+@roff_tcache@thread-specific caching for small objects, in order to make it
-@roff_tcache@make it possible to completely avoid synchronization for most small
+@roff_tcache@possible to completely avoid synchronization for most small
-@roff_tcache@and medium allocation requests.
+@roff_tcache@allocation requests.
 @roff_tcache@Such caching allows very fast allocation in the common case, but it
 @roff_tcache@increases memory usage and fragmentation, since a bounded number of
 @roff_tcache@objects can remain allocated in each thread cache.
@@ -567,27 +577,23 @@ Chunks are always aligned to multiples of the chunk size.
 This alignment makes it possible to find metadata for user objects very
 quickly.
 .Pp
-User objects are broken into four categories according to size: small, medium,
+User objects are broken into three categories according to size: small, large,
-large, and huge.
+and huge.
 Small objects are smaller than one page.
-Medium objects range from one page to an upper limit determined at run time (see
-the
-.Dq M
-option).
 Large objects are smaller than the chunk size.
 Huge objects are a multiple of the chunk size.
-Small, medium, and large objects are managed by arenas; huge objects are managed
+Small and large objects are managed by arenas; huge objects are managed
 separately in a single data structure that is shared by all threads.
 Huge objects are used by applications infrequently enough that this single
 data structure is not a scalability issue.
 .Pp
 Each chunk that is managed by an arena tracks its contents as runs of
-contiguous pages (unused, backing a set of small or medium objects, or backing
+contiguous pages (unused, backing a set of small objects, or backing one large
-one large object).
+object).
 The combination of chunk alignment and chunk page maps makes it possible to
 determine all metadata regarding small and large allocations in constant time.
 .Pp
-Small and medium objects are managed in groups by page runs.
+Small objects are managed in groups by page runs.
 Each run maintains a bitmap that tracks which regions are in use.
 @roff_tiny@Allocation requests that are no more than half the quantum (8 or 16,
 @roff_tiny@depending on architecture) are rounded up to the nearest power of
@@ -607,13 +613,7 @@ Allocation requests that are more than the minimum subpage-multiple size class,
 but no more than the maximum subpage-multiple size class are rounded up to the
 nearest multiple of the subpage size (256).
 Allocation requests that are more than the maximum subpage-multiple size class,
-but no more than the maximum medium size class (see the
+but small enough to fit in an arena-managed chunk (see the
-.Dq M
-option) are rounded up to the nearest medium size class; spacing is an
-automatically determined power of two and ranges from the subpage size to the
-page size.
-Allocation requests that are more than the maximum medium size class, but small
-enough to fit in an arena-managed chunk (see the
 .Dq K
 option), are rounded up to the nearest run size.
 Allocation requests that are too large to fit in an arena-managed chunk are
@@ -772,7 +772,7 @@ option.
 @roff_xmalloc@option.
 @roff_xmalloc@.Ed
 .\"-----------------------------------------------------------------------------
-@roff_tcache@.It Sy "opt.lg_tcache_nslots (size_t) r-"
+@roff_tcache@.It Sy "opt.tcache (bool) r-"
 @roff_tcache@.Bd -ragged -offset indent -compact
 @roff_tcache@See the
 @roff_tcache@.Dq H
@@ -807,7 +807,7 @@ option.
 @roff_prof@option.
 @roff_prof@.Ed
 .\"-----------------------------------------------------------------------------
-@roff_prof@.It Sy "opt.lg_prof_interval (size_t) r-"
+@roff_prof@.It Sy "opt.lg_prof_interval (ssize_t) r-"
 @roff_prof@.Bd -ragged -offset indent -compact
 @roff_prof@See the
 @roff_prof@.Dq I
@@ -842,13 +842,6 @@ See the
 option.
 .Ed
 .\"-----------------------------------------------------------------------------
-.It Sy "opt.lg_medium_max (size_t) r-"
-.Bd -ragged -offset indent -compact
-See the
-.Dq M
-option.
-.Ed
-.\"-----------------------------------------------------------------------------
 .It Sy "opt.lg_dirty_mult (ssize_t) r-"
 .Bd -ragged -offset indent -compact
 See the
@@ -904,11 +897,6 @@ Subpage size class interval.
 Page size.
 .Ed
 .\"-----------------------------------------------------------------------------
-.It Sy "arenas.medium (size_t) r-"
-.Bd -ragged -offset indent -compact
-Medium size class interval.
-.Ed
-.\"-----------------------------------------------------------------------------
 .It Sy "arenas.chunksize (size_t) r-"
 .Bd -ragged -offset indent -compact
 Chunk size.
@@ -956,15 +944,10 @@ Minimum subpage-spaced size class.
 Maximum subpage-spaced size class.
 .Ed
 .\"-----------------------------------------------------------------------------
-.It Sy "arenas.medium_min (size_t) r-"
+@roff_tcache@.It Sy "arenas.tcache_max (size_t) r-"
-.Bd -ragged -offset indent -compact
+@roff_tcache@.Bd -ragged -offset indent -compact
-Minimum medium-spaced size class.
+@roff_tcache@Maximum thread-cached size class.
-.Ed
+@roff_tcache@.Ed
-.\"-----------------------------------------------------------------------------
-.It Sy "arenas.medium_max (size_t) r-"
-.Bd -ragged -offset indent -compact
-Maximum medium-spaced size class.
-.Ed
 .\"-----------------------------------------------------------------------------
 .It Sy "arenas.ntbins (unsigned) r-"
 .Bd -ragged -offset indent -compact
@@ -986,16 +969,16 @@ Number of cacheline-spaced bin size classes.
 Number of subpage-spaced bin size classes.
 .Ed
 .\"-----------------------------------------------------------------------------
-.It Sy "arenas.nmbins (unsigned) r-"
-.Bd -ragged -offset indent -compact
-Number of medium-spaced bin size classes.
-.Ed
-.\"-----------------------------------------------------------------------------
 .It Sy "arenas.nbins (unsigned) r-"
 .Bd -ragged -offset indent -compact
 Total number of bin size classes.
 .Ed
 .\"-----------------------------------------------------------------------------
+@roff_tcache@.It Sy "arenas.nhbins (unsigned) r-"
+@roff_tcache@.Bd -ragged -offset indent -compact
+@roff_tcache@Total number of thread cache bin size classes.
+@roff_tcache@.Ed
+.\"-----------------------------------------------------------------------------
 .It Sy "arenas.bin.<i>.size (size_t) r-"
 .Bd -ragged -offset indent -compact
 Maximum size supported by size class.
@@ -1021,6 +1004,14 @@ Total number of large size classes.
 Maximum size supported by this large size class.
 .Ed
 .\"-----------------------------------------------------------------------------
+@roff_prof@.It Sy "prof.active (bool) rw"
+@roff_prof@.Bd -ragged -offset indent -compact
+@roff_prof@Control whether sampling is currently active.
+@roff_prof@See the
+@roff_prof@.Dq E
+@roff_prof@option for additional information.
+@roff_prof@.Ed
+.\"-----------------------------------------------------------------------------
 @roff_prof@.It Sy "prof.dump (const char *) -w"
 @roff_prof@.Bd -ragged -offset indent -compact
 @roff_prof@Dump a memory profile to the specified file, or if NULL is specified,
@@ -1138,27 +1129,17 @@ has not been called.
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.small.nmalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of small allocation requests.
+@roff_stats@Cumulative number of allocation requests served by small bins.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.small.ndalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of small deallocation requests.
+@roff_stats@Cumulative number of small objects returned to bins.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
-@roff_stats@.It Sy "stats.arenas.<i>.medium.allocated (size_t) r-"
+@roff_stats@.It Sy "stats.arenas.<i>.small.nrequests (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Number of bytes currently allocated by medium objects.
+@roff_stats@Cumulative number of small allocation requests.
-@roff_stats@.Ed
-.\"-----------------------------------------------------------------------------
-@roff_stats@.It Sy "stats.arenas.<i>.medium.nmalloc (uint64_t) r-"
-@roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of medium allocation requests.
-@roff_stats@.Ed
-.\"-----------------------------------------------------------------------------
-@roff_stats@.It Sy "stats.arenas.<i>.medium.ndalloc (uint64_t) r-"
-@roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of medium deallocation requests.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.large.allocated (size_t) r-"
@@ -1168,12 +1149,34 @@ has not been called.
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.large.nmalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of large allocation requests.
+@roff_stats@Cumulative number of large allocation requests served directly by
+@roff_stats@the arena.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.large.ndalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of large deallocation requests.
+@roff_stats@Cumulative number of large deallocation requests served directly by
+@roff_stats@the arena.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.large.nrequests (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of large allocation requests.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.allocated (size_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Current number of bytes allocated by bin.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nmalloc (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of allocations served by bin.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.ndalloc (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of allocations returned to bin.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nrequests (uint64_t) r-"
@@ -1212,6 +1215,18 @@ has not been called.
 @roff_stats@Current number of runs.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.lruns.<j>.nmalloc (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of allocation requests for this size class served
+@roff_stats@directly by the arena.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.lruns.<j>.ndalloc (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of deallocation requests for this size class
+@roff_stats@served directly by the arena.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.lruns.<j>.nrequests (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
 @roff_stats@Cumulative number of allocation requests for this size class.
@@ -1315,12 +1330,12 @@ strings forming the errors and warnings if for some reason the
 file descriptor is not suitable for this.
 .Va @jemalloc_prefix@malloc_message
 takes the
-.Fa w4opaque
+.Fa cbopaque
 pointer argument that is
 .Dv NULL
 unless overridden by the arguments in a call to
 .Fn @jemalloc_prefix@malloc_stats_print ,
-followed by four string pointers.
+followed by a string pointer.
 Please note that doing anything which tries to allocate memory in this function
 is likely to result in a crash or deadlock.
 .Pp

jemalloc/include/jemalloc/internal/arena.h

@@ -18,7 +18,7 @@
 
 #ifdef JEMALLOC_TINY
    /* Smallest size class to support. */
-#  define LG_TINY_MIN		1
+#  define LG_TINY_MIN		LG_SIZEOF_PTR
 #endif
 
 /*
@@ -35,23 +35,6 @@
  */
 #define	LG_CSPACE_MAX_DEFAULT	9
 
-/*
- * Maximum medium size class.  This must not be more than 1/4 of a chunk
- * (LG_MEDIUM_MAX_DEFAULT <= LG_CHUNK_DEFAULT - 2).
- */
-#define	LG_MEDIUM_MAX_DEFAULT	15
-
-/* Return the smallest medium size class that is >= s. */
-#define	MEDIUM_CEILING(s)						\
-	(((s) + mspace_mask) & ~mspace_mask)
-
-/*
- * Soft limit on the number of medium size classes.  Spacing between medium
- * size classes never exceeds pagesize, which can force more than NBINS_MAX
- * medium size classes.
- */
-#define	NMBINS_MAX	16
-
 /*
  * RUN_MAX_OVRHD indicates maximum desired run header overhead.  Runs are sized
  * as small as possible such that this setting is still honored, without
@@ -73,12 +56,6 @@
 #define	RUN_MAX_OVRHD		0x0000003dU
 #define	RUN_MAX_OVRHD_RELAX	0x00001800U
 
-/* Put a cap on small object run size.  This overrides RUN_MAX_OVRHD. */
-#define	RUN_MAX_SMALL							\
-	(arena_maxclass <= (1U << (CHUNK_MAP_LG_PG_RANGE + PAGE_SHIFT))	\
-	    ? arena_maxclass : (1U << (CHUNK_MAP_LG_PG_RANGE +		\
-	    PAGE_SHIFT)))
-
 /*
  * The minimum ratio of active:dirty pages per arena is computed as:
  *
@@ -101,14 +78,23 @@ typedef struct arena_s arena_t;
 
 /* Each element of the chunk map corresponds to one page within the chunk. */
 struct arena_chunk_map_s {
+	union {
 		/*
 		 * Linkage for run trees.  There are two disjoint uses:
 		 *
-	 * 1) arena_t's runs_avail tree.
+		 * 1) arena_t's runs_avail_{clean,dirty} trees.
-	 * 2) arena_run_t conceptually uses this linkage for in-use non-full
+		 * 2) arena_run_t conceptually uses this linkage for in-use
-	 *    runs, rather than directly embedding linkage.
+		 *    non-full runs, rather than directly embedding linkage.
 		 */
-	rb_node(arena_chunk_map_t)	link;
+		rb_node(arena_chunk_map_t)	rb_link;
+		/*
+		 * List of runs currently in purgatory.  arena_chunk_purge()
+		 * temporarily allocates runs that contain dirty pages while
+		 * purging, so that other threads cannot use the runs while the
+		 * purging thread is operating without the arena lock held.
+		 */
+		ql_elm(arena_chunk_map_t)	ql_link;
+	}				u;
 
 #ifdef JEMALLOC_PROF
 	/* Profile counters, used for large object runs. */
@@ -119,14 +105,13 @@ struct arena_chunk_map_s {
 	 * Run address (or size) and various flags are stored together.  The bit
 	 * layout looks like (assuming 32-bit system):
 	 *
-	 *   ???????? ???????? ????cccc ccccdzla
+	 *   ???????? ???????? ????---- ----dzla
 	 *
 	 * ? : Unallocated: Run address for first/last pages, unset for internal
 	 *                  pages.
-	 *     Small/medium: Don't care.
+	 *     Small: Run page offset.
 	 *     Large: Run size for first page, unset for trailing pages.
 	 * - : Unused.
-	 * c : refcount (could overflow for PAGE_SIZE >= 128 KiB)
 	 * d : dirty?
 	 * z : zeroed?
 	 * l : large?
@@ -136,39 +121,50 @@ struct arena_chunk_map_s {
 	 *
 	 * p : run page offset
 	 * s : run size
+	 * c : size class (used only if prof_promote is true)
 	 * x : don't care
 	 * - : 0
-	 * [dzla] : bit set
+	 * + : 1
+	 * [DZLA] : bit set
+	 * [dzla] : bit unset
 	 *
-	 *   Unallocated:
+	 *   Unallocated (clean):
-	 *     ssssssss ssssssss ssss---- --------
+	 *     ssssssss ssssssss ssss---- ----dz--
-	 *     xxxxxxxx xxxxxxxx xxxx---- ----d---
+	 *     xxxxxxxx xxxxxxxx xxxx---- -----Zxx
-	 *     ssssssss ssssssss ssss---- -----z--
+	 *     ssssssss ssssssss ssss---- ----dZ--
 	 *
-	 *   Small/medium:
+	 *   Unallocated (dirty):
-	 *     pppppppp ppppcccc cccccccc cccc---a
+	 *     ssssssss ssssssss ssss---- ----D---
-	 *     pppppppp ppppcccc cccccccc cccc---a
+	 *     xxxxxxxx xxxxxxxx xxxx---- ----xxxx
-	 *     pppppppp ppppcccc cccccccc cccc---a
+	 *     ssssssss ssssssss ssss---- ----D---
+	 *
+	 *   Small:
+	 *     pppppppp pppppppp pppp---- ----d--a
+	 *     pppppppp pppppppp pppp---- -------a
+	 *     pppppppp pppppppp pppp---- ----d--a
 	 *
 	 *   Large:
-	 *     ssssssss ssssssss ssss---- ------la
+	 *     ssssssss ssssssss ssss++++ ++++D-la
-	 *     -------- -------- -------- ------la
+	 *     xxxxxxxx xxxxxxxx xxxx---- ----xxxx
-	 *     -------- -------- -------- ------la
+	 *     -------- -------- -------- ----D-la
+	 *
+	 *   Large (sampled, size <= PAGE_SIZE):
+	 *     ssssssss ssssssss sssscccc ccccD-la
+	 *
+	 *   Large (not sampled, size == PAGE_SIZE):
+	 *     ssssssss ssssssss ssss++++ ++++D-la
 	 */
 	size_t				bits;
-#define	CHUNK_MAP_PG_MASK	((size_t)0xfff00000U)
+#ifdef JEMALLOC_PROF
-#define	CHUNK_MAP_PG_SHIFT	20
+#define	CHUNK_MAP_CLASS_SHIFT	4
-#define	CHUNK_MAP_LG_PG_RANGE	12
+#define	CHUNK_MAP_CLASS_MASK	((size_t)0xff0U)
-
+#endif
-#define	CHUNK_MAP_RC_MASK	((size_t)0xffff0U)
-#define	CHUNK_MAP_RC_ONE	((size_t)0x00010U)
-
 #define	CHUNK_MAP_FLAGS_MASK	((size_t)0xfU)
 #define	CHUNK_MAP_DIRTY		((size_t)0x8U)
 #define	CHUNK_MAP_ZEROED	((size_t)0x4U)
 #define	CHUNK_MAP_LARGE		((size_t)0x2U)
 #define	CHUNK_MAP_ALLOCATED	((size_t)0x1U)
-#define	CHUNK_MAP_KEY		(CHUNK_MAP_DIRTY | CHUNK_MAP_ALLOCATED)
+#define	CHUNK_MAP_KEY		CHUNK_MAP_ALLOCATED
 };
 typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
 typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
@@ -178,11 +174,11 @@ struct arena_chunk_s {
 	/* Arena that owns the chunk. */
 	arena_t		*arena;
 
-	/* Linkage for the arena's chunks_dirty tree. */
+	/* Linkage for the arena's chunks_dirty list. */
-	rb_node(arena_chunk_t) link_dirty;
+	ql_elm(arena_chunk_t) link_dirty;
 
 	/*
-	 * True if the chunk is currently in the chunks_dirty tree, due to
+	 * True if the chunk is currently in the chunks_dirty list, due to
 	 * having at some point contained one or more dirty pages.  Removal
 	 * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible.
 	 */
@@ -205,17 +201,25 @@ struct arena_run_s {
 	/* Bin this run is associated with. */
 	arena_bin_t	*bin;
 
-	/* Index of first element that might have a free region. */
+	/* Stack of available freed regions, or NULL. */
-	unsigned	regs_minelm;
+	void		*avail;
+
+	/* Next region that has never been allocated, or run boundary. */
+	void		*next;
 
 	/* Number of free regions in run. */
 	unsigned	nfree;
-
-	/* Bitmask of in-use regions (0: in use, 1: free). */
-	unsigned	regs_mask[1]; /* Dynamically sized. */
 };
 
 struct arena_bin_s {
+	/*
+	 * All operations on runcur, runs, and stats require that lock be
+	 * locked.  Run allocation/deallocation are protected by the arena lock,
+	 * which may be acquired while holding one or more bin locks, but not
+	 * vise versa.
+	 */
+	malloc_mutex_t	lock;
+
 	/*
 	 * Current run being used to service allocations of this bin's size
 	 * class.
@@ -240,9 +244,6 @@ struct arena_bin_s {
 	/* Total number of regions in a run for this bin's size class. */
 	uint32_t	nregs;
 
-	/* Number of elements in a run's regs_mask for this bin's size class. */
-	uint32_t	regs_mask_nelms;
-
 #ifdef JEMALLOC_PROF
 	/*
 	 * Offset of first (prof_cnt_t *) in a run header for this bin's size
@@ -269,7 +270,10 @@ struct arena_s {
 	/* This arena's index within the arenas array. */
 	unsigned		ind;
 
-	/* All operations on this arena require that lock be locked. */
+	/*
+	 * All non-bin-related operations on this arena require that lock be
+	 * locked.
+	 */
 	malloc_mutex_t		lock;
 
 #ifdef JEMALLOC_STATS
@@ -287,8 +291,8 @@ struct arena_s {
 	uint64_t		prof_accumbytes;
 #endif
 
-	/* Tree of dirty-page-containing chunks this arena manages. */
+	/* List of dirty-page-containing chunks this arena manages. */
-	arena_chunk_tree_t	chunks_dirty;
+	ql_head(arena_chunk_t)	chunks_dirty;
 
 	/*
 	 * In order to avoid rapid chunk allocation/deallocation when an arena
@@ -314,10 +318,27 @@ struct arena_s {
 	size_t			ndirty;
 
 	/*
-	 * Size/address-ordered tree of this arena's available runs.  This tree
+	 * Approximate number of pages being purged.  It is possible for
-	 * is used for first-best-fit run allocation.
+	 * multiple threads to purge dirty pages concurrently, and they use
+	 * npurgatory to indicate the total number of pages all threads are
+	 * attempting to purge.
 	 */
-	arena_avail_tree_t	runs_avail;
+	size_t			npurgatory;
+
+	/*
+	 * Size/address-ordered trees of this arena's available runs.  The trees
+	 * are used for first-best-fit run allocation.  The dirty tree contains
+	 * runs with dirty pages (i.e. very likely to have been touched and
+	 * therefore have associated physical pages), whereas the clean tree
+	 * contains runs with pages that either have no associated physical
+	 * pages, or have pages that the kernel may recycle at any time due to
+	 * previous madvise(2) calls.  The dirty tree is used in preference to
+	 * the clean tree for allocations, because using dirty pages reduces
+	 * the amount of dirty purging necessary to keep the active:dirty page
+	 * ratio below the purge threshold.
+	 */
+	arena_avail_tree_t	runs_avail_clean;
+	arena_avail_tree_t	runs_avail_dirty;
 
 	/*
 	 * bins is used to store trees of free regions of the following sizes,
@@ -371,7 +392,6 @@ struct arena_s {
 
 extern size_t	opt_lg_qspace_max;
 extern size_t	opt_lg_cspace_max;
-extern size_t	opt_lg_medium_max;
 extern ssize_t		opt_lg_dirty_mult;
 extern uint8_t const	*small_size2bin;
 
@@ -384,9 +404,7 @@ extern uint8_t const	*small_size2bin;
 extern unsigned		nqbins; /* Number of quantum-spaced bins. */
 extern unsigned		ncbins; /* Number of cacheline-spaced bins. */
 extern unsigned		nsbins; /* Number of subpage-spaced bins. */
-extern unsigned		nmbins; /* Number of medium bins. */
 extern unsigned		nbins;
-extern unsigned		mbin0; /* mbin offset (nbins - nmbins). */
 #ifdef JEMALLOC_TINY
 #  define		tspace_max	((size_t)(QUANTUM >> 1))
 #endif
@@ -397,18 +415,12 @@ extern size_t		cspace_max;
 extern size_t		sspace_min;
 extern size_t		sspace_max;
 #define			small_maxclass	sspace_max
-#define			medium_min	PAGE_SIZE
-extern size_t		medium_max;
-#define			bin_maxclass	medium_max
 
-/* Spacing between medium size classes. */
+#define			nlclasses (chunk_npages - arena_chunk_header_npages)
-extern size_t		lg_mspace;
-extern size_t		mspace_mask;
-
-#define			nlclasses	((chunksize - PAGE_SIZE) >> PAGE_SHIFT)
 
 #ifdef JEMALLOC_TCACHE
-void	arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
+void	arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
+    size_t binind
 #  ifdef JEMALLOC_PROF
     , uint64_t prof_accumbytes
 #  endif
@@ -418,12 +430,14 @@ void	arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
 void	arena_prof_accum(arena_t *arena, uint64_t accumbytes);
 #endif
 void	*arena_malloc_small(arena_t *arena, size_t size, bool zero);
-void	*arena_malloc_medium(arena_t *arena, size_t size, bool zero);
+void	*arena_malloc_large(arena_t *arena, size_t size, bool zero);
 void	*arena_malloc(size_t size, bool zero);
 void	*arena_palloc(arena_t *arena, size_t alignment, size_t size,
     size_t alloc_size);
 size_t	arena_salloc(const void *ptr);
 #ifdef JEMALLOC_PROF
+void	arena_prof_promoted(const void *ptr, size_t size);
+size_t	arena_salloc_demote(const void *ptr);
 prof_thr_cnt_t	*arena_prof_cnt_get(const void *ptr);
 void	arena_prof_cnt_set(const void *ptr, prof_thr_cnt_t *cnt);
 #endif
@@ -469,17 +483,53 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 		tcache_t *tcache;
 
 		if ((tcache = tcache_get()) != NULL)
-			tcache_dalloc(tcache, ptr);
+			tcache_dalloc_small(tcache, ptr);
 		else {
 #endif
-			malloc_mutex_lock(&arena->lock);
+			arena_run_t *run;
+			arena_bin_t *bin;
+
+			run = (arena_run_t *)((uintptr_t)chunk +
+			    (uintptr_t)((pageind - (mapelm->bits >>
+			    PAGE_SHIFT)) << PAGE_SHIFT));
+			assert(run->magic == ARENA_RUN_MAGIC);
+			assert(((uintptr_t)ptr - ((uintptr_t)run +
+			    (uintptr_t)run->bin->reg0_offset)) %
+			    run->bin->reg_size == 0);
+			bin = run->bin;
+			malloc_mutex_lock(&bin->lock);
 			arena_dalloc_bin(arena, chunk, ptr, mapelm);
-			malloc_mutex_unlock(&arena->lock);
+			malloc_mutex_unlock(&bin->lock);
 #ifdef JEMALLOC_TCACHE
 		}
 #endif
-	} else
+	} else {
+#ifdef JEMALLOC_TCACHE
+		size_t size = mapelm->bits & ~PAGE_MASK;
+
+		assert(((uintptr_t)ptr & PAGE_MASK) == 0);
+		if (size <= tcache_maxclass) {
+			tcache_t *tcache;
+
+			if ((tcache = tcache_get()) != NULL)
+				tcache_dalloc_large(tcache, ptr, size);
+			else {
+				malloc_mutex_lock(&arena->lock);
 				arena_dalloc_large(arena, chunk, ptr);
+				malloc_mutex_unlock(&arena->lock);
+			}
+		} else {
+			malloc_mutex_lock(&arena->lock);
+			arena_dalloc_large(arena, chunk, ptr);
+			malloc_mutex_unlock(&arena->lock);
+		}
+#else
+		assert(((uintptr_t)ptr & PAGE_MASK) == 0);
+		malloc_mutex_lock(&arena->lock);
+		arena_dalloc_large(arena, chunk, ptr);
+		malloc_mutex_unlock(&arena->lock);
+#endif
+	}
 }
 #endif
 

jemalloc/include/jemalloc/internal/ctl.h

@@ -33,6 +33,13 @@ struct ctl_arena_stats_s {
 	size_t			pdirty;
 #ifdef JEMALLOC_STATS
 	arena_stats_t		astats;
+
+	/* Aggregate stats for small size classes, based on bin stats. */
+	size_t			allocated_small;
+	uint64_t		nmalloc_small;
+	uint64_t		ndalloc_small;
+	uint64_t		nrequests_small;
+
 	malloc_bin_stats_t	*bstats;	/* nbins elements. */
 	malloc_large_stats_t	*lstats;	/* nlclasses elements. */
 #endif
@@ -75,17 +82,19 @@ bool	ctl_boot(void);
 #define	xmallctl(name, oldp, oldlenp, newp, newlen) do {		\
 	if (JEMALLOC_P(mallctl)(name, oldp, oldlenp, newp, newlen)	\
 	    != 0) {							\
-		malloc_write4("<jemalloc>: Invalid xmallctl(\"", name,	\
+		malloc_write("<jemalloc>: Invalid xmallctl(\"");	\
-		    "\", ...) call\n", "");				\
+		malloc_write(name);					\
+		malloc_write("\", ...) call\n");			\
 		abort();						\
 	}								\
 } while (0)
 
 #define	xmallctlnametomib(name, mibp, miblenp) do {			\
 	if (JEMALLOC_P(mallctlnametomib)(name, mibp, miblenp) != 0) {	\
-		malloc_write4(						\
+		malloc_write(						\
-		    "<jemalloc>: Invalid xmallctlnametomib(\"", name,	\
+		    "<jemalloc>: Invalid xmallctlnametomib(\"");	\
-		    "\", ...) call\n", "");				\
+		malloc_write(name);					\
+		malloc_write("\", ...) call\n");			\
 		abort();						\
 	}								\
 } while (0)
@@ -93,9 +102,8 @@ bool	ctl_boot(void);
 #define	xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do {	\
 	if (JEMALLOC_P(mallctlbymib)(mib, miblen, oldp, oldlenp, newp,	\
 	    newlen) != 0) {						\
-		malloc_write4(						\
+		malloc_write(						\
-		    "<jemalloc>: Invalid xmallctlbymib() call\n", "",	\
+		    "<jemalloc>: Invalid xmallctlbymib() call\n");	\
-		    "", "");						\
 		abort();						\
 	}								\
 } while (0)

jemalloc/include/jemalloc/internal/jemalloc_internal.h.in

@@ -36,8 +36,7 @@
 #include "jemalloc/internal/qr.h"
 #include "jemalloc/internal/ql.h"
 
-extern void	(*JEMALLOC_P(malloc_message))(void *w4opaque, const char *p1,
+extern void	(*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s);
-    const char *p2, const char *p3, const char *p4);
 
 /*
  * Define a custom assert() in order to reduce the chances of deadlock during
@@ -47,10 +46,14 @@ extern void	(*JEMALLOC_P(malloc_message))(void *w4opaque, const char *p1,
 #  define assert(e) do {						\
 	if (!(e)) {							\
 		char line_buf[UMAX2S_BUFSIZE];				\
-		malloc_write4("<jemalloc>: ", __FILE__, ":",		\
+		malloc_write("<jemalloc>: ");				\
-		    umax2s(__LINE__, 10, line_buf));			\
+		malloc_write(__FILE__);					\
-		malloc_write4(": Failed assertion: ", "\"", #e,		\
+		malloc_write(":");					\
-		    "\"\n");						\
+		malloc_write(umax2s(__LINE__, 10, line_buf));		\
+		malloc_write(": Failed assertion: ");			\
+		malloc_write("\"");					\
+		malloc_write(#e);					\
+		malloc_write("\"\n");					\
 		abort();						\
 	}								\
 } while (0)
@@ -282,8 +285,7 @@ arena_t	*choose_arena_hard(void);
 #include "jemalloc/internal/huge.h"
 
 #ifndef JEMALLOC_ENABLE_INLINE
-void	malloc_write4(const char *p1, const char *p2, const char *p3,
+void	malloc_write(const char *s);
-    const char *p4);
 arena_t	*choose_arena(void);
 #endif
 
@@ -293,10 +295,10 @@ arena_t	*choose_arena(void);
  * JEMALLOC_P(malloc_message)(...) throughout the code.
  */
 JEMALLOC_INLINE void
-malloc_write4(const char *p1, const char *p2, const char *p3, const char *p4)
+malloc_write(const char *s)
 {
 
-	JEMALLOC_P(malloc_message)(NULL, p1, p2, p3, p4);
+	JEMALLOC_P(malloc_message)(NULL, s);
 }
 
 /*
@@ -513,7 +515,11 @@ isalloc(const void *ptr)
 		/* Region. */
 		assert(chunk->arena->magic == ARENA_MAGIC);
 
+#ifdef JEMALLOC_PROF
+		ret = arena_salloc_demote(ptr);
+#else
 		ret = arena_salloc(ptr);
+#endif
 	} else
 		ret = huge_salloc(ptr);
 

jemalloc/include/jemalloc/internal/mutex.h

@@ -25,6 +25,7 @@ bool	malloc_mutex_init(malloc_mutex_t *mutex);
 
 #ifndef JEMALLOC_ENABLE_INLINE
 void	malloc_mutex_lock(malloc_mutex_t *mutex);
+bool	malloc_mutex_trylock(malloc_mutex_t *mutex);
 void	malloc_mutex_unlock(malloc_mutex_t *mutex);
 #endif
 
@@ -37,6 +38,16 @@ malloc_mutex_lock(malloc_mutex_t *mutex)
 		pthread_mutex_lock(mutex);
 }
 
+JEMALLOC_INLINE bool
+malloc_mutex_trylock(malloc_mutex_t *mutex)
+{
+
+	if (isthreaded)
+		return (pthread_mutex_trylock(mutex) != 0);
+	else
+		return (false);
+}
+
 JEMALLOC_INLINE void
 malloc_mutex_unlock(malloc_mutex_t *mutex)
 {

jemalloc/include/jemalloc/internal/prof.h

@@ -119,9 +119,16 @@ struct prof_ctx_s {
 #ifdef JEMALLOC_H_EXTERNS
 
 extern bool	opt_prof;
+/*
+ * Even if opt_prof is true, sampling can be temporarily disabled by setting
+ * opt_prof_active to false.  No locking is used when updating opt_prof_active,
+ * so there are no guarantees regarding how long it will take for all threads
+ * to notice state changes.
+ */
+extern bool	opt_prof_active;
 extern size_t	opt_lg_prof_bt_max; /* Maximum backtrace depth. */
 extern size_t	opt_lg_prof_sample; /* Mean bytes between samples. */
-extern size_t	opt_lg_prof_interval; /* lg(prof_interval). */
+extern ssize_t	opt_lg_prof_interval; /* lg(prof_interval). */
 extern bool	opt_prof_udump; /* High-water memory dumping. */
 extern bool	opt_prof_leak; /* Dump leak summary at exit. */
 
@@ -134,6 +141,12 @@ extern bool	opt_prof_leak; /* Dump leak summary at exit. */
  */
 extern uint64_t	prof_interval;
 
+/*
+ * If true, promote small sampled objects to large objects, since small run
+ * headers do not have embedded profile context pointers.
+ */
+extern bool	prof_promote;
+
 bool	prof_init(prof_t *prof, bool master);
 void	prof_destroy(prof_t *prof);
 

jemalloc/include/jemalloc/internal/stats.h

@@ -31,8 +31,24 @@ struct tcache_bin_stats_s {
 
 struct malloc_bin_stats_s {
 	/*
-	 * Number of allocation requests that corresponded to the size of this
+	 * Current number of bytes allocated, including objects currently
-	 * bin.
+	 * cached by tcache.
+	 */
+	size_t		allocated;
+
+	/*
+	 * Total number of allocation/deallocation requests served directly by
+	 * the bin.  Note that tcache may allocate an object, then recycle it
+	 * many times, resulting many increments to nrequests, but only one
+	 * each to nmalloc and ndalloc.
+	 */
+	uint64_t	nmalloc;
+	uint64_t	ndalloc;
+
+	/*
+	 * Number of allocation requests that correspond to the size of this
+	 * bin.  This includes requests served by tcache, though tcache only
+	 * periodically merges into this counter.
 	 */
 	uint64_t	nrequests;
 
@@ -62,7 +78,18 @@ struct malloc_bin_stats_s {
 
 struct malloc_large_stats_s {
 	/*
-	 * Number of allocation requests that corresponded to this size class.
+	 * Total number of allocation/deallocation requests served directly by
+	 * the arena.  Note that tcache may allocate an object, then recycle it
+	 * many times, resulting many increments to nrequests, but only one
+	 * each to nmalloc and ndalloc.
+	 */
+	uint64_t	nmalloc;
+	uint64_t	ndalloc;
+
+	/*
+	 * Number of allocation requests that correspond to this size class.
+	 * This includes requests served by tcache, though tcache only
+	 * periodically merges into this counter.
 	 */
 	uint64_t	nrequests;
 
@@ -87,17 +114,10 @@ struct arena_stats_s {
 	uint64_t	purged;
 
 	/* Per-size-category statistics. */
-	size_t		allocated_small;
-	uint64_t	nmalloc_small;
-	uint64_t	ndalloc_small;
-
-	size_t		allocated_medium;
-	uint64_t	nmalloc_medium;
-	uint64_t	ndalloc_medium;
-
 	size_t		allocated_large;
 	uint64_t	nmalloc_large;
 	uint64_t	ndalloc_large;
+	uint64_t	nrequests_large;
 
 	/*
 	 * One element for each possible size class, including sizes that
@@ -136,14 +156,13 @@ extern bool	opt_stats_print;
 
 char	*umax2s(uintmax_t x, unsigned base, char *s);
 #ifdef JEMALLOC_STATS
-void malloc_cprintf(void (*write4)(void *, const char *, const char *,
+void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque,
-    const char *, const char *), void *w4opaque, const char *format, ...)
+    const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4));
-    JEMALLOC_ATTR(format(printf, 3, 4));
 void	malloc_printf(const char *format, ...)
     JEMALLOC_ATTR(format(printf, 1, 2));
 #endif
-void	stats_print(void (*write4)(void *, const char *, const char *,
+void	stats_print(void (*write)(void *, const char *), void *cbopaque,
-    const char *, const char *), void *w4opaque, const char *opts);
+    const char *opts);
 
 #endif /* JEMALLOC_H_EXTERNS */
 /******************************************************************************/

jemalloc/include/jemalloc/internal/tcache.h

@@ -6,10 +6,20 @@ typedef struct tcache_bin_s tcache_bin_t;
 typedef struct tcache_s tcache_t;
 
 /*
- * Default number of cache slots for each bin in the thread cache (0:
+ * Absolute maximum number of cache slots for each small bin in the thread
- * disabled).
+ * cache.  This is an additional constraint beyond that imposed as: twice the
+ * number of regions per run for this size class.
+ *
+ * This constant must be an even number.
  */
-#define LG_TCACHE_NSLOTS_DEFAULT	7
+#define	TCACHE_NSLOTS_SMALL_MAX		200
+
+/* Number of cache slots for large size classes. */
+#define	TCACHE_NSLOTS_LARGE		20
+
+/* (1U << opt_lg_tcache_maxclass) is used to compute tcache_maxclass. */
+#define	LG_TCACHE_MAXCLASS_DEFAULT	15
+
 /*
  * (1U << opt_lg_tcache_gc_sweep) is the approximate number of allocation
  * events between full GC sweeps (-1: disabled).  Integer rounding may cause
@@ -29,7 +39,8 @@ struct tcache_bin_s {
 	unsigned	low_water;	/* Min # cached since last GC. */
 	unsigned	high_water;	/* Max # cached since last GC. */
 	unsigned	ncached;	/* # of cached objects. */
-	void		*slots[1];	/* Dynamically sized. */
+	unsigned	ncached_max;	/* Upper limit on ncached. */
+	void		*avail;		/* Chain of available objects. */
 };
 
 struct tcache_s {
@@ -42,14 +53,15 @@ struct tcache_s {
 	arena_t		*arena;		/* This thread's arena. */
 	unsigned	ev_cnt;		/* Event count since incremental GC. */
 	unsigned	next_gc_bin;	/* Next bin to GC. */
-	tcache_bin_t	*tbins[1];	/* Dynamically sized. */
+	tcache_bin_t	tbins[1];	/* Dynamically sized. */
 };
 
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
-extern size_t	opt_lg_tcache_nslots;
+extern bool	opt_tcache;
+extern ssize_t	opt_lg_tcache_maxclass;
 extern ssize_t	opt_lg_tcache_gc_sweep;
 
 /* Map of thread-specific caches. */
@@ -57,24 +69,30 @@ extern __thread tcache_t	*tcache_tls
     JEMALLOC_ATTR(tls_model("initial-exec"));
 
 /*
- * Number of cache slots for each bin in the thread cache, or 0 if tcache is
+ * Number of tcache bins.  There are nbins small-object bins, plus 0 or more
- * disabled.
+ * large-object bins.
  */
-extern size_t			tcache_nslots;
+extern size_t			nhbins;
+
+/* Maximum cached size class. */
+extern size_t			tcache_maxclass;
 
 /* Number of tcache allocation/deallocation events between incremental GCs. */
 extern unsigned			tcache_gc_incr;
 
-void	tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
+void	tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
-#ifdef JEMALLOC_PROF
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+    , tcache_t *tcache
+#endif
+    );
+void	tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
     , tcache_t *tcache
 #endif
     );
 tcache_t *tcache_create(arena_t *arena);
-void	tcache_bin_destroy(tcache_t *tcache, tcache_bin_t *tbin,
+void	*tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin,
-    unsigned binind);
+    size_t binind);
-void	*tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind);
-tcache_bin_t *tcache_bin_create(arena_t *arena);
 void	tcache_destroy(tcache_t *tcache);
 #ifdef JEMALLOC_STATS
 void	tcache_stats_merge(tcache_t *tcache, arena_t *arena);
@@ -88,9 +106,11 @@ void	tcache_boot(void);
 #ifndef JEMALLOC_ENABLE_INLINE
 void	tcache_event(tcache_t *tcache);
 tcache_t *tcache_get(void);
-void	*tcache_bin_alloc(tcache_bin_t *tbin);
+void	*tcache_alloc_easy(tcache_bin_t *tbin);
-void	*tcache_alloc(tcache_t *tcache, size_t size, bool zero);
+void	*tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
-void	tcache_dalloc(tcache_t *tcache, void *ptr);
+void	*tcache_alloc_large(tcache_t *tcache, size_t size, bool zero);
+void	tcache_dalloc_small(tcache_t *tcache, void *ptr);
+void	tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
@@ -99,7 +119,7 @@ tcache_get(void)
 {
 	tcache_t *tcache;
 
-	if (isthreaded == false || tcache_nslots == 0)
+	if ((isthreaded & opt_tcache) == false)
 		return (NULL);
 
 	tcache = tcache_tls;
@@ -124,85 +144,75 @@ tcache_event(tcache_t *tcache)
 
 	tcache->ev_cnt++;
 	assert(tcache->ev_cnt <= tcache_gc_incr);
-	if (tcache->ev_cnt >= tcache_gc_incr) {
+	if (tcache->ev_cnt == tcache_gc_incr) {
 		size_t binind = tcache->next_gc_bin;
-		tcache_bin_t *tbin = tcache->tbins[binind];
+		tcache_bin_t *tbin = &tcache->tbins[binind];
 
-		if (tbin != NULL) {
-			if (tbin->high_water == 0) {
-				/*
-				 * This bin went completely unused for an
-				 * entire GC cycle, so throw away the tbin.
-				 */
-				assert(tbin->ncached == 0);
-				tcache_bin_destroy(tcache, tbin, binind);
-				tcache->tbins[binind] = NULL;
-			} else {
 		if (tbin->low_water > 0) {
 			/*
-					 * Flush (ceiling) half of the objects
+			 * Flush (ceiling) 3/4 of the objects below the low
-					 * below the low water mark.
+			 * water mark.
 			 */
-					tcache_bin_flush(tbin, binind,
+			if (binind < nbins) {
-					    tbin->ncached - (tbin->low_water >>
+				tcache_bin_flush_small(tbin, binind,
-					    1) - (tbin->low_water & 1)
+				    tbin->ncached - tbin->low_water +
-#ifdef JEMALLOC_PROF
+				    (tbin->low_water >> 2)
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+				    , tcache
+#endif
+				    );
+			} else {
+				tcache_bin_flush_large(tbin, binind,
+				    tbin->ncached - tbin->low_water +
+				    (tbin->low_water >> 2)
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 				    , tcache
 #endif
 				    );
 			}
+		}
 		tbin->low_water = tbin->ncached;
 		tbin->high_water = tbin->ncached;
-			}
-		}
 
 		tcache->next_gc_bin++;
-		if (tcache->next_gc_bin == nbins)
+		if (tcache->next_gc_bin == nhbins)
 			tcache->next_gc_bin = 0;
 		tcache->ev_cnt = 0;
 	}
 }
 
 JEMALLOC_INLINE void *
-tcache_bin_alloc(tcache_bin_t *tbin)
+tcache_alloc_easy(tcache_bin_t *tbin)
 {
+	void *ret;
 
 	if (tbin->ncached == 0)
 		return (NULL);
 	tbin->ncached--;
 	if (tbin->ncached < tbin->low_water)
 		tbin->low_water = tbin->ncached;
-	return (tbin->slots[tbin->ncached]);
+	ret = tbin->avail;
+	tbin->avail = *(void **)ret;
+	return (ret);
 }
 
 JEMALLOC_INLINE void *
-tcache_alloc(tcache_t *tcache, size_t size, bool zero)
+tcache_alloc_small(tcache_t *tcache, size_t size, bool zero)
 {
 	void *ret;
-	tcache_bin_t *tbin;
 	size_t binind;
+	tcache_bin_t *tbin;
 
-	if (size <= small_maxclass)
 	binind = small_size2bin[size];
-	else {
-		binind = mbin0 + ((MEDIUM_CEILING(size) - medium_min) >>
-		    lg_mspace);
-	}
 	assert(binind < nbins);
-	tbin = tcache->tbins[binind];
+	tbin = &tcache->tbins[binind];
-	if (tbin == NULL) {
+	ret = tcache_alloc_easy(tbin);
-		tbin = tcache_bin_create(tcache->arena);
-		if (tbin == NULL)
-			return (NULL);
-		tcache->tbins[binind] = tbin;
-	}
-
-	ret = tcache_bin_alloc(tbin);
 	if (ret == NULL) {
-		ret = tcache_alloc_hard(tcache, tbin, binind);
+		ret = tcache_alloc_small_hard(tcache, tbin, binind);
 		if (ret == NULL)
 			return (NULL);
 	}
+	assert(arena_salloc(ret) == tcache->arena->bins[binind].reg_size);
 
 	if (zero == false) {
 #ifdef JEMALLOC_FILL
@@ -224,8 +234,58 @@ tcache_alloc(tcache_t *tcache, size_t size, bool zero)
 	return (ret);
 }
 
+JEMALLOC_INLINE void *
+tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
+{
+	void *ret;
+	size_t binind;
+	tcache_bin_t *tbin;
+
+	size = PAGE_CEILING(size);
+	assert(size <= tcache_maxclass);
+	binind = nbins + (size >> PAGE_SHIFT) - 1;
+	assert(binind < nhbins);
+	tbin = &tcache->tbins[binind];
+	ret = tcache_alloc_easy(tbin);
+	if (ret == NULL) {
+		/*
+		 * Only allocate one large object at a time, because it's quite
+		 * expensive to create one and not use it.
+		 */
+		ret = arena_malloc_large(tcache->arena, size, zero);
+		if (ret == NULL)
+			return (NULL);
+	} else {
+#ifdef JEMALLOC_PROF
+		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
+		size_t pageind = (unsigned)(((uintptr_t)ret - (uintptr_t)chunk)
+		    >> PAGE_SHIFT);
+		chunk->map[pageind].bits |= CHUNK_MAP_CLASS_MASK;
+#endif
+		if (zero == false) {
+#ifdef JEMALLOC_FILL
+			if (opt_junk)
+				memset(ret, 0xa5, size);
+			else if (opt_zero)
+				memset(ret, 0, size);
+#endif
+		} else
+			memset(ret, 0, size);
+
+#ifdef JEMALLOC_STATS
+		tbin->tstats.nrequests++;
+#endif
+#ifdef JEMALLOC_PROF
+		tcache->prof_accumbytes += size;
+#endif
+	}
+
+	tcache_event(tcache);
+	return (ret);
+}
+
 JEMALLOC_INLINE void
-tcache_dalloc(tcache_t *tcache, void *ptr)
+tcache_dalloc_small(tcache_t *tcache, void *ptr)
 {
 	arena_t *arena;
 	arena_chunk_t *chunk;
@@ -235,13 +295,14 @@ tcache_dalloc(tcache_t *tcache, void *ptr)
 	size_t pageind, binind;
 	arena_chunk_map_t *mapelm;
 
+	assert(arena_salloc(ptr) <= small_maxclass);
+
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	arena = chunk->arena;
 	pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT);
 	mapelm = &chunk->map[pageind];
 	run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
-	    ((mapelm->bits & CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) <<
+	    (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT));
-	    PAGE_SHIFT));
 	assert(run->magic == ARENA_RUN_MAGIC);
 	bin = run->bin;
 	binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) /
@@ -250,29 +311,62 @@ tcache_dalloc(tcache_t *tcache, void *ptr)
 
 #ifdef JEMALLOC_FILL
 	if (opt_junk)
-		memset(ptr, 0x5a, arena->bins[binind].reg_size);
+		memset(ptr, 0x5a, bin->reg_size);
 #endif
 
-	tbin = tcache->tbins[binind];
+	tbin = &tcache->tbins[binind];
-	if (tbin == NULL) {
+	if (tbin->ncached == tbin->ncached_max) {
-		tbin = tcache_bin_create(choose_arena());
+		tcache_bin_flush_small(tbin, binind, (tbin->ncached_max >> 1)
-		if (tbin == NULL) {
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-			malloc_mutex_lock(&arena->lock);
-			arena_dalloc_bin(arena, chunk, ptr, mapelm);
-			malloc_mutex_unlock(&arena->lock);
-			return;
-		}
-		tcache->tbins[binind] = tbin;
-	}
-
-	if (tbin->ncached == tcache_nslots)
-		tcache_bin_flush(tbin, binind, (tcache_nslots >> 1)
-#ifdef JEMALLOC_PROF
 		    , tcache
 #endif
 		    );
-	assert(tbin->ncached < tcache_nslots);
+	}
-	tbin->slots[tbin->ncached] = ptr;
+	assert(tbin->ncached < tbin->ncached_max);
+	*(void **)ptr = tbin->avail;
+	tbin->avail = ptr;
+	tbin->ncached++;
+	if (tbin->ncached > tbin->high_water)
+		tbin->high_water = tbin->ncached;
+
+	tcache_event(tcache);
+}
+
+JEMALLOC_INLINE void
+tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size)
+{
+	arena_t *arena;
+	arena_chunk_t *chunk;
+	size_t pageind, binind;
+	tcache_bin_t *tbin;
+	arena_chunk_map_t *mapelm;
+
+	assert((size & PAGE_MASK) == 0);
+	assert(arena_salloc(ptr) > small_maxclass);
+	assert(arena_salloc(ptr) <= tcache_maxclass);
+
+	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+	arena = chunk->arena;
+	pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT);
+	mapelm = &chunk->map[pageind];
+	binind = nbins + (size >> PAGE_SHIFT) - 1;
+
+#ifdef JEMALLOC_FILL
+	if (opt_junk)
+		memset(ptr, 0x5a, bin->reg_size);
+#endif
+
+	tbin = &tcache->tbins[binind];
+	if (tbin->ncached == tbin->ncached_max) {
+		tcache_bin_flush_large(tbin, binind, (tbin->ncached_max >> 1)
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+		    , tcache
+#endif
+		    );
+	}
+	assert(tbin->ncached < tbin->ncached_max);
+	*(void **)ptr = tbin->avail;
+	tbin->avail = ptr;
 	tbin->ncached++;
 	if (tbin->ncached > tbin->high_water)
 		tbin->high_water = tbin->ncached;

jemalloc/include/jemalloc/jemalloc.h.in

@@ -17,8 +17,7 @@ extern "C" {
 #endif
 
 extern const char	*JEMALLOC_P(malloc_options);
-extern void		(*JEMALLOC_P(malloc_message))(void *, const char *p1,
+extern void		(*JEMALLOC_P(malloc_message))(void *, const char *);
-    const char *p2, const char *p3, const char *p4);
 
 void	*JEMALLOC_P(malloc)(size_t size) JEMALLOC_ATTR(malloc);
 void	*JEMALLOC_P(calloc)(size_t num, size_t size) JEMALLOC_ATTR(malloc);
@@ -28,9 +27,8 @@ void	*JEMALLOC_P(realloc)(void *ptr, size_t size);
 void	JEMALLOC_P(free)(void *ptr);
 
 size_t	JEMALLOC_P(malloc_usable_size)(const void *ptr);
-void	JEMALLOC_P(malloc_stats_print)(void (*write4)(void *, const char *,
+void	JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *),
-    const char *, const char *, const char *), void *w4opaque,
+    void *cbopaque, const char *opts);
-    const char *opts);
 int	JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp,
     void *newp, size_t newlen);
 int	JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp,

jemalloc/include/jemalloc/jemalloc_defs.h.in

@@ -1,11 +1,6 @@
 #ifndef JEMALLOC_DEFS_H_
 #define	JEMALLOC_DEFS_H_
 
-/*
- * jemalloc version string.
- */
-#undef JEMALLOC_VERSION
-
 /*
  * If JEMALLOC_PREFIX is defined, it will cause all public APIs to be prefixed.
  * This makes it possible, with some care, to use multiple allocators
@@ -61,9 +56,9 @@
 #undef JEMALLOC_TINY
 
 /*
- * JEMALLOC_TCACHE enables a thread-specific caching layer for small and medium
+ * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects.
- * objects.  This makes it possible to allocate/deallocate objects without any
+ * This makes it possible to allocate/deallocate objects without any locking
- * locking when the cache is in the steady state.
+ * when the cache is in the steady state.
  */
 #undef JEMALLOC_TCACHE
 

jemalloc/src/chunk_mmap.c

@@ -52,8 +52,9 @@ pages_map(void *addr, size_t size)
 			char buf[STRERROR_BUF];
 
 			strerror_r(errno, buf, sizeof(buf));
-			malloc_write4("<jemalloc>", ": Error in munmap(): ",
+			malloc_write("<jemalloc>: Error in munmap(): ");
-			    buf, "\n");
+			malloc_write(buf);
+			malloc_write("\n");
 			if (opt_abort)
 				abort();
 		}
@@ -73,7 +74,9 @@ pages_unmap(void *addr, size_t size)
 		char buf[STRERROR_BUF];
 
 		strerror_r(errno, buf, sizeof(buf));
-		malloc_write4("<jemalloc>", ": Error in munmap(): ", buf, "\n");
+		malloc_write("<jemalloc>: Error in munmap(): ");
+		malloc_write(buf);
+		malloc_write("\n");
 		if (opt_abort)
 			abort();
 	}

jemalloc/src/chunk_swap.c

@@ -297,15 +297,17 @@ chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed)
 			char buf[STRERROR_BUF];
 
 			strerror_r(errno, buf, sizeof(buf));
-			malloc_write4("<jemalloc>",
+			malloc_write(
-			    ": Error in mmap(..., MAP_FIXED, ...): ",
+			    "<jemalloc>: Error in mmap(..., MAP_FIXED, ...): ");
-			    buf, "\n");
+			malloc_write(buf);
+			malloc_write("\n");
 			if (opt_abort)
 				abort();
 			if (munmap(vaddr, voff) == -1) {
 				strerror_r(errno, buf, sizeof(buf));
-				malloc_write4("<jemalloc>",
+				malloc_write("<jemalloc>: Error in munmap(): ");
-				    ": Error in munmap(): ", buf, "\n");
+				malloc_write(buf);
+				malloc_write("\n");
 			}
 			ret = true;
 			goto RETURN;

jemalloc/src/ctl.c

@@ -24,6 +24,12 @@ const ctl_node_t	*n##_index(const size_t *mib, size_t miblen,	\
 static bool	ctl_arena_init(ctl_arena_stats_t *astats);
 #endif
 static void	ctl_arena_clear(ctl_arena_stats_t *astats);
+#ifdef JEMALLOC_STATS
+static void	ctl_arena_stats_amerge(ctl_arena_stats_t *cstats,
+    arena_t *arena);
+static void	ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
+    ctl_arena_stats_t *astats);
+#endif
 static void	ctl_arena_refresh(arena_t *arena, unsigned i);
 static void	ctl_refresh(void);
 static bool	ctl_init(void);
@@ -64,11 +70,12 @@ CTL_PROTO(opt_xmalloc)
 CTL_PROTO(opt_zero)
 #endif
 #ifdef JEMALLOC_TCACHE
-CTL_PROTO(opt_lg_tcache_nslots)
+CTL_PROTO(opt_tcache)
 CTL_PROTO(opt_lg_tcache_gc_sweep)
 #endif
 #ifdef JEMALLOC_PROF
 CTL_PROTO(opt_prof)
+CTL_PROTO(opt_prof_active)
 CTL_PROTO(opt_lg_prof_bt_max)
 CTL_PROTO(opt_lg_prof_sample)
 CTL_PROTO(opt_lg_prof_interval)
@@ -78,7 +85,6 @@ CTL_PROTO(opt_prof_leak)
 CTL_PROTO(opt_stats_print)
 CTL_PROTO(opt_lg_qspace_max)
 CTL_PROTO(opt_lg_cspace_max)
-CTL_PROTO(opt_lg_medium_max)
 CTL_PROTO(opt_lg_dirty_mult)
 CTL_PROTO(opt_lg_chunk)
 #ifdef JEMALLOC_SWAP
@@ -96,7 +102,6 @@ CTL_PROTO(arenas_quantum)
 CTL_PROTO(arenas_cacheline)
 CTL_PROTO(arenas_subpage)
 CTL_PROTO(arenas_pagesize)
-CTL_PROTO(arenas_medium)
 CTL_PROTO(arenas_chunksize)
 #ifdef JEMALLOC_TINY
 CTL_PROTO(arenas_tspace_min)
@@ -108,16 +113,20 @@ CTL_PROTO(arenas_cspace_min)
 CTL_PROTO(arenas_cspace_max)
 CTL_PROTO(arenas_sspace_min)
 CTL_PROTO(arenas_sspace_max)
-CTL_PROTO(arenas_medium_min)
+#ifdef JEMALLOC_TCACHE
-CTL_PROTO(arenas_medium_max)
+CTL_PROTO(arenas_tcache_max)
+#endif
 CTL_PROTO(arenas_ntbins)
 CTL_PROTO(arenas_nqbins)
 CTL_PROTO(arenas_ncbins)
 CTL_PROTO(arenas_nsbins)
-CTL_PROTO(arenas_nmbins)
 CTL_PROTO(arenas_nbins)
+#ifdef JEMALLOC_TCACHE
+CTL_PROTO(arenas_nhbins)
+#endif
 CTL_PROTO(arenas_nlruns)
 #ifdef JEMALLOC_PROF
+CTL_PROTO(prof_active)
 CTL_PROTO(prof_dump)
 CTL_PROTO(prof_interval)
 #endif
@@ -131,12 +140,14 @@ CTL_PROTO(stats_huge_ndalloc)
 CTL_PROTO(stats_arenas_i_small_allocated)
 CTL_PROTO(stats_arenas_i_small_nmalloc)
 CTL_PROTO(stats_arenas_i_small_ndalloc)
-CTL_PROTO(stats_arenas_i_medium_allocated)
+CTL_PROTO(stats_arenas_i_small_nrequests)
-CTL_PROTO(stats_arenas_i_medium_nmalloc)
-CTL_PROTO(stats_arenas_i_medium_ndalloc)
 CTL_PROTO(stats_arenas_i_large_allocated)
 CTL_PROTO(stats_arenas_i_large_nmalloc)
 CTL_PROTO(stats_arenas_i_large_ndalloc)
+CTL_PROTO(stats_arenas_i_large_nrequests)
+CTL_PROTO(stats_arenas_i_bins_j_allocated)
+CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
+CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
 CTL_PROTO(stats_arenas_i_bins_j_nrequests)
 #ifdef JEMALLOC_TCACHE
 CTL_PROTO(stats_arenas_i_bins_j_nfills)
@@ -147,6 +158,8 @@ CTL_PROTO(stats_arenas_i_bins_j_nreruns)
 CTL_PROTO(stats_arenas_i_bins_j_highruns)
 CTL_PROTO(stats_arenas_i_bins_j_curruns)
 INDEX_PROTO(stats_arenas_i_bins_j)
+CTL_PROTO(stats_arenas_i_lruns_j_nmalloc)
+CTL_PROTO(stats_arenas_i_lruns_j_ndalloc)
 CTL_PROTO(stats_arenas_i_lruns_j_nrequests)
 CTL_PROTO(stats_arenas_i_lruns_j_highruns)
 CTL_PROTO(stats_arenas_i_lruns_j_curruns)
@@ -230,11 +243,12 @@ static const ctl_node_t opt_node[] = {
 	{NAME("zero"),			CTL(opt_zero)},
 #endif
 #ifdef JEMALLOC_TCACHE
-	{NAME("lg_tcache_nslots"),	CTL(opt_lg_tcache_nslots)},
+	{NAME("tcache"),		CTL(opt_tcache)},
 	{NAME("lg_tcache_gc_sweep"),	CTL(opt_lg_tcache_gc_sweep)},
 #endif
 #ifdef JEMALLOC_PROF
 	{NAME("prof"),			CTL(opt_prof)},
+	{NAME("prof_active"),		CTL(opt_prof_active)},
 	{NAME("lg_prof_bt_max"),	CTL(opt_lg_prof_bt_max)},
 	{NAME("lg_prof_sample"),	CTL(opt_lg_prof_sample)},
 	{NAME("lg_prof_interval"),	CTL(opt_lg_prof_interval)},
@@ -244,7 +258,6 @@ static const ctl_node_t opt_node[] = {
 	{NAME("stats_print"),		CTL(opt_stats_print)},
 	{NAME("lg_qspace_max"),		CTL(opt_lg_qspace_max)},
 	{NAME("lg_cspace_max"),		CTL(opt_lg_cspace_max)},
-	{NAME("lg_medium_max"),		CTL(opt_lg_medium_max)},
 	{NAME("lg_dirty_mult"),		CTL(opt_lg_dirty_mult)},
 	{NAME("lg_chunk"),		CTL(opt_lg_chunk)}
 #ifdef JEMALLOC_SWAP
@@ -284,7 +297,6 @@ static const ctl_node_t arenas_node[] = {
 	{NAME("cacheline"),		CTL(arenas_cacheline)},
 	{NAME("subpage"),		CTL(arenas_subpage)},
 	{NAME("pagesize"),		CTL(arenas_pagesize)},
-	{NAME("medium"),		CTL(arenas_medium)},
 	{NAME("chunksize"),		CTL(arenas_chunksize)},
 #ifdef JEMALLOC_TINY
 	{NAME("tspace_min"),		CTL(arenas_tspace_min)},
@@ -296,14 +308,17 @@ static const ctl_node_t arenas_node[] = {
 	{NAME("cspace_max"),		CTL(arenas_cspace_max)},
 	{NAME("sspace_min"),		CTL(arenas_sspace_min)},
 	{NAME("sspace_max"),		CTL(arenas_sspace_max)},
-	{NAME("medium_min"),		CTL(arenas_medium_min)},
+#ifdef JEMALLOC_TCACHE
-	{NAME("medium_max"),		CTL(arenas_medium_max)},
+	{NAME("tcache_max"),		CTL(arenas_tcache_max)},
+#endif
 	{NAME("ntbins"),		CTL(arenas_ntbins)},
 	{NAME("nqbins"),		CTL(arenas_nqbins)},
 	{NAME("ncbins"),		CTL(arenas_ncbins)},
 	{NAME("nsbins"),		CTL(arenas_nsbins)},
-	{NAME("nmbins"),		CTL(arenas_nmbins)},
 	{NAME("nbins"),			CTL(arenas_nbins)},
+#ifdef JEMALLOC_TCACHE
+	{NAME("nhbins"),		CTL(arenas_nhbins)},
+#endif
 	{NAME("bin"),			CHILD(arenas_bin)},
 	{NAME("nlruns"),		CTL(arenas_nlruns)},
 	{NAME("lrun"),			CHILD(arenas_lrun)}
@@ -311,6 +326,7 @@ static const ctl_node_t arenas_node[] = {
 
 #ifdef JEMALLOC_PROF
 static const ctl_node_t	prof_node[] = {
+	{NAME("active"),	CTL(prof_active)},
 	{NAME("dump"),		CTL(prof_dump)},
 	{NAME("interval"),	CTL(prof_interval)}
 };
@@ -332,22 +348,21 @@ static const ctl_node_t stats_huge_node[] = {
 static const ctl_node_t stats_arenas_i_small_node[] = {
 	{NAME("allocated"),		CTL(stats_arenas_i_small_allocated)},
 	{NAME("nmalloc"),		CTL(stats_arenas_i_small_nmalloc)},
-	{NAME("ndalloc"),		CTL(stats_arenas_i_small_ndalloc)}
+	{NAME("ndalloc"),		CTL(stats_arenas_i_small_ndalloc)},
-};
+	{NAME("nrequests"),		CTL(stats_arenas_i_small_nrequests)}
-
-static const ctl_node_t stats_arenas_i_medium_node[] = {
-	{NAME("allocated"),		CTL(stats_arenas_i_medium_allocated)},
-	{NAME("nmalloc"),		CTL(stats_arenas_i_medium_nmalloc)},
-	{NAME("ndalloc"),		CTL(stats_arenas_i_medium_ndalloc)}
 };
 
 static const ctl_node_t stats_arenas_i_large_node[] = {
 	{NAME("allocated"),		CTL(stats_arenas_i_large_allocated)},
 	{NAME("nmalloc"),		CTL(stats_arenas_i_large_nmalloc)},
-	{NAME("ndalloc"),		CTL(stats_arenas_i_large_ndalloc)}
+	{NAME("ndalloc"),		CTL(stats_arenas_i_large_ndalloc)},
+	{NAME("nrequests"),		CTL(stats_arenas_i_large_nrequests)}
 };
 
 static const ctl_node_t stats_arenas_i_bins_j_node[] = {
+	{NAME("allocated"),		CTL(stats_arenas_i_bins_j_allocated)},
+	{NAME("nmalloc"),		CTL(stats_arenas_i_bins_j_nmalloc)},
+	{NAME("ndalloc"),		CTL(stats_arenas_i_bins_j_ndalloc)},
 	{NAME("nrequests"),		CTL(stats_arenas_i_bins_j_nrequests)},
 #ifdef JEMALLOC_TCACHE
 	{NAME("nfills"),		CTL(stats_arenas_i_bins_j_nfills)},
@@ -367,6 +382,8 @@ static const ctl_node_t stats_arenas_i_bins_node[] = {
 };
 
 static const ctl_node_t stats_arenas_i_lruns_j_node[] = {
+	{NAME("nmalloc"),		CTL(stats_arenas_i_lruns_j_nmalloc)},
+	{NAME("ndalloc"),		CTL(stats_arenas_i_lruns_j_ndalloc)},
 	{NAME("nrequests"),		CTL(stats_arenas_i_lruns_j_nrequests)},
 	{NAME("highruns"),		CTL(stats_arenas_i_lruns_j_highruns)},
 	{NAME("curruns"),		CTL(stats_arenas_i_lruns_j_curruns)}
@@ -390,7 +407,6 @@ static const ctl_node_t stats_arenas_i_node[] = {
 	{NAME("nmadvise"),		CTL(stats_arenas_i_nmadvise)},
 	{NAME("purged"),		CTL(stats_arenas_i_purged)},
 	{NAME("small"),			CHILD(stats_arenas_i_small)},
-	{NAME("medium"),		CHILD(stats_arenas_i_medium)},
 	{NAME("large"),			CHILD(stats_arenas_i_large)},
 	{NAME("bins"),			CHILD(stats_arenas_i_bins)},
 	{NAME("lruns"),		CHILD(stats_arenas_i_lruns)}
@@ -485,11 +501,80 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
 	astats->pdirty = 0;
 #ifdef JEMALLOC_STATS
 	memset(&astats->astats, 0, sizeof(arena_stats_t));
+	astats->allocated_small = 0;
+	astats->nmalloc_small = 0;
+	astats->ndalloc_small = 0;
+	astats->nrequests_small = 0;
 	memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t));
 	memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t));
 #endif
 }
 
+#ifdef JEMALLOC_STATS
+static void
+ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
+{
+	unsigned i;
+
+	arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty,
+	    &cstats->astats, cstats->bstats, cstats->lstats);
+
+	for (i = 0; i < nbins; i++) {
+		cstats->allocated_small += cstats->bstats[i].allocated;
+		cstats->nmalloc_small += cstats->bstats[i].nmalloc;
+		cstats->ndalloc_small += cstats->bstats[i].ndalloc;
+		cstats->nrequests_small += cstats->bstats[i].nrequests;
+	}
+}
+
+static void
+ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
+{
+	unsigned i;
+
+	sstats->pactive += astats->pactive;
+	sstats->pdirty += astats->pdirty;
+
+	sstats->astats.mapped += astats->astats.mapped;
+	sstats->astats.npurge += astats->astats.npurge;
+	sstats->astats.nmadvise += astats->astats.nmadvise;
+	sstats->astats.purged += astats->astats.purged;
+
+	sstats->allocated_small += astats->allocated_small;
+	sstats->nmalloc_small += astats->nmalloc_small;
+	sstats->ndalloc_small += astats->ndalloc_small;
+	sstats->nrequests_small += astats->nrequests_small;
+
+	sstats->astats.allocated_large += astats->astats.allocated_large;
+	sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
+	sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
+	sstats->astats.nrequests_large += astats->astats.nrequests_large;
+
+	for (i = 0; i < nlclasses; i++) {
+		sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc;
+		sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc;
+		sstats->lstats[i].nrequests += astats->lstats[i].nrequests;
+		sstats->lstats[i].highruns += astats->lstats[i].highruns;
+		sstats->lstats[i].curruns += astats->lstats[i].curruns;
+	}
+
+	for (i = 0; i < nbins; i++) {
+		sstats->bstats[i].allocated += astats->bstats[i].allocated;
+		sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
+		sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;
+		sstats->bstats[i].nrequests += astats->bstats[i].nrequests;
+#ifdef JEMALLOC_TCACHE
+		sstats->bstats[i].nfills += astats->bstats[i].nfills;
+		sstats->bstats[i].nflushes += astats->bstats[i].nflushes;
+#endif
+		sstats->bstats[i].nruns += astats->bstats[i].nruns;
+		sstats->bstats[i].reruns += astats->bstats[i].reruns;
+		sstats->bstats[i].highruns += astats->bstats[i].highruns;
+		sstats->bstats[i].curruns += astats->bstats[i].curruns;
+	}
+}
+#endif
+
 static void
 ctl_arena_refresh(arena_t *arena, unsigned i)
 {
@@ -498,13 +583,10 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
 
 	ctl_arena_clear(astats);
 
-	malloc_mutex_lock(&arena->lock);
 #ifdef JEMALLOC_STATS
-	arena_stats_merge(arena, &astats->pactive, &astats->pdirty,
+	ctl_arena_stats_amerge(astats, arena);
-	    &astats->astats, astats->bstats, astats->lstats);
 	/* Merge into sum stats as well. */
-	arena_stats_merge(arena, &sstats->pactive, &sstats->pdirty,
+	ctl_arena_stats_smerge(sstats, astats);
-	    &sstats->astats, sstats->bstats, sstats->lstats);
 #else
 	astats->pactive += arena->nactive;
 	astats->pdirty += arena->ndirty;
@@ -512,7 +594,6 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
 	sstats->pactive += arena->nactive;
 	sstats->pdirty += arena->ndirty;
 #endif
-	malloc_mutex_unlock(&arena->lock);
 }
 
 static void
@@ -553,8 +634,7 @@ ctl_refresh(void)
 	}
 
 #ifdef JEMALLOC_STATS
-	ctl_stats.allocated = ctl_stats.arenas[narenas].astats.allocated_small
+	ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small
-	    + ctl_stats.arenas[narenas].astats.allocated_medium
 	    + ctl_stats.arenas[narenas].astats.allocated_large
 	    + ctl_stats.huge.allocated;
 	ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT)
@@ -1070,21 +1150,21 @@ CTL_RO_GEN(opt_xmalloc, opt_xmalloc, bool)
 CTL_RO_GEN(opt_zero, opt_zero, bool)
 #endif
 #ifdef JEMALLOC_TCACHE
-CTL_RO_GEN(opt_lg_tcache_nslots, opt_lg_tcache_nslots, size_t)
+CTL_RO_GEN(opt_tcache, opt_tcache, bool)
 CTL_RO_GEN(opt_lg_tcache_gc_sweep, opt_lg_tcache_gc_sweep, ssize_t)
 #endif
 #ifdef JEMALLOC_PROF
 CTL_RO_GEN(opt_prof, opt_prof, bool)
+CTL_RO_GEN(opt_prof_active, opt_prof_active, bool)
 CTL_RO_GEN(opt_lg_prof_bt_max, opt_lg_prof_bt_max, size_t)
 CTL_RO_GEN(opt_lg_prof_sample, opt_lg_prof_sample, size_t)
-CTL_RO_GEN(opt_lg_prof_interval, opt_lg_prof_interval, size_t)
+CTL_RO_GEN(opt_lg_prof_interval, opt_lg_prof_interval, ssize_t)
 CTL_RO_GEN(opt_prof_udump, opt_prof_udump, bool)
 CTL_RO_GEN(opt_prof_leak, opt_prof_leak, bool)
 #endif
 CTL_RO_GEN(opt_stats_print, opt_stats_print, bool)
 CTL_RO_GEN(opt_lg_qspace_max, opt_lg_qspace_max, size_t)
 CTL_RO_GEN(opt_lg_cspace_max, opt_lg_cspace_max, size_t)
-CTL_RO_GEN(opt_lg_medium_max, opt_lg_medium_max, size_t)
 CTL_RO_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t)
 CTL_RO_GEN(opt_lg_chunk, opt_lg_chunk, size_t)
 #ifdef JEMALLOC_SWAP
@@ -1145,7 +1225,6 @@ CTL_RO_GEN(arenas_quantum, QUANTUM, size_t)
 CTL_RO_GEN(arenas_cacheline, CACHELINE, size_t)
 CTL_RO_GEN(arenas_subpage, SUBPAGE, size_t)
 CTL_RO_GEN(arenas_pagesize, PAGE_SIZE, size_t)
-CTL_RO_GEN(arenas_medium, (1U << lg_mspace), size_t)
 CTL_RO_GEN(arenas_chunksize, chunksize, size_t)
 #ifdef JEMALLOC_TINY
 CTL_RO_GEN(arenas_tspace_min, (1U << LG_TINY_MIN), size_t)
@@ -1157,19 +1236,46 @@ CTL_RO_GEN(arenas_cspace_min, cspace_min, size_t)
 CTL_RO_GEN(arenas_cspace_max, cspace_max, size_t)
 CTL_RO_GEN(arenas_sspace_min, sspace_min, size_t)
 CTL_RO_GEN(arenas_sspace_max, sspace_max, size_t)
-CTL_RO_GEN(arenas_medium_min, medium_min, size_t)
+#ifdef JEMALLOC_TCACHE
-CTL_RO_GEN(arenas_medium_max, medium_max, size_t)
+CTL_RO_GEN(arenas_tcache_max, tcache_maxclass, size_t)
+#endif
 CTL_RO_GEN(arenas_ntbins, ntbins, unsigned)
 CTL_RO_GEN(arenas_nqbins, nqbins, unsigned)
 CTL_RO_GEN(arenas_ncbins, ncbins, unsigned)
 CTL_RO_GEN(arenas_nsbins, nsbins, unsigned)
-CTL_RO_GEN(arenas_nmbins, nmbins, unsigned)
 CTL_RO_GEN(arenas_nbins, nbins, unsigned)
+#ifdef JEMALLOC_TCACHE
+CTL_RO_GEN(arenas_nhbins, nhbins, unsigned)
+#endif
 CTL_RO_GEN(arenas_nlruns, nlclasses, size_t)
 
 /******************************************************************************/
 
 #ifdef JEMALLOC_PROF
+static int
+prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
+    void *newp, size_t newlen)
+{
+	int ret;
+	bool oldval;
+
+	oldval = opt_prof_active;
+	if (newp != NULL) {
+		/*
+		 * The memory barriers will tend to make opt_prof_active
+		 * propagate faster on systems with weak memory ordering.
+		 */
+		mb_write();
+		WRITE(opt_prof_active, bool);
+		mb_write();
+	}
+	READ(oldval, bool);
+
+	ret = 0;
+RETURN:
+	return (ret);
+}
+
 static int
 prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
     void *newp, size_t newlen)
@@ -1203,24 +1309,28 @@ CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t)
 CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t)
 CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t)
 CTL_RO_GEN(stats_arenas_i_small_allocated,
-    ctl_stats.arenas[mib[2]].astats.allocated_small, size_t)
+    ctl_stats.arenas[mib[2]].allocated_small, size_t)
 CTL_RO_GEN(stats_arenas_i_small_nmalloc,
-    ctl_stats.arenas[mib[2]].astats.nmalloc_small, uint64_t)
+    ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t)
 CTL_RO_GEN(stats_arenas_i_small_ndalloc,
-    ctl_stats.arenas[mib[2]].astats.ndalloc_small, uint64_t)
+    ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
-CTL_RO_GEN(stats_arenas_i_medium_allocated,
+CTL_RO_GEN(stats_arenas_i_small_nrequests,
-    ctl_stats.arenas[mib[2]].astats.allocated_medium, size_t)
+    ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
-CTL_RO_GEN(stats_arenas_i_medium_nmalloc,
-    ctl_stats.arenas[mib[2]].astats.nmalloc_medium, uint64_t)
-CTL_RO_GEN(stats_arenas_i_medium_ndalloc,
-    ctl_stats.arenas[mib[2]].astats.ndalloc_medium, uint64_t)
 CTL_RO_GEN(stats_arenas_i_large_allocated,
     ctl_stats.arenas[mib[2]].astats.allocated_large, size_t)
 CTL_RO_GEN(stats_arenas_i_large_nmalloc,
     ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t)
 CTL_RO_GEN(stats_arenas_i_large_ndalloc,
     ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t)
+CTL_RO_GEN(stats_arenas_i_large_nrequests,
+    ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t)
 
+CTL_RO_GEN(stats_arenas_i_bins_j_allocated,
+    ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t)
+CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc,
+    ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t)
+CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc,
+    ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t)
 CTL_RO_GEN(stats_arenas_i_bins_j_nrequests,
     ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t)
 #ifdef JEMALLOC_TCACHE
@@ -1247,6 +1357,10 @@ stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j)
 	return (super_stats_arenas_i_bins_j_node);
 }
 
+CTL_RO_GEN(stats_arenas_i_lruns_j_nmalloc,
+    ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t)
+CTL_RO_GEN(stats_arenas_i_lruns_j_ndalloc,
+    ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t)
 CTL_RO_GEN(stats_arenas_i_lruns_j_nrequests,
     ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t)
 CTL_RO_GEN(stats_arenas_i_lruns_j_curruns,

jemalloc/src/jemalloc.c

@@ -52,17 +52,9 @@
  *   |          |                  |     3584 |
  *   |          |                  |     3840 |
  *   |========================================|
- *   | Medium                      |    4 KiB |
+ *   | Large                       |    4 KiB |
- *   |                             |    6 KiB |
  *   |                             |    8 KiB |
- *   |                             |      ... |
+ *   |                             |   12 KiB |
- *   |                             |   28 KiB |
- *   |                             |   30 KiB |
- *   |                             |   32 KiB |
- *   |========================================|
- *   | Large                       |   36 KiB |
- *   |                             |   40 KiB |
- *   |                             |   44 KiB |
  *   |                             |      ... |
  *   |                             | 1012 KiB |
  *   |                             | 1016 KiB |
@@ -76,9 +68,8 @@
  *
  * Different mechanisms are used accoding to category:
  *
- *   Small/medium : Each size class is segregated into its own set of runs.
+ *   Small: Each size class is segregated into its own set of runs.  Each run
- *                  Each run maintains a bitmap of which regions are
+ *          maintains a bitmap of which regions are free/allocated.
- *                  free/allocated.
  *
  *   Large : Each allocation is backed by a dedicated run.  Metadata are stored
  *           in the associated arena chunk header maps.
@@ -151,8 +142,7 @@ static int	opt_narenas_lshift = 0;
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
-static void	wrtmessage(void *w4opaque, const char *p1, const char *p2,
+static void	wrtmessage(void *cbopaque, const char *s);
-    const char *p3, const char *p4);
 static void	stats_print_atexit(void);
 static unsigned	malloc_ncpus(void);
 static bool	malloc_init_hard(void);
@@ -168,20 +158,14 @@ JEMALLOC_ATTR(visibility("hidden"))
 static
 #endif
 void
-wrtmessage(void *w4opaque, const char *p1, const char *p2, const char *p3,
+wrtmessage(void *cbopaque, const char *s)
-    const char *p4)
 {
 
-	if (write(STDERR_FILENO, p1, strlen(p1)) < 0
+	write(STDERR_FILENO, s, strlen(s));
-	    || write(STDERR_FILENO, p2, strlen(p2)) < 0
-	    || write(STDERR_FILENO, p3, strlen(p3)) < 0
-	    || write(STDERR_FILENO, p4, strlen(p4)) < 0)
-		return;
 }
 
-void	(*JEMALLOC_P(malloc_message))(void *, const char *p1, const char *p2,
+void	(*JEMALLOC_P(malloc_message))(void *, const char *s)
-    const char *p3, const char *p4) JEMALLOC_ATTR(visibility("default")) =
+    JEMALLOC_ATTR(visibility("default")) = wrtmessage;
-    wrtmessage;
 
 /******************************************************************************/
 /*
@@ -209,7 +193,7 @@ arenas_extend(unsigned ind)
 	 * by using arenas[0].  In practice, this is an extremely unlikely
 	 * failure.
 	 */
-	malloc_write4("<jemalloc>", ": Error initializing arena\n", "", "");
+	malloc_write("<jemalloc>: Error initializing arena\n");
 	if (opt_abort)
 		abort();
 
@@ -259,6 +243,11 @@ stats_print_atexit(void)
 		if (arena != NULL) {
 			tcache_t *tcache;
 
+			/*
+			 * tcache_stats_merge() locks bins, so if any code is
+			 * introduced that acquires both arena and bin locks in
+			 * the opposite order, deadlocks may result.
+			 */
 			malloc_mutex_lock(&arena->lock);
 			ql_foreach(tcache, &arena->tcache_ql, link) {
 				tcache_stats_merge(tcache, arena);
@@ -471,6 +460,12 @@ MALLOC_OUT:
 						opt_lg_dirty_mult--;
 					break;
 #ifdef JEMALLOC_PROF
+				case 'e':
+					opt_prof_active = false;
+					break;
+				case 'E':
+					opt_prof_active = true;
+					break;
 				case 'f':
 					opt_prof = false;
 					break;
@@ -489,18 +484,15 @@ MALLOC_OUT:
 						opt_lg_tcache_gc_sweep++;
 					break;
 				case 'h':
-					if (opt_lg_tcache_nslots > 0)
+					opt_tcache = false;
-						opt_lg_tcache_nslots--;
 					break;
 				case 'H':
-					if (opt_lg_tcache_nslots + 1 <
+					opt_tcache = true;
-					    (sizeof(size_t) << 3))
-						opt_lg_tcache_nslots++;
 					break;
 #endif
 #ifdef JEMALLOC_PROF
 				case 'i':
-					if (opt_lg_prof_interval > 0)
+					if (opt_lg_prof_interval >= 0)
 						opt_lg_prof_interval--;
 					break;
 				case 'I':
@@ -520,14 +512,10 @@ MALLOC_OUT:
 				case 'k':
 					/*
 					 * Chunks always require at least one
-					 * header page, plus enough room to
+					 * header page, plus one data page.
-					 * hold a run for the largest medium
-					 * size class (one page more than the
-					 * size).
 					 */
 					if ((1U << (opt_lg_chunk - 1)) >=
-					    (2U << PAGE_SHIFT) + (1U <<
+					    (2U << PAGE_SHIFT))
-					    opt_lg_medium_max))
 						opt_lg_chunk--;
 					break;
 				case 'K':
@@ -543,15 +531,17 @@ MALLOC_OUT:
 					opt_prof_leak = true;
 					break;
 #endif
+#ifdef JEMALLOC_TCACHE
 				case 'm':
-					if (opt_lg_medium_max > PAGE_SHIFT)
+					if (opt_lg_tcache_maxclass >= 0)
-						opt_lg_medium_max--;
+						opt_lg_tcache_maxclass--;
 					break;
 				case 'M':
-					if (opt_lg_medium_max + 1 <
+					if (opt_lg_tcache_maxclass + 1 <
-					    opt_lg_chunk)
+					    (sizeof(size_t) << 3))
-						opt_lg_medium_max++;
+						opt_lg_tcache_maxclass++;
 					break;
+#endif
 				case 'n':
 					opt_narenas_lshift--;
 					break;
@@ -627,10 +617,11 @@ MALLOC_OUT:
 
 					cbuf[0] = opts[j];
 					cbuf[1] = '\0';
-					malloc_write4("<jemalloc>",
+					malloc_write(
-					    ": Unsupported character "
+					    "<jemalloc>: Unsupported character "
-					    "in malloc options: '", cbuf,
+					    "in malloc options: '");
-					    "'\n");
+					malloc_write(cbuf);
+					malloc_write("'\n");
 				}
 				}
 			}
@@ -640,8 +631,7 @@ MALLOC_OUT:
 	/* Register fork handlers. */
 	if (pthread_atfork(jemalloc_prefork, jemalloc_postfork,
 	    jemalloc_postfork) != 0) {
-		malloc_write4("<jemalloc>", ": Error in pthread_atfork()\n", "",
+		malloc_write("<jemalloc>: Error in pthread_atfork()\n");
-		    "");
 		if (opt_abort)
 			abort();
 	}
@@ -654,8 +644,7 @@ MALLOC_OUT:
 	if (opt_stats_print) {
 		/* Print statistics at exit. */
 		if (atexit(stats_print_atexit) != 0) {
-			malloc_write4("<jemalloc>", ": Error in atexit()\n", "",
+			malloc_write("<jemalloc>: Error in atexit()\n");
-			    "");
 			if (opt_abort)
 				abort();
 		}
@@ -736,33 +725,7 @@ MALLOC_OUT:
 		 * For SMP systems, create more than one arena per CPU by
 		 * default.
 		 */
-#ifdef JEMALLOC_TCACHE
-		if (tcache_nslots
-#  ifdef JEMALLOC_PROF
-		    /*
-		     * Profile data storage concurrency is directly linked to
-		     * the number of arenas, so only drop the number of arenas
-		     * on behalf of enabled tcache if profiling is disabled.
-		     */
-		    && opt_prof == false
-#  endif
-		    ) {
-			/*
-			 * Only large object allocation/deallocation is
-			 * guaranteed to acquire an arena mutex, so we can get
-			 * away with fewer arenas than without thread caching.
-			 */
-			opt_narenas_lshift += 1;
-		} else {
-#endif
-			/*
-			 * All allocations must acquire an arena mutex, so use
-			 * plenty of arenas.
-			 */
 		opt_narenas_lshift += 2;
-#ifdef JEMALLOC_TCACHE
-		}
-#endif
 	}
 
 	/* Determine how many arenas to use. */
@@ -866,9 +829,8 @@ JEMALLOC_P(malloc)(size_t size)
 		else {
 #  ifdef JEMALLOC_XMALLOC
 			if (opt_xmalloc) {
-				malloc_write4("<jemalloc>",
+				malloc_write("<jemalloc>: Error in malloc(): "
-				    ": Error in malloc(): invalid size 0\n", "",
+				    "invalid size 0\n");
-				    "");
 				abort();
 			}
 #  endif
@@ -879,21 +841,28 @@ JEMALLOC_P(malloc)(size_t size)
 	}
 
 #ifdef JEMALLOC_PROF
-	if (opt_prof && (cnt = prof_alloc_prep(size)) == NULL) {
+	if (opt_prof) {
+		if ((cnt = prof_alloc_prep(size)) == NULL) {
 			ret = NULL;
 			goto OOM;
 		}
+		if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && size <=
+		    small_maxclass) {
+			ret = imalloc(small_maxclass+1);
+			if (ret != NULL)
+				arena_prof_promoted(ret, size);
+		} else
+			ret = imalloc(size);
+	} else
 #endif
-
 		ret = imalloc(size);
 
 OOM:
 	if (ret == NULL) {
 #ifdef JEMALLOC_XMALLOC
 		if (opt_xmalloc) {
-			malloc_write4("<jemalloc>",
+			malloc_write("<jemalloc>: Error in malloc(): "
-			    ": Error in malloc(): out of memory\n", "",
+			    "out of memory\n");
-			    "");
 			abort();
 		}
 #endif
@@ -933,9 +902,9 @@ JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
 			else {
 #  ifdef JEMALLOC_XMALLOC
 				if (opt_xmalloc) {
-					malloc_write4("<jemalloc>",
+					malloc_write("<jemalloc>: Error in "
-					    ": Error in posix_memalign(): "
+					    "posix_memalign(): invalid size "
-					    "invalid size 0\n", "", "");
+					    "0\n");
 					abort();
 				}
 #  endif
@@ -952,9 +921,8 @@ JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
 		    || alignment < sizeof(void *)) {
 #ifdef JEMALLOC_XMALLOC
 			if (opt_xmalloc) {
-				malloc_write4("<jemalloc>",
+				malloc_write("<jemalloc>: Error in "
-				    ": Error in posix_memalign(): "
+				    "posix_memalign(): invalid alignment\n");
-				    "invalid alignment\n", "", "");
 				abort();
 			}
 #endif
@@ -964,21 +932,32 @@ JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
 		}
 
 #ifdef JEMALLOC_PROF
-		if (opt_prof && (cnt = prof_alloc_prep(size)) == NULL) {
+		if (opt_prof) {
+			if ((cnt = prof_alloc_prep(size)) == NULL) {
 				result = NULL;
 				ret = EINVAL;
+			} else {
+				if (prof_promote && (uintptr_t)cnt !=
+				    (uintptr_t)1U && size <= small_maxclass) {
+					result = ipalloc(alignment,
+					    small_maxclass+1);
+					if (result != NULL) {
+						arena_prof_promoted(result,
+						    size);
+					}
+				} else
+					result = ipalloc(alignment, size);
+			}
 		} else
 #endif
-
 			result = ipalloc(alignment, size);
 	}
 
 	if (result == NULL) {
 #ifdef JEMALLOC_XMALLOC
 		if (opt_xmalloc) {
-			malloc_write4("<jemalloc>",
+			malloc_write("<jemalloc>: Error in posix_memalign(): "
-			": Error in posix_memalign(): out of memory\n",
+			    "out of memory\n");
-			"", "");
 			abort();
 		}
 #endif
@@ -1039,21 +1018,28 @@ JEMALLOC_P(calloc)(size_t num, size_t size)
 	}
 
 #ifdef JEMALLOC_PROF
-	if (opt_prof && (cnt = prof_alloc_prep(num_size)) == NULL) {
+	if (opt_prof) {
+		if ((cnt = prof_alloc_prep(num_size)) == NULL) {
 			ret = NULL;
 			goto RETURN;
 		}
+		if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && num_size
+		    <= small_maxclass) {
+			ret = icalloc(small_maxclass+1);
+			if (ret != NULL)
+				arena_prof_promoted(ret, num_size);
+		} else
+			ret = icalloc(num_size);
+	} else
 #endif
-
 		ret = icalloc(num_size);
 
 RETURN:
 	if (ret == NULL) {
 #ifdef JEMALLOC_XMALLOC
 		if (opt_xmalloc) {
-			malloc_write4("<jemalloc>",
+			malloc_write("<jemalloc>: Error in calloc(): out of "
-			    ": Error in calloc(): out of memory\n", "",
+			    "memory\n");
-			    "");
 			abort();
 		}
 #endif
@@ -1119,9 +1105,15 @@ JEMALLOC_P(realloc)(void *ptr, size_t size)
 				ret = NULL;
 				goto OOM;
 			}
-		}
+			if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U &&
+			    size <= small_maxclass) {
+				ret = iralloc(ptr, small_maxclass+1);
+				if (ret != NULL)
+					arena_prof_promoted(ret, size);
+			} else
+				ret = iralloc(ptr, size);
+		} else
 #endif
-
 			ret = iralloc(ptr, size);
 
 #ifdef JEMALLOC_PROF
@@ -1130,9 +1122,8 @@ OOM:
 		if (ret == NULL) {
 #ifdef JEMALLOC_XMALLOC
 			if (opt_xmalloc) {
-				malloc_write4("<jemalloc>",
+				malloc_write("<jemalloc>: Error in realloc(): "
-				    ": Error in realloc(): out of "
+				    "out of memory\n");
-				    "memory\n", "", "");
 				abort();
 			}
 #endif
@@ -1153,8 +1144,21 @@ OOM:
 			ret = NULL;
 		} else {
 #ifdef JEMALLOC_PROF
-			if (opt_prof && (cnt = prof_alloc_prep(size)) == NULL) {
+			if (opt_prof) {
+				if ((cnt = prof_alloc_prep(size)) == NULL)
 					ret = NULL;
+				else {
+					if (prof_promote && (uintptr_t)cnt !=
+					    (uintptr_t)1U && size <=
+					    small_maxclass) {
+						ret = imalloc(small_maxclass+1);
+						if (ret != NULL) {
+							arena_prof_promoted(ret,
+							    size);
+						}
+					} else
+						ret = imalloc(size);
+				}
 			} else
 #endif
 				ret = imalloc(size);
@@ -1163,9 +1167,8 @@ OOM:
 		if (ret == NULL) {
 #ifdef JEMALLOC_XMALLOC
 			if (opt_xmalloc) {
-				malloc_write4("<jemalloc>",
+				malloc_write("<jemalloc>: Error in realloc(): "
-				    ": Error in realloc(): out of "
+				    "out of memory\n");
-				    "memory\n", "", "");
 				abort();
 			}
 #endif
@@ -1239,11 +1242,11 @@ JEMALLOC_P(malloc_swap_enable)(const int *fds, unsigned nfds, int prezeroed)
 
 JEMALLOC_ATTR(visibility("default"))
 void
-JEMALLOC_P(malloc_stats_print)(void (*write4)(void *, const char *,
+JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *),
-    const char *, const char *, const char *), void *w4opaque, const char *opts)
+    void *cbopaque, const char *opts)
 {
 
-	stats_print(write4, w4opaque, opts);
+	stats_print(write_cb, cbopaque, opts);
 }
 
 JEMALLOC_ATTR(visibility("default"))

jemalloc/src/mutex.c

@@ -28,9 +28,8 @@ pthread_create_once(void)
 
 	pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create");
 	if (pthread_create_fptr == NULL) {
-		malloc_write4("<jemalloc>",
+		malloc_write("<jemalloc>: Error in dlsym(RTLD_NEXT, "
-		    ": Error in dlsym(RTLD_NEXT, \"pthread_create\")\n", "",
+		    "\"pthread_create\")\n");
-		    "");
 		abort();
 	}
 

jemalloc/src/prof.c

@@ -18,13 +18,15 @@
 /* Data. */
 
 bool		opt_prof = false;
+bool		opt_prof_active = true;
 size_t		opt_lg_prof_bt_max = LG_PROF_BT_MAX_DEFAULT;
 size_t		opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
-size_t		opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
+ssize_t		opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
 bool		opt_prof_udump = false;
 bool		opt_prof_leak = false;
 
 uint64_t	prof_interval;
+bool		prof_promote;
 
 /*
  * Global hash of (prof_bt_t *)-->(prof_ctx_t *).  This is the master data
@@ -511,6 +513,24 @@ prof_lookup(prof_bt_t *bt)
 	return (ret);
 }
 
+static inline void
+prof_sample_threshold_update(void)
+{
+	uint64_t r;
+	double u;
+
+	/*
+	 * Compute prof_sample_threshold as a geometrically distributed random
+	 * variable with mean (2^opt_lg_prof_sample).
+	 */
+	prn64(r, 53, prof_sample_prn_state, (uint64_t)1125899906842625LLU,
+	    1058392653243283975);
+	u = (double)r * (1.0/9007199254740992.0L);
+	prof_sample_threshold = (uint64_t)(log(u) /
+	    log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample))))
+	    + (uint64_t)1U;
+}
+
 prof_thr_cnt_t *
 prof_alloc_prep(size_t size)
 {
@@ -518,14 +538,36 @@ prof_alloc_prep(size_t size)
 	void *vec[prof_bt_max];
 	prof_bt_t bt;
 
+	if (opt_prof_active == false) {
+		/* Sampling is currently inactive, so avoid sampling. */
+		ret = (prof_thr_cnt_t *)(uintptr_t)1U;
+	} else if (opt_lg_prof_sample == 0) {
 		/*
-	 * Determine whether to capture a backtrace based on whether size is
+		 * Don't bother with sampling logic, since sampling interval is
-	 * enough for prof_accum to reach prof_sample_threshold.  However,
+		 * 1.
-	 * delay updating these variables until prof_{m,re}alloc(), because we
+		 */
-	 * don't know for sure that the allocation will succeed.
+		bt_init(&bt, vec);
+		prof_backtrace(&bt, 2, prof_bt_max);
+		ret = prof_lookup(&bt);
+	} else {
+		if (prof_sample_threshold == 0) {
+			/*
+			 * Initialize.  Seed the prng differently for each
+			 * thread.
+			 */
+			prof_sample_prn_state = (uint64_t)(uintptr_t)&size;
+			prof_sample_threshold_update();
+		}
+
+		/*
+		 * Determine whether to capture a backtrace based on whether
+		 * size is enough for prof_accum to reach
+		 * prof_sample_threshold.  However, delay updating these
+		 * variables until prof_{m,re}alloc(), because we don't know
+		 * for sure that the allocation will succeed.
 		 *
-	 * Use subtraction rather than addition to avoid potential integer
+		 * Use subtraction rather than addition to avoid potential
-	 * overflow.
+		 * integer overflow.
 		 */
 		if (size >= prof_sample_threshold - prof_sample_accum) {
 			bt_init(&bt, vec);
@@ -533,6 +575,7 @@ prof_alloc_prep(size_t size)
 			ret = prof_lookup(&bt);
 		} else
 			ret = (prof_thr_cnt_t *)(uintptr_t)1U;
+	}
 
 	return (ret);
 }
@@ -574,24 +617,6 @@ prof_cnt_set(const void *ptr, prof_thr_cnt_t *cnt)
 		huge_prof_cnt_set(ptr, cnt);
 }
 
-static inline void
-prof_sample_threshold_update(void)
-{
-	uint64_t r;
-	double u;
-
-	/*
-	 * Compute prof_sample_threshold as a geometrically distributed random
-	 * variable with mean (2^opt_lg_prof_sample).
-	 */
-	prn64(r, 53, prof_sample_prn_state, (uint64_t)1125899906842625LLU,
-	    1058392653243283975);
-	u = (double)r * (1.0/9007199254740992.0L);
-	prof_sample_threshold = (uint64_t)(log(u) /
-	    log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample))))
-	    + (uint64_t)1U;
-}
-
 static inline void
 prof_sample_accum_update(size_t size)
 {
@@ -604,18 +629,10 @@ prof_sample_accum_update(size_t size)
 		return;
 	}
 
-	if (prof_sample_threshold == 0) {
-		/* Initialize.  Seed the prng differently for each thread. */
-		prof_sample_prn_state = (uint64_t)(uintptr_t)&size;
-		prof_sample_threshold_update();
-	}
-
 	/* Take care to avoid integer overflow. */
 	if (size >= prof_sample_threshold - prof_sample_accum) {
 		prof_sample_accum -= (prof_sample_threshold - size);
-		/*
+		/* Compute new prof_sample_threshold. */
-		 * Compute new geometrically distributed prof_sample_threshold.
-		 */
 		prof_sample_threshold_update();
 		while (prof_sample_accum >= prof_sample_threshold) {
 			prof_sample_accum -= prof_sample_threshold;
@@ -726,9 +743,8 @@ prof_flush(bool propagate_err)
 	err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
 	if (err == -1) {
 		if (propagate_err == false) {
-			malloc_write4("<jemalloc>",
+			malloc_write("<jemalloc>: write() failed during heap "
-			    ": write() failed during heap profile flush", "\n",
+			    "profile flush\n");
-			    "");
 			if (opt_abort)
 				abort();
 		}
@@ -912,12 +928,12 @@ prof_dump(const char *filename, bool leakcheck, bool propagate_err)
 	prof_dump_fd = creat(filename, 0644);
 	if (prof_dump_fd == -1) {
 		if (propagate_err == false) {
-			malloc_write4("<jemalloc>",
+			malloc_write("<jemalloc>: creat(\"");
-			    ": creat(\"", filename, "\", 0644) failed\n");
+			malloc_write(filename);
+			malloc_write("\", 0644) failed\n");
 			if (opt_abort)
 				abort();
 		}
-		prof_leave();
 		goto ERROR;
 	}
 
@@ -968,15 +984,17 @@ prof_dump(const char *filename, bool leakcheck, bool propagate_err)
 	prof_leave();
 
 	if (leakcheck && cnt_all.curbytes != 0) {
-		malloc_write4("<jemalloc>: Leak summary: ",
+		malloc_write("<jemalloc>: Leak summary: ");
-		    umax2s(cnt_all.curbytes, 10, buf),
+		malloc_write(umax2s(cnt_all.curbytes, 10, buf));
-		    (cnt_all.curbytes != 1) ? " bytes" : " byte", ", ");
+		malloc_write((cnt_all.curbytes != 1) ? " bytes, " : " byte, ");
-		malloc_write4(umax2s(cnt_all.curobjs, 10, buf),
+		malloc_write(umax2s(cnt_all.curobjs, 10, buf));
-		    (cnt_all.curobjs != 1) ? " objects" : " object", ", ", "");
+		malloc_write((cnt_all.curobjs != 1) ? " objects, " :
-		malloc_write4(umax2s(leak_nctx, 10, buf),
+		    " object, ");
-		    (leak_nctx != 1) ? " contexts" : " context", "\n", "");
+		malloc_write(umax2s(leak_nctx, 10, buf));
-		malloc_write4("<jemalloc>: Run pprof on \"",
+		malloc_write((leak_nctx != 1) ? " contexts\n" : " context\n");
-		    filename, "\" for leak detail\n", "");
+		malloc_write("<jemalloc>: Run pprof on \"");
+		malloc_write(filename);
+		malloc_write("\" for leak detail\n");
 	}
 
 	return (false);
@@ -1236,18 +1254,10 @@ prof_boot0(void)
 {
 
 	/*
-	 * opt_prof must be in its final state before any arenas are
+	 * opt_prof and prof_promote must be in their final state before any
-	 * initialized, so this function must be executed early.
+	 * arenas are initialized, so this function must be executed early.
 	 */
 
-	if (opt_lg_prof_sample > 0) {
-		/*
-		 * Disable leak checking, since not all allocations will be
-		 * sampled.
-		 */
-		opt_prof_leak = false;
-	}
-
 	if (opt_prof_leak && opt_prof == false) {
 		/*
 		 * Enable opt_prof, but in such a way that profiles are never
@@ -1256,8 +1266,15 @@ prof_boot0(void)
 		opt_prof = true;
 		opt_prof_udump = false;
 		prof_interval = 0;
-	} else if (opt_prof)
+	} else if (opt_prof) {
-		prof_interval = (((uint64_t)1U) << opt_lg_prof_interval);
+		if (opt_lg_prof_interval >= 0) {
+			prof_interval = (((uint64_t)1U) <<
+			    opt_lg_prof_interval);
+		} else
+			prof_interval = 0;
+	}
+
+	prof_promote = (opt_prof && opt_lg_prof_sample > PAGE_SHIFT);
 }
 
 bool
@@ -1272,8 +1289,8 @@ prof_boot1(void)
 			return (true);
 		if (pthread_key_create(&bt2cnt_tsd, bt2cnt_thread_cleanup)
 		    != 0) {
-			malloc_write4("<jemalloc>",
+			malloc_write(
-			    ": Error in pthread_key_create()\n", "", "");
+			    "<jemalloc>: Error in pthread_key_create()\n");
 			abort();
 		}
 
@@ -1288,8 +1305,7 @@ prof_boot1(void)
 		enq_udump = false;
 
 		if (atexit(prof_fdump) != 0) {
-			malloc_write4("<jemalloc>", ": Error in atexit()\n", "",
+			malloc_write("<jemalloc>: Error in atexit()\n");
-			    "");
 			if (opt_abort)
 				abort();
 		}

jemalloc/src/stats.c

@@ -43,15 +43,14 @@ bool	opt_stats_print = false;
 /* Function prototypes for non-inline static functions. */
 
 #ifdef JEMALLOC_STATS
-static void	malloc_vcprintf(void (*write4)(void *, const char *,
+static void	malloc_vcprintf(void (*write_cb)(void *, const char *),
-    const char *, const char *, const char *), void *w4opaque,
+    void *cbopaque, const char *format, va_list ap);
-    const char *format, va_list ap);
+static void	stats_arena_bins_print(void (*write_cb)(void *, const char *),
-static void	stats_arena_bins_print(void (*write4)(void *, const char *,
+    void *cbopaque, unsigned i);
-    const char *, const char *, const char *), void *w4opaque, unsigned i);
+static void	stats_arena_lruns_print(void (*write_cb)(void *, const char *),
-static void	stats_arena_lruns_print(void (*write4)(void *, const char *,
+    void *cbopaque, unsigned i);
-    const char *, const char *, const char *), void *w4opaque, unsigned i);
+static void	stats_arena_print(void (*write_cb)(void *, const char *),
-static void	stats_arena_print(void (*write4)(void *, const char *,
+    void *cbopaque, unsigned i);
-    const char *, const char *, const char *), void *w4opaque, unsigned i);
 #endif
 
 /******************************************************************************/
@@ -97,23 +96,23 @@ umax2s(uintmax_t x, unsigned base, char *s)
 
 #ifdef JEMALLOC_STATS
 static void
-malloc_vcprintf(void (*write4)(void *, const char *, const char *, const char *,
+malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *), void *w4opaque, const char *format, va_list ap)
+    const char *format, va_list ap)
 {
 	char buf[4096];
 
-	if (write4 == NULL) {
+	if (write_cb == NULL) {
 		/*
-		 * The caller did not provide an alternate write4 callback
+		 * The caller did not provide an alternate write_cb callback
-		 * function, so use the default one.  malloc_write4() is an
+		 * function, so use the default one.  malloc_write() is an
 		 * inline function, so use malloc_message() directly here.
 		 */
-		write4 = JEMALLOC_P(malloc_message);
+		write_cb = JEMALLOC_P(malloc_message);
-		w4opaque = NULL;
+		cbopaque = NULL;
 	}
 
 	vsnprintf(buf, sizeof(buf), format, ap);
-	write4(w4opaque, buf, "", "", "");
+	write_cb(cbopaque, buf);
 }
 
 /*
@@ -122,13 +121,13 @@ malloc_vcprintf(void (*write4)(void *, const char *, const char *, const char *,
  */
 JEMALLOC_ATTR(format(printf, 3, 4))
 void
-malloc_cprintf(void (*write4)(void *, const char *, const char *, const char *,
+malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *), void *w4opaque, const char *format, ...)
+    const char *format, ...)
 {
 	va_list ap;
 
 	va_start(ap, format);
-	malloc_vcprintf(write4, w4opaque, format, ap);
+	malloc_vcprintf(write_cb, cbopaque, format, ap);
 	va_end(ap);
 }
 
@@ -149,8 +148,8 @@ malloc_printf(const char *format, ...)
 
 #ifdef JEMALLOC_STATS
 static void
-stats_arena_bins_print(void (*write4)(void *, const char *, const char *,
+stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *, const char *), void *w4opaque, unsigned i)
+    unsigned i)
 {
 	size_t pagesize;
 	bool config_tcache;
@@ -160,13 +159,15 @@ stats_arena_bins_print(void (*write4)(void *, const char *, const char *,
 
 	CTL_GET("config.tcache", &config_tcache, bool);
 	if (config_tcache) {
-		malloc_cprintf(write4, w4opaque,
+		malloc_cprintf(write_cb, cbopaque,
-		    "bins:     bin    size regs pgs  nrequests    "
+		    "bins:     bin    size regs pgs    allocated      nmalloc"
-		    "nfills  nflushes   newruns    reruns maxruns curruns\n");
+		    "      ndalloc    nrequests       nfills     nflushes"
-	} else {
-		malloc_cprintf(write4, w4opaque,
-		    "bins:     bin    size regs pgs  nrequests   "
 		    "      newruns       reruns      maxruns      curruns\n");
+	} else {
+		malloc_cprintf(write_cb, cbopaque,
+		    "bins:     bin    size regs pgs    allocated      nmalloc"
+		    "      ndalloc      newruns       reruns      maxruns"
+		    "      curruns\n");
 	}
 	CTL_GET("arenas.nbins", &nbins, unsigned);
 	for (j = 0, gap_start = UINT_MAX; j < nbins; j++) {
@@ -178,20 +179,21 @@ stats_arena_bins_print(void (*write4)(void *, const char *, const char *,
 				gap_start = j;
 		} else {
 			unsigned ntbins_, nqbins, ncbins, nsbins;
-			size_t reg_size, run_size;
+			size_t reg_size, run_size, allocated;
 			uint32_t nregs;
-			uint64_t nrequests, nfills, nflushes, reruns;
+			uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes;
+			uint64_t reruns;
 			size_t highruns, curruns;
 
 			if (gap_start != UINT_MAX) {
 				if (j > gap_start + 1) {
 					/* Gap of more than one size class. */
-					malloc_cprintf(write4, w4opaque,
+					malloc_cprintf(write_cb, cbopaque,
 					    "[%u..%u]\n", gap_start,
 					    j - 1);
 				} else {
 					/* Gap of one size class. */
-					malloc_cprintf(write4, w4opaque,
+					malloc_cprintf(write_cb, cbopaque,
 					    "[%u]\n", gap_start);
 				}
 				gap_start = UINT_MAX;
@@ -203,9 +205,15 @@ stats_arena_bins_print(void (*write4)(void *, const char *, const char *,
 			CTL_J_GET("arenas.bin.0.size", &reg_size, size_t);
 			CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t);
 			CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t);
+			CTL_IJ_GET("stats.arenas.0.bins.0.allocated",
+			    &allocated, size_t);
+			CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc",
+			    &nmalloc, uint64_t);
+			CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc",
+			    &ndalloc, uint64_t);
+			if (config_tcache) {
 				CTL_IJ_GET("stats.arenas.0.bins.0.nrequests",
 				    &nrequests, uint64_t);
-			if (config_tcache) {
 				CTL_IJ_GET("stats.arenas.0.bins.0.nfills",
 				    &nfills, uint64_t);
 				CTL_IJ_GET("stats.arenas.0.bins.0.nflushes",
@@ -218,61 +226,67 @@ stats_arena_bins_print(void (*write4)(void *, const char *, const char *,
 			CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns,
 			    size_t);
 			if (config_tcache) {
-				malloc_cprintf(write4, w4opaque,
+				malloc_cprintf(write_cb, cbopaque,
-				    "%13u %1s %5zu %4u %3zu %10"PRIu64
+				    "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64
-				    " %9"PRIu64" %9"PRIu64" %9"PRIu64""
+				    " %12"PRIu64" %12"PRIu64" %12"PRIu64
-				    " %9"PRIu64" %7zu %7zu\n",
+				    " %12"PRIu64" %12"PRIu64" %12"PRIu64
+				    " %12zu %12zu\n",
 				    j,
 				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
 				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
-				    j < ntbins_ + nqbins + ncbins + nsbins ? "S"
+				    "S",
-				    : "M",
 				    reg_size, nregs, run_size / pagesize,
-				    nrequests, nfills, nflushes, nruns, reruns,
+				    allocated, nmalloc, ndalloc, nrequests,
-				    highruns, curruns);
+				    nfills, nflushes, nruns, reruns, highruns,
-			} else {
-				malloc_cprintf(write4, w4opaque,
-				    "%13u %1s %5zu %4u %3zu %10"PRIu64
-				    " %9"PRIu64" %9"PRIu64" %7zu %7zu\n",
-				    j,
-				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
-				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
-				    j < ntbins_ + nqbins + ncbins + nsbins ? "S"
-				    : "M",
-				    reg_size, nregs, run_size / pagesize,
-				    nrequests, nruns, reruns, highruns,
 				    curruns);
+			} else {
+				malloc_cprintf(write_cb, cbopaque,
+				    "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64
+				    " %12"PRIu64" %12"PRIu64" %12"PRIu64
+				    " %12zu %12zu\n",
+				    j,
+				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
+				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
+				    "S",
+				    reg_size, nregs, run_size / pagesize,
+				    allocated, nmalloc, ndalloc, nruns, reruns,
+				    highruns, curruns);
 			}
 		}
 	}
 	if (gap_start != UINT_MAX) {
 		if (j > gap_start + 1) {
 			/* Gap of more than one size class. */
-			malloc_cprintf(write4, w4opaque, "[%u..%u]\n",
+			malloc_cprintf(write_cb, cbopaque, "[%u..%u]\n",
 			    gap_start, j - 1);
 		} else {
 			/* Gap of one size class. */
-			malloc_cprintf(write4, w4opaque, "[%u]\n", gap_start);
+			malloc_cprintf(write_cb, cbopaque, "[%u]\n", gap_start);
 		}
 	}
 }
 
 static void
-stats_arena_lruns_print(void (*write4)(void *, const char *, const char *,
+stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *, const char *), void *w4opaque, unsigned i)
+    unsigned i)
 {
 	size_t pagesize, nlruns, j;
 	ssize_t gap_start;
 
 	CTL_GET("arenas.pagesize", &pagesize, size_t);
 
-	malloc_cprintf(write4, w4opaque,
+	malloc_cprintf(write_cb, cbopaque,
-	    "large:   size pages nrequests   maxruns   curruns\n");
+	    "large:   size pages      nmalloc      ndalloc    nrequests"
+	    "      maxruns      curruns\n");
 	CTL_GET("arenas.nlruns", &nlruns, size_t);
 	for (j = 0, gap_start = -1; j < nlruns; j++) {
-		uint64_t nrequests;
+		uint64_t nmalloc, ndalloc, nrequests;
 		size_t run_size, highruns, curruns;
 
+		CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc,
+		    uint64_t);
+		CTL_IJ_GET("stats.arenas.0.lruns.0.ndalloc", &ndalloc,
+		    uint64_t);
 		CTL_IJ_GET("stats.arenas.0.lruns.0.nrequests", &nrequests,
 		    uint64_t);
 		if (nrequests == 0) {
@@ -285,32 +299,31 @@ stats_arena_lruns_print(void (*write4)(void *, const char *, const char *,
 			CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns,
 			    size_t);
 			if (gap_start != -1) {
-				malloc_cprintf(write4, w4opaque, "[%zu]\n",
+				malloc_cprintf(write_cb, cbopaque, "[%zu]\n",
 				    j - gap_start);
 				gap_start = -1;
 			}
-			malloc_cprintf(write4, w4opaque,
+			malloc_cprintf(write_cb, cbopaque,
-			    "%13zu %5zu %9"PRIu64" %9zu %9zu\n",
+			    "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64
-			    run_size, run_size / pagesize, nrequests, highruns,
+			    " %12zu %12zu\n",
-			    curruns);
+			    run_size, run_size / pagesize, nmalloc, ndalloc,
+			    nrequests, highruns, curruns);
 		}
 	}
 	if (gap_start != -1)
-		malloc_cprintf(write4, w4opaque, "[%zu]\n", j - gap_start);
+		malloc_cprintf(write_cb, cbopaque, "[%zu]\n", j - gap_start);
 }
 
 static void
-stats_arena_print(void (*write4)(void *, const char *, const char *,
+stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *, const char *), void *w4opaque, unsigned i)
+    unsigned i)
 {
 	size_t pagesize, pactive, pdirty, mapped;
 	uint64_t npurge, nmadvise, purged;
 	size_t small_allocated;
-	uint64_t small_nmalloc, small_ndalloc;
+	uint64_t small_nmalloc, small_ndalloc, small_nrequests;
-	size_t medium_allocated;
-	uint64_t medium_nmalloc, medium_ndalloc;
 	size_t large_allocated;
-	uint64_t large_nmalloc, large_ndalloc;
+	uint64_t large_nmalloc, large_ndalloc, large_nrequests;
 
 	CTL_GET("arenas.pagesize", &pagesize, size_t);
 
@@ -319,50 +332,47 @@ stats_arena_print(void (*write4)(void *, const char *, const char *,
 	CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t);
 	CTL_I_GET("stats.arenas.0.nmadvise", &nmadvise, uint64_t);
 	CTL_I_GET("stats.arenas.0.purged", &purged, uint64_t);
-	malloc_cprintf(write4, w4opaque,
+	malloc_cprintf(write_cb, cbopaque,
 	    "dirty pages: %zu:%zu active:dirty, %"PRIu64" sweep%s,"
 	    " %"PRIu64" madvise%s, %"PRIu64" purged\n",
 	    pactive, pdirty, npurge, npurge == 1 ? "" : "s",
 	    nmadvise, nmadvise == 1 ? "" : "s", purged);
 
-	malloc_cprintf(write4, w4opaque,
+	malloc_cprintf(write_cb, cbopaque,
-	    "            allocated      nmalloc      ndalloc\n");
+	    "            allocated      nmalloc      ndalloc    nrequests\n");
 	CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t);
 	CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t);
 	CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t);
-	malloc_cprintf(write4, w4opaque,
+	CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t);
-	    "small:   %12zu %12"PRIu64" %12"PRIu64"\n",
+	malloc_cprintf(write_cb, cbopaque,
-	    small_allocated, small_nmalloc, small_ndalloc);
+	    "small:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
-	CTL_I_GET("stats.arenas.0.medium.allocated", &medium_allocated, size_t);
+	    small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
-	CTL_I_GET("stats.arenas.0.medium.nmalloc", &medium_nmalloc, uint64_t);
-	CTL_I_GET("stats.arenas.0.medium.ndalloc", &medium_ndalloc, uint64_t);
-	malloc_cprintf(write4, w4opaque,
-	    "medium:  %12zu %12"PRIu64" %12"PRIu64"\n",
-	    medium_allocated, medium_nmalloc, medium_ndalloc);
 	CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t);
 	CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t);
 	CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t);
-	malloc_cprintf(write4, w4opaque,
+	CTL_I_GET("stats.arenas.0.large.nrequests", &large_nrequests, uint64_t);
-	    "large:   %12zu %12"PRIu64" %12"PRIu64"\n",
+	malloc_cprintf(write_cb, cbopaque,
-	    large_allocated, large_nmalloc, large_ndalloc);
+	    "large:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
-	malloc_cprintf(write4, w4opaque,
+	    large_allocated, large_nmalloc, large_ndalloc, large_nrequests);
-	    "total:   %12zu %12"PRIu64" %12"PRIu64"\n",
+	malloc_cprintf(write_cb, cbopaque,
-	    small_allocated + medium_allocated + large_allocated,
+	    "total:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
-	    small_nmalloc + medium_nmalloc + large_nmalloc,
+	    small_allocated + large_allocated,
-	    small_ndalloc + medium_ndalloc + large_ndalloc);
+	    small_nmalloc + large_nmalloc,
-	malloc_cprintf(write4, w4opaque, "active:  %12zu\n",
+	    small_ndalloc + large_ndalloc,
+	    small_nrequests + large_nrequests);
+	malloc_cprintf(write_cb, cbopaque, "active:  %12zu\n",
 	    pactive * pagesize );
 	CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t);
-	malloc_cprintf(write4, w4opaque, "mapped:  %12zu\n", mapped);
+	malloc_cprintf(write_cb, cbopaque, "mapped:  %12zu\n", mapped);
 
-	stats_arena_bins_print(write4, w4opaque, i);
+	stats_arena_bins_print(write_cb, cbopaque, i);
-	stats_arena_lruns_print(write4, w4opaque, i);
+	stats_arena_lruns_print(write_cb, cbopaque, i);
 }
 #endif
 
 void
-stats_print(void (*write4)(void *, const char *, const char *, const char *,
+stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
-    const char *), void *w4opaque, const char *opts)
+    const char *opts)
 {
 	uint64_t epoch;
 	size_t u64sz;
@@ -378,14 +388,14 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 	u64sz = sizeof(uint64_t);
 	xmallctl("epoch", &epoch, &u64sz, &epoch, sizeof(uint64_t));
 
-	if (write4 == NULL) {
+	if (write_cb == NULL) {
 		/*
-		 * The caller did not provide an alternate write4 callback
+		 * The caller did not provide an alternate write_cb callback
-		 * function, so use the default one.  malloc_write4() is an
+		 * function, so use the default one.  malloc_write() is an
 		 * inline function, so use malloc_message() directly here.
 		 */
-		write4 = JEMALLOC_P(malloc_message);
+		write_cb = JEMALLOC_P(malloc_message);
-		w4opaque = NULL;
+		cbopaque = NULL;
 	}
 
 	if (opts != NULL) {
@@ -413,9 +423,10 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 		}
 	}
 
-	write4(w4opaque, "___ Begin jemalloc statistics ___\n", "", "", "");
+	write_cb(cbopaque, "___ Begin jemalloc statistics ___\n");
 	if (general) {
 		int err;
+		const char *cpv;
 		bool bv;
 		unsigned uv;
 		ssize_t ssv;
@@ -424,142 +435,175 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 		bsz = sizeof(bool);
 		ssz = sizeof(size_t);
 
+		CTL_GET("version", &cpv, const char *);
+		write_cb(cbopaque, "Version: ");
+		write_cb(cbopaque, cpv);
+		write_cb(cbopaque, "\n");
 		CTL_GET("config.debug", &bv, bool);
-		write4(w4opaque, "Assertions ", bv ? "enabled" : "disabled",
+		write_cb(cbopaque, "Assertions ");
-		    "\n", "");
+		write_cb(cbopaque, bv ? "enabled" : "disabled");
+		write_cb(cbopaque, "\n");
 
-		write4(w4opaque, "Boolean JEMALLOC_OPTIONS: ", "", "", "");
+		write_cb(cbopaque, "Boolean JEMALLOC_OPTIONS: ");
 		if ((err = JEMALLOC_P(mallctl)("opt.abort", &bv, &bsz, NULL, 0))
 		    == 0)
-			write4(w4opaque, bv ? "A" : "a", "", "", "");
+			write_cb(cbopaque, bv ? "A" : "a");
+		if ((err = JEMALLOC_P(mallctl)("prof.active", &bv, &bsz,
+		    NULL, 0)) == 0)
+			write_cb(cbopaque, bv ? "E" : "e");
 		if ((err = JEMALLOC_P(mallctl)("opt.prof", &bv, &bsz, NULL, 0))
 		    == 0)
-			write4(w4opaque, bv ? "F" : "f", "", "", "");
+			write_cb(cbopaque, bv ? "F" : "f");
+		if ((err = JEMALLOC_P(mallctl)("opt.tcache", &bv, &bsz, NULL,
+		    0)) == 0)
+			write_cb(cbopaque, bv ? "H" : "h");
 		if ((err = JEMALLOC_P(mallctl)("opt.junk", &bv, &bsz, NULL, 0))
 		    == 0)
-			write4(w4opaque, bv ? "J" : "j", "", "", "");
+			write_cb(cbopaque, bv ? "J" : "j");
 		if ((err = JEMALLOC_P(mallctl)("opt.prof_leak", &bv, &bsz, NULL,
 		    0)) == 0)
-			write4(w4opaque, bv ? "L" : "l", "", "", "");
+			write_cb(cbopaque, bv ? "L" : "l");
 		if ((err = JEMALLOC_P(mallctl)("opt.overcommit", &bv, &bsz,
 		    NULL, 0)) == 0)
-			write4(w4opaque, bv ? "O" : "o", "", "", "");
+			write_cb(cbopaque, bv ? "O" : "o");
-		write4(w4opaque, "P", "", "", "");
+		if ((err = JEMALLOC_P(mallctl)("opt.stats_print", &bv, &bsz,
+		    NULL, 0)) == 0)
+			write_cb(cbopaque, bv ? "P" : "p");
 		if ((err = JEMALLOC_P(mallctl)("opt.prof_udump", &bv, &bsz,
 		    NULL, 0)) == 0)
-			write4(w4opaque, bv ? "U" : "u", "", "", "");
+			write_cb(cbopaque, bv ? "U" : "u");
 		if ((err = JEMALLOC_P(mallctl)("opt.sysv", &bv, &bsz, NULL, 0))
 		    == 0)
-			write4(w4opaque, bv ? "V" : "v", "", "", "");
+			write_cb(cbopaque, bv ? "V" : "v");
 		if ((err = JEMALLOC_P(mallctl)("opt.xmalloc", &bv, &bsz, NULL,
 		    0)) == 0)
-			write4(w4opaque, bv ? "X" : "x", "", "", "");
+			write_cb(cbopaque, bv ? "X" : "x");
 		if ((err = JEMALLOC_P(mallctl)("opt.zero", &bv, &bsz, NULL, 0))
 		    == 0)
-			write4(w4opaque, bv ? "Z" : "z", "", "", "");
+			write_cb(cbopaque, bv ? "Z" : "z");
-		write4(w4opaque, "\n", "", "", "");
+		write_cb(cbopaque, "\n");
 
-		write4(w4opaque, "CPUs: ", umax2s(ncpus, 10, s), "\n", "");
+		write_cb(cbopaque, "CPUs: ");
+		write_cb(cbopaque, umax2s(ncpus, 10, s));
+		write_cb(cbopaque, "\n");
 
 		CTL_GET("arenas.narenas", &uv, unsigned);
-		write4(w4opaque, "Max arenas: ", umax2s(uv, 10, s), "\n", "");
+		write_cb(cbopaque, "Max arenas: ");
+		write_cb(cbopaque, umax2s(uv, 10, s));
+		write_cb(cbopaque, "\n");
 
-		write4(w4opaque, "Pointer size: ", umax2s(sizeof(void *), 10,
+		write_cb(cbopaque, "Pointer size: ");
-		    s), "\n", "");
+		write_cb(cbopaque, umax2s(sizeof(void *), 10, s));
+		write_cb(cbopaque, "\n");
 
 		CTL_GET("arenas.quantum", &sv, size_t);
-		write4(w4opaque, "Quantum size: ", umax2s(sv, 10, s), "\n", "");
+		write_cb(cbopaque, "Quantum size: ");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, "\n");
 
 		CTL_GET("arenas.cacheline", &sv, size_t);
-		write4(w4opaque, "Cacheline size (assumed): ", umax2s(sv, 10,
+		write_cb(cbopaque, "Cacheline size (assumed): ");
-		    s), "\n", "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, "\n");
 
 		CTL_GET("arenas.subpage", &sv, size_t);
-		write4(w4opaque, "Subpage spacing: ", umax2s(sv, 10, s), "\n",
+		write_cb(cbopaque, "Subpage spacing: ");
-		    "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
-
+		write_cb(cbopaque, "\n");
-		CTL_GET("arenas.medium", &sv, size_t);
-		write4(w4opaque, "Medium spacing: ", umax2s(sv, 10, s), "\n",
-		    "");
 
 		if ((err = JEMALLOC_P(mallctl)("arenas.tspace_min", &sv, &ssz,
 		    NULL, 0)) == 0) {
-			write4(w4opaque, "Tiny 2^n-spaced sizes: [", umax2s(sv,
+			write_cb(cbopaque, "Tiny 2^n-spaced sizes: [");
-			    10, s), "..", "");
+			write_cb(cbopaque, umax2s(sv, 10, s));
+			write_cb(cbopaque, "..");
 
 			CTL_GET("arenas.tspace_max", &sv, size_t);
-			write4(w4opaque, umax2s(sv, 10, s), "]\n", "", "");
+			write_cb(cbopaque, umax2s(sv, 10, s));
+			write_cb(cbopaque, "]\n");
 		}
 
 		CTL_GET("arenas.qspace_min", &sv, size_t);
-		write4(w4opaque, "Quantum-spaced sizes: [", umax2s(sv, 10, s),
+		write_cb(cbopaque, "Quantum-spaced sizes: [");
-		    "..", "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, "..");
 		CTL_GET("arenas.qspace_max", &sv, size_t);
-		write4(w4opaque, umax2s(sv, 10, s), "]\n", "", "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, "]\n");
 
 		CTL_GET("arenas.cspace_min", &sv, size_t);
-		write4(w4opaque, "Cacheline-spaced sizes: [", umax2s(sv, 10, s),
+		write_cb(cbopaque, "Cacheline-spaced sizes: [");
-		    "..", "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, "..");
 		CTL_GET("arenas.cspace_max", &sv, size_t);
-		write4(w4opaque, umax2s(sv, 10, s), "]\n", "", "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, "]\n");
 
 		CTL_GET("arenas.sspace_min", &sv, size_t);
-		write4(w4opaque, "Subpage-spaced sizes: [", umax2s(sv, 10, s),
+		write_cb(cbopaque, "Subpage-spaced sizes: [");
-		    "..", "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, "..");
 		CTL_GET("arenas.sspace_max", &sv, size_t);
-		write4(w4opaque, umax2s(sv, 10, s), "]\n", "", "");
+		write_cb(cbopaque, umax2s(sv, 10, s));
-
+		write_cb(cbopaque, "]\n");
-		CTL_GET("arenas.medium_min", &sv, size_t);
-		write4(w4opaque, "Medium sizes: [", umax2s(sv, 10, s), "..",
-		    "");
-		CTL_GET("arenas.medium_max", &sv, size_t);
-		write4(w4opaque, umax2s(sv, 10, s), "]\n", "", "");
 
 		CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t);
 		if (ssv >= 0) {
-			write4(w4opaque,
+			write_cb(cbopaque,
-			    "Min active:dirty page ratio per arena: ",
+			    "Min active:dirty page ratio per arena: ");
-			    umax2s((1U << ssv), 10, s), ":1\n", "");
+			write_cb(cbopaque, umax2s((1U << ssv), 10, s));
+			write_cb(cbopaque, ":1\n");
 		} else {
-			write4(w4opaque,
+			write_cb(cbopaque,
-			    "Min active:dirty page ratio per arena: N/A\n", "",
+			    "Min active:dirty page ratio per arena: N/A\n");
-			    "", "");
 		}
-		if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_nslots", &sv,
+		if ((err = JEMALLOC_P(mallctl)("arenas.tcache_max", &sv,
 		    &ssz, NULL, 0)) == 0) {
-			size_t tcache_nslots, tcache_gc_sweep;
+			write_cb(cbopaque,
-
+			    "Maximum thread-cached size class: ");
-			tcache_nslots = (1U << sv);
+			write_cb(cbopaque, umax2s(sv, 10, s));
-			write4(w4opaque, "Thread cache slots per size class: ",
+			write_cb(cbopaque, "\n");
-			    tcache_nslots ? umax2s(tcache_nslots, 10, s) :
+		}
-			    "N/A", "\n", "");
+		if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_gc_sweep", &ssv,
-
+		    &ssz, NULL, 0)) == 0) {
-			CTL_GET("opt.lg_tcache_gc_sweep", &ssv, ssize_t);
+			size_t tcache_gc_sweep = (1U << ssv);
-			tcache_gc_sweep = (1U << ssv);
+			bool tcache_enabled;
-			write4(w4opaque, "Thread cache GC sweep interval: ",
+			CTL_GET("opt.tcache", &tcache_enabled, bool);
-			    tcache_nslots && ssv >= 0 ? umax2s(tcache_gc_sweep,
+			write_cb(cbopaque, "Thread cache GC sweep interval: ");
-			    10, s) : "N/A", "\n", "");
+			write_cb(cbopaque, tcache_enabled && ssv >= 0 ?
+			    umax2s(tcache_gc_sweep, 10, s) : "N/A");
+			write_cb(cbopaque, "\n");
 		}
 		if ((err = JEMALLOC_P(mallctl)("opt.prof", &bv, &bsz, NULL, 0))
 		   == 0 && bv) {
-			xmallctl("opt.lg_prof_bt_max", &sv, &ssz, NULL, 0);
+			CTL_GET("opt.lg_prof_bt_max", &sv, size_t);
-			write4(w4opaque, "Maximum profile backtrace depth: ",
+			write_cb(cbopaque, "Maximum profile backtrace depth: ");
-			    umax2s((1U << sv), 10, s), "\n", "");
+			write_cb(cbopaque, umax2s((1U << sv), 10, s));
+			write_cb(cbopaque, "\n");
 
-			xmallctl("opt.lg_prof_sample", &sv, &ssz, NULL, 0);
+			CTL_GET("opt.lg_prof_sample", &sv, size_t);
-			write4(w4opaque, "Average profile sample interval: ",
+			write_cb(cbopaque, "Average profile sample interval: ");
-			    umax2s((1U << sv), 10, s), "", "");
+			write_cb(cbopaque, umax2s((1U << sv), 10, s));
-			write4(w4opaque, " (2^", umax2s(sv, 10, s), ")\n", "");
+			write_cb(cbopaque, " (2^");
+			write_cb(cbopaque, umax2s(sv, 10, s));
+			write_cb(cbopaque, ")\n");
 
-			xmallctl("opt.lg_prof_interval", &sv, &ssz, NULL, 0);
+			CTL_GET("opt.lg_prof_interval", &ssv, ssize_t);
-			write4(w4opaque, "Average profile dump interval: ",
+			write_cb(cbopaque, "Average profile dump interval: ");
-			    umax2s((1U << sv), 10, s), "", "");
+			if (ssv >= 0) {
-			write4(w4opaque, " (2^", umax2s(sv, 10, s), ")\n", "");
+				write_cb(cbopaque, umax2s((1U << ssv), 10, s));
+				write_cb(cbopaque, " (2^");
+				write_cb(cbopaque, umax2s(ssv, 10, s));
+				write_cb(cbopaque, ")\n");
+			} else
+				write_cb(cbopaque, "N/A\n");
 		}
 		CTL_GET("arenas.chunksize", &sv, size_t);
-		write4(w4opaque, "Chunk size: ", umax2s(sv, 10, s), "", "");
+		write_cb(cbopaque, "Chunk size: ");
+		write_cb(cbopaque, umax2s(sv, 10, s));
 		CTL_GET("opt.lg_chunk", &sv, size_t);
-		write4(w4opaque, " (2^", umax2s(sv, 10, s), ")\n", "");
+		write_cb(cbopaque, " (2^");
+		write_cb(cbopaque, umax2s(sv, 10, s));
+		write_cb(cbopaque, ")\n");
 	}
 
 #ifdef JEMALLOC_STATS
@@ -577,7 +621,7 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 		CTL_GET("stats.allocated", &allocated, size_t);
 		CTL_GET("stats.active", &active, size_t);
 		CTL_GET("stats.mapped", &mapped, size_t);
-		malloc_cprintf(write4, w4opaque,
+		malloc_cprintf(write_cb, cbopaque,
 		    "Allocated: %zu, active: %zu, mapped: %zu\n", allocated,
 		    active, mapped);
 
@@ -589,17 +633,17 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 		    NULL, 0)) == 0) {
 			size_t lg_chunk;
 
-			malloc_cprintf(write4, w4opaque, "chunks: nchunks   "
+			malloc_cprintf(write_cb, cbopaque, "chunks: nchunks   "
 			    "highchunks    curchunks   swap_avail\n");
 			CTL_GET("opt.lg_chunk", &lg_chunk, size_t);
-			malloc_cprintf(write4, w4opaque,
+			malloc_cprintf(write_cb, cbopaque,
 			    "  %13"PRIu64"%13zu%13zu%13zu\n",
 			    chunks_total, chunks_high, chunks_current,
 			    swap_avail << lg_chunk);
 		} else {
-			malloc_cprintf(write4, w4opaque, "chunks: nchunks   "
+			malloc_cprintf(write_cb, cbopaque, "chunks: nchunks   "
 			    "highchunks    curchunks\n");
-			malloc_cprintf(write4, w4opaque,
+			malloc_cprintf(write_cb, cbopaque,
 			    "  %13"PRIu64"%13zu%13zu\n",
 			    chunks_total, chunks_high, chunks_current);
 		}
@@ -608,9 +652,9 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 		CTL_GET("stats.huge.nmalloc", &huge_nmalloc, uint64_t);
 		CTL_GET("stats.huge.ndalloc", &huge_ndalloc, uint64_t);
 		CTL_GET("stats.huge.allocated", &huge_allocated, size_t);
-		malloc_cprintf(write4, w4opaque,
+		malloc_cprintf(write_cb, cbopaque,
 		    "huge: nmalloc      ndalloc    allocated\n");
-		malloc_cprintf(write4, w4opaque,
+		malloc_cprintf(write_cb, cbopaque,
 		    " %12"PRIu64" %12"PRIu64" %12zu\n",
 		    huge_nmalloc, huge_ndalloc, huge_allocated);
 
@@ -633,9 +677,9 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 
 				if (ninitialized > 1) {
 					/* Print merged arena stats. */
-					malloc_cprintf(write4, w4opaque,
+					malloc_cprintf(write_cb, cbopaque,
 					    "\nMerged arenas stats:\n");
-					stats_arena_print(write4, w4opaque,
+					stats_arena_print(write_cb, cbopaque,
 					    narenas);
 				}
 			}
@@ -658,15 +702,16 @@ stats_print(void (*write4)(void *, const char *, const char *, const char *,
 
 				for (i = 0; i < narenas; i++) {
 					if (initialized[i]) {
-						malloc_cprintf(write4, w4opaque,
+						malloc_cprintf(write_cb,
+						    cbopaque,
 						    "\narenas[%u]:\n", i);
-						stats_arena_print(write4,
+						stats_arena_print(write_cb,
-						    w4opaque, i);
+						    cbopaque, i);
 					}
 				}
 			}
 		}
 	}
 #endif /* #ifdef JEMALLOC_STATS */
-	write4(w4opaque, "--- End jemalloc statistics ---\n", "", "", "");
+	write_cb(cbopaque, "--- End jemalloc statistics ---\n");
 }

jemalloc/src/tcache.c

@@ -4,7 +4,8 @@
 /******************************************************************************/
 /* Data. */
 
-size_t	opt_lg_tcache_nslots = LG_TCACHE_NSLOTS_DEFAULT;
+bool	opt_tcache = true;
+ssize_t	opt_lg_tcache_maxclass = LG_TCACHE_MAXCLASS_DEFAULT;
 ssize_t	opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT;
 
 /* Map of thread-specific caches. */
@@ -16,7 +17,8 @@ __thread tcache_t	*tcache_tls JEMALLOC_ATTR(tls_model("initial-exec"));
  */
 static pthread_key_t		tcache_tsd;
 
-size_t				tcache_nslots;
+size_t				nhbins;
+size_t				tcache_maxclass;
 unsigned			tcache_gc_incr;
 
 /******************************************************************************/
@@ -27,11 +29,11 @@ static void	tcache_thread_cleanup(void *arg);
 /******************************************************************************/
 
 void *
-tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
+tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 {
 	void *ret;
 
-	arena_tcache_fill(tcache->arena, tbin, binind
+	arena_tcache_fill_small(tcache->arena, tbin, binind
 #ifdef JEMALLOC_PROF
 	    , tcache->prof_accumbytes
 #endif
@@ -39,38 +41,54 @@ tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 #ifdef JEMALLOC_PROF
 	tcache->prof_accumbytes = 0;
 #endif
-	ret = tcache_bin_alloc(tbin);
+	ret = tcache_alloc_easy(tbin);
 
 	return (ret);
 }
 
 void
-tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
+tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
-#ifdef JEMALLOC_PROF
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
     , tcache_t *tcache
 #endif
     )
 {
-	arena_chunk_t *chunk;
+	void *flush, *deferred, *ptr;
-	arena_t *arena;
+	unsigned i, nflush, ndeferred;
-	void *ptr;
+
-	unsigned i, ndeferred, ncached;
+	assert(binind < nbins);
+	assert(rem <= tbin->ncached);
+
+	for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
+	    flush = deferred, nflush = ndeferred) {
+		/* Lock the arena bin associated with the first object. */
+		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
+		arena_t *arena = chunk->arena;
+		arena_bin_t *bin = &arena->bins[binind];
 
-	for (ndeferred = tbin->ncached - rem; ndeferred > 0;) {
-		ncached = ndeferred;
-		/* Lock the arena associated with the first object. */
-		chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(tbin->slots[0]);
-		arena = chunk->arena;
-		malloc_mutex_lock(&arena->lock);
 #ifdef JEMALLOC_PROF
 		if (arena == tcache->arena) {
+			malloc_mutex_lock(&arena->lock);
 			arena_prof_accum(arena, tcache->prof_accumbytes);
+			malloc_mutex_unlock(&arena->lock);
 			tcache->prof_accumbytes = 0;
 		}
 #endif
-		/* Deallocate every object that belongs to the locked arena. */
+
-		for (i = ndeferred = 0; i < ncached; i++) {
+		malloc_mutex_lock(&bin->lock);
-			ptr = tbin->slots[i];
+#ifdef JEMALLOC_STATS
+		if (arena == tcache->arena) {
+			bin->stats.nflushes++;
+			bin->stats.nrequests += tbin->tstats.nrequests;
+			tbin->tstats.nrequests = 0;
+		}
+#endif
+		deferred = NULL;
+		ndeferred = 0;
+		for (i = 0; i < nflush; i++) {
+			ptr = flush;
+			assert(ptr != NULL);
+			flush = *(void **)ptr;
 			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 			if (chunk->arena == arena) {
 				size_t pageind = (((uintptr_t)ptr -
@@ -79,124 +97,126 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
 				    &chunk->map[pageind];
 				arena_dalloc_bin(arena, chunk, ptr, mapelm);
 			} else {
+				/*
+				 * This object was allocated via a different
+				 * arena bin than the one that is currently
+				 * locked.  Stash the object, so that it can be
+				 * handled in a future pass.
+				 */
+				*(void **)ptr = deferred;
+				deferred = ptr;
+				ndeferred++;
+			}
+		}
+		malloc_mutex_unlock(&bin->lock);
+
+		if (flush != NULL) {
+			/*
+			 * This was the first pass, and rem cached objects
+			 * remain.
+			 */
+			tbin->avail = flush;
+		}
+	}
+
+	tbin->ncached = rem;
+	if (tbin->ncached < tbin->low_water)
+		tbin->low_water = tbin->ncached;
+}
+
+void
+tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+    , tcache_t *tcache
+#endif
+    )
+{
+	void *flush, *deferred, *ptr;
+	unsigned i, nflush, ndeferred;
+
+	assert(binind < nhbins);
+	assert(rem <= tbin->ncached);
+
+	for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
+	    flush = deferred, nflush = ndeferred) {
+		/* Lock the arena associated with the first object. */
+		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
+		arena_t *arena = chunk->arena;
+
+		malloc_mutex_lock(&arena->lock);
+#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
+		if (arena == tcache->arena) {
+#endif
+#ifdef JEMALLOC_PROF
+			arena_prof_accum(arena, tcache->prof_accumbytes);
+			tcache->prof_accumbytes = 0;
+#endif
+#ifdef JEMALLOC_STATS
+			arena->stats.nrequests_large += tbin->tstats.nrequests;
+			arena->stats.lstats[binind - nbins].nrequests +=
+			    tbin->tstats.nrequests;
+			tbin->tstats.nrequests = 0;
+#endif
+#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
+		}
+#endif
+		deferred = NULL;
+		ndeferred = 0;
+		for (i = 0; i < nflush; i++) {
+			ptr = flush;
+			assert(ptr != NULL);
+			flush = *(void **)ptr;
+			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+			if (chunk->arena == arena)
+				arena_dalloc_large(arena, chunk, ptr);
+			else {
 				/*
 				 * This object was allocated via a different
 				 * arena than the one that is currently locked.
 				 * Stash the object, so that it can be handled
 				 * in a future pass.
 				 */
-				tbin->slots[ndeferred] = ptr;
+				*(void **)ptr = deferred;
+				deferred = ptr;
 				ndeferred++;
 			}
 		}
-#ifdef JEMALLOC_STATS
-		arena->bins[binind].stats.nflushes++;
-		{
-			arena_bin_t *bin = &arena->bins[binind];
-			bin->stats.nrequests += tbin->tstats.nrequests;
-			if (bin->reg_size <= small_maxclass) {
-				arena->stats.nmalloc_small +=
-				    tbin->tstats.nrequests;
-			} else {
-				arena->stats.nmalloc_medium +=
-				    tbin->tstats.nrequests;
-			}
-			tbin->tstats.nrequests = 0;
-		}
-#endif
 		malloc_mutex_unlock(&arena->lock);
-	}
 
-	if (rem > 0) {
+		if (flush != NULL) {
 			/*
-		 * Shift the remaining valid pointers to the base of the slots
+			 * This was the first pass, and rem cached objects
-		 * array.
+			 * remain.
 			 */
-		memmove(&tbin->slots[0], &tbin->slots[tbin->ncached - rem],
+			tbin->avail = flush;
-		    rem * sizeof(void *));
 		}
+	}
+
 	tbin->ncached = rem;
-}
+	if (tbin->ncached < tbin->low_water)
-
+		tbin->low_water = tbin->ncached;
-tcache_bin_t *
-tcache_bin_create(arena_t *arena)
-{
-	tcache_bin_t *ret;
-	size_t tsize;
-
-	tsize = sizeof(tcache_bin_t) + (sizeof(void *) * (tcache_nslots - 1));
-	if (tsize <= small_maxclass)
-		ret = (tcache_bin_t *)arena_malloc_small(arena, tsize, false);
-	else if (tsize <= bin_maxclass)
-		ret = (tcache_bin_t *)arena_malloc_medium(arena, tsize, false);
-	else
-		ret = (tcache_bin_t *)imalloc(tsize);
-	if (ret == NULL)
-		return (NULL);
-#ifdef JEMALLOC_STATS
-	memset(&ret->tstats, 0, sizeof(tcache_bin_stats_t));
-#endif
-	ret->low_water = 0;
-	ret->high_water = 0;
-	ret->ncached = 0;
-
-	return (ret);
-}
-
-void
-tcache_bin_destroy(tcache_t *tcache, tcache_bin_t *tbin, unsigned binind)
-{
-	arena_t *arena;
-	arena_chunk_t *chunk;
-	size_t pageind, tsize;
-	arena_chunk_map_t *mapelm;
-
-	chunk = CHUNK_ADDR2BASE(tbin);
-	arena = chunk->arena;
-	pageind = (((uintptr_t)tbin - (uintptr_t)chunk) >> PAGE_SHIFT);
-	mapelm = &chunk->map[pageind];
-
-#ifdef JEMALLOC_STATS
-	if (tbin->tstats.nrequests != 0) {
-		arena_t *arena = tcache->arena;
-		arena_bin_t *bin = &arena->bins[binind];
-		malloc_mutex_lock(&arena->lock);
-		bin->stats.nrequests += tbin->tstats.nrequests;
-		if (bin->reg_size <= small_maxclass)
-			arena->stats.nmalloc_small += tbin->tstats.nrequests;
-		else
-			arena->stats.nmalloc_medium += tbin->tstats.nrequests;
-		malloc_mutex_unlock(&arena->lock);
-	}
-#endif
-
-	assert(tbin->ncached == 0);
-	tsize = sizeof(tcache_bin_t) + (sizeof(void *) * (tcache_nslots - 1));
-	if (tsize <= bin_maxclass) {
-		malloc_mutex_lock(&arena->lock);
-		arena_dalloc_bin(arena, chunk, tbin, mapelm);
-		malloc_mutex_unlock(&arena->lock);
-	} else
-		idalloc(tbin);
 }
 
 tcache_t *
 tcache_create(arena_t *arena)
 {
 	tcache_t *tcache;
+	size_t size;
+	unsigned i;
 
-	if (sizeof(tcache_t) + (sizeof(tcache_bin_t *) * (nbins - 1)) <=
+	size = sizeof(tcache_t) + (sizeof(tcache_bin_t) * (nhbins - 1));
-	    small_maxclass) {
+	/*
-		tcache = (tcache_t *)arena_malloc_small(arena, sizeof(tcache_t)
+	 * Round up to the nearest multiple of the cacheline size, in order to
-		    + (sizeof(tcache_bin_t *) * (nbins - 1)), true);
+	 * avoid the possibility of false cacheline sharing.
-	} else if (sizeof(tcache_t) + (sizeof(tcache_bin_t *) * (nbins - 1)) <=
+	 *
-	    bin_maxclass) {
+	 * That this works relies on the same logic as in ipalloc().
-		tcache = (tcache_t *)arena_malloc_medium(arena, sizeof(tcache_t)
+	 */
-		    + (sizeof(tcache_bin_t *) * (nbins - 1)), true);
+	size = (size + CACHELINE_MASK) & (-CACHELINE);
-	} else {
+
-		tcache = (tcache_t *)icalloc(sizeof(tcache_t) +
+	if (size <= small_maxclass)
-		    (sizeof(tcache_bin_t *) * (nbins - 1)));
+		tcache = (tcache_t *)arena_malloc_small(arena, size, true);
-	}
+	else
+		tcache = (tcache_t *)icalloc(size);
 
 	if (tcache == NULL)
 		return (NULL);
@@ -210,6 +230,16 @@ tcache_create(arena_t *arena)
 #endif
 
 	tcache->arena = arena;
+	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
+	for (i = 0; i < nbins; i++) {
+		if ((arena->bins[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) {
+			tcache->tbins[i].ncached_max = (arena->bins[i].nregs <<
+			    1);
+		} else
+			tcache->tbins[i].ncached_max = TCACHE_NSLOTS_SMALL_MAX;
+	}
+	for (; i < nhbins; i++)
+		tcache->tbins[i].ncached_max = TCACHE_NSLOTS_LARGE;
 
 	tcache_tls = tcache;
 	pthread_setspecific(tcache_tsd, tcache);
@@ -226,20 +256,47 @@ tcache_destroy(tcache_t *tcache)
 	/* Unlink from list of extant tcaches. */
 	malloc_mutex_lock(&tcache->arena->lock);
 	ql_remove(&tcache->arena->tcache_ql, tcache, link);
-	tcache_stats_merge(tcache, tcache->arena);
 	malloc_mutex_unlock(&tcache->arena->lock);
+	tcache_stats_merge(tcache, tcache->arena);
 #endif
 
 	for (i = 0; i < nbins; i++) {
-		tcache_bin_t *tbin = tcache->tbins[i];
+		tcache_bin_t *tbin = &tcache->tbins[i];
-		if (tbin != NULL) {
+		tcache_bin_flush_small(tbin, i, 0
-			tcache_bin_flush(tbin, i, 0
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
-#ifdef JEMALLOC_PROF
 		    , tcache
 #endif
 		    );
-			tcache_bin_destroy(tcache, tbin, i);
+
+#ifdef JEMALLOC_STATS
+		if (tbin->tstats.nrequests != 0) {
+			arena_t *arena = tcache->arena;
+			arena_bin_t *bin = &arena->bins[i];
+			malloc_mutex_lock(&bin->lock);
+			bin->stats.nrequests += tbin->tstats.nrequests;
+			malloc_mutex_unlock(&bin->lock);
 		}
+#endif
+	}
+
+	for (; i < nhbins; i++) {
+		tcache_bin_t *tbin = &tcache->tbins[i];
+		tcache_bin_flush_large(tbin, i, 0
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+		    , tcache
+#endif
+		    );
+
+#ifdef JEMALLOC_STATS
+		if (tbin->tstats.nrequests != 0) {
+			arena_t *arena = tcache->arena;
+			malloc_mutex_lock(&arena->lock);
+			arena->stats.nrequests_large += tbin->tstats.nrequests;
+			arena->stats.lstats[i - nbins].nrequests +=
+			    tbin->tstats.nrequests;
+			malloc_mutex_unlock(&arena->lock);
+		}
+#endif
 	}
 
 #ifdef JEMALLOC_PROF
@@ -250,16 +307,20 @@ tcache_destroy(tcache_t *tcache)
 	}
 #endif
 
-	if (arena_salloc(tcache) <= bin_maxclass) {
+	if (arena_salloc(tcache) <= small_maxclass) {
 		arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
 		arena_t *arena = chunk->arena;
 		size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >>
 		    PAGE_SHIFT);
 		arena_chunk_map_t *mapelm = &chunk->map[pageind];
+		arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
+		    (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) <<
+		    PAGE_SHIFT));
+		arena_bin_t *bin = run->bin;
 
-		malloc_mutex_lock(&arena->lock);
+		malloc_mutex_lock(&bin->lock);
 		arena_dalloc_bin(arena, chunk, tcache, mapelm);
-		malloc_mutex_unlock(&arena->lock);
+		malloc_mutex_unlock(&bin->lock);
 	} else
 		idalloc(tcache);
 }
@@ -284,33 +345,43 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 	unsigned i;
 
 	/* Merge and reset tcache stats. */
-	for (i = 0; i < mbin0; i++) {
+	for (i = 0; i < nbins; i++) {
 		arena_bin_t *bin = &arena->bins[i];
-		tcache_bin_t *tbin = tcache->tbins[i];
+		tcache_bin_t *tbin = &tcache->tbins[i];
-		if (tbin != NULL) {
+		malloc_mutex_lock(&bin->lock);
 		bin->stats.nrequests += tbin->tstats.nrequests;
-			arena->stats.nmalloc_small += tbin->tstats.nrequests;
+		malloc_mutex_unlock(&bin->lock);
 		tbin->tstats.nrequests = 0;
 	}
-	}
+
-	for (; i < nbins; i++) {
+	for (; i < nhbins; i++) {
-		arena_bin_t *bin = &arena->bins[i];
+		malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins];
-		tcache_bin_t *tbin = tcache->tbins[i];
+		tcache_bin_t *tbin = &tcache->tbins[i];
-		if (tbin != NULL) {
+		arena->stats.nrequests_large += tbin->tstats.nrequests;
-			bin->stats.nrequests += tbin->tstats.nrequests;
+		lstats->nrequests += tbin->tstats.nrequests;
-			arena->stats.nmalloc_medium += tbin->tstats.nrequests;
 		tbin->tstats.nrequests = 0;
 	}
 }
-}
 #endif
 
 void
 tcache_boot(void)
 {
 
-	if (opt_lg_tcache_nslots > 0) {
+	if (opt_tcache) {
-		tcache_nslots = (1U << opt_lg_tcache_nslots);
+		/*
+		 * If necessary, clamp opt_lg_tcache_maxclass, now that
+		 * small_maxclass and arena_maxclass are known.
+		 */
+		if (opt_lg_tcache_maxclass < 0 || (1U <<
+		    opt_lg_tcache_maxclass) < small_maxclass)
+			tcache_maxclass = small_maxclass;
+		else if ((1U << opt_lg_tcache_maxclass) > arena_maxclass)
+			tcache_maxclass = arena_maxclass;
+		else
+			tcache_maxclass = (1U << opt_lg_tcache_maxclass);
+
+		nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT);
 
 		/* Compute incremental GC event threshold. */
 		if (opt_lg_tcache_gc_sweep >= 0) {
@@ -319,14 +390,11 @@ tcache_boot(void)
 			    0) ? 0 : 1);
 		} else
 			tcache_gc_incr = 0;
-	} else
-		tcache_nslots = 0;
 
-	if (tcache_nslots != 0) {
 		if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) !=
 		    0) {
-			malloc_write4("<jemalloc>",
+			malloc_write(
-			    ": Error in pthread_key_create()\n", "", "");
+			    "<jemalloc>: Error in pthread_key_create()\n");
 			abort();
 		}
 	}